Makefile: Rebuild targets only if Makefile content changes

As changes to our Makefile may require a rebuild of dependencies, e.g.
because of a version change, those targets need to depend on the
Makefile itself. Naturally, this is a heuristic that's going to be wrong
often, but it's better than using stale artifacts for our build.
Unfortunately, this breaks the caching mechanism used in CI as
timestamps of files won't be preserved across differnt jobs. As a
result, the build cache is going to be invalidated all the time.

Let's improve the situation by not depending on filestamps anymore, but
instead on the Makefile's contents. It's quite easy to achieve by
depending on a generated Makefile.sha256 file instead of the real
Makefile. This file is being generated by a rule that's always
executing, but the target will only get updated in case the hash of the
source file actually changed. Like this, only real content changes will
cause the timestamp of that file to get updated.

In order to make use of this in our CI, this commit also puts the
generated Makefile.sha256 into our cache.

2 files changed, 23 insertions, 8 deletions

diff --git a/Makefile b/Makefile
index a47aad1ec..e25007b53 100644
--- a/Makefile
+++ b/Makefile
@@ -347,36 +347,43 @@ ${PROTOC}: ${BUILD_DIR}/protoc.zip | ${BUILD_DIR}
 	${Q}mkdir -p ${BUILD_DIR}/protoc
 	cd ${BUILD_DIR}/protoc && unzip ${BUILD_DIR}/protoc.zip
 
-${BUILD_DIR}/protoc.zip: Makefile | ${BUILD_DIR}
+# This is a build hack to avoid excessive rebuilding of targets. Instead of
+# depending on the timestamp of the Makefile, which will change e.g. between
+# jobs of a CI pipeline, we start depending on its hash. Like this, we only
+# rebuild if the Makefile actually has changed contents.
+${BUILD_DIR}/Makefile.sha256: FORCE | ${BUILD_DIR}
+	${Q}shasum -a256 -c $@ >/dev/null 2>&1 || >$@ shasum -a256 Makefile
+
+${BUILD_DIR}/protoc.zip: ${BUILD_DIR}/Makefile.sha256
 	${Q}if [ -z "${PROTOC_URL}" ]; then echo "Cannot generate protos on unsupported platform ${OS}" && exit 1; fi
 	curl -o $@.tmp --silent --show-error -L ${PROTOC_URL}
 	${Q}printf '${PROTOC_HASH}  $@.tmp' | shasum -a256 -c -
 	${Q}mv $@.tmp $@
 
-${BUILD_DIR}/git_full_bins.tgz: Makefile | ${BUILD_DIR}
+${BUILD_DIR}/git_full_bins.tgz: ${BUILD_DIR}/Makefile.sha256
 	curl -o $@.tmp --silent --show-error -L ${GIT_BINARIES_URL}
 	${Q}printf '${GIT_BINARIES_HASH}  $@.tmp' | shasum -a256 -c -
 	${Q}mv $@.tmp $@
 
-${GOIMPORTS}: Makefile ${BUILD_DIR}/go.mod | ${BUILD_DIR}/bin
+${GOIMPORTS}: ${BUILD_DIR}/Makefile.sha256 ${BUILD_DIR}/go.mod
 	${Q}cd ${BUILD_DIR} && go get golang.org/x/tools/cmd/goimports@2538eef75904eff384a2551359968e40c207d9d2
 
-${GO_JUNIT_REPORT}: Makefile ${BUILD_DIR}/go.mod | ${BUILD_DIR}/bin
+${GO_JUNIT_REPORT}: ${BUILD_DIR}/Makefile.sha256 ${BUILD_DIR}/go.mod
 	${Q}cd ${BUILD_DIR} && go get github.com/jstemmer/go-junit-report@984a47ca6b0a7d704c4b589852051b4d7865aa17
 
 ${GITALYFMT}: | ${BUILD_DIR}/bin
 	${Q}go build -o $@ ${SOURCE_DIR}/internal/cmd/gitalyfmt
 
-${GO_LICENSES}: Makefile ${BUILD_DIR}/go.mod | ${BUILD_DIR}/bin
+${GO_LICENSES}: ${BUILD_DIR}/Makefile.sha256 ${BUILD_DIR}/go.mod
 	${Q}cd ${BUILD_DIR} && go get github.com/google/go-licenses@0fa8c766a59182ce9fd94169ddb52abe568b7f4e
 
-${PROTOC_GEN_GO}: Makefile ${BUILD_DIR}/go.mod | ${BUILD_DIR}/bin
+${PROTOC_GEN_GO}: ${BUILD_DIR}/Makefile.sha256 ${BUILD_DIR}/go.mod
 	${Q}cd ${BUILD_DIR} && go get github.com/golang/protobuf/protoc-gen-go@v${PROTOC_GEN_GO_VERSION}
 
-${PROTOC_GEN_GITALY}: ${BUILD_DIR}/go.mod proto-lint | ${BUILD_DIR}/bin
+${PROTOC_GEN_GITALY}: ${BUILD_DIR}/go.mod proto-lint
 	${Q}go build -o $@ gitlab.com/gitlab-org/gitaly/proto/go/internal/cmd/protoc-gen-gitaly
 
-${GOLANGCI_LINT}: Makefile ${BUILD_DIR}/go.mod | ${BUILD_DIR}/bin
+${GOLANGCI_LINT}: ${BUILD_DIR}/Makefile.sha256 ${BUILD_DIR}/go.mod
 	${Q}cd ${BUILD_DIR} && go get github.com/golangci/golangci-lint/cmd/golangci-lint@v${GOLANGCI_LINT_VERSION}
 
 ${TEST_REPO}:
@@ -397,3 +404,6 @@ ${TEST_REPO_GIT}:
 
 ${GITLAB_SHELL_DIR}/config.yml: ${GITLAB_SHELL_DIR}/config.yml.example
 	cp $< $@
+
+.PHONY: FORCE
+FORCE:
diff --git a/changelogs/unreleased/pks-makefile-hash.yml b/changelogs/unreleased/pks-makefile-hash.yml
new file mode 100644
index 000000000..aa0ea63dd
--- /dev/null
+++ b/changelogs/unreleased/pks-makefile-hash.yml
@@ -0,0 +1,5 @@
+---
+title: Rebuild targets only if Makefile content changes
+merge_request: 2492
+author:
+type: added