From 8751343f230ee12a009546c8e6e5ad7c3c09db61 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 6 Oct 2022 12:44:57 +0200 Subject: Makefile: Group together recipes to build our tools Reorder the recipes so that our tools-related ones are grouped together. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 600e3193f..adf7e3a01 100644 --- a/Makefile +++ b/Makefile @@ -694,9 +694,6 @@ ${PROTOC}: ${DEPENDENCY_DIR}/protoc.version | ${TOOLS_DIR} ${Q}cmake --build "${PROTOC_BUILD_DIR}" --target install -- -j $(shell nproc) ${Q}cp "${PROTOC_INSTALL_DIR}"/bin/protoc ${PROTOC} -${TOOLS_DIR}/%: ${TOOLS_DIR}/%.version - ${Q}GOBIN=${TOOLS_DIR} go install ${TOOL_PACKAGE}@${TOOL_VERSION} - ${PROTOC_GEN_GITALY_LINT}: proto | ${TOOLS_DIR} ${Q}go build -o $@ ${SOURCE_DIR}/tools/protoc-gen-gitaly-lint @@ -704,6 +701,9 @@ ${PROTOC_GEN_GITALY_PROTOLIST}: | ${TOOLS_DIR} ${Q}go build -o $@ ${SOURCE_DIR}/tools/protoc-gen-gitaly-protolist # External tools +${TOOLS_DIR}/%: ${TOOLS_DIR}/%.version + ${Q}GOBIN=${TOOLS_DIR} go install ${TOOL_PACKAGE}@${TOOL_VERSION} + ${GOCOVER_COBERTURA}: TOOL_PACKAGE = github.com/t-yuki/gocover-cobertura ${GOCOVER_COBERTURA}: TOOL_VERSION = ${GOCOVER_COBERTURA_VERSION} ${GOFUMPT}: TOOL_PACKAGE = mvdan.cc/gofumpt -- cgit v1.2.3 From 0cd4b87513b7d76d30c2d424161d3635b13dba21 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 10 Oct 2022 16:09:20 +0200 Subject: Makefile: Track Go tool versions via separate Go modules Right now we track versions of our Go tooling directly in our Makefile. While this is simple, it has several drawbacks: - We're susceptible to supply-chain attacks in case an adversary manages to replace the code used to build any of our tools. - We cannot use proper dependencies in our Makefile, which adds the need for `*.version` files. - It is hard to build the tools outside of our Makefile as we don't have a way to properly pull in the correct version. - Upgrading our tooling requires us to manually hunt down new releases for all of our tools. We can fix these issues by following the approach that is efficially recommended by the Go project [1]: every tool has its own Go module in `tools/` with a "tool.go" file that imports the tool of interest. Like this we can use Go's normal tooling to keep track of versions: - We record hashes of the tool's sources as well as all of its dependencies, making supply-chain attacks almost impossible. - We can now provide proper dependencies in our Makefile: every tool depends on "tool.go", "go.mod" and "go.sum". If any of them changes we need to rebuild. - The tools can be installed in the correct version simply by using `go install` with the correct `go.mod` file. - Upgrading tools is as simple as running `go get -u`, so no more manual hunting for new versions. While these benefits are great on their own already, we can go even further with this refactoring: now that each tool has its own `go.mod` file we can adapt the Renovate bot to pick up these files. This means that we don't have to remember upgrading at all anymore, but instead the bot will automatically upgrade them for us. [1]: https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module --- Makefile | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index adf7e3a01..f11be6d9d 100644 --- a/Makefile +++ b/Makefile @@ -97,25 +97,13 @@ ifdef GITALY_TESTING_ENABLE_SHA256 endif # Dependency versions -GOLANGCI_LINT_VERSION ?= v1.48.0 -PROTOLINT_VERSION ?= v0.38.1 -GOCOVER_COBERTURA_VERSION ?= aaee18c8195c3f2d90e5ef80ca918d265463842a -GOFUMPT_VERSION ?= v0.4.0 -GOIMPORTS_VERSION ?= v0.1.10 -GOTESTSUM_VERSION ?= v1.8.1 -GO_LICENSES_VERSION ?= v1.2.1 # https://pkg.go.dev/github.com/protocolbuffers/protobuf PROTOC_VERSION ?= v21.1 -# https://pkg.go.dev/google.golang.org/protobuf -PROTOC_GEN_GO_VERSION ?= v1.28.0 -# https://pkg.go.dev/google.golang.org/grpc/cmd/protoc-gen-go-grpc -PROTOC_GEN_GO_GRPC_VERSION?= v1.2.0 # Git2Go and libgit2 may need to be updated in sync. Please refer to # https://github.com/libgit2/git2go/#which-go-version-to-use for a # compatibility matrix. GIT2GO_VERSION ?= v33 LIBGIT2_VERSION ?= v1.3.2 -DELVE_VERSION ?= v1.9.1 # protoc target PROTOC_REPO_URL ?= https://github.com/protocolbuffers/protobuf @@ -639,8 +627,6 @@ ${DEPENDENCY_DIR}/git-%.version: dependency-version | ${DEPENDENCY_DIR} ${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${GIT_VERSION}.${GIT_EXTRA_VERSION} ${GIT_BUILD_OPTIONS} ${GIT_PATCHES}" ] || >$@ echo -n "${GIT_VERSION}.${GIT_EXTRA_VERSION} ${GIT_BUILD_OPTIONS} ${GIT_PATCHES}" ${DEPENDENCY_DIR}/protoc.version: dependency-version | ${DEPENDENCY_DIR} ${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${PROTOC_VERSION} ${PROTOC_BUILD_OPTIONS}" ] || >$@ echo -n "${PROTOC_VERSION} ${PROTOC_BUILD_OPTIONS}" -${TOOLS_DIR}/%.version: dependency-version | ${TOOLS_DIR} - ${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${TOOL_VERSION}" ] || >$@ echo -n "${TOOL_VERSION}" ${LIBGIT2_INSTALL_DIR}/lib/libgit2.a: ${DEPENDENCY_DIR}/libgit2.version ${Q}${GIT} -c init.defaultBranch=master init ${GIT_QUIET} ${LIBGIT2_SOURCE_DIR} @@ -701,29 +687,19 @@ ${PROTOC_GEN_GITALY_PROTOLIST}: | ${TOOLS_DIR} ${Q}go build -o $@ ${SOURCE_DIR}/tools/protoc-gen-gitaly-protolist # External tools -${TOOLS_DIR}/%: ${TOOLS_DIR}/%.version - ${Q}GOBIN=${TOOLS_DIR} go install ${TOOL_PACKAGE}@${TOOL_VERSION} +${TOOLS_DIR}/%: ${SOURCE_DIR}/tools/%/tool.go ${SOURCE_DIR}/tools/%/go.mod ${SOURCE_DIR}/tools/%/go.sum | ${TOOLS_DIR} + ${Q}GOBIN=${TOOLS_DIR} go install -modfile ${SOURCE_DIR}/tools/$*/go.mod ${TOOL_PACKAGE} ${GOCOVER_COBERTURA}: TOOL_PACKAGE = github.com/t-yuki/gocover-cobertura -${GOCOVER_COBERTURA}: TOOL_VERSION = ${GOCOVER_COBERTURA_VERSION} ${GOFUMPT}: TOOL_PACKAGE = mvdan.cc/gofumpt -${GOFUMPT}: TOOL_VERSION = ${GOFUMPT_VERSION} ${GOIMPORTS}: TOOL_PACKAGE = golang.org/x/tools/cmd/goimports -${GOIMPORTS}: TOOL_VERSION = ${GOIMPORTS_VERSION} ${GOLANGCI_LINT}: TOOL_PACKAGE = github.com/golangci/golangci-lint/cmd/golangci-lint -${GOLANGCI_LINT}: TOOL_VERSION = ${GOLANGCI_LINT_VERSION} ${PROTOLINT}: TOOL_PACKAGE = github.com/yoheimuta/protolint/cmd/protolint -${PROTOLINT}: TOOL_VERSION = ${PROTOLINT_VERSION} ${GOTESTSUM}: TOOL_PACKAGE = gotest.tools/gotestsum -${GOTESTSUM}: TOOL_VERSION = ${GOTESTSUM_VERSION} ${GO_LICENSES}: TOOL_PACKAGE = github.com/google/go-licenses -${GO_LICENSES}: TOOL_VERSION = ${GO_LICENSES_VERSION} ${PROTOC_GEN_GO}: TOOL_PACKAGE = google.golang.org/protobuf/cmd/protoc-gen-go -${PROTOC_GEN_GO}: TOOL_VERSION = ${PROTOC_GEN_GO_VERSION} ${PROTOC_GEN_GO_GRPC}:TOOL_PACKAGE = google.golang.org/grpc/cmd/protoc-gen-go-grpc -${PROTOC_GEN_GO_GRPC}:TOOL_VERSION = ${PROTOC_GEN_GO_GRPC_VERSION} ${DELVE}: TOOL_PACKAGE = github.com/go-delve/delve/cmd/dlv -${DELVE}: TOOL_VERSION = ${DELVE_VERSION} ${TEST_REPO_MIRROR}: ${GIT} clone --mirror ${GIT_QUIET} https://gitlab.com/gitlab-org/gitlab-test.git $@ -- cgit v1.2.3 From f2047336a0a94b4aff2809bb0995aae82b4e8f97 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 6 Oct 2022 14:51:11 +0200 Subject: Makefile: Regroup remaining dependency versions Now that most dependency versions are tracked via `go.mod` files it makes more sense to keep the remaining versions in our Makefile close to where they are used. So let's regroup them for improved locality. --- Makefile | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index f11be6d9d..c5b1b6fa6 100644 --- a/Makefile +++ b/Makefile @@ -96,17 +96,9 @@ ifdef GITALY_TESTING_ENABLE_SHA256 GIT2GO_BUILD_TAGS := ${GIT2GO_BUILD_TAGS},gitaly_test_sha256 endif -# Dependency versions -# https://pkg.go.dev/github.com/protocolbuffers/protobuf -PROTOC_VERSION ?= v21.1 -# Git2Go and libgit2 may need to be updated in sync. Please refer to -# https://github.com/libgit2/git2go/#which-go-version-to-use for a -# compatibility matrix. -GIT2GO_VERSION ?= v33 -LIBGIT2_VERSION ?= v1.3.2 - # protoc target -PROTOC_REPO_URL ?= https://github.com/protocolbuffers/protobuf +PROTOC_VERSION ?= v21.1 +PROTOC_REPO_URL ?= https://github.com/protocolbuffers/protobuf PROTOC_SOURCE_DIR ?= ${DEPENDENCY_DIR}/protobuf/source PROTOC_BUILD_DIR ?= ${DEPENDENCY_DIR}/protobuf/build PROTOC_INSTALL_DIR ?= ${DEPENDENCY_DIR}/protobuf/install @@ -186,6 +178,11 @@ ifdef GIT_FIPS_BUILD_OPTIONS endif # libgit2 target +# Git2Go and libgit2 may need to be updated in sync. Please refer to +# https://github.com/libgit2/git2go/#which-go-version-to-use for a +# compatibility matrix. +GIT2GO_VERSION ?= v33 +LIBGIT2_VERSION ?= v1.3.2 LIBGIT2_REPO_URL ?= https://gitlab.com/libgit2/libgit2 LIBGIT2_SOURCE_DIR ?= ${DEPENDENCY_DIR}/libgit2/source LIBGIT2_BUILD_DIR ?= ${DEPENDENCY_DIR}/libgit2/build -- cgit v1.2.3