Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlessio Caiazza <acaiazza@gitlab.com>2020-04-22 10:58:39 +0300
committerAlessio Caiazza <acaiazza@gitlab.com>2020-04-22 10:58:39 +0300
commitd149fe2578bfb474e761e3dc27c1196c20dd6668 (patch)
treece8a9558b8f9b274c571bc213a33c35194e84fd9
parentc6acc7dcb4dbd3adc380824d76cfc89a72cb845a (diff)
parent66fd5d1b9018ebf5427141c733234060b45bf626 (diff)
Merge commit '66fd5d1b9018ebf5427141c733234060b45bf626' into 12-10-stable
This was the last commit deployed with RC44
-rw-r--r--.gitlab-ci.yml37
-rw-r--r--.golangci.yml19
-rw-r--r--CHANGELOG.md16
-rw-r--r--Dockerfile2
-rw-r--r--NOTICE46
-rw-r--r--README.md2
-rw-r--r--_support/Makefile.template13
-rw-r--r--_support/instrumented-cluster/docker-compose.yml2
-rw-r--r--_support/terraform/main.tf30
-rw-r--r--auth/extract_test.go66
-rw-r--r--auth/rpccredentials.go8
-rw-r--r--auth/token.go25
-rw-r--r--changelogs/unreleased/cc-git-2-26-0.yml5
-rw-r--r--changelogs/unreleased/jc-add-metric-for-node-health.yml5
-rw-r--r--changelogs/unreleased/jc-call-hook-rpc.yml5
-rw-r--r--changelogs/unreleased/jc-fix-praefect-server-info.yml5
-rw-r--r--changelogs/unreleased/jc-operations-service-call-hook-rpcs.yml5
-rw-r--r--changelogs/unreleased/jc-pass-token-into-hook.yml5
-rw-r--r--changelogs/unreleased/jc-revert-revert-846222e0.yml5
-rw-r--r--changelogs/unreleased/po-remove-inforef-ff.yml5
-rw-r--r--changelogs/unreleased/ps-ctx-persistence-timeout.yml5
-rw-r--r--changelogs/unreleased/ps-drop-auth-v1.yml5
-rw-r--r--changelogs/unreleased/ps-linter-unconvert.yml5
-rw-r--r--changelogs/unreleased/ps-proxyheaderwhitelist-race.yml5
-rw-r--r--changelogs/unreleased/sh-add-sql-migrate-status.yml5
-rw-r--r--changelogs/unreleased/sh-fix-local-elector-locking.yml5
-rw-r--r--changelogs/unreleased/sh-improve-sql-leader-election.yml5
-rw-r--r--changelogs/unreleased/sh-make-ignore-unknown-default.yml5
-rw-r--r--changelogs/unreleased/sh-support-sql-migrate-ignore-unknown.yml5
-rw-r--r--changelogs/unreleased/smh-dataloss-cmd.yml5
-rw-r--r--client/address_parser.go4
-rw-r--r--client/dial.go12
-rw-r--r--cmd/gitaly-hooks/README.md39
-rw-r--r--cmd/gitaly-hooks/hooks.go266
-rw-r--r--cmd/gitaly-hooks/hooks_test.go152
-rw-r--r--cmd/gitaly-ssh/auth_test.go10
-rw-r--r--cmd/gitaly-ssh/main.go2
-rw-r--r--cmd/gitaly-ssh/receive_pack.go2
-rw-r--r--cmd/gitaly-ssh/upload_archive.go2
-rw-r--r--cmd/gitaly-ssh/upload_pack.go2
-rw-r--r--cmd/gitaly-ssh/upload_pack_test.go3
-rw-r--r--cmd/praefect/main.go55
-rw-r--r--cmd/praefect/subcmd.go97
-rw-r--r--cmd/praefect/subcmd_dataloss.go111
-rw-r--r--cmd/praefect/subcmd_dataloss_test.go141
-rw-r--r--cmd/praefect/subcmd_pingnodes.go15
-rw-r--r--cmd/praefect/subcmd_reconcile.go59
-rw-r--r--cmd/praefect/subcmd_sqldown.go60
-rw-r--r--cmd/praefect/subcmd_sqlstatus.go55
-rw-r--r--config.praefect.toml.example4
-rw-r--r--doc/design_diskcache.md1
-rw-r--r--doc/sql_migrations.md37
-rw-r--r--go.mod1
-rw-r--r--go.sum2
-rw-r--r--internal/blackbox/prometheus.go6
-rw-r--r--internal/bootstrap/server_factory.go5
-rw-r--r--internal/cache/cachedb.go6
-rw-r--r--internal/cache/cachedb_test.go8
-rw-r--r--internal/cache/keyer.go14
-rw-r--r--internal/cache/walker_test.go35
-rw-r--r--internal/command/command.go13
-rw-r--r--internal/command/command_test.go88
-rw-r--r--internal/command/spawntoken.go4
-rw-r--r--internal/config/config.go9
-rw-r--r--internal/git/bitmap.go4
-rw-r--r--internal/git/dirs.go4
-rw-r--r--internal/git/dirs_test.go2
-rw-r--r--internal/git/hooks/hooks.go3
-rw-r--r--internal/git/log/tag.go2
-rw-r--r--internal/git/receivepack.go38
-rw-r--r--internal/git/stats/reference_discovery.go4
-rw-r--r--internal/git/uploadpack.go2
-rw-r--r--internal/gitlabshell/env.go4
-rw-r--r--internal/helper/fstype/detect_linux.go2
-rw-r--r--internal/helper/housekeeping/housekeeping.go4
-rw-r--r--internal/helper/storage.go2
-rw-r--r--internal/log/hook.go5
-rw-r--r--internal/metadata/featureflag/feature_flags.go4
-rw-r--r--internal/metadata/featureflag/grpc_header.go27
-rw-r--r--internal/metadata/featureflag/grpc_header_test.go15
-rw-r--r--internal/middleware/cache/cache.go7
-rw-r--r--internal/middleware/cache/cache_test.go13
-rw-r--r--internal/middleware/limithandler/metrics.go4
-rw-r--r--internal/praefect/auth_test.go27
-rw-r--r--internal/praefect/config/config.go18
-rw-r--r--internal/praefect/config/config_test.go45
-rw-r--r--internal/praefect/coordinator.go17
-rw-r--r--internal/praefect/coordinator_test.go9
-rw-r--r--internal/praefect/dataloss_check_test.go124
-rw-r--r--internal/praefect/datastore/datastore.go4
-rw-r--r--internal/praefect/datastore/glsql/postgres.go3
-rw-r--r--internal/praefect/datastore/glsql/testing.go20
-rw-r--r--internal/praefect/datastore/init_test.go4
-rw-r--r--internal/praefect/datastore/memory.go71
-rw-r--r--internal/praefect/datastore/memory_test.go137
-rw-r--r--internal/praefect/datastore/migrations/20200324001604_add_sql_election_tables.go34
-rw-r--r--internal/praefect/datastore/postgres.go49
-rw-r--r--internal/praefect/datastore/postgres_test.go32
-rw-r--r--internal/praefect/datastore/queue.go41
-rw-r--r--internal/praefect/datastore/queue_test.go4
-rw-r--r--internal/praefect/grpc-proxy/proxy/helper_test.go4
-rw-r--r--internal/praefect/helper_test.go32
-rw-r--r--internal/praefect/info_service_test.go2
-rw-r--r--internal/praefect/metadata/server.go110
-rw-r--r--internal/praefect/metrics/prometheus.go10
-rw-r--r--internal/praefect/models/node.go13
-rw-r--r--internal/praefect/models/node_test.go16
-rw-r--r--internal/praefect/nodes/init_test.go22
-rw-r--r--internal/praefect/nodes/local_elector.go113
-rw-r--r--internal/praefect/nodes/local_elector_test.go70
-rw-r--r--internal/praefect/nodes/manager.go52
-rw-r--r--internal/praefect/nodes/manager_test.go54
-rw-r--r--internal/praefect/nodes/sql_elector.go456
-rw-r--r--internal/praefect/nodes/sql_elector_test.go163
-rw-r--r--internal/praefect/replicator_test.go17
-rw-r--r--internal/praefect/server_test.go21
-rw-r--r--internal/praefect/service/info/dataloss.go28
-rw-r--r--internal/praefect/service/info/server.go2
-rw-r--r--internal/praefect/service/server/info.go84
-rw-r--r--internal/rubyserver/proxy.go13
-rw-r--r--internal/rubyserver/proxy_test.go1
-rw-r--r--internal/rubyserver/rubyserver.go2
-rw-r--r--internal/safe/file_writer_test.go8
-rw-r--r--internal/server/auth_test.go29
-rw-r--r--internal/server/server.go12
-rw-r--r--internal/service/cleanup/internalrefs/cleaner.go6
-rw-r--r--internal/service/commit/commits_helper.go4
-rw-r--r--internal/service/commit/count_commits.go6
-rw-r--r--internal/service/commit/count_diverging_commits.go4
-rw-r--r--internal/service/commit/isancestor.go4
-rw-r--r--internal/service/commit/languages.go2
-rw-r--r--internal/service/commit/last_commit_for_path_test.go2
-rw-r--r--internal/service/commit/list_files.go4
-rw-r--r--internal/service/commit/list_last_commits_for_tree.go2
-rw-r--r--internal/service/commit/list_last_commits_for_tree_test.go2
-rw-r--r--internal/service/commit/raw_blame.go4
-rw-r--r--internal/service/commit/tree_entries.go4
-rw-r--r--internal/service/conflicts/resolve_conflicts_test.go3
-rw-r--r--internal/service/diff/commit.go6
-rw-r--r--internal/service/hooks/post_receive_test.go20
-rw-r--r--internal/service/hooks/pre_receive.go20
-rw-r--r--internal/service/hooks/pre_receive_test.go14
-rw-r--r--internal/service/hooks/testhelper_test.go2
-rw-r--r--internal/service/hooks/update_test.go16
-rw-r--r--internal/service/objectpool/alternates.go4
-rw-r--r--internal/service/objectpool/fetch_into_object_pool_test.go6
-rw-r--r--internal/service/objectpool/get.go4
-rw-r--r--internal/service/operations/cherry_pick_test.go3
-rw-r--r--internal/service/operations/rebase_test.go8
-rw-r--r--internal/service/operations/submodules_test.go2
-rw-r--r--internal/service/operations/tags_test.go18
-rw-r--r--internal/service/operations/testhelper_test.go37
-rw-r--r--internal/service/ref/delete_refs.go2
-rw-r--r--internal/service/ref/refs_test.go40
-rw-r--r--internal/service/register.go4
-rw-r--r--internal/service/remote/fetch_internal_remote_test.go3
-rw-r--r--internal/service/repository/archive_test.go4
-rw-r--r--internal/service/repository/create_test.go1
-rw-r--r--internal/service/repository/fetch_test.go25
-rw-r--r--internal/service/repository/fork_test.go1
-rw-r--r--internal/service/repository/gc.go4
-rw-r--r--internal/service/repository/gc_test.go4
-rw-r--r--internal/service/repository/redirecting_test_server_test.go2
-rw-r--r--internal/service/repository/rename_test.go12
-rw-r--r--internal/service/repository/repack_test.go10
-rw-r--r--internal/service/repository/replicate.go2
-rw-r--r--internal/service/repository/replicate_test.go8
-rw-r--r--internal/service/repository/size.go12
-rw-r--r--internal/service/repository/snapshot.go4
-rw-r--r--internal/service/repository/testhelper_test.go4
-rw-r--r--internal/service/server/disk_stats_test.go6
-rw-r--r--internal/service/server/info.go7
-rw-r--r--internal/service/server/info_test.go21
-rw-r--r--internal/service/server/server.go10
-rw-r--r--internal/service/server/storage_status_unix.go4
-rw-r--r--internal/service/smarthttp/cache.go12
-rw-r--r--internal/service/smarthttp/inforefs.go4
-rw-r--r--internal/service/smarthttp/inforefs_test.go33
-rw-r--r--internal/service/smarthttp/receive_pack.go13
-rw-r--r--internal/service/smarthttp/receive_pack_test.go73
-rw-r--r--internal/service/smarthttp/testhelper_test.go26
-rw-r--r--internal/service/smarthttp/upload_pack.go6
-rw-r--r--internal/service/ssh/receive_pack.go18
-rw-r--r--internal/service/ssh/testhelper_test.go10
-rw-r--r--internal/service/ssh/upload_pack.go6
-rw-r--r--internal/stream/std_stream.go5
-rw-r--r--internal/testhelper/branch.go4
-rw-r--r--internal/testhelper/commit.go9
-rw-r--r--internal/testhelper/githttp.go3
-rw-r--r--internal/testhelper/hook_env.go12
-rw-r--r--internal/testhelper/interface.go25
-rw-r--r--internal/testhelper/interface_test.go9
-rw-r--r--internal/testhelper/remote.go3
-rw-r--r--internal/testhelper/tag.go3
-rw-r--r--internal/testhelper/test_hook.go5
-rw-r--r--internal/testhelper/testdata/home/.gitconfig3
-rw-r--r--internal/testhelper/testhelper.go116
-rw-r--r--internal/testhelper/testserver.go44
-rw-r--r--ruby/Gemfile.lock2
-rw-r--r--ruby/gitlab-shell/lib/gitlab_config.rb46
-rw-r--r--ruby/gitlab-shell/lib/gitlab_init.rb10
-rw-r--r--ruby/gitlab-shell/lib/gitlab_net.rb30
-rw-r--r--ruby/gitlab-shell/spec/gitlab_config_spec.rb116
-rw-r--r--ruby/gitlab-shell/spec/gitlab_custom_hook_spec.rb2
-rw-r--r--ruby/gitlab-shell/spec/gitlab_logger_spec.rb2
-rw-r--r--ruby/gitlab-shell/spec/gitlab_net_spec.rb73
-rw-r--r--ruby/gitlab-shell/spec/spec_helper.rb4
-rw-r--r--ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-none.yml46
-rw-r--r--ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-ok.yml46
-rw-r--r--ruby/gitlab-shell/spec/vcr_cassettes/notify-post-receive.yml46
-rw-r--r--ruby/lib/gitlab/config.rb4
-rw-r--r--ruby/lib/gitlab/git/hook.rb5
-rw-r--r--ruby/lib/gitlab/git/repository.rb10
-rw-r--r--ruby/spec/support/helpers/gitlab_shell_helper.rb3
-rw-r--r--ruby/spec/support/helpers/testdata/home/.gitconfig3
215 files changed, 3751 insertions, 1331 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e7998d861..cf6d4e670 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,13 +4,13 @@ stages:
- publish
default:
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.24
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.26
tags:
- gitlab-org
variables:
DOCKER_DRIVER: overlay2
- CUSTOM_IMAGE_VERSION: ruby-2.6-golang-1.13-git-2.24
+ CUSTOM_IMAGE_VERSION: ruby-2.6-golang-1.13-git-2.26
SAST_DISABLE_DIND: "true"
SAST_DEFAULT_ANALYZERS: "gosec"
@@ -97,33 +97,41 @@ proto:
build:go1.14:
<<: *build_definition
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.24
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.26
build:go1.13:
<<: *build_definition
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.22
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
binaries_go1.14:
<<: *assemble_definition
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.24
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.26
binaries_go1.13:
<<: *assemble_definition
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.22
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
+
+test:go1.14-git-2.26-ruby-2.6:
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.26
+ <<: *test_definition
test:go1.14-git-2.24-ruby-2.6:
image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.24
<<: *test_definition
-test:go1.13-git-2.21-ruby-2.6:
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.21
+test:go1.14-git-2.22-ruby-2.6:
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.14-git-2.26
<<: *test_definition
-test:go1.13-git-2.22-ruby-2.6:
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.22
+test:go1.13-git-2.26-ruby-2.6:
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.26
<<: *test_definition
test:go1.13-git-2.24-ruby-2.6:
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.22
+ <<: *test_definition
+
+test:go1.13-git-2.22-ruby-2.6:
image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
<<: *test_definition
@@ -245,14 +253,15 @@ praefect_sql_test:
- make test-postgres
lint:
+ # Only Go 1.13 currently officially supported
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
stage: test
+ retry: 2
script:
- go version
- make lint
- # Temporary allowed to fail because https://github.com/golangci/golangci-lint/issues/896
- # The job fails frequently https://gitlab.com/gitlab-org/gitaly/-/jobs/494211383
- # and it stops further development process, so it is temporary measure.
- # Must be fixed under: https://gitlab.com/gitlab-org/gitaly/-/issues/2605
+ # TODO: remove the allowing of failures once golangci-lint is fixed
+ # https://gitlab.com/gitlab-org/gitaly/-/issues/2652
allow_failure: true
code_navigation:
diff --git a/.golangci.yml b/.golangci.yml
index ddc73d4d2..40f340537 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -4,22 +4,19 @@ run:
timeout: 5m
modules-download-mode: readonly
-# all available settings of specific linters
-linters-settings:
- staticcheck:
- checks:
- - inherit
-
# list of useful linters could be found at https://github.com/golangci/awesome-go-linters
linters:
disable-all: true
enable:
- - golint
- - goimports # https://godoc.org/golang.org/x/tools/cmd/goimports
- - staticcheck # https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
- - gosimple # https://github.com/dominikh/go-tools/tree/master/cmd/gosimple
- - unused
+ - goimports
+ - stylecheck
+ - deadcode
- govet
+ - ineffassign
+ - megacheck
+ - varcheck
+ - misspell
+ - unconvert
issues:
# Excluding configuration per-path, per-linter, per-text and per-source
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f113f1ea2..bca3836fc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
# Gitaly changelog
+## 12.9.4
+
+- No changes.
+
+## 12.9.3
+
+- No changes.
+
## 12.9.2
- No changes.
@@ -66,6 +74,10 @@
- Simplify loading of required Ruby files. !1942
+## 12.8.9
+
+- No changes.
+
## 12.8.7
- No changes.
@@ -106,6 +118,10 @@
- Praefect sub-commands: avoid garbage in logs. !1819
+## 12.7.9
+
+- No changes.
+
## 12.7.8
- No changes.
diff --git a/Dockerfile b/Dockerfile
index 8396ea028..37a57e430 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,7 +4,7 @@
# See the _support/load-cluster/docker-compose.yml for an example of how to use
# this image
#
-FROM registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
+FROM registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.26
RUN mkdir -p /app/ruby
diff --git a/NOTICE b/NOTICE
index 247d037cc..503167b2e 100644
--- a/NOTICE
+++ b/NOTICE
@@ -671,6 +671,30 @@ The above copyright notice and this permission notice shall be included in all c
THE SOFTWARE IS PROVIDED &#34;AS IS&#34;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+LICENSE - github.com/mattn/go-runewidth
+The MIT License (MIT)
+
+Copyright (c) 2016 Yasuhiro Matsumoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the &#34;Software&#34;), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED &#34;AS IS&#34;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
LICENSE - github.com/matttproud/golang_protobuf_extensions/pbutil
Apache License
Version 2.0, January 2004
@@ -879,6 +903,28 @@ NOTICE - github.com/matttproud/golang_protobuf_extensions/pbutil
Copyright 2012 Matt T. Proud (matt.proud@gmail.com)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+LICENSE.md - github.com/olekukonko/tablewriter
+Copyright (C) 2014 by Oleku Konko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the &#34;Software&#34;), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED &#34;AS IS&#34;, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
LICENSE - github.com/opentracing/opentracing-go
The MIT License (MIT)
diff --git a/README.md b/README.md
index aea4e2d3b..aec604d7d 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ Gitaly requires Go 1.13.9 or newer and Ruby 2.6. Run `make` to download
and compile Ruby dependencies, and to compile the Gitaly Go
executable.
-Gitaly uses `git`. Versions `2.22.0` and `2.24.0` are supported.
+Gitaly uses `git`. Versions `2.24.x` and `2.26.x` are supported.
## Configuration
diff --git a/_support/Makefile.template b/_support/Makefile.template
index 33cc8828c..feae6dcce 100644
--- a/_support/Makefile.template
+++ b/_support/Makefile.template
@@ -11,7 +11,7 @@ INSTALL_DEST_DIR := $(DESTDIR)$(PREFIX)/bin/
BUNDLE_FLAGS ?= {{ .BundleFlags }}
ASSEMBLY_ROOT ?= {{ .BuildDir }}/assembly
BUILD_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
-SHELL = /usr/bin/env bash -eo pipefail
+SHELL = /usr/bin/env bash -e
unexport GOROOT
export GOBIN = {{ .BuildDir }}/bin
@@ -100,7 +100,7 @@ test: test-go rspec rspec-gitlab-shell
.PHONY: test-go
test-go: prepare-tests {{ .GoJunitReport }}
- @cd {{ .SourceDir }} && go test -v -tags "$(BUILD_TAGS)" -count=1 {{ join .AllPackages " " }} 2>&1 | tee >({{ .GoJunitReport }} > report-${CI_JOB_NAME}.xml)
+ @cd {{ .SourceDir }} && go test -v -tags "$(BUILD_TAGS)" -count=1 {{ join .AllPackages " " }} 2>&1 | tee >({{ .GoJunitReport }} > report-${CI_JOB_NAME}.xml) && exit $${PIPESTATUS[0]}
.PHONY: test-with-proxies
test-with-proxies: prepare-tests
@@ -115,7 +115,7 @@ test-with-praefect: build prepare-tests
.PHONY: race-go
race-go: prepare-tests {{ .GoJunitReport }}
- @cd {{ .SourceDir }} && go test -v -tags "$(BUILD_TAGS)" -race {{ join .AllPackages " " }} 2>&1 | tee >({{ .GoJunitReport }} > report-${CI_JOB_NAME}.xml)
+ @cd {{ .SourceDir }} && go test -v -tags "$(BUILD_TAGS)" -race {{ join .AllPackages " " }} 2>&1 | tee >({{ .GoJunitReport }} > report-${CI_JOB_NAME}.xml) && exit $${PIPESTATUS[0]}
.PHONY: rspec
rspec: assemble-go prepare-tests
@@ -128,7 +128,7 @@ rspec-gitlab-shell: {{ .GitlabShellDir }}/config.yml assemble-go prepare-tests
.PHONY: test-postgres
test-postgres: prepare-tests
- @cd {{ .SourceDir }} && go test -tags postgres -count=1 gitlab.com/gitlab-org/gitaly/internal/praefect/datastore/...
+ @cd {{ .SourceDir }} && go test -tags postgres -count=1 gitlab.com/gitlab-org/gitaly/internal/praefect/...
.PHONY: verify
verify: check-mod-tidy check-formatting notice-up-to-date check-proto rubocop
@@ -141,9 +141,8 @@ check-mod-tidy:
.PHONY: lint
lint: {{ .GoLint }}
@cd {{ .SourceDir }} && \
- {{ .GoLint }} run --out-format tab --config .golangci.yml; \
- EXIT_CODE=$$?;\
- exit $$EXIT_CODE
+ {{ .GoLint }} cache clean; \
+ {{ .GoLint }} run --out-format tab --config .golangci.yml
.PHONY: check-formatting
check-formatting: {{ .GitalyFmt }}
diff --git a/_support/instrumented-cluster/docker-compose.yml b/_support/instrumented-cluster/docker-compose.yml
index bcd491740..bde60b474 100644
--- a/_support/instrumented-cluster/docker-compose.yml
+++ b/_support/instrumented-cluster/docker-compose.yml
@@ -2,7 +2,7 @@ version: "2.4"
services:
gitaly1repos:
- image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.24
+ image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6-golang-1.13-git-2.26
volumes:
- gitalydata1:/repositories
- ./gitaly1repos/setup.sh:/setup.sh
diff --git a/_support/terraform/main.tf b/_support/terraform/main.tf
index 629ac8a1d..f7ab4d52c 100644
--- a/_support/terraform/main.tf
+++ b/_support/terraform/main.tf
@@ -14,7 +14,9 @@ variable "startup_script" {
EOF
}
variable "gitaly_machine_type" { default = "n1-standard-2" }
+variable "praefect_machine_type" { default = "n1-standard-1" }
variable "gitaly_disk_size" { default = "100" }
+variable "praefect_disk_size" { default = "10" }
variable "praefect_sql_password" { }
provider "google" {
@@ -42,9 +44,14 @@ resource "google_sql_database_instance" "praefect_sql" {
ip_configuration{
ipv4_enabled = true
- authorized_networks {
- name = "praefect"
- value = google_compute_instance.praefect.network_interface[0].access_config[0].nat_ip
+ dynamic "authorized_networks" {
+ for_each = google_compute_instance.praefect
+ iterator = praefect
+
+ content {
+ name = "praefect-${praefect.key}"
+ value = praefect.value.network_interface[0].access_config[0].nat_ip
+ }
}
}
}
@@ -97,12 +104,14 @@ output "gitlab_external_ip" {
}
resource "google_compute_instance" "praefect" {
- name = format("%s-praefect", var.praefect_demo_cluster_name)
- machine_type = "n1-standard-1"
+ count = 1
+ name = "${var.praefect_demo_cluster_name}-praefect-${count.index + 1}"
+ machine_type = var.praefect_machine_type
boot_disk {
initialize_params {
image = var.os_image
+ size = var.praefect_disk_size
}
}
@@ -118,10 +127,17 @@ resource "google_compute_instance" "praefect" {
}
output "praefect_internal_ip" {
- value = google_compute_instance.praefect.network_interface[0].network_ip
+ value = {
+ for instance in google_compute_instance.praefect:
+ instance.name => instance.network_interface[0].network_ip
+ }
}
+
output "praefect_ssh_ip" {
- value = google_compute_instance.praefect.network_interface[0].access_config[0].nat_ip
+ value = {
+ for instance in google_compute_instance.praefect:
+ instance.name => instance.network_interface[0].access_config[0].nat_ip
+ }
}
resource "google_compute_instance" "gitaly" {
diff --git a/auth/extract_test.go b/auth/extract_test.go
index 4274785c4..510fb1790 100644
--- a/auth/extract_test.go
+++ b/auth/extract_test.go
@@ -7,51 +7,8 @@ import (
"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
"github.com/stretchr/testify/require"
- "google.golang.org/grpc/codes"
- "google.golang.org/grpc/credentials"
- "google.golang.org/grpc/metadata"
- "google.golang.org/grpc/status"
)
-func TestCheckTokenV1(t *testing.T) {
- secret := "secret 1234"
-
- testCases := []struct {
- desc string
- md metadata.MD
- code codes.Code
- }{
- {
- desc: "ok",
- md: credsMD(t, RPCCredentials(secret)),
- code: codes.OK,
- },
- {
- desc: "denied",
- md: credsMD(t, RPCCredentials("wrong secret")),
- code: codes.PermissionDenied,
- },
- {
- desc: "invalid, not bearer",
- md: credsMD(t, &invalidCreds{"foobar"}),
- code: codes.Unauthenticated,
- },
- {
- desc: "invalid, bearer but not base64",
- md: credsMD(t, &invalidCreds{"Bearer foo!!bar"}),
- code: codes.Unauthenticated,
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.desc, func(t *testing.T) {
- ctx := metadata.NewIncomingContext(context.Background(), tc.md)
- err := CheckToken(ctx, secret, time.Now())
- require.Equal(t, tc.code, status.Code(err), "expected grpc code in error %v", err)
- })
- }
-}
-
func TestCheckTokenV2(t *testing.T) {
targetTime := time.Unix(1535671600, 0)
secret := []byte("foo")
@@ -97,9 +54,14 @@ func TestCheckTokenV2(t *testing.T) {
result: errDenied,
},
{
+ desc: "Invalid token format",
+ token: "foo.bar",
+ result: errUnauthenticated,
+ },
+ {
desc: "Empty token",
token: "",
- result: errDenied,
+ result: errUnauthenticated,
},
}
@@ -113,19 +75,3 @@ func TestCheckTokenV2(t *testing.T) {
})
}
}
-
-func credsMD(t *testing.T, creds credentials.PerRPCCredentials) metadata.MD {
- md, err := creds.GetRequestMetadata(context.Background())
- require.NoError(t, err)
- return metadata.New(md)
-}
-
-type invalidCreds struct {
- authHeader string
-}
-
-func (invalidCreds) RequireTransportSecurity() bool { return false }
-
-func (ic *invalidCreds) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
- return map[string]string{"authorization": ic.authHeader}, nil
-}
diff --git a/auth/rpccredentials.go b/auth/rpccredentials.go
index 923ee8567..9ebf19d15 100644
--- a/auth/rpccredentials.go
+++ b/auth/rpccredentials.go
@@ -9,14 +9,6 @@ import (
"google.golang.org/grpc/credentials"
)
-// RPCCredentials can be used with grpc.WithPerRPCCredentials to create a
-// grpc.DialOption that uses v1 Gitaly authentication for authentication
-// with a Gitaly server. The shared secret must match the one used on the
-// Gitaly server.
-func RPCCredentials(sharedSecret string) credentials.PerRPCCredentials {
- return &rpcCredentialsV2{sharedSecret: sharedSecret}
-}
-
// RPCCredentialsV2 can be used with grpc.WithPerRPCCredentials to create
// a grpc.DialOption that inserts an V2 (HMAC) token with the current
// timestamp for authentication with a Gitaly server. The shared secret
diff --git a/auth/token.go b/auth/token.go
index 9355750aa..dee53227c 100644
--- a/auth/token.go
+++ b/auth/token.go
@@ -4,9 +4,8 @@ import (
"context"
"crypto/hmac"
"crypto/sha256"
- "crypto/subtle"
- "encoding/base64"
"encoding/hex"
+ "fmt"
"strconv"
"strings"
"time"
@@ -58,17 +57,7 @@ func CheckToken(ctx context.Context, secret string, targetTime time.Time) error
return errUnauthenticated
}
- switch authInfo.Version {
- case "v1":
- decodedToken, err := base64.StdEncoding.DecodeString(authInfo.Message)
- if err != nil {
- return errUnauthenticated
- }
-
- if tokensEqual(decodedToken, []byte(secret)) {
- return nil
- }
- case "v2":
+ if authInfo.Version == "v2" {
if v2HmacInfoValid(authInfo.Message, authInfo.SignedMessage, []byte(secret), targetTime, timestampThreshold) {
return nil
}
@@ -77,10 +66,6 @@ func CheckToken(ctx context.Context, secret string, targetTime time.Time) error
return errDenied
}
-func tokensEqual(tok1, tok2 []byte) bool {
- return subtle.ConstantTimeCompare(tok1, tok2) == 1
-}
-
// ExtractAuthInfo returns an `AuthInfo` with the data extracted from `ctx`
func ExtractAuthInfo(ctx context.Context) (*AuthInfo, error) {
token, err := grpc_auth.AuthFromMD(ctx, "bearer")
@@ -89,12 +74,10 @@ func ExtractAuthInfo(ctx context.Context) (*AuthInfo, error) {
return nil, err
}
- split := strings.SplitN(string(token), ".", 3)
+ split := strings.SplitN(token, ".", 3)
- // v1 is base64-encoded using base64.StdEncoding, which cannot contain a ".".
- // A v1 token cannot slip through here.
if len(split) != 3 {
- return &AuthInfo{Version: "v1", Message: token}, nil
+ return nil, fmt.Errorf("invalid token format")
}
version, sig, msg := split[0], split[1], split[2]
diff --git a/changelogs/unreleased/cc-git-2-26-0.yml b/changelogs/unreleased/cc-git-2-26-0.yml
new file mode 100644
index 000000000..9f0e5cb9e
--- /dev/null
+++ b/changelogs/unreleased/cc-git-2-26-0.yml
@@ -0,0 +1,5 @@
+---
+title: Upgrade to Git 2.26
+merge_request: 1983
+author:
+type: deprecated
diff --git a/changelogs/unreleased/jc-add-metric-for-node-health.yml b/changelogs/unreleased/jc-add-metric-for-node-health.yml
new file mode 100644
index 000000000..e3173e7ae
--- /dev/null
+++ b/changelogs/unreleased/jc-add-metric-for-node-health.yml
@@ -0,0 +1,5 @@
+---
+title: Adding metrics to track which nodes are up and down
+merge_request: 2019
+author:
+type: added
diff --git a/changelogs/unreleased/jc-call-hook-rpc.yml b/changelogs/unreleased/jc-call-hook-rpc.yml
new file mode 100644
index 000000000..093e1d2c1
--- /dev/null
+++ b/changelogs/unreleased/jc-call-hook-rpc.yml
@@ -0,0 +1,5 @@
+---
+title: Call Hook RPCs from gitaly-hooks binary
+merge_request: 1740
+author:
+type: performance
diff --git a/changelogs/unreleased/jc-fix-praefect-server-info.yml b/changelogs/unreleased/jc-fix-praefect-server-info.yml
new file mode 100644
index 000000000..828390ca6
--- /dev/null
+++ b/changelogs/unreleased/jc-fix-praefect-server-info.yml
@@ -0,0 +1,5 @@
+---
+title: Modify Praefect's server info implementation
+merge_request: 1991
+author:
+type: fixed
diff --git a/changelogs/unreleased/jc-operations-service-call-hook-rpcs.yml b/changelogs/unreleased/jc-operations-service-call-hook-rpcs.yml
new file mode 100644
index 000000000..6d4cab2b6
--- /dev/null
+++ b/changelogs/unreleased/jc-operations-service-call-hook-rpcs.yml
@@ -0,0 +1,5 @@
+---
+title: Call hook rpcs from operations service
+merge_request: 2034
+author:
+type: added
diff --git a/changelogs/unreleased/jc-pass-token-into-hook.yml b/changelogs/unreleased/jc-pass-token-into-hook.yml
new file mode 100644
index 000000000..bee806f51
--- /dev/null
+++ b/changelogs/unreleased/jc-pass-token-into-hook.yml
@@ -0,0 +1,5 @@
+---
+title: Pass gitaly token into gitaly-hooks
+merge_request: 2035
+author:
+type: fixed
diff --git a/changelogs/unreleased/jc-revert-revert-846222e0.yml b/changelogs/unreleased/jc-revert-revert-846222e0.yml
new file mode 100644
index 000000000..cdbe3c84c
--- /dev/null
+++ b/changelogs/unreleased/jc-revert-revert-846222e0.yml
@@ -0,0 +1,5 @@
+---
+title: Exercise Operations service tests with auth
+merge_request: 2059
+author:
+type: fixed
diff --git a/changelogs/unreleased/po-remove-inforef-ff.yml b/changelogs/unreleased/po-remove-inforef-ff.yml
new file mode 100644
index 000000000..5def8d352
--- /dev/null
+++ b/changelogs/unreleased/po-remove-inforef-ff.yml
@@ -0,0 +1,5 @@
+---
+title: Remove feature flags for InfoRef cache
+merge_request: 2038
+author:
+type: other
diff --git a/changelogs/unreleased/ps-ctx-persistence-timeout.yml b/changelogs/unreleased/ps-ctx-persistence-timeout.yml
new file mode 100644
index 000000000..ca6419faa
--- /dev/null
+++ b/changelogs/unreleased/ps-ctx-persistence-timeout.yml
@@ -0,0 +1,5 @@
+---
+title: 'Praefect: avoid early request cancellation when queueing replication jobs'
+merge_request: 2062
+author:
+type: fixed
diff --git a/changelogs/unreleased/ps-drop-auth-v1.yml b/changelogs/unreleased/ps-drop-auth-v1.yml
new file mode 100644
index 000000000..f5ef196db
--- /dev/null
+++ b/changelogs/unreleased/ps-drop-auth-v1.yml
@@ -0,0 +1,5 @@
+---
+title: Drop support for Gitaly v1 authentication
+merge_request: 2024
+author:
+type: deprecated
diff --git a/changelogs/unreleased/ps-linter-unconvert.yml b/changelogs/unreleased/ps-linter-unconvert.yml
new file mode 100644
index 000000000..1bdb73684
--- /dev/null
+++ b/changelogs/unreleased/ps-linter-unconvert.yml
@@ -0,0 +1,5 @@
+---
+title: 'Static code analysis: unconvert'
+merge_request: 2046
+author:
+type: other
diff --git a/changelogs/unreleased/ps-proxyheaderwhitelist-race.yml b/changelogs/unreleased/ps-proxyheaderwhitelist-race.yml
new file mode 100644
index 000000000..d7a400caf
--- /dev/null
+++ b/changelogs/unreleased/ps-proxyheaderwhitelist-race.yml
@@ -0,0 +1,5 @@
+---
+title: Race condition on ProxyHeaderWhitelist
+merge_request: 2025
+author:
+type: fixed
diff --git a/changelogs/unreleased/sh-add-sql-migrate-status.yml b/changelogs/unreleased/sh-add-sql-migrate-status.yml
new file mode 100644
index 000000000..40e5a3574
--- /dev/null
+++ b/changelogs/unreleased/sh-add-sql-migrate-status.yml
@@ -0,0 +1,5 @@
+---
+title: Add Praefect command to show migration status
+merge_request: 2041
+author:
+type: added
diff --git a/changelogs/unreleased/sh-fix-local-elector-locking.yml b/changelogs/unreleased/sh-fix-local-elector-locking.yml
new file mode 100644
index 000000000..2c5cf4def
--- /dev/null
+++ b/changelogs/unreleased/sh-fix-local-elector-locking.yml
@@ -0,0 +1,5 @@
+---
+title: Fix localElector locking issues
+merge_request: 2030
+author:
+type: fixed
diff --git a/changelogs/unreleased/sh-improve-sql-leader-election.yml b/changelogs/unreleased/sh-improve-sql-leader-election.yml
new file mode 100644
index 000000000..0106c3ade
--- /dev/null
+++ b/changelogs/unreleased/sh-improve-sql-leader-election.yml
@@ -0,0 +1,5 @@
+---
+title: Add SQL-based election for shard primaries
+merge_request: 1979
+author:
+type: added
diff --git a/changelogs/unreleased/sh-make-ignore-unknown-default.yml b/changelogs/unreleased/sh-make-ignore-unknown-default.yml
new file mode 100644
index 000000000..980c66cb6
--- /dev/null
+++ b/changelogs/unreleased/sh-make-ignore-unknown-default.yml
@@ -0,0 +1,5 @@
+---
+title: Make Praefect sql-migrate ignore unknown migrations by default
+merge_request: 2058
+author:
+type: changed
diff --git a/changelogs/unreleased/sh-support-sql-migrate-ignore-unknown.yml b/changelogs/unreleased/sh-support-sql-migrate-ignore-unknown.yml
new file mode 100644
index 000000000..1247b62b7
--- /dev/null
+++ b/changelogs/unreleased/sh-support-sql-migrate-ignore-unknown.yml
@@ -0,0 +1,5 @@
+---
+title: Support ignoring unknown Praefect migrations
+merge_request: 2039
+author:
+type: changed
diff --git a/changelogs/unreleased/smh-dataloss-cmd.yml b/changelogs/unreleased/smh-dataloss-cmd.yml
new file mode 100644
index 000000000..1e81d2b83
--- /dev/null
+++ b/changelogs/unreleased/smh-dataloss-cmd.yml
@@ -0,0 +1,5 @@
+---
+title: Praefect dataloss subcommand
+merge_request: 2057
+author:
+type: added
diff --git a/client/address_parser.go b/client/address_parser.go
index a052342ae..1d0d560cb 100644
--- a/client/address_parser.go
+++ b/client/address_parser.go
@@ -11,11 +11,11 @@ import (
func extractHostFromRemoteURL(rawAddress string) (hostAndPort string, err error) {
u, err := url.Parse(rawAddress)
if err != nil {
- return "", err
+ return "", fmt.Errorf("failed to parse remote addresses: %w", err)
}
if u.Path != "" {
- return "", fmt.Errorf("remote addresses should not have a path")
+ return "", fmt.Errorf("remote addresses should not have a path: %q", u.Path)
}
if u.Host == "" {
diff --git a/client/dial.go b/client/dial.go
index 4a1072c52..e1a6fd12b 100644
--- a/client/dial.go
+++ b/client/dial.go
@@ -31,17 +31,17 @@ func DialContext(ctx context.Context, rawAddress string, connOpts []grpc.DialOpt
switch getConnectionType(rawAddress) {
case invalidConnection:
- return nil, fmt.Errorf("invalid connection string: %s", rawAddress)
+ return nil, fmt.Errorf("invalid connection string: %q", rawAddress)
case tlsConnection:
canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to extract host for 'tls' connection: %w", err)
}
certPool, err := gitaly_x509.SystemCertPool()
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to get system certificat pool for 'tls' connection: %w", err)
}
creds := credentials.NewClientTLSFromCert(certPool, "")
@@ -50,7 +50,7 @@ func DialContext(ctx context.Context, rawAddress string, connOpts []grpc.DialOpt
case tcpConnection:
canonicalAddress, err = extractHostFromRemoteURL(rawAddress) // Ensure the form: "host:port" ...
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to extract host for 'tcp' connection: %w", err)
}
connOpts = append(connOpts, grpc.WithInsecure())
@@ -65,7 +65,7 @@ func DialContext(ctx context.Context, rawAddress string, connOpts []grpc.DialOpt
grpc.WithContextDialer(func(ctx context.Context, addr string) (conn net.Conn, err error) {
path, err := extractPathFromSocketURL(addr)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to extract host for 'unix' connection: %w", err)
}
d := net.Dialer{}
@@ -81,7 +81,7 @@ func DialContext(ctx context.Context, rawAddress string, connOpts []grpc.DialOpt
conn, err := grpc.DialContext(ctx, canonicalAddress, connOpts...)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to dial %q connection: %w", canonicalAddress, err)
}
return conn, nil
diff --git a/cmd/gitaly-hooks/README.md b/cmd/gitaly-hooks/README.md
new file mode 100644
index 000000000..806ae00d4
--- /dev/null
+++ b/cmd/gitaly-hooks/README.md
@@ -0,0 +1,39 @@
+# gitaly-hooks
+
+`gitaly-hooks` is a binary that is the single point of entry for git hooks through gitaly.
+
+## How is it invoked?
+
+`gitaly-hooks` has the following subcommands:
+
+| subcommand | purpose | arguments | stdin |
+|--------------|-------------------------------------------------|--------------------------------------|---------------------------------------------|
+| `check` | checks if the hooks can reach the gitlab server | none | none |
+| `pre-receive` | used as the git pre-receive hook | none | `<old-value>` SP `<new-value>` SP `<ref-name>` LF |
+| `update` | used as the git update hook | `<ref-name>` `<old-object>` `<new-object>` | none
+| `post-receive` | used as the git post-receive hook | none | `<old-value>` SP `<new-value>` SP `<ref-name>` LF |
+
+## Where is it invoked from?
+
+There are two main code paths that call `gitaly-hooks`.
+
+### git receive-pack (SSH & HTTP)
+
+We have two RPCs that perform the `git receive-pack` function, [SSHReceivePack](https://gitlab.com/gitlab-org/gitaly/-/blob/master/internal/service/ssh/receive_pack.go) and [PostReceivePack](https://gitlab.com/gitlab-org/gitaly/-/blob/master/internal/service/smarthttp/receive_pack.go).
+
+Both of these RPCs, when executing `git receive-pack`, set `core.hooksPath` to the path of the `gitaly-hooks` binary. [That happens here in `ReceivePackConfig`](https://gitlab.com/gitlab-org/gitaly/-/blob/master/internal/git/receivepack.go).
+
+### Operations service RPCs
+
+In the [operations service](https://gitlab.com/gitlab-org/gitaly/-/tree/master/internal/service/operations) there are RPCs that call out to `gitaly-ruby`, which then do certain operations that execute git hooks.
+This is accomplished through the `with_hooks` method [here](https://gitlab.com/gitlab-org/gitaly/-/blob/master/ruby/lib/gitlab/git/operation_service.rb). Eventually the [`hook.rb`](https://gitlab.com/gitlab-org/gitaly/-/blob/master/ruby/lib/gitlab/git/hook.rb) is
+called, which then calls the `gitaly-hooks` binary. This method doesn't rely on git to run the hooks. Instead, the arguments and input to the
+hooks are built in ruby and then get shelled out to `gitaly-hooks`.
+
+## What does gitaly-hooks do?
+
+`gitaly-hooks` will take the arguments and make an RPC call to `PreReceiveHook`, `UpdateHook`, or `PostReceiveHook` accordingly. These RPCs then call out to the [ruby hooks](https://gitlab.com/gitlab-org/gitaly/-/tree/master/ruby/gitlab-shell/hooks). The goal is to port these ruby hooks into Go.
+
+**Note:**
+Currently `gitaly-hooks` will only make an RPC call to `PreReceiveHook`, `UpdateHook`, or `PostReceiveHook` if a feature flag `gitaly_hook_rpc` is enabled. Otherwise, `gitaly-hooks` falls back to calling the ruby hooks directly.
+
diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go
index 1bc42aa59..ab88b235c 100644
--- a/cmd/gitaly-hooks/hooks.go
+++ b/cmd/gitaly-hooks/hooks.go
@@ -4,23 +4,35 @@ import (
"context"
"errors"
"fmt"
+ "io"
"log"
"os"
"os/exec"
"path/filepath"
+ "strconv"
+ "strings"
"github.com/BurntSushi/toml"
+ "github.com/golang/protobuf/jsonpb"
+ gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
+ "gitlab.com/gitlab-org/gitaly/client"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/gitlabshell"
gitalylog "gitlab.com/gitlab-org/gitaly/internal/log"
+ "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/metadata"
+ "gitlab.com/gitlab-org/gitaly/internal/stream"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+ "gitlab.com/gitlab-org/gitaly/streamio"
+ "google.golang.org/grpc"
)
func main() {
var logger = gitalylog.NewHookLogger()
if len(os.Args) < 2 {
- logger.Fatal(errors.New("requires hook name"))
+ logger.Fatalf("requires hook name. args: %v", os.Args)
}
subCmd := os.Args[1]
@@ -36,55 +48,224 @@ func main() {
os.Exit(status)
}
- gitalyRubyDir := os.Getenv("GITALY_RUBY_DIR")
- if gitalyRubyDir == "" {
- logger.Fatal(errors.New("GITALY_RUBY_DIR not set"))
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ if os.Getenv(featureflag.HooksRPCEnvVar) != "true" {
+ executeScript(ctx, subCmd, logger)
+ return
}
- rubyHookPath := filepath.Join(gitalyRubyDir, "gitlab-shell", "hooks", subCmd)
+ repository, err := repositoryFromEnv()
+ if err != nil {
+ logger.Fatalf("error when getting repository: %v", err)
+ }
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
+ conn, err := gitalyFromEnv()
+ if err != nil {
+ logger.Fatalf("error when connecting to gitaly: %v", err)
+ }
- var hookCmd *exec.Cmd
+ hookClient := gitalypb.NewHookServiceClient(conn)
+
+ hookStatus := int32(1)
switch subCmd {
case "update":
args := os.Args[2:]
if len(args) != 3 {
- logger.Fatal(errors.New("update hook missing required arguments"))
+ logger.Fatalf("hook %q is missing required arguments", subCmd)
}
+ ref, oldValue, newValue := args[0], args[1], args[2]
- hookCmd = exec.Command(rubyHookPath, args...)
- case "pre-receive", "post-receive":
- hookCmd = exec.Command(rubyHookPath)
+ req := &gitalypb.UpdateHookRequest{
+ Repository: repository,
+ EnvironmentVariables: glValues(),
+ Ref: []byte(ref),
+ OldValue: oldValue,
+ NewValue: newValue,
+ }
+
+ updateHookStream, err := hookClient.UpdateHook(ctx, req)
+ if err != nil {
+ logger.Fatalf("error when starting command for %q: %v", subCmd, err)
+ }
+
+ if hookStatus, err = stream.Handler(func() (stream.StdoutStderrResponse, error) {
+ return updateHookStream.Recv()
+ }, noopSender, os.Stdout, os.Stderr); err != nil {
+ logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
+ }
+ case "pre-receive":
+ preReceiveHookStream, err := hookClient.PreReceiveHook(ctx)
+ if err != nil {
+ logger.Fatalf("error when getting preReceiveHookStream client for %q: %v", subCmd, err)
+ }
+
+ environment := glValues()
+ if env, ok := os.LookupEnv(metadata.PraefectEnvKey); ok {
+ praefectEnv := fmt.Sprintf("%s=%s", metadata.PraefectEnvKey, env)
+ environment = append(environment, praefectEnv)
+ }
+ if err := preReceiveHookStream.Send(&gitalypb.PreReceiveHookRequest{
+ Repository: repository,
+ EnvironmentVariables: environment,
+ }); err != nil {
+ logger.Fatalf("error when sending request for %q: %v", subCmd, err)
+ }
+
+ f := sendFunc(streamio.NewWriter(func(p []byte) error {
+ return preReceiveHookStream.Send(&gitalypb.PreReceiveHookRequest{Stdin: p})
+ }), preReceiveHookStream, os.Stdin)
+
+ if hookStatus, err = stream.Handler(func() (stream.StdoutStderrResponse, error) {
+ return preReceiveHookStream.Recv()
+ }, f, os.Stdout, os.Stderr); err != nil {
+ logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
+ }
+ case "post-receive":
+ postReceiveHookStream, err := hookClient.PostReceiveHook(ctx)
+ if err != nil {
+ logger.Fatalf("error when getting stream client for %q: %v", subCmd, err)
+ }
+
+ if err := postReceiveHookStream.Send(&gitalypb.PostReceiveHookRequest{
+ Repository: repository,
+ EnvironmentVariables: glValues(),
+ GitPushOptions: gitPushOptions(),
+ }); err != nil {
+ logger.Fatalf("error when sending request for %q: %v", subCmd, err)
+ }
+
+ f := sendFunc(streamio.NewWriter(func(p []byte) error {
+ return postReceiveHookStream.Send(&gitalypb.PostReceiveHookRequest{Stdin: p})
+ }), postReceiveHookStream, os.Stdin)
+
+ if hookStatus, err = stream.Handler(func() (stream.StdoutStderrResponse, error) {
+ return postReceiveHookStream.Recv()
+ }, f, os.Stdout, os.Stderr); err != nil {
+ logger.Fatalf("error when receiving data for %q: %v", subCmd, err)
+ }
default:
- logger.Fatal(errors.New("hook name invalid"))
+ logger.Fatalf("subcommand name invalid: %q", subCmd)
}
- cmd, err := command.New(ctx, hookCmd, os.Stdin, os.Stdout, os.Stderr, os.Environ()...)
+ os.Exit(int(hookStatus))
+}
+
+func noopSender(c chan error) {}
+
+func repositoryFromEnv() (*gitalypb.Repository, error) {
+ repoString, ok := os.LookupEnv("GITALY_REPO")
+ if !ok {
+ return nil, errors.New("GITALY_REPO not found")
+ }
+
+ var repo gitalypb.Repository
+ if err := jsonpb.UnmarshalString(repoString, &repo); err != nil {
+ return nil, fmt.Errorf("unmarshal JSON %q: %w", repoString, err)
+ }
+
+ pwd, err := os.Getwd()
if err != nil {
- logger.Fatalf("error when starting command for %v: %v", rubyHookPath, err)
+ return nil, fmt.Errorf("can't define working directory: %w", err)
}
- if err = cmd.Wait(); err != nil {
- os.Exit(1)
+ gitObjDirAbs, ok := os.LookupEnv("GIT_OBJECT_DIRECTORY")
+ if ok {
+ gitObjDir, err := filepath.Rel(pwd, gitObjDirAbs)
+ if err != nil {
+ return nil, fmt.Errorf("can't define rel path %q: %w", gitObjDirAbs, err)
+ }
+ repo.GitObjectDirectory = gitObjDir
+ }
+ gitAltObjDirsAbs, ok := os.LookupEnv("GIT_ALTERNATE_OBJECT_DIRECTORIES")
+ if ok {
+ var gitAltObjDirs []string
+ for _, gitAltObjDirAbs := range strings.Split(gitAltObjDirsAbs, ":") {
+ gitAltObjDir, err := filepath.Rel(pwd, gitAltObjDirAbs)
+ if err != nil {
+ return nil, fmt.Errorf("can't define rel path %q: %w", gitAltObjDirAbs, err)
+ }
+ gitAltObjDirs = append(gitAltObjDirs, gitAltObjDir)
+ }
+
+ repo.GitAlternateObjectDirectories = gitAltObjDirs
+ }
+
+ return &repo, nil
+}
+
+func gitalyFromEnv() (*grpc.ClientConn, error) {
+ gitalySocket := os.Getenv("GITALY_SOCKET")
+ if gitalySocket == "" {
+ return nil, errors.New("GITALY_SOCKET not set")
+ }
+
+ gitalyToken, ok := os.LookupEnv("GITALY_TOKEN")
+ if !ok {
+ return nil, errors.New("GITALY_TOKEN not set")
+ }
+
+ dialOpts := client.DefaultDialOpts
+ if gitalyToken != "" {
+ dialOpts = append(dialOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(gitalyToken)))
+ }
+
+ conn, err := client.Dial("unix://"+gitalySocket, dialOpts)
+ if err != nil {
+ return nil, fmt.Errorf("error when dialing: %w", err)
+ }
+
+ return conn, nil
+}
+
+func glValues() []string {
+ var glEnvVars []string
+ for _, kv := range os.Environ() {
+ if strings.HasPrefix(kv, "GL_") {
+ glEnvVars = append(glEnvVars, kv)
+ }
+ }
+
+ return glEnvVars
+}
+
+func gitPushOptions() []string {
+ var gitPushOptions []string
+
+ gitPushOptionCount, err := strconv.Atoi(os.Getenv("GIT_PUSH_OPTION_COUNT"))
+ if err != nil {
+ return gitPushOptions
+ }
+
+ for i := 0; i < gitPushOptionCount; i++ {
+ gitPushOptions = append(gitPushOptions, os.Getenv(fmt.Sprintf("GIT_PUSH_OPTION_%d", i)))
+ }
+
+ return gitPushOptions
+}
+
+func sendFunc(reqWriter io.Writer, stream grpc.ClientStream, stdin io.Reader) func(errC chan error) {
+ return func(errC chan error) {
+ _, errSend := io.Copy(reqWriter, stdin)
+ stream.CloseSend()
+ errC <- errSend
}
}
func check(configPath string) (int, error) {
cfgFile, err := os.Open(configPath)
if err != nil {
- return 1, fmt.Errorf("error when opening config file: %v", err)
+ return 1, fmt.Errorf("failed to open config file: %w", err)
}
defer cfgFile.Close()
var c config.Cfg
if _, err := toml.DecodeReader(cfgFile, &c); err != nil {
- fmt.Println(err)
- return 1, err
+ return 1, fmt.Errorf("failed to decode toml: %w", err)
}
cmd := exec.Command(filepath.Join(c.GitlabShell.Dir, "bin", "check"))
@@ -96,8 +277,51 @@ func check(configPath string) (int, error) {
if status, ok := command.ExitStatus(err); ok {
return status, nil
}
- return 1, err
+ return 1, fmt.Errorf("failed to run %q: %w", cmd.String(), err)
}
return 0, nil
}
+
+func executeScript(ctx context.Context, subCmd string, logger *gitalylog.HookLogger) {
+ gitalyRubyDir := os.Getenv("GITALY_RUBY_DIR")
+ if gitalyRubyDir == "" {
+ logger.Fatalf("GITALY_RUBY_DIR not set")
+ }
+
+ rubyHookPath := filepath.Join(gitalyRubyDir, "gitlab-shell", "hooks", subCmd)
+
+ var hookCmd *exec.Cmd
+
+ switch subCmd {
+ case "update":
+ args := os.Args[2:]
+ requireArgs := 3
+ if len(args) != requireArgs {
+ logger.Fatalf("update hook requires %d arguments, got: %d", requireArgs, len(args))
+ }
+
+ hookCmd = exec.Command(rubyHookPath, args...)
+ case "pre-receive", "post-receive":
+ hookCmd = exec.Command(rubyHookPath)
+ default:
+ logger.Fatalf("subcommand name invalid: %v", subCmd)
+ }
+
+ cmd, err := command.New(ctx, hookCmd, os.Stdin, os.Stdout, os.Stderr, os.Environ()...)
+ if err != nil {
+ logger.Fatalf("error when starting command %q: %v", hookCmd.String(), err)
+ }
+
+ if err := cmd.Wait(); err != nil {
+ logger.Errorf("error when executing ruby hook: %v", err)
+ var exitError *exec.ExitError
+ if errors.As(err, &exitError) {
+ os.Exit(exitError.ExitCode())
+ } else {
+ os.Exit(1)
+ }
+ }
+
+ os.Exit(0)
+}
diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go
index 30955507f..f9f247512 100644
--- a/cmd/gitaly-hooks/hooks_test.go
+++ b/cmd/gitaly-hooks/hooks_test.go
@@ -15,7 +15,11 @@ import (
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/config"
+ "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
+ hook "gitlab.com/gitlab-org/gitaly/internal/service/hooks"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+ "google.golang.org/grpc/reflection"
)
func TestMain(m *testing.M) {
@@ -33,7 +37,7 @@ func testMain(m *testing.M) int {
}
func TestHooksPrePostReceive(t *testing.T) {
- _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
+ testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
defer cleanupFn()
secretToken := "secret token"
@@ -80,15 +84,40 @@ func TestHooksPrePostReceive(t *testing.T) {
gitObjectDirRegex := regexp.MustCompile(`(?m)^GIT_OBJECT_DIRECTORY=(.*)$`)
gitAlternateObjectDirRegex := regexp.MustCompile(`(?m)^GIT_ALTERNATE_OBJECT_DIRECTORIES=(.*)$`)
+ token := "abc123"
+ socket, stop := runHookServiceServer(t, token)
+ defer stop()
+
+ testCases := []struct {
+ hookName string
+ callRPC bool
+ }{
+ {
+ hookName: "pre-receive",
+ callRPC: false,
+ },
+ {
+ hookName: "post-receive",
+ callRPC: false,
+ },
+ {
+ hookName: "pre-receive",
+ callRPC: true,
+ },
+ {
+ hookName: "post-receive",
+ callRPC: true,
+ },
+ }
- for _, hook := range []string{"pre-receive", "post-receive"} {
- t.Run(hook, func(t *testing.T) {
- customHookOutputPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, hook)
+ for _, tc := range testCases {
+ t.Run(fmt.Sprintf("hookName: %s, calling rpc: %v", tc.hookName, tc.callRPC), func(t *testing.T) {
+ customHookOutputPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, tc.hookName)
defer cleanup()
var stderr, stdout bytes.Buffer
stdin := bytes.NewBuffer([]byte(changes))
- hookPath, err := filepath.Abs(fmt.Sprintf("../../ruby/git-hooks/%s", hook))
+ hookPath, err := filepath.Abs(fmt.Sprintf("../../ruby/git-hooks/%s", tc.hookName))
require.NoError(t, err)
cmd := exec.Command(hookPath)
cmd.Stderr = &stderr
@@ -97,6 +126,9 @@ func TestHooksPrePostReceive(t *testing.T) {
cmd.Env = testhelper.EnvForHooks(
t,
tempGitlabShellDir,
+ socket,
+ token,
+ testRepo,
testhelper.GlHookValues{
GLID: glID,
GLUsername: glUsername,
@@ -107,6 +139,10 @@ func TestHooksPrePostReceive(t *testing.T) {
},
gitPushOptions...,
)
+
+ if tc.callRPC {
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=true", featureflag.HooksRPCEnvVar))
+ }
cmd.Dir = testRepoPath
require.NoError(t, cmd.Run())
@@ -117,17 +153,18 @@ func TestHooksPrePostReceive(t *testing.T) {
require.Contains(t, output, "GL_USERNAME="+glUsername)
require.Contains(t, output, "GL_ID="+glID)
require.Contains(t, output, "GL_REPOSITORY="+glRepository)
- if hook != "pre-receive" {
- require.Contains(t, output, "GL_PROTOCOL="+glProtocol)
- }
- gitObjectDirMatches := gitObjectDirRegex.FindStringSubmatch(output)
- require.Len(t, gitObjectDirMatches, 2)
- require.Equal(t, gitObjectDir, gitObjectDirMatches[1])
+ if tc.hookName == "pre-receive" {
+ gitObjectDirMatches := gitObjectDirRegex.FindStringSubmatch(output)
+ require.Len(t, gitObjectDirMatches, 2)
+ require.Equal(t, gitObjectDir, gitObjectDirMatches[1])
- gitAlternateObjectDirMatches := gitAlternateObjectDirRegex.FindStringSubmatch(output)
- require.Len(t, gitAlternateObjectDirMatches, 2)
- require.Equal(t, strings.Join(gitAlternateObjectDirs, ":"), gitAlternateObjectDirMatches[1])
+ gitAlternateObjectDirMatches := gitAlternateObjectDirRegex.FindStringSubmatch(output)
+ require.Len(t, gitAlternateObjectDirMatches, 2)
+ require.Equal(t, strings.Join(gitAlternateObjectDirs, ":"), gitAlternateObjectDirMatches[1])
+ } else {
+ require.Contains(t, output, "GL_PROTOCOL="+glProtocol)
+ }
})
}
}
@@ -142,16 +179,11 @@ func TestHooksUpdate(t *testing.T) {
defer cleanup()
testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: "http://www.example.com"})
- _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
- defer cleanupFn()
os.Symlink(filepath.Join(config.Config.GitlabShell.Dir, "config.yml"), filepath.Join(tempGitlabShellDir, "config.yml"))
testhelper.WriteShellSecretFile(t, tempGitlabShellDir, "the wrong token")
- customHookOutputPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, "update")
- defer cleanup()
-
gitlabShellDir := config.Config.GitlabShell.Dir
defer func() {
config.Config.GitlabShell.Dir = gitlabShellDir
@@ -159,22 +191,45 @@ func TestHooksUpdate(t *testing.T) {
config.Config.GitlabShell.Dir = tempGitlabShellDir
+ token := "abc123"
+ socket, stop := runHookServiceServer(t, token)
+ defer stop()
+
require.NoError(t, os.MkdirAll(filepath.Join(tempGitlabShellDir, "hooks", "update.d"), 0755))
testhelper.MustRunCommand(t, nil, "cp", "testdata/update", filepath.Join(tempGitlabShellDir, "hooks", "update.d", "update"))
- tempFilePath := filepath.Join(testRepoPath, "tempfile")
- refval, oldval, newval := "refval", "oldval", "newval"
- var stdout, stderr bytes.Buffer
+ for _, callRPC := range []bool{true, false} {
+ t.Run(fmt.Sprintf("call rpc: %t", callRPC), func(t *testing.T) {
+ testHooksUpdate(t, tempGitlabShellDir, socket, token, testhelper.GlHookValues{
+ GLID: glID,
+ GLUsername: glUsername,
+ GLRepo: glRepository,
+ GLProtocol: glProtocol,
+ }, callRPC)
+ })
+ }
+}
+
+func testHooksUpdate(t *testing.T, gitlabShellDir, socket, token string, glValues testhelper.GlHookValues, callRPC bool) {
+ testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
+ defer cleanupFn()
+ refval, oldval, newval := "refval", "oldval", "newval"
updateHookPath, err := filepath.Abs("../../ruby/git-hooks/update")
require.NoError(t, err)
cmd := exec.Command(updateHookPath, refval, oldval, newval)
- cmd.Env = testhelper.EnvForHooks(t, tempGitlabShellDir, testhelper.GlHookValues{
- GLID: glID,
- GLUsername: glUsername,
- GLRepo: glRepository,
- GLProtocol: glProtocol,
- })
+ cmd.Env = testhelper.EnvForHooks(t, gitlabShellDir, socket, token, testRepo, glValues)
+ cmd.Dir = testRepoPath
+ tempFilePath := filepath.Join(testRepoPath, "tempfile")
+
+ customHookOutputPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, "update")
+ defer cleanup()
+
+ var stdout, stderr bytes.Buffer
+
+ if callRPC {
+ cmd.Env = append(cmd.Env, fmt.Sprintf("%s=true", featureflag.HooksRPCEnvVar))
+ }
cmd.Stdout = &stdout
cmd.Stderr = &stderr
cmd.Dir = testRepoPath
@@ -187,17 +242,16 @@ func TestHooksUpdate(t *testing.T) {
var inputs []string
- f, err := os.Open(tempFilePath)
+ b, err := ioutil.ReadFile(tempFilePath)
require.NoError(t, err)
- require.NoError(t, json.NewDecoder(f).Decode(&inputs))
+ require.NoError(t, json.Unmarshal(b, &inputs))
require.Equal(t, []string{refval, oldval, newval}, inputs)
- require.NoError(t, f.Close())
output := string(testhelper.MustReadFile(t, customHookOutputPath))
- require.Contains(t, output, "GL_USERNAME="+glUsername)
- require.Contains(t, output, "GL_ID="+glID)
- require.Contains(t, output, "GL_REPOSITORY="+glRepository)
- require.Contains(t, output, "GL_PROTOCOL="+glProtocol)
+ require.Contains(t, output, "GL_USERNAME="+glValues.GLUsername)
+ require.Contains(t, output, "GL_ID="+glValues.GLID)
+ require.Contains(t, output, "GL_REPOSITORY="+glValues.GLRepo)
+ require.Contains(t, output, "GL_PROTOCOL="+glValues.GLProtocol)
}
func TestHooksPostReceiveFailed(t *testing.T) {
@@ -211,7 +265,7 @@ func TestHooksPostReceiveFailed(t *testing.T) {
tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t)
defer cleanup()
- _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
+ testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
defer cleanupFn()
// By setting the last parameter to false, the post-receive API call will
@@ -244,12 +298,16 @@ func TestHooksPostReceiveFailed(t *testing.T) {
customHookOutputPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, "post-receive")
defer cleanup()
+ token := "abc123"
+ socket, stop := runHookServiceServer(t, token)
+ defer stop()
+
var stdout, stderr bytes.Buffer
postReceiveHookPath, err := filepath.Abs("../../ruby/git-hooks/post-receive")
require.NoError(t, err)
cmd := exec.Command(postReceiveHookPath)
- cmd.Env = testhelper.EnvForHooks(t, tempGitlabShellDir, testhelper.GlHookValues{
+ cmd.Env = testhelper.EnvForHooks(t, tempGitlabShellDir, socket, token, testRepo, testhelper.GlHookValues{
GLID: glID,
GLUsername: glUsername,
GLRepo: glRepository,
@@ -283,6 +341,9 @@ func TestHooksNotAllowed(t *testing.T) {
tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t)
defer cleanup()
+ testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
+ defer cleanupFn()
+
c := testhelper.GitlabTestServerOptions{
User: "",
Password: "",
@@ -295,8 +356,6 @@ func TestHooksNotAllowed(t *testing.T) {
}
ts := testhelper.NewGitlabTestServer(t, c)
defer ts.Close()
- _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
- defer cleanupFn()
testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL})
testhelper.WriteShellSecretFile(t, tempGitlabShellDir, "the wrong token")
@@ -310,6 +369,9 @@ func TestHooksNotAllowed(t *testing.T) {
defer cleanup()
config.Config.GitlabShell.Dir = tempGitlabShellDir
+ token := "abc123"
+ socket, stop := runHookServiceServer(t, token)
+ defer stop()
var stderr, stdout bytes.Buffer
@@ -319,7 +381,7 @@ func TestHooksNotAllowed(t *testing.T) {
cmd.Stderr = &stderr
cmd.Stdout = &stdout
cmd.Stdin = strings.NewReader(changes)
- cmd.Env = testhelper.EnvForHooks(t, tempGitlabShellDir, testhelper.GlHookValues{
+ cmd.Env = testhelper.EnvForHooks(t, tempGitlabShellDir, socket, token, testRepo, testhelper.GlHookValues{
GLID: glID,
GLUsername: glUsername,
GLRepo: glRepository,
@@ -428,3 +490,13 @@ func TestCheckBadCreds(t *testing.T) {
require.Equal(t, "Check GitLab API access: ", stdout.String())
require.Equal(t, "FAILED. code: 401\n", stderr.String())
}
+
+func runHookServiceServer(t *testing.T, token string) (string, func()) {
+ server := testhelper.NewServerWithAuth(t, nil, nil, token)
+
+ gitalypb.RegisterHookServiceServer(server.GrpcServer(), hook.NewServer())
+ reflection.Register(server.GrpcServer())
+ require.NoError(t, server.Start())
+
+ return server.Socket(), server.Stop
+}
diff --git a/cmd/gitaly-ssh/auth_test.go b/cmd/gitaly-ssh/auth_test.go
index 55bac2e2f..1956400f2 100644
--- a/cmd/gitaly-ssh/auth_test.go
+++ b/cmd/gitaly-ssh/auth_test.go
@@ -39,13 +39,13 @@ func TestConnectivity(t *testing.T) {
require.NoError(t, os.RemoveAll(relativeSocketPath))
require.NoError(t, os.Symlink(socketPath, relativeSocketPath))
- tcpServer, tcpPort := runServer(t, server.NewInsecure, "tcp", "localhost:0")
+ tcpServer, tcpPort := runServer(t, server.NewInsecure, config.Config, "tcp", "localhost:0")
defer tcpServer.Stop()
- tlsServer, tlsPort := runServer(t, server.NewSecure, "tcp", "localhost:0")
+ tlsServer, tlsPort := runServer(t, server.NewSecure, config.Config, "tcp", "localhost:0")
defer tlsServer.Stop()
- unixServer, _ := runServer(t, server.NewInsecure, "unix", socketPath)
+ unixServer, _ := runServer(t, server.NewInsecure, config.Config, "unix", socketPath)
defer unixServer.Stop()
testCases := []struct {
@@ -116,8 +116,8 @@ func TestConnectivity(t *testing.T) {
}
}
-func runServer(t *testing.T, newServer func(rubyServer *rubyserver.Server) *grpc.Server, connectionType string, addr string) (*grpc.Server, int) {
- srv := newServer(nil)
+func runServer(t *testing.T, newServer func(rubyServer *rubyserver.Server, cfg config.Cfg) *grpc.Server, cfg config.Cfg, connectionType string, addr string) (*grpc.Server, int) {
+ srv := newServer(nil, cfg)
listener, err := net.Listen(connectionType, addr)
require.NoError(t, err)
diff --git a/cmd/gitaly-ssh/main.go b/cmd/gitaly-ssh/main.go
index 5c70de194..175a14724 100644
--- a/cmd/gitaly-ssh/main.go
+++ b/cmd/gitaly-ssh/main.go
@@ -121,7 +121,7 @@ func getConnection(url string) (*grpc.ClientConn, error) {
func dialOpts() []grpc.DialOption {
connOpts := client.DefaultDialOpts
if token := os.Getenv("GITALY_TOKEN"); token != "" {
- connOpts = append(connOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token)))
+ connOpts = append(connOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token)))
}
// Add grpc client interceptors
diff --git a/cmd/gitaly-ssh/receive_pack.go b/cmd/gitaly-ssh/receive_pack.go
index 97ffee280..70b2cb1b9 100644
--- a/cmd/gitaly-ssh/receive_pack.go
+++ b/cmd/gitaly-ssh/receive_pack.go
@@ -14,7 +14,7 @@ import (
func receivePack(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
var request gitalypb.SSHReceivePackRequest
if err := jsonpb.UnmarshalString(req, &request); err != nil {
- return 0, fmt.Errorf("json unmarshal: %v", err)
+ return 0, fmt.Errorf("json unmarshal: %w", err)
}
ctx, cancel := context.WithCancel(ctx)
diff --git a/cmd/gitaly-ssh/upload_archive.go b/cmd/gitaly-ssh/upload_archive.go
index a9c32f30d..dcb9eb2cf 100644
--- a/cmd/gitaly-ssh/upload_archive.go
+++ b/cmd/gitaly-ssh/upload_archive.go
@@ -14,7 +14,7 @@ import (
func uploadArchive(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
var request gitalypb.SSHUploadArchiveRequest
if err := jsonpb.UnmarshalString(req, &request); err != nil {
- return 0, fmt.Errorf("json unmarshal: %v", err)
+ return 0, fmt.Errorf("json unmarshal: %w", err)
}
ctx, cancel := context.WithCancel(ctx)
diff --git a/cmd/gitaly-ssh/upload_pack.go b/cmd/gitaly-ssh/upload_pack.go
index 954e47a6b..25c856325 100644
--- a/cmd/gitaly-ssh/upload_pack.go
+++ b/cmd/gitaly-ssh/upload_pack.go
@@ -21,7 +21,7 @@ const (
func uploadPack(ctx context.Context, conn *grpc.ClientConn, req string) (int32, error) {
var request gitalypb.SSHUploadPackRequest
if err := jsonpb.UnmarshalString(req, &request); err != nil {
- return 0, fmt.Errorf("json unmarshal: %v", err)
+ return 0, fmt.Errorf("json unmarshal: %w", err)
}
request.GitConfigOptions = append([]string{GitConfigShowAllRefs}, request.GitConfigOptions...)
diff --git a/cmd/gitaly-ssh/upload_pack_test.go b/cmd/gitaly-ssh/upload_pack_test.go
index 80c12e9e9..8d528cd87 100644
--- a/cmd/gitaly-ssh/upload_pack_test.go
+++ b/cmd/gitaly-ssh/upload_pack_test.go
@@ -8,6 +8,7 @@ import (
"github.com/golang/protobuf/jsonpb"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/updateref"
"gitlab.com/gitlab-org/gitaly/internal/server"
@@ -41,7 +42,7 @@ func TestVisibilityOfHiddenRefs(t *testing.T) {
socketPath := testhelper.GetTemporaryGitalySocketFileName()
- unixServer, _ := runServer(t, server.NewInsecure, "unix", socketPath)
+ unixServer, _ := runServer(t, server.NewInsecure, config.Config, "unix", socketPath)
defer unixServer.Stop()
wd, err := os.Getwd()
diff --git a/cmd/praefect/main.go b/cmd/praefect/main.go
index 9814d795b..ec6bc8076 100644
--- a/cmd/praefect/main.go
+++ b/cmd/praefect/main.go
@@ -14,7 +14,17 @@
//
// The subcommand "sql-migrate" will apply any outstanding SQL migrations.
//
-// praefect -config PATH_TO_CONFIG sql-migrate
+// praefect -config PATH_TO_CONFIG sql-migrate [-ignore-unknown=true|false]
+//
+// By default, the migration will ignore any unknown migrations that are
+// not known by the Praefect binary.
+//
+// "-ignore-unknown=false" will disable this behavior.
+//
+// The subcommand "sql-migrate-status" will show which SQL migrations have
+// been applied and which ones have not:
+//
+// praefect -config PATH_TO_CONFIG sql-migrate-status
//
// Dial Nodes
//
@@ -43,6 +53,20 @@
// reference storage is omitted, Praefect will perform the check against the
// current primary. If the primary is the same as the target, an error will
// occur.
+//
+// Dataloss
+//
+// The subcommand "dataloss" helps identify dataloss cases during a given
+// timeframe by checking for dead replication jobs. This can be useful to
+// quantify the impact of a primary node failure.
+//
+// praefect -config PATH_TO_CONFIG dataloss -from RFC3339_TIME -to RFC3339_TIME
+//
+// "-from" specifies the inclusive beginning of a timerange to check.
+//
+// "-to" specifies the exclusive ending of a timerange to check.
+//
+// If a timerange is not specified, dead jobs from last six hours are fetched by default.
package main
import (
@@ -83,6 +107,17 @@ var (
const progname = "praefect"
func main() {
+ flag.Usage = func() {
+ cmds := []string{}
+ for k := range subcommands {
+ cmds = append(cmds, k)
+ }
+
+ printfErr("Usage of %s:\n", progname)
+ flag.PrintDefaults()
+ printfErr(" subcommand (optional)\n")
+ printfErr("\tOne of %s\n", strings.Join(cmds, ", "))
+ }
flag.Parse()
// If invoked with -version
@@ -176,7 +211,18 @@ func run(cfgs []starter.Config, conf config.Config) error {
return err
}
- nodeManager, err := nodes.NewManager(logger, conf, nodeLatencyHistogram)
+ var db *sql.DB
+
+ if conf.NeedsSQL() {
+ dbConn, closedb, err := initDatabase(logger, conf)
+ if err != nil {
+ return err
+ }
+ defer closedb()
+ db = dbConn
+ }
+
+ nodeManager, err := nodes.NewManager(logger, conf, db, nodeLatencyHistogram)
if err != nil {
return err
}
@@ -192,11 +238,6 @@ func run(cfgs []starter.Config, conf config.Config) error {
}
if conf.PostgresQueueEnabled {
- db, closedb, err := initDatabase(logger, conf)
- if err != nil {
- return err
- }
- defer closedb()
ds.ReplicationEventQueue = datastore.NewPostgresReplicationEventQueue(db)
} else {
ds.ReplicationEventQueue = datastore.NewMemoryReplicationEventQueue()
diff --git a/cmd/praefect/subcmd.go b/cmd/praefect/subcmd.go
index a29882a90..7a4222d80 100644
--- a/cmd/praefect/subcmd.go
+++ b/cmd/praefect/subcmd.go
@@ -3,6 +3,7 @@ package main
import (
"context"
"database/sql"
+ "flag"
"fmt"
"os"
"os/signal"
@@ -16,7 +17,22 @@ import (
"google.golang.org/grpc"
)
-const invocationPrefix = progname + " -config CONFIG_TOML"
+type subcmd interface {
+ FlagSet() *flag.FlagSet
+ Exec(flags *flag.FlagSet, config config.Config) error
+}
+
+var (
+ subcommands = map[string]subcmd{
+ "sql-ping": &sqlPingSubcommand{},
+ "sql-migrate": &sqlMigrateSubcommand{},
+ "dial-nodes": &dialNodesSubcommand{},
+ "reconcile": &reconcileSubcommand{},
+ "sql-migrate-down": &sqlMigrateDownSubcommand{},
+ "sql-migrate-status": &sqlMigrateStatusSubcommand{},
+ "dataloss": newDatalossSubcommand(),
+ }
+)
// subCommand returns an exit code, to be fed into os.Exit.
func subCommand(conf config.Config, arg0 string, argRest []string) int {
@@ -28,65 +44,82 @@ func subCommand(conf config.Config, arg0 string, argRest []string) int {
os.Exit(130) // indicates program was interrupted
}()
- switch arg0 {
- case "sql-ping":
- return sqlPing(conf)
- case "sql-migrate":
- return sqlMigrate(conf)
- case subCmdSQLMigrateDown:
- return sqlMigrateDown(conf, argRest)
- case "dial-nodes":
- return dialNodes(conf)
- case "reconcile":
- return reconcile(conf, argRest)
- default:
+ subcmd, ok := subcommands[arg0]
+ if !ok {
printfErr("%s: unknown subcommand: %q\n", progname, arg0)
return 1
}
+
+ flags := subcmd.FlagSet()
+
+ if err := flags.Parse(argRest); err != nil {
+ printfErr("%s\n", err)
+ return 1
+ }
+
+ if err := subcmd.Exec(flags, conf); err != nil {
+ printfErr("%s\n", err)
+ return 1
+ }
+
+ return 0
}
-func sqlPing(conf config.Config) int {
+type sqlPingSubcommand struct{}
+
+func (s *sqlPingSubcommand) FlagSet() *flag.FlagSet {
+ return flag.NewFlagSet("sql-ping", flag.ExitOnError)
+}
+
+func (s *sqlPingSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
const subCmd = progname + " sql-ping"
- db, clean, code := openDB(conf.DB)
- if code != 0 {
- return code
+ db, clean, err := openDB(conf.DB)
+ if err != nil {
+ return err
}
defer clean()
if err := datastore.CheckPostgresVersion(db); err != nil {
- printfErr("%s: fail: %v\n", subCmd, err)
- return 1
+ return fmt.Errorf("%s: fail: %v", subCmd, err)
}
fmt.Printf("%s: OK\n", subCmd)
- return 0
+ return nil
}
-func sqlMigrate(conf config.Config) int {
+type sqlMigrateSubcommand struct {
+ ignoreUnknown bool
+}
+
+func (s *sqlMigrateSubcommand) FlagSet() *flag.FlagSet {
+ flags := flag.NewFlagSet("sql-migrate", flag.ExitOnError)
+ flags.BoolVar(&s.ignoreUnknown, "ignore-unknown", true, "ignore unknown migrations (default is true)")
+ return flags
+}
+
+func (s *sqlMigrateSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
const subCmd = progname + " sql-migrate"
- db, clean, code := openDB(conf.DB)
- if code != 0 {
- return code
+ db, clean, err := openDB(conf.DB)
+ if err != nil {
+ return err
}
defer clean()
- n, err := glsql.Migrate(db)
+ n, err := glsql.Migrate(db, s.ignoreUnknown)
if err != nil {
- printfErr("%s: fail: %v\n", subCmd, err)
- return 1
+ return fmt.Errorf("%s: fail: %v", subCmd, err)
}
fmt.Printf("%s: OK (applied %d migrations)\n", subCmd, n)
- return 0
+ return nil
}
-func openDB(conf config.DB) (*sql.DB, func(), int) {
+func openDB(conf config.DB) (*sql.DB, func(), error) {
db, err := glsql.OpenDB(conf)
if err != nil {
- printfErr("sql open: %v\n", err)
- return nil, nil, 1
+ return nil, nil, fmt.Errorf("sql open: %v", err)
}
clean := func() {
@@ -95,7 +128,7 @@ func openDB(conf config.DB) (*sql.DB, func(), int) {
}
}
- return db, clean, 0
+ return db, clean, nil
}
func printfErr(format string, a ...interface{}) (int, error) {
diff --git a/cmd/praefect/subcmd_dataloss.go b/cmd/praefect/subcmd_dataloss.go
new file mode 100644
index 000000000..7785d4f95
--- /dev/null
+++ b/cmd/praefect/subcmd_dataloss.go
@@ -0,0 +1,111 @@
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "sort"
+ "time"
+
+ "github.com/golang/protobuf/ptypes"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
+
+var errFromNotBeforeTo = errors.New("'from' must be a time before 'to'")
+
+type timeFlag time.Time
+
+func (tf *timeFlag) String() string {
+ return time.Time(*tf).Format(time.RFC3339)
+}
+
+func (tf *timeFlag) Set(v string) error {
+ t, err := time.Parse(time.RFC3339, v)
+ *tf = timeFlag(t)
+ return err
+}
+
+type datalossSubcommand struct {
+ output io.Writer
+ from time.Time
+ to time.Time
+}
+
+func newDatalossSubcommand() *datalossSubcommand {
+ now := time.Now()
+ return &datalossSubcommand{
+ output: os.Stdout,
+ from: now.Add(-6 * time.Hour),
+ to: now,
+ }
+}
+
+func (cmd *datalossSubcommand) FlagSet() *flag.FlagSet {
+ fs := flag.NewFlagSet("dataloss", flag.ContinueOnError)
+ fs.Var((*timeFlag)(&cmd.from), "from", "inclusive beginning of timerange")
+ fs.Var((*timeFlag)(&cmd.to), "to", "exclusive ending of timerange")
+ return fs
+}
+
+func (cmd *datalossSubcommand) Exec(_ *flag.FlagSet, cfg config.Config) error {
+ nodeAddr, err := getNodeAddress(cfg)
+ if err != nil {
+ return err
+ }
+
+ if !cmd.from.Before(cmd.to) {
+ return errFromNotBeforeTo
+ }
+
+ pbFrom, err := ptypes.TimestampProto(cmd.from)
+ if err != nil {
+ return fmt.Errorf("invalid 'from': %v", err)
+ }
+
+ pbTo, err := ptypes.TimestampProto(cmd.to)
+ if err != nil {
+ return fmt.Errorf("invalid 'to': %v", err)
+ }
+
+ conn, err := subCmdDial(nodeAddr, cfg.Auth.Token)
+ if err != nil {
+ return fmt.Errorf("error dialing: %v", err)
+ }
+ defer func() {
+ if err := conn.Close(); err != nil {
+ log.Printf("error closing connection: %v", err)
+ }
+ }()
+
+ client := gitalypb.NewPraefectInfoServiceClient(conn)
+ resp, err := client.DatalossCheck(context.Background(), &gitalypb.DatalossCheckRequest{
+ From: pbFrom,
+ To: pbTo,
+ })
+ if err != nil {
+ return fmt.Errorf("error checking: %v", err)
+ }
+
+ keys := make([]string, 0, len(resp.ByRelativePath))
+ for k := range resp.ByRelativePath {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ if _, err := fmt.Fprintf(cmd.output, "Failed replication jobs between [%s, %s):\n", cmd.from, cmd.to); err != nil {
+ return fmt.Errorf("error writing output: %v", err)
+ }
+
+ for _, proj := range keys {
+ if _, err := fmt.Fprintf(cmd.output, "%s: %d jobs\n", proj, resp.ByRelativePath[proj]); err != nil {
+ return fmt.Errorf("error writing output: %v", err)
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/praefect/subcmd_dataloss_test.go b/cmd/praefect/subcmd_dataloss_test.go
new file mode 100644
index 000000000..f300a9ddf
--- /dev/null
+++ b/cmd/praefect/subcmd_dataloss_test.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "flag"
+ "fmt"
+ "net"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/golang/protobuf/ptypes/timestamp"
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/internal/testhelper"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+ "google.golang.org/grpc"
+)
+
+func TestTimeFlag(t *testing.T) {
+ for _, tc := range []struct {
+ input string
+ expected time.Time
+ }{
+ {
+ input: "2020-02-03T14:15:16Z",
+ expected: time.Date(2020, 2, 3, 14, 15, 16, 0, time.UTC),
+ },
+ {
+ input: "2020-02-03T14:15:16+02:00",
+ expected: time.Date(2020, 2, 3, 14, 15, 16, 0, time.FixedZone("UTC+2", 2*60*60)),
+ },
+ {
+ input: "",
+ },
+ } {
+ t.Run(tc.input, func(t *testing.T) {
+ var actual time.Time
+ fs := flag.NewFlagSet("dataloss", flag.ContinueOnError)
+ fs.Var((*timeFlag)(&actual), "time", "")
+
+ err := fs.Parse([]string{"-time", tc.input})
+ if !tc.expected.IsZero() {
+ require.NoError(t, err)
+ }
+
+ require.True(t, tc.expected.Equal(actual))
+ })
+ }
+}
+
+type mockPraefectInfoService struct {
+ gitalypb.UnimplementedPraefectInfoServiceServer
+ DatalossCheckFunc func(context.Context, *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error)
+}
+
+func (m mockPraefectInfoService) DatalossCheck(ctx context.Context, r *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error) {
+ return m.DatalossCheckFunc(ctx, r)
+}
+
+func TestDatalossSubcommand(t *testing.T) {
+ tmp, clean := testhelper.TempDir(t)
+ defer clean()
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ ln, err := net.Listen("unix", filepath.Join(tmp, "gitaly.sock"))
+ require.NoError(t, err)
+ defer ln.Close()
+
+ mockSvc := &mockPraefectInfoService{}
+ srv := grpc.NewServer()
+ gitalypb.RegisterPraefectInfoServiceServer(srv, mockSvc)
+ go func() { require.NoError(t, srv.Serve(ln)) }()
+ defer srv.Stop()
+
+ // verify the mock service is up
+ addr := fmt.Sprintf("%s://%s", ln.Addr().Network(), ln.Addr())
+ cc, err := grpc.DialContext(ctx, addr, grpc.WithBlock(), grpc.WithInsecure())
+ require.NoError(t, err)
+ defer cc.Close()
+
+ for _, tc := range []struct {
+ desc string
+ args []string
+ datalossCheck func(context.Context, *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error)
+ output string
+ error error
+ }{
+ {
+ desc: "from equals to",
+ args: []string{"-from=2020-01-02T00:00:00Z", "-to=2020-01-02T00:00:00Z"},
+ error: errFromNotBeforeTo,
+ },
+ {
+ desc: "from after to",
+ args: []string{"-from=2020-01-02T00:00:00Z", "-to=2020-01-01T00:00:00Z"},
+ error: errFromNotBeforeTo,
+ },
+ {
+ desc: "no dead jobs",
+ args: []string{"-from=2020-01-02T00:00:00Z", "-to=2020-01-03T00:00:00Z"},
+ datalossCheck: func(ctx context.Context, req *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error) {
+ require.Equal(t, req.GetFrom(), &timestamp.Timestamp{Seconds: 1577923200})
+ require.Equal(t, req.GetTo(), &timestamp.Timestamp{Seconds: 1578009600})
+ return &gitalypb.DatalossCheckResponse{}, nil
+ },
+ output: "Failed replication jobs between [2020-01-02 00:00:00 +0000 UTC, 2020-01-03 00:00:00 +0000 UTC):\n",
+ },
+ {
+ desc: "success",
+ args: []string{"-from=2020-01-02T00:00:00Z", "-to=2020-01-03T00:00:00Z"},
+ datalossCheck: func(_ context.Context, req *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error) {
+ require.Equal(t, req.GetFrom(), &timestamp.Timestamp{Seconds: 1577923200})
+ require.Equal(t, req.GetTo(), &timestamp.Timestamp{Seconds: 1578009600})
+ return &gitalypb.DatalossCheckResponse{ByRelativePath: map[string]int64{
+ "test-repo/relative-path/2": 4,
+ "test-repo/relative-path/1": 1,
+ "test-repo/relative-path/3": 2,
+ }}, nil
+ },
+ output: `Failed replication jobs between [2020-01-02 00:00:00 +0000 UTC, 2020-01-03 00:00:00 +0000 UTC):
+test-repo/relative-path/1: 1 jobs
+test-repo/relative-path/2: 4 jobs
+test-repo/relative-path/3: 2 jobs
+`,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ mockSvc.DatalossCheckFunc = tc.datalossCheck
+ cmd := newDatalossSubcommand()
+ output := &bytes.Buffer{}
+ cmd.output = output
+ require.NoError(t, cmd.FlagSet().Parse(tc.args))
+ require.Equal(t, tc.error, cmd.Exec(cmd.FlagSet(), config.Config{SocketPath: ln.Addr().String()}))
+ require.Equal(t, tc.output, output.String())
+ })
+ }
+}
diff --git a/cmd/praefect/subcmd_pingnodes.go b/cmd/praefect/subcmd_pingnodes.go
index b3bf24fec..742f8eb12 100644
--- a/cmd/praefect/subcmd_pingnodes.go
+++ b/cmd/praefect/subcmd_pingnodes.go
@@ -3,6 +3,7 @@ package main
import (
"context"
"errors"
+ "flag"
"fmt"
"log"
"sync"
@@ -45,7 +46,13 @@ func flattenNodes(conf config.Config) map[string]*nodePing {
return nodeByAddress
}
-func dialNodes(conf config.Config) int {
+type dialNodesSubcommand struct{}
+
+func (s *dialNodesSubcommand) FlagSet() *flag.FlagSet {
+ return flag.NewFlagSet("dial-nodes", flag.ExitOnError)
+}
+
+func (s *dialNodesSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
nodes := flattenNodes(conf)
var wg sync.WaitGroup
@@ -58,14 +65,14 @@ func dialNodes(conf config.Config) int {
}
wg.Wait()
- exitCode := 0
+ var err error
for _, n := range nodes {
if n.err != nil {
- exitCode = 1
+ err = n.err
}
}
- return exitCode
+ return err
}
func (npr *nodePing) dial() (*grpc.ClientConn, error) {
diff --git a/cmd/praefect/subcmd_reconcile.go b/cmd/praefect/subcmd_reconcile.go
index 21f3ed7fc..42f64d6b4 100644
--- a/cmd/praefect/subcmd_reconcile.go
+++ b/cmd/praefect/subcmd_reconcile.go
@@ -19,32 +19,44 @@ type nodeReconciler struct {
referenceStorage string
}
-func reconcile(conf config.Config, subCmdArgs []string) int {
- var (
- fs = flag.NewFlagSet("reconcile", flag.ExitOnError)
- vs = fs.String("virtual", "", "virtual storage for target storage")
- t = fs.String("target", "", "target storage to reconcile")
- r = fs.String("reference", "", "reference storage to reconcile (optional)")
- )
-
- if err := fs.Parse(subCmdArgs); err != nil {
- log.Printf("unable to parse args %v: %s", subCmdArgs, err)
- return 1
- }
+type reconcileSubcommand struct {
+ virtual string
+ target string
+ reference string
+}
+
+func (s *reconcileSubcommand) FlagSet() *flag.FlagSet {
+ fs := flag.NewFlagSet("reconcile", flag.ExitOnError)
+ fs.StringVar(&s.virtual, "virtual", "", "virtual storage for target storage")
+ fs.StringVar(&s.target, "target", "", "target storage to reconcile")
+ fs.StringVar(&s.reference, "reference", "", "reference storage to reconcile (optional)")
+ return fs
+}
+func (s *reconcileSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
nr := nodeReconciler{
conf: conf,
- virtualStorage: *vs,
- targetStorage: *t,
- referenceStorage: *r,
+ virtualStorage: s.virtual,
+ targetStorage: s.target,
+ referenceStorage: s.reference,
}
if err := nr.reconcile(); err != nil {
- log.Print("unable to reconcile: ", err)
- return 1
+ return fmt.Errorf("unable to reconcile: %s", err)
}
- return 0
+ return nil
+}
+
+func getNodeAddress(cfg config.Config) (string, error) {
+ switch {
+ case cfg.SocketPath != "":
+ return "unix://" + cfg.SocketPath, nil
+ case cfg.ListenAddr != "":
+ return "tcp://" + cfg.ListenAddr, nil
+ default:
+ return "", errors.New("no Praefect address configured")
+ }
}
func (nr nodeReconciler) reconcile() error {
@@ -52,14 +64,9 @@ func (nr nodeReconciler) reconcile() error {
return err
}
- var nodeAddr string
- switch {
- case nr.conf.SocketPath != "":
- nodeAddr = "unix://" + nr.conf.SocketPath
- case nr.conf.ListenAddr != "":
- nodeAddr = "tcp://" + nr.conf.ListenAddr
- default:
- return errors.New("no Praefect address configured")
+ nodeAddr, err := getNodeAddress(nr.conf)
+ if err != nil {
+ return err
}
cc, err := subCmdDial(nodeAddr, nr.conf.Auth.Token)
diff --git a/cmd/praefect/subcmd_sqldown.go b/cmd/praefect/subcmd_sqldown.go
index f34395d6c..2748de0d0 100644
--- a/cmd/praefect/subcmd_sqldown.go
+++ b/cmd/praefect/subcmd_sqldown.go
@@ -1,6 +1,7 @@
package main
import (
+ "errors"
"flag"
"fmt"
"strconv"
@@ -9,43 +10,28 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore"
)
-const subCmdSQLMigrateDown = "sql-migrate-down"
-
-func sqlMigrateDown(conf config.Config, args []string) int {
- cmd := &sqlMigrateDownCmd{Config: conf}
- return cmd.Run(args)
+type sqlMigrateDownSubcommand struct {
+ force bool
}
-type sqlMigrateDownCmd struct{ config.Config }
-
-func (*sqlMigrateDownCmd) prefix() string { return progname + " " + subCmdSQLMigrateDown }
-
-func (*sqlMigrateDownCmd) invocation() string { return invocationPrefix + " " + subCmdSQLMigrateDown }
-
-func (smd *sqlMigrateDownCmd) Run(args []string) int {
- flagset := flag.NewFlagSet(smd.prefix(), flag.ExitOnError)
- flagset.Usage = func() {
- printfErr("usage: %s [-f] MAX_MIGRATIONS\n", smd.invocation())
- }
- force := flagset.Bool("f", false, "apply down-migrations (default is dry run)")
-
- _ = flagset.Parse(args) // No error check because flagset is set to ExitOnError
-
- if flagset.NArg() != 1 {
- flagset.Usage()
- return 1
+func (s *sqlMigrateDownSubcommand) FlagSet() *flag.FlagSet {
+ flags := flag.NewFlagSet("sql-migrate-down", flag.ExitOnError)
+ flags.Usage = func() {
+ flag.PrintDefaults()
+ printfErr(" MAX_MIGRATIONS\n")
+ printfErr("\tNumber of migrations to roll back\n")
}
+ flags.BoolVar(&s.force, "f", false, "apply down-migrations (default is dry run)")
+ return flags
+}
- if err := smd.run(*force, flagset.Arg(0)); err != nil {
- printfErr("%s: fail: %v\n", smd.prefix(), err)
- return 1
+func (s *sqlMigrateDownSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
+ if flags.NArg() != 1 {
+ flags.Usage()
+ return errors.New("invalid usage")
}
- return 0
-}
-
-func (smd *sqlMigrateDownCmd) run(force bool, maxString string) error {
- maxMigrations, err := strconv.Atoi(maxString)
+ maxMigrations, err := strconv.Atoi(flags.Arg(0))
if err != nil {
return err
}
@@ -54,26 +40,26 @@ func (smd *sqlMigrateDownCmd) run(force bool, maxString string) error {
return fmt.Errorf("number of migrations to roll back must be 1 or more")
}
- if force {
- n, err := datastore.MigrateDown(smd.Config, maxMigrations)
+ if s.force {
+ n, err := datastore.MigrateDown(conf, maxMigrations)
if err != nil {
return err
}
- fmt.Printf("%s: OK (applied %d \"down\" migrations)\n", smd.prefix(), n)
+ fmt.Printf("OK (applied %d \"down\" migrations)\n", n)
return nil
}
- planned, err := datastore.MigrateDownPlan(smd.Config, maxMigrations)
+ planned, err := datastore.MigrateDownPlan(conf, maxMigrations)
if err != nil {
return err
}
- fmt.Printf("%s: DRY RUN -- would roll back:\n\n", smd.prefix())
+ fmt.Printf("DRY RUN -- would roll back:\n\n")
for _, id := range planned {
fmt.Printf("- %s\n", id)
}
- fmt.Printf("\nTo apply these migrations run: %s -f %d\n", smd.invocation(), maxMigrations)
+ fmt.Printf("\nTo apply these migrations run with -f\n")
return nil
}
diff --git a/cmd/praefect/subcmd_sqlstatus.go b/cmd/praefect/subcmd_sqlstatus.go
new file mode 100644
index 000000000..f71d9d200
--- /dev/null
+++ b/cmd/praefect/subcmd_sqlstatus.go
@@ -0,0 +1,55 @@
+package main
+
+import (
+ "flag"
+ "os"
+ "sort"
+
+ "github.com/olekukonko/tablewriter"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/datastore"
+)
+
+type sqlMigrateStatusSubcommand struct{}
+
+func (s *sqlMigrateStatusSubcommand) FlagSet() *flag.FlagSet {
+ return flag.NewFlagSet("sql-migrate-status", flag.ExitOnError)
+}
+
+func (s *sqlMigrateStatusSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error {
+ migrations, err := datastore.MigrateStatus(conf)
+ if err != nil {
+ return err
+ }
+
+ table := tablewriter.NewWriter(os.Stdout)
+ table.SetHeader([]string{"Migration", "Applied"})
+ table.SetColWidth(60)
+
+ // Display the rows in order of name
+ var keys []string
+ for k := range migrations {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ for _, k := range keys {
+ m := migrations[k]
+ applied := "no"
+
+ if m.Unknown {
+ applied = "unknown migration"
+ } else if m.Migrated {
+ applied = m.AppliedAt.String()
+ }
+
+ table.Append([]string{
+ k,
+ applied,
+ })
+ }
+
+ table.Render()
+
+ return err
+}
diff --git a/config.praefect.toml.example b/config.praefect.toml.example
index d45978796..534f9b9a2 100644
--- a/config.praefect.toml.example
+++ b/config.praefect.toml.example
@@ -26,6 +26,10 @@ listen_addr = "127.0.0.1:2305"
# as shard. listen_addr should be unique for all nodes.
# Requires the protocol to be defined, e.g. tcp://host.tld:1234
+[failover]
+enabled = true
+election_strategy = "local" # Options: local, sql
+
[[virtual_storage]]
name = 'praefect'
diff --git a/doc/design_diskcache.md b/doc/design_diskcache.md
index b931f6388..4aeb07d0d 100644
--- a/doc/design_diskcache.md
+++ b/doc/design_diskcache.md
@@ -46,6 +46,7 @@ specific repository state:
RPC request (digest) │ ┌──────┐
Gitaly version (string) ├─────│SHA256│─────▶ Cache key
RPC Method (string) │ └──────┘
+ Feature flags (string) │
─────┘
```
diff --git a/doc/sql_migrations.md b/doc/sql_migrations.md
index 617da054a..e6832fdd4 100644
--- a/doc/sql_migrations.md
+++ b/doc/sql_migrations.md
@@ -12,6 +12,43 @@ Praefect SQL migrations should be applied automatically when you deploy Praefect
praefect -config /path/to/config.toml sql-migrate
```
+By default, the migration will ignore any unknown migrations that are
+not known by the Praefect binary.
+
+The `-ignore-unknown=false` will disable this behavior:
+
+```shell
+praefect -config /path/to/config.toml sql-migrate -ignore-unknown=false
+```
+
+## Showing the status of migrations
+
+To see which migrations have been applied, run:
+
+```
+praefect -config /path/to/config.toml sql-migrate-status
+```
+
+For example, the output may look like:
+
+```
++----------------------------------------+--------------------------------------+
+| MIGRATION | APPLIED |
++----------------------------------------+--------------------------------------+
+| 20200109161404_hello_world | 2020-02-26 16:00:32.486129 -0800 PST |
+| 20200113151438_1_test_migration | 2020-02-26 16:00:32.486871 -0800 PST |
+| 20200224220728_job_queue | 2020-03-25 16:27:21.384917 -0700 PDT |
+| 20200324001604_add_sql_election_tables | no |
+| 20200401010230_add_some_table | unknown migration |
++----------------------------------------+--------------------------------------+
+```
+
+The first column contains the migration ID, and the second contains one of three items:
+
+1. The date on which the migration was applied
+2. `no` if the migration has not yet been applied
+3. `unknown migration` if the migration is not known by the current Praefect binary
+
## Rolling back migrations
Rolling back SQL migrations in Praefect works a little differently
diff --git a/go.mod b/go.mod
index e863dcfc6..067f4d24f 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/kelseyhightower/envconfig v1.3.0
github.com/lib/pq v1.2.0
+ github.com/olekukonko/tablewriter v0.0.2
github.com/prometheus/client_golang v1.0.0
github.com/prometheus/procfs v0.0.3 // indirect
github.com/rubenv/sql-migrate v0.0.0-20191213152630-06338513c237
diff --git a/go.sum b/go.sum
index ca408701c..c46e84daa 100644
--- a/go.sum
+++ b/go.sum
@@ -188,6 +188,7 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do=
github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -203,6 +204,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16MZ91cS6o=
github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
diff --git a/internal/blackbox/prometheus.go b/internal/blackbox/prometheus.go
index 6eacbb7c4..7929ad5e4 100644
--- a/internal/blackbox/prometheus.go
+++ b/internal/blackbox/prometheus.go
@@ -6,11 +6,11 @@ import (
)
var (
- getFirstPacket = newGauge("get_first_packet_seconds", "Time to first Git packet in GET /info/refs reponse")
- getTotalTime = newGauge("get_total_time_seconds", "Time to receive entire GET /info/refs reponse")
+ getFirstPacket = newGauge("get_first_packet_seconds", "Time to first Git packet in GET /info/refs response")
+ getTotalTime = newGauge("get_total_time_seconds", "Time to receive entire GET /info/refs response")
getAdvertisedRefs = newGauge("get_advertised_refs", "Number of Git refs advertised in GET /info/refs")
wantedRefs = newGauge("wanted_refs", "Number of Git refs selected for (fake) Git clone (branches + tags)")
- postTotalTime = newGauge("post_total_time_seconds", "Time to receive entire POST /upload-pack reponse")
+ postTotalTime = newGauge("post_total_time_seconds", "Time to receive entire POST /upload-pack response")
postFirstProgressPacket = newGauge("post_first_progress_packet_seconds", "Time to first progress band Git packet in POST /upload-pack response")
postFirstPackPacket = newGauge("post_first_pack_packet_seconds", "Time to first pack band Git packet in POST /upload-pack response")
postPackBytes = newGauge("post_pack_bytes", "Number of pack band bytes in POST /upload-pack response")
diff --git a/internal/bootstrap/server_factory.go b/internal/bootstrap/server_factory.go
index c754fbf53..30bfea16b 100644
--- a/internal/bootstrap/server_factory.go
+++ b/internal/bootstrap/server_factory.go
@@ -4,6 +4,7 @@ import (
"net"
"sync"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
"gitlab.com/gitlab-org/gitaly/internal/server"
"google.golang.org/grpc"
@@ -68,14 +69,14 @@ func (s *GitalyServerFactory) Serve(l net.Listener, secure bool) error {
func (s *GitalyServerFactory) get(secure bool) *grpc.Server {
if secure {
if s.secure == nil {
- s.secure = server.NewSecure(s.ruby)
+ s.secure = server.NewSecure(s.ruby, config.Config)
}
return s.secure
}
if s.insecure == nil {
- s.insecure = server.NewInsecure(s.ruby)
+ s.insecure = server.NewInsecure(s.ruby, config.Config)
}
return s.insecure
diff --git a/internal/cache/cachedb.go b/internal/cache/cachedb.go
index f756fa31e..a70724596 100644
--- a/internal/cache/cachedb.go
+++ b/internal/cache/cachedb.go
@@ -9,7 +9,7 @@ import (
"sync"
"github.com/golang/protobuf/proto"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/safe"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
)
@@ -85,7 +85,7 @@ func (sdb *StreamDB) GetStream(ctx context.Context, repo *gitalypb.Repository, r
return nil, err
}
- grpc_logrus.Extract(ctx).
+ ctxlogrus.Extract(ctx).
WithField("stream_path", respPath).
Info("getting stream")
@@ -120,7 +120,7 @@ func (sdb *StreamDB) PutStream(ctx context.Context, repo *gitalypb.Repository, r
return err
}
- grpc_logrus.Extract(ctx).
+ ctxlogrus.Extract(ctx).
WithField("stream_path", reqPath).
Info("putting stream")
diff --git a/internal/cache/cachedb_test.go b/internal/cache/cachedb_test.go
index 65ea3ff79..3f0b40fd0 100644
--- a/internal/cache/cachedb_test.go
+++ b/internal/cache/cachedb_test.go
@@ -14,6 +14,7 @@ import (
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/cache"
"gitlab.com/gitlab-org/gitaly/internal/config"
+ "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
)
@@ -91,6 +92,13 @@ func TestStreamDBNaiveKeyer(t *testing.T) {
require.NoError(t, db.PutStream(ctx, req1.Repository, req1, strings.NewReader(expectStream2)))
expectGetHit(expectStream2, req1)
+ // enabled feature flags affect caching
+ oldCtx := ctx
+ ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, "meow")
+ expectGetMiss(req1)
+ ctx = oldCtx
+ expectGetHit(expectStream2, req1)
+
// start critical section without closing
repo1Lease, err := keyer.StartLease(req1.Repository)
require.NoError(t, err)
diff --git a/internal/cache/keyer.go b/internal/cache/keyer.go
index 350c13851..2deea4d36 100644
--- a/internal/cache/keyer.go
+++ b/internal/cache/keyer.go
@@ -9,13 +9,16 @@ import (
"io/ioutil"
"os"
"path/filepath"
+ "sort"
+ "strings"
"time"
"github.com/golang/protobuf/proto"
"github.com/google/uuid"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/helper"
+ "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/internal/safe"
"gitlab.com/gitlab-org/gitaly/internal/tempdir"
"gitlab.com/gitlab-org/gitaly/internal/version"
@@ -106,7 +109,7 @@ func updateLatest(ctx context.Context, repo *gitalypb.Repository) (string, error
return "", err
}
- grpc_logrus.Extract(ctx).
+ ctxlogrus.Extract(ctx).
WithField("diskcache", nextGenID).
Infof("diskcache state change")
@@ -297,7 +300,8 @@ func latestPath(repoStateDir string) string { return filepath.Join(repoStateDir,
// compositeKeyHashHex returns a hex encoded string that is a SHA256 hash sum of
// the composite key made up of the following properties: Gitaly version, gRPC
-// method, repo cache current generation ID, protobuf request
+// method, repo cache current generation ID, protobuf request, and enabled
+// feature flags.
func compositeKeyHashHex(ctx context.Context, genID string, req proto.Message) (string, error) {
method, ok := grpc.Method(ctx)
if !ok {
@@ -311,11 +315,15 @@ func compositeKeyHashHex(ctx context.Context, genID string, req proto.Message) (
h := sha256.New()
+ ffs := featureflag.AllEnabledFlags(ctx)
+ sort.Strings(ffs)
+
for _, i := range []string{
version.GetVersion(),
method,
genID,
string(reqSum),
+ strings.Join(ffs, " "),
} {
_, err := h.Write(prefixLen(i))
if err != nil {
diff --git a/internal/cache/walker_test.go b/internal/cache/walker_test.go
index 50e4a55c5..ef7ba95f9 100644
--- a/internal/cache/walker_test.go
+++ b/internal/cache/walker_test.go
@@ -104,8 +104,6 @@ func setupDiskCacheWalker(t testing.TB) func() {
},
}
- satisfyConfigValidation(tmpPath)
-
cleanup := func() {
config.Config.Storages = oldStorages
require.NoError(t, os.RemoveAll(tmpPath))
@@ -114,39 +112,6 @@ func setupDiskCacheWalker(t testing.TB) func() {
return cleanup
}
-// satisfyConfigValidation puts garbage values in the config file to satisfy
-// validation
-func satisfyConfigValidation(tmpPath string) error {
- config.Config.ListenAddr = "meow"
-
- if err := os.MkdirAll(filepath.Join(tmpPath, "hooks"), 0700); err != nil {
- return err
- }
- if err := os.MkdirAll(filepath.Join(tmpPath, "git-hooks"), 0700); err != nil {
- return err
- }
-
- for _, filePath := range []string{
- filepath.Join("ruby", "git-hooks", "pre-receive"),
- filepath.Join("ruby", "git-hooks", "post-receive"),
- filepath.Join("ruby", "git-hooks", "update"),
- } {
- if err := ioutil.WriteFile(filepath.Join(tmpPath, filePath), nil, 0755); err != nil {
- return err
- }
- }
- config.Config.GitlabShell = config.GitlabShell{
- Dir: filepath.Join(tmpPath, "gitlab-shell"),
- }
- config.Config.Ruby = config.Ruby{
- Dir: filepath.Join(tmpPath, "ruby"),
- }
-
- config.Config.BinDir = filepath.Join(tmpPath, "bin")
-
- return nil
-}
-
func pollCountersUntil(t testing.TB, expectRemovals int) {
// poll injected mock prometheus counters until expected events occur
timeout := time.After(time.Second)
diff --git a/internal/command/command.go b/internal/command/command.go
index ae374f3a8..3477c33ea 100644
--- a/internal/command/command.go
+++ b/internal/command/command.go
@@ -14,9 +14,8 @@ import (
"syscall"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/sirupsen/logrus"
- log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/config"
)
@@ -127,7 +126,7 @@ func GitPath() string {
// This shouldn't happen outside of testing, SetGitPath should be called by
// main.go to ensure correctness of the configuration on start-up.
if err := config.SetGitPath(); err != nil {
- log.Fatal(err) // Bail out.
+ logrus.Fatal(err) // Bail out.
}
}
@@ -182,7 +181,7 @@ func New(ctx context.Context, cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.
logPid := -1
defer func() {
- grpc_logrus.Extract(ctx).WithFields(log.Fields{
+ ctxlogrus.Extract(ctx).WithFields(logrus.Fields{
"pid": logPid,
"path": cmd.Path,
"args": cmd.Args,
@@ -235,7 +234,7 @@ func New(ctx context.Context, cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.
command.stderrCloser = &noopWriteCloser{stderr}
close(command.stderrDone)
} else {
- command.stderrCloser = escapeNewlineWriter(grpc_logrus.Extract(ctx).WriterLevel(log.ErrorLevel), command.stderrDone, MaxStderrBytes)
+ command.stderrCloser = escapeNewlineWriter(ctxlogrus.Extract(ctx).WriterLevel(logrus.ErrorLevel), command.stderrDone, MaxStderrBytes)
}
cmd.Stderr = command.stderrCloser
@@ -392,7 +391,7 @@ func (c *Command) logProcessComplete(ctx context.Context, exitCode int) {
userTime := cmd.ProcessState.UserTime()
realTime := time.Since(c.startTime)
- entry := grpc_logrus.Extract(ctx).WithFields(log.Fields{
+ entry := ctxlogrus.Extract(ctx).WithFields(logrus.Fields{
"pid": cmd.ProcessState.Pid(),
"path": cmd.Path,
"args": cmd.Args,
@@ -403,7 +402,7 @@ func (c *Command) logProcessComplete(ctx context.Context, exitCode int) {
})
if rusage, ok := cmd.ProcessState.SysUsage().(*syscall.Rusage); ok {
- entry = entry.WithFields(log.Fields{
+ entry = entry.WithFields(logrus.Fields{
"command.maxrss": rusage.Maxrss,
"command.inblock": rusage.Inblock,
"command.oublock": rusage.Oublock,
diff --git a/internal/command/command_test.go b/internal/command/command_test.go
index dbf241bd7..782635242 100644
--- a/internal/command/command_test.go
+++ b/internal/command/command_test.go
@@ -1,12 +1,14 @@
package command
import (
+ "bufio"
"bytes"
"context"
"fmt"
"io"
"os"
"os/exec"
+ "regexp"
"strings"
"testing"
"time"
@@ -195,10 +197,16 @@ func TestCommandStdErr(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- var stdout, stderr bytes.Buffer
+ var stdout bytes.Buffer
+
+ expectedMessage := `hello world\\nhello world\\nhello world\\nhello world\\nhello world\\n`
+
+ r, w := io.Pipe()
+ defer r.Close()
+ defer w.Close()
logger := logrus.New()
- logger.SetOutput(&stderr)
+ logger.SetOutput(w)
ctx = ctxlogrus.ToContext(ctx, logrus.NewEntry(logger))
@@ -208,18 +216,23 @@ func TestCommandStdErr(t *testing.T) {
require.Error(t, cmd.Wait())
assert.Empty(t, stdout.Bytes())
- logger.Exit(0)
- assert.Equal(t, `hello world\nhello world\nhello world\nhello world\nhello world\n`, stderr.String())
+ b := bufio.NewReader(r)
+ line, err := b.ReadString('\n')
+ require.NoError(t, err)
+ require.Equal(t, expectedMessage, extractMessage(line))
}
func TestCommandStdErrLargeOutput(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- var stdout, stderr bytes.Buffer
+ var stdout bytes.Buffer
+ r, w := io.Pipe()
+ defer r.Close()
+ defer w.Close()
logger := logrus.New()
- logger.SetOutput(&stderr)
+ logger.SetOutput(w)
ctx = ctxlogrus.ToContext(ctx, logrus.NewEntry(logger))
@@ -229,18 +242,29 @@ func TestCommandStdErrLargeOutput(t *testing.T) {
require.Error(t, cmd.Wait())
assert.Empty(t, stdout.Bytes())
- logger.Exit(0)
- assert.True(t, stderr.Len() <= MaxStderrBytes)
+ b := bufio.NewReader(r)
+ line, err := b.ReadString('\n')
+ require.NoError(t, err)
+
+ // the logrus printer prints with %q, so with an escaped newline it will add an extra \ escape to the
+ // output. So for the test we can take out the extra \ since it was logrus that added it, not the command
+ // https://github.com/sirupsen/logrus/blob/master/text_formatter.go#L324
+ msg := strings.Replace(extractMessage(line), `\\n`, `\n`, -1)
+ require.LessOrEqual(t, len(msg), MaxStderrBytes)
}
func TestCommandStdErrBinaryNullBytes(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- var stdout, stderr bytes.Buffer
+ var stdout bytes.Buffer
+
+ r, w := io.Pipe()
+ defer r.Close()
+ defer w.Close()
logger := logrus.New()
- logger.SetOutput(&stderr)
+ logger.SetOutput(w)
ctx = ctxlogrus.ToContext(ctx, logrus.NewEntry(logger))
@@ -250,18 +274,23 @@ func TestCommandStdErrBinaryNullBytes(t *testing.T) {
require.Error(t, cmd.Wait())
assert.Empty(t, stdout.Bytes())
- logger.Exit(0)
- assert.NotEmpty(t, stderr.Bytes())
+ b := bufio.NewReader(r)
+ line, err := b.ReadString('\n')
+ require.NoError(t, err)
+ require.NotEmpty(t, extractMessage(line))
}
func TestCommandStdErrLongLine(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- var stdout, stderr bytes.Buffer
+ var stdout bytes.Buffer
+ r, w := io.Pipe()
+ defer r.Close()
+ defer w.Close()
logger := logrus.New()
- logger.SetOutput(&stderr)
+ logger.SetOutput(w)
ctx = ctxlogrus.ToContext(ctx, logrus.NewEntry(logger))
@@ -271,19 +300,23 @@ func TestCommandStdErrLongLine(t *testing.T) {
require.Error(t, cmd.Wait())
assert.Empty(t, stdout.Bytes())
- logger.Exit(0)
- assert.NotEmpty(t, stderr.Bytes())
- assert.Equal(t, fmt.Sprintf("%s\\n%s", strings.Repeat("a", StderrBufferSize), strings.Repeat("b", StderrBufferSize)), stderr.String())
+ b := bufio.NewReader(r)
+ line, err := b.ReadString('\n')
+ require.NoError(t, err)
+ require.Contains(t, line, fmt.Sprintf(`%s\\n%s`, strings.Repeat("a", StderrBufferSize), strings.Repeat("b", StderrBufferSize)))
}
func TestCommandStdErrMaxBytes(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- var stdout, stderr bytes.Buffer
+ var stdout bytes.Buffer
+ r, w := io.Pipe()
+ defer r.Close()
+ defer w.Close()
logger := logrus.New()
- logger.SetOutput(&stderr)
+ logger.SetOutput(w)
ctx = ctxlogrus.ToContext(ctx, logrus.NewEntry(logger))
@@ -293,6 +326,19 @@ func TestCommandStdErrMaxBytes(t *testing.T) {
require.Error(t, cmd.Wait())
assert.Empty(t, stdout.Bytes())
- logger.Exit(0)
- assert.NotEmpty(t, stderr.Bytes())
+ b := bufio.NewReader(r)
+ line, err := b.ReadString('\n')
+ require.NoError(t, err)
+ require.NotEmpty(t, extractMessage(line))
+}
+
+var logMsgRegex = regexp.MustCompile(`msg="(.+)"`)
+
+func extractMessage(logMessage string) string {
+ subMatches := logMsgRegex.FindStringSubmatch(logMessage)
+ if len(subMatches) != 2 {
+ return ""
+ }
+
+ return subMatches[1]
}
diff --git a/internal/command/spawntoken.go b/internal/command/spawntoken.go
index a3bfddeba..8c5b21783 100644
--- a/internal/command/spawntoken.go
+++ b/internal/command/spawntoken.go
@@ -5,7 +5,7 @@ import (
"fmt"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/kelseyhightower/envconfig"
"github.com/prometheus/client_golang/prometheus"
)
@@ -80,5 +80,5 @@ func logTime(ctx context.Context, start time.Time, msg string) {
return
}
- grpc_logrus.Extract(ctx).WithField("spawn_queue_ms", delta.Seconds()*1000).Info(msg)
+ ctxlogrus.Extract(ctx).WithField("spawn_queue_ms", delta.Seconds()*1000).Info(msg)
}
diff --git a/internal/config/config.go b/internal/config/config.go
index 0f041f2a9..2779b09a9 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -199,7 +199,7 @@ func (h *hookErrs) Add(err error) {
}
func validateHooks() error {
- if os.Getenv("GITALY_TESTING_NO_GIT_HOOKS") == "1" {
+ if SkipHooks() {
return nil
}
@@ -278,6 +278,10 @@ func validateStorages() error {
return nil
}
+func SkipHooks() bool {
+ return os.Getenv("GITALY_TESTING_NO_GIT_HOOKS") == "1"
+}
+
// SetGitPath populates the variable GitPath with the path to the `git`
// executable. It warns if no path was specified in the configuration.
func SetGitPath() error {
@@ -319,8 +323,7 @@ func (c Cfg) Storage(storageName string) (Storage, bool) {
func validateBinDir() error {
if err := validateIsDirectory(Config.BinDir, "bin_dir"); err != nil {
log.WithError(err).Warn("Gitaly bin directory is not configured")
- // TODO this must become a fatal error
- return nil
+ return err
}
var err error
diff --git a/internal/git/bitmap.go b/internal/git/bitmap.go
index 01f55b059..f557557f7 100644
--- a/internal/git/bitmap.go
+++ b/internal/git/bitmap.go
@@ -6,7 +6,7 @@ import (
"strconv"
"strings"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags"
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitaly/internal/git/packfile"
@@ -28,7 +28,7 @@ func init() { prometheus.MustRegister(badBitmapRequestCount) }
// repoPath, and if it finds any, it logs a warning. This is to help us
// investigate https://gitlab.com/gitlab-org/gitaly/issues/1728.
func WarnIfTooManyBitmaps(ctx context.Context, repo repository.GitRepo) {
- logEntry := grpc_logrus.Extract(ctx)
+ logEntry := ctxlogrus.Extract(ctx)
storageRoot, err := helper.GetStorageByName(repo.GetStorageName())
if err != nil {
diff --git a/internal/git/dirs.go b/internal/git/dirs.go
index e619b6ccf..0a192ae3b 100644
--- a/internal/git/dirs.go
+++ b/internal/git/dirs.go
@@ -8,7 +8,7 @@ import (
"path/filepath"
"strings"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
)
// alternateOutsideStorageError is returned when an alternates file contains an
@@ -44,7 +44,7 @@ func AlternateObjectDirectories(ctx context.Context, storageRoot, repoPath strin
}
func altObjectDirs(ctx context.Context, storagePrefix, objDir string, depth int) ([]string, error) {
- logEntry := grpc_logrus.Extract(ctx)
+ logEntry := ctxlogrus.Extract(ctx)
const maxAlternatesDepth = 5 // Taken from https://github.com/git/git/blob/v2.23.0/sha1-file.c#L575
if depth > maxAlternatesDepth {
logEntry.WithField("objdir", objDir).Warn("ignoring deeply nested alternate object directory")
diff --git a/internal/git/dirs_test.go b/internal/git/dirs_test.go
index ae73b912a..e0bb87e0c 100644
--- a/internal/git/dirs_test.go
+++ b/internal/git/dirs_test.go
@@ -50,7 +50,7 @@ func TestObjectDirsNoAlternates(t *testing.T) {
}
func TestObjectDirsOutsideStorage(t *testing.T) {
- tmp, clean := testhelper.TempDir(t, "")
+ tmp, clean := testhelper.TempDir(t)
defer clean()
storageRoot := filepath.Join(tmp, "storage-root")
diff --git a/internal/git/hooks/hooks.go b/internal/git/hooks/hooks.go
index e8c1f325f..264db79e7 100644
--- a/internal/git/hooks/hooks.go
+++ b/internal/git/hooks/hooks.go
@@ -2,7 +2,6 @@ package hooks
import (
"fmt"
- "os"
"path"
"gitlab.com/gitlab-org/gitaly/internal/config"
@@ -20,7 +19,7 @@ func Path() string {
return Override
}
- if os.Getenv("GITALY_TESTING_NO_GIT_HOOKS") == "1" {
+ if config.SkipHooks() {
return "/var/empty"
}
diff --git a/internal/git/log/tag.go b/internal/git/log/tag.go
index 1f251fa3a..0b71ef3df 100644
--- a/internal/git/log/tag.go
+++ b/internal/git/log/tag.go
@@ -121,7 +121,7 @@ func buildAnnotatedTag(b *catfile.Batch, tagID, name string, header *tagHeader,
// tags contain the signature block in the message:
// https://github.com/git/git/blob/master/Documentation/technical/signature-format.txt#L12
- index := bytes.Index([]byte(tag.Message), []byte("-----BEGIN"))
+ index := bytes.Index(tag.Message, []byte("-----BEGIN"))
if index > 0 {
signature := string(tag.Message[index : bytes.Index(tag.Message[index:], []byte("\n"))+index])
tag.SignatureType = detectSignatureType(signature)
diff --git a/internal/git/receivepack.go b/internal/git/receivepack.go
index 7c09d96da..6397497bc 100644
--- a/internal/git/receivepack.go
+++ b/internal/git/receivepack.go
@@ -1,10 +1,17 @@
package git
import (
+ "context"
+ "errors"
"fmt"
+ "os"
+ "github.com/golang/protobuf/jsonpb"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
"gitlab.com/gitlab-org/gitaly/internal/gitlabshell"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/metadata"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
)
// ReceivePackRequest abstracts away the different requests that end up
@@ -13,16 +20,41 @@ type ReceivePackRequest interface {
GetGlId() string
GetGlUsername() string
GetGlRepository() string
+ GetRepository() *gitalypb.Repository
}
-// HookEnv is information we pass down to the Git hooks during
+var jsonpbMarshaller = &jsonpb.Marshaler{}
+
+// ReceivePackHookEnv is information we pass down to the Git hooks during
// git-receive-pack.
-func HookEnv(req ReceivePackRequest) []string {
- return append([]string{
+func ReceivePackHookEnv(ctx context.Context, req ReceivePackRequest) ([]string, error) {
+ repo, err := jsonpbMarshaller.MarshalToString(req.GetRepository())
+ if err != nil {
+ return nil, err
+ }
+
+ env := append([]string{
fmt.Sprintf("GL_ID=%s", req.GetGlId()),
fmt.Sprintf("GL_USERNAME=%s", req.GetGlUsername()),
fmt.Sprintf("GL_REPOSITORY=%s", req.GetGlRepository()),
+ fmt.Sprintf("GITALY_SOCKET=" + config.GitalyInternalSocketPath()),
+ fmt.Sprintf("GITALY_REPO=%s", repo),
+ fmt.Sprintf("GITALY_TOKEN=%s", config.Config.Auth.Token),
}, gitlabshell.Env()...)
+
+ praefect, err := metadata.ExtractPraefectServer(ctx)
+ if err == nil {
+ praefectEnv, err := praefect.Env()
+ if err != nil {
+ return nil, err
+ }
+
+ env = append(env, praefectEnv)
+ } else if !errors.Is(os.ErrNotExist, err) {
+ return nil, err
+ }
+
+ return env, nil
}
// ReceivePackConfig contains config options we want to enforce when
diff --git a/internal/git/stats/reference_discovery.go b/internal/git/stats/reference_discovery.go
index bcf1dc344..0bd83628d 100644
--- a/internal/git/stats/reference_discovery.go
+++ b/internal/git/stats/reference_discovery.go
@@ -94,12 +94,12 @@ func (d *ReferenceDiscovery) Parse(body io.Reader) error {
return errors.New("invalid first reference line")
}
- ref := strings.SplitN(string(split[0]), " ", 2)
+ ref := strings.SplitN(split[0], " ", 2)
if len(ref) != 2 {
return errors.New("invalid reference line")
}
d.Refs = append(d.Refs, Reference{Oid: ref[0], Name: ref[1]})
- d.Caps = strings.Split(string(split[1]), " ")
+ d.Caps = strings.Split(split[1], " ")
state = referenceDiscoveryExpectRef
case referenceDiscoveryExpectRef:
diff --git a/internal/git/uploadpack.go b/internal/git/uploadpack.go
index 9481e6e16..f09481c56 100644
--- a/internal/git/uploadpack.go
+++ b/internal/git/uploadpack.go
@@ -1,6 +1,6 @@
package git
-// UploadPackFilterConfig confins config options that are required to allow
+// UploadPackFilterConfig confines config options that are required to allow
// partial-clone filters.
func UploadPackFilterConfig() []Option {
return []Option{
diff --git a/internal/gitlabshell/env.go b/internal/gitlabshell/env.go
index 79bae0646..4db3c4fd0 100644
--- a/internal/gitlabshell/env.go
+++ b/internal/gitlabshell/env.go
@@ -1,6 +1,8 @@
package gitlabshell
-import "gitlab.com/gitlab-org/gitaly/internal/config"
+import (
+ "gitlab.com/gitlab-org/gitaly/internal/config"
+)
// Env is a helper that returns a slice with environment variables used by gitlab shell
func Env() []string {
diff --git a/internal/helper/fstype/detect_linux.go b/internal/helper/fstype/detect_linux.go
index 87ef046a9..3bacecb72 100644
--- a/internal/helper/fstype/detect_linux.go
+++ b/internal/helper/fstype/detect_linux.go
@@ -10,7 +10,7 @@ func detectFileSystem(path string) string {
// This explicit cast to int64 is required for systems where the syscall
// returns an int32 instead.
- fsType, found := magicMap[int64(stat.Type)]
+ fsType, found := magicMap[int64(stat.Type)] //nolint:unconvert
if !found {
return unknownFS
}
diff --git a/internal/helper/housekeeping/housekeeping.go b/internal/helper/housekeeping/housekeeping.go
index 41ee05247..7dfc57c60 100644
--- a/internal/helper/housekeeping/housekeeping.go
+++ b/internal/helper/housekeeping/housekeeping.go
@@ -7,7 +7,7 @@ import (
"strings"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
)
@@ -64,7 +64,7 @@ func Perform(ctx context.Context, repoPath string) error {
}
func myLogger(ctx context.Context) *log.Entry {
- return grpc_logrus.Extract(ctx).WithField("system", "housekeeping")
+ return ctxlogrus.Extract(ctx).WithField("system", "housekeeping")
}
// FixDirectoryPermissions does a recursive directory walk to look for
diff --git a/internal/helper/storage.go b/internal/helper/storage.go
index 6efa280e3..542e62d69 100644
--- a/internal/helper/storage.go
+++ b/internal/helper/storage.go
@@ -84,7 +84,7 @@ func InjectGitalyServers(ctx context.Context, name, address, token string) (cont
func DialServer(gitalyServer map[string]string) (*grpc.ClientConn, error) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(gitalyServer["token"])),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(gitalyServer["token"])),
}
conn, err := client.Dial(gitalyServer["address"], connOpts)
diff --git a/internal/log/hook.go b/internal/log/hook.go
index 54789ab3d..a8219b376 100644
--- a/internal/log/hook.go
+++ b/internal/log/hook.go
@@ -40,3 +40,8 @@ func (h *HookLogger) Fatalf(format string, a ...interface{}) {
fmt.Fprintf(os.Stderr, "error executing git hook")
h.logger.Fatalf(format, a...)
}
+
+// Fatalf logs a formatted error at the Fatal level
+func (h *HookLogger) Errorf(format string, a ...interface{}) {
+ h.logger.Errorf(format, a...)
+}
diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go
index c2178e554..af3beb7c8 100644
--- a/internal/metadata/featureflag/feature_flags.go
+++ b/internal/metadata/featureflag/feature_flags.go
@@ -8,10 +8,6 @@ const (
LinguistFileCountStats = "linguist_file_count_stats"
// HooksRPC will invoke update, pre receive, and post receive hooks by using RPCs
HooksRPC = "hooks_rpc"
- // CacheInvalidator controls the tracking of repo state via gRPC
- // annotations (i.e. accessor and mutator RPC's). This enables cache
- // invalidation by changing state when the repo is modified.
- CacheInvalidator = "cache_invalidator"
)
const (
diff --git a/internal/metadata/featureflag/grpc_header.go b/internal/metadata/featureflag/grpc_header.go
index 00d6e77ab..10a2472ec 100644
--- a/internal/metadata/featureflag/grpc_header.go
+++ b/internal/metadata/featureflag/grpc_header.go
@@ -2,7 +2,6 @@ package featureflag
import (
"context"
- "fmt"
"strconv"
"strings"
@@ -55,7 +54,31 @@ func IsDisabled(ctx context.Context, flag string) bool {
return !IsEnabled(ctx, flag)
}
+const ffPrefix = "gitaly-feature-"
+
// HeaderKey returns the feature flag key to be used in the metadata map
func HeaderKey(flag string) string {
- return fmt.Sprintf("gitaly-feature-%s", strings.ReplaceAll(flag, "_", "-"))
+ return ffPrefix + strings.ReplaceAll(flag, "_", "-")
+}
+
+// AllEnabledFlags returns all feature flags that use the Gitaly metadata
+// prefix and are enabled. Note: results will not be sorted.
+func AllEnabledFlags(ctx context.Context) []string {
+ md, ok := metadata.FromIncomingContext(ctx)
+ if !ok {
+ return nil
+ }
+
+ ffs := make([]string, 0, len(md))
+
+ for k, v := range md {
+ if !strings.HasPrefix(k, ffPrefix) {
+ continue
+ }
+ if len(v) > 0 && v[0] == "true" {
+ ffs = append(ffs, strings.TrimPrefix(k, ffPrefix))
+ }
+ }
+
+ return ffs
}
diff --git a/internal/metadata/featureflag/grpc_header_test.go b/internal/metadata/featureflag/grpc_header_test.go
index 4fceaf87c..55007fc86 100644
--- a/internal/metadata/featureflag/grpc_header_test.go
+++ b/internal/metadata/featureflag/grpc_header_test.go
@@ -33,3 +33,18 @@ func TestGRPCMetadataFeatureFlag(t *testing.T) {
})
}
}
+
+func TestAllEnabledFlags(t *testing.T) {
+ ctx := metadata.NewIncomingContext(
+ context.Background(),
+ metadata.New(
+ map[string]string{
+ ffPrefix + "meow": "true",
+ ffPrefix + "foo": "true",
+ ffPrefix + "woof": "false", // not enabled
+ ffPrefix + "bar": "TRUE", // not enabled
+ },
+ ),
+ )
+ assert.ElementsMatch(t, AllEnabledFlags(ctx), []string{"meow", "foo"})
+}
diff --git a/internal/middleware/cache/cache.go b/internal/middleware/cache/cache.go
index 79d8066e0..5f02a89e7 100644
--- a/internal/middleware/cache/cache.go
+++ b/internal/middleware/cache/cache.go
@@ -9,7 +9,6 @@ import (
"github.com/golang/protobuf/proto"
"github.com/sirupsen/logrus"
diskcache "gitlab.com/gitlab-org/gitaly/internal/cache"
- "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc"
@@ -38,8 +37,7 @@ func shouldIgnore(fullMethod string) bool {
// repository in a gRPC stream based RPC
func StreamInvalidator(ci Invalidator, reg *protoregistry.Registry) grpc.StreamServerInterceptor {
return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
- if !featureflag.IsEnabled(ss.Context(), featureflag.CacheInvalidator) ||
- shouldIgnore(info.FullMethod) {
+ if shouldIgnore(info.FullMethod) {
return handler(srv, ss)
}
@@ -66,8 +64,7 @@ func StreamInvalidator(ci Invalidator, reg *protoregistry.Registry) grpc.StreamS
// repository in a gRPC unary RPC
func UnaryInvalidator(ci Invalidator, reg *protoregistry.Registry) grpc.UnaryServerInterceptor {
return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
- if !featureflag.IsEnabled(ctx, featureflag.CacheInvalidator) ||
- shouldIgnore(info.FullMethod) {
+ if shouldIgnore(info.FullMethod) {
return handler(ctx, req)
}
diff --git a/internal/middleware/cache/cache_test.go b/internal/middleware/cache/cache_test.go
index d01087235..9bcb2343b 100644
--- a/internal/middleware/cache/cache_test.go
+++ b/internal/middleware/cache/cache_test.go
@@ -13,7 +13,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
diskcache "gitlab.com/gitlab-org/gitaly/internal/cache"
- "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/internal/middleware/cache"
"gitlab.com/gitlab-org/gitaly/internal/middleware/cache/testdata"
"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
@@ -32,22 +31,16 @@ func TestInvalidators(t *testing.T) {
srvr := grpc.NewServer(
grpc.StreamInterceptor(
- grpc.StreamServerInterceptor(
- cache.StreamInvalidator(mCache, reg),
- ),
+ cache.StreamInvalidator(mCache, reg),
),
grpc.UnaryInterceptor(
- grpc.UnaryServerInterceptor(
- cache.UnaryInvalidator(mCache, reg),
- ),
+ cache.UnaryInvalidator(mCache, reg),
),
)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
- ctx = featureflag.OutgoingCtxWithFeatureFlag(ctx, featureflag.CacheInvalidator)
-
svc := &testSvc{}
cli, cc, cleanup := newTestSvc(t, ctx, srvr, svc)
@@ -168,7 +161,7 @@ func streamFileDesc(t testing.TB) *descriptor.FileDescriptorProto {
return fdp
}
-func newTestSvc(t testing.TB, ctx context.Context, srvr *grpc.Server, svc testdata.TestServiceServer) (testdata.TestServiceClient, *grpc.ClientConn, func()) { //nolint:golint
+func newTestSvc(t testing.TB, ctx context.Context, srvr *grpc.Server, svc testdata.TestServiceServer) (testdata.TestServiceClient, *grpc.ClientConn, func()) {
healthSrvr := health.NewServer()
grpc_health_v1.RegisterHealthServer(srvr, healthSrvr)
healthSrvr.SetServingStatus("TestService", grpc_health_v1.HealthCheckResponse_SERVING)
diff --git a/internal/middleware/limithandler/metrics.go b/internal/middleware/limithandler/metrics.go
index 3e5f68f97..78c628cb9 100644
--- a/internal/middleware/limithandler/metrics.go
+++ b/internal/middleware/limithandler/metrics.go
@@ -5,7 +5,7 @@ import (
"strings"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/prometheus/client_golang/prometheus"
)
@@ -82,7 +82,7 @@ func (c *promMonitor) Enter(ctx context.Context, acquireTime time.Duration) {
c.inprogressGauge.Inc()
if acquireTime > acquireDurationLogThreshold {
- logger := grpc_logrus.Extract(ctx)
+ logger := ctxlogrus.Extract(ctx)
logger.WithField("acquire_ms", acquireTime.Seconds()*1000).Info("Rate limit acquire wait")
}
diff --git a/internal/praefect/auth_test.go b/internal/praefect/auth_test.go
index 7fce795b2..5539d2a5e 100644
--- a/internal/praefect/auth_test.go
+++ b/internal/praefect/auth_test.go
@@ -42,11 +42,6 @@ func TestAuthFailures(t *testing.T) {
code: codes.Unauthenticated,
},
{
- desc: "wrong secret",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials("foobar"))},
- code: codes.PermissionDenied,
- },
- {
desc: "wrong secret new auth",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2("foobar"))},
code: codes.PermissionDenied,
@@ -88,23 +83,7 @@ func TestAuthSuccess(t *testing.T) {
}{
{desc: "no auth, not required"},
{
- desc: "v1 incorrect auth, not required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials("incorrect"))},
- token: token,
- },
- {
- desc: "v1 correct auth, not required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token))},
- token: token,
- },
- {
- desc: "v1 correct auth, required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token))},
- token: token,
- required: true,
- },
- {
- desc: "v2 correct new auth, not required",
+ desc: "v2 correct auth, not required",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token))},
token: token,
},
@@ -114,7 +93,7 @@ func TestAuthSuccess(t *testing.T) {
token: token,
},
{
- desc: "v2 correct new auth, required",
+ desc: "v2 correct auth, required",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token))},
token: token,
required: true,
@@ -194,7 +173,7 @@ func runServer(t *testing.T, token string, required bool) (*Server, string, func
ReplicationEventQueue: datastore.NewMemoryReplicationEventQueue(),
}
- nodeMgr, err := nodes.NewManager(logEntry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(logEntry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
registry := protoregistry.New()
diff --git a/internal/praefect/config/config.go b/internal/praefect/config/config.go
index da7f76015..1b01b33b2 100644
--- a/internal/praefect/config/config.go
+++ b/internal/praefect/config/config.go
@@ -14,6 +14,11 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/praefect/models"
)
+type Failover struct {
+ Enabled bool `toml:"enabled"`
+ ElectionStrategy string `toml:"election_strategy"`
+}
+
// Config is a container for everything found in the TOML config file
type Config struct {
ListenAddr string `toml:"listen_addr"`
@@ -29,6 +34,8 @@ type Config struct {
Prometheus prometheus.Config `toml:"prometheus"`
Auth auth.Config `toml:"auth"`
DB `toml:"database"`
+ Failover Failover `toml:"failover"`
+ // Keep for legacy reasons: remove after Omnibus has switched
FailoverEnabled bool `toml:"failover_enabled"`
PostgresQueueEnabled bool `toml:"postgres_queue_enabled"`
}
@@ -63,6 +70,12 @@ func FromFile(filePath string) (Config, error) {
config.Nodes = nil
}
+ // TODO: Remove this after failover_enabled has moved under a separate failover section. This is for
+ // backwards compatibility only
+ if config.FailoverEnabled {
+ config.Failover.Enabled = true
+ }
+
return *config, err
}
@@ -144,6 +157,11 @@ func (c Config) Validate() error {
return nil
}
+// NeedsSQL returns true if the driver for SQL needs to be initialized
+func (c Config) NeedsSQL() bool {
+ return c.PostgresQueueEnabled || (c.Failover.Enabled && c.Failover.ElectionStrategy == "sql")
+}
+
// DB holds Postgres client configuration data.
type DB struct {
Host string `toml:"host"`
diff --git a/internal/praefect/config/config_test.go b/internal/praefect/config/config_test.go
index 07e4fb449..4a01db1a4 100644
--- a/internal/praefect/config/config_test.go
+++ b/internal/praefect/config/config_test.go
@@ -265,3 +265,48 @@ func TestToPQString(t *testing.T) {
})
}
}
+
+func TestNeedsSQL(t *testing.T) {
+ testCases := []struct {
+ desc string
+ config Config
+ expected bool
+ }{
+ {
+ desc: "default",
+ config: Config{},
+ expected: false,
+ },
+ {
+ desc: "PostgreSQL queue enabled",
+ config: Config{PostgresQueueEnabled: true},
+ expected: true,
+ },
+ {
+ desc: "Failover enabled with default election strategy",
+ config: Config{Failover: Failover{Enabled: true}},
+ expected: false,
+ },
+ {
+ desc: "Failover enabled with SQL election strategy",
+ config: Config{Failover: Failover{Enabled: true, ElectionStrategy: "sql"}},
+ expected: true,
+ },
+ {
+ desc: "Both PostgresQL and SQL election strategy enabled",
+ config: Config{PostgresQueueEnabled: true, Failover: Failover{Enabled: true, ElectionStrategy: "sql"}},
+ expected: true,
+ },
+ {
+ desc: "Both PostgresQL and SQL election strategy enabled but failover disabled",
+ config: Config{PostgresQueueEnabled: true, Failover: Failover{Enabled: false, ElectionStrategy: "sql"}},
+ expected: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ require.Equal(t, tc.expected, tc.config.NeedsSQL())
+ })
+ }
+}
diff --git a/internal/praefect/coordinator.go b/internal/praefect/coordinator.go
index 361f24602..eab186a9c 100644
--- a/internal/praefect/coordinator.go
+++ b/internal/praefect/coordinator.go
@@ -3,6 +3,7 @@ package praefect
import (
"context"
"fmt"
+ "sync"
"github.com/golang/protobuf/proto"
"github.com/sirupsen/logrus"
@@ -11,6 +12,7 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore"
"gitlab.com/gitlab-org/gitaly/internal/praefect/grpc-proxy/proxy"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/metadata"
"gitlab.com/gitlab-org/gitaly/internal/praefect/nodes"
"gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -77,6 +79,11 @@ func NewCoordinator(l logrus.FieldLogger, ds datastore.Datastore, nodeMgr nodes.
}
func (c *Coordinator) directRepositoryScopedMessage(ctx context.Context, mi protoregistry.MethodInfo, peeker proxy.StreamModifier, fullMethodName string, m proto.Message) (*proxy.StreamParameters, error) {
+ ctx, err := metadata.InjectPraefectServer(ctx, c.conf)
+ if err != nil {
+ return nil, fmt.Errorf("could not inject Praefect server")
+ }
+
targetRepo, err := mi.TargetRepo(m)
if err != nil {
return nil, helper.ErrInvalidArgument(err)
@@ -216,7 +223,10 @@ func (c *Coordinator) createReplicaJobs(
return func() {
correlationID := c.ensureCorrelationID(ctx, targetRepo)
+ var wg sync.WaitGroup
for _, secondary := range secondaries {
+ wg.Add(1)
+
event := datastore.ReplicationEvent{
Job: datastore.ReplicationJob{
Change: change,
@@ -228,13 +238,11 @@ func (c *Coordinator) createReplicaJobs(
Meta: datastore.Params{metadatahandler.CorrelationIDKey: correlationID},
}
- // TODO: it could happen that there won't be enough time to enqueue replication events
- // do we need to create another ctx with another timeout?
- // https://gitlab.com/gitlab-org/gitaly/-/issues/2586
go func() {
+ defer wg.Done()
_, err := c.datastore.Enqueue(ctx, event)
if err != nil {
- c.log.WithFields(logrus.Fields{
+ c.log.WithError(err).WithFields(logrus.Fields{
logWithReplSource: event.Job.SourceNodeStorage,
logWithReplTarget: event.Job.TargetNodeStorage,
logWithReplChange: event.Job.Change,
@@ -243,6 +251,7 @@ func (c *Coordinator) createReplicaJobs(
}
}()
}
+ wg.Wait()
}
}
diff --git a/internal/praefect/coordinator_test.go b/internal/praefect/coordinator_test.go
index 6abfa023f..b09bedcad 100644
--- a/internal/praefect/coordinator_test.go
+++ b/internal/praefect/coordinator_test.go
@@ -19,6 +19,7 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/testhelper/promtest"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"gitlab.com/gitlab-org/labkit/correlation"
+ "google.golang.org/grpc/metadata"
)
var testLogger = logrus.New()
@@ -75,7 +76,7 @@ func TestStreamDirector(t *testing.T) {
entry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(entry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(entry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
r := protoregistry.New()
require.NoError(t, r.RegisterFiles(protoregistry.GitalyProtoFileDescriptors...))
@@ -96,6 +97,10 @@ func TestStreamDirector(t *testing.T) {
require.NoError(t, err)
require.Equal(t, address, streamParams.Conn().Target())
+ md, ok := metadata.FromOutgoingContext(streamParams.Context())
+ require.True(t, ok)
+ require.Contains(t, md, "praefect-server")
+
mi, err := coordinator.registry.LookupMethod(fullMethod)
require.NoError(t, err)
@@ -200,7 +205,7 @@ func TestAbsentCorrelationID(t *testing.T) {
entry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(entry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(entry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
coordinator := NewCoordinator(entry, ds, nodeMgr, conf, protoregistry.GitalyProtoPreregistered)
diff --git a/internal/praefect/dataloss_check_test.go b/internal/praefect/dataloss_check_test.go
new file mode 100644
index 000000000..7b86dd362
--- /dev/null
+++ b/internal/praefect/dataloss_check_test.go
@@ -0,0 +1,124 @@
+package praefect
+
+import (
+ "testing"
+
+ "github.com/golang/protobuf/ptypes"
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/datastore"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/mock"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/models"
+ "gitlab.com/gitlab-org/gitaly/internal/testhelper"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
+
+func TestDatalossCheck(t *testing.T) {
+ cfg := config.Config{
+ VirtualStorages: []*config.VirtualStorage{
+ {
+ Name: "praefect",
+ Nodes: []*models.Node{
+ {
+ DefaultPrimary: true,
+ Storage: "not-needed",
+ Address: "tcp::/this-doesnt-matter",
+ },
+ },
+ },
+ },
+ }
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ rq := datastore.NewMemoryReplicationEventQueue()
+ const targetNode = "test-node"
+ killJobs := func(t *testing.T) {
+ t.Helper()
+ for {
+ jobs, err := rq.Dequeue(ctx, targetNode, 1)
+ require.NoError(t, err)
+ if len(jobs) == 0 {
+ // all jobs dead
+ break
+ }
+
+ state := datastore.JobStateFailed
+ if jobs[0].Attempt == 0 {
+ state = datastore.JobStateDead
+ }
+
+ _, err = rq.Acknowledge(ctx, state, []uint64{jobs[0].ID})
+ require.NoError(t, err)
+ }
+ }
+
+ beforeTimerange, err := rq.Enqueue(ctx, datastore.ReplicationEvent{
+ Job: datastore.ReplicationJob{
+ RelativePath: "repo/before-timerange",
+ },
+ })
+ require.NoError(t, err)
+ expectedDeadJobs := map[string]int64{"repo/dead-job": 1, "repo/multiple-dead-jobs": 2}
+ for relPath, count := range expectedDeadJobs {
+ for i := int64(0); i < count; i++ {
+ _, err := rq.Enqueue(ctx, datastore.ReplicationEvent{
+ Job: datastore.ReplicationJob{
+ RelativePath: relPath,
+ TargetNodeStorage: targetNode,
+ },
+ })
+ require.NoError(t, err)
+ }
+ }
+ killJobs(t)
+
+ // add some non-dead jobs
+ for relPath, state := range map[string]datastore.JobState{
+ "repo/completed-job": datastore.JobStateCompleted,
+ "repo/cancelled-job": datastore.JobStateCancelled,
+ } {
+ _, err := rq.Enqueue(ctx, datastore.ReplicationEvent{
+ Job: datastore.ReplicationJob{
+ RelativePath: relPath,
+ TargetNodeStorage: targetNode,
+ },
+ })
+ require.NoError(t, err)
+
+ jobs, err := rq.Dequeue(ctx, targetNode, 1)
+ require.NoError(t, err)
+
+ _, err = rq.Acknowledge(ctx, state, []uint64{jobs[0].ID})
+ require.NoError(t, err)
+ }
+
+ afterTimerange, err := rq.Enqueue(ctx, datastore.ReplicationEvent{
+ Job: datastore.ReplicationJob{
+ RelativePath: "repo/after-timerange",
+ },
+ })
+ require.NoError(t, err)
+ killJobs(t)
+
+ cc, _, clean := runPraefectServerWithMock(t, cfg,
+ datastore.Datastore{ReplicationEventQueue: rq},
+ map[string]mock.SimpleServiceServer{
+ "not-needed": &mock.UnimplementedSimpleServiceServer{},
+ },
+ )
+ defer clean()
+
+ pbFrom, err := ptypes.TimestampProto(beforeTimerange.CreatedAt)
+ require.NoError(t, err)
+ pbTo, err := ptypes.TimestampProto(afterTimerange.CreatedAt)
+ require.NoError(t, err)
+
+ resp, err := gitalypb.NewPraefectInfoServiceClient(cc).DatalossCheck(ctx, &gitalypb.DatalossCheckRequest{
+ From: pbFrom,
+ To: pbTo,
+ })
+ require.NoError(t, err)
+ require.Equal(t, &gitalypb.DatalossCheckResponse{ByRelativePath: expectedDeadJobs}, resp)
+}
diff --git a/internal/praefect/datastore/datastore.go b/internal/praefect/datastore/datastore.go
index 0e5f14654..4b71fa55b 100644
--- a/internal/praefect/datastore/datastore.go
+++ b/internal/praefect/datastore/datastore.go
@@ -100,8 +100,8 @@ type ReplJob struct {
RelativePath string // source for replication
State JobState
Attempts int
- Params Params // additional information required to run the job
- CorrelationID string // from original request
+ Params Params // additional information required to run the job
+ CorrelationID string // from original request
CreatedAt time.Time // when has the job been created?
}
diff --git a/internal/praefect/datastore/glsql/postgres.go b/internal/praefect/datastore/glsql/postgres.go
index ee837e86a..cee10faa1 100644
--- a/internal/praefect/datastore/glsql/postgres.go
+++ b/internal/praefect/datastore/glsql/postgres.go
@@ -30,8 +30,9 @@ func OpenDB(conf config.DB) (*sql.DB, error) {
}
// Migrate will apply all pending SQL migrations.
-func Migrate(db *sql.DB) (int, error) {
+func Migrate(db *sql.DB, ignoreUnknown bool) (int, error) {
migrationSource := &migrate.MemoryMigrationSource{Migrations: migrations.All()}
+ migrate.SetIgnoreUnknown(ignoreUnknown)
return migrate.Exec(db, "postgres", migrationSource, migrate.Up)
}
diff --git a/internal/praefect/datastore/glsql/testing.go b/internal/praefect/datastore/glsql/testing.go
index 8bff18850..91efa6286 100644
--- a/internal/praefect/datastore/glsql/testing.go
+++ b/internal/praefect/datastore/glsql/testing.go
@@ -54,6 +54,8 @@ func (db DB) TruncateAll(t testing.TB) {
"replication_queue_job_lock",
"replication_queue",
"replication_queue_lock",
+ "node_status",
+ "shard_primaries",
)
}
@@ -80,7 +82,7 @@ func GetDB(t testing.TB, database string) DB {
testDBInitOnce.Do(func() {
sqlDB := initGitalyTestDB(t, database)
- _, mErr := Migrate(sqlDB)
+ _, mErr := Migrate(sqlDB, false)
require.NoError(t, mErr, "failed to run database migration")
testDB = DB{DB: sqlDB}
})
@@ -90,9 +92,9 @@ func GetDB(t testing.TB, database string) DB {
return testDB
}
-func initGitalyTestDB(t testing.TB, database string) *sql.DB {
- t.Helper()
-
+// GetDBConfig returns the database configuration determined by
+// environment variables. See GetDB() for the list of variables.
+func GetDBConfig(t testing.TB, database string) config.DB {
getEnvFromGDK(t)
host, hostFound := os.LookupEnv("PGHOST")
@@ -104,13 +106,19 @@ func initGitalyTestDB(t testing.TB, database string) *sql.DB {
require.NoError(t, pErr, "PGPORT must be a port number of the Postgres database listens for incoming connections")
// connect to 'postgres' database first to re-create testing database from scratch
- dbCfg := config.DB{
+ return config.DB{
Host: host,
Port: portNumber,
- DBName: "postgres",
+ DBName: database,
SSLMode: "disable",
User: os.Getenv("PGUSER"),
}
+}
+
+func initGitalyTestDB(t testing.TB, database string) *sql.DB {
+ t.Helper()
+
+ dbCfg := GetDBConfig(t, "postgres")
postgresDB, oErr := OpenDB(dbCfg)
require.NoError(t, oErr, "failed to connect to 'postgres' database")
diff --git a/internal/praefect/datastore/init_test.go b/internal/praefect/datastore/init_test.go
index bfbffa166..a9601791f 100644
--- a/internal/praefect/datastore/init_test.go
+++ b/internal/praefect/datastore/init_test.go
@@ -7,6 +7,7 @@ import (
"os"
"testing"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore/glsql"
)
@@ -19,4 +20,5 @@ func TestMain(m *testing.M) {
os.Exit(code)
}
-func getDB(t testing.TB) glsql.DB { return glsql.GetDB(t, "datastore") }
+func getDB(t testing.TB) glsql.DB { return glsql.GetDB(t, "datastore") }
+func getDBConfig(t testing.TB) config.DB { return glsql.GetDBConfig(t, "datastore") }
diff --git a/internal/praefect/datastore/memory.go b/internal/praefect/datastore/memory.go
index 4f6f72938..64826d78d 100644
--- a/internal/praefect/datastore/memory.go
+++ b/internal/praefect/datastore/memory.go
@@ -2,22 +2,37 @@ package datastore
import (
"context"
+ "errors"
"fmt"
"sync"
"time"
)
+var (
+ errDeadAckedAsFailed = errors.New("job acknowledged as failed with no attempts left, should be 'dead'")
+)
+
// NewMemoryReplicationEventQueue return in-memory implementation of the ReplicationEventQueue.
func NewMemoryReplicationEventQueue() ReplicationEventQueue {
- return &memoryReplicationEventQueue{dequeued: map[uint64]struct{}{}}
+ return &memoryReplicationEventQueue{
+ dequeued: map[uint64]struct{}{},
+ maxDeadJobs: 1000,
+ }
+}
+
+type deadJob struct {
+ createdAt time.Time
+ relativePath string
}
// memoryReplicationEventQueue implements queue interface with in-memory implementation of storage
type memoryReplicationEventQueue struct {
sync.RWMutex
- seq uint64 // used to generate unique identifiers for events
- queued []ReplicationEvent // all new events stored as queue
- dequeued map[uint64]struct{} // all events dequeued, but not yet acknowledged
+ seq uint64 // used to generate unique identifiers for events
+ queued []ReplicationEvent // all new events stored as queue
+ dequeued map[uint64]struct{} // all events dequeued, but not yet acknowledged
+ maxDeadJobs int // maximum amount of dead jobs to hold in memory
+ deadJobs []deadJob // dead jobs stored for reporting purposes
}
// nextID returns a new sequential ID for new events.
@@ -50,11 +65,10 @@ func (s *memoryReplicationEventQueue) Dequeue(_ context.Context, nodeStorage str
for i := 0; i < len(s.queued); i++ {
event := s.queued[i]
- hasMoreAttempts := event.Attempt > 0
isForTargetStorage := event.Job.TargetNodeStorage == nodeStorage
isReadyOrFailed := event.State == JobStateReady || event.State == JobStateFailed
- if hasMoreAttempts && isForTargetStorage && isReadyOrFailed {
+ if isForTargetStorage && isReadyOrFailed {
updatedAt := time.Now().UTC()
event.Attempt--
event.State = JobStateInProgress
@@ -101,6 +115,10 @@ func (s *memoryReplicationEventQueue) Acknowledge(_ context.Context, state JobSt
return nil, fmt.Errorf("event not in progress, can't be acknowledged: %d [%s]", s.queued[i].ID, s.queued[i].State)
}
+ if s.queued[i].Attempt == 0 && state == JobStateFailed {
+ return nil, errDeadAckedAsFailed
+ }
+
updatedAt := time.Now().UTC()
s.queued[i].State = state
s.queued[i].UpdatedAt = &updatedAt
@@ -110,12 +128,7 @@ func (s *memoryReplicationEventQueue) Acknowledge(_ context.Context, state JobSt
switch state {
case JobStateCompleted, JobStateCancelled, JobStateDead:
// this event is fully processed and could be removed
- s.remove(i)
- case JobStateFailed:
- if s.queued[i].Attempt == 0 {
- // out of luck for this replication event, remove from queue as no more attempts available
- s.remove(i)
- }
+ s.remove(i, state)
}
break
}
@@ -124,9 +137,39 @@ func (s *memoryReplicationEventQueue) Acknowledge(_ context.Context, state JobSt
return result, nil
}
-// remove deletes i-th element from slice and from tracking map.
+// CountDeadReplicationJobs returns the dead replication job counts by relative path within the given timerange.
+// The timerange beginning is inclusive and ending is exclusive. The in-memory queue stores only the most recent
+// 1000 dead jobs.
+func (s *memoryReplicationEventQueue) CountDeadReplicationJobs(ctx context.Context, from, to time.Time) (map[string]int64, error) {
+ s.RLock()
+ defer s.RUnlock()
+
+ from = from.Add(-time.Nanosecond)
+ dead := map[string]int64{}
+ for _, job := range s.deadJobs {
+ if job.createdAt.After(from) && job.createdAt.Before(to) {
+ dead[job.relativePath]++
+ }
+ }
+
+ return dead, nil
+}
+
+// remove deletes i-th element from the queue and from the in-flight tracking map.
// It doesn't check 'i' for the out of range and must be called with lock protection.
-func (s *memoryReplicationEventQueue) remove(i int) {
+// If state is JobStateDead, the event will be added to the dead job tracker.
+func (s *memoryReplicationEventQueue) remove(i int, state JobState) {
+ if state == JobStateDead {
+ if len(s.deadJobs) >= s.maxDeadJobs {
+ s.deadJobs = s.deadJobs[1:]
+ }
+
+ s.deadJobs = append(s.deadJobs, deadJob{
+ createdAt: s.queued[i].CreatedAt,
+ relativePath: s.queued[i].Job.RelativePath,
+ })
+ }
+
delete(s.dequeued, s.queued[i].ID)
s.queued = append(s.queued[:i], s.queued[i+1:]...)
}
diff --git a/internal/praefect/datastore/memory_test.go b/internal/praefect/datastore/memory_test.go
index 1f33ff200..9eaefcadf 100644
--- a/internal/praefect/datastore/memory_test.go
+++ b/internal/praefect/datastore/memory_test.go
@@ -1,13 +1,144 @@
package datastore
import (
+ "fmt"
"sync"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
)
+func ContractTestCountDeadReplicationJobs(t *testing.T, q ReplicationEventQueue) {
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ // take the time here to include the also the
+ // completed and cancelled jobs in timerange
+ beforeOldest := time.Now()
+
+ const target = "target"
+ ackJobsToDeath := func(t *testing.T) {
+ t.Helper()
+
+ for {
+ jobs, err := q.Dequeue(ctx, target, 1)
+ require.NoError(t, err)
+ if len(jobs) == 0 {
+ break
+ }
+
+ for _, job := range jobs {
+ state := JobStateFailed
+ if job.Attempt == 0 {
+ state = JobStateDead
+ }
+
+ _, err := q.Acknowledge(ctx, state, []uint64{job.ID})
+ require.NoError(t, err)
+ }
+ }
+ }
+
+ // postgres only handles timestamps with a microsecond resolution thus
+ // we have to work with the time in microsecond sized steps
+ const tick = time.Microsecond
+
+ // add some other job states to the datastore to ensure they are not counted
+ for relPath, state := range map[string]JobState{"repo/completed-job": JobStateCompleted, "repo/cancelled-job": JobStateCancelled} {
+ _, err := q.Enqueue(ctx, ReplicationEvent{Job: ReplicationJob{RelativePath: relPath, TargetNodeStorage: target}})
+ require.NoError(t, err)
+
+ jobs, err := q.Dequeue(ctx, target, 1)
+ require.NoError(t, err)
+
+ _, err = q.Acknowledge(ctx, state, []uint64{jobs[0].ID})
+ require.NoError(t, err)
+ }
+
+ oldest, err := q.Enqueue(ctx, ReplicationEvent{Job: ReplicationJob{RelativePath: "old", TargetNodeStorage: target}})
+ require.NoError(t, err)
+
+ afterOldest := oldest.CreatedAt.Add(tick)
+
+ dead, err := q.CountDeadReplicationJobs(ctx, beforeOldest, afterOldest)
+ require.NoError(t, err)
+ require.Empty(t, dead, "should not include ready jobs")
+
+ jobs, err := q.Dequeue(ctx, target, 1)
+ require.NoError(t, err)
+
+ _, err = q.Acknowledge(ctx, JobStateFailed, []uint64{jobs[0].ID})
+ require.NoError(t, err)
+
+ dead, err = q.CountDeadReplicationJobs(ctx, beforeOldest, afterOldest)
+ require.NoError(t, err)
+ require.Empty(t, dead, "should not include failed jobs")
+
+ ackJobsToDeath(t)
+ dead, err = q.CountDeadReplicationJobs(ctx, beforeOldest, afterOldest)
+ require.NoError(t, err)
+ require.Equal(t, map[string]int64{"old": 1}, dead, "should include dead job")
+
+ middle, err := q.Enqueue(ctx, ReplicationEvent{Job: ReplicationJob{RelativePath: "new", TargetNodeStorage: target}})
+ require.NoError(t, err)
+
+ ackJobsToDeath(t)
+ dead, err = q.CountDeadReplicationJobs(ctx, beforeOldest, middle.CreatedAt.Add(tick))
+ require.NoError(t, err)
+ require.Equal(t, map[string]int64{"old": 1, "new": 1}, dead, "should include both dead jobs")
+
+ newest, err := q.Enqueue(ctx, ReplicationEvent{Job: ReplicationJob{RelativePath: "new", TargetNodeStorage: target}})
+ require.NoError(t, err)
+
+ ackJobsToDeath(t)
+ dead, err = q.CountDeadReplicationJobs(ctx, beforeOldest, newest.CreatedAt.Add(tick))
+ require.NoError(t, err)
+ require.Equal(t, map[string]int64{"old": 1, "new": 2}, dead, "dead job are grouped by relative path")
+
+ dead, err = q.CountDeadReplicationJobs(ctx, middle.CreatedAt, newest.CreatedAt.Add(-tick))
+ require.NoError(t, err)
+ require.Equal(t, map[string]int64{"new": 1}, dead, "should only count the in-between dead job")
+}
+
+func TestMemoryCountDeadReplicationJobs(t *testing.T) {
+ ContractTestCountDeadReplicationJobs(t, NewMemoryReplicationEventQueue())
+}
+
+func TestMemoryCountDeadReplicationJobsLimit(t *testing.T) {
+ q := NewMemoryReplicationEventQueue().(*memoryReplicationEventQueue)
+ q.maxDeadJobs = 2
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ const target = "target"
+
+ beforeAll := time.Now()
+ for i := 0; i < q.maxDeadJobs+1; i++ {
+ job, err := q.Enqueue(ctx, ReplicationEvent{Job: ReplicationJob{RelativePath: fmt.Sprintf("job-%d", i), TargetNodeStorage: target}})
+ require.NoError(t, err)
+
+ for i := 0; i < job.Attempt; i++ {
+ _, err := q.Dequeue(ctx, target, 1)
+ require.NoError(t, err)
+
+ state := JobStateFailed
+ if i == job.Attempt-1 {
+ state = JobStateDead
+ }
+
+ _, err = q.Acknowledge(ctx, state, []uint64{job.ID})
+ require.NoError(t, err)
+ }
+ }
+
+ dead, err := q.CountDeadReplicationJobs(ctx, beforeAll, time.Now())
+ require.NoError(t, err)
+ require.Equal(t, map[string]int64{"job-1": 1, "job-2": 1}, dead, "should only include the last two dead jobs")
+}
+
func TestMemoryReplicationEventQueue(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
@@ -124,7 +255,11 @@ func TestMemoryReplicationEventQueue(t *testing.T) {
}
require.Equal(t, expAttempt3, dequeuedAttempt3[0])
- acknowledgedAttempt3, err := queue.Acknowledge(ctx, JobStateFailed, []uint64{event1.ID})
+ ackFailedNoAttemptsLeft, err := queue.Acknowledge(ctx, JobStateFailed, []uint64{event1.ID})
+ require.Error(t, errDeadAckedAsFailed, err)
+ require.Empty(t, ackFailedNoAttemptsLeft)
+
+ acknowledgedAttempt3, err := queue.Acknowledge(ctx, JobStateDead, []uint64{event1.ID})
require.NoError(t, err)
require.Equal(t, []uint64{event1.ID}, acknowledgedAttempt3, "one event must be acknowledged")
diff --git a/internal/praefect/datastore/migrations/20200324001604_add_sql_election_tables.go b/internal/praefect/datastore/migrations/20200324001604_add_sql_election_tables.go
new file mode 100644
index 000000000..7a96769b2
--- /dev/null
+++ b/internal/praefect/datastore/migrations/20200324001604_add_sql_election_tables.go
@@ -0,0 +1,34 @@
+package migrations
+
+import migrate "github.com/rubenv/sql-migrate"
+
+func init() {
+ m := &migrate.Migration{
+ Id: "20200324001604_add_sql_election_tables",
+ Up: []string{
+ `CREATE TABLE node_status (
+ id BIGSERIAL PRIMARY KEY,
+ praefect_name VARCHAR(511) NOT NULL,
+ shard_name VARCHAR(255) NOT NULL,
+ node_name VARCHAR(255) NOT NULL,
+ last_contact_attempt_at TIMESTAMP WITH TIME ZONE,
+ last_seen_active_at TIMESTAMP WITH TIME ZONE)`,
+ "CREATE UNIQUE INDEX shard_node_names_on_node_status_idx ON node_status (praefect_name, shard_name, node_name)",
+ "CREATE INDEX shard_name_on_node_status_idx ON node_status (shard_name, node_name)",
+
+ `CREATE TABLE shard_primaries (
+ id BIGSERIAL PRIMARY KEY,
+ shard_name VARCHAR(255) NOT NULL,
+ node_name VARCHAR(255) NOT NULL,
+ elected_by_praefect VARCHAR(255) NOT NULL,
+ elected_at TIMESTAMP WITH TIME ZONE NOT NULL)`,
+ "CREATE UNIQUE INDEX shard_name_on_shard_primaries_idx ON shard_primaries (shard_name)",
+ },
+ Down: []string{
+ "DROP TABLE shard_primaries",
+ "DROP TABLE node_status",
+ },
+ }
+
+ allMigrations = append(allMigrations, m)
+}
diff --git a/internal/praefect/datastore/postgres.go b/internal/praefect/datastore/postgres.go
index 0cb05e34a..8e6d1744d 100644
--- a/internal/praefect/datastore/postgres.go
+++ b/internal/praefect/datastore/postgres.go
@@ -12,6 +12,14 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore/migrations"
)
+// MigrationStatusRow represents an entry in the schema migrations table.
+// If the migration is in the database but is not listed, Unknown will be true.
+type MigrationStatusRow struct {
+ Migrated bool
+ Unknown bool
+ AppliedAt time.Time
+}
+
// CheckPostgresVersion checks the server version of the Postgres DB
// specified in conf. This is a diagnostic for the Praefect Postgres
// rollout. https://gitlab.com/gitlab-org/gitaly/issues/1755
@@ -66,6 +74,47 @@ func MigrateDown(conf config.Config, max int) (int, error) {
return migrate.ExecMax(db, sqlMigrateDialect, migrationSource(), migrate.Down, max)
}
+// MigrateStatus returns the status of database migrations. The key of the map
+// indexes the migration ID.
+func MigrateStatus(conf config.Config) (map[string]*MigrationStatusRow, error) {
+ db, err := glsql.OpenDB(conf.DB)
+ if err != nil {
+ return nil, fmt.Errorf("sql open: %v", err)
+ }
+ defer db.Close()
+
+ migrations, err := migrationSource().FindMigrations()
+ if err != nil {
+ return nil, err
+ }
+
+ records, err := migrate.GetMigrationRecords(db, sqlMigrateDialect)
+ if err != nil {
+ return nil, err
+ }
+
+ rows := make(map[string]*MigrationStatusRow)
+
+ for _, m := range migrations {
+ rows[m.Id] = &MigrationStatusRow{
+ Migrated: false,
+ }
+ }
+
+ for _, r := range records {
+ if rows[r.Id] == nil {
+ rows[r.Id] = &MigrationStatusRow{
+ Unknown: true,
+ }
+ }
+
+ rows[r.Id].Migrated = true
+ rows[r.Id].AppliedAt = r.AppliedAt
+ }
+
+ return rows, nil
+}
+
func migrationSource() *migrate.MemoryMigrationSource {
return &migrate.MemoryMigrationSource{Migrations: migrations.All()}
}
diff --git a/internal/praefect/datastore/postgres_test.go b/internal/praefect/datastore/postgres_test.go
new file mode 100644
index 000000000..c23c84af6
--- /dev/null
+++ b/internal/praefect/datastore/postgres_test.go
@@ -0,0 +1,32 @@
+// +build postgres
+
+package datastore
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+)
+
+func TestMigrateStatus(t *testing.T) {
+ db := getDB(t)
+
+ config := config.Config{
+ DB: getDBConfig(t),
+ }
+
+ _, err := db.Exec("INSERT INTO schema_migrations VALUES ('2020_01_01_test', NOW())")
+ require.NoError(t, err)
+
+ rows, err := MigrateStatus(config)
+ require.NoError(t, err)
+
+ m := rows["20200109161404_hello_world"]
+ require.True(t, m.Migrated)
+ require.False(t, m.Unknown)
+
+ m = rows["2020_01_01_test"]
+ require.True(t, m.Migrated)
+ require.True(t, m.Unknown)
+}
diff --git a/internal/praefect/datastore/queue.go b/internal/praefect/datastore/queue.go
index f9fbd41c5..16e893a3f 100644
--- a/internal/praefect/datastore/queue.go
+++ b/internal/praefect/datastore/queue.go
@@ -8,6 +8,7 @@ import (
"fmt"
"time"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore/glsql"
)
@@ -21,6 +22,9 @@ type ReplicationEventQueue interface {
// It only updates events that are in 'in_progress' state.
// It returns list of ids that was actually acknowledged.
Acknowledge(ctx context.Context, state JobState, ids []uint64) ([]uint64, error)
+ // CountDeadReplicationJobs returns the dead replication job counts by relative path within the
+ // given timerange. The timerange beginning is inclusive and ending is exclusive.
+ CountDeadReplicationJobs(ctx context.Context, from, to time.Time) (map[string]int64, error)
}
func allowToAck(state JobState) error {
@@ -148,6 +152,43 @@ type PostgresReplicationEventQueue struct {
qc glsql.Querier
}
+// CountDeadReplicationJobs returns the dead replication job counts by relative path within the
+// given timerange. The timerange beginning is inclusive and ending is exclusive.
+func (rq PostgresReplicationEventQueue) CountDeadReplicationJobs(ctx context.Context, from, to time.Time) (map[string]int64, error) {
+ const q = `
+ SELECT job->>'relative_path', count(*)
+ FROM replication_queue
+ WHERE state = 'dead'
+ AND created_at >= $1
+ AND created_at < $2
+ GROUP BY job->>'relative_path';
+ `
+
+ rows, err := rq.qc.QueryContext(ctx, q, from.UTC(), to.UTC())
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if err := rows.Close(); err != nil {
+ ctxlogrus.Extract(ctx).WithError(err).Error("error closing database rows")
+ }
+ }()
+
+ out := map[string]int64{}
+ for rows.Next() {
+ var relativePath string
+ var count int64
+
+ if err := rows.Scan(&relativePath, &count); err != nil {
+ return nil, err
+ }
+
+ out[relativePath] = count
+ }
+
+ return out, rows.Err()
+}
+
func (rq PostgresReplicationEventQueue) Enqueue(ctx context.Context, event ReplicationEvent) (ReplicationEvent, error) {
query := `
WITH insert_lock AS (
diff --git a/internal/praefect/datastore/queue_test.go b/internal/praefect/datastore/queue_test.go
index 56010f86c..5a0bd91ad 100644
--- a/internal/praefect/datastore/queue_test.go
+++ b/internal/praefect/datastore/queue_test.go
@@ -12,6 +12,10 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
)
+func TestPostgresReplicationEventQueue_CountDeadReplicationJobs(t *testing.T) {
+ ContractTestCountDeadReplicationJobs(t, PostgresReplicationEventQueue{getDB(t).DB})
+}
+
func TestPostgresReplicationEventQueue_Enqueue(t *testing.T) {
db := getDB(t)
diff --git a/internal/praefect/grpc-proxy/proxy/helper_test.go b/internal/praefect/grpc-proxy/proxy/helper_test.go
index 7e001b8c6..c240f8d71 100644
--- a/internal/praefect/grpc-proxy/proxy/helper_test.go
+++ b/internal/praefect/grpc-proxy/proxy/helper_test.go
@@ -18,7 +18,7 @@ func newListener(tb testing.TB) net.Listener {
return listener
}
-func newBackendPinger(tb testing.TB, ctx context.Context) (*grpc.ClientConn, *interceptPinger, func()) { //nolint:golint
+func newBackendPinger(tb testing.TB, ctx context.Context) (*grpc.ClientConn, *interceptPinger, func()) {
ip := &interceptPinger{}
done := make(chan struct{})
@@ -52,7 +52,7 @@ func newBackendPinger(tb testing.TB, ctx context.Context) (*grpc.ClientConn, *in
return cc, ip, cleanup
}
-func newProxy(tb testing.TB, ctx context.Context, director proxy.StreamDirector, svc, method string) (*grpc.ClientConn, func()) { //nolint:golint
+func newProxy(tb testing.TB, ctx context.Context, director proxy.StreamDirector, svc, method string) (*grpc.ClientConn, func()) {
proxySrvr := grpc.NewServer(
grpc.CustomCodec(proxy.NewCodec()),
grpc.UnknownServiceHandler(proxy.TransparentHandler(director)),
diff --git a/internal/praefect/helper_test.go b/internal/praefect/helper_test.go
index 1d1353cde..e0275c856 100644
--- a/internal/praefect/helper_test.go
+++ b/internal/praefect/helper_test.go
@@ -74,11 +74,7 @@ func testConfig(backends int) config.Config {
// setupServer wires all praefect dependencies together via dependency
// injection
-func setupServer(t testing.TB, conf config.Config, nodeMgr nodes.Manager, l *logrus.Entry, r *protoregistry.Registry) *Server {
- ds := datastore.Datastore{
- ReplicasDatastore: datastore.NewInMemory(conf),
- ReplicationEventQueue: datastore.NewMemoryReplicationEventQueue(),
- }
+func setupServer(t testing.TB, conf config.Config, nodeMgr nodes.Manager, ds datastore.Datastore, l *logrus.Entry, r *protoregistry.Registry) *Server {
coordinator := NewCoordinator(l, ds, nodeMgr, conf, r)
var defaultNode *models.Node
@@ -99,7 +95,7 @@ func setupServer(t testing.TB, conf config.Config, nodeMgr nodes.Manager, l *log
// config.Nodes. There must be a 1-to-1 mapping between backend server and
// configured storage node.
// requires there to be only 1 virtual storage
-func runPraefectServerWithMock(t *testing.T, conf config.Config, backends map[string]mock.SimpleServiceServer) (mock.SimpleServiceClient, *Server, testhelper.Cleanup) {
+func runPraefectServerWithMock(t *testing.T, conf config.Config, ds datastore.Datastore, backends map[string]mock.SimpleServiceServer) (*grpc.ClientConn, *Server, testhelper.Cleanup) {
require.Len(t, conf.VirtualStorages, 1)
require.Equal(t, len(backends), len(conf.VirtualStorages[0].Nodes),
"mock server count doesn't match config nodes")
@@ -117,21 +113,21 @@ func runPraefectServerWithMock(t *testing.T, conf config.Config, backends map[st
conf.VirtualStorages[0].Nodes[i] = node
}
- nodeMgr, err := nodes.NewManager(testhelper.DiscardTestEntry(t), conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(testhelper.DiscardTestEntry(t), conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
nodeMgr.Start(1*time.Millisecond, 5*time.Millisecond)
r := protoregistry.New()
require.NoError(t, r.RegisterFiles(mustLoadProtoReg(t)))
- prf := setupServer(t, conf, nodeMgr, log.Default(), r)
+ prf := setupServer(t, conf, nodeMgr, ds, log.Default(), r)
listener, port := listenAvailPort(t)
t.Logf("praefect listening on port %d", port)
errQ := make(chan error)
- prf.RegisterServices(nodeMgr, conf, datastore.Datastore{})
+ prf.RegisterServices(nodeMgr, conf, ds)
go func() {
errQ <- prf.Serve(listener, false)
}()
@@ -152,7 +148,7 @@ func runPraefectServerWithMock(t *testing.T, conf config.Config, backends map[st
require.NoError(t, prf.Shutdown(ctx))
}
- return mock.NewSimpleServiceClient(cc), prf, cleanup
+ return cc, prf, cleanup
}
func noopBackoffFunc() (backoff, backoffReset) {
@@ -167,7 +163,15 @@ func runPraefectServerWithGitaly(t *testing.T, conf config.Config) (*grpc.Client
require.Len(t, conf.VirtualStorages, 1)
var cleanups []testhelper.Cleanup
- _, backendAddr, cleanupGitaly := runInternalGitalyServer(t, conf.VirtualStorages[0].Nodes[0].Token)
+ var storages []gconfig.Storage
+ for _, node := range conf.VirtualStorages[0].Nodes {
+ storages = append(storages, gconfig.Storage{
+ Name: node.Storage,
+ Path: testhelper.GitlabTestStoragePath(),
+ })
+ }
+
+ _, backendAddr, cleanupGitaly := runInternalGitalyServer(t, storages, conf.VirtualStorages[0].Nodes[0].Token)
cleanups = append(cleanups, cleanupGitaly)
for i, node := range conf.VirtualStorages[0].Nodes {
@@ -181,7 +185,7 @@ func runPraefectServerWithGitaly(t *testing.T, conf config.Config) (*grpc.Client
}
logEntry := log.Default()
- nodeMgr, err := nodes.NewManager(testhelper.DiscardTestEntry(t), conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(testhelper.DiscardTestEntry(t), conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
nodeMgr.Start(1*time.Millisecond, 5*time.Millisecond)
@@ -233,7 +237,7 @@ func runPraefectServerWithGitaly(t *testing.T, conf config.Config) (*grpc.Client
return cc, prf, cleanup
}
-func runInternalGitalyServer(t *testing.T, token string) (*grpc.Server, string, func()) {
+func runInternalGitalyServer(t *testing.T, storages []gconfig.Storage, token string) (*grpc.Server, string, func()) {
streamInt := []grpc.StreamServerInterceptor{auth.StreamServerInterceptor(internalauth.Config{Token: token})}
unaryInt := []grpc.UnaryServerInterceptor{auth.UnaryServerInterceptor(internalauth.Config{Token: token})}
@@ -247,7 +251,7 @@ func runInternalGitalyServer(t *testing.T, token string) (*grpc.Server, string,
internalListener, err := net.Listen("unix", internalSocket)
require.NoError(t, err)
- gitalypb.RegisterServerServiceServer(server, gitalyserver.NewServer())
+ gitalypb.RegisterServerServiceServer(server, gitalyserver.NewServer(storages))
gitalypb.RegisterRepositoryServiceServer(server, repository.NewServer(RubyServer, internalSocket))
gitalypb.RegisterInternalGitalyServer(server, internalgitaly.NewServer(gconfig.Config.Storages))
healthpb.RegisterHealthServer(server, health.NewServer())
diff --git a/internal/praefect/info_service_test.go b/internal/praefect/info_service_test.go
index b35769bd4..5ee9d96cd 100644
--- a/internal/praefect/info_service_test.go
+++ b/internal/praefect/info_service_test.go
@@ -44,7 +44,7 @@ func TestInfoService_RepositoryReplicas(t *testing.T) {
gconfig.Config.Storages = storages
}(gconfig.Config.Storages)
- tempDir, cleanupTempDir := testhelper.TempDir(t, "praefect-test")
+ tempDir, cleanupTempDir := testhelper.TempDir(t)
defer cleanupTempDir()
for _, node := range conf.VirtualStorages[0].Nodes {
diff --git a/internal/praefect/metadata/server.go b/internal/praefect/metadata/server.go
new file mode 100644
index 000000000..61af49a7c
--- /dev/null
+++ b/internal/praefect/metadata/server.go
@@ -0,0 +1,110 @@
+package metadata
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "os"
+
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "google.golang.org/grpc/metadata"
+)
+
+const (
+ PraefectMetadataKey = "praefect-server"
+ PraefectEnvKey = "PRAEFECT_SERVER"
+)
+
+type PraefectServer struct {
+ Address string `json:"address"`
+ Token string `json:"token"`
+}
+
+// InjectPraefectServer injects Praefect connection metadata into an incoming context
+func InjectPraefectServer(ctx context.Context, conf config.Config) (context.Context, error) {
+ var address string
+ if conf.ListenAddr != "" {
+ address = conf.ListenAddr
+ } else if conf.SocketPath != "" {
+ address = "unix://" + conf.SocketPath
+ }
+
+ praefectServer := PraefectServer{
+ Address: address,
+ Token: conf.Auth.Token,
+ }
+
+ marshalled, err := json.Marshal(praefectServer)
+ if err != nil {
+ return nil, err
+ }
+
+ md, ok := metadata.FromIncomingContext(ctx)
+ if !ok {
+ md = metadata.New(map[string]string{})
+ } else {
+ md = md.Copy()
+ }
+ md.Set(PraefectMetadataKey, base64.StdEncoding.EncodeToString(marshalled))
+
+ return metadata.NewIncomingContext(ctx, md), nil
+}
+
+// ExtractPraefectServer extracts `PraefectServer` from an incoming context. In
+// case the metadata key is not set, the function will return `os.ErrNotExist`.
+func ExtractPraefectServer(ctx context.Context) (p *PraefectServer, err error) {
+ md, ok := metadata.FromIncomingContext(ctx)
+ if !ok {
+ return nil, os.ErrNotExist
+ }
+
+ encoded := md[PraefectMetadataKey]
+ if len(encoded) == 0 {
+ return nil, os.ErrNotExist
+ }
+
+ decoded, err := base64.StdEncoding.DecodeString(encoded[0])
+ if err != nil {
+ return nil, fmt.Errorf("failed decoding base64: %v", err)
+ }
+
+ if err := json.Unmarshal(decoded, &p); err != nil {
+ return nil, fmt.Errorf("failed unmarshalling json: %v", err)
+ }
+
+ return
+}
+
+// PraefectFromEnv extracts `PraefectServer` from the environment variable
+// `PraefectEnvKey`. In case the variable is not set, the function will return
+// `os.ErrNotExist`.
+func PraefectFromEnv() (*PraefectServer, error) {
+ encoded, ok := os.LookupEnv(PraefectEnvKey)
+ if !ok {
+ return nil, os.ErrNotExist
+ }
+
+ decoded, err := base64.StdEncoding.DecodeString(encoded)
+ if err != nil {
+ return nil, fmt.Errorf("failed decoding base64: %w", err)
+ }
+
+ p := PraefectServer{}
+ if err := json.Unmarshal(decoded, &p); err != nil {
+ return nil, err
+ }
+
+ return &p, nil
+}
+
+// Env encodes the `PraefectServer` and returns an environment variable.
+func (p PraefectServer) Env() (string, error) {
+ marshalled, err := json.Marshal(p)
+ if err != nil {
+ return "", err
+ }
+
+ encoded := base64.StdEncoding.EncodeToString(marshalled)
+ return fmt.Sprintf("%s=%s", PraefectEnvKey, encoded), nil
+}
diff --git a/internal/praefect/metrics/prometheus.go b/internal/praefect/metrics/prometheus.go
index 47bdc1626..48ac91708 100644
--- a/internal/praefect/metrics/prometheus.go
+++ b/internal/praefect/metrics/prometheus.go
@@ -22,7 +22,6 @@ func RegisterReplicationDelay(conf promconfig.Config) (metrics.HistogramVec, err
return replicationDelay, prometheus.Register(replicationDelay)
}
-
// RegisterReplicationLatency creates and registers a prometheus histogram
// to observe replication latency times
func RegisterReplicationLatency(conf promconfig.Config) (metrics.HistogramVec, error) {
@@ -83,6 +82,14 @@ var PrimaryGauge = prometheus.NewGaugeVec(
}, []string{"virtual_storage", "gitaly_storage"},
)
+var NodeLastHealthcheckGauge = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "gitaly",
+ Subsystem: "praefect",
+ Name: "node_last_healthcheck_up",
+ }, []string{"gitaly_storage"},
+)
+
var ChecksumMismatchCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "gitaly",
@@ -96,5 +103,6 @@ func init() {
MethodTypeCounter,
PrimaryGauge,
ChecksumMismatchCounter,
+ NodeLastHealthcheckGauge,
)
}
diff --git a/internal/praefect/models/node.go b/internal/praefect/models/node.go
index 7c7144165..57cf3ac46 100644
--- a/internal/praefect/models/node.go
+++ b/internal/praefect/models/node.go
@@ -1,6 +1,7 @@
package models
import (
+ "encoding/json"
"fmt"
)
@@ -12,6 +13,18 @@ type Node struct {
DefaultPrimary bool `toml:"primary"`
}
+func (n Node) MarshalJSON() ([]byte, error) {
+ return json.Marshal(&struct {
+ Storage string `json:"storage"`
+ Address string `json:"address"`
+ Primary bool `json:"primary"`
+ }{
+ Storage: n.Storage,
+ Address: n.Address,
+ Primary: n.DefaultPrimary,
+ })
+}
+
// String prints out the node attributes but hiding the token
func (n Node) String() string {
return fmt.Sprintf("storage_name: %s, address: %s, primary: %v", n.Storage, n.Address, n.DefaultPrimary)
diff --git a/internal/praefect/models/node_test.go b/internal/praefect/models/node_test.go
index 64942711f..7cefdf7f1 100644
--- a/internal/praefect/models/node_test.go
+++ b/internal/praefect/models/node_test.go
@@ -1,6 +1,7 @@
package models
import (
+ "encoding/json"
"testing"
"github.com/stretchr/testify/require"
@@ -34,3 +35,18 @@ func TestRepository_Clone(t *testing.T) {
clone.Replicas[0].Address = "0.0.0.3"
require.Equal(t, "0.0.0.1", src.Replicas[0].Address)
}
+
+func TestNode_MarshalJSON(t *testing.T) {
+ token := "secretToken"
+ node := &Node{
+ Storage: "storage",
+ Address: "address",
+ Token: token,
+ DefaultPrimary: true,
+ }
+
+ b, err := json.Marshal(node)
+ require.NoError(t, err)
+ require.NotContains(t, string(b), "token")
+ require.NotContains(t, string(b), token)
+}
diff --git a/internal/praefect/nodes/init_test.go b/internal/praefect/nodes/init_test.go
new file mode 100644
index 000000000..70c50dd30
--- /dev/null
+++ b/internal/praefect/nodes/init_test.go
@@ -0,0 +1,22 @@
+// +build postgres
+
+package nodes
+
+import (
+ "log"
+ "os"
+ "testing"
+
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/datastore/glsql"
+)
+
+func TestMain(m *testing.M) {
+ code := m.Run()
+ // Clean closes connection to database once all tests are done
+ if err := glsql.Clean(); err != nil {
+ log.Fatalln(err, "database disconnection failure")
+ }
+ os.Exit(code)
+}
+
+func getDB(t testing.TB) glsql.DB { return glsql.GetDB(t, "nodes") }
diff --git a/internal/praefect/nodes/local_elector.go b/internal/praefect/nodes/local_elector.go
index aa9b332de..dc3d73a68 100644
--- a/internal/praefect/nodes/local_elector.go
+++ b/internal/praefect/nodes/local_elector.go
@@ -5,12 +5,13 @@ import (
"sync"
"time"
+ "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/praefect/metrics"
)
type nodeCandidate struct {
+ m sync.RWMutex
node Node
- primary bool
statuses []bool
}
@@ -24,13 +25,30 @@ type localElector struct {
shardName string
nodes []*nodeCandidate
primaryNode *nodeCandidate
+ log logrus.FieldLogger
}
// healthcheckThreshold is the number of consecutive healthpb.HealthCheckResponse_SERVING necessary
// for deeming a node "healthy"
const healthcheckThreshold = 3
+func (n *nodeCandidate) checkNode(ctx context.Context) {
+ status, _ := n.node.check(ctx)
+
+ n.m.Lock()
+ defer n.m.Unlock()
+
+ n.statuses = append(n.statuses, status)
+
+ if len(n.statuses) > healthcheckThreshold {
+ n.statuses = n.statuses[1:]
+ }
+}
+
func (n *nodeCandidate) isHealthy() bool {
+ n.m.RLock()
+ defer n.m.RUnlock()
+
if len(n.statuses) < healthcheckThreshold {
return false
}
@@ -44,39 +62,28 @@ func (n *nodeCandidate) isHealthy() bool {
return true
}
-func newLocalElector(name string, failoverEnabled bool) *localElector {
+func newLocalElector(name string, failoverEnabled bool, log logrus.FieldLogger, ns []*nodeStatus) *localElector {
+ nodes := make([]*nodeCandidate, len(ns))
+ for i, n := range ns {
+ nodes[i] = &nodeCandidate{
+ node: n,
+ }
+ }
+
return &localElector{
shardName: name,
failoverEnabled: failoverEnabled,
+ log: log,
+ nodes: nodes,
+ primaryNode: nodes[0],
}
}
-// addNode registers a primary or secondary in the internal
-// datastore.
-func (s *localElector) addNode(node Node, primary bool) {
- localNode := nodeCandidate{
- node: node,
- primary: primary,
- statuses: make([]bool, 0),
- }
-
- s.m.Lock()
- defer s.m.Unlock()
-
- if primary {
- s.primaryNode = &localNode
- }
-
- s.nodes = append(s.nodes, &localNode)
-}
-
// Start launches a Goroutine to check the state of the nodes and
// continuously monitor their health via gRPC health checks.
-func (s *localElector) start(bootstrapInterval, monitorInterval time.Duration) error {
+func (s *localElector) start(bootstrapInterval, monitorInterval time.Duration) {
s.bootstrap(bootstrapInterval)
go s.monitor(monitorInterval)
-
- return nil
}
func (s *localElector) bootstrap(d time.Duration) {
@@ -96,50 +103,51 @@ func (s *localElector) monitor(d time.Duration) {
ticker := time.NewTicker(d)
defer ticker.Stop()
+ ctx := context.Background()
+
for {
<-ticker.C
- ctx := context.Background()
- s.checkNodes(ctx)
+ err := s.checkNodes(ctx)
+
+ if err != nil {
+ s.log.WithError(err).Warn("error checking nodes")
+ }
}
}
// checkNodes issues a gRPC health check for each node managed by the
// shard.
-func (s *localElector) checkNodes(ctx context.Context) {
- s.m.Lock()
+func (s *localElector) checkNodes(ctx context.Context) error {
defer s.updateMetrics()
- defer s.m.Unlock()
for _, n := range s.nodes {
- status, _ := n.node.check(ctx)
- n.statuses = append(n.statuses, status)
-
- if len(n.statuses) > healthcheckThreshold {
- n.statuses = n.statuses[1:]
- }
+ n.checkNode(ctx)
}
+ s.m.Lock()
+ defer s.m.Unlock()
+
if s.primaryNode != nil && s.primaryNode.isHealthy() {
- return
+ return nil
}
var newPrimary *nodeCandidate
for _, node := range s.nodes {
- if !node.primary && node.isHealthy() {
+ if node != s.primaryNode && node.isHealthy() {
newPrimary = node
break
}
}
if newPrimary == nil {
- return
+ return ErrPrimaryNotHealthy
}
- s.primaryNode.primary = false
s.primaryNode = newPrimary
- newPrimary.primary = true
+
+ return nil
}
// GetPrimary gets the primary of a shard. If no primary exists, it will
@@ -147,27 +155,29 @@ func (s *localElector) checkNodes(ctx context.Context) {
// ErrPrimaryNotHealthy.
func (s *localElector) GetPrimary() (Node, error) {
s.m.RLock()
- defer s.m.RUnlock()
+ primary := s.primaryNode
+ s.m.RUnlock()
- if s.primaryNode == nil {
+ if primary == nil {
return nil, ErrPrimaryNotHealthy
}
- if s.failoverEnabled && !s.primaryNode.isHealthy() {
- return s.primaryNode.node, ErrPrimaryNotHealthy
+ if s.failoverEnabled && !primary.isHealthy() {
+ return primary.node, ErrPrimaryNotHealthy
}
- return s.primaryNode.node, nil
+ return primary.node, nil
}
// GetSecondaries gets the secondaries of a shard
func (s *localElector) GetSecondaries() ([]Node, error) {
s.m.RLock()
- defer s.m.RUnlock()
+ primary := s.primaryNode
+ s.m.RUnlock()
var secondaries []Node
for _, n := range s.nodes {
- if !n.primary {
+ if n != primary {
secondaries = append(secondaries, n.node)
}
}
@@ -177,15 +187,16 @@ func (s *localElector) GetSecondaries() ([]Node, error) {
func (s *localElector) updateMetrics() {
s.m.RLock()
- defer s.m.RUnlock()
+ primary := s.primaryNode
+ s.m.RUnlock()
- for _, node := range s.nodes {
+ for _, n := range s.nodes {
var val float64
- if node.primary {
+ if n == primary {
val = 1
}
- metrics.PrimaryGauge.WithLabelValues(s.shardName, node.node.GetStorage()).Set(val)
+ metrics.PrimaryGauge.WithLabelValues(s.shardName, n.node.GetStorage()).Set(val)
}
}
diff --git a/internal/praefect/nodes/local_elector_test.go b/internal/praefect/nodes/local_elector_test.go
index 16f3a3875..e816392dc 100644
--- a/internal/praefect/nodes/local_elector_test.go
+++ b/internal/praefect/nodes/local_elector_test.go
@@ -1,6 +1,7 @@
package nodes
import (
+ "sync"
"testing"
"time"
@@ -11,10 +12,9 @@ import (
"google.golang.org/grpc"
)
-func TestPrimaryAndSecondaries(t *testing.T) {
+func setupElector(t *testing.T) (*localElector, []*nodeStatus, *grpc.ClientConn, *grpc.Server) {
socket := testhelper.GetTemporaryGitalySocketFileName()
svr, _ := testhelper.NewServerWithHealth(t, socket)
- defer svr.Stop()
cc, err := grpc.Dial(
"unix://"+socket,
@@ -24,30 +24,72 @@ func TestPrimaryAndSecondaries(t *testing.T) {
require.NoError(t, err)
storageName := "default"
- mockHistogramVec := promtest.NewMockHistogramVec()
+ mockHistogramVec0, mockHistogramVec1 := promtest.NewMockHistogramVec(), promtest.NewMockHistogramVec()
- cs := newConnectionStatus(models.Node{Storage: storageName}, cc, testhelper.DiscardTestEntry(t), mockHistogramVec)
- strategy := newLocalElector(storageName, true)
+ cs := newConnectionStatus(models.Node{Storage: storageName}, cc, testhelper.DiscardTestEntry(t), mockHistogramVec0)
+ secondary := newConnectionStatus(models.Node{Storage: storageName}, cc, testhelper.DiscardTestEntry(t), mockHistogramVec1)
+ ns := []*nodeStatus{cs, secondary}
+ logger := testhelper.NewTestLogger(t).WithField("test", t.Name())
+ strategy := newLocalElector(storageName, true, logger, ns)
- strategy.addNode(cs, true)
strategy.bootstrap(time.Second)
+ return strategy, ns, cc, svr
+}
+
+func TestPrimaryAndSecondaries(t *testing.T) {
+ strategy, ns, _, svr := setupElector(t)
+ defer svr.Stop()
+
primary, err := strategy.GetPrimary()
require.NoError(t, err)
- require.Equal(t, primary, cs)
+ require.Equal(t, ns[0], primary)
secondaries, err := strategy.GetSecondaries()
require.NoError(t, err)
- require.Equal(t, 0, len(secondaries))
+ require.Equal(t, 1, len(secondaries))
+ require.Equal(t, ns[1], secondaries[0])
+}
- secondary := newConnectionStatus(models.Node{Storage: storageName}, cc, testhelper.DiscardTestEntry(t), nil)
- strategy.addNode(secondary, false)
+func TestConcurrentCheckWithPrimary(t *testing.T) {
+ strategy, ns, _, svr := setupElector(t)
+ defer svr.Stop()
- secondaries, err = strategy.GetSecondaries()
+ iterations := 10
+ var wg sync.WaitGroup
+ start := make(chan bool)
+ wg.Add(2)
- require.NoError(t, err)
- require.Equal(t, 1, len(secondaries))
- require.Equal(t, secondary, secondaries[0])
+ go func() {
+ defer wg.Done()
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ <-start
+
+ for i := 0; i < iterations; i++ {
+ strategy.checkNodes(ctx)
+ }
+ }()
+
+ go func() {
+ defer wg.Done()
+ start <- true
+
+ for i := 0; i < iterations; i++ {
+ primary, err := strategy.GetPrimary()
+ require.Equal(t, ns[0], primary)
+ require.NoError(t, err)
+
+ secondaries, err := strategy.GetSecondaries()
+ require.NoError(t, err)
+ require.Equal(t, 1, len(secondaries))
+ require.Equal(t, ns[1], secondaries[0])
+ }
+ }()
+
+ wg.Wait()
}
diff --git a/internal/praefect/nodes/manager.go b/internal/praefect/nodes/manager.go
index ea2e57feb..0ef67ed5d 100644
--- a/internal/praefect/nodes/manager.go
+++ b/internal/praefect/nodes/manager.go
@@ -2,6 +2,7 @@ package nodes
import (
"context"
+ "database/sql"
"errors"
"time"
@@ -12,6 +13,7 @@ import (
"gitlab.com/gitlab-org/gitaly/client"
"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/internal/praefect/grpc-proxy/proxy"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/metrics"
"gitlab.com/gitlab-org/gitaly/internal/praefect/models"
prommetrics "gitlab.com/gitlab-org/gitaly/internal/prometheus/metrics"
correlation "gitlab.com/gitlab-org/labkit/correlation/grpc"
@@ -46,14 +48,14 @@ type Mgr struct {
log *logrus.Entry
// strategies is a map of strategies keyed on virtual storage name
strategies map[string]leaderElectionStrategy
+ db *sql.DB
}
// leaderElectionStrategy defines the interface by which primary and
// secondaries are managed.
type leaderElectionStrategy interface {
- start(bootstrapInterval, monitorInterval time.Duration) error
- addNode(node Node, primary bool)
- checkNodes(context.Context)
+ start(bootstrapInterval, monitorInterval time.Duration)
+ checkNodes(context.Context) error
Shard
}
@@ -63,23 +65,17 @@ type leaderElectionStrategy interface {
var ErrPrimaryNotHealthy = errors.New("primary is not healthy")
// NewManager creates a new NodeMgr based on virtual storage configs
-func NewManager(log *logrus.Entry, c config.Config, latencyHistogram prommetrics.HistogramVec, dialOpts ...grpc.DialOption) (*Mgr, error) {
- mgr := Mgr{
- log: log,
- failoverEnabled: c.FailoverEnabled}
-
- mgr.strategies = make(map[string]leaderElectionStrategy)
+func NewManager(log *logrus.Entry, c config.Config, db *sql.DB, latencyHistogram prommetrics.HistogramVec, dialOpts ...grpc.DialOption) (*Mgr, error) {
+ strategies := make(map[string]leaderElectionStrategy, len(c.VirtualStorages))
for _, virtualStorage := range c.VirtualStorages {
- strategy := newLocalElector(virtualStorage.Name, c.FailoverEnabled)
- mgr.strategies[virtualStorage.Name] = strategy
-
+ ns := make([]*nodeStatus, 1, len(virtualStorage.Nodes))
for _, node := range virtualStorage.Nodes {
conn, err := client.Dial(node.Address,
append(
[]grpc.DialOption{
grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.NewCodec())),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(node.Token)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(node.Token)),
grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(
grpc_prometheus.StreamClientInterceptor,
grpctracing.StreamClientTracingInterceptor(),
@@ -95,22 +91,40 @@ func NewManager(log *logrus.Entry, c config.Config, latencyHistogram prommetrics
if err != nil {
return nil, err
}
- ns := newConnectionStatus(*node, conn, log, latencyHistogram)
+ cs := newConnectionStatus(*node, conn, log, latencyHistogram)
+ if node.DefaultPrimary {
+ ns[0] = cs
+ } else {
+ ns = append(ns, cs)
+ }
+ }
- strategy.addNode(ns, node.DefaultPrimary)
+ if c.Failover.ElectionStrategy == "sql" {
+ strategies[virtualStorage.Name] = newSQLElector(virtualStorage.Name, c, defaultFailoverTimeoutSeconds, defaultActivePraefectSeconds, db, log, ns)
+ } else {
+ strategies[virtualStorage.Name] = newLocalElector(virtualStorage.Name, c.Failover.Enabled, log, ns)
}
}
- return &mgr, nil
+ return &Mgr{
+ log: log,
+ db: db,
+ failoverEnabled: c.Failover.Enabled,
+ strategies: strategies,
+ }, nil
}
// Start will bootstrap the node manager by calling healthcheck on the nodes as well as kicking off
// the monitoring process. Start must be called before NodeMgr can be used.
func (n *Mgr) Start(bootstrapInterval, monitorInterval time.Duration) {
if n.failoverEnabled {
+ n.log.Info("Starting failover checks")
+
for _, strategy := range n.strategies {
strategy.start(bootstrapInterval, monitorInterval)
}
+ } else {
+ n.log.Info("Failover checks are disabled")
}
}
@@ -200,5 +214,11 @@ func (n *nodeStatus) check(ctx context.Context) (bool, error) {
}).Warn("error when pinging healthcheck")
}
+ var gaugeValue float64
+ if status {
+ gaugeValue = 1
+ }
+ metrics.NodeLastHealthcheckGauge.WithLabelValues(n.GetStorage()).Set(gaugeValue)
+
return status, err
}
diff --git a/internal/praefect/nodes/manager_test.go b/internal/praefect/nodes/manager_test.go
index 185c640d6..a4f95661b 100644
--- a/internal/praefect/nodes/manager_test.go
+++ b/internal/praefect/nodes/manager_test.go
@@ -52,6 +52,52 @@ func TestNodeStatus(t *testing.T) {
require.False(t, status)
}
+func TestPrimaryIsSecond(t *testing.T) {
+ virtualStorages := []*config.VirtualStorage{
+ {
+ Name: "virtual-storage-0",
+ Nodes: []*models.Node{
+ {
+ Storage: "praefect-internal-0",
+ Address: "unix://socket0",
+ DefaultPrimary: false,
+ },
+ {
+ Storage: "praefect-internal-1",
+ Address: "unix://socket1",
+ DefaultPrimary: true,
+ },
+ },
+ },
+ }
+
+ conf := config.Config{
+ VirtualStorages: virtualStorages,
+ Failover: config.Failover{Enabled: false},
+ }
+
+ mockHistogram := promtest.NewMockHistogramVec()
+ nm, err := NewManager(testhelper.DiscardTestEntry(t), conf, nil, mockHistogram)
+ require.NoError(t, err)
+
+ shard, err := nm.GetShard("virtual-storage-0")
+ require.NoError(t, err)
+
+ primary, err := shard.GetPrimary()
+ require.NoError(t, err)
+
+ secondaries, err := shard.GetSecondaries()
+ require.Len(t, secondaries, 1)
+ require.NoError(t, err)
+
+ require.Equal(t, virtualStorages[0].Nodes[1].Storage, primary.GetStorage())
+ require.Equal(t, virtualStorages[0].Nodes[1].Address, primary.GetAddress())
+
+ require.Len(t, secondaries, 1)
+ require.Equal(t, virtualStorages[0].Nodes[0].Storage, secondaries[0].GetStorage())
+ require.Equal(t, virtualStorages[0].Nodes[0].Address, secondaries[0].GetAddress())
+}
+
func TestNodeManager(t *testing.T) {
internalSocket0, internalSocket1 := testhelper.GetTemporaryGitalySocketFileName(), testhelper.GetTemporaryGitalySocketFileName()
srv0, healthSrv0 := testhelper.NewServerWithHealth(t, internalSocket0)
@@ -79,18 +125,18 @@ func TestNodeManager(t *testing.T) {
confWithFailover := config.Config{
VirtualStorages: virtualStorages,
- FailoverEnabled: true,
+ Failover: config.Failover{Enabled: true},
}
confWithoutFailover := config.Config{
VirtualStorages: virtualStorages,
- FailoverEnabled: false,
+ Failover: config.Failover{Enabled: false},
}
mockHistogram := promtest.NewMockHistogramVec()
- nm, err := NewManager(testhelper.DiscardTestEntry(t), confWithFailover, mockHistogram)
+ nm, err := NewManager(testhelper.DiscardTestEntry(t), confWithFailover, nil, mockHistogram)
require.NoError(t, err)
- nmWithoutFailover, err := NewManager(testhelper.DiscardTestEntry(t), confWithoutFailover, mockHistogram)
+ nmWithoutFailover, err := NewManager(testhelper.DiscardTestEntry(t), confWithoutFailover, nil, mockHistogram)
require.NoError(t, err)
nm.Start(1*time.Millisecond, 5*time.Second)
diff --git a/internal/praefect/nodes/sql_elector.go b/internal/praefect/nodes/sql_elector.go
new file mode 100644
index 000000000..babab116c
--- /dev/null
+++ b/internal/praefect/nodes/sql_elector.go
@@ -0,0 +1,456 @@
+package nodes
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "math"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/sirupsen/logrus"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/metrics"
+)
+
+const (
+ defaultFailoverTimeoutSeconds = 10
+ defaultActivePraefectSeconds = 60
+)
+
+type sqlCandidate struct {
+ Node
+}
+
+// sqlElector manages the primary election for one virtual storage (aka
+// shard). It enables multiple, redundant Praefect processes to run,
+// which is needed to eliminate a single point of failure in Gitaly High
+// Avaiability.
+//
+// The sqlElector is responsible for:
+//
+// 1. Monitoring and updating the status of all nodes within the shard.
+// 2. Electing a new primary of the shard based on the health.
+//
+// Every Praefect node periodically (every second) performs a health check RPC with a Gitaly node.
+// 1. For each node, Praefect updates a row in a new table
+// (`node_status`) with the following information:
+//
+// a. The name of the Praefect instance (`praefect_name`)
+// b. The name of the virtual storage name (`shard_name`)
+// c. The name of the Gitaly storage name (`storage_name`)
+// d. The timestamp of the last time Praefect tried to reach that node (`last_contact_attempt_at`)
+// e. The timestamp of the last successful health check (`last_seen_active_at`)
+//
+// 2. Once the health checks are complete, Praefect node does a `SELECT` from
+// `node_status` to determine healthy nodes. A healthy node is
+// defined by:
+// a. A node that has a recent successful error check (e.g. one in
+// the last 10 s).
+// b. A majority of the available Praefect nodes have entries that
+// match the two above.
+//
+// To determine the majority, we use a lightweight service discovery
+// protocol: a Praefect node is deemed a voting member if the
+// `praefect_name` has a recent `last_contact_attempt_at` in the
+// `node_status` table. The name is derived from a combination
+// of the hostname and listening port/socket.
+//
+// The primary of each shard is listed in the
+// `shard_primaries`. If the current primary is in the healthy
+// node list, then sqlElector updates its internal state to match.
+//
+// Otherwise, if there is no primary or it is unhealthy, any Praefect node
+// can elect a new primary by choosing candidate from the healthy node
+// list.
+type sqlElector struct {
+ m sync.RWMutex
+ praefectName string
+ shardName string
+ nodes []*sqlCandidate
+ primaryNode *sqlCandidate
+ db *sql.DB
+ log logrus.FieldLogger
+ failoverSeconds int
+ activePraefectSeconds int
+}
+
+func newSQLElector(name string, c config.Config, failoverTimeoutSeconds int, activePraefectSeconds int, db *sql.DB, log logrus.FieldLogger, ns []*nodeStatus) *sqlElector {
+ praefectName := getPraefectName(c, log)
+
+ log = log.WithField("praefectName", praefectName)
+ log.Info("Using SQL election strategy")
+
+ nodes := make([]*sqlCandidate, len(ns))
+ for i, n := range ns {
+ nodes[i] = &sqlCandidate{Node: n}
+ }
+
+ return &sqlElector{
+ praefectName: praefectName,
+ shardName: name,
+ db: db,
+ log: log,
+ failoverSeconds: failoverTimeoutSeconds,
+ activePraefectSeconds: activePraefectSeconds,
+ nodes: nodes,
+ primaryNode: nodes[0],
+ }
+}
+
+// Generate a Praefect name so that each Praefect process can report
+// node statuses independently. This will enable us to do a SQL
+// election to determine which nodes are active. Ideally this name
+// doesn't change across restarts since that may temporarily make it
+// look like there are more Praefect processes active for
+// determining a quorum.
+func getPraefectName(c config.Config, log logrus.FieldLogger) string {
+ name, err := os.Hostname()
+
+ if err != nil {
+ name = uuid.New().String()
+ log.WithError(err).WithFields(logrus.Fields{
+ "praefectName": name,
+ }).Warn("unable to determine Praefect hostname, using randomly generated UUID")
+ }
+
+ if c.ListenAddr != "" {
+ return fmt.Sprintf("%s:%s", name, c.ListenAddr)
+ }
+
+ return fmt.Sprintf("%s:%s", name, c.SocketPath)
+}
+
+// start launches a Goroutine to check the state of the nodes and
+// continuously monitor their health via gRPC health checks.
+func (s *sqlElector) start(bootstrapInterval, monitorInterval time.Duration) {
+ s.bootstrap(bootstrapInterval)
+ go s.monitor(monitorInterval)
+}
+
+func (s *sqlElector) bootstrap(d time.Duration) {
+ ctx := context.Background()
+ s.checkNodes(ctx)
+}
+
+func (s *sqlElector) monitor(d time.Duration) {
+ ticker := time.NewTicker(d)
+ defer ticker.Stop()
+
+ ctx := context.Background()
+
+ for {
+ <-ticker.C
+ s.checkNodes(ctx)
+ }
+}
+
+func (s *sqlElector) checkNodes(ctx context.Context) error {
+ var wg sync.WaitGroup
+
+ defer s.updateMetrics()
+
+ for _, n := range s.nodes {
+ wg.Add(1)
+
+ go func(n Node) {
+ defer wg.Done()
+ result, _ := n.check(ctx)
+ if err := s.updateNode(n, result); err != nil {
+ s.log.WithError(err).WithFields(logrus.Fields{
+ "shard": s.shardName,
+ "storage": n.GetStorage(),
+ "address": n.GetAddress(),
+ }).Error("error checking node")
+ }
+ }(n)
+ }
+
+ wg.Wait()
+
+ err := s.validateAndUpdatePrimary()
+
+ if err != nil {
+ s.log.WithError(err).Error("unable to validate primary")
+ return err
+ }
+
+ // The attempt to elect a primary may have conflicted with another
+ // node attempting to elect a primary. We check the database again
+ // to see the current state.
+ candidate, err := s.lookupPrimary()
+
+ if err != nil {
+ s.log.WithError(err).Error("error looking up primary")
+ return err
+ }
+
+ s.setPrimary(candidate)
+ return nil
+}
+
+func (s *sqlElector) setPrimary(candidate *sqlCandidate) {
+ s.m.Lock()
+ defer s.m.Unlock()
+
+ if candidate != s.primaryNode {
+ var oldPrimary string
+ var newPrimary string
+
+ if s.primaryNode != nil {
+ oldPrimary = s.primaryNode.GetStorage()
+ }
+
+ if candidate != nil {
+ newPrimary = candidate.GetStorage()
+ }
+
+ s.log.WithFields(logrus.Fields{
+ "oldPrimary": oldPrimary,
+ "newPrimary": newPrimary,
+ "shard": s.shardName}).Info("primary node changed")
+
+ s.primaryNode = candidate
+ }
+}
+
+func (s *sqlElector) updateNode(node Node, result bool) error {
+ var q string
+
+ if result {
+ q = `INSERT INTO node_status (praefect_name, shard_name, node_name, last_contact_attempt_at, last_seen_active_at)
+VALUES ($1, $2, $3, NOW(), NOW())
+ON CONFLICT (praefect_name, shard_name, node_name)
+DO UPDATE SET
+last_contact_attempt_at = NOW(),
+last_seen_active_at = NOW()`
+ } else {
+ // Omit the last_seen_active_at since we weren't successful at contacting this node
+ q = `INSERT INTO node_status (praefect_name, shard_name, node_name, last_contact_attempt_at)
+VALUES ($1, $2, $3, NOW())
+ON CONFLICT (praefect_name, shard_name, node_name)
+DO UPDATE SET
+last_contact_attempt_at = NOW()`
+ }
+
+ _, err := s.db.Exec(q, s.praefectName, s.shardName, node.GetStorage())
+
+ if err != nil {
+ s.log.Errorf("Error updating node: %s", err)
+ }
+
+ return err
+}
+
+// GetPrimary gets the primary of a shard by checking the state of the
+// database and updating the internal state. If no primary exists, it
+// will be nil. If a primary has been elected but is down, err will be
+// ErrPrimaryNotHealthy.
+func (s *sqlElector) GetPrimary() (Node, error) {
+ primary, err := s.lookupPrimary()
+
+ if err != nil {
+ return nil, err
+ }
+
+ // Update the internal state so that calls to GetSecondaries() will be
+ // consistent with GetPrimary()
+ s.setPrimary(primary)
+
+ if primary == nil {
+ return nil, ErrPrimaryNotHealthy
+ }
+
+ return primary, nil
+}
+
+// GetSecondaries gets the secondaries of a shard. It uses the internal
+// state to determine the primary so that calls to GetSecondaries() will
+// be consistent with the first call to GetPrimary().
+func (s *sqlElector) GetSecondaries() ([]Node, error) {
+ s.m.RLock()
+ primaryNode := s.primaryNode
+ s.m.RUnlock()
+
+ var secondaries []Node
+ for _, n := range s.nodes {
+ if primaryNode != n {
+ secondaries = append(secondaries, n)
+ }
+ }
+
+ return secondaries, nil
+}
+
+func (s *sqlElector) updateMetrics() {
+ s.m.RLock()
+ primary := s.primaryNode
+ s.m.RUnlock()
+
+ for _, node := range s.nodes {
+ var val float64
+
+ if primary == node {
+ val = 1
+ }
+
+ metrics.PrimaryGauge.WithLabelValues(s.shardName, node.GetStorage()).Set(val)
+ }
+}
+
+func (s *sqlElector) getQuorumCount() (int, error) {
+ // This is crude form of service discovery. Find how many active
+ // Praefect nodes based on whether they attempted to update entries.
+ q := `SELECT COUNT (DISTINCT praefect_name) FROM node_status WHERE shard_name = $1 AND last_contact_attempt_at >= NOW() - $2::INTERVAL SECOND`
+
+ var totalCount int
+
+ if err := s.db.QueryRow(q, s.shardName, s.activePraefectSeconds).Scan(&totalCount); err != nil {
+ return 0, fmt.Errorf("error retrieving quorum count: %v", err)
+ }
+
+ if totalCount <= 1 {
+ return 1, nil
+ }
+
+ quorumCount := int(math.Ceil(float64(totalCount) / 2))
+
+ return quorumCount, nil
+}
+
+func (s *sqlElector) lookupNodeByName(name string) *sqlCandidate {
+ for _, n := range s.nodes {
+ if n.GetStorage() == name {
+ return n
+ }
+ }
+
+ return nil
+}
+
+func nodeInSlice(candidates []*sqlCandidate, node *sqlCandidate) bool {
+ for _, n := range candidates {
+ if n == node {
+ return true
+ }
+ }
+
+ return false
+}
+
+func (s *sqlElector) demotePrimary() error {
+ s.setPrimary(nil)
+
+ q := "DELETE FROM shard_primaries WHERE shard_name = $1"
+ _, err := s.db.Exec(q, s.shardName)
+
+ return err
+}
+
+func (s *sqlElector) electNewPrimary(candidates []*sqlCandidate) error {
+ // Arbitrarily pick the first candidate from the list. Note that the
+ // candidate list has already been ranked in order of highest number
+ // of Praefect nodes that have reached it and the name of the
+ // Praefect identifier.
+ newPrimary := candidates[0]
+
+ q := `INSERT INTO shard_primaries (elected_by_praefect, shard_name, node_name, elected_at)
+ SELECT $1::VARCHAR, $2::VARCHAR, $3::VARCHAR, NOW()
+ WHERE $3 != COALESCE((SELECT node_name FROM shard_primaries WHERE shard_name = $2::VARCHAR), '')
+ ON CONFLICT (shard_name)
+ DO UPDATE SET elected_by_praefect = EXCLUDED.elected_by_praefect
+ , node_name = EXCLUDED.node_name
+ , elected_at = EXCLUDED.elected_at
+ `
+ _, err := s.db.Exec(q, s.praefectName, s.shardName, newPrimary.GetStorage())
+
+ if err != nil {
+ s.log.Errorf("error updating new primary: %s", err)
+ }
+
+ return nil
+}
+
+func (s *sqlElector) validateAndUpdatePrimary() error {
+ quorumCount, err := s.getQuorumCount()
+
+ if err != nil {
+ return err
+ }
+
+ // Retrieves candidates, ranked by the ones that are the most active
+ q := `SELECT node_name FROM node_status
+ WHERE shard_name = $1 AND last_seen_active_at >= NOW() - $2::INTERVAL SECOND
+ GROUP BY node_name
+ HAVING COUNT(praefect_name) >= $3
+ ORDER BY COUNT(node_name) DESC, node_name ASC`
+
+ rows, err := s.db.Query(q, s.shardName, s.failoverSeconds, quorumCount)
+
+ if err != nil {
+ return fmt.Errorf("error retrieving candidates: %v", err)
+ }
+ defer rows.Close()
+
+ var candidates []*sqlCandidate
+
+ for rows.Next() {
+ var name string
+ if err := rows.Scan(&name); err != nil {
+ return fmt.Errorf("error retrieving candidate rows: %v", err)
+ }
+
+ node := s.lookupNodeByName(name)
+
+ if node != nil {
+ candidates = append(candidates, node)
+ } else {
+ s.log.Errorf("unknown candidate node name found: %s", name)
+ }
+ }
+
+ if err = rows.Err(); err != nil {
+ return err
+ }
+
+ // Check if primary is in this list
+ primaryNode, err := s.lookupPrimary()
+
+ if err != nil {
+ s.log.WithError(err).Error("error looking up primary")
+ return err
+ }
+
+ if len(candidates) == 0 {
+ return s.demotePrimary()
+ }
+
+ if primaryNode == nil || !nodeInSlice(candidates, primaryNode) {
+ return s.electNewPrimary(candidates)
+ }
+
+ return nil
+}
+
+func (s *sqlElector) lookupPrimary() (*sqlCandidate, error) {
+ var primaryName string
+
+ q := `SELECT node_name FROM shard_primaries WHERE shard_name = $1`
+
+ if err := s.db.QueryRow(q, s.shardName).Scan(&primaryName); err != nil {
+ if err == sql.ErrNoRows {
+ return nil, nil
+ }
+
+ return nil, fmt.Errorf("error looking up primary: %v", err)
+ }
+
+ var primaryNode *sqlCandidate
+ if primaryName != "" {
+ primaryNode = s.lookupNodeByName(primaryName)
+ }
+
+ return primaryNode, nil
+}
diff --git a/internal/praefect/nodes/sql_elector_test.go b/internal/praefect/nodes/sql_elector_test.go
new file mode 100644
index 000000000..faf3d99a3
--- /dev/null
+++ b/internal/praefect/nodes/sql_elector_test.go
@@ -0,0 +1,163 @@
+// +build postgres
+
+package nodes
+
+import (
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/models"
+ "gitlab.com/gitlab-org/gitaly/internal/testhelper"
+ "gitlab.com/gitlab-org/gitaly/internal/testhelper/promtest"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+var shardName string = "test-shard-0"
+
+func TestGetPrimaryAndSecondaries(t *testing.T) {
+ db := getDB(t)
+
+ logger := testhelper.NewTestLogger(t).WithField("test", t.Name())
+ praefectSocket := testhelper.GetTemporaryGitalySocketFileName()
+ socketName := "unix://" + praefectSocket
+
+ conf := config.Config{
+ SocketPath: socketName,
+ }
+
+ internalSocket0 := testhelper.GetTemporaryGitalySocketFileName()
+ srv0, _ := testhelper.NewServerWithHealth(t, internalSocket0)
+ defer srv0.Stop()
+
+ cc0, err := grpc.Dial(
+ "unix://"+internalSocket0,
+ grpc.WithInsecure(),
+ )
+ require.NoError(t, err)
+
+ storageName := "default"
+ mockHistogramVec0 := promtest.NewMockHistogramVec()
+ cs0 := newConnectionStatus(models.Node{Storage: storageName + "-0"}, cc0, testhelper.DiscardTestEntry(t), mockHistogramVec0)
+
+ ns := []*nodeStatus{cs0}
+ elector := newSQLElector(shardName, conf, 1, defaultActivePraefectSeconds, db.DB, logger, ns)
+ require.Contains(t, elector.praefectName, ":"+socketName)
+ require.Equal(t, elector.shardName, shardName)
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+ err = elector.checkNodes(ctx)
+ db.RequireRowsInTable(t, "shard_primaries", 1)
+
+ elector.demotePrimary()
+ db.RequireRowsInTable(t, "shard_primaries", 0)
+
+ // Ensure the primary state is reflected immediately
+ primary, err := elector.GetPrimary()
+ require.Error(t, err)
+ require.Equal(t, nil, primary)
+
+ secondaries, err := elector.GetSecondaries()
+ require.NoError(t, err)
+ require.Equal(t, 1, len(secondaries))
+}
+
+func TestBasicFailover(t *testing.T) {
+ db := getDB(t)
+
+ logger := testhelper.NewTestLogger(t).WithField("test", t.Name())
+ praefectSocket := testhelper.GetTemporaryGitalySocketFileName()
+ socketName := "unix://" + praefectSocket
+
+ conf := config.Config{
+ SocketPath: socketName,
+ }
+
+ internalSocket0, internalSocket1 := testhelper.GetTemporaryGitalySocketFileName(), testhelper.GetTemporaryGitalySocketFileName()
+ srv0, healthSrv0 := testhelper.NewServerWithHealth(t, internalSocket0)
+ defer srv0.Stop()
+
+ srv1, healthSrv1 := testhelper.NewServerWithHealth(t, internalSocket1)
+ defer srv1.Stop()
+
+ cc0, err := grpc.Dial(
+ "unix://"+internalSocket0,
+ grpc.WithInsecure(),
+ )
+ require.NoError(t, err)
+
+ cc1, err := grpc.Dial(
+ "unix://"+internalSocket1,
+ grpc.WithInsecure(),
+ )
+
+ require.NoError(t, err)
+
+ storageName := "default"
+ mockHistogramVec0, mockHistogramVec1 := promtest.NewMockHistogramVec(), promtest.NewMockHistogramVec()
+ cs0 := newConnectionStatus(models.Node{Storage: storageName + "-0"}, cc0, testhelper.DiscardTestEntry(t), mockHistogramVec0)
+ cs1 := newConnectionStatus(models.Node{Storage: storageName + "-1"}, cc1, testhelper.DiscardTestEntry(t), mockHistogramVec1)
+
+ ns := []*nodeStatus{cs0, cs1}
+ elector := newSQLElector(shardName, conf, 1, defaultActivePraefectSeconds, db.DB, logger, ns)
+
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+ err = elector.checkNodes(ctx)
+
+ require.NoError(t, err)
+ db.RequireRowsInTable(t, "node_status", 2)
+ db.RequireRowsInTable(t, "shard_primaries", 1)
+
+ require.Equal(t, cs0, elector.primaryNode.Node)
+ primary, err := elector.GetPrimary()
+ require.NoError(t, err)
+ require.Equal(t, cs0.GetStorage(), primary.GetStorage())
+
+ secondaries, err := elector.GetSecondaries()
+ require.NoError(t, err)
+ require.Equal(t, 1, len(secondaries))
+ require.Equal(t, cs1.GetStorage(), secondaries[0].GetStorage())
+
+ // Bring first node down
+ healthSrv0.SetServingStatus("", grpc_health_v1.HealthCheckResponse_UNKNOWN)
+
+ // Primary should remain even after the first check
+ err = elector.checkNodes(ctx)
+ require.NoError(t, err)
+ primary, err = elector.GetPrimary()
+ require.NoError(t, err)
+
+ // Wait for stale timeout to expire
+ time.Sleep(1 * time.Second)
+
+ // Expect that the other node is promoted
+ err = elector.checkNodes(ctx)
+ require.NoError(t, err)
+
+ db.RequireRowsInTable(t, "node_status", 2)
+ db.RequireRowsInTable(t, "shard_primaries", 1)
+ primary, err = elector.GetPrimary()
+ require.NoError(t, err)
+ require.Equal(t, cs1.GetStorage(), primary.GetStorage())
+
+ // Bring second node down
+ healthSrv1.SetServingStatus("", grpc_health_v1.HealthCheckResponse_UNKNOWN)
+
+ // Wait for stale timeout to expire
+ time.Sleep(1 * time.Second)
+ err = elector.checkNodes(ctx)
+ require.NoError(t, err)
+
+ db.RequireRowsInTable(t, "node_status", 2)
+ // No new candidates
+ db.RequireRowsInTable(t, "shard_primaries", 0)
+ primary, err = elector.GetPrimary()
+ require.Error(t, ErrPrimaryNotHealthy, err)
+ secondaries, err = elector.GetSecondaries()
+ require.NoError(t, err)
+ require.Equal(t, 2, len(secondaries))
+}
diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go
index 3b45dad7b..d47de04ac 100644
--- a/internal/praefect/replicator_test.go
+++ b/internal/praefect/replicator_test.go
@@ -148,7 +148,7 @@ func TestProcessReplicationJob(t *testing.T) {
entry := testhelper.DiscardTestEntry(t)
replicator.log = entry
- nodeMgr, err := nodes.NewManager(entry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(entry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
nodeMgr.Start(1*time.Millisecond, 5*time.Millisecond)
@@ -225,7 +225,7 @@ func TestPropagateReplicationJob(t *testing.T) {
}
logEntry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(logEntry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(logEntry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
nodeMgr.Start(1*time.Millisecond, 5*time.Millisecond)
@@ -393,7 +393,7 @@ func TestConfirmReplication(t *testing.T) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(testhelper.RepositoryAuthToken)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(testhelper.RepositoryAuthToken)),
}
conn, err := grpc.Dial(srvSocketPath, connOpts...)
require.NoError(t, err)
@@ -526,7 +526,7 @@ func TestProcessBacklog_FailedJobs(t *testing.T) {
logEntry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(logEntry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(logEntry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
replMgr := NewReplMgr("default", logEntry, ds, nodeMgr)
@@ -664,7 +664,7 @@ func TestProcessBacklog_Success(t *testing.T) {
logEntry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(logEntry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(logEntry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
replMgr := NewReplMgr(conf.VirtualStorages[0].Name, logEntry, ds, nodeMgr)
@@ -706,7 +706,7 @@ func TestBackoff(t *testing.T) {
}
func runFullGitalyServer(t *testing.T) (*grpc.Server, string) {
- server := serverPkg.NewInsecure(RubyServer)
+ server := serverPkg.NewInsecure(RubyServer, gitaly_config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
@@ -754,7 +754,7 @@ func newReplicationService(tb testing.TB) (*grpc.Server, string) {
func newRepositoryClient(t *testing.T, serverSocketPath string) (gitalypb.RepositoryServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(testhelper.RepositoryAuthToken)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(testhelper.RepositoryAuthToken)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
@@ -767,7 +767,6 @@ func newRepositoryClient(t *testing.T, serverSocketPath string) (gitalypb.Reposi
var RubyServer = &rubyserver.Server{}
func TestMain(m *testing.M) {
- testhelper.ConfigureGitalySSH()
testhelper.Configure()
os.Exit(testMain(m))
}
@@ -783,6 +782,8 @@ func testMain(m *testing.M) int {
log.Fatal(err)
}
+ testhelper.ConfigureGitalySSH()
+
if err := RubyServer.Start(); err != nil {
log.Fatal(err)
}
diff --git a/internal/praefect/server_test.go b/internal/praefect/server_test.go
index ade82de87..865ec286a 100644
--- a/internal/praefect/server_test.go
+++ b/internal/praefect/server_test.go
@@ -51,9 +51,11 @@ func TestServerRouteServerAccessor(t *testing.T) {
}
)
- cli, _, cleanup := runPraefectServerWithMock(t, conf, backends)
+ cc, _, cleanup := runPraefectServerWithMock(t, conf, datastore.Datastore{}, backends)
defer cleanup()
+ cli := mock.NewSimpleServiceClient(cc)
+
expectReq := &mock.SimpleRequest{Value: 1}
done := make(chan struct{})
@@ -105,7 +107,8 @@ func TestGitalyServerInfo(t *testing.T) {
metadata, err := client.ServerInfo(ctx, &gitalypb.ServerInfoRequest{})
require.NoError(t, err)
- require.Len(t, metadata.GetStorageStatuses(), len(conf.VirtualStorages[0].Nodes))
+ require.Len(t, metadata.GetStorageStatuses(), len(conf.VirtualStorages))
+ require.Equal(t, conf.VirtualStorages[0].Name, metadata.GetStorageStatuses()[0].StorageName)
require.Equal(t, version.GetVersion(), metadata.GetServerVersion())
gitVersion, err := git.Version()
@@ -134,13 +137,13 @@ func TestGitalyServerInfoBadNode(t *testing.T) {
}
entry := testhelper.DiscardTestEntry(t)
- nodeMgr, err := nodes.NewManager(entry, conf, promtest.NewMockHistogramVec())
+ nodeMgr, err := nodes.NewManager(entry, conf, nil, promtest.NewMockHistogramVec())
require.NoError(t, err)
registry := protoregistry.New()
require.NoError(t, registry.RegisterFiles(protoregistry.GitalyProtoFileDescriptors...))
- srv := setupServer(t, conf, nodeMgr, entry, registry)
+ srv := setupServer(t, conf, nodeMgr, datastore.Datastore{}, entry, registry)
listener, port := listenAvailPort(t)
go func() {
@@ -275,7 +278,7 @@ func TestWarnDuplicateAddrs(t *testing.T) {
tLogger, hook := test.NewNullLogger()
- setupServer(t, conf, nil, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
+ setupServer(t, conf, nil, datastore.Datastore{}, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
for _, entry := range hook.Entries {
require.NotContains(t, entry.Message, "more than one backend node")
@@ -302,7 +305,7 @@ func TestWarnDuplicateAddrs(t *testing.T) {
tLogger, hook = test.NewNullLogger()
- setupServer(t, conf, nil, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
+ setupServer(t, conf, nil, datastore.Datastore{}, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
var found bool
for _, entry := range hook.Entries {
@@ -348,7 +351,7 @@ func TestWarnDuplicateAddrs(t *testing.T) {
tLogger, hook = test.NewNullLogger()
- setupServer(t, conf, nil, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
+ setupServer(t, conf, nil, datastore.Datastore{}, logrus.NewEntry(tLogger), nil) // instantiates a praefect server and triggers warning
for _, entry := range hook.Entries {
require.NotContains(t, entry.Message, "more than one backend node")
@@ -559,13 +562,17 @@ func TestRepoRename(t *testing.T) {
require.NoError(t, err)
require.True(t, resp.GetExists(), "repo with new name must exist")
require.DirExists(t, expNewPath0, "must be renamed on secondary from %q to %q", path0, expNewPath0)
+ defer func() { require.NoError(t, os.RemoveAll(expNewPath0)) }()
// the renaming of the repo on the secondary servers is not deterministic
// since it relies on eventually consistent replication
pollUntilRemoved(t, path1, time.After(10*time.Second))
require.DirExists(t, expNewPath1, "must be renamed on secondary from %q to %q", path1, expNewPath1)
+ defer func() { require.NoError(t, os.RemoveAll(expNewPath1)) }()
+
pollUntilRemoved(t, path2, time.After(10*time.Second))
require.DirExists(t, expNewPath2, "must be renamed on secondary from %q to %q", path2, expNewPath2)
+ defer func() { require.NoError(t, os.RemoveAll(expNewPath2)) }()
}
func tempStoragePath(t testing.TB) string {
diff --git a/internal/praefect/service/info/dataloss.go b/internal/praefect/service/info/dataloss.go
new file mode 100644
index 000000000..9e863d2b2
--- /dev/null
+++ b/internal/praefect/service/info/dataloss.go
@@ -0,0 +1,28 @@
+package info
+
+import (
+ "context"
+
+ "github.com/golang/protobuf/ptypes"
+ "gitlab.com/gitlab-org/gitaly/internal/helper"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
+
+func (s *Server) DatalossCheck(ctx context.Context, req *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error) {
+ from, err := ptypes.Timestamp(req.GetFrom())
+ if err != nil {
+ return nil, helper.ErrInvalidArgumentf("invalid 'from': %v", err)
+ }
+
+ to, err := ptypes.Timestamp(req.GetTo())
+ if err != nil {
+ return nil, helper.ErrInvalidArgumentf("invalid 'to': %v", err)
+ }
+
+ dead, err := s.queue.CountDeadReplicationJobs(ctx, from, to)
+ if err != nil {
+ return nil, err
+ }
+
+ return &gitalypb.DatalossCheckResponse{ByRelativePath: dead}, nil
+}
diff --git a/internal/praefect/service/info/server.go b/internal/praefect/service/info/server.go
index 46702d80a..bbd728152 100644
--- a/internal/praefect/service/info/server.go
+++ b/internal/praefect/service/info/server.go
@@ -2,6 +2,7 @@ package info
import (
"context"
+ "time"
"gitlab.com/gitlab-org/gitaly/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/internal/praefect/datastore"
@@ -12,6 +13,7 @@ import (
// Queue is a subset of the datastore.ReplicationEventQueue functionality needed by this service
type Queue interface {
Enqueue(ctx context.Context, event datastore.ReplicationEvent) (datastore.ReplicationEvent, error)
+ CountDeadReplicationJobs(ctx context.Context, from, to time.Time) (map[string]int64, error)
}
// compile time assertion that Queue is satisfied by
diff --git a/internal/praefect/service/server/info.go b/internal/praefect/service/server/info.go
index 64360f171..27a7f6c2b 100644
--- a/internal/praefect/service/server/info.go
+++ b/internal/praefect/service/server/info.go
@@ -4,9 +4,8 @@ import (
"context"
"sync"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/helper"
- "gitlab.com/gitlab-org/gitaly/internal/praefect/nodes"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"golang.org/x/sync/errgroup"
)
@@ -16,8 +15,13 @@ import (
func (s *Server) ServerInfo(ctx context.Context, in *gitalypb.ServerInfoRequest) (*gitalypb.ServerInfoResponse, error) {
var once sync.Once
- var nodes []nodes.Node
- for _, virtualStorage := range s.conf.VirtualStorages {
+ var gitVersion, serverVersion string
+
+ g, ctx := errgroup.WithContext(ctx)
+
+ storageStatuses := make([]*gitalypb.ServerInfoResponse_StorageStatus, len(s.conf.VirtualStorages))
+
+ for i, virtualStorage := range s.conf.VirtualStorages {
shard, err := s.nodeMgr.GetShard(virtualStorage.Name)
if err != nil {
return nil, err
@@ -28,32 +32,48 @@ func (s *Server) ServerInfo(ctx context.Context, in *gitalypb.ServerInfoRequest)
return nil, err
}
- secondaries, err := shard.GetSecondaries()
- if err != nil {
- return nil, err
- }
-
- nodes = append(append(nodes, primary), secondaries...)
- }
- var gitVersion, serverVersion string
-
- g, ctx := errgroup.WithContext(ctx)
-
- storageStatuses := make([][]*gitalypb.ServerInfoResponse_StorageStatus, len(nodes))
-
- for i, node := range nodes {
i := i
- node := node
+ virtualStorage := virtualStorage
g.Go(func() error {
- client := gitalypb.NewServerServiceClient(node.GetConnection())
+ client := gitalypb.NewServerServiceClient(primary.GetConnection())
resp, err := client.ServerInfo(ctx, &gitalypb.ServerInfoRequest{})
if err != nil {
- grpc_logrus.Extract(ctx).WithField("storage", node.GetStorage()).WithError(err).Error("error getting sever info")
+ ctxlogrus.Extract(ctx).WithField("storage", primary.GetStorage()).WithError(err).Error("error getting server info")
return nil
}
- storageStatuses[i] = resp.GetStorageStatuses()
+ // From the perspective of the praefect client, a server info call should result in the server infos
+ // of virtual storages. Each virtual storage has one or more nodes, but only the primary node's server info
+ // needs to be returned. It's a common pattern in gitaly configs for all gitaly nodes in a fleet to use the same config.toml
+ // whereby there are many storage names but only one of them is actually used by any given gitaly node:
+ //
+ // below is the config.toml for all three internal gitaly nodes
+ // [[storage]]
+ // name = "internal-gitaly-0"
+ // path = "/var/opt/gitlab/git-data"
+ //
+ // [storage]]
+ // name = "internal-gitaly-1"
+ // path = "/var/opt/gitlab/git-data"
+ //
+ // [[storage]]
+ // name = "internal-gitaly-2"
+ // path = "/var/opt/gitlab/git-data"
+ //
+ // technically, any storage's storage status can be returned in the virtual storage's server info,
+ // but to be consistent we will choose the storage with the same name as the internal gitaly storage name.
+ for _, storageStatus := range resp.GetStorageStatuses() {
+ if storageStatus.StorageName == primary.GetStorage() {
+ storageStatuses[i] = storageStatus
+ // the storage name in the response needs to be rewritten to be the virtual storage name
+ // because the praefect client has no concept of internal gitaly nodes that are behind praefect.
+ // From the perspective of the praefect client, the primary internal gitaly node's storage status is equivalent
+ // to the virtual storage's storage status.
+ storageStatuses[i].StorageName = virtualStorage.Name
+ break
+ }
+ }
once.Do(func() {
gitVersion, serverVersion = resp.GetGitVersion(), resp.GetServerVersion()
@@ -67,13 +87,21 @@ func (s *Server) ServerInfo(ctx context.Context, in *gitalypb.ServerInfoRequest)
return nil, helper.ErrInternal(err)
}
- var response gitalypb.ServerInfoResponse
+ return &gitalypb.ServerInfoResponse{
+ ServerVersion: serverVersion,
+ GitVersion: gitVersion,
+ StorageStatuses: filterEmptyStorageStatuses(storageStatuses),
+ }, nil
+}
+
+func filterEmptyStorageStatuses(storageStatuses []*gitalypb.ServerInfoResponse_StorageStatus) []*gitalypb.ServerInfoResponse_StorageStatus {
+ var n int
for _, storageStatus := range storageStatuses {
- response.StorageStatuses = append(response.StorageStatuses, storageStatus...)
+ if storageStatus != nil {
+ storageStatuses[n] = storageStatus
+ n++
+ }
}
-
- response.GitVersion, response.ServerVersion = gitVersion, serverVersion
-
- return &response, nil
+ return storageStatuses[:n]
}
diff --git a/internal/rubyserver/proxy.go b/internal/rubyserver/proxy.go
index 05a35bac8..87052c900 100644
--- a/internal/rubyserver/proxy.go
+++ b/internal/rubyserver/proxy.go
@@ -11,10 +11,6 @@ import (
"google.golang.org/grpc/metadata"
)
-// ProxyHeaderWhitelist is the list of http/2 headers that will be
-// forwarded as-is to gitaly-ruby.
-var ProxyHeaderWhitelist = []string{"gitaly-servers"}
-
// Headers prefixed with this string get whitelisted automatically
const rubyFeaturePrefix = "gitaly-feature-ruby-"
@@ -65,17 +61,18 @@ func setHeaders(ctx context.Context, repo *gitalypb.Repository, mustExist bool)
repoAltDirsHeader, repoAltDirsCombined,
)
+ // list of http/2 headers that will be forwarded as-is to gitaly-ruby
+ proxyHeaderWhitelist := []string{"gitaly-servers"}
+
if inMD, ok := metadata.FromIncomingContext(ctx); ok {
// Automatically whitelist any Ruby-specific feature flag
for header := range inMD {
if strings.HasPrefix(header, rubyFeaturePrefix) {
- // TODO: this changes state of the global variable without any synchronization
- // https://gitlab.com/gitlab-org/gitaly/-/issues/2614
- ProxyHeaderWhitelist = append(ProxyHeaderWhitelist, header)
+ proxyHeaderWhitelist = append(proxyHeaderWhitelist, header)
}
}
- for _, header := range ProxyHeaderWhitelist {
+ for _, header := range proxyHeaderWhitelist {
for _, v := range inMD[header] {
md = metadata.Join(md, metadata.Pairs(header, v))
}
diff --git a/internal/rubyserver/proxy_test.go b/internal/rubyserver/proxy_test.go
index 03936f41c..7233a57d9 100644
--- a/internal/rubyserver/proxy_test.go
+++ b/internal/rubyserver/proxy_test.go
@@ -31,7 +31,6 @@ func TestSetHeadersPreservesWhitelistedMetadata(t *testing.T) {
defer cancel()
key := "gitaly-servers"
- require.Contains(t, ProxyHeaderWhitelist, key, "sanity check")
value := "test-value"
inCtx := metadata.NewIncomingContext(ctx, metadata.Pairs(key, value))
diff --git a/internal/rubyserver/rubyserver.go b/internal/rubyserver/rubyserver.go
index d3e8863f8..48c63bb84 100644
--- a/internal/rubyserver/rubyserver.go
+++ b/internal/rubyserver/rubyserver.go
@@ -96,6 +96,8 @@ func (s *Server) start() error {
"GITALY_RUBY_DIR="+cfg.Ruby.Dir,
"GITALY_VERSION="+version.GetVersion(),
"GITALY_GIT_HOOKS_DIR="+hooks.Path(),
+ "GITALY_SOCKET="+config.GitalyInternalSocketPath(),
+ "GITALY_TOKEN="+cfg.Auth.Token,
"GITALY_RUGGED_GIT_CONFIG_SEARCH_PATH="+cfg.Ruby.RuggedGitConfigSearchPath)
env = append(env, gitlabshell.Env()...)
diff --git a/internal/safe/file_writer_test.go b/internal/safe/file_writer_test.go
index 08b498f0b..308e3eb7a 100644
--- a/internal/safe/file_writer_test.go
+++ b/internal/safe/file_writer_test.go
@@ -15,7 +15,7 @@ import (
)
func TestFile(t *testing.T) {
- dir, cleanup := testhelper.TempDir(t, t.Name())
+ dir, cleanup := testhelper.TempDir(t)
defer cleanup()
filePath := filepath.Join(dir, "test_file_contents")
@@ -41,7 +41,7 @@ func TestFile(t *testing.T) {
}
func TestFileRace(t *testing.T) {
- dir, cleanup := testhelper.TempDir(t, t.Name())
+ dir, cleanup := testhelper.TempDir(t)
defer cleanup()
filePath := filepath.Join(dir, "test_file_contents")
@@ -68,7 +68,7 @@ func TestFileRace(t *testing.T) {
}
func TestFileCloseBeforeCommit(t *testing.T) {
- dir, cleanup := testhelper.TempDir(t, t.Name())
+ dir, cleanup := testhelper.TempDir(t)
defer cleanup()
dstPath := filepath.Join(dir, "safety_meow")
@@ -87,7 +87,7 @@ func TestFileCloseBeforeCommit(t *testing.T) {
}
func TestFileCommitBeforeClose(t *testing.T) {
- dir, cleanup := testhelper.TempDir(t, t.Name())
+ dir, cleanup := testhelper.TempDir(t)
defer cleanup()
dstPath := filepath.Join(dir, "safety_meow")
diff --git a/internal/server/auth_test.go b/internal/server/auth_test.go
index 9366d2ada..8b43a78e9 100644
--- a/internal/server/auth_test.go
+++ b/internal/server/auth_test.go
@@ -79,11 +79,6 @@ func TestAuthFailures(t *testing.T) {
},
{
desc: "wrong secret",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials("foobar"))},
- code: codes.PermissionDenied,
- },
- {
- desc: "wrong secret new auth",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2("foobar"))},
code: codes.PermissionDenied,
},
@@ -117,23 +112,7 @@ func TestAuthSuccess(t *testing.T) {
}{
{desc: "no auth, not required"},
{
- desc: "v1 incorrect auth, not required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials("incorrect"))},
- token: token,
- },
- {
- desc: "v1 correct auth, not required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token))},
- token: token,
- },
- {
- desc: "v1 correct auth, required",
- opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token))},
- token: token,
- required: true,
- },
- {
- desc: "v2 correct new auth, not required",
+ desc: "v2 correct auth, not required",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token))},
token: token,
},
@@ -143,7 +122,7 @@ func TestAuthSuccess(t *testing.T) {
token: token,
},
{
- desc: "v2 correct new auth, required",
+ desc: "v2 correct auth, required",
opts: []grpc.DialOption{grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token))},
token: token,
required: true,
@@ -187,7 +166,7 @@ func healthCheck(conn *grpc.ClientConn) error {
}
func runServer(t *testing.T) (*grpc.Server, string) {
- srv := NewInsecure(nil)
+ srv := NewInsecure(nil, config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
@@ -204,7 +183,7 @@ func runSecureServer(t *testing.T) (*grpc.Server, string) {
KeyPath: "testdata/gitalykey.pem",
}
- srv := NewSecure(nil)
+ srv := NewSecure(nil, config.Config)
listener, err := net.Listen("tcp", "localhost:9999")
require.NoError(t, err)
diff --git a/internal/server/server.go b/internal/server/server.go
index e4f4896b4..2240a4442 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -69,7 +69,7 @@ func init() {
// createNewServer returns a GRPC server with all Gitaly services and interceptors set up.
// allows for specifying secure = true to enable tls credentials
-func createNewServer(rubyServer *rubyserver.Server, secure bool) *grpc.Server {
+func createNewServer(rubyServer *rubyserver.Server, cfg config.Cfg, secure bool) *grpc.Server {
ctxTagOpts := []grpc_ctxtags.Option{
grpc_ctxtags.WithFieldExtractorForInitialReq(fieldextractors.FieldExtractor),
}
@@ -123,7 +123,7 @@ func createNewServer(rubyServer *rubyserver.Server, secure bool) *grpc.Server {
server := grpc.NewServer(opts...)
- service.RegisterAll(server, rubyServer)
+ service.RegisterAll(server, cfg, rubyServer)
reflection.Register(server)
grpc_prometheus.Register(server)
@@ -132,13 +132,13 @@ func createNewServer(rubyServer *rubyserver.Server, secure bool) *grpc.Server {
}
// NewInsecure returns a GRPC server with all Gitaly services and interceptors set up.
-func NewInsecure(rubyServer *rubyserver.Server) *grpc.Server {
- return createNewServer(rubyServer, false)
+func NewInsecure(rubyServer *rubyserver.Server, cfg config.Cfg) *grpc.Server {
+ return createNewServer(rubyServer, cfg, false)
}
// NewSecure returns a GRPC server enabling TLS credentials
-func NewSecure(rubyServer *rubyserver.Server) *grpc.Server {
- return createNewServer(rubyServer, true)
+func NewSecure(rubyServer *rubyserver.Server, cfg config.Cfg) *grpc.Server {
+ return createNewServer(rubyServer, cfg, true)
}
// CleanupInternalSocketDir will clean up the directory for internal sockets if it is a generated temp dir
diff --git a/internal/service/cleanup/internalrefs/cleaner.go b/internal/service/cleanup/internalrefs/cleaner.go
index 6efb4d618..5dd6889d4 100644
--- a/internal/service/cleanup/internalrefs/cleaner.go
+++ b/internal/service/cleanup/internalrefs/cleaner.go
@@ -7,7 +7,7 @@ import (
"io"
"strings"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/updateref"
@@ -96,7 +96,7 @@ func (c *Cleaner) processEntry(oldSHA, newSHA string) error {
return nil
}
- grpc_logrus.Extract(c.ctx).WithFields(log.Fields{
+ ctxlogrus.Extract(c.ctx).WithFields(log.Fields{
"sha": oldSHA,
"refs": refs,
}).Info("removing internal references")
@@ -128,7 +128,7 @@ func buildLookupTable(ctx context.Context, repo *gitalypb.Repository) (map[strin
return nil, err
}
- logger := grpc_logrus.Extract(ctx)
+ logger := ctxlogrus.Extract(ctx)
out := make(map[string][]string)
scanner := bufio.NewScanner(cmd)
diff --git a/internal/service/commit/commits_helper.go b/internal/service/commit/commits_helper.go
index d18f44fec..3c419bd68 100644
--- a/internal/service/commit/commits_helper.go
+++ b/internal/service/commit/commits_helper.go
@@ -3,7 +3,7 @@ package commit
import (
"context"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/log"
"gitlab.com/gitlab-org/gitaly/internal/helper/chunk"
@@ -39,7 +39,7 @@ func sendCommits(ctx context.Context, sender chunk.Sender, repo *gitalypb.Reposi
if err := cmd.Wait(); err != nil {
// We expect this error to be caused by non-existing references. In that
// case, we just log the error and send no commits to the `sender`.
- grpc_logrus.Extract(ctx).WithError(err).Info("ignoring git-log error")
+ ctxlogrus.Extract(ctx).WithError(err).Info("ignoring git-log error")
}
return nil
diff --git a/internal/service/commit/count_commits.go b/internal/service/commit/count_commits.go
index 51916195f..a02515d7f 100644
--- a/internal/service/commit/count_commits.go
+++ b/internal/service/commit/count_commits.go
@@ -8,7 +8,7 @@ import (
"strconv"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc/codes"
@@ -55,11 +55,11 @@ func (s *server) CountCommits(ctx context.Context, in *gitalypb.CountCommitsRequ
var count int64
countStr, readAllErr := ioutil.ReadAll(cmd)
if readAllErr != nil {
- grpc_logrus.Extract(ctx).WithError(err).Info("ignoring git rev-list error")
+ ctxlogrus.Extract(ctx).WithError(err).Info("ignoring git rev-list error")
}
if err := cmd.Wait(); err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Info("ignoring git rev-list error")
+ ctxlogrus.Extract(ctx).WithError(err).Info("ignoring git rev-list error")
count = 0
} else if readAllErr == nil {
var err error
diff --git a/internal/service/commit/count_diverging_commits.go b/internal/service/commit/count_diverging_commits.go
index 1593d8b57..52750feb9 100644
--- a/internal/service/commit/count_diverging_commits.go
+++ b/internal/service/commit/count_diverging_commits.go
@@ -79,12 +79,12 @@ func findLeftRightCount(ctx context.Context, repo *gitalypb.Repository, from, to
return 0, 0, fmt.Errorf("invalid output from git rev-list --left-right: %v", string(countStr))
}
- leftCount, err = strconv.ParseInt(string(counts[0]), 10, 32)
+ leftCount, err = strconv.ParseInt(counts[0], 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("invalid left count value: %v", counts[0])
}
- rightCount, err = strconv.ParseInt(string(counts[1]), 10, 32)
+ rightCount, err = strconv.ParseInt(counts[1], 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("invalid right count value: %v", counts[1])
}
diff --git a/internal/service/commit/isancestor.go b/internal/service/commit/isancestor.go
index 2b56a6acf..6bff2ac62 100644
--- a/internal/service/commit/isancestor.go
+++ b/internal/service/commit/isancestor.go
@@ -3,7 +3,7 @@ package commit
import (
"context"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -25,7 +25,7 @@ func (s *server) CommitIsAncestor(ctx context.Context, in *gitalypb.CommitIsAnce
// Assumes that `path`, `ancestorID` and `childID` are populated :trollface:
func commitIsAncestorName(ctx context.Context, repo *gitalypb.Repository, ancestorID, childID string) (bool, error) {
- grpc_logrus.Extract(ctx).WithFields(log.Fields{
+ ctxlogrus.Extract(ctx).WithFields(log.Fields{
"ancestorSha": ancestorID,
"childSha": childID,
}).Debug("commitIsAncestor")
diff --git a/internal/service/commit/languages.go b/internal/service/commit/languages.go
index 72d66a1c4..19ee7ffe2 100644
--- a/internal/service/commit/languages.go
+++ b/internal/service/commit/languages.go
@@ -82,7 +82,7 @@ func (*server) CommitLanguages(ctx context.Context, req *gitalypb.CommitLanguage
Share: float32(100*count) / float32(total),
Color: linguist.Color(lang),
FileCount: uint32(fileCountStats[lang]),
- Bytes: uint64(stats[lang]),
+ Bytes: stats[lang],
}
resp.Languages = append(resp.Languages, l)
}
diff --git a/internal/service/commit/last_commit_for_path_test.go b/internal/service/commit/last_commit_for_path_test.go
index 484d0ed9e..2b6f2ec75 100644
--- a/internal/service/commit/last_commit_for_path_test.go
+++ b/internal/service/commit/last_commit_for_path_test.go
@@ -77,7 +77,7 @@ func TestSuccessfulLastCommitForPathRequest(t *testing.T) {
request := &gitalypb.LastCommitForPathRequest{
Repository: testRepo,
Revision: []byte(testCase.revision),
- Path: []byte(testCase.path),
+ Path: testCase.path,
}
ctx, cancel := context.WithCancel(context.Background())
diff --git a/internal/service/commit/list_files.go b/internal/service/commit/list_files.go
index 95b455897..441cae9a5 100644
--- a/internal/service/commit/list_files.go
+++ b/internal/service/commit/list_files.go
@@ -4,7 +4,7 @@ import (
"fmt"
"io"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/lstree"
@@ -15,7 +15,7 @@ import (
)
func (s *server) ListFiles(in *gitalypb.ListFilesRequest, stream gitalypb.CommitService_ListFilesServer) error {
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"Revision": in.GetRevision(),
}).Debug("ListFiles")
diff --git a/internal/service/commit/list_last_commits_for_tree.go b/internal/service/commit/list_last_commits_for_tree.go
index 05535ca9e..e938b8631 100644
--- a/internal/service/commit/list_last_commits_for_tree.go
+++ b/internal/service/commit/list_last_commits_for_tree.go
@@ -68,7 +68,7 @@ func listLastCommitsForTree(in *gitalypb.ListLastCommitsForTreeRequest, stream g
}
for _, entry := range entries[offset:limit] {
- commit, err := log.LastCommitForPath(ctx, c, repo, string(in.GetRevision()), entry.Path)
+ commit, err := log.LastCommitForPath(ctx, c, repo, in.GetRevision(), entry.Path)
if err != nil {
return err
}
diff --git a/internal/service/commit/list_last_commits_for_tree_test.go b/internal/service/commit/list_last_commits_for_tree_test.go
index eecb8a6e5..4b9c22412 100644
--- a/internal/service/commit/list_last_commits_for_tree_test.go
+++ b/internal/service/commit/list_last_commits_for_tree_test.go
@@ -353,7 +353,7 @@ func TestNonUtf8ListLastCommitsForTreeRequest(t *testing.T) {
request := &gitalypb.ListLastCommitsForTreeRequest{
Repository: testRepo,
- Revision: string(commitID),
+ Revision: commitID,
Limit: 100,
Offset: 0,
}
diff --git a/internal/service/commit/raw_blame.go b/internal/service/commit/raw_blame.go
index 6b923187d..6b7aacac5 100644
--- a/internal/service/commit/raw_blame.go
+++ b/internal/service/commit/raw_blame.go
@@ -4,7 +4,7 @@ import (
"fmt"
"io"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
@@ -44,7 +44,7 @@ func (s *server) RawBlame(in *gitalypb.RawBlameRequest, stream gitalypb.CommitSe
}
if err := cmd.Wait(); err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Info("ignoring git-blame error")
+ ctxlogrus.Extract(ctx).WithError(err).Info("ignoring git-blame error")
}
return nil
diff --git a/internal/service/commit/tree_entries.go b/internal/service/commit/tree_entries.go
index d0c36f34d..70c386271 100644
--- a/internal/service/commit/tree_entries.go
+++ b/internal/service/commit/tree_entries.go
@@ -3,7 +3,7 @@ package commit
import (
"fmt"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/catfile"
@@ -86,7 +86,7 @@ func (c *treeEntriesSender) Send() error { return c.stream.Send(c.response) }
func (c *treeEntriesSender) Reset() { c.response = &gitalypb.GetTreeEntriesResponse{} }
func (s *server) GetTreeEntries(in *gitalypb.GetTreeEntriesRequest, stream gitalypb.CommitService_GetTreeEntriesServer) error {
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"Revision": in.Revision,
"Path": in.Path,
}).Debug("GetTreeEntries")
diff --git a/internal/service/conflicts/resolve_conflicts_test.go b/internal/service/conflicts/resolve_conflicts_test.go
index 8c92b3a0e..7dea2b29a 100644
--- a/internal/service/conflicts/resolve_conflicts_test.go
+++ b/internal/service/conflicts/resolve_conflicts_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git/log"
serverPkg "gitlab.com/gitlab-org/gitaly/internal/server"
"gitlab.com/gitlab-org/gitaly/internal/service/conflicts"
@@ -308,7 +309,7 @@ func TestFailedResolveConflictsRequestDueToValidation(t *testing.T) {
}
func runFullServer(t *testing.T) (*grpc.Server, string) {
- server := serverPkg.NewInsecure(conflicts.RubyServer)
+ server := serverPkg.NewInsecure(conflicts.RubyServer, config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
listener, err := net.Listen("unix", serverSocketPath)
diff --git a/internal/service/diff/commit.go b/internal/service/diff/commit.go
index ccabef38b..654fe4db6 100644
--- a/internal/service/diff/commit.go
+++ b/internal/service/diff/commit.go
@@ -4,7 +4,7 @@ import (
"context"
"fmt"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/diff"
"gitlab.com/gitlab-org/gitaly/internal/git"
@@ -19,7 +19,7 @@ type requestWithLeftRightCommitIds interface {
}
func (s *server) CommitDiff(in *gitalypb.CommitDiffRequest, stream gitalypb.DiffService_CommitDiffServer) error {
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"LeftCommitId": in.LeftCommitId,
"RightCommitId": in.RightCommitId,
"IgnoreWhitespaceChange": in.IgnoreWhitespaceChange,
@@ -117,7 +117,7 @@ func (s *server) CommitDiff(in *gitalypb.CommitDiffRequest, stream gitalypb.Diff
}
func (s *server) CommitDelta(in *gitalypb.CommitDeltaRequest, stream gitalypb.DiffService_CommitDeltaServer) error {
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"LeftCommitId": in.LeftCommitId,
"RightCommitId": in.RightCommitId,
"Paths": logPaths(in.Paths),
diff --git a/internal/service/hooks/post_receive_test.go b/internal/service/hooks/post_receive_test.go
index fe7b4145d..451c322f7 100644
--- a/internal/service/hooks/post_receive_test.go
+++ b/internal/service/hooks/post_receive_test.go
@@ -67,7 +67,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=key_id", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key_id", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
GitPushOptions: []string{"option0", "option1"}},
status: 0,
stdout: "OK",
@@ -78,7 +78,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBuffer(nil),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=key_id", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key_id", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
GitPushOptions: []string{"option0"},
},
status: 1,
@@ -90,7 +90,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
GitPushOptions: []string{"option0"},
},
status: 1,
@@ -102,7 +102,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
GitPushOptions: []string{"option0"},
},
status: 1,
@@ -114,7 +114,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL="},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=", "GL_REPOSITORY=repository"},
GitPushOptions: []string{"option0"},
},
status: 1,
@@ -125,12 +125,8 @@ func TestPostReceive(t *testing.T) {
desc: "missing gl_repository value",
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
- Repository: &gitalypb.Repository{
- StorageName: testRepo.GetStorageName(),
- RelativePath: testRepo.GetRelativePath(),
- GlRepository: "",
- },
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ Repository: testRepo,
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY="},
GitPushOptions: []string{"option0"},
},
status: 1,
@@ -142,7 +138,7 @@ func TestPostReceive(t *testing.T) {
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PostReceiveHookRequest{
Repository: testRepo,
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
diff --git a/internal/service/hooks/pre_receive.go b/internal/service/hooks/pre_receive.go
index 6458c68b5..0564bb71c 100644
--- a/internal/service/hooks/pre_receive.go
+++ b/internal/service/hooks/pre_receive.go
@@ -2,11 +2,11 @@ package hook
import (
"errors"
- "fmt"
"os/exec"
"path/filepath"
"gitlab.com/gitlab-org/gitaly/internal/config"
+ "gitlab.com/gitlab-org/gitaly/internal/git/alternates"
"gitlab.com/gitlab-org/gitaly/internal/gitlabshell"
"gitlab.com/gitlab-org/gitaly/internal/helper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -19,8 +19,15 @@ type hookRequest interface {
}
func hookRequestEnv(req hookRequest) []string {
- return append(gitlabshell.Env(),
- append(req.GetEnvironmentVariables(), fmt.Sprintf("GL_REPOSITORY=%s", req.GetRepository().GetGlRepository()))...)
+ return append(gitlabshell.Env(), req.GetEnvironmentVariables()...)
+}
+
+func preReceiveEnv(req hookRequest) ([]string, error) {
+ _, env, err := alternates.PathAndEnv(req.GetRepository())
+ if err != nil {
+ return nil, err
+ }
+ return append(hookRequestEnv(req), env...), nil
}
func gitlabShellHook(hookName string) string {
@@ -52,12 +59,17 @@ func (s *server) PreReceiveHook(stream gitalypb.HookService_PreReceiveHookServer
c := exec.Command(gitlabShellHook("pre-receive"))
c.Dir = repoPath
+ env, err := preReceiveEnv(firstRequest)
+ if err != nil {
+ return helper.ErrInternal(err)
+ }
+
status, err := streamCommandResponse(
stream.Context(),
stdin,
stdout, stderr,
c,
- hookRequestEnv(firstRequest),
+ env,
)
if err != nil {
diff --git a/internal/service/hooks/pre_receive_test.go b/internal/service/hooks/pre_receive_test.go
index 2a2d40a57..e1d89dccf 100644
--- a/internal/service/hooks/pre_receive_test.go
+++ b/internal/service/hooks/pre_receive_test.go
@@ -73,6 +73,7 @@ func TestPreReceive(t *testing.T) {
"GL_ID=key-123",
"GL_PROTOCOL=protocol",
"GL_USERNAME=username",
+ "GL_REPOSITORY=repository",
},
},
status: 0,
@@ -88,6 +89,7 @@ func TestPreReceive(t *testing.T) {
"GL_ID=key-123",
"GL_PROTOCOL=protocol",
"GL_USERNAME=username",
+ "GL_REPOSITORY=repository",
},
},
status: 1,
@@ -103,6 +105,7 @@ func TestPreReceive(t *testing.T) {
"GL_ID=key-123",
"GL_USERNAME=username",
"GL_PROTOCOL=",
+ "GL_REPOSITORY=repository",
},
},
status: 1,
@@ -118,6 +121,7 @@ func TestPreReceive(t *testing.T) {
"GL_ID=",
"GL_PROTOCOL=protocol",
"GL_USERNAME=username",
+ "GL_REPOSITORY=repository",
},
},
status: 1,
@@ -133,6 +137,7 @@ func TestPreReceive(t *testing.T) {
"GL_ID=key-123",
"GL_PROTOCOL=protocol",
"GL_USERNAME=",
+ "GL_REPOSITORY=repository",
},
},
status: 1,
@@ -143,15 +148,12 @@ func TestPreReceive(t *testing.T) {
desc: "missing gl_repository",
stdin: bytes.NewBufferString("a\nb\nc\nd\ne\nf\ng"),
req: gitalypb.PreReceiveHookRequest{
- Repository: &gitalypb.Repository{
- StorageName: testRepo.GetStorageName(),
- RelativePath: testRepo.GetRelativePath(),
- GlRepository: "",
- },
+ Repository: testRepo,
EnvironmentVariables: []string{
"GL_ID=key-123",
- "GL_PROTOCOL=",
+ "GL_PROTOCOL=protocol",
"GL_USERNAME=username",
+ "GL_REPOSITORY=",
},
},
status: 1,
diff --git a/internal/service/hooks/testhelper_test.go b/internal/service/hooks/testhelper_test.go
index 25a59c200..8311bc869 100644
--- a/internal/service/hooks/testhelper_test.go
+++ b/internal/service/hooks/testhelper_test.go
@@ -21,7 +21,7 @@ func TestMain(m *testing.M) {
func newHooksClient(t *testing.T, serverSocketPath string) (gitalypb.HookServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(config.Config.Auth.Token)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.Config.Auth.Token)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
diff --git a/internal/service/hooks/update_test.go b/internal/service/hooks/update_test.go
index 81e0f0639..1155371d3 100644
--- a/internal/service/hooks/update_test.go
+++ b/internal/service/hooks/update_test.go
@@ -68,7 +68,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 0,
stdout: "OK",
@@ -81,7 +81,7 @@ func TestUpdate(t *testing.T) {
Ref: nil,
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -94,7 +94,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -107,7 +107,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -120,7 +120,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -133,7 +133,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=", "GL_PROTOCOL=protocol", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -146,7 +146,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL="},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=", "GL_REPOSITORY=repository"},
},
status: 1,
stdout: "",
@@ -163,7 +163,7 @@ func TestUpdate(t *testing.T) {
Ref: []byte("master"),
OldValue: "a",
NewValue: "b",
- EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol"},
+ EnvironmentVariables: []string{"GL_ID=key-123", "GL_USERNAME=username", "GL_PROTOCOL=protocol", "GL_REPOSITORY="},
},
status: 1,
stdout: "",
diff --git a/internal/service/objectpool/alternates.go b/internal/service/objectpool/alternates.go
index db0206ec8..c43e074ea 100644
--- a/internal/service/objectpool/alternates.go
+++ b/internal/service/objectpool/alternates.go
@@ -11,7 +11,7 @@ import (
"strings"
"time"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/helper"
"gitlab.com/gitlab-org/gitaly/internal/helper/text"
@@ -182,7 +182,7 @@ func removeAlternatesIfOk(ctx context.Context, repo *gitalypb.Repository, altFil
return
}
- logger := grpc_logrus.Extract(ctx)
+ logger := ctxlogrus.Extract(ctx)
// If we would do a os.Rename, and then someone else comes and clobbers
// our file, it's gone forever. This trick with os.Link and os.Rename
diff --git a/internal/service/objectpool/fetch_into_object_pool_test.go b/internal/service/objectpool/fetch_into_object_pool_test.go
index d689c0215..425ea621f 100644
--- a/internal/service/objectpool/fetch_into_object_pool_test.go
+++ b/internal/service/objectpool/fetch_into_object_pool_test.go
@@ -57,7 +57,7 @@ func TestFetchIntoObjectPool_Success(t *testing.T) {
require.Len(t, packFiles, 1, "ensure commits got packed")
packContents := testhelper.MustRunCommand(t, nil, "git", "-C", pool.FullPath(), "verify-pack", "-v", packFiles[0])
- require.Contains(t, string(packContents), string(repoCommit))
+ require.Contains(t, string(packContents), repoCommit)
_, err = client.FetchIntoObjectPool(ctx, req)
require.NoError(t, err, "calling FetchIntoObjectPool twice should be OK")
@@ -65,12 +65,12 @@ func TestFetchIntoObjectPool_Success(t *testing.T) {
}
func TestFetchIntoObjectPool_CollectLogStatistics(t *testing.T) {
- defer func(tl func(tb testhelper.TB) *logrus.Logger) {
+ defer func(tl func(tb testing.TB) *logrus.Logger) {
testhelper.NewTestLogger = tl
}(testhelper.NewTestLogger)
logBuffer := &bytes.Buffer{}
- testhelper.NewTestLogger = func(tb testhelper.TB) *logrus.Logger {
+ testhelper.NewTestLogger = func(tb testing.TB) *logrus.Logger {
return &logrus.Logger{Out: logBuffer, Formatter: &logrus.JSONFormatter{}, Level: logrus.InfoLevel}
}
diff --git a/internal/service/objectpool/get.go b/internal/service/objectpool/get.go
index c04d27139..d5500dbc2 100644
--- a/internal/service/objectpool/get.go
+++ b/internal/service/objectpool/get.go
@@ -4,7 +4,7 @@ import (
"context"
"errors"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git/objectpool"
"gitlab.com/gitlab-org/gitaly/internal/helper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -18,7 +18,7 @@ func (s *server) GetObjectPool(ctx context.Context, in *gitalypb.GetObjectPoolRe
objectPool, err := objectpool.FromRepo(in.GetRepository())
if err != nil {
- grpc_logrus.Extract(ctx).
+ ctxlogrus.Extract(ctx).
WithError(err).
WithField("storage", in.GetRepository().GetStorageName()).
WithField("storage", in.GetRepository().GetRelativePath()).
diff --git a/internal/service/operations/cherry_pick_test.go b/internal/service/operations/cherry_pick_test.go
index 509a28327..ab829a2e4 100644
--- a/internal/service/operations/cherry_pick_test.go
+++ b/internal/service/operations/cherry_pick_test.go
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git/log"
serverPkg "gitlab.com/gitlab-org/gitaly/internal/server"
"gitlab.com/gitlab-org/gitaly/internal/service/operations"
@@ -411,7 +412,7 @@ func TestFailedUserCherryPickRequestDueToCommitError(t *testing.T) {
}
func runFullServerWithHooks(t *testing.T) (*grpc.Server, string) {
- server := serverPkg.NewInsecure(operations.RubyServer)
+ server := serverPkg.NewInsecure(operations.RubyServer, config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
listener, err := net.Listen("unix", serverSocketPath)
diff --git a/internal/service/operations/rebase_test.go b/internal/service/operations/rebase_test.go
index 228284530..da254dced 100644
--- a/internal/service/operations/rebase_test.go
+++ b/internal/service/operations/rebase_test.go
@@ -387,7 +387,7 @@ func TestFailedUserRebaseConfirmableDueToGitError(t *testing.T) {
firstResponse, err := rebaseStream.Recv()
require.NoError(t, err, "receive first response")
- require.Contains(t, firstResponse.GitError, "error: Failed to merge in the changes.")
+ require.Contains(t, firstResponse.GitError, "CONFLICT (content): Merge conflict in README.md")
err = testhelper.ReceiveEOFWithTimeout(func() error {
_, err = rebaseStream.Recv()
@@ -427,13 +427,13 @@ func recvTimeout(bidi gitalypb.OperationService_UserRebaseConfirmableClient, tim
}
}
-func buildHeaderRequest(repo *gitalypb.Repository, user *gitalypb.User, rebaseId string, branchName string, branchSha string, remoteRepo *gitalypb.Repository, remoteBranch string) *gitalypb.UserRebaseConfirmableRequest { // nolint:golint
+func buildHeaderRequest(repo *gitalypb.Repository, user *gitalypb.User, rebaseID string, branchName string, branchSha string, remoteRepo *gitalypb.Repository, remoteBranch string) *gitalypb.UserRebaseConfirmableRequest {
return &gitalypb.UserRebaseConfirmableRequest{
UserRebaseConfirmableRequestPayload: &gitalypb.UserRebaseConfirmableRequest_Header_{
- &gitalypb.UserRebaseConfirmableRequest_Header{
+ Header: &gitalypb.UserRebaseConfirmableRequest_Header{
Repository: repo,
User: user,
- RebaseId: rebaseId,
+ RebaseId: rebaseID,
Branch: []byte(branchName),
BranchSha: branchSha,
RemoteRepository: remoteRepo,
diff --git a/internal/service/operations/submodules_test.go b/internal/service/operations/submodules_test.go
index d85632567..0bfd7735c 100644
--- a/internal/service/operations/submodules_test.go
+++ b/internal/service/operations/submodules_test.go
@@ -77,7 +77,7 @@ func TestSuccessfulUserUpdateSubmoduleRequest(t *testing.T) {
parser := lstree.NewParser(bytes.NewReader(entry))
parsedEntry, err := parser.NextEntry()
require.NoError(t, err)
- require.Equal(t, testCase.submodule, string(parsedEntry.Path))
+ require.Equal(t, testCase.submodule, parsedEntry.Path)
require.Equal(t, testCase.commitSha, parsedEntry.Oid)
})
}
diff --git a/internal/service/operations/tags_test.go b/internal/service/operations/tags_test.go
index d4ed59b3b..474d13d03 100644
--- a/internal/service/operations/tags_test.go
+++ b/internal/service/operations/tags_test.go
@@ -209,6 +209,14 @@ func TestSuccessfulUserCreateTagRequest(t *testing.T) {
}
func TestSuccessfulGitHooksForUserCreateTagRequest(t *testing.T) {
+ testSuccessfulGitHooksForUserCreateTagRequest(t, false)
+}
+
+func TestSuccessfulGitHooksForUserCreateTagRequestWithHookRPCs(t *testing.T) {
+ testSuccessfulGitHooksForUserCreateTagRequest(t, true)
+}
+
+func testSuccessfulGitHooksForUserCreateTagRequest(t *testing.T, callHookRPC bool) {
serverSocketPath, stop := runOperationServiceServer(t)
defer stop()
@@ -236,6 +244,13 @@ func TestSuccessfulGitHooksForUserCreateTagRequest(t *testing.T) {
User: user,
}
+ ctx, cancel := testhelper.Context()
+ defer cancel()
+
+ if callHookRPC {
+ ctx = outgoingCtxWithRubyFeatureFlag(ctx, "call-hook-rpc")
+ }
+
for _, hookName := range GitlabHooks {
t.Run(hookName, func(t *testing.T) {
defer exec.Command("git", "-C", testRepoPath, "tag", "-d", tagName).Run()
@@ -243,9 +258,6 @@ func TestSuccessfulGitHooksForUserCreateTagRequest(t *testing.T) {
hookOutputTempPath, cleanup := testhelper.WriteEnvToCustomHook(t, testRepoPath, hookName)
defer cleanup()
- ctx, cancel := testhelper.Context()
- defer cancel()
-
response, err := client.UserCreateTag(ctx, request)
require.NoError(t, err)
require.Empty(t, response.PreReceiveError)
diff --git a/internal/service/operations/testhelper_test.go b/internal/service/operations/testhelper_test.go
index c9a59c580..408cb4846 100644
--- a/internal/service/operations/testhelper_test.go
+++ b/internal/service/operations/testhelper_test.go
@@ -1,17 +1,24 @@
package operations
import (
+ "context"
+ "fmt"
+ "net"
"os"
"path/filepath"
+ "strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
+ gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
"gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
+ hook "gitlab.com/gitlab-org/gitaly/internal/service/hooks"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc"
+ "google.golang.org/grpc/metadata"
"google.golang.org/grpc/reflection"
)
@@ -57,6 +64,11 @@ func testMain(m *testing.M) int {
testhelper.ConfigureGitalySSH()
testhelper.ConfigureGitalyHooksBinary()
+ defer func(token string) {
+ config.Config.Auth.Token = token
+ }(config.Config.Auth.Token)
+ config.Config.Auth.Token = testhelper.RepositoryAuthToken
+
if err := RubyServer.Start(); err != nil {
log.Fatal(err)
}
@@ -66,19 +78,29 @@ func testMain(m *testing.M) int {
}
func runOperationServiceServer(t *testing.T) (string, func()) {
- srv := testhelper.NewServer(t, nil, nil)
+ srv := testhelper.NewServerWithAuth(t, nil, nil, config.Config.Auth.Token)
gitalypb.RegisterOperationServiceServer(srv.GrpcServer(), &server{ruby: RubyServer})
+ gitalypb.RegisterHookServiceServer(srv.GrpcServer(), hook.NewServer())
reflection.Register(srv.GrpcServer())
require.NoError(t, srv.Start())
+ internalSocket := config.GitalyInternalSocketPath()
+ internalListener, err := net.Listen("unix", internalSocket)
+ require.NoError(t, err)
+
+ go func() {
+ srv.GrpcServer().Serve(internalListener)
+ }()
+
return "unix://" + srv.Socket(), srv.Stop
}
func newOperationClient(t *testing.T, serverSocketPath string) (gitalypb.OperationServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.Config.Auth.Token)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
@@ -100,3 +122,16 @@ func SetupAndStartGitlabServer(t *testing.T, glID, glRepository string, gitPushO
GitPushOptions: gitPushOptions,
})
}
+
+func outgoingCtxWithRubyFeatureFlag(ctx context.Context, flag string) context.Context {
+ md, ok := metadata.FromOutgoingContext(ctx)
+ if !ok {
+ md = metadata.New(map[string]string{})
+ }
+ md.Set(rubyHeaderKey(flag), "true")
+ return metadata.NewOutgoingContext(ctx, md)
+}
+
+func rubyHeaderKey(flag string) string {
+ return fmt.Sprintf("gitaly-feature-ruby-%s", strings.ReplaceAll(flag, "_", "-"))
+}
diff --git a/internal/service/ref/delete_refs.go b/internal/service/ref/delete_refs.go
index 44ff3a17b..a5fbde68a 100644
--- a/internal/service/ref/delete_refs.go
+++ b/internal/service/ref/delete_refs.go
@@ -72,7 +72,7 @@ func refsToRemove(ctx context.Context, req *gitalypb.DeleteRefsRequest) ([]strin
continue
}
- refs = append(refs, string(refName))
+ refs = append(refs, refName)
}
if err != nil {
diff --git a/internal/service/ref/refs_test.go b/internal/service/ref/refs_test.go
index c71efba88..c9159c67e 100644
--- a/internal/service/ref/refs_test.go
+++ b/internal/service/ref/refs_test.go
@@ -515,7 +515,7 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) {
expectedTags := []*gitalypb.Tag{
{
Name: []byte(commitID),
- Id: string(commitTagID),
+ Id: commitTagID,
TargetCommit: gitCommit,
Message: []byte("commit tag with a commit sha as the name"),
MessageSize: 40,
@@ -528,7 +528,7 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) {
},
{
Name: []byte("tag-of-tag"),
- Id: string(tagOfTagID),
+ Id: tagOfTagID,
TargetCommit: gitCommit,
Message: []byte("tag of a tag"),
MessageSize: 12,
@@ -619,7 +619,7 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) {
},
{
Name: []byte("v1.2.0"),
- Id: string(annotatedTagID),
+ Id: annotatedTagID,
Message: []byte("Blob tag"),
MessageSize: 8,
Tagger: &gitalypb.CommitAuthor{
@@ -631,26 +631,26 @@ func TestSuccessfulFindAllTagsRequest(t *testing.T) {
},
{
Name: []byte("v1.3.0"),
- Id: string(commitID),
+ Id: commitID,
TargetCommit: gitCommit,
},
{
Name: []byte("v1.4.0"),
- Id: string(blobID),
+ Id: blobID,
},
{
Name: []byte("v1.5.0"),
- Id: string(commitID),
+ Id: commitID,
TargetCommit: gitCommit,
},
{
Name: []byte("v1.6.0"),
- Id: string(bigCommitID),
+ Id: bigCommitID,
TargetCommit: bigCommit,
},
{
Name: []byte("v1.7.0"),
- Id: string(bigMessageTag1ID),
+ Id: bigMessageTag1ID,
Message: []byte(bigMessage[:helper.MaxCommitOrTagMessageSize]),
MessageSize: int64(len(bigMessage)),
TargetCommit: gitCommit,
@@ -727,11 +727,11 @@ func TestFindAllTagNestedTags(t *testing.T) {
for depth := 0; depth < tc.depth; depth++ {
tagName := fmt.Sprintf("tag-depth-%d", depth)
tagMessage := fmt.Sprintf("a commit %d deep", depth)
- tagID = string(testhelper.CreateTag(t, testRepoCopyPath, tagName, tagID, &testhelper.CreateTagOpts{Message: tagMessage}))
+ tagID = testhelper.CreateTag(t, testRepoCopyPath, tagName, tagID, &testhelper.CreateTagOpts{Message: tagMessage})
expectedTag := &gitalypb.Tag{
Name: []byte(tagName),
- Id: string(tagID),
+ Id: tagID,
Message: []byte(tagMessage),
MessageSize: int64(len([]byte(tagMessage))),
Tagger: &gitalypb.CommitAuthor{
@@ -1431,7 +1431,7 @@ func TestSuccessfulFindTagRequest(t *testing.T) {
expectedTags := []*gitalypb.Tag{
{
Name: []byte(commitID),
- Id: string(commitTagID),
+ Id: commitTagID,
TargetCommit: gitCommit,
Message: []byte("commit tag with a commit sha as the name"),
MessageSize: 40,
@@ -1444,7 +1444,7 @@ func TestSuccessfulFindTagRequest(t *testing.T) {
},
{
Name: []byte("tag-of-tag"),
- Id: string(tagOfTagID),
+ Id: tagOfTagID,
TargetCommit: gitCommit,
Message: []byte("tag of a tag"),
MessageSize: 12,
@@ -1536,7 +1536,7 @@ func TestSuccessfulFindTagRequest(t *testing.T) {
},
{
Name: []byte("v1.2.0"),
- Id: string(annotatedTagID),
+ Id: annotatedTagID,
Message: []byte("Blob tag"),
MessageSize: 8,
Tagger: &gitalypb.CommitAuthor{
@@ -1548,26 +1548,26 @@ func TestSuccessfulFindTagRequest(t *testing.T) {
},
{
Name: []byte("v1.3.0"),
- Id: string(commitID),
+ Id: commitID,
TargetCommit: gitCommit,
},
{
Name: []byte("v1.4.0"),
- Id: string(blobID),
+ Id: blobID,
},
{
Name: []byte("v1.5.0"),
- Id: string(commitID),
+ Id: commitID,
TargetCommit: gitCommit,
},
{
Name: []byte("v1.6.0"),
- Id: string(bigCommitID),
+ Id: bigCommitID,
TargetCommit: bigCommit,
},
{
Name: []byte("v1.7.0"),
- Id: string(bigMessageTag1ID),
+ Id: bigMessageTag1ID,
Message: []byte(bigMessage[:helper.MaxCommitOrTagMessageSize]),
MessageSize: int64(len(bigMessage)),
TargetCommit: gitCommit,
@@ -1650,11 +1650,11 @@ func TestFindTagNestedTag(t *testing.T) {
for depth := 0; depth < tc.depth; depth++ {
tagName = fmt.Sprintf("tag-depth-%d", depth)
tagMessage = fmt.Sprintf("a commit %d deep", depth)
- tagID = string(testhelper.CreateTag(t, testRepoCopyPath, tagName, tagID, &testhelper.CreateTagOpts{Message: tagMessage}))
+ tagID = testhelper.CreateTag(t, testRepoCopyPath, tagName, tagID, &testhelper.CreateTagOpts{Message: tagMessage})
}
expectedTag := &gitalypb.Tag{
Name: []byte(tagName),
- Id: string(tagID),
+ Id: tagID,
Message: []byte(tagMessage),
MessageSize: int64(len([]byte(tagMessage))),
Tagger: &gitalypb.CommitAuthor{
diff --git a/internal/service/register.go b/internal/service/register.go
index 412defcf3..4015f99fc 100644
--- a/internal/service/register.go
+++ b/internal/service/register.go
@@ -52,7 +52,7 @@ var (
// RegisterAll will register all the known grpc services with
// the specified grpc service instance
-func RegisterAll(grpcServer *grpc.Server, rubyServer *rubyserver.Server) {
+func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver.Server) {
gitalypb.RegisterBlobServiceServer(grpcServer, blob.NewServer(rubyServer))
gitalypb.RegisterCleanupServiceServer(grpcServer, cleanup.NewServer())
gitalypb.RegisterCommitServiceServer(grpcServer, commit.NewServer())
@@ -70,7 +70,7 @@ func RegisterAll(grpcServer *grpc.Server, rubyServer *rubyserver.Server) {
gitalypb.RegisterWikiServiceServer(grpcServer, wiki.NewServer(rubyServer))
gitalypb.RegisterConflictsServiceServer(grpcServer, conflicts.NewServer(rubyServer))
gitalypb.RegisterRemoteServiceServer(grpcServer, remote.NewServer(rubyServer))
- gitalypb.RegisterServerServiceServer(grpcServer, server.NewServer())
+ gitalypb.RegisterServerServiceServer(grpcServer, server.NewServer(cfg.Storages))
gitalypb.RegisterObjectPoolServiceServer(grpcServer, objectpool.NewServer())
gitalypb.RegisterHookServiceServer(grpcServer, hook.NewServer())
gitalypb.RegisterInternalGitalyServer(grpcServer, internalgitaly.NewServer(config.Config.Storages))
diff --git a/internal/service/remote/fetch_internal_remote_test.go b/internal/service/remote/fetch_internal_remote_test.go
index e752bffb3..09a89b06b 100644
--- a/internal/service/remote/fetch_internal_remote_test.go
+++ b/internal/service/remote/fetch_internal_remote_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
serverPkg "gitlab.com/gitlab-org/gitaly/internal/server"
"gitlab.com/gitlab-org/gitaly/internal/service/remote"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
@@ -114,7 +115,7 @@ func TestFailedFetchInternalRemoteDueToValidations(t *testing.T) {
}
func runFullServer(t *testing.T) (*grpc.Server, string) {
- server := serverPkg.NewInsecure(remote.RubyServer)
+ server := serverPkg.NewInsecure(remote.RubyServer, config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
listener, err := net.Listen("unix", serverSocketPath)
diff --git a/internal/service/repository/archive_test.go b/internal/service/repository/archive_test.go
index a7afef228..f6a7e2bad 100644
--- a/internal/service/repository/archive_test.go
+++ b/internal/service/repository/archive_test.go
@@ -9,7 +9,6 @@ import (
"testing"
"github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
@@ -214,9 +213,6 @@ func TestGetArchiveFailure(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.desc, func(t *testing.T) {
ctx, cancel := testhelper.Context()
- // enable cache invalidator to test against bad inputs
- // See https://gitlab.com/gitlab-org/gitaly/issues/2262
- ctx = featureflag.OutgoingCtxWithFeatureFlag(ctx, featureflag.CacheInvalidator)
defer cancel()
req := &gitalypb.GetArchiveRequest{
diff --git a/internal/service/repository/create_test.go b/internal/service/repository/create_test.go
index 889784043..9fe052bb5 100644
--- a/internal/service/repository/create_test.go
+++ b/internal/service/repository/create_test.go
@@ -35,6 +35,7 @@ func TestCreateRepositorySuccess(t *testing.T) {
req := &gitalypb.CreateRepositoryRequest{Repository: repo}
_, err = client.CreateRepository(ctx, req)
require.NoError(t, err)
+ defer func() { require.NoError(t, os.RemoveAll(repoDir)) }()
fi, err := os.Stat(repoDir)
require.NoError(t, err)
diff --git a/internal/service/repository/fetch_test.go b/internal/service/repository/fetch_test.go
index 43500dc2e..9257f1a90 100644
--- a/internal/service/repository/fetch_test.go
+++ b/internal/service/repository/fetch_test.go
@@ -32,8 +32,11 @@ func TestFetchSourceBranchSourceRepositorySuccess(t *testing.T) {
md := testhelper.GitalyServersMetadata(t, serverSocketPath)
ctx := metadata.NewOutgoingContext(ctxOuter, md)
- targetRepo, _ := newTestRepo(t, "fetch-source-target.git")
- sourceRepo, sourcePath := newTestRepo(t, "fetch-source-source.git")
+ targetRepo, _, cleanup := newTestRepo(t, "fetch-source-target.git")
+ defer cleanup()
+
+ sourceRepo, sourcePath, cleanup := newTestRepo(t, "fetch-source-source.git")
+ defer cleanup()
sourceBranch := "fetch-source-branch-test-branch"
newCommitID := testhelper.CreateCommit(t, sourcePath, sourceBranch, nil)
@@ -68,7 +71,8 @@ func TestFetchSourceBranchSameRepositorySuccess(t *testing.T) {
md := testhelper.GitalyServersMetadata(t, serverSocketPath)
ctx := metadata.NewOutgoingContext(ctxOuter, md)
- repo, repoPath := newTestRepo(t, "fetch-source-source.git")
+ repo, repoPath, cleanup := newTestRepo(t, "fetch-source-source.git")
+ defer cleanup()
sourceBranch := "fetch-source-branch-test-branch"
newCommitID := testhelper.CreateCommit(t, repoPath, sourceBranch, nil)
@@ -103,8 +107,11 @@ func TestFetchSourceBranchBranchNotFound(t *testing.T) {
md := testhelper.GitalyServersMetadata(t, serverSocketPath)
ctx := metadata.NewOutgoingContext(ctxOuter, md)
- targetRepo, _ := newTestRepo(t, "fetch-source-target.git")
- sourceRepo, _ := newTestRepo(t, "fetch-source-source.git")
+ targetRepo, _, cleanup := newTestRepo(t, "fetch-source-target.git")
+ defer cleanup()
+
+ sourceRepo, _, cleanup := newTestRepo(t, "fetch-source-source.git")
+ defer cleanup()
sourceBranch := "does-not-exist"
targetRef := "refs/tmp/fetch-source-branch-test"
@@ -168,7 +175,7 @@ func TestFetchFullServerRequiresAuthentication(t *testing.T) {
testhelper.RequireGrpcError(t, err, codes.Unauthenticated)
}
-func newTestRepo(t *testing.T, relativePath string) (*gitalypb.Repository, string) {
+func newTestRepo(t *testing.T, relativePath string) (*gitalypb.Repository, string, func()) {
_, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
defer cleanupFn()
@@ -180,11 +187,11 @@ func newTestRepo(t *testing.T, relativePath string) (*gitalypb.Repository, strin
require.NoError(t, os.RemoveAll(repoPath))
testhelper.MustRunCommand(t, nil, "git", "clone", "--bare", testRepoPath, repoPath)
- return repo, repoPath
+ return repo, repoPath, func() { require.NoError(t, os.RemoveAll(repoPath)) }
}
func runFullServer(t *testing.T) (*grpc.Server, string) {
- server := serverPkg.NewInsecure(repository.RubyServer)
+ server := serverPkg.NewInsecure(repository.RubyServer, config.Config)
serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
listener, err := net.Listen("unix", serverSocketPath)
@@ -201,7 +208,7 @@ func runFullServer(t *testing.T) (*grpc.Server, string) {
}
func runFullSecureServer(t *testing.T) (*grpc.Server, string, testhelper.Cleanup) {
- server := serverPkg.NewSecure(repository.RubyServer)
+ server := serverPkg.NewSecure(repository.RubyServer, config.Config)
listener, addr := testhelper.GetLocalhostListener(t)
errQ := make(chan error)
diff --git a/internal/service/repository/fork_test.go b/internal/service/repository/fork_test.go
index e04ef8c06..7f603823b 100644
--- a/internal/service/repository/fork_test.go
+++ b/internal/service/repository/fork_test.go
@@ -86,6 +86,7 @@ func TestSuccessfulCreateForkRequest(t *testing.T) {
_, err = client.CreateFork(ctx, req)
require.NoError(t, err)
+ defer func() { require.NoError(t, os.RemoveAll(forkedRepoPath)) }()
testhelper.MustRunCommand(t, nil, "git", "-C", forkedRepoPath, "fsck")
diff --git a/internal/service/repository/gc.go b/internal/service/repository/gc.go
index dcf57fc5a..ab4099855 100644
--- a/internal/service/repository/gc.go
+++ b/internal/service/repository/gc.go
@@ -8,7 +8,7 @@ import (
"os"
"path/filepath"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/catfile"
@@ -20,7 +20,7 @@ import (
)
func (*server) GarbageCollect(ctx context.Context, in *gitalypb.GarbageCollectRequest) (*gitalypb.GarbageCollectResponse, error) {
- ctxlogger := grpc_logrus.Extract(ctx)
+ ctxlogger := ctxlogrus.Extract(ctx)
ctxlogger.WithFields(log.Fields{
"WriteBitmaps": in.GetCreateBitmap(),
}).Debug("GarbageCollect")
diff --git a/internal/service/repository/gc_test.go b/internal/service/repository/gc_test.go
index 8b54cf8d3..c3823bc38 100644
--- a/internal/service/repository/gc_test.go
+++ b/internal/service/repository/gc_test.go
@@ -124,12 +124,12 @@ func TestGarbageCollectSuccess(t *testing.T) {
}
func TestGarbageCollectLogStatistics(t *testing.T) {
- defer func(tl func(tb testhelper.TB) *logrus.Logger) {
+ defer func(tl func(tb testing.TB) *logrus.Logger) {
testhelper.NewTestLogger = tl
}(testhelper.NewTestLogger)
logBuffer := &bytes.Buffer{}
- testhelper.NewTestLogger = func(tb testhelper.TB) *logrus.Logger {
+ testhelper.NewTestLogger = func(tb testing.TB) *logrus.Logger {
return &logrus.Logger{Out: logBuffer, Formatter: &logrus.JSONFormatter{}, Level: logrus.InfoLevel}
}
diff --git a/internal/service/repository/redirecting_test_server_test.go b/internal/service/repository/redirecting_test_server_test.go
index d9dee1840..f5a200488 100644
--- a/internal/service/repository/redirecting_test_server_test.go
+++ b/internal/service/repository/redirecting_test_server_test.go
@@ -38,7 +38,7 @@ func StartRedirectingTestServer() (*RedirectingTestServerState, *httptest.Server
}
func TestRedirectingServerRedirects(t *testing.T) {
- dir, cleanup := testhelper.TempDir(t, t.Name())
+ dir, cleanup := testhelper.TempDir(t)
defer cleanup()
httpServerState, redirectingServer := StartRedirectingTestServer()
diff --git a/internal/service/repository/rename_test.go b/internal/service/repository/rename_test.go
index 4edf51f92..2727f243a 100644
--- a/internal/service/repository/rename_test.go
+++ b/internal/service/repository/rename_test.go
@@ -1,7 +1,7 @@
package repository
import (
- "path/filepath"
+ "os"
"testing"
"github.com/stretchr/testify/require"
@@ -21,12 +21,7 @@ func TestRenameRepositorySuccess(t *testing.T) {
testRepo, _, cleanupFn := testhelper.NewTestRepo(t)
defer cleanupFn()
- tempDir, cleanupTempDir := testhelper.TempDir(t, t.Name())
- defer cleanupTempDir()
-
- destinationPath := filepath.Join(tempDir, "a", "new", "location")
-
- req := &gitalypb.RenameRepositoryRequest{Repository: testRepo, RelativePath: destinationPath}
+ req := &gitalypb.RenameRepositoryRequest{Repository: testRepo, RelativePath: "a-new-location"}
ctx, cancel := testhelper.Context()
defer cancel()
@@ -34,9 +29,10 @@ func TestRenameRepositorySuccess(t *testing.T) {
_, err := client.RenameRepository(ctx, req)
require.NoError(t, err)
- newDirectory, err := helper.GetPath(&gitalypb.Repository{StorageName: "default", RelativePath: destinationPath})
+ newDirectory, err := helper.GetPath(&gitalypb.Repository{StorageName: "default", RelativePath: req.RelativePath})
require.NoError(t, err)
require.DirExists(t, newDirectory)
+ defer func() { require.NoError(t, os.RemoveAll(newDirectory)) }()
require.True(t, helper.IsGitDirectory(newDirectory), "moved Git repository has been corrupted")
diff --git a/internal/service/repository/repack_test.go b/internal/service/repository/repack_test.go
index c35fe1769..b3b0d584d 100644
--- a/internal/service/repository/repack_test.go
+++ b/internal/service/repository/repack_test.go
@@ -50,12 +50,12 @@ func TestRepackIncrementalSuccess(t *testing.T) {
}
func TestRepackIncrementalCollectLogStatistics(t *testing.T) {
- defer func(tl func(tb testhelper.TB) *logrus.Logger) {
+ defer func(tl func(tb testing.TB) *logrus.Logger) {
testhelper.NewTestLogger = tl
}(testhelper.NewTestLogger)
logBuffer := &bytes.Buffer{}
- testhelper.NewTestLogger = func(tb testhelper.TB) *logrus.Logger {
+ testhelper.NewTestLogger = func(tb testing.TB) *logrus.Logger {
return &logrus.Logger{Out: logBuffer, Formatter: &logrus.JSONFormatter{}, Level: logrus.InfoLevel}
}
@@ -114,7 +114,7 @@ func TestRepackLocal(t *testing.T) {
packContents := testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "verify-pack", "-v", packFiles[0])
require.NotContains(t, string(packContents), string(altDirsCommit))
- require.Contains(t, string(packContents), string(repoCommit))
+ require.Contains(t, string(packContents), repoCommit)
}
func TestRepackIncrementalFailure(t *testing.T) {
@@ -199,12 +199,12 @@ func TestRepackFullSuccess(t *testing.T) {
}
func TestRepackFullCollectLogStatistics(t *testing.T) {
- defer func(tl func(tb testhelper.TB) *logrus.Logger) {
+ defer func(tl func(tb testing.TB) *logrus.Logger) {
testhelper.NewTestLogger = tl
}(testhelper.NewTestLogger)
logBuffer := &bytes.Buffer{}
- testhelper.NewTestLogger = func(tb testhelper.TB) *logrus.Logger {
+ testhelper.NewTestLogger = func(tb testing.TB) *logrus.Logger {
return &logrus.Logger{Out: logBuffer, Formatter: &logrus.JSONFormatter{}, Level: logrus.InfoLevel}
}
diff --git a/internal/service/repository/replicate.go b/internal/service/repository/replicate.go
index 35080fafc..3952bd242 100644
--- a/internal/service/repository/replicate.go
+++ b/internal/service/repository/replicate.go
@@ -271,7 +271,7 @@ func (s *server) getOrCreateConnection(address, token string) (*grpc.ClientConn,
connOpts := []grpc.DialOption{grpc.WithInsecure()}
if token != "" {
- connOpts = append(connOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(token)))
+ connOpts = append(connOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(token)))
}
cc, ok = s.connsByAddress[address]
diff --git a/internal/service/repository/replicate_test.go b/internal/service/repository/replicate_test.go
index 2dd9a8fbf..033cb42ff 100644
--- a/internal/service/repository/replicate_test.go
+++ b/internal/service/repository/replicate_test.go
@@ -25,7 +25,7 @@ import (
)
func TestReplicateRepository(t *testing.T) {
- tmpPath, cleanup := testhelper.TempDir(t, t.Name())
+ tmpPath, cleanup := testhelper.TempDir(t)
defer cleanup()
replicaPath := filepath.Join(tmpPath, "replica")
@@ -104,7 +104,7 @@ func TestReplicateRepository(t *testing.T) {
)
// if an unreachable object has been replicated, that means snapshot replication was used
- testhelper.MustRunCommand(t, nil, "git", "-C", targetRepoPath, "cat-file", "-p", string(blobID))
+ testhelper.MustRunCommand(t, nil, "git", "-C", targetRepoPath, "cat-file", "-p", blobID)
}
func TestReplicateRepositoryInvalidArguments(t *testing.T) {
@@ -197,7 +197,7 @@ func TestReplicateRepositoryInvalidArguments(t *testing.T) {
}
func TestReplicateRepository_BadRepository(t *testing.T) {
- tmpPath, cleanup := testhelper.TempDir(t, t.Name())
+ tmpPath, cleanup := testhelper.TempDir(t)
defer cleanup()
replicaPath := filepath.Join(tmpPath, "replica")
@@ -254,7 +254,7 @@ func TestReplicateRepository_BadRepository(t *testing.T) {
}
func TestReplicateRepository_FailedFetchInternalRemote(t *testing.T) {
- tmpPath, cleanup := testhelper.TempDir(t, t.Name())
+ tmpPath, cleanup := testhelper.TempDir(t)
defer cleanup()
replicaPath := filepath.Join(tmpPath, "replica")
diff --git a/internal/service/repository/size.go b/internal/service/repository/size.go
index 63adc13b9..b966f4bd8 100644
--- a/internal/service/repository/size.go
+++ b/internal/service/repository/size.go
@@ -8,7 +8,7 @@ import (
"os/exec"
"strconv"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/helper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
@@ -35,30 +35,30 @@ func (s *server) GetObjectDirectorySize(ctx context.Context, in *gitalypb.GetObj
func getPathSize(ctx context.Context, path string) int64 {
cmd, err := command.New(ctx, exec.Command("du", "-sk", path), nil, nil, nil)
if err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Warn("ignoring du command error")
+ ctxlogrus.Extract(ctx).WithError(err).Warn("ignoring du command error")
return 0
}
sizeLine, err := ioutil.ReadAll(cmd)
if err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Warn("ignoring command read error")
+ ctxlogrus.Extract(ctx).WithError(err).Warn("ignoring command read error")
return 0
}
if err := cmd.Wait(); err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Warn("ignoring du wait error")
+ ctxlogrus.Extract(ctx).WithError(err).Warn("ignoring du wait error")
return 0
}
sizeParts := bytes.Split(sizeLine, []byte("\t"))
if len(sizeParts) != 2 {
- grpc_logrus.Extract(ctx).Warn(fmt.Sprintf("ignoring du malformed output: %q", sizeLine))
+ ctxlogrus.Extract(ctx).Warn(fmt.Sprintf("ignoring du malformed output: %q", sizeLine))
return 0
}
size, err := strconv.ParseInt(string(sizeParts[0]), 10, 0)
if err != nil {
- grpc_logrus.Extract(ctx).WithError(err).Warn("ignoring parsing size error")
+ ctxlogrus.Extract(ctx).WithError(err).Warn("ignoring parsing size error")
return 0
}
diff --git a/internal/service/repository/snapshot.go b/internal/service/repository/snapshot.go
index 4cb4d2bea..acfd833af 100644
--- a/internal/service/repository/snapshot.go
+++ b/internal/service/repository/snapshot.go
@@ -7,7 +7,7 @@ import (
"path/filepath"
"regexp"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/archive"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/helper"
@@ -90,7 +90,7 @@ func addAlternateFiles(ctx context.Context, repository *gitalypb.Repository, bui
altObjDirs, err := git.AlternateObjectDirectories(ctx, storageRoot, repoPath)
if err != nil {
- grpc_logrus.Extract(ctx).WithField("error", err).Warn("error getting alternate object directories")
+ ctxlogrus.Extract(ctx).WithField("error", err).Warn("error getting alternate object directories")
return nil
}
diff --git a/internal/service/repository/testhelper_test.go b/internal/service/repository/testhelper_test.go
index 24101410d..ae61359dc 100644
--- a/internal/service/repository/testhelper_test.go
+++ b/internal/service/repository/testhelper_test.go
@@ -36,7 +36,7 @@ var (
func newRepositoryClient(t *testing.T, serverSocketPath string) (gitalypb.RepositoryServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(config.Config.Auth.Token)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.Config.Auth.Token)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
@@ -52,7 +52,7 @@ var RunRepoServer = runRepoServer
func newSecureRepoClient(t *testing.T, serverSocketPath string, pool *x509.CertPool) (gitalypb.RepositoryServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(pool, "")),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(config.Config.Auth.Token)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.Config.Auth.Token)),
}
conn, err := client.Dial(serverSocketPath, connOpts)
diff --git a/internal/service/server/disk_stats_test.go b/internal/service/server/disk_stats_test.go
index 2ba75bbd5..265c2fafa 100644
--- a/internal/service/server/disk_stats_test.go
+++ b/internal/service/server/disk_stats_test.go
@@ -12,7 +12,7 @@ import (
)
func TestStorageDiskStatistics(t *testing.T) {
- server, serverSocketPath := runServer(t)
+ server, serverSocketPath := runServer(t, config.Config.Storages)
defer server.Stop()
client, conn := newServerClient(t, serverSocketPath)
@@ -58,7 +58,7 @@ func getSpaceStats(t *testing.T, path string) (available int64, used int64) {
require.NoError(t, err)
// Redundant conversions to handle differences between unix families
- available = int64(stats.Bavail) * int64(stats.Bsize)
- used = (int64(stats.Blocks) - int64(stats.Bfree)) * int64(stats.Bsize)
+ available = int64(stats.Bavail) * int64(stats.Bsize) //nolint:unconvert
+ used = (int64(stats.Blocks) - int64(stats.Bfree)) * int64(stats.Bsize) //nolint:unconvert
return
}
diff --git a/internal/service/server/info.go b/internal/service/server/info.go
index 636a302ba..9218be54e 100644
--- a/internal/service/server/info.go
+++ b/internal/service/server/info.go
@@ -6,8 +6,7 @@ import (
"os"
"path"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
- "gitlab.com/gitlab-org/gitaly/internal/config"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/helper"
"gitlab.com/gitlab-org/gitaly/internal/helper/fstype"
@@ -23,13 +22,13 @@ func (s *server) ServerInfo(ctx context.Context, in *gitalypb.ServerInfoRequest)
}
var storageStatuses []*gitalypb.ServerInfoResponse_StorageStatus
- for _, shard := range config.Config.Storages {
+ for _, shard := range s.storages {
readable, writeable := shardCheck(shard.Path)
fsType := fstype.FileSystem(shard.Path)
gitalyMetadata, err := storage.ReadMetadataFile(shard)
if err != nil {
- grpc_logrus.Extract(ctx).WithField("storage", shard).WithError(err).Error("reading gitaly metadata file")
+ ctxlogrus.Extract(ctx).WithField("storage", shard).WithError(err).Error("reading gitaly metadata file")
}
storageStatuses = append(storageStatuses, &gitalypb.ServerInfoResponse_StorageStatus{
diff --git a/internal/service/server/info_test.go b/internal/service/server/info_test.go
index 9a2d636ef..817bd0967 100644
--- a/internal/service/server/info_test.go
+++ b/internal/service/server/info_test.go
@@ -21,7 +21,13 @@ import (
)
func TestGitalyServerInfo(t *testing.T) {
- server, serverSocketPath := runServer(t)
+ // Setup storage paths
+ testStorages := []config.Storage{
+ {Name: "default", Path: testhelper.GitlabTestStoragePath()},
+ {Name: "broken", Path: "/does/not/exist"},
+ }
+
+ server, serverSocketPath := runServer(t, testStorages)
defer server.Stop()
client, conn := newServerClient(t, serverSocketPath)
@@ -30,11 +36,6 @@ func TestGitalyServerInfo(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
- // Setup storage paths
- testStorages := []config.Storage{
- {Name: "default", Path: testhelper.GitlabTestStoragePath()},
- {Name: "broken", Path: "/does/not/exist"},
- }
defer func(oldStorages []config.Storage) {
config.Config.Storages = oldStorages
}(config.Config.Storages)
@@ -68,7 +69,7 @@ func TestGitalyServerInfo(t *testing.T) {
require.Equal(t, metadata.GitalyFilesystemID, c.GetStorageStatuses()[0].FilesystemId)
}
-func runServer(t *testing.T) (*grpc.Server, string) {
+func runServer(t *testing.T, storages []config.Storage) (*grpc.Server, string) {
authConfig := internalauth.Config{Token: testhelper.RepositoryAuthToken}
streamInt := []grpc.StreamServerInterceptor{auth.StreamServerInterceptor(authConfig)}
unaryInt := []grpc.UnaryServerInterceptor{auth.UnaryServerInterceptor(authConfig)}
@@ -81,7 +82,7 @@ func runServer(t *testing.T) (*grpc.Server, string) {
t.Fatal(err)
}
- gitalypb.RegisterServerServiceServer(server, NewServer())
+ gitalypb.RegisterServerServiceServer(server, NewServer(storages))
reflection.Register(server)
go server.Serve(listener)
@@ -90,7 +91,7 @@ func runServer(t *testing.T) (*grpc.Server, string) {
}
func TestServerNoAuth(t *testing.T) {
- srv, path := runServer(t)
+ srv, path := runServer(t, config.Config.Storages)
defer srv.Stop()
connOpts := []grpc.DialOption{
@@ -114,7 +115,7 @@ func TestServerNoAuth(t *testing.T) {
func newServerClient(t *testing.T, serverSocketPath string) (gitalypb.ServerServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
- grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(testhelper.RepositoryAuthToken)),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(testhelper.RepositoryAuthToken)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
diff --git a/internal/service/server/server.go b/internal/service/server/server.go
index 182803fe6..d80631a0b 100644
--- a/internal/service/server/server.go
+++ b/internal/service/server/server.go
@@ -1,12 +1,16 @@
package server
-import "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+import (
+ "gitlab.com/gitlab-org/gitaly/internal/config"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
+)
type server struct {
+ storages []config.Storage
gitalypb.UnimplementedServerServiceServer
}
// NewServer creates a new instance of a grpc ServerServiceServer
-func NewServer() gitalypb.ServerServiceServer {
- return &server{}
+func NewServer(storages []config.Storage) gitalypb.ServerServiceServer {
+ return &server{storages: storages}
}
diff --git a/internal/service/server/storage_status_unix.go b/internal/service/server/storage_status_unix.go
index f5b63fc31..71d1f7e24 100644
--- a/internal/service/server/storage_status_unix.go
+++ b/internal/service/server/storage_status_unix.go
@@ -16,8 +16,8 @@ func getStorageStatus(shard config.Storage) (*gitalypb.DiskStatisticsResponse_St
}
// Redundant conversions to handle differences between unix families
- available := int64(stats.Bavail) * int64(stats.Bsize)
- used := (int64(stats.Blocks) - int64(stats.Bfree)) * int64(stats.Bsize)
+ available := int64(stats.Bavail) * int64(stats.Bsize) //nolint:unconvert
+ used := (int64(stats.Blocks) - int64(stats.Bfree)) * int64(stats.Bsize) //nolint:unconvert
return &gitalypb.DiskStatisticsResponse_StorageStatus{
StorageName: shard.Name,
diff --git a/internal/service/smarthttp/cache.go b/internal/service/smarthttp/cache.go
index 7666c1af5..93314f473 100644
--- a/internal/service/smarthttp/cache.go
+++ b/internal/service/smarthttp/cache.go
@@ -5,11 +5,10 @@ import (
"io"
"sync"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/cache"
- "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@@ -45,18 +44,13 @@ func init() {
prometheus.MustRegister(hitMissTotals)
}
-// UploadPackCacheFeatureFlagKey enables cache usage in InfoRefsUploadPack RPC
-const UploadPackCacheFeatureFlagKey = "inforef-uploadpack-cache"
-
func tryCache(ctx context.Context, in *gitalypb.InfoRefsRequest, w io.Writer, missFn func(io.Writer) error) error {
- if !featureflag.IsEnabled(ctx, UploadPackCacheFeatureFlagKey) ||
- !featureflag.IsEnabled(ctx, featureflag.CacheInvalidator) ||
- len(in.GetGitConfigOptions()) > 0 ||
+ if len(in.GetGitConfigOptions()) > 0 ||
len(in.GetGitProtocol()) > 0 {
return missFn(w)
}
- logger := grpc_logrus.Extract(ctx).WithFields(log.Fields{"service": uploadPackSvc})
+ logger := ctxlogrus.Extract(ctx).WithFields(log.Fields{"service": uploadPackSvc})
logger.Debug("Attempting to fetch cached response")
countAttempt()
diff --git a/internal/service/smarthttp/inforefs.go b/internal/service/smarthttp/inforefs.go
index 1d3fe52f9..724afd736 100644
--- a/internal/service/smarthttp/inforefs.go
+++ b/internal/service/smarthttp/inforefs.go
@@ -5,7 +5,7 @@ import (
"fmt"
"io"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/pktline"
@@ -40,7 +40,7 @@ func (s *server) InfoRefsReceivePack(in *gitalypb.InfoRefsRequest, stream gitaly
}
func handleInfoRefs(ctx context.Context, service string, req *gitalypb.InfoRefsRequest, w io.Writer) error {
- grpc_logrus.Extract(ctx).WithFields(log.Fields{
+ ctxlogrus.Extract(ctx).WithFields(log.Fields{
"service": service,
}).Debug("handleInfoRefs")
diff --git a/internal/service/smarthttp/inforefs_test.go b/internal/service/smarthttp/inforefs_test.go
index 9606fb1c4..05bbc8649 100644
--- a/internal/service/smarthttp/inforefs_test.go
+++ b/internal/service/smarthttp/inforefs_test.go
@@ -24,7 +24,6 @@ import (
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
"google.golang.org/grpc/codes"
- "google.golang.org/grpc/metadata"
)
func TestSuccessfulInfoRefsUploadPack(t *testing.T) {
@@ -310,14 +309,8 @@ func TestCacheInfoRefsUploadPack(t *testing.T) {
)
}
- // if feature-flag is disabled, we should not find a cached response
assertNormalResponse()
- testhelper.AssertPathNotExists(t, pathToCachedResponse(t, rpcRequest))
-
- // enable feature flag, and we expect to find the cached response
- ctx = enableCacheFeatureFlag(ctx)
- assertNormalResponse()
- require.FileExists(t, pathToCachedResponse(t, rpcRequest))
+ require.FileExists(t, pathToCachedResponse(t, ctx, rpcRequest))
replacedContents := []string{
"first line",
@@ -327,24 +320,19 @@ func TestCacheInfoRefsUploadPack(t *testing.T) {
}
// replace cached response file to prove the info-ref uses the cache
- replaceCachedResponse(t, rpcRequest, strings.Join(replacedContents, "\n"))
+ replaceCachedResponse(t, ctx, rpcRequest, strings.Join(replacedContents, "\n"))
response, err := makeInfoRefsUploadPackRequest(ctx, t, serverSocketPath, rpcRequest)
require.NoError(t, err)
assertGitRefAdvertisement(t, "InfoRefsUploadPack", string(response),
replacedContents[0], replacedContents[3], replacedContents[1:3],
)
- // disable feature-flag to show replaced response no longer used
- ctx = context.Background()
- assertNormalResponse()
-
// invalidate cache for repository
ender, err := cache.LeaseKeyer{}.StartLease(rpcRequest.Repository)
require.NoError(t, err)
require.NoError(t, ender.EndLease(setInfoRefsUploadPackMethod(context.Background())))
// replaced cache response is no longer valid
- ctx = enableCacheFeatureFlag(ctx)
assertNormalResponse()
// failed requests should not cache response
@@ -359,7 +347,7 @@ func TestCacheInfoRefsUploadPack(t *testing.T) {
_, err = makeInfoRefsUploadPackRequest(ctx, t, serverSocketPath, invalidReq)
testhelper.RequireGrpcError(t, err, codes.Internal)
- testhelper.AssertPathNotExists(t, pathToCachedResponse(t, invalidReq))
+ testhelper.AssertPathNotExists(t, pathToCachedResponse(t, ctx, invalidReq))
}
func createInvalidRepo(t testing.TB, repo *gitalypb.Repository) func() {
@@ -371,18 +359,11 @@ func createInvalidRepo(t testing.TB, repo *gitalypb.Repository) func() {
return func() { require.NoError(t, os.RemoveAll(repoDir)) }
}
-func replaceCachedResponse(t testing.TB, req *gitalypb.InfoRefsRequest, newContents string) {
- path := pathToCachedResponse(t, req)
+func replaceCachedResponse(t testing.TB, ctx context.Context, req *gitalypb.InfoRefsRequest, newContents string) {
+ path := pathToCachedResponse(t, ctx, req)
require.NoError(t, ioutil.WriteFile(path, []byte(newContents), 0644))
}
-func enableCacheFeatureFlag(ctx context.Context) context.Context {
- return metadata.NewOutgoingContext(ctx, metadata.New(map[string]string{
- featureflag.HeaderKey(UploadPackCacheFeatureFlagKey): "true",
- featureflag.HeaderKey(featureflag.CacheInvalidator): "true",
- }))
-}
-
func clearCache(t testing.TB) {
for _, storage := range config.Config.Storages {
require.NoError(t, os.RemoveAll(tempdir.CacheDir(storage)))
@@ -393,8 +374,8 @@ func setInfoRefsUploadPackMethod(ctx context.Context) context.Context {
return testhelper.SetCtxGrpcMethod(ctx, "/gitaly.SmartHTTPService/InfoRefsUploadPack")
}
-func pathToCachedResponse(t testing.TB, req *gitalypb.InfoRefsRequest) string {
- ctx := setInfoRefsUploadPackMethod(context.Background())
+func pathToCachedResponse(t testing.TB, ctx context.Context, req *gitalypb.InfoRefsRequest) string {
+ ctx = setInfoRefsUploadPackMethod(ctx)
path, err := cache.LeaseKeyer{}.KeyPath(ctx, req.GetRepository(), req)
require.NoError(t, err)
return path
diff --git a/internal/service/smarthttp/receive_pack.go b/internal/service/smarthttp/receive_pack.go
index ed11ce77f..4a6ee992d 100644
--- a/internal/service/smarthttp/receive_pack.go
+++ b/internal/service/smarthttp/receive_pack.go
@@ -4,7 +4,7 @@ import (
"fmt"
"strconv"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/git"
@@ -23,7 +23,7 @@ func (s *server) PostReceivePack(stream gitalypb.SmartHTTPService_PostReceivePac
return err
}
- grpc_logrus.Extract(ctx).WithFields(log.Fields{
+ ctxlogrus.Extract(ctx).WithFields(log.Fields{
"GlID": req.GlId,
"GlRepository": req.GlRepository,
"GlUsername": req.GlUsername,
@@ -42,7 +42,11 @@ func (s *server) PostReceivePack(stream gitalypb.SmartHTTPService_PostReceivePac
return stream.Send(&gitalypb.PostReceivePackResponse{Data: p})
})
- env := append(git.HookEnv(req), "GL_PROTOCOL=http")
+ hookEnv, err := git.ReceivePackHookEnv(ctx, req)
+ if err != nil {
+ return err
+ }
+ env := append(hookEnv, "GL_PROTOCOL=http")
repoPath, err := helper.GetRepoPath(req.Repository)
if err != nil {
@@ -82,6 +86,9 @@ func validateReceivePackRequest(req *gitalypb.PostReceivePackRequest) error {
if req.Data != nil {
return status.Errorf(codes.InvalidArgument, "PostReceivePack: non-empty Data")
}
+ if req.Repository == nil {
+ return helper.ErrInvalidArgumentf("PostReceivePack: empty Repository")
+ }
return nil
}
diff --git a/internal/service/smarthttp/receive_pack_test.go b/internal/service/smarthttp/receive_pack_test.go
index a774b2656..d8c9e0a45 100644
--- a/internal/service/smarthttp/receive_pack_test.go
+++ b/internal/service/smarthttp/receive_pack_test.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/ioutil"
+ "net"
"os"
"path"
"path/filepath"
@@ -18,10 +19,14 @@ import (
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
"gitlab.com/gitlab-org/gitaly/internal/helper/text"
+ "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
+ hook "gitlab.com/gitlab-org/gitaly/internal/service/hooks"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/streamio"
+ "google.golang.org/grpc"
"google.golang.org/grpc/codes"
+ "google.golang.org/grpc/reflection"
)
func TestSuccessfulReceivePackRequest(t *testing.T) {
@@ -327,13 +332,34 @@ func TestInvalidTimezone(t *testing.T) {
testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "show", commit)
}
-func TestPostReceivePackToHooks(t *testing.T) {
+func drainPostReceivePackResponse(stream gitalypb.SmartHTTPService_PostReceivePackClient) error {
+ var err error
+ for err == nil {
+ _, err = stream.Recv()
+ }
+ return err
+}
+
+func TestPostReceivePackToHooks_WithRPC(t *testing.T) {
+ testPostReceivePackToHooks(t, true)
+}
+
+func TestPostReceivePackToHooks_WithoutRPC(t *testing.T) {
+ testPostReceivePackToHooks(t, false)
+}
+
+func testPostReceivePackToHooks(t *testing.T, callRPC bool) {
secretToken := "secret token"
glRepository := "some_repo"
glID := "key-123"
- socket, stop := runSmartHTTPServer(t)
- defer stop()
+ defer func(token string) {
+ config.Config.Auth.Token = token
+ }(config.Config.Auth.Token)
+ config.Config.Auth.Token = "abc123"
+
+ server, socket := runSmartHTTPHookServiceServer(t)
+ defer server.Stop()
client, conn := newSmartHTTPClient(t, "unix://"+socket)
defer conn.Close()
@@ -341,13 +367,16 @@ func TestPostReceivePackToHooks(t *testing.T) {
tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t)
defer cleanup()
- gitlabShellDir := config.Config.GitlabShell.Dir
- defer func() {
+ defer func(gitlabShellDir string) {
config.Config.GitlabShell.Dir = gitlabShellDir
- }()
-
+ }(config.Config.GitlabShell.Dir)
config.Config.GitlabShell.Dir = tempGitlabShellDir
+ defer func(override string) {
+ hooks.Override = override
+ }(hooks.Override)
+ hooks.Override = ""
+
repo, testRepoPath, cleanup := testhelper.NewTestRepo(t)
defer cleanup()
@@ -385,6 +414,10 @@ func TestPostReceivePackToHooks(t *testing.T) {
ctx, cancel := testhelper.Context()
defer cancel()
+ if callRPC {
+ ctx = featureflag.OutgoingCtxWithFeatureFlag(ctx, featureflag.HooksRPC)
+ }
+
stream, err := client.PostReceivePack(ctx)
require.NoError(t, err)
@@ -398,12 +431,28 @@ func TestPostReceivePackToHooks(t *testing.T) {
expectedResponse := "0030\x01000eunpack ok\n0019ok refs/heads/master\n00000000"
require.Equal(t, expectedResponse, string(response), "Expected response to be %q, got %q", expectedResponse, response)
+ require.Error(t, drainPostReceivePackResponse(stream), io.EOF)
}
-func drainPostReceivePackResponse(stream gitalypb.SmartHTTPService_PostReceivePackClient) error {
- var err error
- for err == nil {
- _, err = stream.Recv()
+func runSmartHTTPHookServiceServer(t *testing.T) (*grpc.Server, string) {
+ server := testhelper.NewTestGrpcServer(t, nil, nil)
+
+ serverSocketPath := testhelper.GetTemporaryGitalySocketFileName()
+ listener, err := net.Listen("unix", serverSocketPath)
+ if err != nil {
+ t.Fatal(err)
}
- return err
+ internalListener, err := net.Listen("unix", config.GitalyInternalSocketPath())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ gitalypb.RegisterSmartHTTPServiceServer(server, NewServer())
+ gitalypb.RegisterHookServiceServer(server, hook.NewServer())
+ reflection.Register(server)
+
+ go server.Serve(listener)
+ go server.Serve(internalListener)
+
+ return server, "unix://" + serverSocketPath
}
diff --git a/internal/service/smarthttp/testhelper_test.go b/internal/service/smarthttp/testhelper_test.go
index 1e867961e..08ec3309a 100644
--- a/internal/service/smarthttp/testhelper_test.go
+++ b/internal/service/smarthttp/testhelper_test.go
@@ -1,11 +1,18 @@
package smarthttp
import (
+ "log"
"os"
+ "path/filepath"
"testing"
"github.com/stretchr/testify/require"
+ gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
+ diskcache "gitlab.com/gitlab-org/gitaly/internal/cache"
+ "gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
+ "gitlab.com/gitlab-org/gitaly/internal/middleware/cache"
+ "gitlab.com/gitlab-org/gitaly/internal/praefect/protoregistry"
"gitlab.com/gitlab-org/gitaly/internal/testhelper"
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc"
@@ -24,13 +31,29 @@ func TestMain(m *testing.M) {
func testMain(m *testing.M) int {
hooks.Override = "/"
+ cwd, err := os.Getwd()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ config.Config.Ruby.Dir = filepath.Join(cwd, "../../../ruby")
+
testhelper.ConfigureGitalyHooksBinary()
return m.Run()
}
func runSmartHTTPServer(t *testing.T, serverOpts ...ServerOpt) (string, func()) {
- srv := testhelper.NewServer(t, nil, nil)
+ keyer := diskcache.LeaseKeyer{}
+
+ srv := testhelper.NewServer(t,
+ []grpc.StreamServerInterceptor{
+ cache.StreamInvalidator(keyer, protoregistry.GitalyProtoPreregistered),
+ },
+ []grpc.UnaryServerInterceptor{
+ cache.UnaryInvalidator(keyer, protoregistry.GitalyProtoPreregistered),
+ },
+ )
gitalypb.RegisterSmartHTTPServiceServer(srv.GrpcServer(), NewServer(serverOpts...))
reflection.Register(srv.GrpcServer())
@@ -43,6 +66,7 @@ func runSmartHTTPServer(t *testing.T, serverOpts ...ServerOpt) (string, func())
func newSmartHTTPClient(t *testing.T, serverSocketPath string) (gitalypb.SmartHTTPServiceClient, *grpc.ClientConn) {
connOpts := []grpc.DialOption{
grpc.WithInsecure(),
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.Config.Auth.Token)),
}
conn, err := grpc.Dial(serverSocketPath, connOpts...)
if err != nil {
diff --git a/internal/service/smarthttp/upload_pack.go b/internal/service/smarthttp/upload_pack.go
index ff513a42c..63854d72d 100644
--- a/internal/service/smarthttp/upload_pack.go
+++ b/internal/service/smarthttp/upload_pack.go
@@ -5,7 +5,7 @@ import (
"fmt"
"io"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/git"
"gitlab.com/gitlab-org/gitaly/internal/git/stats"
@@ -47,7 +47,7 @@ func (s *server) PostUploadPack(stream gitalypb.SmartHTTPService_PostUploadPackS
stats, err := stats.ParsePackfileNegotiation(pr)
if err != nil {
- grpc_logrus.Extract(stream.Context()).WithError(err).Debug("failed parsing packfile negotiation")
+ ctxlogrus.Extract(stream.Context()).WithError(err).Debug("failed parsing packfile negotiation")
return
}
stats.UpdateMetrics(s.packfileNegotiationMetrics)
@@ -109,7 +109,7 @@ func (s *server) PostUploadPack(stream gitalypb.SmartHTTPService_PostUploadPackS
return status.Errorf(codes.Unavailable, "PostUploadPack: %v", err)
}
- grpc_logrus.Extract(ctx).WithField("request_sha", fmt.Sprintf("%x", h.Sum(nil))).WithField("response_bytes", respBytes).Info("request details")
+ ctxlogrus.Extract(ctx).WithField("request_sha", fmt.Sprintf("%x", h.Sum(nil))).WithField("response_bytes", respBytes).Info("request details")
return nil
}
diff --git a/internal/service/ssh/receive_pack.go b/internal/service/ssh/receive_pack.go
index af03f68b8..b2aede673 100644
--- a/internal/service/ssh/receive_pack.go
+++ b/internal/service/ssh/receive_pack.go
@@ -1,10 +1,11 @@
package ssh
import (
+ "errors"
"fmt"
"strconv"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/git"
@@ -20,7 +21,7 @@ func (s *server) SSHReceivePack(stream gitalypb.SSHService_SSHReceivePackServer)
return helper.ErrInternal(err)
}
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"GlID": req.GlId,
"GlRepository": req.GlRepository,
"GlUsername": req.GlUsername,
@@ -53,7 +54,11 @@ func sshReceivePack(stream gitalypb.SSHService_SSHReceivePackServer, req *gitaly
return stream.Send(&gitalypb.SSHReceivePackResponse{Stderr: p})
})
- env := append(git.HookEnv(req), "GL_PROTOCOL=ssh")
+ hookEnv, err := git.ReceivePackHookEnv(ctx, req)
+ if err != nil {
+ return err
+ }
+ env := append(hookEnv, "GL_PROTOCOL=ssh")
repoPath, err := helper.GetRepoPath(req.Repository)
if err != nil {
@@ -93,10 +98,13 @@ func sshReceivePack(stream gitalypb.SSHService_SSHReceivePackServer, req *gitaly
func validateFirstReceivePackRequest(req *gitalypb.SSHReceivePackRequest) error {
if req.GlId == "" {
- return fmt.Errorf("empty GlId")
+ return errors.New("empty GlId")
}
if req.Stdin != nil {
- return fmt.Errorf("non-empty data in first request")
+ return errors.New("non-empty data in first request")
+ }
+ if req.Repository == nil {
+ return errors.New("repository is empty")
}
return nil
diff --git a/internal/service/ssh/testhelper_test.go b/internal/service/ssh/testhelper_test.go
index b9a437039..8ef0b490d 100644
--- a/internal/service/ssh/testhelper_test.go
+++ b/internal/service/ssh/testhelper_test.go
@@ -23,7 +23,6 @@ const (
var (
testRepo *gitalypb.Repository
gitalySSHPath string
- cwd string
)
func TestMain(m *testing.M) {
@@ -35,7 +34,6 @@ func testMain(m *testing.M) int {
defer testhelper.MustHaveNoChildProcess()
hooks.Override = "/"
- cwd = mustGetCwd()
err := os.RemoveAll(testPath)
if err != nil {
@@ -51,14 +49,6 @@ func testMain(m *testing.M) int {
return m.Run()
}
-func mustGetCwd() string {
- wd, err := os.Getwd()
- if err != nil {
- log.Panic(err)
- }
- return wd
-}
-
func runSSHServer(t *testing.T, serverOpts ...ServerOpt) (string, func()) {
srv := testhelper.NewServer(t, nil, nil)
diff --git a/internal/service/ssh/upload_pack.go b/internal/service/ssh/upload_pack.go
index 6731942f6..cf1a4c527 100644
--- a/internal/service/ssh/upload_pack.go
+++ b/internal/service/ssh/upload_pack.go
@@ -6,7 +6,7 @@ import (
"io"
"sync"
- grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
+ "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
log "github.com/sirupsen/logrus"
"gitlab.com/gitlab-org/gitaly/internal/command"
"gitlab.com/gitlab-org/gitaly/internal/git"
@@ -30,7 +30,7 @@ func (s *server) SSHUploadPack(stream gitalypb.SSHService_SSHUploadPackServer) e
repository = req.Repository.GlRepository
}
- grpc_logrus.Extract(stream.Context()).WithFields(log.Fields{
+ ctxlogrus.Extract(stream.Context()).WithFields(log.Fields{
"GlRepository": repository,
"GitConfigOptions": req.GitConfigOptions,
"GitProtocol": req.GitProtocol,
@@ -100,7 +100,7 @@ func (s *server) sshUploadPack(stream gitalypb.SSHService_SSHUploadPackServer, r
stats, err := stats.ParsePackfileNegotiation(pr)
if err != nil {
- grpc_logrus.Extract(stream.Context()).WithError(err).Debug("failed parsing packfile negotiation")
+ ctxlogrus.Extract(stream.Context()).WithError(err).Debug("failed parsing packfile negotiation")
return
}
stats.UpdateMetrics(s.packfileNegotiationMetrics)
diff --git a/internal/stream/std_stream.go b/internal/stream/std_stream.go
index 908f149c8..a6ba0f0a2 100644
--- a/internal/stream/std_stream.go
+++ b/internal/stream/std_stream.go
@@ -7,12 +7,17 @@ import (
"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
)
+// StdoutStderrResponse is an interface for RPC responses that need to stream stderr and stdout
type StdoutStderrResponse interface {
GetExitStatus() *gitalypb.ExitStatus
GetStderr() []byte
GetStdout() []byte
}
+// Sender is a function that sends input data to the stream
+type Sender func(chan error)
+
+// Handler takes care of sending and receiving to and from the stream
func Handler(recv func() (StdoutStderrResponse, error), send func(chan error), stdout, stderr io.Writer) (int32, error) {
var (
exitStatus int32
diff --git a/internal/testhelper/branch.go b/internal/testhelper/branch.go
index c8984e1a0..617a851ce 100644
--- a/internal/testhelper/branch.go
+++ b/internal/testhelper/branch.go
@@ -1,7 +1,9 @@
package testhelper
+import "testing"
+
// CreateRemoteBranch creates a new remote branch
-func CreateRemoteBranch(t TB, repoPath, remoteName, branchName, ref string) {
+func CreateRemoteBranch(t testing.TB, repoPath, remoteName, branchName, ref string) {
MustRunCommand(t, nil, "git", "-C", repoPath, "update-ref",
"refs/remotes/"+remoteName+"/"+branchName, ref)
}
diff --git a/internal/testhelper/commit.go b/internal/testhelper/commit.go
index a6b60131e..d8199882b 100644
--- a/internal/testhelper/commit.go
+++ b/internal/testhelper/commit.go
@@ -7,6 +7,7 @@ import (
"os/exec"
"path"
"strings"
+ "testing"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/helper/text"
@@ -24,7 +25,7 @@ const (
)
// CreateCommit makes a new empty commit and updates the named branch to point to it.
-func CreateCommit(t TB, repoPath, branchName string, opts *CreateCommitOpts) string {
+func CreateCommit(t testing.TB, repoPath, branchName string, opts *CreateCommitOpts) string {
message := "message"
// The ID of an arbitrary commit known to exist in the test repository.
parentID := "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863"
@@ -61,7 +62,7 @@ func CreateCommit(t TB, repoPath, branchName string, opts *CreateCommitOpts) str
// CreateCommitInAlternateObjectDirectory runs a command such that its created
// objects will live in an alternate objects directory. It returns the current
// head after the command is run and the alternate objects directory path
-func CreateCommitInAlternateObjectDirectory(t TB, repoPath, altObjectsDir string, cmd *exec.Cmd) (currentHead []byte) {
+func CreateCommitInAlternateObjectDirectory(t testing.TB, repoPath, altObjectsDir string, cmd *exec.Cmd) (currentHead []byte) {
gitPath := path.Join(repoPath, ".git")
altObjectsPath := path.Join(gitPath, altObjectsDir)
@@ -91,7 +92,7 @@ func CreateCommitInAlternateObjectDirectory(t TB, repoPath, altObjectsDir string
// specified name. This enables testing situations where the filepath is not
// possible due to filesystem constraints (e.g. non-UTF characters). The commit
// ID is returned.
-func CommitBlobWithName(t TB, testRepoPath, blobID, fileName, commitMessage string) string {
+func CommitBlobWithName(t testing.TB, testRepoPath, blobID, fileName, commitMessage string) string {
mktreeIn := strings.NewReader(fmt.Sprintf("100644 blob %s\t%s", blobID, fileName))
treeID := text.ChompBytes(MustRunCommand(t, mktreeIn, "git", "-C", testRepoPath, "mktree"))
@@ -104,7 +105,7 @@ func CommitBlobWithName(t TB, testRepoPath, blobID, fileName, commitMessage stri
}
// CreateCommitOnNewBranch creates a branch and a commit, returning the commit sha and the branch name respectivelyi
-func CreateCommitOnNewBranch(t TB, repoPath string) (string, string) {
+func CreateCommitOnNewBranch(t testing.TB, repoPath string) (string, string) {
nonce, err := text.RandomHex(4)
require.NoError(t, err)
newBranch := "branch-" + nonce
diff --git a/internal/testhelper/githttp.go b/internal/testhelper/githttp.go
index c694ed800..0c1416978 100644
--- a/internal/testhelper/githttp.go
+++ b/internal/testhelper/githttp.go
@@ -6,12 +6,13 @@ import (
"net/http"
"net/http/cgi"
"path/filepath"
+ "testing"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/config"
)
-func GitServer(t TB, repoPath string, middleware func(http.ResponseWriter, *http.Request, http.Handler)) (int, func() error) {
+func GitServer(t testing.TB, repoPath string, middleware func(http.ResponseWriter, *http.Request, http.Handler)) (int, func() error) {
require.NoError(t, ioutil.WriteFile(filepath.Join(repoPath, "git-daemon-export-ok"), nil, 0644))
listener, err := net.Listen("tcp", "127.0.0.1:0")
diff --git a/internal/testhelper/hook_env.go b/internal/testhelper/hook_env.go
index 8f5356d26..fd06c2019 100644
--- a/internal/testhelper/hook_env.go
+++ b/internal/testhelper/hook_env.go
@@ -2,11 +2,12 @@ package testhelper
import (
"io/ioutil"
+ "log"
"os"
"path"
"path/filepath"
+ "testing"
- log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/internal/config"
"gitlab.com/gitlab-org/gitaly/internal/git/hooks"
@@ -14,7 +15,7 @@ import (
// CaptureHookEnv creates a bogus 'update' Git hook to sniff out what
// environment variables get set for hooks.
-func CaptureHookEnv(t TB) (hookPath string, cleanup func()) {
+func CaptureHookEnv(t testing.TB) (hookPath string, cleanup func()) {
var err error
oldOverride := hooks.Override
hooks.Override, err = filepath.Abs("testdata/scratch/hooks")
@@ -37,11 +38,8 @@ env | grep -e ^GIT -e ^GL_ > `+hookOutputFile+"\n"), 0755))
// ConfigureGitalyHooksBinary builds gitaly-hooks command for tests
func ConfigureGitalyHooksBinary() {
- var err error
-
- config.Config.BinDir, err = filepath.Abs("testdata/gitaly-libexec")
- if err != nil {
- log.Fatal(err)
+ if config.Config.BinDir == "" {
+ log.Fatal("config.Config.BinDir must be set")
}
goBuildArgs := []string{
diff --git a/internal/testhelper/interface.go b/internal/testhelper/interface.go
deleted file mode 100644
index c86ece6f3..000000000
--- a/internal/testhelper/interface.go
+++ /dev/null
@@ -1,25 +0,0 @@
-package testhelper
-
-// TB is an interface that matches that of testing.TB.
-// Using TB rather than testing.TB prevents side effects from
-// importing testing package such as extra flags being added to the default
-// flag set. TB should be used in testing utilities that should be
-// importable by tests in other packages, namely anything that is not in a
-// *_test.go file.
-type TB interface {
- Error(args ...interface{})
- Errorf(format string, args ...interface{})
- Fail()
- FailNow()
- Failed() bool
- Fatal(args ...interface{})
- Fatalf(format string, args ...interface{})
- Helper()
- Log(args ...interface{})
- Logf(format string, args ...interface{})
- Name() string
- Skip(args ...interface{})
- SkipNow()
- Skipf(format string, args ...interface{})
- Skipped() bool
-}
diff --git a/internal/testhelper/interface_test.go b/internal/testhelper/interface_test.go
deleted file mode 100644
index 24b3a1122..000000000
--- a/internal/testhelper/interface_test.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package testhelper
-
-import (
- "testing"
-)
-
-// test that TB interface is a subset of testing.TB,
-// compiling fails if this is not the case.
-var _ TB = (testing.TB)(nil)
diff --git a/internal/testhelper/remote.go b/internal/testhelper/remote.go
index bdb483b2d..d5d1f199c 100644
--- a/internal/testhelper/remote.go
+++ b/internal/testhelper/remote.go
@@ -2,10 +2,11 @@ package testhelper
import (
"strings"
+ "testing"
)
// RemoteExists tests if the repository at repoPath has a Git remote named remoteName.
-func RemoteExists(t TB, repoPath string, remoteName string) bool {
+func RemoteExists(t testing.TB, repoPath string, remoteName string) bool {
if remoteName == "" {
t.Fatal("empty remote name")
}
diff --git a/internal/testhelper/tag.go b/internal/testhelper/tag.go
index 8d3fc72d6..3b94e84fd 100644
--- a/internal/testhelper/tag.go
+++ b/internal/testhelper/tag.go
@@ -3,6 +3,7 @@ package testhelper
import (
"bytes"
"fmt"
+ "testing"
"gitlab.com/gitlab-org/gitaly/internal/helper/text"
)
@@ -14,7 +15,7 @@ type CreateTagOpts struct {
}
// CreateTag creates a new tag.
-func CreateTag(t TB, repoPath, tagName, targetID string, opts *CreateTagOpts) string {
+func CreateTag(t testing.TB, repoPath, tagName, targetID string, opts *CreateTagOpts) string {
var message string
force := false
diff --git a/internal/testhelper/test_hook.go b/internal/testhelper/test_hook.go
index e28eb5e35..8d0b0264c 100644
--- a/internal/testhelper/test_hook.go
+++ b/internal/testhelper/test_hook.go
@@ -2,6 +2,7 @@ package testhelper
import (
"io/ioutil"
+ "testing"
log "github.com/sirupsen/logrus"
)
@@ -10,7 +11,7 @@ import (
var NewTestLogger = DiscardTestLogger
// DiscardTestLogger created a logrus hook that discards everything.
-func DiscardTestLogger(tb TB) *log.Logger {
+func DiscardTestLogger(tb testing.TB) *log.Logger {
logger := log.New()
logger.Out = ioutil.Discard
@@ -18,6 +19,6 @@ func DiscardTestLogger(tb TB) *log.Logger {
}
// DiscardTestLogger created a logrus entry that discards everything.
-func DiscardTestEntry(tb TB) *log.Entry {
+func DiscardTestEntry(tb testing.TB) *log.Entry {
return log.NewEntry(DiscardTestLogger(tb))
}
diff --git a/internal/testhelper/testdata/home/.gitconfig b/internal/testhelper/testdata/home/.gitconfig
new file mode 100644
index 000000000..29ace58d5
--- /dev/null
+++ b/internal/testhelper/testdata/home/.gitconfig
@@ -0,0 +1,3 @@
+[user]
+ email = you@example.com
+ name = Your Name
diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go
index dbf2b3497..bdc415e1c 100644
--- a/internal/testhelper/testhelper.go
+++ b/internal/testhelper/testhelper.go
@@ -19,6 +19,7 @@ import (
"strings"
"sync"
"syscall"
+ "testing"
"time"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
@@ -68,8 +69,17 @@ func Configure() {
config.Config.InternalSocketDir = dir
+ if err := os.MkdirAll("testdata/gitaly-libexec", 0755); err != nil {
+ log.Fatal(err)
+ }
+ config.Config.BinDir, err = filepath.Abs("testdata/gitaly-libexec")
+ if err != nil {
+ log.Fatal(err)
+ }
+
for _, f := range []func() error{
ConfigureRuby,
+ ConfigureGit,
config.Validate,
} {
if err := f(); err != nil {
@@ -82,7 +92,7 @@ func Configure() {
}
// MustReadFile returns the content of a file or fails at once.
-func MustReadFile(t TB, filename string) []byte {
+func MustReadFile(t testing.TB, filename string) []byte {
content, err := ioutil.ReadFile(filename)
if err != nil {
t.Fatal(err)
@@ -102,7 +112,7 @@ func GitlabTestStoragePath() string {
// GitalyServersMetadata returns a metadata pair for gitaly-servers to be used in
// inter-gitaly operations.
-func GitalyServersMetadata(t TB, serverSocketPath string) metadata.MD {
+func GitalyServersMetadata(t testing.TB, serverSocketPath string) metadata.MD {
gitalyServers := storage.GitalyServers{
"default": {
"address": serverSocketPath,
@@ -146,7 +156,7 @@ func TestRepository() *gitalypb.Repository {
}
// RequireGrpcError asserts the passed err is of the same code as expectedCode.
-func RequireGrpcError(t TB, err error, expectedCode codes.Code) {
+func RequireGrpcError(t testing.TB, err error, expectedCode codes.Code) {
if err == nil {
t.Fatal("Expected an error, got nil")
}
@@ -159,7 +169,7 @@ func RequireGrpcError(t TB, err error, expectedCode codes.Code) {
}
// MustRunCommand runs a command with an optional standard input and returns the standard output, or fails.
-func MustRunCommand(t TB, stdin io.Reader, name string, args ...string) []byte {
+func MustRunCommand(t testing.TB, stdin io.Reader, name string, args ...string) []byte {
cmd := exec.Command(name, args...)
if name == "git" {
@@ -271,7 +281,7 @@ func GetTemporaryGitalySocketFileName() string {
// GetLocalhostListener listens on the next available TCP port and returns
// the listener and the localhost address (host:port) string.
-func GetLocalhostListener(t TB) (net.Listener, string) {
+func GetLocalhostListener(t testing.TB) (net.Listener, string) {
l, err := net.Listen("tcp", "localhost:0")
require.NoError(t, err)
@@ -280,6 +290,32 @@ func GetLocalhostListener(t TB) (net.Listener, string) {
return l, addr
}
+// ConfigureGit configures git for test purpose
+func ConfigureGit() error {
+ _, currentFile, _, ok := runtime.Caller(0)
+ if !ok {
+ return fmt.Errorf("could not get caller info")
+ }
+
+ goenvCmd := exec.Command("go", "env", "GOCACHE")
+ goCacheBytes, err := goenvCmd.Output()
+ goCache := strings.TrimSpace(string(goCacheBytes))
+ if err != nil {
+ return err
+ }
+
+ // set GOCACHE env to current go cache location, otherwise if it's default it would be overwritten by setting HOME
+ err = os.Setenv("GOCACHE", goCache)
+ if err != nil {
+ return err
+ }
+
+ testHome := filepath.Join(filepath.Dir(currentFile), "testdata/home")
+
+ // overwrite HOME env variable so user global .gitconfig doesn't influence tests
+ return os.Setenv("HOME", testHome)
+}
+
// ConfigureRuby configures Ruby settings for test purposes at run time.
func ConfigureRuby() error {
if dir := os.Getenv("GITALY_TEST_RUBY_DIR"); len(dir) > 0 {
@@ -312,7 +348,7 @@ func GetGitEnvData() (string, error) {
}
// NewTestGrpcServer creates a GRPC Server for testing purposes
-func NewTestGrpcServer(tb TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor) *grpc.Server {
+func NewTestGrpcServer(tb testing.TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor) *grpc.Server {
logger := NewTestLogger(tb)
logrusEntry := log.NewEntry(logger).WithField("test", tb.Name())
@@ -385,7 +421,7 @@ func Context() (context.Context, func()) {
}
// CreateRepo creates a temporary directory for a repo, without initializing it
-func CreateRepo(t TB, storagePath, relativePath string) *gitalypb.Repository {
+func CreateRepo(t testing.TB, storagePath, relativePath string) *gitalypb.Repository {
require.NoError(t, os.MkdirAll(filepath.Dir(storagePath), 0755), "making repo parent dir")
return &gitalypb.Repository{
StorageName: "default",
@@ -395,16 +431,16 @@ func CreateRepo(t TB, storagePath, relativePath string) *gitalypb.Repository {
}
// InitBareRepo creates a new bare repository
-func InitBareRepo(t TB) (*gitalypb.Repository, string, func()) {
+func InitBareRepo(t testing.TB) (*gitalypb.Repository, string, func()) {
return initRepo(t, true)
}
// InitRepoWithWorktree creates a new repository with a worktree
-func InitRepoWithWorktree(t TB) (*gitalypb.Repository, string, func()) {
+func InitRepoWithWorktree(t testing.TB) (*gitalypb.Repository, string, func()) {
return initRepo(t, false)
}
-func initRepo(t TB, bare bool) (*gitalypb.Repository, string, func()) {
+func initRepo(t testing.TB, bare bool) (*gitalypb.Repository, string, func()) {
storagePath := GitlabTestStoragePath()
relativePath := NewRepositoryName(t, bare)
repoPath := filepath.Join(storagePath, relativePath)
@@ -421,21 +457,21 @@ func initRepo(t TB, bare bool) (*gitalypb.Repository, string, func()) {
repo.RelativePath = path.Join(repo.RelativePath, ".git")
}
- return repo, repoPath, func() { os.RemoveAll(repoPath) }
+ return repo, repoPath, func() { require.NoError(t, os.RemoveAll(repoPath)) }
}
// NewTestRepo creates a bare copy of the test repository.
-func NewTestRepo(t TB) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
+func NewTestRepo(t testing.TB) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
return cloneTestRepo(t, true)
}
// NewTestRepoWithWorktree creates a copy of the test repository with a
// worktree. This is allows you to run normal 'non-bare' Git commands.
-func NewTestRepoWithWorktree(t TB) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
+func NewTestRepoWithWorktree(t testing.TB) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
return cloneTestRepo(t, false)
}
-func cloneTestRepo(t TB, bare bool) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
+func cloneTestRepo(t testing.TB, bare bool) (repo *gitalypb.Repository, repoPath string, cleanup func()) {
storagePath := GitlabTestStoragePath()
relativePath := NewRepositoryName(t, bare)
repoPath = filepath.Join(storagePath, relativePath)
@@ -454,7 +490,7 @@ func cloneTestRepo(t TB, bare bool) (repo *gitalypb.Repository, repoPath string,
MustRunCommand(t, nil, "git", append(args, testRepoPath, repoPath)...)
- return repo, repoPath, func() { os.RemoveAll(repoPath) }
+ return repo, repoPath, func() { require.NoError(t, os.RemoveAll(repoPath)) }
}
// AddWorktreeArgs returns git command arguments for adding a worktree at the
@@ -464,17 +500,14 @@ func AddWorktreeArgs(repoPath, worktreeName string) []string {
}
// AddWorktree creates a worktree in the repository path for tests
-func AddWorktree(t TB, repoPath string, worktreeName string) {
+func AddWorktree(t testing.TB, repoPath string, worktreeName string) {
MustRunCommand(t, nil, "git", AddWorktreeArgs(repoPath, worktreeName)...)
}
// ConfigureGitalySSH configures the gitaly-ssh command for tests
func ConfigureGitalySSH() {
- var err error
-
- config.Config.BinDir, err = filepath.Abs("testdata/gitaly-libexec")
- if err != nil {
- log.Fatal(err)
+ if config.Config.BinDir == "" {
+ log.Fatal("config.Config.BinDir must be set")
}
goBuildArgs := []string{
@@ -487,14 +520,14 @@ func ConfigureGitalySSH() {
}
// GetRepositoryRefs gives a list of each repository ref as a string
-func GetRepositoryRefs(t TB, repoPath string) string {
+func GetRepositoryRefs(t testing.TB, repoPath string) string {
refs := MustRunCommand(t, nil, "git", "-C", repoPath, "for-each-ref")
return string(refs)
}
// AssertPathNotExists asserts true if the path doesn't exist, false otherwise
-func AssertPathNotExists(t TB, path string) {
+func AssertPathNotExists(t testing.TB, path string) {
_, err := os.Stat(path)
assert.True(t, os.IsNotExist(err), "file should not exist: %s", path)
}
@@ -502,7 +535,7 @@ func AssertPathNotExists(t TB, path string) {
// newDiskHash generates a random directory path following the Rails app's
// approach in the hashed storage module, formatted as '[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{64}'.
// https://gitlab.com/gitlab-org/gitlab/-/blob/f5c7d8eb1dd4eee5106123e04dec26d277ff6a83/app/models/storage/hashed.rb#L38-43
-func newDiskHash(t TB) string {
+func newDiskHash(t testing.TB) string {
// rails app calculates a sha256 and uses its hex representation
// as the directory path
b, err := text.RandomHex(sha256.Size)
@@ -512,7 +545,7 @@ func newDiskHash(t TB) string {
// NewRepositoryName returns a random repository hash
// in format '@hashed/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{64}(.git)?'.
-func NewRepositoryName(t TB, bare bool) string {
+func NewRepositoryName(t testing.TB, bare bool) string {
suffix := ""
if bare {
suffix = ".git"
@@ -523,47 +556,46 @@ func NewRepositoryName(t TB, bare bool) string {
// NewTestObjectPoolName returns a random pool repository name
// in format '@pools/[0-9a-z]{2}/[0-9a-z]{2}/[0-9a-z]{64}.git'.
-func NewTestObjectPoolName(t TB) string {
+func NewTestObjectPoolName(t testing.TB) string {
return filepath.Join("@pools", newDiskHash(t)+".git")
}
// CreateLooseRef creates a ref that points to master
-func CreateLooseRef(t TB, repoPath, refName string) {
+func CreateLooseRef(t testing.TB, repoPath, refName string) {
relRefPath := fmt.Sprintf("refs/heads/%s", refName)
MustRunCommand(t, nil, "git", "-C", repoPath, "update-ref", relRefPath, "master")
require.FileExists(t, filepath.Join(repoPath, relRefPath), "ref must be in loose file")
}
// TempDir is a wrapper around ioutil.TempDir that provides a cleanup function.
-// The returned temp directory will be created in the directory specified by
-// environment variable TEST_TEMP_DIR_PATH. If that variable is unset, the
-// relative folder "./testdata/tmp" to this source file will be used.
-func TempDir(t TB, prefix string) (string, func() error) {
+func TempDir(t testing.TB) (string, func()) {
_, currentFile, _, ok := runtime.Caller(0)
if !ok {
log.Fatal("Could not get caller info")
}
rootTmpDir := path.Join(path.Dir(currentFile), "testdata/tmp")
- dirPath, err := ioutil.TempDir(rootTmpDir, prefix)
+ tmpDir, err := ioutil.TempDir(rootTmpDir, "")
require.NoError(t, err)
- return dirPath, func() error {
- return os.RemoveAll(dirPath)
- }
+ return tmpDir, func() { require.NoError(t, os.RemoveAll(tmpDir)) }
}
// GitObjectMustExist is a test assertion that fails unless the git repo in repoPath contains sha
-func GitObjectMustExist(t TB, repoPath, sha string) {
+func GitObjectMustExist(t testing.TB, repoPath, sha string) {
gitObjectExists(t, repoPath, sha, true)
}
// GitObjectMustNotExist is a test assertion that fails unless the git repo in repoPath contains sha
-func GitObjectMustNotExist(t TB, repoPath, sha string) {
+func GitObjectMustNotExist(t testing.TB, repoPath, sha string) {
gitObjectExists(t, repoPath, sha, false)
}
-func gitObjectExists(t TB, repoPath, sha string, exists bool) {
+func gitObjectExists(t testing.TB, repoPath, sha string, exists bool) {
cmd := exec.Command("git", "-C", repoPath, "cat-file", "-e", sha)
+ cmd.Env = []string{
+ "GIT_ALLOW_PROTOCOL=", // To prevent partial clone reaching remote repo over SSH
+ }
+
if exists {
require.NoError(t, cmd.Run(), "checking for object should succeed")
return
@@ -576,16 +608,16 @@ func gitObjectExists(t TB, repoPath, sha string, exists bool) {
type Cleanup func()
// GetGitObjectDirSize gets the number of 1k blocks of a git object directory
-func GetGitObjectDirSize(t TB, repoPath string) int64 {
+func GetGitObjectDirSize(t testing.TB, repoPath string) int64 {
return getGitDirSize(t, repoPath, "objects")
}
// GetGitPackfileDirSize gets the number of 1k blocks of a git object directory
-func GetGitPackfileDirSize(t TB, repoPath string) int64 {
+func GetGitPackfileDirSize(t testing.TB, repoPath string) int64 {
return getGitDirSize(t, repoPath, "objects", "pack")
}
-func getGitDirSize(t TB, repoPath string, subdirs ...string) int64 {
+func getGitDirSize(t testing.TB, repoPath string, subdirs ...string) int64 {
cmd := exec.Command("du", "-s", "-k", filepath.Join(append([]string{repoPath}, subdirs...)...))
output, err := cmd.Output()
require.NoError(t, err)
@@ -609,7 +641,7 @@ func GrpcErrorHasMessage(grpcError error, msg string) bool {
}
// dump the env vars that the custom hooks receives to a file
-func WriteEnvToCustomHook(t TB, repoPath, hookName string) (string, func()) {
+func WriteEnvToCustomHook(t testing.TB, repoPath, hookName string) (string, func()) {
hookOutputTemp, err := ioutil.TempFile("", "")
require.NoError(t, err)
require.NoError(t, hookOutputTemp.Close())
diff --git a/internal/testhelper/testserver.go b/internal/testhelper/testserver.go
index b2fa98fb7..efdf1dbe5 100644
--- a/internal/testhelper/testserver.go
+++ b/internal/testhelper/testserver.go
@@ -14,9 +14,11 @@ import (
"path/filepath"
"regexp"
"strings"
+ "testing"
"time"
"github.com/BurntSushi/toml"
+ "github.com/golang/protobuf/jsonpb"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags"
@@ -30,9 +32,9 @@ import (
praefectconfig "gitlab.com/gitlab-org/gitaly/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/internal/praefect/models"
serverauth "gitlab.com/gitlab-org/gitaly/internal/server/auth"
+ "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
- "google.golang.org/grpc/health/grpc_health_v1"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
"gopkg.in/yaml.v2"
)
@@ -75,7 +77,7 @@ func NewTestServer(srv *grpc.Server, opts ...TestServerOpt) *TestServer {
}
// NewServerWithAuth creates a new test server with authentication
-func NewServerWithAuth(tb TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor, token string, opts ...TestServerOpt) *TestServer {
+func NewServerWithAuth(tb testing.TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor, token string, opts ...TestServerOpt) *TestServer {
if token != "" {
if PraefectEnabled() {
opts = append(opts, WithToken(token))
@@ -201,7 +203,7 @@ func (p *TestServer) Start() error {
opts := []grpc.DialOption{grpc.WithInsecure()}
if p.token != "" {
- opts = append(opts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(p.token)))
+ opts = append(opts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(p.token)))
}
conn, err := grpc.Dial("unix://"+praefectServerSocketPath, opts...)
@@ -239,7 +241,7 @@ func waitForPraefectStartup(conn *grpc.ClientConn) error {
}
// NewServer creates a Server for testing purposes
-func NewServer(tb TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor, opts ...TestServerOpt) *TestServer {
+func NewServer(tb testing.TB, streamInterceptors []grpc.StreamServerInterceptor, unaryInterceptors []grpc.UnaryServerInterceptor, opts ...TestServerOpt) *TestServer {
logger := NewTestLogger(tb)
logrusEntry := log.NewEntry(logger).WithField("test", tb.Name())
@@ -261,7 +263,7 @@ func NewServer(tb TB, streamInterceptors []grpc.StreamServerInterceptor, unaryIn
var changeLineRegex = regexp.MustCompile("^[a-f0-9]{40} [a-f0-9]{40} refs/[^ ]+$")
-func handleAllowed(t TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
+func handleAllowed(t testing.TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, r.ParseForm())
require.Equal(t, http.MethodPost, r.Method, "expected http post")
@@ -333,7 +335,7 @@ func handleAllowed(t TB, options GitlabTestServerOptions) func(w http.ResponseWr
}
}
-func handlePreReceive(t TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
+func handlePreReceive(t testing.TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, r.ParseForm())
require.Equal(t, http.MethodPost, r.Method)
@@ -350,7 +352,7 @@ func handlePreReceive(t TB, options GitlabTestServerOptions) func(w http.Respons
}
}
-func handlePostReceive(t TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
+func handlePostReceive(t testing.TB, options GitlabTestServerOptions) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
require.NoError(t, r.ParseForm())
require.Equal(t, http.MethodPost, r.Method)
@@ -409,7 +411,7 @@ type GitlabTestServerOptions struct {
}
// NewGitlabTestServer returns a mock gitlab server that responds to the hook api endpoints
-func NewGitlabTestServer(t TB, options GitlabTestServerOptions) *httptest.Server {
+func NewGitlabTestServer(t testing.TB, options GitlabTestServerOptions) *httptest.Server {
mux := http.NewServeMux()
mux.Handle("/api/v4/internal/allowed", http.HandlerFunc(handleAllowed(t, options)))
mux.Handle("/api/v4/internal/pre_receive", http.HandlerFunc(handlePreReceive(t, options)))
@@ -421,7 +423,7 @@ func NewGitlabTestServer(t TB, options GitlabTestServerOptions) *httptest.Server
// CreateTemporaryGitlabShellDir creates a temporary gitlab shell directory. It returns the path to the directory
// and a cleanup function
-func CreateTemporaryGitlabShellDir(t TB) (string, func()) {
+func CreateTemporaryGitlabShellDir(t testing.TB) (string, func()) {
tempDir, err := ioutil.TempDir("", "gitlab-shell")
require.NoError(t, err)
return tempDir, func() {
@@ -431,7 +433,7 @@ func CreateTemporaryGitlabShellDir(t TB) (string, func()) {
// WriteTemporaryGitlabShellConfigFile writes a gitlab shell config.yml in a temporary directory. It returns the path
// and a cleanup function
-func WriteTemporaryGitlabShellConfigFile(t TB, dir string, config GitlabShellConfig) (string, func()) {
+func WriteTemporaryGitlabShellConfigFile(t testing.TB, dir string, config GitlabShellConfig) (string, func()) {
out, err := yaml.Marshal(&config)
require.NoError(t, err)
@@ -445,7 +447,7 @@ func WriteTemporaryGitlabShellConfigFile(t TB, dir string, config GitlabShellCon
// WriteTemporaryGitalyConfigFile writes a gitaly toml file into a temporary directory. It returns the path to
// the file as well as a cleanup function
-func WriteTemporaryGitalyConfigFile(t TB, tempDir string) (string, func()) {
+func WriteTemporaryGitalyConfigFile(t testing.TB, tempDir string) (string, func()) {
path := filepath.Join(tempDir, "config.toml")
contents := fmt.Sprintf(`
[gitlab-shell]
@@ -463,11 +465,16 @@ type GlHookValues struct {
GitAlternateObjectDirs []string
}
+var jsonpbMarshaller jsonpb.Marshaler
+
// EnvForHooks generates a set of environment variables for gitaly hooks
-func EnvForHooks(t TB, gitlabShellDir string, glHookValues GlHookValues, gitPushOptions ...string) []string {
+func EnvForHooks(t testing.TB, gitlabShellDir, gitalySocket, gitalyToken string, repo *gitalypb.Repository, glHookValues GlHookValues, gitPushOptions ...string) []string {
rubyDir, err := filepath.Abs("../../ruby")
require.NoError(t, err)
+ repoString, err := jsonpbMarshaller.MarshalToString(repo)
+ require.NoError(t, err)
+
env := append(append([]string{
fmt.Sprintf("GITALY_BIN_DIR=%s", config.Config.BinDir),
fmt.Sprintf("GITALY_RUBY_DIR=%s", rubyDir),
@@ -475,6 +482,9 @@ func EnvForHooks(t TB, gitlabShellDir string, glHookValues GlHookValues, gitPush
fmt.Sprintf("GL_REPOSITORY=%s", glHookValues.GLRepo),
fmt.Sprintf("GL_PROTOCOL=%s", glHookValues.GLProtocol),
fmt.Sprintf("GL_USERNAME=%s", glHookValues.GLUsername),
+ fmt.Sprintf("GITALY_SOCKET=%s", gitalySocket),
+ fmt.Sprintf("GITALY_TOKEN=%s", gitalyToken),
+ fmt.Sprintf("GITALY_REPO=%v", repoString),
fmt.Sprintf("GITALY_GITLAB_SHELL_DIR=%s", gitlabShellDir),
fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir),
"GITALY_LOG_LEVEL=info",
@@ -492,7 +502,7 @@ func EnvForHooks(t TB, gitlabShellDir string, glHookValues GlHookValues, gitPush
}
// WriteShellSecretFile writes a .gitlab_shell_secret file in the specified directory
-func WriteShellSecretFile(t TB, dir, secretToken string) {
+func WriteShellSecretFile(t testing.TB, dir, secretToken string) {
require.NoError(t, ioutil.WriteFile(filepath.Join(dir, ".gitlab_shell_secret"), []byte(secretToken), 0644))
}
@@ -508,11 +518,11 @@ type HTTPSettings struct {
Password string `yaml:"password"`
}
-func NewServerWithHealth(t TB, socketName string) (*grpc.Server, *health.Server) {
+func NewServerWithHealth(t testing.TB, socketName string) (*grpc.Server, *health.Server) {
srv := NewTestGrpcServer(t, nil, nil)
healthSrvr := health.NewServer()
- grpc_health_v1.RegisterHealthServer(srv, healthSrvr)
- healthSrvr.SetServingStatus("", grpc_health_v1.HealthCheckResponse_SERVING)
+ healthpb.RegisterHealthServer(srv, healthSrvr)
+ healthSrvr.SetServingStatus("", healthpb.HealthCheckResponse_SERVING)
lis, err := net.Listen("unix", socketName)
require.NoError(t, err)
@@ -522,7 +532,7 @@ func NewServerWithHealth(t TB, socketName string) (*grpc.Server, *health.Server)
return srv, healthSrvr
}
-func SetupAndStartGitlabServer(t TB, c *GitlabTestServerOptions) func() {
+func SetupAndStartGitlabServer(t testing.TB, c *GitlabTestServerOptions) func() {
ts := NewGitlabTestServer(t, *c)
WriteTemporaryGitlabShellConfigFile(t, config.Config.GitlabShell.Dir, GitlabShellConfig{GitlabURL: ts.URL})
diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock
index acafd8f43..508f8f895 100644
--- a/ruby/Gemfile.lock
+++ b/ruby/Gemfile.lock
@@ -140,7 +140,7 @@ GEM
optimist (>= 3.0.0)
rdoc (6.2.0)
redis (4.1.3)
- rouge (3.17.0)
+ rouge (3.18.0)
rspec (3.8.0)
rspec-core (~> 3.8.0)
rspec-expectations (~> 3.8.0)
diff --git a/ruby/gitlab-shell/lib/gitlab_config.rb b/ruby/gitlab-shell/lib/gitlab_config.rb
index 19f03c346..9020ff7f9 100644
--- a/ruby/gitlab-shell/lib/gitlab_config.rb
+++ b/ruby/gitlab-shell/lib/gitlab_config.rb
@@ -2,44 +2,46 @@ require 'yaml'
class GitlabConfig
def secret_file
- fetch_from_legacy_config('secret_file',File.join(ROOT_PATH, '.gitlab_shell_secret'))
+ fetch_from_config('secret_file', fetch_from_legacy_config('secret_file', File.join(ROOT_PATH, '.gitlab_shell_secret')))
end
# Pass a default value because this is called from a repo's context; in which
# case, the repo's hooks directory should be the default.
#
def custom_hooks_dir(default: nil)
- fetch_from_legacy_config('custom_hooks_dir', File.join(ROOT_PATH, 'hooks'))
+ fetch_from_config('custom_hooks_dir', fetch_from_legacy_config('custom_hooks_dir', File.join(ROOT_PATH, 'hooks')))
end
def gitlab_url
- fetch_from_legacy_config('gitlab_url',"http://localhost:8080").sub(%r{/*$}, '')
+ fetch_from_config('gitlab_url', fetch_from_legacy_config('gitlab_url',"http://localhost:8080").sub(%r{/*$}, ''))
end
def http_settings
- fetch_from_legacy_config('http_settings', {})
+ fetch_from_config('http_settings', fetch_from_legacy_config('http_settings', {}))
end
def log_file
- return File.join(LOG_PATH, 'gitlab-shell.log') unless LOG_PATH.empty?
+ log_path = Pathname.new(fetch_from_config('log_path', LOG_PATH))
- fetch_from_legacy_config('log_file', File.join(ROOT_PATH, 'gitlab-shell.log'))
+ log_path = ROOT_PATH if log_path === ''
+
+ return log_path.join('gitlab-shell.log')
end
def log_level
- return LOG_LEVEL unless LOG_LEVEL.empty?
+ log_level = fetch_from_config('log_level', LOG_LEVEL)
+
+ return log_level unless log_level.empty?
- fetch_from_legacy_config('log_level', 'INFO')
+ 'INFO'
end
def log_format
- return LOG_FORMAT unless LOG_FORMAT.empty?
+ log_format = fetch_from_config('log_format', LOG_FORMAT)
- fetch_from_legacy_config('log_format', 'text')
- end
+ return log_format unless log_format.empty?
- def metrics_log_file
- fetch_from_legacy_config('metrics_log_file', File.join(ROOT_PATH, 'gitlab-shell-metrics.log'))
+ 'text'
end
def to_json
@@ -51,7 +53,6 @@ class GitlabConfig
log_file: log_file,
log_level: log_level,
log_format: log_format,
- metrics_log_file: metrics_log_file
}.to_json
end
@@ -61,8 +62,23 @@ class GitlabConfig
private
+ def fetch_from_config(key, default)
+ value = config[key]
+
+ return default if value.nil? || value.empty?
+
+ value
+ end
+
+ def config
+ @config ||= JSON.parse(ENV.fetch('GITALY_GITLAB_SHELL_CONFIG', '{}'))
+ end
+
def legacy_config
# TODO: deprecate @legacy_config that is parsing the gitlab-shell config.yml
- @legacy_config ||= YAML.load_file(File.join(ROOT_PATH, 'config.yml'))
+ legacy_file = ROOT_PATH.join('config.yml')
+ return {} unless legacy_file.exist?
+
+ @legacy_config ||= YAML.load_file(legacy_file)
end
end
diff --git a/ruby/gitlab-shell/lib/gitlab_init.rb b/ruby/gitlab-shell/lib/gitlab_init.rb
index ce54e2d45..949135493 100644
--- a/ruby/gitlab-shell/lib/gitlab_init.rb
+++ b/ruby/gitlab-shell/lib/gitlab_init.rb
@@ -1,8 +1,10 @@
# GITLAB_SHELL_DIR has been deprecated
-ROOT_PATH = ENV['GITALY_GITLAB_SHELL_DIR'] || ENV['GITLAB_SHELL_DIR'] || File.expand_path('..', __dir__)
-LOG_PATH = ENV.fetch('GITALY_LOG_DIR', "")
-LOG_LEVEL = ENV.fetch('GITALY_LOG_LEVEL', "")
-LOG_FORMAT = ENV.fetch('GITALY_LOG_FORMAT', "")
+require 'pathname'
+
+ROOT_PATH = Pathname.new(ENV['GITALY_GITLAB_SHELL_DIR'] || ENV['GITLAB_SHELL_DIR'] || File.expand_path('..', __dir__)).freeze
+LOG_PATH = Pathname.new(ENV.fetch('GITALY_LOG_DIR', ROOT_PATH))
+LOG_LEVEL = ENV.fetch('GITALY_LOG_LEVEL', 'INFO')
+LOG_FORMAT = ENV.fetch('GITALY_LOG_FORMAT', 'text')
# We are transitioning parts of gitlab-shell into the gitaly project. In
# gitaly, GITALY_EMBEDDED will be true.
diff --git a/ruby/gitlab-shell/lib/gitlab_net.rb b/ruby/gitlab-shell/lib/gitlab_net.rb
index 74996e712..812be9ae8 100644
--- a/ruby/gitlab-shell/lib/gitlab_net.rb
+++ b/ruby/gitlab-shell/lib/gitlab_net.rb
@@ -41,40 +41,10 @@ class GitlabNet # rubocop:disable Metrics/ClassLength
GitAccessStatus.new(false, resp.code, API_INACCESSIBLE_MESSAGE)
end
- def broadcast_message
- resp = get("#{internal_api_endpoint}/broadcast_message")
- JSON.parse(resp.body) rescue {}
- end
-
- def merge_request_urls(gl_repository, repo_path, changes)
- changes = changes.join("\n") unless changes.is_a?(String)
- changes = changes.encode('UTF-8', 'ASCII', invalid: :replace, replace: '')
- url = "#{internal_api_endpoint}/merge_request_urls?project=#{URI.escape(repo_path)}&changes=#{URI.escape(changes)}"
- url += "&gl_repository=#{URI.escape(gl_repository)}" if gl_repository
- resp = get(url)
-
- if resp.code == '200'
- JSON.parse(resp.body)
- else
- []
- end
- rescue
- []
- end
-
def check
get("#{internal_api_endpoint}/check", options: { read_timeout: CHECK_TIMEOUT })
end
- def notify_post_receive(gl_repository, repo_path)
- params = { gl_repository: gl_repository, project: repo_path }
- resp = post("#{internal_api_endpoint}/notify_post_receive", params)
-
- resp.code == '200'
- rescue
- false
- end
-
def post_receive(gl_repository, gl_id, changes, push_options)
params = {
gl_repository: gl_repository,
diff --git a/ruby/gitlab-shell/spec/gitlab_config_spec.rb b/ruby/gitlab-shell/spec/gitlab_config_spec.rb
index 6de0cd08e..b0f12381c 100644
--- a/ruby/gitlab-shell/spec/gitlab_config_spec.rb
+++ b/ruby/gitlab-shell/spec/gitlab_config_spec.rb
@@ -1,54 +1,112 @@
require_relative 'spec_helper'
require_relative '../lib/gitlab_config'
+require 'json'
describe GitlabConfig do
let(:config) { GitlabConfig.new }
- let(:config_data) { {} }
- before { expect(YAML).to receive(:load_file).and_return(config_data) }
+ context "without ENV['GITALY_GITLAB_SHELL_CONFIG'] is passed in" do
+ let(:config_data) { {} }
- describe '#gitlab_url' do
- let(:url) { 'http://test.com' }
+ before { expect(YAML).to receive(:load_file).and_return(config_data) }
- subject { config.gitlab_url }
+ describe '#gitlab_url' do
+ let(:url) { 'http://test.com' }
- before { config_data['gitlab_url'] = url }
+ subject { config.gitlab_url }
- it { is_expected.not_to be_empty }
- it { is_expected.to eq(url) }
-
- context 'remove trailing slashes' do
- before { config_data['gitlab_url'] = url + '//' }
+ before { config_data['gitlab_url'] = url }
+ it { is_expected.not_to be_empty }
it { is_expected.to eq(url) }
+
+ context 'remove trailing slashes' do
+ before { config_data['gitlab_url'] = url + '//' }
+
+ it { is_expected.to eq(url) }
+ end
end
- end
- describe '#log_format' do
- subject { config.log_format }
+ describe '#secret_file' do
+ subject { config.secret_file }
- it 'returns "text" by default' do
- is_expected.to eq('text')
+ it 'returns ".gitlab_shell_secret" by default' do
+ is_expected.to eq(File.join(File.expand_path('..', __dir__),'.gitlab_shell_secret'))
+ end
+ end
+
+ describe '#fetch_from_legacy_config' do
+ let(:key) { 'yaml_key' }
+
+ where(:yaml_value, :default, :expected_value) do
+ [
+ ['a', 'b', 'a'],
+ [nil, 'b', 'b'],
+ ['a', nil, 'a'],
+ [nil, {}, {}]
+ ]
+ end
+
+ with_them do
+ it 'returns the correct value' do
+ config_data[key] = yaml_value
+
+ expect(config.fetch_from_legacy_config(key, default)).to eq(expected_value)
+ end
+ end
end
end
- describe '#fetch_from_legacy_config' do
- let(:key) { 'yaml_key' }
+ context "when ENV['GITALY_GITLAB_SHELL_CONFIG'] is passed in" do
+ let(:config_data) { {'secret_file': 'path/to/secret/file',
+ 'custom_hooks_dir': '/path/to/custom_hooks',
+ 'gitlab_url': 'http://localhost:123454',
+ 'http_settings': {'user': 'user_123', 'password':'password123', 'ca_file': '/path/to/ca_file', 'ca_path': 'path/to/ca_path'},
+ 'log_path': '/path/to/log',
+ 'log_level': 'myloglevel',
+ 'log_format': 'log_format'} }
+
+ before { allow(ENV).to receive(:fetch).with('GITALY_GITLAB_SHELL_CONFIG', '{}').and_return(config_data.to_json) }
- where(:yaml_value, :default, :expected_value) do
- [
- ['a', 'b', 'a'],
- [nil, 'b', 'b'],
- ['a', nil, 'a'],
- [nil, {}, {}]
- ]
+ describe '#secret_file' do
+ it 'returns the correct secret_file' do
+ expect(config.secret_file).to eq(config_data[:secret_file])
+ end
end
- with_them do
- it 'returns the correct value' do
- config_data[key] = yaml_value
+ describe '#custom_hooks_dir' do
+ it 'returns the correct custom_hooks_dir' do
+ expect(config.custom_hooks_dir).to eq(config_data[:custom_hooks_dir])
+ end
+ end
+
+ describe '#http_settings' do
+ it 'returns the correct http_settings' do
+ expect(config.http_settings).to eq(config_data[:http_settings].transform_keys(&:to_s))
+ end
+ end
+
+ describe '#gitlab_url' do
+ it 'returns the correct gitlab_url' do
+ expect(config.gitlab_url).to eq(config_data[:gitlab_url])
+ end
+ end
+
+ describe '#log_path' do
+ it 'returns the correct log_path' do
+ expect(config.log_file).to eq(Pathname.new(File.join(config_data[:log_path], 'gitlab-shell.log')))
+ end
+ end
+
+ describe '#log_level' do
+ it 'returns the correct log_level' do
+ expect(config.log_level).to eq(config_data[:log_level])
+ end
+ end
- expect(config.fetch_from_legacy_config(key, default)).to eq(expected_value)
+ describe '#log_format' do
+ it 'returns the correct log_format' do
+ expect(config.log_format).to eq(config_data[:log_format])
end
end
end
diff --git a/ruby/gitlab-shell/spec/gitlab_custom_hook_spec.rb b/ruby/gitlab-shell/spec/gitlab_custom_hook_spec.rb
index 50865331b..c19b90f87 100644
--- a/ruby/gitlab-shell/spec/gitlab_custom_hook_spec.rb
+++ b/ruby/gitlab-shell/spec/gitlab_custom_hook_spec.rb
@@ -99,7 +99,7 @@ describe GitlabCustomHook do
FileUtils.symlink(File.join(tmp_root_path, 'hooks'), File.join(tmp_repo_path, 'hooks'))
FileUtils.symlink(File.join(ROOT_PATH, 'config.yml.example'), File.join(tmp_root_path, 'config.yml'))
- stub_const('ROOT_PATH', tmp_root_path)
+ stub_const('ROOT_PATH', Pathname.new(tmp_root_path))
end
after do
diff --git a/ruby/gitlab-shell/spec/gitlab_logger_spec.rb b/ruby/gitlab-shell/spec/gitlab_logger_spec.rb
index 0da647350..0523a4cc3 100644
--- a/ruby/gitlab-shell/spec/gitlab_logger_spec.rb
+++ b/ruby/gitlab-shell/spec/gitlab_logger_spec.rb
@@ -142,7 +142,7 @@ describe GitlabLogger do
test_logger_status = system('bin/test-logger', msg)
expect(test_logger_status).to eq(true)
- grep_status = system('grep', '-q', '-e', msg, GitlabConfig.new.log_file)
+ grep_status = system('grep', '-q', '-e', msg, GitlabConfig.new.log_file.to_s)
expect(grep_status).to eq(true)
end
end
diff --git a/ruby/gitlab-shell/spec/gitlab_net_spec.rb b/ruby/gitlab-shell/spec/gitlab_net_spec.rb
index a6f3cd8e9..f8161e222 100644
--- a/ruby/gitlab-shell/spec/gitlab_net_spec.rb
+++ b/ruby/gitlab-shell/spec/gitlab_net_spec.rb
@@ -41,58 +41,6 @@ describe GitlabNet, vcr: true do
end
end
- describe '#broadcast_message' do
- context "broadcast message exists" do
- it 'should return message' do
- VCR.use_cassette("broadcast_message-ok") do
- result = gitlab_net.broadcast_message
- expect(result["message"]).to eq("Message")
- end
- end
- end
-
- context "broadcast message doesn't exist" do
- it 'should return nil' do
- VCR.use_cassette("broadcast_message-none") do
- result = gitlab_net.broadcast_message
- expect(result).to eq({})
- end
- end
- end
- end
-
- describe '#merge_request_urls' do
- let(:gl_repository) { "project-1" }
- let(:changes) { "123456 789012 refs/heads/test\n654321 210987 refs/tags/tag" }
- let(:encoded_changes) { "123456%20789012%20refs/heads/test%0A654321%20210987%20refs/tags/tag" }
-
- it "sends the given arguments as encoded URL parameters" do
- expect(gitlab_net).to receive(:get).with("#{internal_api_endpoint}/merge_request_urls?project=#{project}&changes=#{encoded_changes}&gl_repository=#{gl_repository}")
-
- gitlab_net.merge_request_urls(gl_repository, project, changes)
- end
-
- it "omits the gl_repository parameter if it's nil" do
- expect(gitlab_net).to receive(:get).with("#{internal_api_endpoint}/merge_request_urls?project=#{project}&changes=#{encoded_changes}")
-
- gitlab_net.merge_request_urls(nil, project, changes)
- end
-
- it "returns an empty array when the result cannot be parsed as JSON" do
- response = double(:response, code: '200', body: '')
- allow(gitlab_net).to receive(:get).and_return(response)
-
- expect(gitlab_net.merge_request_urls(gl_repository, project, changes)).to eq([])
- end
-
- it "returns an empty array when the result's status is not 200" do
- response = double(:response, code: '500', body: '[{}]')
- allow(gitlab_net).to receive(:get).and_return(response)
-
- expect(gitlab_net.merge_request_urls(gl_repository, project, changes)).to eq([])
- end
- end
-
describe '#pre_receive' do
let(:gl_repository) { "project-1" }
let(:params) { { gl_repository: gl_repository } }
@@ -160,27 +108,6 @@ describe GitlabNet, vcr: true do
end
end
- describe '#notify_post_receive' do
- let(:gl_repository) { 'project-1' }
- let(:repo_path) { '/path/to/my/repo.git' }
- let(:params) do
- { gl_repository: gl_repository, project: repo_path }
- end
-
- it 'sets the arguments as form parameters' do
- VCR.use_cassette('notify-post-receive') do
- expect_any_instance_of(Net::HTTP::Post).to receive(:set_form_data).with(hash_including(params))
- gitlab_net.notify_post_receive(gl_repository, repo_path)
- end
- end
-
- it 'returns true if notification was succesful' do
- VCR.use_cassette('notify-post-receive') do
- expect(gitlab_net.notify_post_receive(gl_repository, repo_path)).to be_truthy
- end
- end
- end
-
describe '#check_access' do
context 'ssh key with access nil, to project' do
it 'should allow push access for host' do
diff --git a/ruby/gitlab-shell/spec/spec_helper.rb b/ruby/gitlab-shell/spec/spec_helper.rb
index be5ff9c6b..684f87b15 100644
--- a/ruby/gitlab-shell/spec/spec_helper.rb
+++ b/ruby/gitlab-shell/spec/spec_helper.rb
@@ -10,8 +10,4 @@ Dir[File.expand_path('support/**/*.rb', __dir__)].each { |f| require f }
RSpec.configure do |config|
config.run_all_when_everything_filtered = true
config.filter_run :focus
-
- config.before(:each) do
- stub_const('ROOT_PATH', File.expand_path('..', __dir__))
- end
end
diff --git a/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-none.yml b/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-none.yml
deleted file mode 100644
index 8162343c2..000000000
--- a/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-none.yml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-http_interactions:
-- request:
- method: get
- uri: http://localhost:3000/api/v4/internal/broadcast_message
- body:
- encoding: US-ASCII
- string: secret_token=0a3938d9d95d807e94d937af3a4fbbea%0A
- headers:
- Accept-Encoding:
- - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
- Accept:
- - "*/*"
- User-Agent:
- - Ruby
- Content-Type:
- - application/x-www-form-urlencoded
- response:
- status:
- code: 200
- message: OK
- headers:
- Cache-Control:
- - max-age=0, private, must-revalidate
- Content-Length:
- - '2'
- Content-Type:
- - application/json
- Date:
- - Wed, 21 Jun 2017 10:44:50 GMT
- Etag:
- - W/"99914b932bd37a50b983c5e7c90ae93b"
- Vary:
- - Origin
- X-Frame-Options:
- - SAMEORIGIN
- X-Request-Id:
- - d31271ab-e21f-4349-a4c3-54f238c075c3
- X-Runtime:
- - '0.254031'
- body:
- encoding: UTF-8
- string: "{}"
- http_version:
- recorded_at: Wed, 21 Jun 2017 10:44:50 GMT
-recorded_with: VCR 2.4.0
diff --git a/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-ok.yml b/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-ok.yml
deleted file mode 100644
index a309c7ed3..000000000
--- a/ruby/gitlab-shell/spec/vcr_cassettes/broadcast_message-ok.yml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-http_interactions:
-- request:
- method: get
- uri: http://localhost:3000/api/v4/internal/broadcast_message
- body:
- encoding: US-ASCII
- string: secret_token=0a3938d9d95d807e94d937af3a4fbbea%0A
- headers:
- Accept-Encoding:
- - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
- Accept:
- - "*/*"
- User-Agent:
- - Ruby
- Content-Type:
- - application/x-www-form-urlencoded
- response:
- status:
- code: 200
- message: OK
- headers:
- Cache-Control:
- - max-age=0, private, must-revalidate
- Content-Length:
- - '153'
- Content-Type:
- - application/json
- Date:
- - Wed, 21 Jun 2017 12:29:13 GMT
- Etag:
- - W/"ce6de457fcc884f50125e81a10f165ce"
- Vary:
- - Origin
- X-Frame-Options:
- - SAMEORIGIN
- X-Request-Id:
- - ebaaf2c2-112e-4b3c-9182-e8714cd2a29c
- X-Runtime:
- - '0.276085'
- body:
- encoding: UTF-8
- string: '{"message":"Message","starts_at":"2017-06-21T12:28:00.000Z","ends_at":"2017-06-21T12:35:00.000Z","color":"#e75e40","font":"#ffffff","id":1,"active":true}'
- http_version:
- recorded_at: Wed, 21 Jun 2017 12:29:13 GMT
-recorded_with: VCR 2.4.0
diff --git a/ruby/gitlab-shell/spec/vcr_cassettes/notify-post-receive.yml b/ruby/gitlab-shell/spec/vcr_cassettes/notify-post-receive.yml
deleted file mode 100644
index 255f41516..000000000
--- a/ruby/gitlab-shell/spec/vcr_cassettes/notify-post-receive.yml
+++ /dev/null
@@ -1,46 +0,0 @@
----
-http_interactions:
-- request:
- method: post
- uri: http://localhost:3000/api/v4/internal/notify_post_receive
- body:
- encoding: US-ASCII
- string: gl_repository=project-1&project=%2Fpath%2Fto%2Fmy%2Frepo.git&secret_token=0a3938d9d95d807e94d937af3a4fbbea%0A
- headers:
- Accept-Encoding:
- - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
- Accept:
- - "*/*"
- User-Agent:
- - Ruby
- Content-Type:
- - application/x-www-form-urlencoded
- response:
- status:
- code: 200
- message: OK
- headers:
- Cache-Control:
- - max-age=0, private, must-revalidate
- Content-Length:
- - '3'
- Content-Type:
- - application/json
- Date:
- - Wed, 21 Jun 2017 12:47:48 GMT
- Etag:
- - W/"3644a684f98ea8fe223c713b77189a77"
- Vary:
- - Origin
- X-Frame-Options:
- - SAMEORIGIN
- X-Request-Id:
- - 407b184d-d6cf-43db-93fa-3f2d97112948
- X-Runtime:
- - '6.626186'
- body:
- encoding: UTF-8
- string: '200'
- http_version:
- recorded_at: Wed, 21 Jun 2017 12:47:48 GMT
-recorded_with: VCR 2.4.0
diff --git a/ruby/lib/gitlab/config.rb b/ruby/lib/gitlab/config.rb
index d644d88aa..1631315f1 100644
--- a/ruby/lib/gitlab/config.rb
+++ b/ruby/lib/gitlab/config.rb
@@ -64,6 +64,10 @@ module Gitlab
@ruby_dir ||= ENV['GITALY_RUBY_DIR']
end
+ def internal_socket
+ @internal_socket ||= ENV['GITALY_SOCKET']
+ end
+
def rbtrace_enabled?
@rbtrace_enabled ||= enabled?(ENV['GITALY_RUBY_RBTRACE_ENABLED'])
end
diff --git a/ruby/lib/gitlab/git/hook.rb b/ruby/lib/gitlab/git/hook.rb
index 7a5e15871..096e67843 100644
--- a/ruby/lib/gitlab/git/hook.rb
+++ b/ruby/lib/gitlab/git/hook.rb
@@ -115,7 +115,10 @@ module Gitlab
'GL_REPOSITORY' => repository.gl_repository,
'GL_PROTOCOL' => GL_PROTOCOL,
'PWD' => repo_path,
- 'GIT_DIR' => repo_path
+ 'GIT_DIR' => repo_path,
+ 'GITALY_REPO' => repository.gitaly_repository.to_json,
+ 'GITALY_SOCKET' => Gitlab.config.gitaly.internal_socket,
+ 'GITALY_HOOK_RPCS_ENABLED' => repository.feature_enabled?('call-hook-rpc').to_s
}
end
end
diff --git a/ruby/lib/gitlab/git/repository.rb b/ruby/lib/gitlab/git/repository.rb
index ccb251fcb..326c4c563 100644
--- a/ruby/lib/gitlab/git/repository.rb
+++ b/ruby/lib/gitlab/git/repository.rb
@@ -41,7 +41,8 @@ module Gitlab
GitalyServer.repo_path(call),
GitalyServer.gl_repository(call),
Gitlab::Git::GitlabProjects.from_gitaly(gitaly_repository, call),
- GitalyServer.repo_alt_dirs(call)
+ GitalyServer.repo_alt_dirs(call),
+ GitalyServer.feature_flags(call)
)
end
@@ -63,7 +64,7 @@ module Gitlab
attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path
- def initialize(gitaly_repository, path, gl_repository, gitlab_projects, combined_alt_dirs = "")
+ def initialize(gitaly_repository, path, gl_repository, gitlab_projects, combined_alt_dirs = "", feature_flags = GitalyServer::FeatureFlags.new({}))
@gitaly_repository = gitaly_repository
@alternate_object_directories = combined_alt_dirs
@@ -75,12 +76,17 @@ module Gitlab
@path = path
@gl_repository = gl_repository
@gitlab_projects = gitlab_projects
+ @feature_flags = feature_flags
end
def ==(other)
[storage, relative_path] == [other.storage, other.relative_path]
end
+ def feature_enabled?(flag)
+ @feature_flags.enabled?(flag)
+ end
+
def add_branch(branch_name, user:, target:)
target_object = Ref.dereference_object(lookup(target))
raise InvalidRef, "target not found: #{target}" unless target_object
diff --git a/ruby/spec/support/helpers/gitlab_shell_helper.rb b/ruby/spec/support/helpers/gitlab_shell_helper.rb
index b38b6ae62..7c1aafb4d 100644
--- a/ruby/spec/support/helpers/gitlab_shell_helper.rb
+++ b/ruby/spec/support/helpers/gitlab_shell_helper.rb
@@ -5,6 +5,9 @@ TMP_DIR_NAME = 'tmp'.freeze
TMP_DIR = File.join(GITALY_RUBY_DIR, TMP_DIR_NAME).freeze
GITLAB_SHELL_DIR = File.join(TMP_DIR, 'gitlab-shell').freeze
+# overwrite HOME env variable so user global .gitconfig doesn't influence tests
+ENV["HOME"] = File.join(File.dirname(__FILE__), "/testdata/home")
+
module GitlabShellHelper
def self.setup_gitlab_shell
Gitlab.config.gitlab_shell.test_global_ivar_override(:path, GITLAB_SHELL_DIR)
diff --git a/ruby/spec/support/helpers/testdata/home/.gitconfig b/ruby/spec/support/helpers/testdata/home/.gitconfig
new file mode 100644
index 000000000..29ace58d5
--- /dev/null
+++ b/ruby/spec/support/helpers/testdata/home/.gitconfig
@@ -0,0 +1,3 @@
+[user]
+ email = you@example.com
+ name = Your Name