diff options
274 files changed, 3352 insertions, 1632 deletions
diff --git a/.golangci.yml b/.golangci.yml index 6d1e9c0f2..88ac72af8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -56,7 +56,7 @@ linters-settings: - (*database/sql.DB).Close - (*database/sql.Rows).Close - (*gitlab.com/gitlab-org/gitaly/v16/client.Pool).Close - - (*gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel.ServerConn).Close + - (*gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel.ServerConn).Close - (*gitlab.com/gitlab-org/gitaly/v16/internal/streamcache.pipe).Close - (*gitlab.com/gitlab-org/gitaly/v16/internal/streamcache.pipeReader).Close - (*google.golang.org/grpc.ClientConn).Close diff --git a/CHANGELOG.md b/CHANGELOG.md index bb7bb57a1..d94e3ec26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Gitaly changelog +## 15.11.4 (2023-05-16) + +No changes. + ## 15.11.3 (2023-05-10) ### Security (1 change) diff --git a/client/dial.go b/client/dial.go index 434b2c727..b29e9f372 100644 --- a/client/dial.go +++ b/client/dial.go @@ -7,9 +7,9 @@ import ( "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/backoff" - "gitlab.com/gitlab-org/gitaly/v16/internal/dnsresolver" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/dnsresolver" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "google.golang.org/grpc" healthpb "google.golang.org/grpc/health/grpc_health_v1" ) diff --git a/client/sidechannel.go b/client/sidechannel.go index 0741d07cc..4375d780a 100644 --- a/client/sidechannel.go +++ b/client/sidechannel.go @@ -5,9 +5,9 @@ import ( "io" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) diff --git a/cmd/gitaly-git2go/featureflags.go b/cmd/gitaly-git2go/featureflags.go index fbe716209..055f18be9 100644 --- a/cmd/gitaly-git2go/featureflags.go +++ b/cmd/gitaly-git2go/featureflags.go @@ -7,8 +7,8 @@ import ( "encoding/gob" "flag" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git2go" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" ) // This subcommand is only called in tests, so we don't want to register it like diff --git a/cmd/gitaly-git2go/main.go b/cmd/gitaly-git2go/main.go index 27ee407f2..419dd4551 100644 --- a/cmd/gitaly-git2go/main.go +++ b/cmd/gitaly-git2go/main.go @@ -13,9 +13,9 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" git "github.com/libgit2/git2go/v34" "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git2go" glog "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/labkit/correlation" ) diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go index 4a87420dc..c6c422daa 100644 --- a/cmd/gitaly-hooks/hooks.go +++ b/cmd/gitaly-hooks/hooks.go @@ -13,11 +13,11 @@ import ( "github.com/sirupsen/logrus" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/client" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/env" gitalylog "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/stream" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go index a4b7fac04..a74ce2eae 100644 --- a/cmd/gitaly-hooks/hooks_test.go +++ b/cmd/gitaly-hooks/hooks_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" @@ -26,11 +27,10 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" gitalylog "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -787,7 +787,9 @@ remote: error executing git hook t.Run(tc.name, func(t *testing.T) { testhelper.SkipQuarantinedTest(t, "https://gitlab.com/gitlab-org/gitaly/-/issues/5105", - "TestGitalyServerReturnsError_packObjects/resource_exhausted_with_LimitError_detail") + "TestGitalyServerReturnsError_packObjects/resource_exhausted_with_LimitError_detail", + "TestGitalyServerReturnsError_packObjects/resource_exhausted_without_LimitError_detail", + ) logDir := testhelper.TempDir(t) diff --git a/cmd/gitaly-ssh/main.go b/cmd/gitaly-ssh/main.go index 813dfc576..96864f65a 100644 --- a/cmd/gitaly-ssh/main.go +++ b/cmd/gitaly-ssh/main.go @@ -11,8 +11,8 @@ import ( "github.com/sirupsen/logrus" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/client" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" internalclient "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/labkit/tracing" "google.golang.org/grpc" ) diff --git a/doc/PROCESS.md b/doc/PROCESS.md index c4e09da5f..ba695920a 100644 --- a/doc/PROCESS.md +++ b/doc/PROCESS.md @@ -137,7 +137,7 @@ in detail in the following sections. /chatops run feature set gitaly_go_find_license true --staging ``` -[gitaly-featureflag-folder]: https://gitlab.com/gitlab-org/gitaly/-/tree/master/internal/metadata/featureflag +[gitaly-featureflag-folder]: https://gitlab.com/gitlab-org/gitaly/-/tree/master/internal/featureflag [test-featureset]: https://gitlab.com/gitlab-org/gitaly/blob/c6fb49b9c6e854c0a2803f53106af501d6006cb8/internal/testhelper/featureset.go#L55-55 ### Flag management with chatops diff --git a/doc/protobuf.md b/doc/protobuf.md index 94831731a..c086caa73 100644 --- a/doc/protobuf.md +++ b/doc/protobuf.md @@ -14,6 +14,11 @@ in their respective subdirectories. The list of RPCs can be Run `make proto` from the root of the repository to regenerate the client libraries after updating .proto files. +## Gitaly RPC documentation + +For documentation generated from the protobuf definitions (in `proto/` directory), +see [Gitaly RPC documentation](https://gitlab-org.gitlab.io/gitaly/). + See [`developers.google.com`](https://developers.google.com/protocol-buffers/docs/proto3) for documentation of the 'proto3' Protocol buffer specification @@ -7,7 +7,7 @@ replace github.com/go-enry/go-license-detector/v4 => github.com/gl-gitaly/go-lic require ( github.com/ProtonMail/go-crypto v0.0.0-20230426101702-58e86b294756 - github.com/beevik/ntp v0.3.0 + github.com/beevik/ntp v0.3.2 github.com/cloudflare/tableflip v1.2.3 github.com/containerd/cgroups v1.1.0 github.com/dgraph-io/badger/v3 v3.2103.5 @@ -27,9 +27,9 @@ require ( github.com/jackc/pgx/v4 v4.18.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/libgit2/git2go/v34 v34.0.0 - github.com/miekg/dns v1.1.53 + github.com/miekg/dns v1.1.54 github.com/olekukonko/tablewriter v0.0.5 - github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 + github.com/opencontainers/runtime-spec v1.1.0-rc.2.0.20230511085824-0983f1d9e08f github.com/opentracing/opentracing-go v1.2.0 github.com/pelletier/go-toml/v2 v2.0.7 github.com/prometheus/client_golang v1.15.1 @@ -41,7 +41,7 @@ require ( gitlab.com/gitlab-org/labkit v1.18.0 go.uber.org/goleak v1.2.1 gocloud.dev v0.29.0 - golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 + golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea golang.org/x/sync v0.2.0 golang.org/x/sys v0.8.0 golang.org/x/time v0.3.0 @@ -188,9 +188,9 @@ require ( go.uber.org/atomic v1.10.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.8.2 // indirect @@ -627,8 +627,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.18.3/go.mod h1:b+psTJn33Q4qGoDaM7ZiO github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= -github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/beevik/ntp v0.3.2 h1:tWwrPRUDbURCK/Voh3Daxl2c60mOiA1z+bOqm4j/aSY= +github.com/beevik/ntp v0.3.2/go.mod h1:hg8DSzEy+KlFOOmR19XMea1MsLaA6vojqM04e2IhuUo= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -1633,8 +1633,8 @@ github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJys github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= -github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= @@ -1775,8 +1775,9 @@ github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/ github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0-rc.2.0.20230511085824-0983f1d9e08f h1:Li3QKbkyXFhrVfvLG+zMVgJRs1lEQ0qQBu77MTrxKSQ= +github.com/opencontainers/runtime-spec v1.1.0-rc.2.0.20230511085824-0983f1d9e08f/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= @@ -2284,8 +2285,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2412,8 +2413,9 @@ golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmL golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2640,8 +2642,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2656,8 +2658,9 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/backup/backup.go b/internal/backup/backup.go index c40e675fe..60fa0cd65 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -470,10 +470,15 @@ func (mgr *Manager) restoreBundle(ctx context.Context, path string, server stora return nil } -func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) error { +func (mgr *Manager) writeCustomHooks(ctx context.Context, repo Repository, path string) (returnErr error) { w := NewLazyWriter(func() (io.WriteCloser, error) { return mgr.sink.GetWriter(ctx, path) }) + defer func() { + if err := w.Close(); err != nil && returnErr == nil { + returnErr = fmt.Errorf("write custom hooks: %w", err) + } + }() if err := repo.GetCustomHooks(ctx, w); err != nil { return fmt.Errorf("write custom hooks: %w", err) } diff --git a/internal/backup/repository.go b/internal/backup/repository.go index 98d134468..6c2523908 100644 --- a/internal/backup/repository.go +++ b/internal/backup/repository.go @@ -213,7 +213,7 @@ func (r *localRepository) ListRefs(ctx context.Context) ([]git.Reference, error) // GetCustomHooks fetches the custom hooks archive. func (r *localRepository) GetCustomHooks(ctx context.Context, out io.Writer) error { if err := repoutil.GetCustomHooks(ctx, r.locator, out, r.repo); err != nil { - return fmt.Errorf("local repository: list refs: %w", err) + return fmt.Errorf("local repository: get custom hooks: %w", err) } return nil } diff --git a/internal/cache/diskcache_test.go b/internal/cache/diskcache_test.go index ec19ddd9b..2db13ecde 100644 --- a/internal/cache/diskcache_test.go +++ b/internal/cache/diskcache_test.go @@ -9,9 +9,9 @@ import ( promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/cache/keyer.go b/internal/cache/keyer.go index 609867750..db9620b23 100644 --- a/internal/cache/keyer.go +++ b/internal/cache/keyer.go @@ -15,9 +15,9 @@ import ( "github.com/google/uuid" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/version" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/cli/error.go b/internal/cli/error.go new file mode 100644 index 000000000..1a33eb7a5 --- /dev/null +++ b/internal/cli/error.go @@ -0,0 +1,12 @@ +package cli + +import "fmt" + +// RequiredFlagError type is needed to produce the same error message as one from the +// github.com/urfave/cli/v2 package. Unfortunately the errRequiredFlags type is not +// exportable, and we can't utilise it. +type RequiredFlagError string + +func (rf RequiredFlagError) Error() string { + return fmt.Sprintf("Required flag %q not set", string(rf)) +} diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index 120f3fdb0..e2c6c48ca 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -14,7 +14,6 @@ import ( "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap/starter" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" @@ -37,10 +36,11 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/env" glog "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/streamcache" "gitlab.com/gitlab-org/gitaly/v16/internal/tempdir" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" diff --git a/internal/cli/praefect/main.go b/internal/cli/praefect/main.go index e30332f74..23a6c7818 100644 --- a/internal/cli/praefect/main.go +++ b/internal/cli/praefect/main.go @@ -10,17 +10,6 @@ // // praefect -config PATH_TO_CONFIG sql-ping // -// # SQL Migrate -// -// The subcommand "sql-migrate" will apply any outstanding SQL migrations. -// -// praefect -config PATH_TO_CONFIG sql-migrate [-ignore-unknown=true|false] -// -// By default, the migration will ignore any unknown migrations that are -// not known by the Praefect binary. -// -// "-ignore-unknown=false" will disable this behavior. -// // The subcommand "sql-migrate-status" will show which SQL migrations have // been applied and which ones have not: // @@ -78,6 +67,10 @@ func NewApp() *cli.App { newDialNodesCommand(), newListStoragesCommand(), newListUntrackedRepositoriesCommand(), + newTrackRepositoryCommand(), + newVerifyCommand(), + newMetadataCommand(), + newSQLMigrateCommand(), }, Flags: []cli.Flag{ &cli.StringFlag{ diff --git a/internal/cli/praefect/serve.go b/internal/cli/praefect/serve.go index 22260e500..328459ef6 100644 --- a/internal/cli/praefect/serve.go +++ b/internal/cli/praefect/serve.go @@ -16,10 +16,11 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap/starter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/sentry" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect" @@ -35,7 +36,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/version" "gitlab.com/gitlab-org/labkit/monitoring" "gitlab.com/gitlab-org/labkit/tracing" diff --git a/internal/cli/praefect/subcmd.go b/internal/cli/praefect/subcmd.go index ef7e2282a..74ddeddf1 100644 --- a/internal/cli/praefect/subcmd.go +++ b/internal/cli/praefect/subcmd.go @@ -34,15 +34,11 @@ const ( func subcommands(logger *logrus.Entry) map[string]subcmd { return map[string]subcmd{ sqlPingCmdName: &sqlPingSubcommand{}, - sqlMigrateCmdName: newSQLMigrateSubCommand(os.Stdout), sqlMigrateDownCmdName: &sqlMigrateDownSubcommand{}, sqlMigrateStatusCmdName: &sqlMigrateStatusSubcommand{}, setReplicationFactorCmdName: newSetReplicatioFactorSubcommand(os.Stdout), removeRepositoryCmdName: newRemoveRepository(logger, os.Stdout), - trackRepositoryCmdName: newTrackRepository(logger, os.Stdout), trackRepositoriesCmdName: newTrackRepositories(logger, os.Stdout), - metadataCmdName: newMetadataSubcommand(os.Stdout), - verifyCmdName: newVerifySubcommand(os.Stdout), } } diff --git a/internal/cli/praefect/subcmd_metadata.go b/internal/cli/praefect/subcmd_metadata.go index 1cce06899..30ef2d516 100644 --- a/internal/cli/praefect/subcmd_metadata.go +++ b/internal/cli/praefect/subcmd_metadata.go @@ -1,96 +1,109 @@ package praefect import ( - "context" "errors" - "flag" "fmt" - "io" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) -const metadataCmdName = "metadata" - -type metadataSubcommand struct { - stdout io.Writer - repositoryID int64 - virtualStorage string - relativePath string -} - -func newMetadataSubcommand(stdout io.Writer) *metadataSubcommand { - return &metadataSubcommand{stdout: stdout} -} - -func (cmd *metadataSubcommand) FlagSet() *flag.FlagSet { - fs := flag.NewFlagSet(metadataCmdName, flag.ContinueOnError) - fs.Int64Var(&cmd.repositoryID, "repository-id", 0, "the repository's ID") - fs.StringVar(&cmd.virtualStorage, "virtual-storage", "", "the repository's virtual storage") - fs.StringVar(&cmd.relativePath, "relative-path", "", "the repository's relative path in the virtual storage") - return fs -} - -func (cmd *metadataSubcommand) println(format string, args ...interface{}) { - fmt.Fprintf(cmd.stdout, format+"\n", args...) +func newMetadataCommand() *cli.Command { + return &cli.Command{ + Name: "metadata", + Usage: "shows metadata information about repository", + Description: "The command provides metadata information about the repository. It includes " + + "identifier of the repository, path on the disk for it and it's replicas, information " + + "about replicas such as if it is assigned or not, its generation, health state, the storage, " + + "if it is a valid primary, etc. It can be invoked by providing repository identifier or " + + "virtual repository name and relative path.", + HideHelpCommand: true, + Action: metadataAction, + Flags: []cli.Flag{ + &cli.Int64Flag{ + Name: "repository-id", + Usage: "the repository's ID", + }, + &cli.StringFlag{ + Name: paramVirtualStorage, + Usage: "the repository's virtual storage", + }, + &cli.StringFlag{ + Name: "relative-path", + Usage: "the repository's relative path in the virtual storage", + }, + }, + Before: func(ctx *cli.Context) error { + if ctx.Args().Present() { + _ = cli.ShowSubcommandHelp(ctx) + return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1) + } + return nil + }, + } } -func (cmd *metadataSubcommand) Exec(flags *flag.FlagSet, cfg config.Config) error { - if flags.NArg() > 0 { - return unexpectedPositionalArgsError{Command: flags.Name()} +func metadataAction(appCtx *cli.Context) error { + logger := log.Default() + conf, err := getConfig(logger, appCtx.String(configFlagName)) + if err != nil { + return err } + repositoryID := appCtx.Int64("repository-id") + virtualStorage := appCtx.String(paramVirtualStorage) + relativePath := appCtx.String("relative-path") + var request gitalypb.GetRepositoryMetadataRequest switch { - case cmd.repositoryID != 0: - if cmd.virtualStorage != "" || cmd.relativePath != "" { + case repositoryID != 0: + if virtualStorage != "" || relativePath != "" { return errors.New("virtual storage and relative path can't be provided with a repository ID") } - request.Query = &gitalypb.GetRepositoryMetadataRequest_RepositoryId{RepositoryId: cmd.repositoryID} - case cmd.virtualStorage != "" || cmd.relativePath != "": - if cmd.virtualStorage == "" { + request.Query = &gitalypb.GetRepositoryMetadataRequest_RepositoryId{RepositoryId: repositoryID} + case virtualStorage != "" || relativePath != "": + if virtualStorage == "" { return errors.New("virtual storage is required with relative path") - } else if cmd.relativePath == "" { + } else if relativePath == "" { return errors.New("relative path is required with virtual storage") } request.Query = &gitalypb.GetRepositoryMetadataRequest_Path_{ Path: &gitalypb.GetRepositoryMetadataRequest_Path{ - VirtualStorage: cmd.virtualStorage, - RelativePath: cmd.relativePath, + VirtualStorage: virtualStorage, + RelativePath: relativePath, }, } default: return errors.New("repository id or virtual storage and relative path required") } - nodeAddr, err := getNodeAddress(cfg) + nodeAddr, err := getNodeAddress(conf) if err != nil { return fmt.Errorf("get node address: %w", err) } - ctx := context.TODO() - conn, err := subCmdDial(ctx, nodeAddr, cfg.Auth.Token, defaultDialTimeout) + conn, err := subCmdDial(appCtx.Context, nodeAddr, conf.Auth.Token, defaultDialTimeout) if err != nil { return fmt.Errorf("dial: %w", err) } defer conn.Close() - metadata, err := gitalypb.NewPraefectInfoServiceClient(conn).GetRepositoryMetadata(ctx, &request) + metadata, err := gitalypb.NewPraefectInfoServiceClient(conn).GetRepositoryMetadata(appCtx.Context, &request) if err != nil { return fmt.Errorf("get metadata: %w", err) } - cmd.println("Repository ID: %d", metadata.RepositoryId) - cmd.println("Virtual Storage: %q", metadata.VirtualStorage) - cmd.println("Relative Path: %q", metadata.RelativePath) - cmd.println("Replica Path: %q", metadata.ReplicaPath) - cmd.println("Primary: %q", metadata.Primary) - cmd.println("Generation: %d", metadata.Generation) - cmd.println("Replicas:") + fmt.Fprintf(appCtx.App.Writer, "Repository ID: %d\n", metadata.RepositoryId) + fmt.Fprintf(appCtx.App.Writer, "Virtual Storage: %q\n", metadata.VirtualStorage) + fmt.Fprintf(appCtx.App.Writer, "Relative Path: %q\n", metadata.RelativePath) + fmt.Fprintf(appCtx.App.Writer, "Replica Path: %q\n", metadata.ReplicaPath) + fmt.Fprintf(appCtx.App.Writer, "Primary: %q\n", metadata.Primary) + fmt.Fprintf(appCtx.App.Writer, "Generation: %d\n", metadata.Generation) + fmt.Fprintf(appCtx.App.Writer, "Replicas:\n") for _, replica := range metadata.Replicas { - cmd.println("- Storage: %q", replica.Storage) - cmd.println(" Assigned: %v", replica.Assigned) + fmt.Fprintf(appCtx.App.Writer, "- Storage: %q\n", replica.Storage) + fmt.Fprintf(appCtx.App.Writer, " Assigned: %v\n", replica.Assigned) generationText := fmt.Sprintf("%d, fully up to date", replica.Generation) if replica.Generation == -1 { @@ -104,10 +117,10 @@ func (cmd *metadataSubcommand) Exec(flags *flag.FlagSet, cfg config.Config) erro verifiedAt = replica.VerifiedAt.AsTime().String() } - cmd.println(" Generation: %s", generationText) - cmd.println(" Healthy: %v", replica.Healthy) - cmd.println(" Valid Primary: %v", replica.ValidPrimary) - cmd.println(" Verified At: %s", verifiedAt) + fmt.Fprintf(appCtx.App.Writer, " Generation: %s\n", generationText) + fmt.Fprintf(appCtx.App.Writer, " Healthy: %v\n", replica.Healthy) + fmt.Fprintf(appCtx.App.Writer, " Valid Primary: %v\n", replica.ValidPrimary) + fmt.Fprintf(appCtx.App.Writer, " Verified At: %s\n", verifiedAt) } return nil } diff --git a/internal/cli/praefect/subcmd_metadata_test.go b/internal/cli/praefect/subcmd_metadata_test.go index bfce0350d..05db30b7e 100644 --- a/internal/cli/praefect/subcmd_metadata_test.go +++ b/internal/cli/praefect/subcmd_metadata_test.go @@ -4,10 +4,12 @@ import ( "bytes" "errors" "fmt" + "io" "testing" "time" "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/info" @@ -22,7 +24,7 @@ func TestMetadataSubcommand(t *testing.T) { ctx := testhelper.Context(t) tx := testdb.New(t).Begin(t) - defer tx.Rollback(t) + t.Cleanup(func() { tx.Rollback(t) }) testdb.SetHealthyNodes(t, ctx, tx, map[string]map[string][]string{ "praefect": {"virtual-storage": {"primary", "secondary-1"}}, @@ -42,7 +44,23 @@ func TestMetadataSubcommand(t *testing.T) { ln, clean := listenAndServe(t, []svcRegistrar{ registerPraefectInfoServer(info.NewServer(config.Config{}, rs, nil, nil, nil)), }) - defer clean() + t.Cleanup(clean) + + conf := config.Config{ + SocketPath: ln.Addr().String(), + VirtualStorages: []*config.VirtualStorage{ + { + Name: "vs-1", + Nodes: []*config.Node{ + { + Storage: "storage-1", + Address: "tcp://1.2.3.4", + }, + }, + }, + }, + } + confPath := writeConfigToFile(t, conf) for _, tc := range []struct { desc string @@ -50,6 +68,11 @@ func TestMetadataSubcommand(t *testing.T) { error error }{ { + desc: "positional arguments", + args: []string{"positional-arg"}, + error: cli.Exit(unexpectedPositionalArgsError{Command: "metadata"}, 1), + }, + { desc: "missing parameters fails", error: errors.New("repository id or virtual storage and relative path required"), }, @@ -87,14 +110,28 @@ func TestMetadataSubcommand(t *testing.T) { args: []string{"-virtual-storage=virtual-storage", "-relative-path=relative-path"}, }, } { + tc := tc t.Run(tc.desc, func(t *testing.T) { - stdout := &bytes.Buffer{} - cmd := newMetadataSubcommand(stdout) + t.Parallel() - fs := cmd.FlagSet() - require.NoError(t, fs.Parse(tc.args)) - err := cmd.Exec(fs, config.Config{SocketPath: ln.Addr().String()}) - testhelper.RequireGrpcError(t, tc.error, err) + var stdout bytes.Buffer + app := cli.App{ + Reader: bytes.NewReader(nil), + Writer: &stdout, + ErrWriter: io.Discard, + HideHelpCommand: true, + Commands: []*cli.Command{ + newMetadataCommand(), + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Value: confPath, + }, + }, + } + err := app.Run(append([]string{progname, "metadata"}, tc.args...)) + require.Equal(t, tc.error, err) if tc.error != nil { return } diff --git a/internal/cli/praefect/subcmd_sql_migrate.go b/internal/cli/praefect/subcmd_sql_migrate.go index 57020bb7b..46d941113 100644 --- a/internal/cli/praefect/subcmd_sql_migrate.go +++ b/internal/cli/praefect/subcmd_sql_migrate.go @@ -1,13 +1,12 @@ package praefect import ( - "flag" "fmt" - "io" "time" migrate "github.com/rubenv/sql-migrate" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/glsql" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/migrations" ) @@ -17,25 +16,42 @@ const ( timeFmt = "2006-01-02T15:04:05" ) -type sqlMigrateSubcommand struct { - w io.Writer - ignoreUnknown bool - verbose bool -} - -func newSQLMigrateSubCommand(writer io.Writer) *sqlMigrateSubcommand { - return &sqlMigrateSubcommand{w: writer} -} - -func (cmd *sqlMigrateSubcommand) FlagSet() *flag.FlagSet { - flags := flag.NewFlagSet(sqlMigrateCmdName, flag.ExitOnError) - flags.BoolVar(&cmd.ignoreUnknown, "ignore-unknown", true, "ignore unknown migrations (default is true)") - flags.BoolVar(&cmd.verbose, "verbose", false, "show text of migration query (default is false)") - return flags +func newSQLMigrateCommand() *cli.Command { + return &cli.Command{ + Name: sqlMigrateCmdName, + Usage: "apply outstanding SQL migrations", + Description: "The sql-migrate subcommand applies outstanding migrations to the configured database.\n" + + "The subcommand doesn't fail if database has migrations unknown to the version of Praefect you're using.\n" + + "To make the subcommand fail on unknown migrations, use the 'ignore-unknown' flag.", + HideHelpCommand: true, + Action: sqlMigrateAction, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "ignore-unknown", + Usage: "ignore unknown migrations", + Value: true, + }, + &cli.BoolFlag{ + Name: "verbose", + Usage: "show text of migration query", + }, + }, + Before: func(ctx *cli.Context) error { + if ctx.Args().Present() { + _ = cli.ShowSubcommandHelp(ctx) + return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1) + } + return nil + }, + } } -func (cmd *sqlMigrateSubcommand) Exec(flags *flag.FlagSet, conf config.Config) error { - const subCmd = progname + " " + sqlMigrateCmdName +func sqlMigrateAction(appCtx *cli.Context) error { + logger := log.Default() + conf, err := getConfig(logger, appCtx.String(configFlagName)) + if err != nil { + return err + } db, clean, err := openDB(conf.DB) if err != nil { @@ -43,8 +59,9 @@ func (cmd *sqlMigrateSubcommand) Exec(flags *flag.FlagSet, conf config.Config) e } defer clean() + ignoreUnknown := appCtx.Bool("ignore-unknown") migrationSet := migrate.MigrationSet{ - IgnoreUnknown: cmd.ignoreUnknown, + IgnoreUnknown: ignoreUnknown, TableName: migrations.MigrationTableName, } @@ -55,40 +72,41 @@ func (cmd *sqlMigrateSubcommand) Exec(flags *flag.FlagSet, conf config.Config) e // Find all migrations that are currently down. planMigrations, _, _ := migrationSet.PlanMigration(db, "postgres", planSource, migrate.Up, 0) + subCmd := progname + " " + appCtx.Command.Name if len(planMigrations) == 0 { - fmt.Fprintf(cmd.w, "%s: all migrations are up\n", subCmd) + fmt.Fprintf(appCtx.App.Writer, "%s: all migrations are up\n", subCmd) return nil } - fmt.Fprintf(cmd.w, "%s: migrations to apply: %d\n\n", subCmd, len(planMigrations)) + fmt.Fprintf(appCtx.App.Writer, "%s: migrations to apply: %d\n\n", subCmd, len(planMigrations)) executed := 0 for _, mig := range planMigrations { - fmt.Fprintf(cmd.w, "= %s %v: migrating\n", time.Now().Format(timeFmt), mig.Id) + fmt.Fprintf(appCtx.App.Writer, "= %s %v: migrating\n", time.Now().Format(timeFmt), mig.Id) start := time.Now() - if cmd.verbose { - fmt.Fprintf(cmd.w, "\t%v\n", mig.Up) + if appCtx.Bool("verbose") { + fmt.Fprintf(appCtx.App.Writer, "\t%v\n", mig.Up) } - n, err := glsql.MigrateSome(mig.Migration, db, cmd.ignoreUnknown) + n, err := glsql.MigrateSome(mig.Migration, db, ignoreUnknown) if err != nil { return fmt.Errorf("%s: fail: %w", time.Now().Format(timeFmt), err) } if n > 0 { - fmt.Fprintf(cmd.w, "== %s %v: applied (%s)\n", time.Now().Format(timeFmt), mig.Id, time.Since(start)) + fmt.Fprintf(appCtx.App.Writer, "== %s %v: applied (%s)\n", time.Now().Format(timeFmt), mig.Id, time.Since(start)) // Additional migrations were run. No harm, but prevents us from tracking their execution duration. if n > 1 { - fmt.Fprintf(cmd.w, "warning: %v additional migrations were applied successfully\n", n-1) + fmt.Fprintf(appCtx.App.Writer, "warning: %v additional migrations were applied successfully\n", n-1) } } else { - fmt.Fprintf(cmd.w, "== %s %v: skipped (%s)\n", time.Now().Format(timeFmt), mig.Id, time.Since(start)) + fmt.Fprintf(appCtx.App.Writer, "== %s %v: skipped (%s)\n", time.Now().Format(timeFmt), mig.Id, time.Since(start)) } executed += n } - fmt.Fprintf(cmd.w, "\n%s: OK (applied %d migrations)\n", subCmd, executed) + fmt.Fprintf(appCtx.App.Writer, "\n%s: OK (applied %d migrations)\n", subCmd, executed) return nil } diff --git a/internal/cli/praefect/subcmd_sql_migrate_test.go b/internal/cli/praefect/subcmd_sql_migrate_test.go index 535c091fa..9fae09ca8 100644 --- a/internal/cli/praefect/subcmd_sql_migrate_test.go +++ b/internal/cli/praefect/subcmd_sql_migrate_test.go @@ -2,11 +2,12 @@ package praefect import ( "bytes" - "flag" "fmt" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/migrations" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" @@ -15,17 +16,28 @@ import ( func TestSubCmdSqlMigrate(t *testing.T) { db := testdb.New(t) dbCfg := testdb.GetConfig(t, db.Name) - cfg := config.Config{DB: dbCfg} + cfg := config.Config{ + ListenAddr: "/dev/null", + VirtualStorages: []*config.VirtualStorage{{Name: "p", Nodes: []*config.Node{{Storage: "s", Address: "localhost"}}}}, + DB: dbCfg, + } + confPath := writeConfigToFile(t, cfg) migrationCt := len(migrations.All()) for _, tc := range []struct { desc string up int - verbose bool + args []string expectedOutput []string + expectedErr error }{ { + desc: "unexpected positional arguments", + args: []string{"positonal-arg"}, + expectedErr: cli.Exit(unexpectedPositionalArgsError{Command: "sql-migrate"}, 1), + }, + { desc: "All migrations up", up: migrationCt, expectedOutput: []string{"praefect sql-migrate: all migrations are up"}, @@ -51,9 +63,9 @@ func TestSubCmdSqlMigrate(t *testing.T) { }, }, { - desc: "Verbose output", - up: 0, - verbose: true, + desc: "Verbose output", + up: 0, + args: []string{"-verbose"}, expectedOutput: []string{ fmt.Sprintf("praefect sql-migrate: migrations to apply: %d", migrationCt), "20200109161404_hello_world: migrating", @@ -67,9 +79,25 @@ func TestSubCmdSqlMigrate(t *testing.T) { testdb.SetMigrations(t, db, cfg, tc.up) var stdout bytes.Buffer - migrateCmd := sqlMigrateSubcommand{w: &stdout, ignoreUnknown: true, verbose: tc.verbose} - assert.NoError(t, migrateCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), cfg)) - + var stderr bytes.Buffer + app := cli.App{ + Reader: bytes.NewReader(nil), + Writer: &stdout, + ErrWriter: &stderr, + HideHelpCommand: true, + Commands: []*cli.Command{ + newSQLMigrateCommand(), + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Value: confPath, + }, + }, + } + err := app.Run(append([]string{progname, sqlMigrateCmdName, "-ignore-unknown"}, tc.args...)) + require.Equal(t, tc.expectedErr, err) + assert.Empty(t, stderr.String()) for _, out := range tc.expectedOutput { assert.Contains(t, stdout.String(), out) } diff --git a/internal/cli/praefect/subcmd_track_repository.go b/internal/cli/praefect/subcmd_track_repository.go index fd5ad5126..1c28cbe28 100644 --- a/internal/cli/praefect/subcmd_track_repository.go +++ b/internal/cli/praefect/subcmd_track_repository.go @@ -4,14 +4,16 @@ import ( "context" "database/sql" "errors" - "flag" "fmt" "io" "math/rand" "time" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" + "github.com/urfave/cli/v2" + glcli "gitlab.com/gitlab-org/gitaly/v16/internal/cli" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -27,13 +29,47 @@ const ( trackRepositoryCmdName = "track-repository" ) -type trackRepository struct { - w io.Writer - logger logrus.FieldLogger - virtualStorage string - relativePath string - authoritativeStorage string - replicateImmediately bool +func newTrackRepositoryCommand() *cli.Command { + return &cli.Command{ + Name: trackRepositoryCmdName, + Usage: "Praefect starts to track given repository", + Description: "This command adds a given repository to be tracked by Praefect.\n" + + "It checks if the repository exists on disk on the authoritative storage,\n" + + "and whether database records are absent from tracking the repository.\n" + + "If the 'replicate-immediately' flag is used, the command will attempt to replicate\n" + + "the repository to the secondaries. The command is blocked until the\n" + + "replication finishes. Otherwise, replication jobs will be created and will " + + "be executed eventually by Praefect in the background.\n", + HideHelpCommand: true, + Action: trackRepositoryAction, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: paramVirtualStorage, + Usage: "name of the repository's virtual storage", + Required: true, + }, + &cli.StringFlag{ + Name: paramRelativePath, + Usage: "relative path to the repository", + Required: true, + }, + &cli.StringFlag{ + Name: paramAuthoritativeStorage, + Usage: "storage with the repository to consider as authoritative", + }, + &cli.BoolFlag{ + Name: "replicate-immediately", + Usage: "kick off a replication immediately", + }, + }, + Before: func(ctx *cli.Context) error { + if ctx.Args().Present() { + _ = cli.ShowSubcommandHelp(ctx) + return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1) + } + return nil + }, + } } type trackRepositoryRequest struct { @@ -44,39 +80,21 @@ type trackRepositoryRequest struct { var errAuthoritativeRepositoryNotExist = errors.New("authoritative repository does not exist") -func newTrackRepository(logger logrus.FieldLogger, w io.Writer) *trackRepository { - return &trackRepository{w: w, logger: logger} -} - -func (cmd *trackRepository) FlagSet() *flag.FlagSet { - fs := flag.NewFlagSet(trackRepositoryCmdName, flag.ExitOnError) - fs.StringVar(&cmd.virtualStorage, paramVirtualStorage, "", "name of the repository's virtual storage") - fs.StringVar(&cmd.relativePath, paramRelativePath, "", "relative path to the repository") - fs.StringVar(&cmd.authoritativeStorage, paramAuthoritativeStorage, "", "storage with the repository to consider as authoritative") - fs.BoolVar(&cmd.replicateImmediately, "replicate-immediately", false, "kick off a replication immediately") - fs.Usage = func() { - printfErr("Description:\n" + - " This command adds a given repository to be tracked by Praefect.\n" + - " It checks if the repository exists on disk on the authoritative storage,\n" + - " and whether database records are absent from tracking the repository.\n" + - " If -replicate-immediately is used, the command will attempt to replicate the repository to the secondaries.\n" + - " Otherwise, replication jobs will be created and will be executed eventually by Praefect itself.\n") - fs.PrintDefaults() +func trackRepositoryAction(appCtx *cli.Context) error { + logger := log.Default() + conf, err := getConfig(logger, appCtx.String(configFlagName)) + if err != nil { + return err } - return fs -} -func (cmd trackRepository) Exec(flags *flag.FlagSet, cfg config.Config) error { - switch { - case flags.NArg() > 0: - return unexpectedPositionalArgsError{Command: flags.Name()} - case cmd.virtualStorage == "": - return requiredParameterError(paramVirtualStorage) - case cmd.relativePath == "": - return requiredParameterError(paramRelativePath) - case cmd.authoritativeStorage == "": - if cfg.Failover.ElectionStrategy == config.ElectionStrategyPerRepository { - return requiredParameterError(paramAuthoritativeStorage) + virtualStorage := appCtx.String(paramVirtualStorage) + relativePath := appCtx.String(paramRelativePath) + authoritativeStorage := appCtx.String(paramAuthoritativeStorage) + replicateImmediately := appCtx.Bool("replicate-immediately") + + if authoritativeStorage == "" { + if conf.Failover.ElectionStrategy == config.ElectionStrategyPerRepository { + return glcli.RequiredFlagError(paramAuthoritativeStorage) } } @@ -84,29 +102,24 @@ func (cmd trackRepository) Exec(flags *flag.FlagSet, cfg config.Config) error { openDBCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - db, err := glsql.OpenDB(openDBCtx, cfg.DB) + db, err := glsql.OpenDB(openDBCtx, conf.DB) if err != nil { return fmt.Errorf("connect to database: %w", err) } defer func() { _ = db.Close() }() - return cmd.exec(ctx, db, cfg) -} - -const trackRepoErrorPrefix = "attempting to track repository in praefect database" - -func (cmd *trackRepository) exec(ctx context.Context, db *sql.DB, cfg config.Config) error { - logger := cmd.logger.WithField("correlation_id", correlation.ExtractFromContext(ctx)) - req := trackRepositoryRequest{ - RelativePath: cmd.relativePath, - AuthoritativeStorage: cmd.authoritativeStorage, - VirtualStorage: cmd.virtualStorage, + RelativePath: relativePath, + AuthoritativeStorage: authoritativeStorage, + VirtualStorage: virtualStorage, } - return req.execRequest(ctx, db, cfg, cmd.w, logger, cmd.replicateImmediately) + logger = logger.WithField("correlation_id", correlation.ExtractFromContext(ctx)) + return req.execRequest(ctx, db, conf, appCtx.App.Writer, logger, replicateImmediately) } +const trackRepoErrorPrefix = "attempting to track repository in praefect database" + func (req *trackRepositoryRequest) execRequest(ctx context.Context, db *sql.DB, cfg config.Config, diff --git a/internal/cli/praefect/subcmd_track_repository_test.go b/internal/cli/praefect/subcmd_track_repository_test.go index 56f33fc17..d721e51d9 100644 --- a/internal/cli/praefect/subcmd_track_repository_test.go +++ b/internal/cli/praefect/subcmd_track_repository_test.go @@ -2,15 +2,14 @@ package praefect import ( "bytes" - "flag" - "net" + "io" "path/filepath" - "strconv" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" @@ -25,62 +24,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) -func TestAddRepository_FlagSet(t *testing.T) { - t.Parallel() - cmd := &trackRepository{} - fs := cmd.FlagSet() - require.NoError(t, fs.Parse([]string{"--virtual-storage", "vs", "--repository", "repo", "--authoritative-storage", "storage-0"})) - require.Equal(t, "vs", cmd.virtualStorage) - require.Equal(t, "repo", cmd.relativePath) - require.Equal(t, "storage-0", cmd.authoritativeStorage) -} - -func TestAddRepository_Exec_invalidArgs(t *testing.T) { - t.Parallel() - t.Run("not all flag values processed", func(t *testing.T) { - cmd := trackRepository{} - flagSet := flag.NewFlagSet("cmd", flag.PanicOnError) - require.NoError(t, flagSet.Parse([]string{"stub"})) - err := cmd.Exec(flagSet, config.Config{}) - require.EqualError(t, err, "cmd doesn't accept positional arguments") - }) - - t.Run("virtual-storage is not set", func(t *testing.T) { - cmd := trackRepository{} - err := cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), config.Config{}) - require.EqualError(t, err, `"virtual-storage" is a required parameter`) - }) - - t.Run("repository is not set", func(t *testing.T) { - cmd := trackRepository{virtualStorage: "stub"} - err := cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), config.Config{}) - require.EqualError(t, err, `"repository" is a required parameter`) - }) - - t.Run("authoritative-storage is not set", func(t *testing.T) { - cmd := trackRepository{virtualStorage: "stub", relativePath: "path/to/repo"} - err := cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), config.Config{Failover: config.Failover{ElectionStrategy: config.ElectionStrategyPerRepository}}) - require.EqualError(t, err, `"authoritative-storage" is a required parameter`) - }) - - t.Run("db connection error", func(t *testing.T) { - listener, addr := testhelper.GetLocalhostListener(t) - require.NoError(t, listener.Close()) - - host, portStr, err := net.SplitHostPort(addr) - require.NoError(t, err) - port, err := strconv.ParseUint(portStr, 10, 16) - require.NoError(t, err) - - cmd := trackRepository{virtualStorage: "stub", relativePath: "stub", authoritativeStorage: "storage-0", logger: testhelper.NewDiscardingLogger(t)} - cfg := config.Config{DB: config.DB{Host: host, Port: int(port), SSLMode: "disable"}} - err = cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), cfg) - require.Error(t, err) - require.Contains(t, err.Error(), "connect to database: send ping: failed to connect to") - }) -} - -func TestAddRepository_Exec(t *testing.T) { +func TestTrackRepositorySubcommand(t *testing.T) { t.Parallel() g1Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-1")) g2Cfg := testcfg.Build(t, testcfg.WithStorages("gitaly-2")) @@ -117,6 +61,7 @@ func TestAddRepository_Exec(t *testing.T) { ElectionStrategy: config.ElectionStrategyPerRepository, }, } + confPath := writeConfigToFile(t, conf) gitalyCC, err := client.Dial(g1Addr, nil) require.NoError(t, err) @@ -138,26 +83,129 @@ func TestAddRepository_Exec(t *testing.T) { } authoritativeStorage := g1Cfg.Storages[0].Name - logger := testhelper.NewDiscardingLogger(t) + repoDS := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) + + relativePathAlreadyExist := "path/to/test/repo_2" + require.NoError(t, createRepoThroughGitaly1(relativePathAlreadyExist)) + require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, relativePathAlreadyExist)) + require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, relativePathAlreadyExist)) + idRelativePathAlreadyExist, err := repoDS.ReserveRepositoryID(ctx, virtualStorageName, relativePathAlreadyExist) + require.NoError(t, err) + require.NoError(t, repoDS.CreateRepository( + ctx, + idRelativePathAlreadyExist, + virtualStorageName, + relativePathAlreadyExist, + relativePathAlreadyExist, + g1Cfg.Storages[0].Name, + nil, + nil, + true, + true, + )) + + runCmd := func(t *testing.T, args []string) (string, error) { + var stdout bytes.Buffer + app := cli.App{ + Reader: bytes.NewReader(nil), + Writer: &stdout, + ErrWriter: io.Discard, + HideHelpCommand: true, + Commands: []*cli.Command{ + newTrackRepositoryCommand(), + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Value: confPath, + }, + }, + } + err := app.Run(append([]string{progname, trackRepositoryCmdName}, args...)) + return stdout.String(), err + } + + t.Run("fails", func(t *testing.T) { + for _, tc := range []struct { + name string + args []string + errorMsg string + }{ + { + name: "positional arguments", + args: []string{"-virtual-storage=v", "-repository=r", "-authoritative-storage=s", "positional-arg"}, + errorMsg: "track-repository doesn't accept positional arguments", + }, + { + name: "virtual-storage is not set", + args: []string{ + "-repository", "path/to/test/repo_1", + "-authoritative-storage", authoritativeStorage, + }, + errorMsg: `Required flag "virtual-storage" not set`, + }, + { + name: "repository is not set", + args: []string{ + "-virtual-storage", virtualStorageName, + "-authoritative-storage", authoritativeStorage, + }, + errorMsg: `Required flag "repository" not set`, + }, + { + name: "authoritative-storage is not set", + args: []string{ + "-virtual-storage", virtualStorageName, + "-repository", "path/to/test/repo_1", + }, + errorMsg: `Required flag "authoritative-storage" not set`, + }, + { + name: "repository does not exist", + args: []string{ + "-virtual-storage", virtualStorageName, + "-repository", "path/to/test/repo_1", + "-authoritative-storage", authoritativeStorage, + }, + errorMsg: "attempting to track repository in praefect database: authoritative repository does not exist", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := runCmd(t, tc.args) + require.EqualError(t, err, tc.errorMsg) + }) + } + }) t.Run("ok", func(t *testing.T) { testCases := []struct { relativePath string desc string replicateImmediately bool - expectedOutput string + repositoryExists bool + expectedOutput []string }{ { - relativePath: "path/to/test/repo1", desc: "force replication", + relativePath: "path/to/test/repo1", replicateImmediately: true, - expectedOutput: "Finished replicating repository to \"gitaly-2\".\n", + expectedOutput: []string{"Finished replicating repository to \"gitaly-2\".\n"}, }, { - relativePath: "path/to/test/repo2", desc: "do not force replication", + relativePath: "path/to/test/repo2", replicateImmediately: false, - expectedOutput: "Added replication job to replicate repository to \"gitaly-2\".\n", + expectedOutput: []string{"Added replication job to replicate repository to \"gitaly-2\".\n"}, + }, + { + desc: "records already exist", + relativePath: relativePathAlreadyExist, + repositoryExists: true, + expectedOutput: []string{ + "repository is already tracked in praefect database", + "Finished adding new repository to be tracked in praefect database.", + "Added replication job to replicate repository to \"gitaly-2\".\n", + }, }, } @@ -178,27 +226,28 @@ func TestAddRepository_Exec(t *testing.T) { nodeMgr.Start(0, time.Hour) defer nodeMgr.Stop() - repoDS := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) exists, err := repoDS.RepositoryExists(ctx, virtualStorageName, tc.relativePath) require.NoError(t, err) - require.False(t, exists) + require.Equal(t, tc.repositoryExists, exists) // create the repo on Gitaly without Praefect knowing - require.NoError(t, createRepoThroughGitaly1(tc.relativePath)) - require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, tc.relativePath)) - require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, tc.relativePath)) - var stdout bytes.Buffer + if !tc.repositoryExists { + require.NoError(t, createRepoThroughGitaly1(tc.relativePath)) + require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, tc.relativePath)) + require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, tc.relativePath)) + } - addRepoCmd := &trackRepository{ - logger: logger, - virtualStorage: virtualStorageName, - relativePath: tc.relativePath, - authoritativeStorage: authoritativeStorage, - replicateImmediately: tc.replicateImmediately, - w: &stdout, + args := []string{ + "-virtual-storage", virtualStorageName, + "-repository", tc.relativePath, + "-authoritative-storage", authoritativeStorage, + } + if tc.replicateImmediately { + args = append(args, "-replicate-immediately") } + stdout, err := runCmd(t, args) + require.NoError(t, err) - require.NoError(t, addRepoCmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)) as := datastore.NewAssignmentStore(db, conf.StorageNames()) repositoryID, err := repoDS.GetRepositoryID(ctx, virtualStorageName, tc.relativePath) @@ -206,14 +255,20 @@ func TestAddRepository_Exec(t *testing.T) { assignments, err := as.GetHostAssignments(ctx, virtualStorageName, repositoryID) require.NoError(t, err) - require.Len(t, assignments, 2) + if tc.repositoryExists { + require.Len(t, assignments, 1) + } else { + require.Len(t, assignments, 2) + assert.Contains(t, assignments, g2Cfg.Storages[0].Name) + } assert.Contains(t, assignments, g1Cfg.Storages[0].Name) - assert.Contains(t, assignments, g2Cfg.Storages[0].Name) exists, err = repoDS.RepositoryExists(ctx, virtualStorageName, tc.relativePath) require.NoError(t, err) assert.True(t, exists) - assert.Contains(t, stdout.String(), tc.expectedOutput) + for _, expectedOutput := range tc.expectedOutput { + assert.Contains(t, stdout, expectedOutput) + } if !tc.replicateImmediately { queue := datastore.NewPostgresReplicationEventQueue(db) @@ -226,54 +281,6 @@ func TestAddRepository_Exec(t *testing.T) { } }) - t.Run("repository does not exist", func(t *testing.T) { - relativePath := "path/to/test/repo_1" - - cmd := &trackRepository{ - w: &bytes.Buffer{}, - logger: testhelper.NewDiscardingLogger(t), - virtualStorage: "praefect", - relativePath: relativePath, - authoritativeStorage: authoritativeStorage, - } - - assert.ErrorIs(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf), errAuthoritativeRepositoryNotExist) - }) - - t.Run("records already exist", func(t *testing.T) { - relativePath := "path/to/test/repo_2" - - require.NoError(t, createRepoThroughGitaly1(relativePath)) - require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, relativePath)) - require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, relativePath)) - - ds := datastore.NewPostgresRepositoryStore(db, conf.StorageNames()) - id, err := ds.ReserveRepositoryID(ctx, virtualStorageName, relativePath) - require.NoError(t, err) - require.NoError(t, ds.CreateRepository( - ctx, - id, - virtualStorageName, - relativePath, - relativePath, - g1Cfg.Storages[0].Name, - nil, - nil, - true, - true, - )) - - cmd := &trackRepository{ - w: &bytes.Buffer{}, - logger: testhelper.NewDiscardingLogger(t), - virtualStorage: virtualStorageName, - relativePath: relativePath, - authoritativeStorage: authoritativeStorage, - } - - assert.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)) - }) - t.Run("replication event exists", func(t *testing.T) { relativePath := "path/to/test/repo_3" @@ -281,19 +288,20 @@ func TestAddRepository_Exec(t *testing.T) { require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, relativePath)) require.NoDirExists(t, filepath.Join(g2Cfg.Storages[0].Path, relativePath)) - var stdout bytes.Buffer - cmd := &trackRepository{ - w: &stdout, - logger: testhelper.NewDiscardingLogger(t), - virtualStorage: virtualStorageName, - relativePath: relativePath, - authoritativeStorage: authoritativeStorage, - } - - assert.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)) + _, err := runCmd(t, []string{ + "-virtual-storage", virtualStorageName, + "-repository", relativePath, + "-authoritative-storage", authoritativeStorage, + }) + require.NoError(t, err) // running the command twice means we try creating the replication event // again, which should log the duplicate but not break the flow. - assert.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf)) - assert.Contains(t, stdout.String(), "replication event queue already has similar entry: replication event \"\" -> \"praefect\" -> \"gitaly-2\" -> \"path/to/test/repo_3\" already exists.") + stdout, err := runCmd(t, []string{ + "-virtual-storage", virtualStorageName, + "-repository", relativePath, + "-authoritative-storage", authoritativeStorage, + }) + require.NoError(t, err) + assert.Contains(t, stdout, "replication event queue already has similar entry: replication event \"\" -> \"praefect\" -> \"gitaly-2\" -> \"path/to/test/repo_3\" already exists.") }) } diff --git a/internal/cli/praefect/subcmd_verify.go b/internal/cli/praefect/subcmd_verify.go index 544353e2c..5e40e8a91 100644 --- a/internal/cli/praefect/subcmd_verify.go +++ b/internal/cli/praefect/subcmd_verify.go @@ -1,74 +1,93 @@ package praefect import ( - "context" "errors" - "flag" "fmt" - "io" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" + "github.com/urfave/cli/v2" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) const verifyCmdName = "verify" -type verifySubcommand struct { - stdout io.Writer - repositoryID int64 - virtualStorage string - storage string -} - -func newVerifySubcommand(stdout io.Writer) *verifySubcommand { - return &verifySubcommand{stdout: stdout} -} - -func (cmd *verifySubcommand) FlagSet() *flag.FlagSet { - fs := flag.NewFlagSet(metadataCmdName, flag.ContinueOnError) - fs.Int64Var(&cmd.repositoryID, "repository-id", 0, "the id of the repository to verify") - fs.StringVar(&cmd.virtualStorage, "virtual-storage", "", "the virtual storage to verify") - fs.StringVar(&cmd.storage, "storage", "", "the storage to verify") - return fs +func newVerifyCommand() *cli.Command { + return &cli.Command{ + Name: verifyCmdName, + Usage: "marks a discrete repository, or repositories of a storage, or repositories of a virtual storage to be verified", + Description: "The command marks a single repository if 'repository-id' flag is provided or a batch of\n" + + "repositories that belong to a particular storage or virtual storage to be checked they exist.\n" + + "The repository existence is confirmed if repository exists on the disk. That verification operation\n" + + "runs in background and executes verification asynchronously.", + HideHelpCommand: true, + Action: verifyAction, + Flags: []cli.Flag{ + &cli.Int64Flag{ + Name: "repository-id", + Usage: "the id of the repository to verify", + }, + &cli.StringFlag{ + Name: paramVirtualStorage, + Usage: "the virtual storage to verify", + }, + &cli.StringFlag{ + Name: "storage", + Usage: "the storage to verify", + }, + }, + Before: func(ctx *cli.Context) error { + if ctx.Args().Present() { + _ = cli.ShowSubcommandHelp(ctx) + return cli.Exit(unexpectedPositionalArgsError{Command: ctx.Command.Name}, 1) + } + return nil + }, + } } -func (cmd *verifySubcommand) Exec(flags *flag.FlagSet, cfg config.Config) error { - if flags.NArg() > 0 { - return unexpectedPositionalArgsError{Command: flags.Name()} +func verifyAction(appCtx *cli.Context) error { + logger := log.Default() + conf, err := getConfig(logger, appCtx.String(configFlagName)) + if err != nil { + return err } + repositoryID := appCtx.Int64("repository-id") + virtualStorage := appCtx.String(paramVirtualStorage) + storage := appCtx.String("storage") + var request gitalypb.MarkUnverifiedRequest switch { - case cmd.repositoryID != 0: - if cmd.virtualStorage != "" || cmd.storage != "" { + case repositoryID != 0: + if virtualStorage != "" || storage != "" { return errors.New("virtual storage and storage can't be provided with a repository ID") } - request.Selector = &gitalypb.MarkUnverifiedRequest_RepositoryId{RepositoryId: cmd.repositoryID} - case cmd.storage != "": - if cmd.virtualStorage == "" { + request.Selector = &gitalypb.MarkUnverifiedRequest_RepositoryId{RepositoryId: repositoryID} + case storage != "": + if virtualStorage == "" { return errors.New("virtual storage must be passed with storage") } request.Selector = &gitalypb.MarkUnverifiedRequest_Storage_{ Storage: &gitalypb.MarkUnverifiedRequest_Storage{ - VirtualStorage: cmd.virtualStorage, - Storage: cmd.storage, + VirtualStorage: virtualStorage, + Storage: storage, }, } - case cmd.virtualStorage != "": - request.Selector = &gitalypb.MarkUnverifiedRequest_VirtualStorage{VirtualStorage: cmd.virtualStorage} + case virtualStorage != "": + request.Selector = &gitalypb.MarkUnverifiedRequest_VirtualStorage{VirtualStorage: virtualStorage} default: return errors.New("(repository id), (virtual storage) or (virtual storage, storage) required") } - nodeAddr, err := getNodeAddress(cfg) + nodeAddr, err := getNodeAddress(conf) if err != nil { return fmt.Errorf("get node address: %w", err) } - ctx := context.TODO() - conn, err := subCmdDial(ctx, nodeAddr, cfg.Auth.Token, defaultDialTimeout) + ctx := appCtx.Context + conn, err := subCmdDial(ctx, nodeAddr, conf.Auth.Token, defaultDialTimeout) if err != nil { return fmt.Errorf("dial: %w", err) } @@ -79,7 +98,7 @@ func (cmd *verifySubcommand) Exec(flags *flag.FlagSet, cfg config.Config) error return fmt.Errorf("verify replicas: %w", err) } - fmt.Fprintf(cmd.stdout, "%d replicas marked unverified\n", response.GetReplicasMarked()) + fmt.Fprintf(appCtx.App.Writer, "%d replicas marked unverified\n", response.GetReplicasMarked()) return nil } diff --git a/internal/cli/praefect/subcmd_verify_test.go b/internal/cli/praefect/subcmd_verify_test.go index 19a1e5d9d..901579f17 100644 --- a/internal/cli/praefect/subcmd_verify_test.go +++ b/internal/cli/praefect/subcmd_verify_test.go @@ -4,9 +4,11 @@ import ( "bytes" "errors" "fmt" + "io" "testing" "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/info" @@ -50,6 +52,11 @@ func TestVerifySubcommand(t *testing.T) { expectedState state }{ { + desc: "positional arguments", + args: []string{"positional-arg"}, + error: cli.Exit(unexpectedPositionalArgsError{Command: "verify"}, 1), + }, + { desc: "no selector given", error: errors.New("(repository id), (virtual storage) or (virtual storage, storage) required"), expectedState: startingState, @@ -187,12 +194,36 @@ func TestVerifySubcommand(t *testing.T) { `) require.NoError(t, err) - stdout := &bytes.Buffer{} - cmd := newVerifySubcommand(stdout) + conf := config.Config{ + SocketPath: ln.Addr().String(), + VirtualStorages: []*config.VirtualStorage{ + { + Name: "vs", + Nodes: []*config.Node{ + {Address: "stub", Storage: "st"}, + }, + }, + }, + } + confPath := writeConfigToFile(t, conf) - fs := cmd.FlagSet() - require.NoError(t, fs.Parse(tc.args)) - err = cmd.Exec(fs, config.Config{SocketPath: ln.Addr().String()}) + var stdout bytes.Buffer + app := cli.App{ + Reader: bytes.NewReader(nil), + Writer: &stdout, + ErrWriter: io.Discard, + HideHelpCommand: true, + Commands: []*cli.Command{ + newVerifyCommand(), + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + Value: confPath, + }, + }, + } + err = app.Run(append([]string{progname, verifyCmdName}, tc.args...)) testhelper.RequireGrpcError(t, tc.error, err) if tc.error != nil { return diff --git a/internal/command/command.go b/internal/command/command.go index 4af839fbf..ea688a156 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/command/commandcounter" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" labkittracing "gitlab.com/gitlab-org/labkit/tracing" ) diff --git a/internal/command/command_test.go b/internal/command/command_test.go index 8eda53261..dd23c85d6 100644 --- a/internal/command/command_test.go +++ b/internal/command/command_test.go @@ -21,7 +21,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/cgroups" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/durationpb" ) func TestNew_environment(t *testing.T) { @@ -196,7 +200,18 @@ func TestNew_spawnTimeout(t *testing.T) { // And after some time we expect that spawning of the command fails due to the configured // timeout. - require.Equal(t, fmt.Errorf("process spawn timed out after 200ms"), <-errCh) + err := <-errCh + var structErr structerr.Error + require.ErrorAs(t, err, &structErr) + details := structErr.Details() + require.Len(t, details, 1) + + limitErr, ok := details[0].(*gitalypb.LimitError) + require.True(t, ok) + + testhelper.RequireGrpcCode(t, err, codes.ResourceExhausted) + require.Equal(t, "process spawn timed out after 200ms", limitErr.ErrorMessage) + require.Equal(t, durationpb.New(0), limitErr.RetryAfter) } func TestCommand_Wait_contextCancellationKillsCommand(t *testing.T) { diff --git a/internal/command/spawntoken.go b/internal/command/spawntoken.go index b49a32192..2b0065bf5 100644 --- a/internal/command/spawntoken.go +++ b/internal/command/spawntoken.go @@ -5,15 +5,15 @@ import ( "fmt" "time" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "github.com/kelseyhightower/envconfig" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/protobuf/types/known/durationpb" ) -const logDurationThreshold = 5 * time.Millisecond - var ( spawnTokens chan struct{} spawnConfig SpawnConfig @@ -63,31 +63,32 @@ func getSpawnToken(ctx context.Context) (putToken func(), err error) { select { case spawnTokens <- struct{}{}: - logTime(ctx, start, "spawn token acquired") + recordTime(ctx, start, "") return func() { <-spawnTokens }, nil case <-time.After(spawnConfig.Timeout): - logTime(ctx, start, "spawn token timeout") + recordTime(ctx, start, "spawn token timeout") spawnTimeoutCount.Inc() - return nil, fmt.Errorf("process spawn timed out after %v", spawnConfig.Timeout) + msg := fmt.Sprintf("process spawn timed out after %v", spawnConfig.Timeout) + return nil, structerr.NewResourceExhausted(msg).WithDetail(&gitalypb.LimitError{ + ErrorMessage: msg, + RetryAfter: durationpb.New(0), + }) case <-ctx.Done(): return nil, ctx.Err() } } -func logTime(ctx context.Context, start time.Time, msg string) { +func recordTime(ctx context.Context, start time.Time, msg string) { delta := time.Since(start) if stats := StatsFromContext(ctx); stats != nil { stats.RecordSum("command.spawn_token_wait_ms", int(delta.Milliseconds())) + if len(msg) != 0 { + stats.RecordMetadata("command.spawn_token_error", msg) + } } - - if delta < logDurationThreshold { - return - } - - ctxlogrus.Extract(ctx).WithField("spawn_queue_ms", delta.Seconds()*1000).Info(msg) } diff --git a/internal/command/spawntoken_test.go b/internal/command/spawntoken_test.go index 014802fff..62f534b7f 100644 --- a/internal/command/spawntoken_test.go +++ b/internal/command/spawntoken_test.go @@ -2,12 +2,19 @@ package command import ( "testing" + "time" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/durationpb" ) func TestGetSpawnToken_CommandStats(t *testing.T) { + t.Parallel() + ctx := testhelper.Context(t) ctx = InitContextStats(ctx) @@ -19,3 +26,41 @@ func TestGetSpawnToken_CommandStats(t *testing.T) { require.NotNil(t, stats) require.Contains(t, stats.Fields(), "command.spawn_token_wait_ms") } + +// This test modifies a global config, hence should never run in parallel +func TestGetSpawnToken_CommandStats_timeout(t *testing.T) { + priorTimeout := spawnConfig.Timeout + priorSpawnTokens := spawnTokens + + spawnConfig.Timeout = 1 * time.Millisecond + spawnTokens = make(chan struct{}, 1) + spawnTokens <- struct{}{} + defer func() { + spawnConfig.Timeout = priorTimeout + spawnTokens = priorSpawnTokens + }() + + ctx := testhelper.Context(t) + ctx = InitContextStats(ctx) + + _, err := getSpawnToken(ctx) + + var structErr structerr.Error + require.ErrorAs(t, err, &structErr) + details := structErr.Details() + require.Len(t, details, 1) + + limitErr, ok := details[0].(*gitalypb.LimitError) + require.True(t, ok) + + testhelper.RequireGrpcCode(t, err, codes.ResourceExhausted) + require.Equal(t, "process spawn timed out after 1ms", limitErr.ErrorMessage) + require.Equal(t, durationpb.New(0), limitErr.RetryAfter) + + stats := StatsFromContext(ctx) + require.NotNil(t, stats) + fields := stats.Fields() + + require.GreaterOrEqual(t, fields["command.spawn_token_wait_ms"], 0) + require.Equal(t, fields["command.spawn_token_error"], "spawn token timeout") +} diff --git a/internal/metadata/featureflag/context.go b/internal/featureflag/context.go index caa0f0e48..caa0f0e48 100644 --- a/internal/metadata/featureflag/context.go +++ b/internal/featureflag/context.go diff --git a/internal/metadata/featureflag/context_test.go b/internal/featureflag/context_test.go index 9dd864373..b41128d70 100644 --- a/internal/metadata/featureflag/context_test.go +++ b/internal/featureflag/context_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/require" - gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "google.golang.org/grpc/metadata" ) diff --git a/internal/metadata/featureflag/featureflag.go b/internal/featureflag/featureflag.go index 0ccf7ba31..0ccf7ba31 100644 --- a/internal/metadata/featureflag/featureflag.go +++ b/internal/featureflag/featureflag.go diff --git a/internal/metadata/featureflag/featureflag_test.go b/internal/featureflag/featureflag_test.go index 4246f0a1d..4246f0a1d 100644 --- a/internal/metadata/featureflag/featureflag_test.go +++ b/internal/featureflag/featureflag_test.go diff --git a/internal/metadata/featureflag/ff_fix_routing_with_additional_repository.go b/internal/featureflag/ff_fix_routing_with_additional_repository.go index 4d01587e0..2cd2b63dd 100644 --- a/internal/metadata/featureflag/ff_fix_routing_with_additional_repository.go +++ b/internal/featureflag/ff_fix_routing_with_additional_repository.go @@ -8,6 +8,6 @@ package featureflag var FixRoutingWithAdditionalRepository = NewFeatureFlag( "fix_routing_with_additional_repository", "v16.0.0", - "", + "https://gitlab.com/gitlab-org/gitaly/-/issues/5134", false, ) diff --git a/internal/metadata/featureflag/ff_geometric_repacking.go b/internal/featureflag/ff_geometric_repacking.go index 8665b9d21..8665b9d21 100644 --- a/internal/metadata/featureflag/ff_geometric_repacking.go +++ b/internal/featureflag/ff_geometric_repacking.go diff --git a/internal/metadata/featureflag/ff_localrepo_read_object_cached.go b/internal/featureflag/ff_localrepo_read_object_cached.go index 40e60d1f1..40e60d1f1 100644 --- a/internal/metadata/featureflag/ff_localrepo_read_object_cached.go +++ b/internal/featureflag/ff_localrepo_read_object_cached.go diff --git a/internal/metadata/featureflag/ff_pack_objects_limiting_remote_ip.go b/internal/featureflag/ff_pack_objects_limiting_remote_ip.go index e4d9a5c1e..e4d9a5c1e 100644 --- a/internal/metadata/featureflag/ff_pack_objects_limiting_remote_ip.go +++ b/internal/featureflag/ff_pack_objects_limiting_remote_ip.go diff --git a/internal/metadata/featureflag/ff_run_cmd_in_cgroup.go b/internal/featureflag/ff_run_cmd_in_cgroup.go index 356bae752..356bae752 100644 --- a/internal/metadata/featureflag/ff_run_cmd_in_cgroup.go +++ b/internal/featureflag/ff_run_cmd_in_cgroup.go diff --git a/internal/git/catfile/cache.go b/internal/git/catfile/cache.go index 63fd357bb..69bebbec1 100644 --- a/internal/git/catfile/cache.go +++ b/internal/git/catfile/cache.go @@ -11,8 +11,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" "gitlab.com/gitlab-org/labkit/correlation" ) diff --git a/internal/git/catfile/object_info_reader_test.go b/internal/git/catfile/object_info_reader_test.go index c21845d37..ae2ea6b29 100644 --- a/internal/git/catfile/object_info_reader_test.go +++ b/internal/git/catfile/object_info_reader_test.go @@ -230,12 +230,10 @@ func TestObjectInfoReader_queue(t *testing.T) { commitInfo := ObjectInfo{ Oid: commitOID, Type: "commit", - Size: func() int64 { - if gittest.ObjectHashIsSHA256() { - return 201 - } - return 177 - }(), + Size: gittest.ObjectHashDependent(t, map[string]int64{ + "sha1": 177, + "sha256": 201, + }), Format: gittest.DefaultObjectHash.Format, } diff --git a/internal/git/command_factory_cgroup_test.go b/internal/git/command_factory_cgroup_test.go index 6336de8f0..39404db23 100644 --- a/internal/git/command_factory_cgroup_test.go +++ b/internal/git/command_factory_cgroup_test.go @@ -6,9 +6,9 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/cgroups" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" ) diff --git a/internal/git/command_options.go b/internal/git/command_options.go index 2fae98a73..e30ff1a45 100644 --- a/internal/git/command_options.go +++ b/internal/git/command_options.go @@ -9,9 +9,9 @@ import ( "strings" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/x509" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/git/execution_environment.go b/internal/git/execution_environment.go index f93132eb0..504dd7c68 100644 --- a/internal/git/execution_environment.go +++ b/internal/git/execution_environment.go @@ -9,8 +9,8 @@ import ( "path/filepath" "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "golang.org/x/sys/unix" ) diff --git a/internal/git/gitpipe/catfile_info.go b/internal/git/gitpipe/catfile_info.go index 537c17e2d..8a25cb5d1 100644 --- a/internal/git/gitpipe/catfile_info.go +++ b/internal/git/gitpipe/catfile_info.go @@ -80,10 +80,10 @@ func CatfileInfo( requestChan := make(chan catfileInfoRequest, 32) go func() { defer func() { - close(requestChan) if atomic.AddInt32(&queueRefcount, -1) == 0 { queueCleanup() } + close(requestChan) }() var i int64 @@ -132,10 +132,10 @@ func CatfileInfo( resultChan := make(chan CatfileInfoResult) go func() { defer func() { - close(resultChan) if atomic.AddInt32(&queueRefcount, -1) == 0 { queueCleanup() } + close(resultChan) }() // It's fine to iterate over the request channel without paying attention to diff --git a/internal/git/gitpipe/catfile_info_test.go b/internal/git/gitpipe/catfile_info_test.go index 624c3b041..e0e51054d 100644 --- a/internal/git/gitpipe/catfile_info_test.go +++ b/internal/git/gitpipe/catfile_info_test.go @@ -75,7 +75,7 @@ func TestCatfileInfo(t *testing.T) { {OID: blobID, ObjectName: []byte("branch-test.txt")}, }, expectedResults: []CatfileInfoResult{ - {ObjectInfo: &catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(43, 55), Format: gittest.DefaultObjectHash.Format}}, + {ObjectInfo: &catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(t, 43, 55), Format: gittest.DefaultObjectHash.Format}}, {ObjectInfo: &catfile.ObjectInfo{Oid: blobID, Type: "blob", Size: 8, Format: gittest.DefaultObjectHash.Format}, ObjectName: []byte("branch-test.txt")}, }, }, @@ -277,6 +277,7 @@ func TestCatfileInfo(t *testing.T) { // We now consume all the input of the iterator. require.True(t, it.Next()) require.False(t, it.Next()) + require.NoError(t, it.Err()) // Which means that the queue should now be unused, so we can again use it. _, err = CatfileInfo(ctx, objectInfoReader, NewRevisionIterator(ctx, input)) @@ -306,8 +307,8 @@ func TestCatfileInfoAllObjects(t *testing.T) { {ObjectInfo: &catfile.ObjectInfo{Oid: blob1, Type: "blob", Size: 6, Format: gittest.DefaultObjectHash.Format}}, {ObjectInfo: &catfile.ObjectInfo{Oid: blob2, Type: "blob", Size: 6, Format: gittest.DefaultObjectHash.Format}}, {ObjectInfo: &catfile.ObjectInfo{Oid: gittest.DefaultObjectHash.EmptyTreeOID, Type: "tree", Size: 0, Format: gittest.DefaultObjectHash.Format}}, - {ObjectInfo: &catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(34, 46), Format: gittest.DefaultObjectHash.Format}}, - {ObjectInfo: &catfile.ObjectInfo{Oid: commit, Type: "commit", Size: hashDependentObjectSize(177, 201), Format: gittest.DefaultObjectHash.Format}}, + {ObjectInfo: &catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(t, 34, 46), Format: gittest.DefaultObjectHash.Format}}, + {ObjectInfo: &catfile.ObjectInfo{Oid: commit, Type: "commit", Size: hashDependentObjectSize(t, 177, 201), Format: gittest.DefaultObjectHash.Format}}, } t.Run("successful", func(t *testing.T) { diff --git a/internal/git/gitpipe/catfile_object_test.go b/internal/git/gitpipe/catfile_object_test.go index 81977fa18..b53d86aea 100644 --- a/internal/git/gitpipe/catfile_object_test.go +++ b/internal/git/gitpipe/catfile_object_test.go @@ -71,11 +71,11 @@ func TestCatfileObject(t *testing.T) { { desc: "revlist result with object names", catfileInfoInputs: []CatfileInfoResult{ - {ObjectInfo: &catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(43, 55)}}, + {ObjectInfo: &catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(t, 43, 55)}}, {ObjectInfo: &catfile.ObjectInfo{Oid: blobID, Type: "blob", Size: 59}, ObjectName: []byte("branch-test.txt")}, }, expectedResults: []CatfileObjectResult{ - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(43, 55)}}}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: treeID, Type: "tree", Size: hashDependentObjectSize(t, 43, 55)}}}, {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: blobID, Type: "blob", Size: 8}}, ObjectName: []byte("branch-test.txt")}, }, }, diff --git a/internal/git/gitpipe/pipeline_test.go b/internal/git/gitpipe/pipeline_test.go index 4d2762b89..b108fcb03 100644 --- a/internal/git/gitpipe/pipeline_test.go +++ b/internal/git/gitpipe/pipeline_test.go @@ -111,9 +111,9 @@ func TestPipeline_revlist(t *testing.T) { WithObjects(), }, expectedResults: []CatfileObjectResult{ - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(66, 90)}}}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(t, 66, 90)}}}, {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: blobB, Type: "blob", Size: 1}}, ObjectName: []byte("blob")}, - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: subtree, Type: "tree", Size: hashDependentObjectSize(35, 47)}}, ObjectName: []byte("subtree")}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: subtree, Type: "tree", Size: hashDependentObjectSize(t, 35, 47)}}, ObjectName: []byte("subtree")}, {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: blobA, Type: "blob", Size: 6}}, ObjectName: []byte("subtree/subblob")}, }, }, @@ -152,10 +152,10 @@ func TestPipeline_revlist(t *testing.T) { WithObjects(), }, expectedResults: []CatfileObjectResult{ - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: commitB, Type: "commit", Size: hashDependentObjectSize(225, 273)}}}, - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(66, 90)}}}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: commitB, Type: "commit", Size: hashDependentObjectSize(t, 225, 273)}}}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: tree, Type: "tree", Size: hashDependentObjectSize(t, 66, 90)}}}, {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: blobB, Type: "blob", Size: 1}}, ObjectName: []byte("blob")}, - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: subtree, Type: "tree", Size: hashDependentObjectSize(35, 47)}}, ObjectName: []byte("subtree")}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: subtree, Type: "tree", Size: hashDependentObjectSize(t, 35, 47)}}, ObjectName: []byte("subtree")}, {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: blobA, Type: "blob", Size: 6}}, ObjectName: []byte("subtree/subblob")}, }, }, @@ -166,7 +166,7 @@ func TestPipeline_revlist(t *testing.T) { commitB.String(), }, expectedResults: []CatfileObjectResult{ - {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: commitB, Type: "commit", Size: hashDependentObjectSize(225, 273)}}}, + {Object: &catfile.Object{ObjectInfo: catfile.ObjectInfo{Oid: commitB, Type: "commit", Size: hashDependentObjectSize(t, 225, 273)}}}, }, }, { diff --git a/internal/git/gitpipe/testhelper_test.go b/internal/git/gitpipe/testhelper_test.go index ba13f75c2..df850aec7 100644 --- a/internal/git/gitpipe/testhelper_test.go +++ b/internal/git/gitpipe/testhelper_test.go @@ -51,9 +51,9 @@ func (ch *chanObjectIterator) ObjectName() []byte { return []byte("idontcare") } -func hashDependentObjectSize(sha1Size, sha256Size int64) int64 { - if gittest.ObjectHashIsSHA256() { - return sha256Size - } - return sha1Size +func hashDependentObjectSize(tb testing.TB, sha1Size, sha256Size int64) int64 { + return gittest.ObjectHashDependent(tb, map[string]int64{ + "sha1": sha1Size, + "sha256": sha256Size, + }) } diff --git a/internal/git/gittest/objects.go b/internal/git/gittest/objects.go index 65d2362ee..e39611d21 100644 --- a/internal/git/gittest/objects.go +++ b/internal/git/gittest/objects.go @@ -20,6 +20,15 @@ func ObjectHashIsSHA256() bool { return DefaultObjectHash.EmptyTreeOID == git.ObjectHashSHA256.EmptyTreeOID } +// ObjectHashDependent returns the value from the given map that is associated with the default +// object hash (e.g. "sha1", "sha256"). Fails in case the map doesn't contain the current object +// hash. +func ObjectHashDependent[T any](tb testing.TB, valuesByObjectHash map[string]T) T { + tb.Helper() + require.Contains(tb, valuesByObjectHash, DefaultObjectHash.Format) + return valuesByObjectHash[DefaultObjectHash.Format] +} + // ListObjects returns a list of all object IDs in the repository. func ListObjects(tb testing.TB, cfg config.Cfg, repoPath string) []git.ObjectID { tb.Helper() diff --git a/internal/git/hooks_options.go b/internal/git/hooks_options.go index de28cab68..41002bede 100644 --- a/internal/git/hooks_options.go +++ b/internal/git/hooks_options.go @@ -5,11 +5,11 @@ import ( "errors" "fmt" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" labkittracing "gitlab.com/gitlab-org/labkit/tracing" diff --git a/internal/git/hooks_options_test.go b/internal/git/hooks_options_test.go index 3dfefd7f2..d96530a64 100644 --- a/internal/git/hooks_options_test.go +++ b/internal/git/hooks_options_test.go @@ -8,7 +8,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" grpcmetadata "google.golang.org/grpc/metadata" diff --git a/internal/git/hooks_payload.go b/internal/git/hooks_payload.go index 7bcf2acc7..2dfd8f15d 100644 --- a/internal/git/hooks_payload.go +++ b/internal/git/hooks_payload.go @@ -7,8 +7,8 @@ import ( "fmt" "strings" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/protobuf/encoding/protojson" diff --git a/internal/git/hooks_payload_test.go b/internal/git/hooks_payload_test.go index c733d8e7e..e73e755a6 100644 --- a/internal/git/hooks_payload_test.go +++ b/internal/git/hooks_payload_test.go @@ -5,9 +5,9 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" diff --git a/internal/git/housekeeping/clean_stale_data.go b/internal/git/housekeeping/clean_stale_data.go index 3e7908236..a21b4367e 100644 --- a/internal/git/housekeeping/clean_stale_data.go +++ b/internal/git/housekeeping/clean_stale_data.go @@ -15,6 +15,7 @@ import ( log "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" @@ -40,10 +41,56 @@ var lockfiles = []string{ "objects/info/commit-graphs/commit-graph-chain.lock", } -type staleFileFinderFn func(context.Context, string) ([]string, error) +type ( + findStaleFileFunc func(context.Context, string) ([]string, error) + cleanupRepoFunc func(context.Context, *localrepo.Repo) (int, error) + cleanupRepoWithTxManagerFunc func(context.Context, *localrepo.Repo, transaction.Manager) (int, error) +) + +// CleanStaleDataConfig is the configuration for running CleanStaleData. It is used to define +// the different types of cleanups we want to run. +type CleanStaleDataConfig struct { + staleFileFinders map[string]findStaleFileFunc + repoCleanups map[string]cleanupRepoFunc + repoCleanupWithTxManagers map[string]cleanupRepoWithTxManagerFunc +} + +// OnlyStaleReferenceLockCleanup returns a config which only contains a +// stale reference lock cleaner with the provided grace period. +func OnlyStaleReferenceLockCleanup(gracePeriod time.Duration) CleanStaleDataConfig { + return CleanStaleDataConfig{ + staleFileFinders: map[string]findStaleFileFunc{ + "reflocks": findStaleReferenceLocks(gracePeriod), + }, + } +} -// CleanStaleData cleans up any stale data in the repository. -func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.Repo) error { +// DefaultStaleDataCleanup is the default configuration for CleanStaleData +// which contains all the cleanup functions. +func DefaultStaleDataCleanup() CleanStaleDataConfig { + return CleanStaleDataConfig{ + staleFileFinders: map[string]findStaleFileFunc{ + "objects": findTemporaryObjects, + "locks": findStaleLockfiles, + "refs": findBrokenLooseReferences, + "reflocks": findStaleReferenceLocks(referenceLockfileGracePeriod), + "packfilelocks": findStalePackFileLocks, + "packedrefslock": findPackedRefsLock, + "packedrefsnew": findPackedRefsNew, + "serverinfo": findServerInfo, + }, + repoCleanups: map[string]cleanupRepoFunc{ + "refsemptydir": removeRefEmptyDirs, + "configsections": pruneEmptyConfigSections, + }, + repoCleanupWithTxManagers: map[string]cleanupRepoWithTxManagerFunc{ + "configkeys": removeUnnecessaryConfig, + }, + } +} + +// CleanStaleData removes any stale data in the repository as per the provided configuration. +func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.Repo, cfg CleanStaleDataConfig) error { span, ctx := tracing.StartSpanIfHasParent(ctx, "housekeeping.CleanStaleData", nil) defer span.Finish() @@ -71,16 +118,7 @@ func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo. }() var filesToPrune []string - for staleFileType, staleFileFinder := range map[string]staleFileFinderFn{ - "objects": findTemporaryObjects, - "locks": findStaleLockfiles, - "refs": findBrokenLooseReferences, - "reflocks": findStaleReferenceLocks, - "packfilelocks": findStalePackFileLocks, - "packedrefslock": findPackedRefsLock, - "packedrefsnew": findPackedRefsNew, - "serverinfo": findServerInfo, - } { + for staleFileType, staleFileFinder := range cfg.staleFileFinders { staleFiles, err := staleFileFinder(ctx, repoPath) if err != nil { return fmt.Errorf("housekeeping failed to find %s: %w", staleFileType, err) @@ -100,31 +138,20 @@ func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo. } } - prunedRefDirs, err := removeRefEmptyDirs(ctx, repo) - staleDataByType["refsemptydir"] = prunedRefDirs - if err != nil { - return fmt.Errorf("housekeeping could not remove empty refs: %w", err) - } - - unnecessaryConfigRegex := "^(http\\..+\\.extraheader|remote\\..+\\.(fetch|mirror|prune|url)|core\\.(commitgraph|sparsecheckout|splitindex))$" - if err := repo.UnsetMatchingConfig(ctx, unnecessaryConfigRegex, m.txManager); err != nil { - if !errors.Is(err, git.ErrNotFound) { - return fmt.Errorf("housekeeping could not unset unnecessary config lines: %w", err) + for repoCleanupName, repoCleanupFn := range cfg.repoCleanups { + cleanupCount, err := repoCleanupFn(ctx, repo) + staleDataByType[repoCleanupName] = cleanupCount + if err != nil { + return fmt.Errorf("housekeeping could not perform cleanup %s: %w", repoCleanupName, err) } - staleDataByType["configkeys"] = 0 - } else { - // If we didn't get an error we know that we've deleted _something_. We just set - // this variable to `1` because we don't count how many keys we have deleted. It's - // probably good enough: we only want to know whether we're still pruning such old - // configuration or not, but typically don't care how many there are so that we know - // when to delete this cleanup of legacy data. - staleDataByType["configkeys"] = 1 } - skippedSections, err := pruneEmptyConfigSections(ctx, repo) - staleDataByType["configsections"] = skippedSections - if err != nil { - return fmt.Errorf("failed pruning empty sections: %w", err) + for repoCleanupName, repoCleanupFn := range cfg.repoCleanupWithTxManagers { + cleanupCount, err := repoCleanupFn(ctx, repo, m.txManager) + staleDataByType[repoCleanupName] = cleanupCount + if err != nil { + return fmt.Errorf("housekeeping could not perform cleanup (with TxManager) %s: %w", repoCleanupName, err) + } } return nil @@ -426,47 +453,68 @@ func findBrokenLooseReferences(ctx context.Context, repoPath string) ([]string, return brokenRefs, nil } -// findStaleReferenceLocks scans the refdb for stale locks for loose references. -func findStaleReferenceLocks(ctx context.Context, repoPath string) ([]string, error) { - var staleReferenceLocks []string +// findStaleReferenceLocks provides a function which scans the refdb for stale locks +// and loose references against the provided grace period. +func findStaleReferenceLocks(gracePeriod time.Duration) findStaleFileFunc { + return func(_ context.Context, repoPath string) ([]string, error) { + var staleReferenceLocks []string - if err := filepath.WalkDir(filepath.Join(repoPath, "refs"), func(path string, dirEntry fs.DirEntry, err error) error { - if err != nil { - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrPermission) { + if err := filepath.WalkDir(filepath.Join(repoPath, "refs"), func(path string, dirEntry fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) || errors.Is(err, fs.ErrPermission) { + return nil + } + + return err + } + + if dirEntry.IsDir() { return nil } - return err - } + if !strings.HasSuffix(dirEntry.Name(), ".lock") { + return nil + } - if dirEntry.IsDir() { - return nil - } + fi, err := dirEntry.Info() + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } - if !strings.HasSuffix(dirEntry.Name(), ".lock") { - return nil - } + return fmt.Errorf("statting reference lock: %w", err) + } - fi, err := dirEntry.Info() - if err != nil { - if errors.Is(err, fs.ErrNotExist) { + if time.Since(fi.ModTime()) < gracePeriod { return nil } - return fmt.Errorf("statting reference lock: %w", err) + staleReferenceLocks = append(staleReferenceLocks, path) + return nil + }); err != nil { + return nil, fmt.Errorf("walking refs: %w", err) } - if time.Since(fi.ModTime()) < referenceLockfileGracePeriod { - return nil + return staleReferenceLocks, nil + } +} + +func removeUnnecessaryConfig(ctx context.Context, repository *localrepo.Repo, txManager transaction.Manager) (int, error) { + unnecessaryConfigRegex := "^(http\\..+\\.extraheader|remote\\..+\\.(fetch|mirror|prune|url)|core\\.(commitgraph|sparsecheckout|splitindex))$" + if err := repository.UnsetMatchingConfig(ctx, unnecessaryConfigRegex, txManager); err != nil { + if !errors.Is(err, git.ErrNotFound) { + return 0, fmt.Errorf("housekeeping could not unset unnecessary config lines: %w", err) } - staleReferenceLocks = append(staleReferenceLocks, path) - return nil - }); err != nil { - return nil, fmt.Errorf("walking refs: %w", err) + return 0, nil } - return staleReferenceLocks, nil + // If we didn't get an error we know that we've deleted _something_. We just set + // this variable to `1` because we don't count how many keys we have deleted. It's + // probably good enough: we only want to know whether we're still pruning such old + // configuration or not, but typically don't care how many there are so that we know + // when to delete this cleanup of legacy data. + return 1, nil } // findPackedRefsLock returns stale lockfiles for the packed-refs file. diff --git a/internal/git/housekeeping/clean_stale_data_test.go b/internal/git/housekeeping/clean_stale_data_test.go index bdb256021..52f5327be 100644 --- a/internal/git/housekeeping/clean_stale_data_test.go +++ b/internal/git/housekeeping/clean_stale_data_test.go @@ -13,10 +13,10 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -203,6 +203,26 @@ func requireCleanStaleDataMetrics(t *testing.T, m *RepositoryManager, metrics cl require.NoError(t, testutil.CollectAndCompare(m, strings.NewReader(builder.String()), "gitaly_housekeeping_pruned_files_total")) } +func requireReferenceLockCleanupMetrics(t *testing.T, m *RepositoryManager, metrics cleanStaleDataMetrics) { + t.Helper() + + var builder strings.Builder + + _, err := builder.WriteString("# HELP gitaly_housekeeping_pruned_files_total Total number of files pruned\n") + require.NoError(t, err) + _, err = builder.WriteString("# TYPE gitaly_housekeeping_pruned_files_total counter\n") + require.NoError(t, err) + + for metric, expectedValue := range map[string]int{ + "reflocks": metrics.reflocks, + } { + _, err := builder.WriteString(fmt.Sprintf("gitaly_housekeeping_pruned_files_total{filetype=%q} %d\n", metric, expectedValue)) + require.NoError(t, err) + } + + require.NoError(t, testutil.CollectAndCompare(m, strings.NewReader(builder.String()), "gitaly_housekeeping_pruned_files_total")) +} + func TestRepositoryManager_CleanStaleData(t *testing.T) { t.Parallel() testcases := []struct { @@ -381,7 +401,7 @@ func TestRepositoryManager_CleanStaleData(t *testing.T) { mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, e := range tc.entries { e.validate(t, repoPath) @@ -486,7 +506,7 @@ func TestRepositoryManager_CleanStaleData_references(t *testing.T) { mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) var actual []string require.NoError(t, filepath.Walk(filepath.Join(repoPath, "refs"), func(path string, info os.FileInfo, _ error) error { @@ -613,7 +633,7 @@ func TestRepositoryManager_CleanStaleData_emptyRefDirs(t *testing.T) { mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, e := range tc.entries { e.validate(t, repoPath) @@ -655,7 +675,7 @@ func TestRepositoryManager_CleanStaleData_withSpecificFile(t *testing.T) { desc string file string subdirs []string - finder staleFileFinderFn + finder findStaleFileFunc expectedMetrics cleanStaleDataMetrics }{ { @@ -738,7 +758,7 @@ func TestRepositoryManager_CleanStaleData_withSpecificFile(t *testing.T) { repo := localrepo.NewTestRepo(t, cfg, repoProto) mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, subcase := range []struct { desc string entry entry @@ -782,7 +802,7 @@ func TestRepositoryManager_CleanStaleData_withSpecificFile(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, subcase.expectedFiles, staleFiles) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) entry.validate(t, repoPath) }) @@ -836,7 +856,7 @@ func TestRepositoryManager_CleanStaleData_serverInfo(t *testing.T) { mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, entry := range entries { entry.validate(t, repoPath) @@ -856,6 +876,9 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { entries []entry expectedReferenceLocks []string expectedMetrics cleanStaleDataMetrics + gracePeriod time.Duration + cfg CleanStaleDataConfig + metricsCompareFn func(t *testing.T, m *RepositoryManager, metrics cleanStaleDataMetrics) }{ { desc: "fresh lock is kept", @@ -865,6 +888,26 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { f("main.lock", withAge(10*time.Minute)), }), }, + gracePeriod: referenceLockfileGracePeriod, + cfg: DefaultStaleDataCleanup(), + }, + { + desc: "fresh lock is deleted when grace period is low", + entries: []entry{ + d("refs", []entry{ + f("main", withAge(10*time.Minute)), + f("main.lock", withAge(10*time.Minute), expectDeletion), + }), + }, + expectedReferenceLocks: []string{ + "refs/main.lock", + }, + expectedMetrics: cleanStaleDataMetrics{ + reflocks: 1, + }, + gracePeriod: time.Second, + cfg: OnlyStaleReferenceLockCleanup(time.Second), + metricsCompareFn: requireReferenceLockCleanupMetrics, }, { desc: "stale lock is deleted", @@ -880,6 +923,8 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { expectedMetrics: cleanStaleDataMetrics{ reflocks: 1, }, + gracePeriod: referenceLockfileGracePeriod, + cfg: DefaultStaleDataCleanup(), }, { desc: "nested reference locks are deleted", @@ -907,6 +952,8 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { expectedMetrics: cleanStaleDataMetrics{ reflocks: 3, }, + gracePeriod: referenceLockfileGracePeriod, + cfg: DefaultStaleDataCleanup(), }, } { tc := tc @@ -933,19 +980,23 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { expectedReferenceLocks = append(expectedReferenceLocks, filepath.Join(repoPath, referenceLock)) } - staleLockfiles, err := findStaleReferenceLocks(ctx, repoPath) + staleLockfiles, err := findStaleReferenceLocks(tc.gracePeriod)(ctx, repoPath) require.NoError(t, err) require.ElementsMatch(t, expectedReferenceLocks, staleLockfiles) mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, tc.cfg)) for _, e := range tc.entries { e.validate(t, repoPath) } - requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics) + if tc.metricsCompareFn != nil { + tc.metricsCompareFn(t, mgr, tc.expectedMetrics) + } else { + requireCleanStaleDataMetrics(t, mgr, tc.expectedMetrics) + } }) } } @@ -1048,7 +1099,7 @@ func TestRepositoryManager_CleanStaleData_missingRepo(t *testing.T) { require.NoError(t, os.RemoveAll(repoPath)) - require.NoError(t, NewManager(cfg.Prometheus, nil).CleanStaleData(ctx, repo)) + require.NoError(t, NewManager(cfg.Prometheus, nil).CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) } func TestRepositoryManager_CleanStaleData_unsetConfiguration(t *testing.T) { @@ -1089,7 +1140,7 @@ func TestRepositoryManager_CleanStaleData_unsetConfiguration(t *testing.T) { mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, `[core] repositoryformatversion = 0 @@ -1128,7 +1179,7 @@ func TestRepositoryManager_CleanStaleData_unsetConfigurationTransactional(t *tes AuthInfo: backchannel.WithID(nil, 1234), }) - require.NoError(t, NewManager(cfg.Prometheus, txManager).CleanStaleData(ctx, repo)) + require.NoError(t, NewManager(cfg.Prometheus, txManager).CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, 2, len(txManager.Votes())) configKeys := gittest.Exec(t, cfg, "-C", repoPath, "config", "--list", "--local", "--name-only") @@ -1180,7 +1231,7 @@ func TestRepositoryManager_CleanStaleData_pruneEmptyConfigSections(t *testing.T) mgr := NewManager(cfg.Prometheus, nil) - require.NoError(t, mgr.CleanStaleData(ctx, repo)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, `[core] repositoryformatversion = 0 filemode = true diff --git a/internal/git/housekeeping/manager.go b/internal/git/housekeeping/manager.go index 36e26fb86..173ad06c2 100644 --- a/internal/git/housekeeping/manager.go +++ b/internal/git/housekeeping/manager.go @@ -14,8 +14,8 @@ import ( // Manager is a housekeeping manager. It is supposed to handle housekeeping tasks for repositories // such as the cleanup of unneeded files and optimizations for the repository's data structures. type Manager interface { - // CleanStaleData removes any stale data in the repository. - CleanStaleData(context.Context, *localrepo.Repo) error + // CleanStaleData removes any stale data in the repository as per the provided configuration. + CleanStaleData(ctx context.Context, repo *localrepo.Repo, cfg CleanStaleDataConfig) error // OptimizeRepository optimizes the repository's data structures such that it can be more // efficiently served. OptimizeRepository(context.Context, *localrepo.Repo, ...OptimizeRepositoryOption) error diff --git a/internal/git/housekeeping/optimization_strategy.go b/internal/git/housekeeping/optimization_strategy.go index fa44476df..37775bca3 100644 --- a/internal/git/housekeeping/optimization_strategy.go +++ b/internal/git/housekeeping/optimization_strategy.go @@ -5,9 +5,9 @@ import ( "math" "time" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" ) const ( @@ -62,16 +62,6 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context return false, RepackObjectsConfig{} } - cfg := RepackObjectsConfig{ - // We cannot write bitmaps when there are alternates as we don't have full closure - // of all objects in the packfile. This also does not currently work with multi pack - // indices. - WriteBitmap: len(s.info.Alternates) == 0, - // We want to always update the multi-pack-index while we're already at it repacking - // some of the objects. - WriteMultiPackIndex: true, - } - // There is a bug in Git that causes geometric repacking to fail in some circumstances when // the repository is connected to an object pool. While we're upstreaming the fix via // https://gitlab.com/gitlab-org/git/-/issues/152 we thus disable geometric repacks in any @@ -85,12 +75,60 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context // have backported them into Git v2.40.0.gl1. So if we detect that the current Git version // does indeed support geometric repacking then we can enable this even when the repository // is part of an object pool. - canUseGeometricRepacking := len(s.info.Alternates) == 0 || s.gitVersion.GeometricRepackingSupportsAlternates() + canUseGeometricRepacking := len(s.info.Alternates.ObjectDirectories) == 0 || s.gitVersion.GeometricRepackingSupportsAlternates() if canUseGeometricRepacking && featureflag.GeometricRepacking.IsEnabled(ctx) { nonCruftPackfilesCount := s.info.Packfiles.Count - s.info.Packfiles.CruftCount timeSinceLastFullRepack := time.Since(s.info.Packfiles.LastFullRepack) + fullRepackCfg := RepackObjectsConfig{ + // We use the full-with-unreachable strategy to also pack all unreachable + // objects into the packfile. This only happens for object pools though, + // as they should never delete objects. + Strategy: RepackObjectsStrategyFullWithUnreachable, + // We cannot write bitmaps when there are alternates as we don't have full + // closure of all objects in the packfile. + WriteBitmap: len(s.info.Alternates.ObjectDirectories) == 0, + // We rewrite all packfiles into a single one and thus change the layout + // that was indexed by the multi-pack-index. We thus need to update it, as + // well. + WriteMultiPackIndex: true, + } + if !s.info.IsObjectPool { + // When we don't have an object pool at hand we want to be able to expire + // unreachable objects. We thus use cruft packs with an expiry date. + fullRepackCfg.Strategy = RepackObjectsStrategyFullWithCruft + fullRepackCfg.CruftExpireBefore = s.expireBefore + } + + geometricRepackCfg := RepackObjectsConfig{ + Strategy: RepackObjectsStrategyGeometric, + // We cannot write bitmaps when there are alternates as we don't have full + // closure of all objects in the packfile. + WriteBitmap: len(s.info.Alternates.ObjectDirectories) == 0, + // We're rewriting packfiles that may be part of the multi-pack-index, so we + // do want to update it to reflect the new layout. + WriteMultiPackIndex: true, + } + + // Incremental repacks only pack unreachable objects into a new pack. As we only + // perform this kind of repack in the case where the overall repository structure + // looks good to us we try to do use the least amount of resources to update them. + // We thus neither update the multi-pack-index nor do we update bitmaps. + incrementalRepackCfg := RepackObjectsConfig{ + Strategy: RepackObjectsStrategyIncrementalWithUnreachable, + WriteBitmap: false, + WriteMultiPackIndex: false, + } + + // When alternative object directories have been modified since our last full repack + // then we have likely joined an object pool since then. This means that we'll want + // to perform a full repack in order to deduplicate objects that are part of the + // object pool. + if s.info.Alternates.LastModified.After(s.info.Packfiles.LastFullRepack) { + return true, fullRepackCfg + } + // It is mandatory for us that we perform regular full repacks in repositories so // that we can evict objects which are unreachable into a separate cruft pack. So in // the case where we have more than one non-cruft packfiles and the time since our @@ -118,30 +156,14 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context // islands into account we can get rid of this condition and only do geometric // repacks. if nonCruftPackfilesCount > 1 && timeSinceLastFullRepack > FullRepackCooldownPeriod { - if s.info.IsObjectPool { - // Using cruft packs would be pointless here as we don't ever want - // to expire unreachable objects. And we don't want to explode - // unreachable objects into loose objects either: for one that'd be - // inefficient, and second they'd only get soaked up by the next - // geometric repack anyway. - // - // So instead, we do a full repack that appends unreachable objects - // to the end of the new packfile. - cfg.Strategy = RepackObjectsStrategyFullWithUnreachable - } else { - cfg.Strategy = RepackObjectsStrategyFullWithCruft - cfg.CruftExpireBefore = s.expireBefore - } - - return true, cfg + return true, fullRepackCfg } // In case both packfiles and loose objects are in a good state, but we don't yet // have a multi-pack-index we perform an incremental repack to generate one. We need // to have multi-pack-indices for the next heuristic, so it's bad if it was missing. if !s.info.Packfiles.MultiPackIndex.Exists { - cfg.Strategy = RepackObjectsStrategyGeometric - return true, cfg + return true, geometricRepackCfg } // Last but not least, we also need to take into account whether new packfiles have @@ -198,8 +220,7 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context untrackedPackfiles := s.info.Packfiles.Count - s.info.Packfiles.MultiPackIndex.PackfileCount if untrackedPackfiles > uint64(actualLimit) { - cfg.Strategy = RepackObjectsStrategyGeometric - return true, cfg + return true, geometricRepackCfg } // If there are loose objects then we want to roll them up into a new packfile. @@ -219,15 +240,52 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context // whether objects are reachable and we don't need to update any data structures // that scale with the repository size. if s.info.LooseObjects.Count > looseObjectLimit { - cfg.Strategy = RepackObjectsStrategyIncrementalWithUnreachable - cfg.WriteBitmap = false - cfg.WriteMultiPackIndex = false - return true, cfg + return true, incrementalRepackCfg } return false, RepackObjectsConfig{} } + fullRepackCfg := RepackObjectsConfig{ + // We use the full-with-unreachable strategy to also pack all unreachable objects + // into the packfile. This only happens for object pools though, as they should + // never delete objects. + // + // Note that this is quite inefficient, as all unreachable objects will be exploded + // into loose objects now. This is fixed in our geometric repacking strategy, where + // we append unreachable objects to the new pack. + Strategy: RepackObjectsStrategyFullWithLooseUnreachable, + // We cannot write bitmaps when there are alternates as we don't have full closure + // of all objects in the packfile. + WriteBitmap: len(s.info.Alternates.ObjectDirectories) == 0, + // We want to always update the multi-pack-index while we're already at it repacking + // some of the objects. + WriteMultiPackIndex: true, + } + if !s.info.IsObjectPool { + // When we don't have an object pool at hand we want to be able to expire + // unreachable objects. We thus use cruft packs with an expiry date. + fullRepackCfg.Strategy = RepackObjectsStrategyFullWithCruft + fullRepackCfg.CruftExpireBefore = s.expireBefore + } + + incrementalRepackCfg := RepackObjectsConfig{ + Strategy: RepackObjectsStrategyIncremental, + // We cannot write bitmaps when there are alternates as we don't have full closure + // of all objects in the packfile. + WriteBitmap: len(s.info.Alternates.ObjectDirectories) == 0, + // We want to always update the multi-pack-index while we're already at it repacking + // some of the objects. + WriteMultiPackIndex: true, + } + + // When alternative object directories have been modified since our last full repack then we + // have likely joined an object pool since then. This means that we'll want to perform a + // full repack in order to deduplicate objects that are part of the object pool. + if s.info.Alternates.LastModified.After(s.info.Packfiles.LastFullRepack) { + return true, fullRepackCfg + } + // Whenever we do an incremental repack we create a new packfile, and as a result Git may // have to look into every one of the packfiles to find objects. This is less efficient the // more packfiles we have, but we cannot repack the whole repository every time either given @@ -268,22 +326,7 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context if uint64(math.Max(lowerLimit, math.Log(float64(s.info.Packfiles.Size/1024/1024))/math.Log(log))) <= s.info.Packfiles.Count { - - // Object pools should neither have unreachable objects, nor should we ever try to - // delete any if there are some. So we disable cruft packs and expiration of them - // for them. - // - // Alternatively, we could enable writing cruft packs, but never expire the objects. - // This is left for another iteration though once we have determined that this is - // even necessary. - if !s.info.IsObjectPool { - cfg.Strategy = RepackObjectsStrategyFullWithCruft - cfg.CruftExpireBefore = s.expireBefore - } else { - cfg.Strategy = RepackObjectsStrategyFullWithLooseUnreachable - } - - return true, cfg + return true, fullRepackCfg } // Most Git commands do not write packfiles directly, but instead write loose objects into @@ -300,15 +343,13 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(ctx context.Context // In our case we typically want to ensure that our repositories are much better packed than // it is necessary on the client side. We thus take a much stricter limit of 1024 objects. if s.info.LooseObjects.Count > looseObjectLimit { - cfg.Strategy = RepackObjectsStrategyIncremental - return true, cfg + return true, incrementalRepackCfg } // In case both packfiles and loose objects are in a good state, but we don't yet have a // multi-pack-index we perform an incremental repack to generate one. if !s.info.Packfiles.MultiPackIndex.Exists { - cfg.Strategy = RepackObjectsStrategyIncremental - return true, cfg + return true, incrementalRepackCfg } return false, RepackObjectsConfig{} @@ -436,7 +477,7 @@ func NewEagerOptimizationStrategy(info stats.RepositoryInfo) EagerOptimizationSt // not have any alternates. func (s EagerOptimizationStrategy) ShouldRepackObjects(ctx context.Context) (bool, RepackObjectsConfig) { cfg := RepackObjectsConfig{ - WriteBitmap: len(s.info.Alternates) == 0, + WriteBitmap: len(s.info.Alternates.ObjectDirectories) == 0, WriteMultiPackIndex: true, } diff --git a/internal/git/housekeeping/optimization_strategy_test.go b/internal/git/housekeeping/optimization_strategy_test.go index 8e9526d7d..165f38ff2 100644 --- a/internal/git/housekeeping/optimization_strategy_test.go +++ b/internal/git/housekeeping/optimization_strategy_test.go @@ -7,9 +7,9 @@ import ( "time" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) @@ -41,7 +41,6 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co Exists: false, }, }, - Alternates: []string{}, }, }, expectedNeeded: true, @@ -64,7 +63,9 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co Exists: false, }, }, - Alternates: []string{"something"}, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: []string{"something"}, + }, }, }, // If we have no bitmap in the repository we'd normally want to fully repack @@ -394,7 +395,9 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co PackfileCount: 1, }, }, - Alternates: []string{"object-pool"}, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: []string{"object-pool"}, + }, }, }, expectedNeeded: true, @@ -417,7 +420,9 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co PackfileCount: 1, }, }, - Alternates: []string{"object-pool"}, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: []string{"object-pool"}, + }, }, }, expectedNeeded: true, @@ -434,6 +439,49 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co }, ), }, + { + desc: "alternates modified after last full repack", + strategy: HeuristicalOptimizationStrategy{ + info: stats.RepositoryInfo{ + Packfiles: stats.PackfilesInfo{ + Count: 1, + MultiPackIndex: stats.MultiPackIndexInfo{ + Exists: true, + PackfileCount: 1, + }, + LastFullRepack: time.Now().Add(-1 * time.Hour), + }, + Alternates: stats.AlternatesInfo{ + LastModified: time.Now(), + }, + }, + }, + expectedNeeded: true, + expectedConfig: RepackObjectsConfig{ + Strategy: RepackObjectsStrategyFullWithCruft, + WriteBitmap: true, + WriteMultiPackIndex: true, + }, + }, + { + desc: "alternates modified before last full repack", + strategy: HeuristicalOptimizationStrategy{ + info: stats.RepositoryInfo{ + Packfiles: stats.PackfilesInfo{ + Count: 1, + MultiPackIndex: stats.MultiPackIndexInfo{ + Exists: true, + PackfileCount: 1, + }, + LastFullRepack: time.Now(), + }, + Alternates: stats.AlternatesInfo{ + LastModified: time.Now().Add(-1 * time.Hour), + }, + }, + }, + expectedNeeded: false, + }, } { t.Run(tc.desc, func(t *testing.T) { repackNeeded, repackCfg := tc.strategy.ShouldRepackObjects(ctx) @@ -529,7 +577,9 @@ func testHeuristicalOptimizationStrategyShouldRepackObjects(t *testing.T, ctx co Exists: true, }, }, - Alternates: tc.alternates, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: tc.alternates, + }, }, expireBefore: expireBefore, } @@ -991,7 +1041,9 @@ func TestEagerOptimizationStrategy(t *testing.T) { desc: "alternate", strategy: EagerOptimizationStrategy{ info: stats.RepositoryInfo{ - Alternates: []string{"path/to/alternate"}, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: []string{"path/to/alternate"}, + }, }, expireBefore: expireBefore, }, @@ -1017,7 +1069,9 @@ func TestEagerOptimizationStrategy(t *testing.T) { strategy: EagerOptimizationStrategy{ info: stats.RepositoryInfo{ IsObjectPool: true, - Alternates: []string{"path/to/alternate"}, + Alternates: stats.AlternatesInfo{ + ObjectDirectories: []string{"path/to/alternate"}, + }, }, expireBefore: expireBefore, }, diff --git a/internal/git/housekeeping/optimize_repository.go b/internal/git/housekeeping/optimize_repository.go index 728457f4f..ec010d33a 100644 --- a/internal/git/housekeeping/optimize_repository.go +++ b/internal/git/housekeeping/optimize_repository.go @@ -158,7 +158,7 @@ func optimizeRepository( }() timer := prometheus.NewTimer(m.tasksLatency.WithLabelValues("clean-stale-data")) - if err := m.CleanStaleData(ctx, repo); err != nil { + if err := m.CleanStaleData(ctx, repo, DefaultStaleDataCleanup()); err != nil { return fmt.Errorf("could not execute houskeeping: %w", err) } timer.ObserveDuration() diff --git a/internal/git/housekeeping/optimize_repository_ext_test.go b/internal/git/housekeeping/optimize_repository_ext_test.go index 9bb10127d..b27fe0a1a 100644 --- a/internal/git/housekeeping/optimize_repository_ext_test.go +++ b/internal/git/housekeeping/optimize_repository_ext_test.go @@ -12,13 +12,13 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/git/housekeeping/optimize_repository_test.go b/internal/git/housekeeping/optimize_repository_test.go index 21fecabfd..419e039ba 100644 --- a/internal/git/housekeeping/optimize_repository_test.go +++ b/internal/git/housekeeping/optimize_repository_test.go @@ -14,8 +14,8 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -24,8 +24,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" gitalycfgprom "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" ) @@ -460,13 +460,20 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { gitVersion, err := gittest.NewCommandFactory(t, cfg).GitVersion(ctx) require.NoError(t, err) - linkRepoToPool := func(t *testing.T, repoPath, poolPath string) { + earlierDate := time.Date(2022, 12, 1, 0, 0, 0, 0, time.Local) + laterDate := time.Date(2022, 12, 1, 12, 0, 0, 0, time.Local) + + linkRepoToPool := func(t *testing.T, repoPath, poolPath string, date time.Time) { t.Helper() + + alternatesPath := filepath.Join(repoPath, "objects", "info", "alternates") + require.NoError(t, os.WriteFile( - filepath.Join(repoPath, "objects", "info", "alternates"), + alternatesPath, []byte(filepath.Join(poolPath, "objects")), perm.PrivateFile, )) + require.NoError(t, os.Chtimes(alternatesPath, date, date)) } readPackfiles := func(t *testing.T, repoPath string) []string { @@ -823,7 +830,8 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { RelativePath: gittest.NewObjectPoolName(t), }) - linkRepoToPool(t, repoPath, poolPath) + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, laterDate)) + linkRepoToPool(t, repoPath, poolPath, earlierDate) return setupData{ repo: localrepo.NewTestRepo(t, cfg, repo), @@ -849,7 +857,10 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { SkipCreationViaService: true, RelativePath: relativePath, }) - linkRepoToPool(t, repoPath, poolPath) + + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, laterDate)) + linkRepoToPool(t, repoPath, poolPath, earlierDate) + gittest.WriteRef(t, cfg, repoPath, "refs/heads/some-branch", commitID) return setupData{ @@ -874,7 +885,8 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { SkipCreationViaService: true, RelativePath: relativePath, }) - linkRepoToPool(t, repoPath, poolPath) + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, laterDate)) + linkRepoToPool(t, repoPath, poolPath, earlierDate) gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(commitID), gittest.WithBranch("some-branch")) return setupData{ @@ -889,6 +901,46 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { }, }, { + desc: "recently linked repository gets a full repack", + setup: func(t *testing.T, relativePath string) setupData { + _, poolPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + RelativePath: gittest.NewObjectPoolName(t), + }) + + repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + RelativePath: relativePath, + }) + gittest.WriteCommit(t, cfg, repoPath) + + // Pretend that the last full repack has happened before creating + // the gitalternates file. This should cause a full repack in order + // to deduplicate all objects. + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, earlierDate)) + linkRepoToPool(t, repoPath, poolPath, laterDate) + + return setupData{ + repo: localrepo.NewTestRepo(t, cfg, repo), + expectedMetrics: []metric{ + {name: "packed_objects_full_with_cruft", status: "success", count: 1}, + {name: "written_multi_pack_index", status: "success", count: 1}, + {name: "total", status: "success", count: 1}, + }, + expectedMetricsForPool: []metric{ + {name: func() string { + if gitVersion.GeometricRepackingSupportsAlternates() { + return geometricOrIncremental(ctx, "packed_objects_full_with_unreachable", "packed_objects_full_with_loose_unreachable") + } + return "packed_objects_full_with_loose_unreachable" + }(), status: "success", count: 1}, + {name: "written_multi_pack_index", status: "success", count: 1}, + {name: "total", status: "success", count: 1}, + }, + } + }, + }, + { desc: "repository with some deduplicated objects and eager strategy", setup: func(t *testing.T, relativePath string) setupData { _, poolPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ @@ -901,7 +953,8 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { SkipCreationViaService: true, RelativePath: relativePath, }) - linkRepoToPool(t, repoPath, poolPath) + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, laterDate)) + linkRepoToPool(t, repoPath, poolPath, earlierDate) gittest.WriteCommit(t, cfg, repoPath, gittest.WithParents(commitID), gittest.WithBranch("some-branch")) return setupData{ @@ -956,7 +1009,8 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) { // past. require.Equal(t, repoPackfiles, readPackfiles(t, poolPath)) - linkRepoToPool(t, repoPath, poolPath) + require.NoError(t, stats.UpdateFullRepackTimestamp(repoPath, laterDate)) + linkRepoToPool(t, repoPath, poolPath, earlierDate) return setupData{ repo: localrepo.NewTestRepo(t, cfg, repo), diff --git a/internal/git/housekeeping/testhelper_test.go b/internal/git/housekeeping/testhelper_test.go index 330346bd8..5334733b0 100644 --- a/internal/git/housekeeping/testhelper_test.go +++ b/internal/git/housekeeping/testhelper_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) diff --git a/internal/git/localrepo/commit_test.go b/internal/git/localrepo/commit_test.go index a95e41273..9238ae7a8 100644 --- a/internal/git/localrepo/commit_test.go +++ b/internal/git/localrepo/commit_test.go @@ -32,10 +32,10 @@ func TestWriteCommit(t *testing.T) { require.NoError(t, err) treeEntryA := TreeEntry{Path: "file", Mode: "100644", OID: blobID} - treeA, err := repo.WriteTree(ctx, []TreeEntry{treeEntryA}) + treeA, err := repo.WriteTree(ctx, []*TreeEntry{&treeEntryA}) require.NoError(t, err) - treeB, err := repo.WriteTree(ctx, []TreeEntry{ + treeB, err := repo.WriteTree(ctx, []*TreeEntry{ {Path: "file", Mode: "100644", OID: changedBlobID}, }) require.NoError(t, err) @@ -203,7 +203,7 @@ func TestWriteCommit_validation(t *testing.T) { blobID, err := repo.WriteBlob(ctx, "", strings.NewReader("foo")) require.NoError(t, err) - treeID, err := repo.WriteTree(ctx, []TreeEntry{ + treeID, err := repo.WriteTree(ctx, []*TreeEntry{ { OID: blobID, Mode: "100644", diff --git a/internal/git/localrepo/config_test.go b/internal/git/localrepo/config_test.go index 25bf4ef0c..9db4f6c64 100644 --- a/internal/git/localrepo/config_test.go +++ b/internal/git/localrepo/config_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/git/localrepo/objects.go b/internal/git/localrepo/objects.go index 7c44bb17c..a7c505d8b 100644 --- a/internal/git/localrepo/objects.go +++ b/internal/git/localrepo/objects.go @@ -11,10 +11,10 @@ import ( "time" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) diff --git a/internal/git/localrepo/objects_test.go b/internal/git/localrepo/objects_test.go index 258adde45..5793732a2 100644 --- a/internal/git/localrepo/objects_test.go +++ b/internal/git/localrepo/objects_test.go @@ -13,13 +13,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/git/localrepo/tree.go b/internal/git/localrepo/tree.go index 3582cadfa..e32014afe 100644 --- a/internal/git/localrepo/tree.go +++ b/internal/git/localrepo/tree.go @@ -4,9 +4,12 @@ import ( "bytes" "context" "fmt" + "sort" + "strings" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" ) // ObjectType is an Enum for the type of object of @@ -37,6 +40,33 @@ func (e Entries) Less(i, j int) bool { return e[i].Type < e[j].Type } +// TreeEntriesByPath allows a slice of *TreeEntry to be sorted by Path +type TreeEntriesByPath []*TreeEntry + +func (b TreeEntriesByPath) Len() int { + return len(b) +} + +func (b TreeEntriesByPath) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b TreeEntriesByPath) Less(i, j int) bool { + iPath, jPath := b[i].Path, b[j].Path + + // git has an edge case for subtrees where they are always appended with + // a '/'. See https://github.com/git/git/blob/v2.40.0/read-cache.c#L491 + if b[i].Type == Tree { + iPath += "/" + } + + if b[j].Type == Tree { + jPath += "/" + } + + return iPath < jPath +} + // ToEnum translates a string representation of the object type into an // ObjectType enum. func ToEnum(s string) ObjectType { @@ -52,19 +82,6 @@ func ToEnum(s string) ObjectType { } } -func fromEnum(t ObjectType) string { - switch t { - case Tree: - return "tree" - case Blob: - return "blob" - case Submodule: - return "commit" - default: - return "unknown" - } -} - // TreeEntry represents an entry of a git tree object. type TreeEntry struct { // OID is the object ID the tree entry refers to. @@ -84,50 +101,46 @@ func (t *TreeEntry) IsBlob() bool { // WriteTree writes a new tree object to the given path. This function does not verify whether OIDs // referred to by tree entries actually exist in the repository. -func (repo *Repo) WriteTree(ctx context.Context, entries []TreeEntry) (git.ObjectID, error) { +func (repo *Repo) WriteTree(ctx context.Context, entries []*TreeEntry) (git.ObjectID, error) { var tree bytes.Buffer + + sort.Stable(TreeEntriesByPath(entries)) + for _, entry := range entries { - entryType := entry.Type - - if entryType == Unknown { - switch entry.Mode { - case "100644": - fallthrough - case "100755": - fallthrough - case "120000": - entryType = Blob - case "040000": - entryType = Tree - case "160000": - entryType = Submodule - } - } + mode := strings.TrimPrefix(entry.Mode, "0") + formattedEntry := fmt.Sprintf("%s %s\000", mode, entry.Path) - oid := entry.OID + oidBytes, err := entry.OID.Bytes() + if err != nil { + return "", err + } - formattedEntry := fmt.Sprintf("%s %s %s\t%s\000", entry.Mode, fromEnum(entryType), oid.String(), entry.Path) if _, err := tree.WriteString(formattedEntry); err != nil { return "", err } + + if _, err := tree.Write(oidBytes); err != nil { + return "", err + } } options := []git.Option{ - git.Flag{Name: "-z"}, - git.Flag{Name: "--missing"}, + git.ValueFlag{Name: "-t", Value: "tree"}, + git.Flag{Name: "-w"}, + git.Flag{Name: "--stdin"}, } var stdout, stderr bytes.Buffer if err := repo.ExecAndWait(ctx, git.Command{ - Name: "mktree", + Name: "hash-object", Flags: options, }, git.WithStdout(&stdout), git.WithStderr(&stderr), git.WithStdin(&tree), ); err != nil { - return "", err + return "", structerr.New("%w", err).WithMetadata("stderr", stderr.String()) } objectHash, err := repo.ObjectHash(ctx) diff --git a/internal/git/localrepo/tree_test.go b/internal/git/localrepo/tree_test.go index c2bb1f897..92dd71575 100644 --- a/internal/git/localrepo/tree_test.go +++ b/internal/git/localrepo/tree_test.go @@ -4,11 +4,14 @@ import ( "bytes" "os" "path/filepath" + "sort" + "strings" "testing" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" ) @@ -17,6 +20,9 @@ func TestWriteTree(t *testing.T) { cfg := testcfg.Build(t) ctx := testhelper.Context(t) + gitVersion, err := gittest.NewCommandFactory(t, cfg).GitVersion(ctx) + require.NoError(t, err) + repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) @@ -28,7 +34,7 @@ func TestWriteTree(t *testing.T) { blobID, err := repo.WriteBlob(ctx, "file", bytes.NewBufferString("foobar\n")) require.NoError(t, err) - treeID, err := repo.WriteTree(ctx, []TreeEntry{ + treeID, err := repo.WriteTree(ctx, []*TreeEntry{ { OID: blobID, Mode: "100644", @@ -44,13 +50,14 @@ func TestWriteTree(t *testing.T) { require.NoError(t, os.Remove(nonExistentBlobPath)) for _, tc := range []struct { - desc string - entries []TreeEntry - expectedEntries []TreeEntry + desc string + entries []*TreeEntry + expectedEntries []TreeEntry + expectedErrString string }{ { desc: "entry with blob OID", - entries: []TreeEntry{ + entries: []*TreeEntry{ { OID: blobID, Mode: "100644", @@ -67,7 +74,7 @@ func TestWriteTree(t *testing.T) { }, { desc: "entry with tree OID", - entries: []TreeEntry{ + entries: []*TreeEntry{ { OID: treeID, Mode: "040000", @@ -84,7 +91,7 @@ func TestWriteTree(t *testing.T) { }, { desc: "mixed tree and blob entries", - entries: []TreeEntry{ + entries: []*TreeEntry{ { OID: treeID, Mode: "040000", @@ -121,7 +128,7 @@ func TestWriteTree(t *testing.T) { }, { desc: "entry with nonexistent object", - entries: []TreeEntry{ + entries: []*TreeEntry{ { OID: nonExistentBlobID, Mode: "100644", @@ -136,12 +143,63 @@ func TestWriteTree(t *testing.T) { }, }, }, + { + desc: "entry with duplicate file", + entries: []*TreeEntry{ + { + OID: blobID, + Mode: "100644", + Path: "file", + }, + { + OID: nonExistentBlobID, + Mode: "100644", + Path: "file", + }, + }, + expectedErrString: "duplicateEntries: contains duplicate file entries", + }, + { + desc: "entry with malformed mode", + entries: []*TreeEntry{ + { + OID: blobID, + Mode: "1006442", + Path: "file", + }, + }, + expectedErrString: "badFilemode: contains bad file modes", + }, + { + desc: "tries to write .git file", + entries: []*TreeEntry{ + { + OID: blobID, + Mode: "040000", + Path: ".git", + }, + }, + expectedErrString: "hasDotgit: contains '.git'", + }, } { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() oid, err := repo.WriteTree(ctx, tc.entries) + if tc.expectedErrString != "" { + if gitVersion.HashObjectFsck() { + switch e := err.(type) { + case structerr.Error: + stderr := e.Metadata()["stderr"].(string) + strings.Contains(stderr, tc.expectedErrString) + default: + strings.Contains(err.Error(), tc.expectedErrString) + } + } + return + } + require.NoError(t, err) output := text.ChompBytes(gittest.Exec(t, cfg, "-C", repoPath, "ls-tree", "-r", string(oid))) @@ -179,3 +237,134 @@ func TestWriteTree(t *testing.T) { }) } } + +func TestTreeEntryByPath(t *testing.T) { + t.Parallel() + + testCases := []struct { + desc string + input []*TreeEntry + expected []*TreeEntry + }{ + { + desc: "all blobs", + input: []*TreeEntry{ + { + Type: Blob, + Path: "abc", + }, + { + Type: Blob, + Path: "ab", + }, + { + Type: Blob, + Path: "a", + }, + }, + expected: []*TreeEntry{ + { + Type: Blob, + Path: "a", + }, + { + Type: Blob, + Path: "ab", + }, + { + Type: Blob, + Path: "abc", + }, + }, + }, + { + desc: "blobs and trees", + input: []*TreeEntry{ + { + Type: Blob, + Path: "abc", + }, + { + Type: Tree, + Path: "ab", + }, + { + Type: Blob, + Path: "a", + }, + }, + expected: []*TreeEntry{ + { + Type: Blob, + Path: "a", + }, + { + Type: Tree, + Path: "ab", + }, + { + Type: Blob, + Path: "abc", + }, + }, + }, + { + desc: "trees get sorted with / appended", + input: []*TreeEntry{ + { + Type: Tree, + Path: "a", + }, + { + Type: Blob, + Path: "a+", + }, + }, + expected: []*TreeEntry{ + { + Type: Blob, + Path: "a+", + }, + { + Type: Tree, + Path: "a", + }, + }, + }, + { + desc: "blobs get sorted without / appended", + input: []*TreeEntry{ + { + Type: Blob, + Path: "a", + }, + { + Type: Blob, + Path: "a+", + }, + }, + expected: []*TreeEntry{ + { + Type: Blob, + Path: "a", + }, + { + Type: Blob, + Path: "a+", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + sort.Stable(TreeEntriesByPath(tc.input)) + + require.Equal( + t, + tc.expected, + tc.input, + ) + }) + } +} diff --git a/internal/git/objectpool/create_test.go b/internal/git/objectpool/create_test.go index d648cd805..7dbc7abff 100644 --- a/internal/git/objectpool/create_test.go +++ b/internal/git/objectpool/create_test.go @@ -8,7 +8,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" @@ -16,6 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go index c4f40825c..36823dd38 100644 --- a/internal/git/objectpool/fetch.go +++ b/internal/git/objectpool/fetch.go @@ -11,6 +11,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/git/updateref" @@ -34,7 +35,7 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *localrepo.Repo return fmt.Errorf("computing origin repo's path: %w", err) } - if err := o.housekeepingManager.CleanStaleData(ctx, o.Repo); err != nil { + if err := o.housekeepingManager.CleanStaleData(ctx, o.Repo, housekeeping.DefaultStaleDataCleanup()); err != nil { return fmt.Errorf("cleaning stale data: %w", err) } diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go index 7188e77ab..d5b516d9e 100644 --- a/internal/git/objectpool/fetch_test.go +++ b/internal/git/objectpool/fetch_test.go @@ -11,11 +11,11 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) @@ -328,7 +328,7 @@ func testObjectPoolLogStats(t *testing.T, ctx context.Context) { IsObjectPool: true, LooseObjects: stats.LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: stats.ReferencesInfo{ LooseReferencesCount: 1, @@ -352,7 +352,7 @@ func testObjectPoolLogStats(t *testing.T, ctx context.Context) { IsObjectPool: true, LooseObjects: stats.LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: stats.ReferencesInfo{ LooseReferencesCount: 1, diff --git a/internal/git/objectpool/link_test.go b/internal/git/objectpool/link_test.go index d2096c5fd..751664c96 100644 --- a/internal/git/objectpool/link_test.go +++ b/internal/git/objectpool/link_test.go @@ -7,11 +7,11 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" diff --git a/internal/git/objectpool/testhelper_test.go b/internal/git/objectpool/testhelper_test.go index 6c610ba15..4efcb5923 100644 --- a/internal/git/objectpool/testhelper_test.go +++ b/internal/git/objectpool/testhelper_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" @@ -13,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -57,9 +57,9 @@ func setupObjectPool(t *testing.T, ctx context.Context) (config.Cfg, *ObjectPool return cfg, pool, repo } -func hashDependentSize(sha1Size, sha256Size uint64) uint64 { - if gittest.ObjectHashIsSHA256() { - return sha256Size - } - return sha1Size +func hashDependentSize(tb testing.TB, sha1Size, sha256Size uint64) uint64 { + return gittest.ObjectHashDependent(tb, map[string]uint64{ + "sha1": sha1Size, + "sha256": sha256Size, + }) } diff --git a/internal/git/remoterepo/repository_test.go b/internal/git/remoterepo/repository_test.go index ba4ef4ddd..5911350ec 100644 --- a/internal/git/remoterepo/repository_test.go +++ b/internal/git/remoterepo/repository_test.go @@ -14,8 +14,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/remoterepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" ) diff --git a/internal/git/stats/repository_info.go b/internal/git/stats/repository_info.go index 5b6cbd82a..f52ba41be 100644 --- a/internal/git/stats/repository_info.go +++ b/internal/git/stats/repository_info.go @@ -1,6 +1,7 @@ package stats import ( + "bufio" "bytes" "context" "encoding/binary" @@ -15,7 +16,6 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" - "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" ) const ( @@ -137,9 +137,8 @@ type RepositoryInfo struct { References ReferencesInfo `json:"references"` // CommitGraph contains information about the repository's commit-graphs. CommitGraph CommitGraphInfo `json:"commit_graph"` - // Alternates is the list of absolute paths of alternate object databases this repository is - // connected to. - Alternates []string `json:"alternates"` + // Alternates contains information about alternate object directories. + Alternates AlternatesInfo `json:"alternates"` } // RepositoryInfoForRepository computes the RepositoryInfo for a repository. @@ -174,7 +173,7 @@ func RepositoryInfoForRepository(repo *localrepo.Repo) (RepositoryInfo, error) { return RepositoryInfo{}, fmt.Errorf("checking commit-graph info: %w", err) } - info.Alternates, err = readAlternates(repo) + info.Alternates, err = AlternatesInfoForRepository(repoPath) if err != nil { return RepositoryInfo{}, fmt.Errorf("reading alterantes: %w", err) } @@ -487,32 +486,70 @@ func hasPrefixAndSuffix(s, prefix, suffix string) bool { return strings.HasPrefix(s, prefix) && strings.HasSuffix(s, suffix) } -func readAlternates(repo *localrepo.Repo) ([]string, error) { - repoPath, err := repo.Path() - if err != nil { - return nil, fmt.Errorf("getting repository path: %w", err) - } +// AlternatesInfo contains information about altenrate object directories the repository is linked +// to. +type AlternatesInfo struct { + // Exists determines whether the `info/alternates` file exists or not. + Exists bool `json:"exists"` + // Paths contains the list of paths to object directories that the repository is linked to. + ObjectDirectories []string `json:"object_directories,omitempty"` + // LastModified is the time when the alternates file has last been modified. Has the zero + // value when the alternates file doesn't exist. + LastModified time.Time `json:"last_modified"` +} - contents, err := os.ReadFile(filepath.Join(repoPath, "objects", "info", "alternates")) +// AlternatesInfoForRepository reads the alternates file and returns information on it. This +// function does not return an error in case the alternates file doesn't exist. Existence can be +// checked via the `Exists` field of the returned `AlternatesInfo` structure. +func AlternatesInfoForRepository(repoPath string) (AlternatesInfo, error) { + file, err := os.Open(filepath.Join(repoPath, "objects", "info", "alternates")) if err != nil { if errors.Is(err, os.ErrNotExist) { - return nil, nil + return AlternatesInfo{ + Exists: false, + }, nil } - return nil, fmt.Errorf("reading alternates: %w", err) + return AlternatesInfo{}, err + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return AlternatesInfo{}, err } - relativeAlternatePaths := strings.Split(text.ChompBytes(contents), "\n") - alternatePaths := make([]string, 0, len(relativeAlternatePaths)) - for _, relativeAlternatePath := range relativeAlternatePaths { - if filepath.IsAbs(relativeAlternatePath) { - alternatePaths = append(alternatePaths, relativeAlternatePath) - } else { - alternatePaths = append(alternatePaths, filepath.Join(repoPath, "objects", relativeAlternatePath)) + var alternatePaths []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Bytes() + + switch { + case len(line) == 0: + // Empty lines are skipped by Git. + continue + case bytes.HasPrefix(line, []byte("#")): + // Lines starting with a '#' are comments and thus need to be skipped. + continue + default: + path := scanner.Text() + + if filepath.IsAbs(path) { + alternatePaths = append(alternatePaths, path) + } else { + alternatePaths = append(alternatePaths, filepath.Join(repoPath, "objects", path)) + } } } + if err := scanner.Err(); err != nil { + return AlternatesInfo{}, fmt.Errorf("scanning alternate paths: %w", err) + } - return alternatePaths, nil + return AlternatesInfo{ + Exists: true, + ObjectDirectories: alternatePaths, + LastModified: stat.ModTime(), + }, nil } // BitmapInfo contains information about a packfile or multi-pack-index bitmap. diff --git a/internal/git/stats/repository_info_test.go b/internal/git/stats/repository_info_test.go index dbfb1360f..a73c660d3 100644 --- a/internal/git/stats/repository_info_test.go +++ b/internal/git/stats/repository_info_test.go @@ -122,6 +122,9 @@ func TestLogObjectInfo(t *testing.T) { targetRepoPath := filepath.Join(storagePath, targetRepoName) gittest.Exec(t, cfg, "clone", "--bare", "--shared", repoPath1, "--reference", repoPath1, "--reference", repoPath2, targetRepoPath) + alternatesStat, err := os.Stat(filepath.Join(targetRepoPath, "objects", "info", "alternates")) + require.NoError(t, err) + LogRepositoryInfo(ctx, localrepo.NewTestRepo(t, cfg, &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, RelativePath: targetRepoName, @@ -135,9 +138,13 @@ func TestLogObjectInfo(t *testing.T) { References: ReferencesInfo{ PackedReferencesSize: uint64(packedRefsStat.Size()), }, - Alternates: []string{ - filepath.Join(repoPath1, "/objects"), - filepath.Join(repoPath2, "/objects"), + Alternates: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + filepath.Join(repoPath1, "/objects"), + filepath.Join(repoPath2, "/objects"), + }, + LastModified: alternatesStat.ModTime(), }, }, repoInfo) }) @@ -159,7 +166,7 @@ func TestLogObjectInfo(t *testing.T) { require.Equal(t, RepositoryInfo{ LooseObjects: LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: ReferencesInfo{ LooseReferencesCount: 1, @@ -179,6 +186,8 @@ func TestRepositoryInfoForRepository(t *testing.T) { }) alternatePath = filepath.Join(alternatePath, "objects") + date := time.Date(2005, 4, 7, 15, 13, 13, 0, time.Local) + for _, tc := range []struct { desc string setup func(t *testing.T, repoPath string) @@ -213,7 +222,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { expectedInfo: RepositoryInfo{ Packfiles: PackfilesInfo{ Count: 1, - Size: hashDependentSize(42, 54), + Size: hashDependentSize(t, 42, 54), ReverseIndexCount: 1, Bitmap: BitmapInfo{ Exists: true, @@ -242,7 +251,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { }, Packfiles: PackfilesInfo{ Count: 1, - Size: hashDependentSize(42, 54), + Size: hashDependentSize(t, 42, 54), ReverseIndexCount: 1, Bitmap: BitmapInfo{ Exists: true, @@ -271,12 +280,19 @@ func TestRepositoryInfoForRepository(t *testing.T) { { desc: "alternates", setup: func(t *testing.T, repoPath string) { - infoAlternatesPath := filepath.Join(repoPath, "objects", "info", "alternates") - require.NoError(t, os.WriteFile(infoAlternatesPath, []byte(alternatePath), perm.PrivateFile)) + writeFileWithMtime(t, + filepath.Join(repoPath, "objects", "info", "alternates"), + []byte(alternatePath), + date, + ) }, expectedInfo: RepositoryInfo{ - Alternates: []string{ - alternatePath, + Alternates: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + alternatePath, + }, + LastModified: date, }, }, }, @@ -292,7 +308,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { expectedInfo: RepositoryInfo{ LooseObjects: LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: ReferencesInfo{ LooseReferencesCount: 1, @@ -314,7 +330,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { expectedInfo: RepositoryInfo{ LooseObjects: LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: ReferencesInfo{ LooseReferencesCount: 1, @@ -341,7 +357,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { expectedInfo: RepositoryInfo{ LooseObjects: LooseObjectsInfo{ Count: 2, - Size: hashDependentSize(142, 158), + Size: hashDependentSize(t, 142, 158), }, References: ReferencesInfo{ LooseReferencesCount: 1, @@ -357,10 +373,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { desc: "last full repack timestamp", setup: func(t *testing.T, repoPath string) { timestampPath := filepath.Join(repoPath, fullRepackTimestampFilename) - require.NoError(t, os.WriteFile(timestampPath, nil, perm.PrivateFile)) - - date := time.Date(2005, 4, 7, 15, 13, 13, 0, time.Local) - require.NoError(t, os.Chtimes(timestampPath, date, date)) + writeFileWithMtime(t, timestampPath, nil, date) }, expectedInfo: RepositoryInfo{ Packfiles: PackfilesInfo{ @@ -371,8 +384,11 @@ func TestRepositoryInfoForRepository(t *testing.T) { { desc: "all together", setup: func(t *testing.T, repoPath string) { - infoAlternatesPath := filepath.Join(repoPath, "objects", "info", "alternates") - require.NoError(t, os.WriteFile(infoAlternatesPath, []byte(alternatePath), perm.PrivateFile)) + writeFileWithMtime(t, + filepath.Join(repoPath, "objects", "info", "alternates"), + []byte(alternatePath), + date, + ) // We write a single packed blob. blobID := gittest.WriteBlob(t, cfg, repoPath, []byte("x")) @@ -397,7 +413,7 @@ func TestRepositoryInfoForRepository(t *testing.T) { }, Packfiles: PackfilesInfo{ Count: 1, - Size: hashDependentSize(42, 54), + Size: hashDependentSize(t, 42, 54), ReverseIndexCount: 1, GarbageCount: 3, GarbageSize: 3, @@ -410,8 +426,12 @@ func TestRepositoryInfoForRepository(t *testing.T) { References: ReferencesInfo{ LooseReferencesCount: 1, }, - Alternates: []string{ - alternatePath, + Alternates: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + alternatePath, + }, + LastModified: date, }, }, }, @@ -431,6 +451,201 @@ func TestRepositoryInfoForRepository(t *testing.T) { } } +func TestAlternatesInfoForRepository(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + date := time.Date(2005, 4, 7, 15, 13, 13, 0, time.Local) + + createRepo := func(t *testing.T) (*gitalypb.Repository, string) { + return gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + } + writeAlternates := func(t *testing.T, repoPath string, alternates []byte) { + writeFileWithMtime(t, + filepath.Join(repoPath, "objects", "info", "alternates"), + alternates, + date, + ) + } + + type setupData struct { + repoPath string + expectedErr error + expectedInfo AlternatesInfo + } + + for _, tc := range []struct { + desc string + setup func(t *testing.T) setupData + }{ + { + desc: "invalid path", + setup: func(t *testing.T) setupData { + return setupData{ + repoPath: "/does/not/exist", + expectedInfo: AlternatesInfo{ + Exists: false, + }, + } + }, + }, + { + desc: "empty repository", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: false, + }, + } + }, + }, + { + desc: "empty alternates file", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, nil) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + LastModified: date, + }, + } + }, + }, + { + desc: "missing trailing newline", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("/absolute")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + "/absolute", + }, + LastModified: date, + }, + } + }, + }, + { + desc: "absolute path", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("/absolute\n")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + "/absolute", + }, + LastModified: date, + }, + } + }, + }, + { + desc: "relative path", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("../../../../../@pools/foo/bar\n")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + filepath.Join(cfg.Storages[0].Path, "@pools", "foo", "bar"), + }, + LastModified: date, + }, + } + }, + }, + { + desc: "mixed absolute and relative paths", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("/absolute\n../../../../../@pools/foo/bar\n")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + "/absolute", + filepath.Join(cfg.Storages[0].Path, "@pools", "foo", "bar"), + }, + LastModified: date, + }, + } + }, + }, + { + desc: "empty lines", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("/first\n\n/second\n")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + "/first", + "/second", + }, + LastModified: date, + }, + } + }, + }, + { + desc: "comment", + setup: func(t *testing.T) setupData { + _, repoPath := createRepo(t) + writeAlternates(t, repoPath, []byte("/first\n# comment\n/second\n")) + + return setupData{ + repoPath: repoPath, + expectedInfo: AlternatesInfo{ + Exists: true, + ObjectDirectories: []string{ + "/first", + "/second", + }, + LastModified: date, + }, + } + }, + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + setup := tc.setup(t) + + info, err := AlternatesInfoForRepository(setup.repoPath) + require.Equal(t, setup.expectedErr, err) + require.Equal(t, setup.expectedInfo, info) + }) + } +} + func TestReferencesInfoForRepository(t *testing.T) { t.Parallel() @@ -576,8 +791,7 @@ func TestCountLooseObjects(t *testing.T) { beforeCutoffDate := cutoffDate.Add(-1 * time.Minute) for _, objectPath := range objectPaths { - require.NoError(t, os.WriteFile(objectPath, []byte("1"), perm.SharedFile)) - require.NoError(t, os.Chtimes(objectPath, afterCutoffDate, afterCutoffDate)) + writeFileWithMtime(t, objectPath, []byte("1"), afterCutoffDate) } // Objects are recent, so with the cutoff-date they shouldn't be counted. @@ -801,7 +1015,7 @@ func TestPackfileInfoForRepository(t *testing.T) { }, expectedInfo: PackfilesInfo{ Count: 1, - Size: hashDependentSize(163, 189), + Size: hashDependentSize(t, 163, 189), ReverseIndexCount: 1, Bitmap: BitmapInfo{ Exists: true, @@ -818,7 +1032,7 @@ func TestPackfileInfoForRepository(t *testing.T) { }, expectedInfo: PackfilesInfo{ Count: 1, - Size: hashDependentSize(163, 189), + Size: hashDependentSize(t, 163, 189), ReverseIndexCount: 1, MultiPackIndex: MultiPackIndexInfo{ Exists: true, @@ -835,7 +1049,7 @@ func TestPackfileInfoForRepository(t *testing.T) { }, expectedInfo: PackfilesInfo{ Count: 1, - Size: hashDependentSize(163, 189), + Size: hashDependentSize(t, 163, 189), ReverseIndexCount: 1, MultiPackIndex: MultiPackIndexInfo{ Exists: true, @@ -861,7 +1075,7 @@ func TestPackfileInfoForRepository(t *testing.T) { }, expectedInfo: PackfilesInfo{ Count: 2, - Size: hashDependentSize(315, 367), + Size: hashDependentSize(t, 315, 367), ReverseIndexCount: 1, GarbageCount: 1, GarbageSize: 1, @@ -886,10 +1100,10 @@ func TestPackfileInfoForRepository(t *testing.T) { }, expectedInfo: PackfilesInfo{ Count: 2, - Size: hashDependentSize(318, 371), + Size: hashDependentSize(t, 318, 371), ReverseIndexCount: 2, CruftCount: 1, - CruftSize: hashDependentSize(156, 183), + CruftSize: hashDependentSize(t, 156, 183), MultiPackIndex: MultiPackIndexInfo{ Exists: true, Version: 1, @@ -1480,9 +1694,15 @@ func TestFullRepackTimestamp(t *testing.T) { }) } -func hashDependentSize(sha1, sha256 uint64) uint64 { - if gittest.DefaultObjectHash.Format == "sha1" { - return sha1 - } - return sha256 +func hashDependentSize(tb testing.TB, sha1, sha256 uint64) uint64 { + return gittest.ObjectHashDependent(tb, map[string]uint64{ + "sha1": sha1, + "sha256": sha256, + }) +} + +func writeFileWithMtime(tb testing.TB, path string, content []byte, date time.Time) { + tb.Helper() + require.NoError(tb, os.WriteFile(path, content, perm.PrivateFile)) + require.NoError(tb, os.Chtimes(path, date, date)) } diff --git a/internal/git/updateref/update_with_hooks.go b/internal/git/updateref/update_with_hooks.go index c2e19b86a..8acbb9065 100644 --- a/internal/git/updateref/update_with_hooks.go +++ b/internal/git/updateref/update_with_hooks.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -16,7 +17,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) diff --git a/internal/git/updateref/update_with_hooks_test.go b/internal/git/updateref/update_with_hooks_test.go index 88753701d..fe3a577dd 100644 --- a/internal/git/updateref/update_with_hooks_test.go +++ b/internal/git/updateref/update_with_hooks_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -18,7 +19,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" hookservice "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" @@ -229,7 +229,7 @@ func TestUpdaterWithHooks_UpdateReference(t *testing.T) { expectedErr: "reference-transaction failure", }, { - desc: "post-receive error is ignored", + desc: "post-receive custom hooks error is ignored", preReceive: func(t *testing.T, ctx context.Context, repo *gitalypb.Repository, pushOptions, env []string, stdin io.Reader, stdout, stderr io.Writer) error { return nil }, @@ -242,9 +242,28 @@ func TestUpdaterWithHooks_UpdateReference(t *testing.T) { postReceive: func(t *testing.T, ctx context.Context, repo *gitalypb.Repository, pushOptions, env []string, stdin io.Reader, stdout, stderr io.Writer) error { _, err := io.Copy(stderr, strings.NewReader("post-receive failure")) require.NoError(t, err) - return errors.New("ignored") + return hook.NewCustomHookError(errors.New("ignored")) + }, + expectedRefDeletion: true, + }, + { + desc: "post-receive non-custom hooks error returned", + preReceive: func(t *testing.T, ctx context.Context, repo *gitalypb.Repository, pushOptions, env []string, stdin io.Reader, stdout, stderr io.Writer) error { + return nil + }, + update: func(t *testing.T, ctx context.Context, repo *gitalypb.Repository, ref, oldValue, newValue string, env []string, stdout, stderr io.Writer) error { + return nil + }, + referenceTransaction: func(t *testing.T, ctx context.Context, state hook.ReferenceTransactionState, env []string, stdin io.Reader) error { + return nil + }, + postReceive: func(t *testing.T, ctx context.Context, repo *gitalypb.Repository, pushOptions, env []string, stdin io.Reader, stdout, stderr io.Writer) error { + _, err := io.Copy(stderr, strings.NewReader("post-receive failure")) + require.NoError(t, err) + return errors.New("uh oh") }, expectedRefDeletion: true, + expectedErr: "running post-receive hooks: uh oh", }, } diff --git a/internal/git/version.go b/internal/git/version.go index 5048007fa..43cff0475 100644 --- a/internal/git/version.go +++ b/internal/git/version.go @@ -75,6 +75,14 @@ func (v Version) IsSupported() bool { return !v.LessThan(minimumVersion) } +// HashObjectFsck detects whether or not the given Git version will do fsck +// checks when git-hash-object writes objects. +func (v Version) HashObjectFsck() bool { + return !v.LessThan(Version{ + major: 2, minor: 40, patch: 0, + }) +} + // PatchIDRespectsBinaries detects whether the given Git version correctly handles binary diffs when // computing a patch ID. Previous to Git v2.39.0, git-patch-id(1) just completely ignored any binary // diffs and thus would consider two diffs the same even if a binary changed. diff --git a/internal/git2go/executor.go b/internal/git2go/executor.go index dea316d95..5ea382faf 100644 --- a/internal/git2go/executor.go +++ b/internal/git2go/executor.go @@ -10,13 +10,13 @@ import ( "strings" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/alternates" "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" glog "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/labkit/correlation" ) diff --git a/internal/git2go/featureflags_test.go b/internal/git2go/featureflags_test.go index 9e71539e1..f5c2b7b8f 100644 --- a/internal/git2go/featureflags_test.go +++ b/internal/git2go/featureflags_test.go @@ -8,11 +8,11 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" ) diff --git a/internal/gitaly/client/dial.go b/internal/gitaly/client/dial.go index def5b5fad..c22bf74e3 100644 --- a/internal/gitaly/client/dial.go +++ b/internal/gitaly/client/dial.go @@ -8,7 +8,7 @@ import ( "net/url" "time" - "gitlab.com/gitlab-org/gitaly/v16/internal/dnsresolver" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/dnsresolver" gitalyx509 "gitlab.com/gitlab-org/gitaly/v16/internal/x509" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc" diff --git a/internal/gitaly/client/dial_test.go b/internal/gitaly/client/dial_test.go index 3bc1385b7..93119ccfd 100644 --- a/internal/gitaly/client/dial_test.go +++ b/internal/gitaly/client/dial_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc" diff --git a/internal/gitaly/config/sentry/sentry.go b/internal/gitaly/config/sentry/sentry.go index e0acc7f9e..a63207201 100644 --- a/internal/gitaly/config/sentry/sentry.go +++ b/internal/gitaly/config/sentry/sentry.go @@ -5,7 +5,7 @@ import ( sentry "github.com/getsentry/sentry-go" log "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/panichandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/panichandler" ) // Config contains configuration for sentry diff --git a/internal/gitaly/hook/custom.go b/internal/gitaly/hook/custom.go index d63fc076e..6ccf968a0 100644 --- a/internal/gitaly/hook/custom.go +++ b/internal/gitaly/hook/custom.go @@ -20,7 +20,22 @@ import ( type customHooksExecutor func(ctx context.Context, args, env []string, stdin io.Reader, stdout, stderr io.Writer) error // CustomHookError is returned in case custom hooks return an error. -type CustomHookError error +type CustomHookError struct { + err error +} + +// NewCustomHookError creates a new CustomHookError. +func NewCustomHookError(e error) CustomHookError { + return CustomHookError{err: e} +} + +func (e CustomHookError) Error() string { + return e.err.Error() +} + +func (e CustomHookError) Unwrap() error { + return e.err +} // newCustomHooksExecutor creates a new hooks executor for custom hooks. Hooks // are looked up and executed in the following order: @@ -84,7 +99,7 @@ func (m *GitLabHookManager) newCustomHooksExecutor(repo *gitalypb.Repository, ho // not be modified, but instead used as-is as the hooks' error // message given that they may contain output that should be shown // to the user. - return CustomHookError(fmt.Errorf("error executing %q: %w", hookFile, err)) + return NewCustomHookError(fmt.Errorf("error executing %q: %w", hookFile, err)) } } diff --git a/internal/gitaly/hook/postreceive_test.go b/internal/gitaly/hook/postreceive_test.go index 08b12655a..468aa76b9 100644 --- a/internal/gitaly/hook/postreceive_test.go +++ b/internal/gitaly/hook/postreceive_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -18,7 +18,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" diff --git a/internal/gitaly/hook/prereceive_test.go b/internal/gitaly/hook/prereceive_test.go index 9362b0ae3..fe2d3c77e 100644 --- a/internal/gitaly/hook/prereceive_test.go +++ b/internal/gitaly/hook/prereceive_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -17,7 +17,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/hook/sidechannel.go b/internal/gitaly/hook/sidechannel.go index cc3d52621..224ed40d7 100644 --- a/internal/gitaly/hook/sidechannel.go +++ b/internal/gitaly/hook/sidechannel.go @@ -12,8 +12,8 @@ import ( "time" "gitlab.com/gitlab-org/gitaly/v16/internal/git" + gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "google.golang.org/grpc/metadata" ) diff --git a/internal/gitaly/hook/sidechannel_test.go b/internal/gitaly/hook/sidechannel_test.go index de4e0523a..da66709e9 100644 --- a/internal/gitaly/hook/sidechannel_test.go +++ b/internal/gitaly/hook/sidechannel_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" grpc_metadata "google.golang.org/grpc/metadata" ) diff --git a/internal/gitaly/hook/transactions_test.go b/internal/gitaly/hook/transactions_test.go index da2e6d1d8..bf708b15e 100644 --- a/internal/gitaly/hook/transactions_test.go +++ b/internal/gitaly/hook/transactions_test.go @@ -9,12 +9,12 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" diff --git a/internal/gitaly/hook/update_test.go b/internal/gitaly/hook/update_test.go index 55d8acc06..ab4b6010b 100644 --- a/internal/gitaly/hook/update_test.go +++ b/internal/gitaly/hook/update_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -15,7 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" diff --git a/internal/gitaly/maintenance/optimize_test.go b/internal/gitaly/maintenance/optimize_test.go index 7f1d00ff3..cdb48f727 100644 --- a/internal/gitaly/maintenance/optimize_test.go +++ b/internal/gitaly/maintenance/optimize_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" @@ -14,6 +13,7 @@ import ( repo "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/repoutil/create.go b/internal/gitaly/repoutil/create.go index f4396697c..07dedc61a 100644 --- a/internal/gitaly/repoutil/create.go +++ b/internal/gitaly/repoutil/create.go @@ -11,6 +11,7 @@ import ( "time" "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" @@ -65,7 +66,7 @@ func Create( locator storage.Locator, gitCmdFactory git.CommandFactory, txManager transaction.Manager, - repository *gitalypb.Repository, + repository repository.GitRepo, seedRepository func(repository *gitalypb.Repository) error, options ...CreateOption, ) error { diff --git a/internal/gitaly/repoutil/lock.go b/internal/gitaly/repoutil/lock.go index c11fc73fd..a3235ce18 100644 --- a/internal/gitaly/repoutil/lock.go +++ b/internal/gitaly/repoutil/lock.go @@ -6,11 +6,11 @@ import ( "path/filepath" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" - "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) // Lock attempts to lock the entire repository directory such that only one @@ -20,7 +20,7 @@ import ( // // Returns the error safe.ErrFileAlreadyLocked if the repository is already // locked. -func Lock(ctx context.Context, locator storage.Locator, repository *gitalypb.Repository) (func(), error) { +func Lock(ctx context.Context, locator storage.Locator, repository repository.GitRepo) (func(), error) { path, err := locator.GetPath(repository) if err != nil { return nil, err diff --git a/internal/gitaly/repoutil/remove.go b/internal/gitaly/repoutil/remove.go new file mode 100644 index 000000000..817876504 --- /dev/null +++ b/internal/gitaly/repoutil/remove.go @@ -0,0 +1,119 @@ +package repoutil + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" + "gitlab.com/gitlab-org/gitaly/v16/internal/safe" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" + "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/voting" +) + +// Remove will remove a repository in a race-free way with proper transactional semantics. +func Remove( + ctx context.Context, + locator storage.Locator, + txManager transaction.Manager, + repository repository.GitRepo, +) error { + path, err := locator.GetPath(repository) + if err != nil { + return structerr.NewInternal("%w", err) + } + + tempDir, err := locator.TempDir(repository.GetStorageName()) + if err != nil { + return structerr.NewInternal("temporary directory: %w", err) + } + + if err := os.MkdirAll(tempDir, perm.SharedDir); err != nil { + return structerr.NewInternal("%w", err) + } + + base := filepath.Base(path) + destDir := filepath.Join(tempDir, base+"+removed") + + // Check whether the repository exists. If not, then there is nothing we can + // remove. Historically, we didn't return an error in this case, which was just + // plain bad RPC design: callers should be able to act on this, and if they don't + // care they may still just return `NotFound` errors. + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return structerr.NewNotFound("repository does not exist") + } + + return structerr.NewInternal("statting repository: %w", err) + } + + // Lock the repository such that it cannot be created or removed by any concurrent + // RPC call. + unlock, err := Lock(ctx, locator, repository) + if err != nil { + if errors.Is(err, safe.ErrFileAlreadyLocked) { + return structerr.NewFailedPrecondition("repository is already locked") + } + return structerr.NewInternal("locking repository for removal: %w", err) + } + defer unlock() + + // Recheck whether the repository still exists after we have taken the lock. It + // could be a concurrent RPC call removed the repository while we have not yet been + // holding the lock. + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return structerr.NewNotFound("repository was concurrently removed") + } + return structerr.NewInternal("re-statting repository: %w", err) + } + + if err := voteOnAction(ctx, txManager, repository, voting.Prepared); err != nil { + return structerr.NewInternal("vote on rename: %w", err) + } + + // We move the repository into our temporary directory first before we start to + // delete it. This is done such that we don't leave behind a partially-removed and + // thus likely corrupt repository. + if err := os.Rename(path, destDir); err != nil { + return structerr.NewInternal("staging repository for removal: %w", err) + } + + if err := os.RemoveAll(destDir); err != nil { + return structerr.NewInternal("removing repository: %w", err) + } + + if err := voteOnAction(ctx, txManager, repository, voting.Committed); err != nil { + return structerr.NewInternal("vote on finalizing: %w", err) + } + + return nil +} + +func voteOnAction( + ctx context.Context, + txManager transaction.Manager, + repo repository.GitRepo, + phase voting.Phase, +) error { + return transaction.RunOnContext(ctx, func(tx txinfo.Transaction) error { + var voteStep string + switch phase { + case voting.Prepared: + voteStep = "pre-remove" + case voting.Committed: + voteStep = "post-remove" + default: + return fmt.Errorf("invalid removal step: %d", phase) + } + + vote := fmt.Sprintf("%s %s", voteStep, repo.GetRelativePath()) + return txManager.Vote(ctx, tx, voting.VoteFromData([]byte(vote)), phase) + }) +} diff --git a/internal/gitaly/repoutil/remove_test.go b/internal/gitaly/repoutil/remove_test.go new file mode 100644 index 000000000..0cbc158f1 --- /dev/null +++ b/internal/gitaly/repoutil/remove_test.go @@ -0,0 +1,92 @@ +package repoutil + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func TestRemove(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + desc string + createRepo func(tb testing.TB, ctx context.Context, cfg config.Cfg) (*gitalypb.Repository, string) + expectedErr error + }{ + { + desc: "success", + createRepo: func(tb testing.TB, ctx context.Context, cfg config.Cfg) (*gitalypb.Repository, string) { + return gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + }, + }, + { + desc: "does not exist", + createRepo: func(tb testing.TB, ctx context.Context, cfg config.Cfg) (*gitalypb.Repository, string) { + repo := &gitalypb.Repository{StorageName: cfg.Storages[0].Name, RelativePath: "/does/not/exist"} + return repo, "" + }, + expectedErr: structerr.NewNotFound("repository does not exist"), + }, + { + desc: "locked", + createRepo: func(tb testing.TB, ctx context.Context, cfg config.Cfg) (*gitalypb.Repository, string) { + repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + + // Simulate a concurrent RPC holding the repository lock. + lockPath := repoPath + ".lock" + require.NoError(t, os.WriteFile(lockPath, []byte{}, perm.SharedFile)) + tb.Cleanup(func() { + require.NoError(t, os.RemoveAll(lockPath)) + }) + + return repo, repoPath + }, + expectedErr: structerr.NewFailedPrecondition("repository is already locked"), + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + locator := config.NewLocator(cfg) + txManager := transaction.NewTrackingManager() + + repo, repoPath := tc.createRepo(t, ctx, cfg) + + if repoPath != "" { + require.DirExists(t, repoPath) + } + + err := Remove(ctx, locator, txManager, repo) + + if tc.expectedErr != nil { + require.Equal(t, tc.expectedErr, err) + return + } + + require.NoError(t, err) + + if repoPath != "" { + require.NoDirExists(t, repoPath) + } + }) + } +} diff --git a/internal/gitaly/server/auth_test.go b/internal/gitaly/server/auth_test.go index 566b5f93b..2e83b9a34 100644 --- a/internal/gitaly/server/auth_test.go +++ b/internal/gitaly/server/auth_test.go @@ -1,5 +1,3 @@ -//go:build !gitaly_test_sha256 - package server import ( @@ -16,7 +14,6 @@ import ( "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" @@ -28,7 +25,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -318,8 +316,8 @@ func TestAuthBeforeLimit(t *testing.T) { repo, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, - Seed: gittest.SeedGitLabTest, }) + commitID := gittest.WriteCommit(t, cfg, repoPath) gitlabURL, cleanup := gitlab.SetupAndStartGitlabServer(t, cfg.GitlabShell.Dir, &gitlab.TestServerOptions{ SecretToken: "secretToken", @@ -352,7 +350,7 @@ sleep %v _, err := client.UserCreateTag(ctx, &gitalypb.UserCreateTagRequest{ Repository: repo, TagName: []byte(fmt.Sprintf("tag-name-%d", i)), - TargetRevision: []byte("c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd"), + TargetRevision: []byte(commitID), User: gittest.TestUser, Message: []byte("a new tag!"), }) diff --git a/internal/gitaly/server/server.go b/internal/gitaly/server/server.go index 89668d1f2..6bdf6a734 100644 --- a/internal/gitaly/server/server.go +++ b/internal/gitaly/server/server.go @@ -8,22 +8,22 @@ import ( grpcmwlogrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" grpcmwtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/server/auth" - "gitlab.com/gitlab-org/gitaly/v16/internal/grpcstats" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/grpcstats" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/cache" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/commandstatshandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/panichandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/sentryhandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/statushandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/fieldextractors" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" gitalylog "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/logsanitizer" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/cache" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/commandstatshandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/panichandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/sentryhandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/statushandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc" diff --git a/internal/gitaly/server/server_factory.go b/internal/gitaly/server/server_factory.go index 847e5387d..10afb3ce3 100644 --- a/internal/gitaly/server/server_factory.go +++ b/internal/gitaly/server/server_factory.go @@ -4,10 +4,10 @@ import ( "sync" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "google.golang.org/grpc" ) diff --git a/internal/gitaly/server/server_factory_test.go b/internal/gitaly/server/server_factory_test.go index c61b374eb..4c54bd720 100644 --- a/internal/gitaly/server/server_factory_test.go +++ b/internal/gitaly/server/server_factory_test.go @@ -16,10 +16,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap/starter" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "golang.org/x/sync/errgroup" diff --git a/internal/gitaly/service/dependencies.go b/internal/gitaly/service/dependencies.go index 90044ff9c..3eb528e75 100644 --- a/internal/gitaly/service/dependencies.go +++ b/internal/gitaly/service/dependencies.go @@ -2,7 +2,6 @@ package service import ( "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" @@ -14,7 +13,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/streamcache" ) diff --git a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-after.txt b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-after.txt index ef577df80..f3584af8e 100644 --- a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-after.txt +++ b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-after.txt @@ -15,7 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v15/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/safe" "gitlab.com/gitlab-org/gitaly/v15/internal/structerr" "gitlab.com/gitlab-org/gitaly/v15/internal/tempdir" diff --git a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-before.txt b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-before.txt index b221a32e5..8f7604a59 100644 --- a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-before.txt +++ b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-before.txt @@ -14,7 +14,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/command" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v15/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/safe" "gitlab.com/gitlab-org/gitaly/v15/internal/structerr" "gitlab.com/gitlab-org/gitaly/v15/internal/transaction/txinfo" diff --git a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-diff.txt b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-diff.txt index cea355b74..bc99ce839 100644 --- a/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-diff.txt +++ b/internal/gitaly/service/diff/testdata/file-with-multiple-chunks-diff.txt @@ -5,7 +5,7 @@ + "gitlab.com/gitlab-org/gitaly/v15/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v15/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v15/internal/safe" "gitlab.com/gitlab-org/gitaly/v15/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v15/internal/tempdir" diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go index c5a904bd7..2ee394c39 100644 --- a/internal/gitaly/service/hook/pack_objects.go +++ b/internal/gitaly/service/hook/pack_objects.go @@ -18,12 +18,12 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "gitlab.com/gitlab-org/gitaly/v16/internal/command" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" gitalyhook "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/stream" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/gitaly/service/hook/pack_objects_test.go b/internal/gitaly/service/hook/pack_objects_test.go index 4c2de55ab..3a606fe5e 100644 --- a/internal/gitaly/service/hook/pack_objects_test.go +++ b/internal/gitaly/service/hook/pack_objects_test.go @@ -18,14 +18,14 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" hookPkg "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/streamcache" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/hook/post_receive_test.go b/internal/gitaly/service/hook/post_receive_test.go index 0551df0ff..651798908 100644 --- a/internal/gitaly/service/hook/post_receive_test.go +++ b/internal/gitaly/service/hook/post_receive_test.go @@ -10,13 +10,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/hook/pre_receive_test.go b/internal/gitaly/service/hook/pre_receive_test.go index caa76df4e..15223a0ca 100644 --- a/internal/gitaly/service/hook/pre_receive_test.go +++ b/internal/gitaly/service/hook/pre_receive_test.go @@ -13,13 +13,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/hook/reference_transaction_test.go b/internal/gitaly/service/hook/reference_transaction_test.go index 793de70e9..ec722b3b0 100644 --- a/internal/gitaly/service/hook/reference_transaction_test.go +++ b/internal/gitaly/service/hook/reference_transaction_test.go @@ -9,10 +9,10 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/hook/server.go b/internal/gitaly/service/hook/server.go index b1c1ed283..55de7f68e 100644 --- a/internal/gitaly/service/hook/server.go +++ b/internal/gitaly/service/hook/server.go @@ -6,7 +6,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" gitalyhook "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/streamcache" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) diff --git a/internal/gitaly/service/hook/update_test.go b/internal/gitaly/service/hook/update_test.go index 775f864f6..257e05c6e 100644 --- a/internal/gitaly/service/hook/update_test.go +++ b/internal/gitaly/service/hook/update_test.go @@ -11,10 +11,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/gitaly/service/internalgitaly/backup_repos.go b/internal/gitaly/service/internalgitaly/backup_repos.go new file mode 100644 index 000000000..ff43b5b5e --- /dev/null +++ b/internal/gitaly/service/internalgitaly/backup_repos.go @@ -0,0 +1,87 @@ +package internalgitaly + +import ( + "errors" + "fmt" + "io" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" +) + +func (s server) BackupRepos(stream gitalypb.InternalGitaly_BackupReposServer) error { + ctx := stream.Context() + + request, err := stream.Recv() + if err != nil { + return fmt.Errorf("backup repos: first request: %w", err) + } + + if err := validateBackupReposRequest(request); err != nil { + return structerr.NewInvalidArgument("backup repos: first request: %w", err) + } + + header := request.GetHeader() + backupID := header.GetBackupId() + + sink, err := backup.ResolveSink(ctx, header.GetStorageUrl()) + if err != nil { + return structerr.NewInvalidArgument("backup repos: resolve sink: %w", err) + } + + locator, err := backup.ResolveLocator("pointer", sink) + if err != nil { + return structerr.NewInvalidArgument("backup repos: resolve locator: %w", err) + } + + manager := backup.NewManagerLocal(sink, locator, s.locator, s.gitCmdFactory, s.catfileCache, backupID) + pipeline := backup.NewLoggingPipeline(ctxlogrus.Extract(ctx)) + + for { + for _, repo := range request.GetRepositories() { + pipeline.Handle(ctx, backup.NewCreateCommand( + manager, + // ServerInfo will be removed once restore methods are added to + // backup.Repository. Even though it is unused it must be + // non-zero so that storage.ExtractGitalyServer is not called. + storage.ServerInfo{Address: "unused"}, + repo, + false, + )) + } + + var err error + request, err = stream.Recv() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return fmt.Errorf("backup repos: receive: %w", err) + } + } + + if err := pipeline.Done(); err != nil { + return fmt.Errorf("backup repos: %w", err) + } + + if err := stream.SendAndClose(&gitalypb.BackupReposResponse{}); err != nil { + return fmt.Errorf("backup repos: %w", err) + } + + return nil +} + +func validateBackupReposRequest(req *gitalypb.BackupReposRequest) error { + header := req.Header + switch { + case header == nil: + return fmt.Errorf("empty Header") + case header.GetBackupId() == "": + return fmt.Errorf("empty BackupId") + case header.GetStorageUrl() == "": + return fmt.Errorf("empty StorageUrl") + } + return nil +} diff --git a/internal/gitaly/service/internalgitaly/backup_repos_test.go b/internal/gitaly/service/internalgitaly/backup_repos_test.go new file mode 100644 index 000000000..6b00d4da0 --- /dev/null +++ b/internal/gitaly/service/internalgitaly/backup_repos_test.go @@ -0,0 +1,161 @@ +package internalgitaly + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/backup" + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" + "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestServerBackupRepos(t *testing.T) { + t.Parallel() + + const backupID = "abc123" + + type setupData struct { + requests []*gitalypb.BackupReposRequest + expectedRepos []*gitalypb.Repository + } + + for _, tc := range []struct { + desc string + setup func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData + expectedErr error + }{ + { + desc: "missing header", + setup: func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData { + return setupData{requests: []*gitalypb.BackupReposRequest{{}}} + }, + expectedErr: status.Error(codes.InvalidArgument, `backup repos: first request: empty Header`), + }, + { + desc: "missing backup ID", + setup: func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData { + return setupData{ + requests: []*gitalypb.BackupReposRequest{{Header: &gitalypb.BackupReposRequest_Header{}}}, + } + }, + expectedErr: status.Error(codes.InvalidArgument, `backup repos: first request: empty BackupId`), + }, + { + desc: "missing storage URL", + setup: func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData { + return setupData{ + requests: []*gitalypb.BackupReposRequest{{Header: &gitalypb.BackupReposRequest_Header{ + BackupId: backupID, + }}}, + } + }, + expectedErr: status.Error(codes.InvalidArgument, `backup repos: first request: empty StorageUrl`), + }, + { + desc: "invalid storage URL", + setup: func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData { + return setupData{ + requests: []*gitalypb.BackupReposRequest{ + { + Header: &gitalypb.BackupReposRequest_Header{ + BackupId: backupID, + StorageUrl: "%invalid%", + }, + }, + }, + } + }, + expectedErr: status.Error(codes.InvalidArgument, `backup repos: resolve sink: parse "%invalid%": invalid URL escape "%in"`), + }, + { + desc: "success", + setup: func(tb testing.TB, ctx context.Context, cfg config.Cfg, storageURL string) setupData { + var repos []*gitalypb.Repository + for i := 0; i < 5; i++ { + repo, repoPath := gittest.CreateRepository(tb, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch)) + repos = append(repos, repo) + } + + return setupData{ + requests: []*gitalypb.BackupReposRequest{ + { + Header: &gitalypb.BackupReposRequest_Header{ + BackupId: backupID, + StorageUrl: storageURL, + }, + Repositories: repos[:len(repos)/2], + }, + { + Repositories: repos[len(repos)/2:], + }, + }, + expectedRepos: repos, + } + }, + }, + } { + tc := tc + + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + storageURL := testhelper.TempDir(t) + setupData := tc.setup(t, ctx, cfg, storageURL) + + catfileCache := catfile.NewCache(cfg) + t.Cleanup(catfileCache.Stop) + + srv := NewServer( + cfg.Storages, + config.NewLocator(cfg), + gittest.NewCommandFactory(t, cfg), + catfileCache, + ) + + client := setupInternalGitalyService(t, cfg, srv) + + stream, err := client.BackupRepos(ctx) + require.NoError(t, err) + + for _, req := range setupData.requests { + err = stream.Send(req) + require.NoError(t, err) + } + + _, err = stream.CloseAndRecv() + if tc.expectedErr == nil { + require.NoError(t, err) + } else { + require.Equal(t, tc.expectedErr, err) + } + + t.Log(storageURL) + sink, err := backup.ResolveSink(ctx, storageURL) + require.NoError(t, err) + + for _, repo := range setupData.expectedRepos { + relativePath := strings.TrimSuffix(repo.GetRelativePath(), ".git") + bundlePath := filepath.Join(relativePath, backupID, "001.bundle") + + r, err := sink.GetReader(ctx, bundlePath) + require.NoError(t, err) + testhelper.MustClose(t, r) + } + }) + } +} diff --git a/internal/gitaly/service/internalgitaly/server.go b/internal/gitaly/service/internalgitaly/server.go index b24ea4ede..d0ce88352 100644 --- a/internal/gitaly/service/internalgitaly/server.go +++ b/internal/gitaly/service/internalgitaly/server.go @@ -1,16 +1,32 @@ package internalgitaly import ( + "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) type server struct { gitalypb.UnimplementedInternalGitalyServer - storages []config.Storage + storages []config.Storage + locator storage.Locator + gitCmdFactory git.CommandFactory + catfileCache catfile.Cache } // NewServer return an instance of the Gitaly service. -func NewServer(storages []config.Storage) gitalypb.InternalGitalyServer { - return &server{storages: storages} +func NewServer( + storages []config.Storage, + locator storage.Locator, + gitCmdFactory git.CommandFactory, + catfileCache catfile.Cache, +) gitalypb.InternalGitalyServer { + return &server{ + storages: storages, + locator: locator, + gitCmdFactory: gitCmdFactory, + catfileCache: catfileCache, + } } diff --git a/internal/gitaly/service/internalgitaly/walkrepos_test.go b/internal/gitaly/service/internalgitaly/walkrepos_test.go index b69a8cf70..de4e5b23d 100644 --- a/internal/gitaly/service/internalgitaly/walkrepos_test.go +++ b/internal/gitaly/service/internalgitaly/walkrepos_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" @@ -70,12 +71,20 @@ func TestWalkRepos(t *testing.T) { os.Chtimes(testRepo2Path, time.Now(), modifiedDate), ) + catfileCache := catfile.NewCache(cfg) + t.Cleanup(catfileCache.Stop) + // to test a directory being deleted during a walk, we must delete a directory after // the file walk has started. To achieve that, we wrap the server to pass down a wrapped // stream that allows us to hook in to stream responses. We then delete 'b' when // the first repo 'a' is being streamed to the client. deleteOnce := sync.Once{} - srv := NewServer([]config.Storage{{Name: storageName, Path: storageRoot}}) + srv := NewServer( + []config.Storage{{Name: storageName, Path: storageRoot}}, + config.NewLocator(cfg), + gittest.NewCommandFactory(t, cfg), + catfileCache, + ) wsrv := &serverWrapper{ srv, func(r *gitalypb.WalkReposRequest, s gitalypb.InternalGitaly_WalkReposServer) error { diff --git a/internal/gitaly/service/objectpool/alternates.go b/internal/gitaly/service/objectpool/alternates.go index 89486b990..db8e4f578 100644 --- a/internal/gitaly/service/objectpool/alternates.go +++ b/internal/gitaly/service/objectpool/alternates.go @@ -10,13 +10,11 @@ import ( "time" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -204,35 +202,17 @@ func (s *server) removeAlternatesIfOk(ctx context.Context, repo *localrepo.Repo, } }() - var err error - var cmd *command.Command - - if featureflag.RevlistForConnectivity.IsEnabled(ctx) { - // The choice here of git rev-list is for performance reasons. - // git fsck --connectivity-only performed badly for large - // repositories. The reasons are detailed in https://lore.kernel.org/git/9304B938-4A59-456B-B091-DBBCAA1823B2@gmail.com/ - cmd, err = repo.Exec(ctx, git.Command{ - Name: "rev-list", - Flags: []git.Option{ - git.Flag{Name: "--objects"}, - git.Flag{Name: "--all"}, - git.Flag{Name: "--quiet"}, - }, - }) - } else { - cmd, err = repo.Exec(ctx, git.Command{ - Name: "fsck", - Flags: []git.Option{git.Flag{Name: "--connectivity-only"}}, - }, git.WithConfig(git.ConfigPair{ - // Starting with Git's f30e4d854b (fsck: verify commit graph when implicitly - // enabled, 2021-10-15), git-fsck(1) will check the commit graph for consistency - // even if `core.commitGraph` is not enabled explicitly. We do not want to verify - // whether the commit graph is consistent though, but only care about connectivity, - // so we now explicitly disable usage of the commit graph. - Key: "core.commitGraph", Value: "false", - })) - } - + // The choice here of git rev-list is for performance reasons. + // git fsck --connectivity-only performed badly for large + // repositories. The reasons are detailed in https://lore.kernel.org/git/9304B938-4A59-456B-B091-DBBCAA1823B2@gmail.com/ + cmd, err := repo.Exec(ctx, git.Command{ + Name: "rev-list", + Flags: []git.Option{ + git.Flag{Name: "--objects"}, + git.Flag{Name: "--all"}, + git.Flag{Name: "--quiet"}, + }, + }) if err != nil { return err } diff --git a/internal/gitaly/service/objectpool/alternates_test.go b/internal/gitaly/service/objectpool/alternates_test.go index 21fc62db7..bb8351d93 100644 --- a/internal/gitaly/service/objectpool/alternates_test.go +++ b/internal/gitaly/service/objectpool/alternates_test.go @@ -1,7 +1,6 @@ package objectpool import ( - "context" "os" "os/exec" "path/filepath" @@ -11,7 +10,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc/codes" @@ -20,15 +18,8 @@ import ( func TestDisconnectGitAlternates(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets( - featureflag.RevlistForConnectivity, - ).Run( - t, - testDisconnectGitAlternates, - ) -} -func testDisconnectGitAlternates(t *testing.T, ctx context.Context) { + ctx := testhelper.Context(t) cfg, repoProto, repoPath, _, client := setup(t, ctx) repo := localrepo.NewTestRepo(t, cfg, repoProto) @@ -67,13 +58,8 @@ func testDisconnectGitAlternates(t *testing.T, ctx context.Context) { func TestDisconnectGitAlternatesNoAlternates(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.RevlistForConnectivity).Run( - t, - testDisconnectGitAlternatesNoAlternates, - ) -} -func testDisconnectGitAlternatesNoAlternates(t *testing.T, ctx context.Context) { + ctx := testhelper.Context(t) cfg, repoProto, repoPath, _, client := setup(t, ctx) repo := localrepo.NewTestRepo(t, cfg, repoProto) @@ -89,13 +75,8 @@ func testDisconnectGitAlternatesNoAlternates(t *testing.T, ctx context.Context) func TestDisconnectGitAlternatesUnexpectedAlternates(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.RevlistForConnectivity).Run( - t, - testDisconnectGitAlternatesUnexpectedAlternates, - ) -} -func testDisconnectGitAlternatesUnexpectedAlternates(t *testing.T, ctx context.Context) { + ctx := testhelper.Context(t) cfg, _, _, _, client := setup(t, ctx) testCases := []struct { @@ -127,13 +108,9 @@ func testDisconnectGitAlternatesUnexpectedAlternates(t *testing.T, ctx context.C func TestRemoveAlternatesIfOk(t *testing.T) { t.Parallel() - testhelper.NewFeatureSets(featureflag.RevlistForConnectivity).Run( - t, - testRemoveAlternatesIfOk, - ) -} -func testRemoveAlternatesIfOk(t *testing.T, ctx context.Context) { + ctx := testhelper.Context(t) + t.Run("pack files are missing", func(t *testing.T) { cfg, repoProto, repoPath, _, _ := setup(t, ctx) srv := server{gitCmdFactory: gittest.NewCommandFactory(t, cfg)} diff --git a/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go b/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go index 585eccf11..176555599 100644 --- a/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go +++ b/internal/gitaly/service/objectpool/fetch_into_object_pool_test.go @@ -13,14 +13,14 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/operations/apply_patch_test.go b/internal/gitaly/service/operations/apply_patch_test.go index 54bf80892..09bdeb0a1 100644 --- a/internal/gitaly/service/operations/apply_patch_test.go +++ b/internal/gitaly/service/operations/apply_patch_test.go @@ -12,13 +12,13 @@ import ( "time" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/operations/branches_test.go b/internal/gitaly/service/operations/branches_test.go index a11bedc9a..ed18aa28d 100644 --- a/internal/gitaly/service/operations/branches_test.go +++ b/internal/gitaly/service/operations/branches_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -18,8 +17,9 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -31,18 +31,6 @@ import ( "google.golang.org/grpc/status" ) -type testTransactionServer struct { - gitalypb.UnimplementedRefTransactionServer - called int -} - -func (s *testTransactionServer) VoteTransaction(ctx context.Context, in *gitalypb.VoteTransactionRequest) (*gitalypb.VoteTransactionResponse, error) { - s.called++ - return &gitalypb.VoteTransactionResponse{ - State: gitalypb.VoteTransactionResponse_COMMIT, - }, nil -} - func TestUserCreateBranch_successful(t *testing.T) { t.Parallel() diff --git a/internal/gitaly/service/operations/merge_test.go b/internal/gitaly/service/operations/merge_test.go index 37c0377f7..d14d60b7b 100644 --- a/internal/gitaly/service/operations/merge_test.go +++ b/internal/gitaly/service/operations/merge_test.go @@ -13,7 +13,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -21,6 +20,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/gitaly/service/operations/rebase_test.go b/internal/gitaly/service/operations/rebase_test.go index 7fee6f34c..8ea7cb2e7 100644 --- a/internal/gitaly/service/operations/rebase_test.go +++ b/internal/gitaly/service/operations/rebase_test.go @@ -12,7 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/operations/squash_test.go b/internal/gitaly/service/operations/squash_test.go index 75eac5a71..41535f731 100644 --- a/internal/gitaly/service/operations/squash_test.go +++ b/internal/gitaly/service/operations/squash_test.go @@ -15,8 +15,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/operations/submodules.go b/internal/gitaly/service/operations/submodules.go index f5daad03c..ba249bcde 100644 --- a/internal/gitaly/service/operations/submodules.go +++ b/internal/gitaly/service/operations/submodules.go @@ -86,14 +86,14 @@ func (s *Server) updateSubmodule(ctx context.Context, quarantineRepo *localrepo. return "", fmt.Errorf("error reading tree: %w", err) } - var newEntries []localrepo.TreeEntry + var newEntries []*localrepo.TreeEntry var newTreeID git.ObjectID for _, entry := range entries { // If the entry's path does not match, then we simply // want to retain this tree entry. if entry.Path != base { - newEntries = append(newEntries, *entry) + newEntries = append(newEntries, entry) continue } @@ -119,7 +119,7 @@ func (s *Server) updateSubmodule(ctx context.Context, quarantineRepo *localrepo. // Otherwise, create a new tree entry submoduleFound = true - newEntries = append(newEntries, localrepo.TreeEntry{ + newEntries = append(newEntries, &localrepo.TreeEntry{ Mode: entry.Mode, Path: entry.Path, OID: replaceWith, diff --git a/internal/gitaly/service/operations/tags.go b/internal/gitaly/service/operations/tags.go index 944c9132f..5b0ec9d46 100644 --- a/internal/gitaly/service/operations/tags.go +++ b/internal/gitaly/service/operations/tags.go @@ -38,15 +38,21 @@ func (s *Server) UserDeleteTag(ctx context.Context, req *gitalypb.UserDeleteTagR } referenceName := git.ReferenceName(fmt.Sprintf("refs/tags/%s", req.TagName)) + repo := s.localrepo(req.GetRepository()) + + objectHash, err := repo.ObjectHash(ctx) + if err != nil { + return nil, fmt.Errorf("detecting object format: %w", err) + } + var revision git.ObjectID - var err error if expectedOldOID := req.GetExpectedOldOid(); expectedOldOID != "" { - revision, err = git.ObjectHashSHA1.FromHex(expectedOldOID) + revision, err = objectHash.FromHex(expectedOldOID) if err != nil { return nil, structerr.NewInvalidArgument("invalid expected old object ID: %w", err).WithMetadata("old_object_id", expectedOldOID) } - revision, err = s.localrepo(req.GetRepository()).ResolveRevision( + revision, err = repo.ResolveRevision( ctx, git.Revision(fmt.Sprintf("%s^{object}", revision)), ) if err != nil { @@ -54,7 +60,7 @@ func (s *Server) UserDeleteTag(ctx context.Context, req *gitalypb.UserDeleteTagR WithMetadata("old_object_id", expectedOldOID) } } else { - revision, err = s.localrepo(req.GetRepository()).ResolveRevision(ctx, referenceName.Revision()) + revision, err = repo.ResolveRevision(ctx, referenceName.Revision()) if err != nil { if errors.Is(err, git.ErrReferenceNotFound) { return nil, structerr.NewFailedPrecondition("tag not found: %s", req.TagName) @@ -63,7 +69,7 @@ func (s *Server) UserDeleteTag(ctx context.Context, req *gitalypb.UserDeleteTagR } } - if err := s.updateReferenceWithHooks(ctx, req.Repository, req.User, nil, referenceName, git.ObjectHashSHA1.ZeroOID, revision); err != nil { + if err := s.updateReferenceWithHooks(ctx, req.Repository, req.User, nil, referenceName, objectHash.ZeroOID, revision); err != nil { var customHookErr updateref.CustomHookError if errors.As(err, &customHookErr) { return &gitalypb.UserDeleteTagResponse{ @@ -129,6 +135,11 @@ func (s *Server) UserCreateTag(ctx context.Context, req *gitalypb.UserCreateTagR return nil, err } + objectHash, err := quarantineRepo.ObjectHash(ctx) + if err != nil { + return nil, fmt.Errorf("detecting object hash: %w", err) + } + tag, tagID, err := s.createTag(ctx, quarantineRepo, targetRevision, req.TagName, req.Message, req.User, committerTime) if err != nil { return nil, err @@ -160,7 +171,7 @@ func (s *Server) UserCreateTag(ctx context.Context, req *gitalypb.UserCreateTagR ) } - if err := s.updateReferenceWithHooks(ctx, req.Repository, req.User, quarantineDir, referenceName, tagID, git.ObjectHashSHA1.ZeroOID); err != nil { + if err := s.updateReferenceWithHooks(ctx, req.Repository, req.User, quarantineDir, referenceName, tagID, objectHash.ZeroOID); err != nil { var notAllowedError hook.NotAllowedError var customHookErr updateref.CustomHookError var updateRefError updateref.Error diff --git a/internal/gitaly/service/operations/tags_test.go b/internal/gitaly/service/operations/tags_test.go index 661be50a7..952643087 100644 --- a/internal/gitaly/service/operations/tags_test.go +++ b/internal/gitaly/service/operations/tags_test.go @@ -1,5 +1,3 @@ -//go:build !gitaly_test_sha256 - package operations import ( @@ -10,12 +8,12 @@ import ( "time" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -250,25 +248,27 @@ func TestUserDeleteTag(t *testing.T) { func TestUserDeleteTag_hooks(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) - - ctx, cfg, repo, repoPath, client := setupOperationsService(t, ctx) - tagNameInput := "to-be-déleted-soon-tag" - - request := &gitalypb.UserDeleteTagRequest{ - Repository: repo, - TagName: []byte(tagNameInput), - User: gittest.TestUser, - } + ctx := testhelper.Context(t) + ctx, cfg, client := setupOperationsServiceWithoutRepo(t, ctx) for _, hookName := range GitlabHooks { + hookName := hookName + t.Run(hookName, func(t *testing.T) { - gittest.Exec(t, cfg, "-C", repoPath, "tag", tagNameInput) + t.Parallel() + + repo, repoPath := gittest.CreateRepository(t, ctx, cfg) + commitID := gittest.WriteCommit(t, cfg, repoPath) + gittest.WriteTag(t, cfg, repoPath, "to-be-deleted", commitID.Revision()) hookOutputTempPath := gittest.WriteEnvToCustomHook(t, repoPath, hookName) - _, err := client.UserDeleteTag(ctx, request) + _, err := client.UserDeleteTag(ctx, &gitalypb.UserDeleteTagRequest{ + Repository: repo, + TagName: []byte("to-be-deleted"), + User: gittest.TestUser, + }) require.NoError(t, err) output := testhelper.MustReadFile(t, hookOutputTempPath) @@ -368,8 +368,11 @@ func TestUserCreateTag_successful(t *testing.T) { message: "This is an annotated tag", expectedResponse: &gitalypb.UserCreateTagResponse{ Tag: &gitalypb.Tag{ - Name: []byte(inputTagName), - Id: "6c6134431f05e3d22726a3876cc1fecea7df18b5", + Name: []byte(inputTagName), + Id: gittest.ObjectHashDependent(t, map[string]string{ + "sha1": "6c6134431f05e3d22726a3876cc1fecea7df18b5", + "sha256": "a9dc4f6d82a3be4e52aad29658cb59e6599498a273d02afac0b4bfd0b79d508d", + }), TargetCommit: targetRevisionCommit, Message: []byte("This is an annotated tag"), MessageSize: 24, @@ -875,10 +878,17 @@ func TestUserCreateTag_nestedTags(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - ctx, cfg, repoProto, repoPath, client := setupOperationsService(t, ctx) + ctx, cfg, client := setupOperationsServiceWithoutRepo(t, ctx) + repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg) repo := localrepo.NewTestRepo(t, cfg, repoProto) + blobID := gittest.WriteBlob(t, cfg, repoPath, []byte("content")) + treeID := gittest.WriteTree(t, cfg, repoPath, []gittest.TreeEntry{ + {Path: "blob", Mode: "100644", OID: blobID}, + }) + commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTree(treeID)) + for _, tc := range []struct { desc string targetObject string @@ -887,17 +897,17 @@ func TestUserCreateTag_nestedTags(t *testing.T) { }{ { desc: "nested tags to commit", - targetObject: "c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd", + targetObject: commitID.String(), targetObjectType: "commit", }, { desc: "nested tags to tree", targetObjectType: "tree", - targetObject: "612036fac47c5d31c212b17268e2f3ba807bce1e", + targetObject: treeID.String(), }, { desc: "nested tags to blob", - targetObject: "dfaa3f97ca337e20154a98ac9d0be76ddd1fcc82", + targetObject: blobID.String(), targetObjectType: "blob", }, } { @@ -1008,7 +1018,10 @@ func TestUserCreateTag_stableTagIDs(t *testing.T) { require.NoError(t, err) require.Equal(t, &gitalypb.Tag{ - Id: "d877784c740f492d74e6073de649a6b046ab3656", + Id: gittest.ObjectHashDependent(t, map[string]string{ + "sha1": "d877784c740f492d74e6073de649a6b046ab3656", + "sha256": "32397ee1fd84d0b06365045712b658cd2fd265e4d8478fc8186e68555abe4002", + }), Name: []byte("happy-tag"), Message: []byte("my message"), MessageSize: 10, @@ -1093,19 +1106,13 @@ func TestUserCreateTag_gitHooks(t *testing.T) { func TestUserDeleteTag_hookFailure(t *testing.T) { t.Parallel() - ctx := testhelper.Context(t) - ctx, cfg, repo, repoPath, client := setupOperationsService(t, ctx) - - tagNameInput := "to-be-deleted-soon-tag" - gittest.Exec(t, cfg, "-C", repoPath, "tag", tagNameInput) - defer gittest.Exec(t, cfg, "-C", repoPath, "tag", "-d", tagNameInput) + ctx := testhelper.Context(t) + ctx, cfg, client := setupOperationsServiceWithoutRepo(t, ctx) - request := &gitalypb.UserDeleteTagRequest{ - Repository: repo, - TagName: []byte(tagNameInput), - User: gittest.TestUser, - } + repo, repoPath := gittest.CreateRepository(t, ctx, cfg) + commitID := gittest.WriteCommit(t, cfg, repoPath) + gittest.WriteTag(t, cfg, repoPath, "to-be-deleted", commitID.Revision()) hookContent := []byte("#!/bin/sh\necho GL_ID=$GL_ID\nexit 1") @@ -1113,12 +1120,16 @@ func TestUserDeleteTag_hookFailure(t *testing.T) { t.Run(hookName, func(t *testing.T) { gittest.WriteCustomHook(t, repoPath, hookName, hookContent) - response, err := client.UserDeleteTag(ctx, request) + response, err := client.UserDeleteTag(ctx, &gitalypb.UserDeleteTagRequest{ + Repository: repo, + TagName: []byte("to-be-deleted"), + User: gittest.TestUser, + }) require.NoError(t, err) require.Contains(t, response.PreReceiveError, "GL_ID="+gittest.TestUser.GlId) tags := gittest.Exec(t, cfg, "-C", repoPath, "tag") - require.Contains(t, string(tags), tagNameInput, "tag name does not exist in tags list") + require.Contains(t, string(tags), "to-be-deleted", "tag name does not exist in tags list") }) } } @@ -1363,7 +1374,7 @@ func TestTagHookOutput(t *testing.T) { t.Parallel() ctx := testhelper.Context(t) - ctx, cfg, repo, repoPath, client := setupOperationsService(t, ctx) + ctx, cfg, client := setupOperationsServiceWithoutRepo(t, ctx) for _, tc := range []struct { desc string @@ -1426,45 +1437,55 @@ func TestTagHookOutput(t *testing.T) { hookType: gitalypb.CustomHookError_HOOK_TYPE_UPDATE, }, } { + tc := tc + hookTC := hookTC + t.Run(hookTC.hook+"/"+tc.desc, func(t *testing.T) { - tagNameInput := "some-tag" - createRequest := &gitalypb.UserCreateTagRequest{ - Repository: repo, - TagName: []byte(tagNameInput), - TargetRevision: []byte("master"), - User: gittest.TestUser, - } - deleteRequest := &gitalypb.UserDeleteTagRequest{ - Repository: repo, - TagName: []byte(tagNameInput), - User: gittest.TestUser, - } + t.Parallel() + + repo, repoPath := gittest.CreateRepository(t, ctx, cfg) + commitID := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch(git.DefaultBranch)) + gittest.WriteTag(t, cfg, repoPath, "to-be-deleted", commitID.Revision()) hookFilename := gittest.WriteCustomHook(t, repoPath, hookTC.hook, []byte(tc.hookContent)) - createResponse, err := client.UserCreateTag(ctx, createRequest) - testhelper.RequireGrpcError(t, structerr.NewPermissionDenied("reference update denied by custom hooks").WithDetail( - &gitalypb.UserCreateTagError{ - Error: &gitalypb.UserCreateTagError_CustomHook{ - CustomHook: &gitalypb.CustomHookError{ - HookType: hookTC.hookType, - Stdout: []byte(tc.expectedStdout), - Stderr: []byte(tc.expectedStderr), + t.Run("UserCreateTag", func(t *testing.T) { + t.Parallel() + + response, err := client.UserCreateTag(ctx, &gitalypb.UserCreateTagRequest{ + Repository: repo, + TagName: []byte("new-tag"), + TargetRevision: []byte(git.DefaultBranch), + User: gittest.TestUser, + }) + + testhelper.RequireGrpcError(t, structerr.NewPermissionDenied("reference update denied by custom hooks").WithDetail( + &gitalypb.UserCreateTagError{ + Error: &gitalypb.UserCreateTagError_CustomHook{ + CustomHook: &gitalypb.CustomHookError{ + HookType: hookTC.hookType, + Stdout: []byte(tc.expectedStdout), + Stderr: []byte(tc.expectedStderr), + }, }, }, - }, - ), err) - require.Nil(t, createResponse) - - defer gittest.Exec(t, cfg, "-C", repoPath, "tag", "-d", tagNameInput) - gittest.Exec(t, cfg, "-C", repoPath, "tag", tagNameInput) - - deleteResponse, err := client.UserDeleteTag(ctx, deleteRequest) - require.NoError(t, err) - deleteResponseOk := &gitalypb.UserDeleteTagResponse{ - PreReceiveError: tc.expectedErr(hookFilename), - } - testhelper.ProtoEqual(t, deleteResponseOk, deleteResponse) + ), err) + require.Nil(t, response) + }) + + t.Run("UserDeleteTag", func(t *testing.T) { + t.Parallel() + + response, err := client.UserDeleteTag(ctx, &gitalypb.UserDeleteTagRequest{ + Repository: repo, + TagName: []byte("to-be-deleted"), + User: gittest.TestUser, + }) + require.NoError(t, err) + testhelper.ProtoEqual(t, &gitalypb.UserDeleteTagResponse{ + PreReceiveError: tc.expectedErr(hookFilename), + }, response) + }) }) } } diff --git a/internal/gitaly/service/operations/testhelper_test.go b/internal/gitaly/service/operations/testhelper_test.go index ff6ada770..9e4d5a07d 100644 --- a/internal/gitaly/service/operations/testhelper_test.go +++ b/internal/gitaly/service/operations/testhelper_test.go @@ -1,5 +1,3 @@ -//go:build !gitaly_test_sha256 - package operations import ( @@ -168,3 +166,15 @@ func setupAndStartGitlabServer(tb testing.TB, glID, glRepository string, cfg con return url } + +type testTransactionServer struct { + gitalypb.UnimplementedRefTransactionServer + called int +} + +func (s *testTransactionServer) VoteTransaction(ctx context.Context, in *gitalypb.VoteTransactionRequest) (*gitalypb.VoteTransactionResponse, error) { + s.called++ + return &gitalypb.VoteTransactionResponse{ + State: gitalypb.VoteTransactionResponse_COMMIT, + }, nil +} diff --git a/internal/gitaly/service/ref/delete_refs_test.go b/internal/gitaly/service/ref/delete_refs_test.go index 33c7639ca..831c21663 100644 --- a/internal/gitaly/service/ref/delete_refs_test.go +++ b/internal/gitaly/service/ref/delete_refs_test.go @@ -14,7 +14,7 @@ import ( hookservice "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/repository/apply_gitattributes_test.go b/internal/gitaly/service/repository/apply_gitattributes_test.go index 77923d770..897c20938 100644 --- a/internal/gitaly/service/repository/apply_gitattributes_test.go +++ b/internal/gitaly/service/repository/apply_gitattributes_test.go @@ -8,10 +8,10 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/repository/create_repository_from_bundle_test.go b/internal/gitaly/service/repository/create_repository_from_bundle_test.go index b44f8419a..e25ada641 100644 --- a/internal/gitaly/service/repository/create_repository_from_bundle_test.go +++ b/internal/gitaly/service/repository/create_repository_from_bundle_test.go @@ -18,8 +18,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/praefectutil" "gitlab.com/gitlab-org/gitaly/v16/internal/tempdir" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/gitaly/service/repository/create_repository_test.go b/internal/gitaly/service/repository/create_repository_test.go index 5f532431d..ed7b1130c 100644 --- a/internal/gitaly/service/repository/create_repository_test.go +++ b/internal/gitaly/service/repository/create_repository_test.go @@ -13,8 +13,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/auth" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/praefectutil" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/gitaly/service/repository/fetch_bundle_test.go b/internal/gitaly/service/repository/fetch_bundle_test.go index e27056ae3..b7eed4669 100644 --- a/internal/gitaly/service/repository/fetch_bundle_test.go +++ b/internal/gitaly/service/repository/fetch_bundle_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gitalyhook "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/repository/fetch_remote_test.go b/internal/gitaly/service/repository/fetch_remote_test.go index b33c0fac4..a14a04e29 100644 --- a/internal/gitaly/service/repository/fetch_remote_test.go +++ b/internal/gitaly/service/repository/fetch_remote_test.go @@ -15,9 +15,9 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/repository/license_test.go b/internal/gitaly/service/repository/license_test.go index 01f1f4f4d..01a703def 100644 --- a/internal/gitaly/service/repository/license_test.go +++ b/internal/gitaly/service/repository/license_test.go @@ -9,9 +9,9 @@ import ( "github.com/go-enry/go-license-detector/v4/licensedb" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/repository/optimize_test.go b/internal/gitaly/service/repository/optimize_test.go index 443be1a6a..f94718a03 100644 --- a/internal/gitaly/service/repository/optimize_test.go +++ b/internal/gitaly/service/repository/optimize_test.go @@ -14,13 +14,13 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/repository/prune_unreachable_objects.go b/internal/gitaly/service/repository/prune_unreachable_objects.go index ae5e54a4f..c2ef446ca 100644 --- a/internal/gitaly/service/repository/prune_unreachable_objects.go +++ b/internal/gitaly/service/repository/prune_unreachable_objects.go @@ -57,7 +57,7 @@ func (s *server) PruneUnreachableObjects( if err := housekeeping.RepackObjects(ctx, repo, housekeeping.RepackObjectsConfig{ Strategy: housekeeping.RepackObjectsStrategyFullWithCruft, WriteMultiPackIndex: true, - WriteBitmap: len(repoInfo.Alternates) == 0, + WriteBitmap: len(repoInfo.Alternates.ObjectDirectories) == 0, CruftExpireBefore: expireBefore, }); err != nil { return nil, structerr.NewInternal("repacking objects: %w", err) diff --git a/internal/gitaly/service/repository/remove.go b/internal/gitaly/service/repository/remove.go index abeab649d..299ecb7fb 100644 --- a/internal/gitaly/service/repository/remove.go +++ b/internal/gitaly/service/repository/remove.go @@ -2,19 +2,10 @@ package repository import ( "context" - "errors" - "fmt" - "os" - "path/filepath" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" - "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" - "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" - "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/voting" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -23,91 +14,10 @@ func (s *server) RemoveRepository(ctx context.Context, in *gitalypb.RemoveReposi if err := service.ValidateRepository(repository); err != nil { return nil, structerr.NewInvalidArgument("%w", err) } - path, err := s.locator.GetPath(repository) - if err != nil { - return nil, structerr.NewInternal("%w", err) - } - - tempDir, err := s.locator.TempDir(repository.GetStorageName()) - if err != nil { - return nil, structerr.NewInternal("temporary directory: %w", err) - } - - if err := os.MkdirAll(tempDir, perm.SharedDir); err != nil { - return nil, structerr.NewInternal("%w", err) - } - - base := filepath.Base(path) - destDir := filepath.Join(tempDir, base+"+removed") - - // Check whether the repository exists. If not, then there is nothing we can - // remove. Historically, we didn't return an error in this case, which was just - // plain bad RPC design: callers should be able to act on this, and if they don't - // care they may still just return `NotFound` errors. - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return nil, structerr.NewNotFound("repository does not exist") - } - - return nil, structerr.NewInternal("statting repository: %w", err) - } - - // Lock the repository such that it cannot be created or removed by any concurrent - // RPC call. - unlock, err := repoutil.Lock(ctx, s.locator, repository) - if err != nil { - if errors.Is(err, safe.ErrFileAlreadyLocked) { - return nil, structerr.NewFailedPrecondition("repository is already locked") - } - return nil, structerr.NewInternal("locking repository for removal: %w", err) - } - defer unlock() - // Recheck whether the repository still exists after we have taken the lock. It - // could be a concurrent RPC call removed the repository while we have not yet been - // holding the lock. - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return nil, structerr.NewNotFound("repository was concurrently removed") - } - return nil, structerr.NewInternal("re-statting repository: %w", err) - } - - if err := s.voteOnAction(ctx, repository, voting.Prepared); err != nil { - return nil, structerr.NewInternal("vote on rename: %w", err) - } - - // We move the repository into our temporary directory first before we start to - // delete it. This is done such that we don't leave behind a partially-removed and - // thus likely corrupt repository. - if err := os.Rename(path, destDir); err != nil { - return nil, structerr.NewInternal("staging repository for removal: %w", err) - } - - if err := os.RemoveAll(destDir); err != nil { - return nil, structerr.NewInternal("removing repository: %w", err) - } - - if err := s.voteOnAction(ctx, repository, voting.Committed); err != nil { - return nil, structerr.NewInternal("vote on finalizing: %w", err) + if err := repoutil.Remove(ctx, s.locator, s.txManager, repository); err != nil { + return nil, err } return &gitalypb.RemoveRepositoryResponse{}, nil } - -func (s *server) voteOnAction(ctx context.Context, repo *gitalypb.Repository, phase voting.Phase) error { - return transaction.RunOnContext(ctx, func(tx txinfo.Transaction) error { - var voteStep string - switch phase { - case voting.Prepared: - voteStep = "pre-remove" - case voting.Committed: - voteStep = "post-remove" - default: - return fmt.Errorf("invalid removal step: %d", phase) - } - - vote := fmt.Sprintf("%s %s", voteStep, repo.GetRelativePath()) - return s.txManager.Vote(ctx, tx, voting.VoteFromData([]byte(vote)), phase) - }) -} diff --git a/internal/gitaly/service/repository/replicate.go b/internal/gitaly/service/repository/replicate.go index 32251f386..10ecb3157 100644 --- a/internal/gitaly/service/repository/replicate.go +++ b/internal/gitaly/service/repository/replicate.go @@ -20,8 +20,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tempdir" diff --git a/internal/gitaly/service/repository/replicate_test.go b/internal/gitaly/service/repository/replicate_test.go index 640c0d93f..6a90e7560 100644 --- a/internal/gitaly/service/repository/replicate_test.go +++ b/internal/gitaly/service/repository/replicate_test.go @@ -17,7 +17,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -26,9 +25,10 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/repository/set_custom_hooks_test.go b/internal/gitaly/service/repository/set_custom_hooks_test.go index c1b9579bf..517b31314 100644 --- a/internal/gitaly/service/repository/set_custom_hooks_test.go +++ b/internal/gitaly/service/repository/set_custom_hooks_test.go @@ -15,8 +15,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/repository/write_ref_test.go b/internal/gitaly/service/repository/write_ref_test.go index 9158f54b7..c12d8d82e 100644 --- a/internal/gitaly/service/repository/write_ref_test.go +++ b/internal/gitaly/service/repository/write_ref_test.go @@ -11,8 +11,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/setup/register.go b/internal/gitaly/service/setup/register.go index 2c40bb220..50f0482c1 100644 --- a/internal/gitaly/service/setup/register.go +++ b/internal/gitaly/service/setup/register.go @@ -144,7 +144,12 @@ func RegisterAll(srv *grpc.Server, deps *service.Dependencies) { deps.GetPackObjectsConcurrencyTracker(), deps.GetPackObjectsLimiter(), )) - gitalypb.RegisterInternalGitalyServer(srv, internalgitaly.NewServer(deps.GetCfg().Storages)) + gitalypb.RegisterInternalGitalyServer(srv, internalgitaly.NewServer( + deps.GetCfg().Storages, + deps.GetLocator(), + deps.GetGitCmdFactory(), + deps.GetCatfileCache(), + )) healthpb.RegisterHealthServer(srv, health.NewServer()) reflection.Register(srv) diff --git a/internal/gitaly/service/smarthttp/inforefs_test.go b/internal/gitaly/service/smarthttp/inforefs_test.go index e78ec4f32..364f132ba 100644 --- a/internal/gitaly/service/smarthttp/inforefs_test.go +++ b/internal/gitaly/service/smarthttp/inforefs_test.go @@ -13,12 +13,12 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/service/smarthttp/receive_pack_test.go b/internal/gitaly/service/smarthttp/receive_pack_test.go index 7e14ca0eb..5052481cf 100644 --- a/internal/gitaly/service/smarthttp/receive_pack_test.go +++ b/internal/gitaly/service/smarthttp/receive_pack_test.go @@ -13,7 +13,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" @@ -21,9 +21,9 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" gitalyhook "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/smarthttp/testhelper_test.go b/internal/gitaly/service/smarthttp/testhelper_test.go index 823323888..0a9a455b3 100644 --- a/internal/gitaly/service/smarthttp/testhelper_test.go +++ b/internal/gitaly/service/smarthttp/testhelper_test.go @@ -6,13 +6,13 @@ import ( "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" hookservice "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/objectpool" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/gitaly/service/smarthttp/upload_pack.go b/internal/gitaly/service/smarthttp/upload_pack.go index 7ff667e40..b28f35591 100644 --- a/internal/gitaly/service/smarthttp/upload_pack.go +++ b/internal/gitaly/service/smarthttp/upload_pack.go @@ -11,7 +11,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) diff --git a/internal/gitaly/service/smarthttp/upload_pack_test.go b/internal/gitaly/service/smarthttp/upload_pack_test.go index 6ec595b23..c75df1496 100644 --- a/internal/gitaly/service/smarthttp/upload_pack_test.go +++ b/internal/gitaly/service/smarthttp/upload_pack_test.go @@ -18,7 +18,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/ssh/receive_pack_test.go b/internal/gitaly/service/ssh/receive_pack_test.go index 710576f79..365ee7ef6 100644 --- a/internal/gitaly/service/ssh/receive_pack_test.go +++ b/internal/gitaly/service/ssh/receive_pack_test.go @@ -12,15 +12,15 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/service/ssh/upload_pack.go b/internal/gitaly/service/ssh/upload_pack.go index b65f3899e..5392f092f 100644 --- a/internal/gitaly/service/ssh/upload_pack.go +++ b/internal/gitaly/service/ssh/upload_pack.go @@ -15,8 +15,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/pktline" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/stream" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/gitaly/service/ssh/upload_pack_test.go b/internal/gitaly/service/ssh/upload_pack_test.go index d56c8ab0d..811a91303 100644 --- a/internal/gitaly/service/ssh/upload_pack_test.go +++ b/internal/gitaly/service/ssh/upload_pack_test.go @@ -16,14 +16,14 @@ import ( promtest "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/gitaly/transaction/manager.go b/internal/gitaly/transaction/manager.go index 1cfc7bcb5..b34c448d9 100644 --- a/internal/gitaly/transaction/manager.go +++ b/internal/gitaly/transaction/manager.go @@ -10,9 +10,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" internalclient "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/txinfo" "gitlab.com/gitlab-org/gitaly/v16/internal/transaction/voting" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/gitaly/transaction/manager_test.go b/internal/gitaly/transaction/manager_test.go index a39a5adc7..b500a73be 100644 --- a/internal/gitaly/transaction/manager_test.go +++ b/internal/gitaly/transaction/manager_test.go @@ -6,11 +6,11 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" diff --git a/internal/gitaly/transaction/voting_test.go b/internal/gitaly/transaction/voting_test.go index c6cb681f5..0116a6b23 100644 --- a/internal/gitaly/transaction/voting_test.go +++ b/internal/gitaly/transaction/voting_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/backchannel/backchannel.go b/internal/grpc/backchannel/backchannel.go index 990e3e5fb..990e3e5fb 100644 --- a/internal/backchannel/backchannel.go +++ b/internal/grpc/backchannel/backchannel.go diff --git a/internal/backchannel/backchannel_example_test.go b/internal/grpc/backchannel/backchannel_example_test.go index dea595fd3..0bc341e6e 100644 --- a/internal/backchannel/backchannel_example_test.go +++ b/internal/grpc/backchannel/backchannel_example_test.go @@ -6,8 +6,8 @@ import ( "net" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/backchannel/backchannel_test.go b/internal/grpc/backchannel/backchannel_test.go index 9ad009a27..a7c8fed6c 100644 --- a/internal/backchannel/backchannel_test.go +++ b/internal/grpc/backchannel/backchannel_test.go @@ -13,7 +13,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc" diff --git a/internal/backchannel/client.go b/internal/grpc/backchannel/client.go index c4526b7cc..c4526b7cc 100644 --- a/internal/backchannel/client.go +++ b/internal/grpc/backchannel/client.go diff --git a/internal/backchannel/registry.go b/internal/grpc/backchannel/registry.go index 407d770f9..407d770f9 100644 --- a/internal/backchannel/registry.go +++ b/internal/grpc/backchannel/registry.go diff --git a/internal/backchannel/server.go b/internal/grpc/backchannel/server.go index a6fe677ee..a6fe677ee 100644 --- a/internal/backchannel/server.go +++ b/internal/grpc/backchannel/server.go diff --git a/internal/dnsresolver/builder.go b/internal/grpc/dnsresolver/builder.go index eaf1e4299..eaf1e4299 100644 --- a/internal/dnsresolver/builder.go +++ b/internal/grpc/dnsresolver/builder.go diff --git a/internal/dnsresolver/builder_test.go b/internal/grpc/dnsresolver/builder_test.go index 7cbbf711c..7cbbf711c 100644 --- a/internal/dnsresolver/builder_test.go +++ b/internal/grpc/dnsresolver/builder_test.go diff --git a/internal/dnsresolver/noop.go b/internal/grpc/dnsresolver/noop.go index 6197653ed..6197653ed 100644 --- a/internal/dnsresolver/noop.go +++ b/internal/grpc/dnsresolver/noop.go diff --git a/internal/dnsresolver/resolver.go b/internal/grpc/dnsresolver/resolver.go index a6ecf3136..a6ecf3136 100644 --- a/internal/dnsresolver/resolver.go +++ b/internal/grpc/dnsresolver/resolver.go diff --git a/internal/dnsresolver/resolver_test.go b/internal/grpc/dnsresolver/resolver_test.go index 0f05d20e3..0f05d20e3 100644 --- a/internal/dnsresolver/resolver_test.go +++ b/internal/grpc/dnsresolver/resolver_test.go diff --git a/internal/dnsresolver/target.go b/internal/grpc/dnsresolver/target.go index 0e1ac3f8a..0e1ac3f8a 100644 --- a/internal/dnsresolver/target.go +++ b/internal/grpc/dnsresolver/target.go diff --git a/internal/dnsresolver/target_test.go b/internal/grpc/dnsresolver/target_test.go index 36ad9bf49..36ad9bf49 100644 --- a/internal/dnsresolver/target_test.go +++ b/internal/grpc/dnsresolver/target_test.go diff --git a/internal/dnsresolver/testhelper_test.go b/internal/grpc/dnsresolver/testhelper_test.go index eded29045..eded29045 100644 --- a/internal/dnsresolver/testhelper_test.go +++ b/internal/grpc/dnsresolver/testhelper_test.go diff --git a/internal/grpcstats/stats.go b/internal/grpc/grpcstats/stats.go index 89782e538..89782e538 100644 --- a/internal/grpcstats/stats.go +++ b/internal/grpc/grpcstats/stats.go diff --git a/internal/grpcstats/stats_test.go b/internal/grpc/grpcstats/stats_test.go index 84d13930c..84d13930c 100644 --- a/internal/grpcstats/stats_test.go +++ b/internal/grpc/grpcstats/stats_test.go diff --git a/internal/grpcstats/testhelper_test.go b/internal/grpc/grpcstats/testhelper_test.go index f57175b92..f57175b92 100644 --- a/internal/grpcstats/testhelper_test.go +++ b/internal/grpc/grpcstats/testhelper_test.go diff --git a/internal/listenmux/mux.go b/internal/grpc/listenmux/mux.go index f5a1ed67a..f5a1ed67a 100644 --- a/internal/listenmux/mux.go +++ b/internal/grpc/listenmux/mux.go diff --git a/internal/listenmux/mux_test.go b/internal/grpc/listenmux/mux_test.go index 76997c0c1..76997c0c1 100644 --- a/internal/listenmux/mux_test.go +++ b/internal/grpc/listenmux/mux_test.go diff --git a/internal/metadata/metadata.go b/internal/grpc/metadata/metadata.go index 83e6e1247..83e6e1247 100644 --- a/internal/metadata/metadata.go +++ b/internal/grpc/metadata/metadata.go diff --git a/internal/metadata/metadata_test.go b/internal/grpc/metadata/metadata_test.go index 7d716f6a7..7d716f6a7 100644 --- a/internal/metadata/metadata_test.go +++ b/internal/grpc/metadata/metadata_test.go diff --git a/internal/middleware/cache/cache.go b/internal/grpc/middleware/cache/cache.go index cb5fbb219..cb5fbb219 100644 --- a/internal/middleware/cache/cache.go +++ b/internal/grpc/middleware/cache/cache.go diff --git a/internal/middleware/cache/cache_test.go b/internal/grpc/middleware/cache/cache_test.go index 588063474..588063474 100644 --- a/internal/middleware/cache/cache_test.go +++ b/internal/grpc/middleware/cache/cache_test.go diff --git a/internal/middleware/cache/export_test.go b/internal/grpc/middleware/cache/export_test.go index 48c1dab84..48c1dab84 100644 --- a/internal/middleware/cache/export_test.go +++ b/internal/grpc/middleware/cache/export_test.go diff --git a/internal/middleware/cache/prometheus.go b/internal/grpc/middleware/cache/prometheus.go index 2d1790e3f..2d1790e3f 100644 --- a/internal/middleware/cache/prometheus.go +++ b/internal/grpc/middleware/cache/prometheus.go diff --git a/internal/middleware/commandstatshandler/commandstatshandler.go b/internal/grpc/middleware/commandstatshandler/commandstatshandler.go index c1029e1d8..c1029e1d8 100644 --- a/internal/middleware/commandstatshandler/commandstatshandler.go +++ b/internal/grpc/middleware/commandstatshandler/commandstatshandler.go diff --git a/internal/middleware/commandstatshandler/commandstatshandler_test.go b/internal/grpc/middleware/commandstatshandler/commandstatshandler_test.go index b8305d74f..483a2cc3d 100644 --- a/internal/middleware/commandstatshandler/commandstatshandler_test.go +++ b/internal/grpc/middleware/commandstatshandler/commandstatshandler_test.go @@ -10,13 +10,13 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/ref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/middleware/featureflag/featureflag_handler.go b/internal/grpc/middleware/featureflag/featureflag_handler.go index 0bd40a3e6..1342b4d5f 100644 --- a/internal/middleware/featureflag/featureflag_handler.go +++ b/internal/grpc/middleware/featureflag/featureflag_handler.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" ) // FieldsProducer adds feature_flags logging fields to gRPC logs. Only enabled flags are available. diff --git a/internal/middleware/featureflag/featureflag_handler_test.go b/internal/grpc/middleware/featureflag/featureflag_handler_test.go index b4834d0c6..5df5850af 100644 --- a/internal/middleware/featureflag/featureflag_handler_test.go +++ b/internal/grpc/middleware/featureflag/featureflag_handler_test.go @@ -9,8 +9,8 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "google.golang.org/grpc" diff --git a/internal/middleware/limithandler/concurrency_limiter.go b/internal/grpc/middleware/limithandler/concurrency_limiter.go index 1f807d9db..52b4c293b 100644 --- a/internal/middleware/limithandler/concurrency_limiter.go +++ b/internal/grpc/middleware/limithandler/concurrency_limiter.go @@ -76,7 +76,6 @@ func (sem *keyedConcurrencyLimiter) acquire(ctx context.Context, limitingKey str } }() default: - sem.monitor.Dropped(ctx, limitingKey, len(sem.queueTokens), "max_size") return ErrMaxQueueSize } } @@ -104,10 +103,8 @@ func (sem *keyedConcurrencyLimiter) acquire(ctx context.Context, limitingKey str case sem.concurrencyTokens <- struct{}{}: return nil case <-ticker.C(): - sem.monitor.Dropped(ctx, limitingKey, len(sem.queueTokens), "max_time") return ErrMaxQueueTime case <-ctx.Done(): - sem.monitor.Dropped(ctx, limitingKey, len(sem.queueTokens), "other") return ctx.Err() } } @@ -120,6 +117,11 @@ func (sem *keyedConcurrencyLimiter) release() { <-sem.concurrencyTokens } +// queueLength returns the length of token queue +func (sem *keyedConcurrencyLimiter) queueLength() int { + return len(sem.queueTokens) +} + // ConcurrencyLimiter contains rate limiter state. type ConcurrencyLimiter struct { // maxConcurrencyLimit is the maximum number of concurrent calls to the limited function. @@ -186,18 +188,22 @@ func (c *ConcurrencyLimiter) Limit(ctx context.Context, limitingKey string, f Li start := time.Now() if err := sem.acquire(ctx, limitingKey); err != nil { + queueTime := time.Since(start) switch err { case ErrMaxQueueSize: + c.monitor.Dropped(ctx, limitingKey, sem.queueLength(), queueTime, "max_size") return nil, structerr.NewResourceExhausted("%w", ErrMaxQueueSize).WithDetail(&gitalypb.LimitError{ ErrorMessage: err.Error(), RetryAfter: durationpb.New(0), }) case ErrMaxQueueTime: + c.monitor.Dropped(ctx, limitingKey, sem.queueLength(), queueTime, "max_time") return nil, structerr.NewResourceExhausted("%w", ErrMaxQueueTime).WithDetail(&gitalypb.LimitError{ ErrorMessage: err.Error(), RetryAfter: durationpb.New(0), }) default: + c.monitor.Dropped(ctx, limitingKey, sem.queueLength(), queueTime, "other") return nil, fmt.Errorf("unexpected error when dequeueing request: %w", err) } } diff --git a/internal/middleware/limithandler/concurrency_limiter_test.go b/internal/grpc/middleware/limithandler/concurrency_limiter_test.go index 0a0fa0734..02adb7d04 100644 --- a/internal/middleware/limithandler/concurrency_limiter_test.go +++ b/internal/grpc/middleware/limithandler/concurrency_limiter_test.go @@ -77,7 +77,7 @@ func (c *counter) Exit(context.Context) { c.exit++ } -func (c *counter) Dropped(_ context.Context, _ string, _ int, reason string) { +func (c *counter) Dropped(_ context.Context, _ string, _ int, _ time.Duration, reason string) { switch reason { case "max_time": c.droppedTime++ diff --git a/internal/middleware/limithandler/middleware.go b/internal/grpc/middleware/limithandler/middleware.go index c532943a4..c532943a4 100644 --- a/internal/middleware/limithandler/middleware.go +++ b/internal/grpc/middleware/limithandler/middleware.go diff --git a/internal/middleware/limithandler/middleware_test.go b/internal/grpc/middleware/limithandler/middleware_test.go index e3a260f32..bd5f28240 100644 --- a/internal/middleware/limithandler/middleware_test.go +++ b/internal/grpc/middleware/limithandler/middleware_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/duration" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/middleware/limithandler/monitor.go b/internal/grpc/middleware/limithandler/monitor.go index da295c563..b251df6d4 100644 --- a/internal/middleware/limithandler/monitor.go +++ b/internal/grpc/middleware/limithandler/monitor.go @@ -14,16 +14,16 @@ type ConcurrencyMonitor interface { Dequeued(ctx context.Context) Enter(ctx context.Context, acquireTime time.Duration) Exit(ctx context.Context) - Dropped(ctx context.Context, key string, length int, message string) + Dropped(ctx context.Context, key string, length int, acquireTime time.Duration, message string) } type noopConcurrencyMonitor struct{} -func (c *noopConcurrencyMonitor) Queued(context.Context, string, int) {} -func (c *noopConcurrencyMonitor) Dequeued(context.Context) {} -func (c *noopConcurrencyMonitor) Enter(context.Context, time.Duration) {} -func (c *noopConcurrencyMonitor) Exit(context.Context) {} -func (c *noopConcurrencyMonitor) Dropped(context.Context, string, int, string) {} +func (c *noopConcurrencyMonitor) Queued(context.Context, string, int) {} +func (c *noopConcurrencyMonitor) Dequeued(context.Context) {} +func (c *noopConcurrencyMonitor) Enter(context.Context, time.Duration) {} +func (c *noopConcurrencyMonitor) Exit(context.Context) {} +func (c *noopConcurrencyMonitor) Dropped(context.Context, string, int, time.Duration, string) {} // NewNoopConcurrencyMonitor returns a noopConcurrencyMonitor func NewNoopConcurrencyMonitor() ConcurrencyMonitor { @@ -99,11 +99,12 @@ func (p *PromMonitor) Exit(ctx context.Context) { } // Dropped is called when a request is dropped. -func (p *PromMonitor) Dropped(ctx context.Context, key string, length int, reason string) { +func (p *PromMonitor) Dropped(ctx context.Context, key string, length int, acquireTime time.Duration, reason string) { if stats := limitStatsFromContext(ctx); stats != nil { stats.SetLimitingKey(p.limitingType, key) stats.SetConcurrencyQueueLength(length) stats.SetConcurrencyDroppedReason(reason) + stats.AddConcurrencyQueueMs(acquireTime.Milliseconds()) } p.requestsDroppedMetric.WithLabelValues(reason).Inc() } diff --git a/internal/grpc/middleware/limithandler/monitor_test.go b/internal/grpc/middleware/limithandler/monitor_test.go new file mode 100644 index 000000000..3b5d6163f --- /dev/null +++ b/internal/grpc/middleware/limithandler/monitor_test.go @@ -0,0 +1,422 @@ +package limithandler + +import ( + "bytes" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + promconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" +) + +func TestNewPerRPCPromMonitor(t *testing.T) { + system := "gitaly" + fullMethod := "fullMethod" + createNewMonitor := func() *PromMonitor { + acquiringSecondsVec := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "acquiring_seconds", + Help: "seconds to acquire", + Buckets: promconfig.DefaultConfig().GRPCLatencyBuckets, + }, + []string{"system", "grpc_service", "grpc_method"}, + ) + inProgressMetric := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "in_progress", + Help: "requests in progress", + }, + []string{"system", "grpc_service", "grpc_method"}, + ) + queuedMetric := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "queued", + Help: "number of queued requests", + }, + []string{"system", "grpc_service", "grpc_method"}, + ) + requestsDroppedMetric := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "dropped", + Help: "number of dropped requests", + }, + []string{ + "system", + "grpc_service", + "grpc_method", + "reason", + }, + ) + return newPerRPCPromMonitor( + system, + fullMethod, + queuedMetric, + inProgressMetric, + acquiringSecondsVec, + requestsDroppedMetric, + ) + } + + t.Run("request is dequeued successfully", func(t *testing.T) { + rpcMonitor := createNewMonitor() + ctx := InitLimitStats(testhelper.Context(t)) + + rpcMonitor.Queued(ctx, fullMethod, 5) + rpcMonitor.Enter(ctx, time.Second) + + expectedMetrics := `# HELP acquiring_seconds seconds to acquire +# TYPE acquiring_seconds histogram +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 1 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 1 +acquiring_seconds_sum{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 +acquiring_seconds_count{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 +# HELP in_progress requests in progress +# TYPE in_progress gauge +in_progress{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 +# HELP queued number of queued requests +# TYPE queued gauge +queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 + +` + require.NoError(t, testutil.CollectAndCompare( + rpcMonitor, + bytes.NewBufferString(expectedMetrics), + "in_progress", + "queued", + "dropped", + "acquiring_seconds", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePerRPC, + "limit.limiting_key": fullMethod, + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + }, stats.Fields()) + + // After the request exists, in_progress and queued gauge decrease + rpcMonitor.Exit(ctx) + rpcMonitor.Dequeued(ctx) + expectedMetrics = `# HELP acquiring_seconds seconds to acquire +# HELP in_progress requests in progress +# TYPE in_progress gauge +in_progress{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +# HELP queued number of queued requests +# TYPE queued gauge +queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 + +` + require.NoError(t, testutil.CollectAndCompare( + rpcMonitor, + bytes.NewBufferString(expectedMetrics), + "in_progress", + "queued", + )) + }) + + t.Run("request is dropped after queueing", func(t *testing.T) { + rpcMonitor := createNewMonitor() + ctx := InitLimitStats(testhelper.Context(t)) + + rpcMonitor.Queued(ctx, fullMethod, 5) + rpcMonitor.Dropped(ctx, fullMethod, 5, time.Second, "load") + + expectedMetrics := `# HELP acquiring_seconds seconds to acquire +# TYPE acquiring_seconds histogram +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 0 +acquiring_seconds_sum{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +acquiring_seconds_count{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +# HELP dropped number of dropped requests +# TYPE dropped counter +dropped{grpc_method="unknown",grpc_service="unknown",reason="load",system="gitaly"} 1 +# HELP in_progress requests in progress +# TYPE in_progress gauge +in_progress{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +# HELP queued number of queued requests +# TYPE queued gauge +queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 + +` + require.NoError(t, testutil.CollectAndCompare( + rpcMonitor, + bytes.NewBufferString(expectedMetrics), + "in_progress", + "queued", + "dropped", + "acquiring_seconds", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePerRPC, + "limit.limiting_key": fullMethod, + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + "limit.concurrency_dropped": "load", + }, stats.Fields()) + }) + + t.Run("request is dropped before queueing", func(t *testing.T) { + rpcMonitor := createNewMonitor() + ctx := InitLimitStats(testhelper.Context(t)) + rpcMonitor.Dropped(ctx, fullMethod, 5, time.Second, "load") + + expectedMetrics := `# HELP acquiring_seconds seconds to acquire +# TYPE acquiring_seconds histogram +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 0 +acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 0 +acquiring_seconds_sum{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +acquiring_seconds_count{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +# HELP dropped number of dropped requests +# TYPE dropped counter +dropped{grpc_method="unknown",grpc_service="unknown",reason="load",system="gitaly"} 1 +# HELP in_progress requests in progress +# TYPE in_progress gauge +in_progress{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 +# HELP queued number of queued requests +# TYPE queued gauge +queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 0 + +` + require.NoError(t, testutil.CollectAndCompare( + rpcMonitor, + bytes.NewBufferString(expectedMetrics), + "in_progress", + "queued", + "dropped", + "acquiring_seconds", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePerRPC, + "limit.limiting_key": fullMethod, + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + "limit.concurrency_dropped": "load", + }, stats.Fields()) + }) +} + +func TestNewPackObjectsConcurrencyMonitor(t *testing.T) { + t.Run("request is dequeued successfully", func(t *testing.T) { + ctx := InitLimitStats(testhelper.Context(t)) + packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor( + promconfig.DefaultConfig().GRPCLatencyBuckets, + ) + + packObjectsConcurrencyMonitor.Queued(ctx, "1234", 5) + packObjectsConcurrencyMonitor.Enter(ctx, time.Second) + + expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds) +# TYPE gitaly_pack_objects_acquiring_seconds histogram +gitaly_pack_objects_acquiring_seconds_bucket{le="0.001"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.005"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.025"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.1"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.5"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="1"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="10"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="30"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="60"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="300"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="1500"} 1 +gitaly_pack_objects_acquiring_seconds_bucket{le="+Inf"} 1 +gitaly_pack_objects_acquiring_seconds_sum 1 +gitaly_pack_objects_acquiring_seconds_count 1 +# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls +# TYPE gitaly_pack_objects_in_progress gauge +gitaly_pack_objects_in_progress 1 +# HELP gitaly_pack_objects_queued Gauge of number of queued calls +# TYPE gitaly_pack_objects_queued gauge +gitaly_pack_objects_queued 1 + +` + require.NoError(t, testutil.CollectAndCompare( + packObjectsConcurrencyMonitor, + bytes.NewBufferString(expectedMetrics), + "gitaly_pack_objects_acquiring_seconds", + "gitaly_pack_objects_in_progress", + "gitaly_pack_objects_queued", + "gitaly_pack_objects_dropped_total", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePackObjects, + "limit.limiting_key": "1234", + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + }, stats.Fields()) + + // After the request exists, in_progress and queued gauge decrease + packObjectsConcurrencyMonitor.Exit(ctx) + packObjectsConcurrencyMonitor.Dequeued(ctx) + expectedMetrics = `# HELP acquiring_seconds seconds to acquire +# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls +# TYPE gitaly_pack_objects_in_progress gauge +gitaly_pack_objects_in_progress 0 +# HELP gitaly_pack_objects_queued Gauge of number of queued calls +# TYPE gitaly_pack_objects_queued gauge +gitaly_pack_objects_queued 0 + +` + require.NoError(t, testutil.CollectAndCompare( + packObjectsConcurrencyMonitor, + bytes.NewBufferString(expectedMetrics), + "gitaly_pack_objects_in_progress", + "gitaly_pack_objects_queued", + )) + }) + + t.Run("request is dropped after queueing", func(t *testing.T) { + ctx := InitLimitStats(testhelper.Context(t)) + packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor( + promconfig.DefaultConfig().GRPCLatencyBuckets, + ) + + packObjectsConcurrencyMonitor.Queued(ctx, "1234", 5) + packObjectsConcurrencyMonitor.Dropped(ctx, "1234", 5, time.Second, "load") + + expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds) +# TYPE gitaly_pack_objects_acquiring_seconds histogram +gitaly_pack_objects_acquiring_seconds_bucket{le="0.001"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.005"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.025"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.1"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.5"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="1"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="10"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="30"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="60"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="300"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="1500"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="+Inf"} 0 +gitaly_pack_objects_acquiring_seconds_sum 0 +gitaly_pack_objects_acquiring_seconds_count 0 +# HELP gitaly_pack_objects_dropped_total Number of requests dropped from the queue +# TYPE gitaly_pack_objects_dropped_total counter +gitaly_pack_objects_dropped_total{reason="load"} 1 +# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls +# TYPE gitaly_pack_objects_in_progress gauge +gitaly_pack_objects_in_progress 0 +# HELP gitaly_pack_objects_queued Gauge of number of queued calls +# TYPE gitaly_pack_objects_queued gauge +gitaly_pack_objects_queued 1 + +` + require.NoError(t, testutil.CollectAndCompare( + packObjectsConcurrencyMonitor, + bytes.NewBufferString(expectedMetrics), + "gitaly_pack_objects_acquiring_seconds", + "gitaly_pack_objects_in_progress", + "gitaly_pack_objects_queued", + "gitaly_pack_objects_dropped_total", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePackObjects, + "limit.limiting_key": "1234", + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + "limit.concurrency_dropped": "load", + }, stats.Fields()) + }) + + t.Run("request is dropped before queueing", func(t *testing.T) { + ctx := InitLimitStats(testhelper.Context(t)) + packObjectsConcurrencyMonitor := NewPackObjectsConcurrencyMonitor( + promconfig.DefaultConfig().GRPCLatencyBuckets, + ) + + packObjectsConcurrencyMonitor.Dropped(ctx, "1234", 5, time.Second, "load") + + expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds) +# TYPE gitaly_pack_objects_acquiring_seconds histogram +gitaly_pack_objects_acquiring_seconds_bucket{le="0.001"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.005"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.025"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.1"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="0.5"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="1"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="10"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="30"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="60"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="300"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="1500"} 0 +gitaly_pack_objects_acquiring_seconds_bucket{le="+Inf"} 0 +gitaly_pack_objects_acquiring_seconds_sum 0 +gitaly_pack_objects_acquiring_seconds_count 0 +# HELP gitaly_pack_objects_dropped_total Number of requests dropped from the queue +# TYPE gitaly_pack_objects_dropped_total counter +gitaly_pack_objects_dropped_total{reason="load"} 1 +# HELP gitaly_pack_objects_in_progress Gauge of number of concurrent in-progress calls +# TYPE gitaly_pack_objects_in_progress gauge +gitaly_pack_objects_in_progress 0 +# HELP gitaly_pack_objects_queued Gauge of number of queued calls +# TYPE gitaly_pack_objects_queued gauge +gitaly_pack_objects_queued 0 + +` + require.NoError(t, testutil.CollectAndCompare( + packObjectsConcurrencyMonitor, + bytes.NewBufferString(expectedMetrics), + "gitaly_pack_objects_acquiring_seconds", + "gitaly_pack_objects_in_progress", + "gitaly_pack_objects_queued", + "gitaly_pack_objects_dropped_total", + )) + + stats := limitStatsFromContext(ctx) + require.NotNil(t, stats) + require.Equal(t, logrus.Fields{ + "limit.limiting_type": TypePackObjects, + "limit.limiting_key": "1234", + "limit.concurrency_queue_ms": int64(1000), + "limit.concurrency_queue_length": 5, + "limit.concurrency_dropped": "load", + }, stats.Fields()) + }) +} diff --git a/internal/middleware/limithandler/rate_limiter.go b/internal/grpc/middleware/limithandler/rate_limiter.go index be09a1f77..be09a1f77 100644 --- a/internal/middleware/limithandler/rate_limiter.go +++ b/internal/grpc/middleware/limithandler/rate_limiter.go diff --git a/internal/middleware/limithandler/rate_limiter_test.go b/internal/grpc/middleware/limithandler/rate_limiter_test.go index 5e88b8f6b..5e88b8f6b 100644 --- a/internal/middleware/limithandler/rate_limiter_test.go +++ b/internal/grpc/middleware/limithandler/rate_limiter_test.go diff --git a/internal/middleware/limithandler/stats.go b/internal/grpc/middleware/limithandler/stats.go index 30926695a..30926695a 100644 --- a/internal/middleware/limithandler/stats.go +++ b/internal/grpc/middleware/limithandler/stats.go diff --git a/internal/middleware/limithandler/stats_interceptor_test.go b/internal/grpc/middleware/limithandler/stats_interceptor_test.go index 0c8ac400c..010e487c8 100644 --- a/internal/middleware/limithandler/stats_interceptor_test.go +++ b/internal/grpc/middleware/limithandler/stats_interceptor_test.go @@ -10,14 +10,14 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/ref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" diff --git a/internal/middleware/limithandler/stats_test.go b/internal/grpc/middleware/limithandler/stats_test.go index 53ccccfd5..53ccccfd5 100644 --- a/internal/middleware/limithandler/stats_test.go +++ b/internal/grpc/middleware/limithandler/stats_test.go diff --git a/internal/middleware/limithandler/testhelper_test.go b/internal/grpc/middleware/limithandler/testhelper_test.go index 137c47cce..137c47cce 100644 --- a/internal/middleware/limithandler/testhelper_test.go +++ b/internal/grpc/middleware/limithandler/testhelper_test.go diff --git a/internal/middleware/metadatahandler/metadatahandler.go b/internal/grpc/middleware/metadatahandler/metadatahandler.go index bae12151e..bae12151e 100644 --- a/internal/middleware/metadatahandler/metadatahandler.go +++ b/internal/grpc/middleware/metadatahandler/metadatahandler.go diff --git a/internal/middleware/metadatahandler/metadatahandler_test.go b/internal/grpc/middleware/metadatahandler/metadatahandler_test.go index 7f665574f..7f665574f 100644 --- a/internal/middleware/metadatahandler/metadatahandler_test.go +++ b/internal/grpc/middleware/metadatahandler/metadatahandler_test.go diff --git a/internal/middleware/panichandler/LICENSE b/internal/grpc/middleware/panichandler/LICENSE index b1bb8825b..b1bb8825b 100644 --- a/internal/middleware/panichandler/LICENSE +++ b/internal/grpc/middleware/panichandler/LICENSE diff --git a/internal/middleware/panichandler/README.md b/internal/grpc/middleware/panichandler/README.md index 03b32ec15..03b32ec15 100644 --- a/internal/middleware/panichandler/README.md +++ b/internal/grpc/middleware/panichandler/README.md diff --git a/internal/middleware/panichandler/panic_handler.go b/internal/grpc/middleware/panichandler/panic_handler.go index e91269d5f..e91269d5f 100644 --- a/internal/middleware/panichandler/panic_handler.go +++ b/internal/grpc/middleware/panichandler/panic_handler.go diff --git a/internal/middleware/sentryhandler/sentryhandler.go b/internal/grpc/middleware/sentryhandler/sentryhandler.go index a6645c552..a6645c552 100644 --- a/internal/middleware/sentryhandler/sentryhandler.go +++ b/internal/grpc/middleware/sentryhandler/sentryhandler.go diff --git a/internal/middleware/sentryhandler/sentryhandler_test.go b/internal/grpc/middleware/sentryhandler/sentryhandler_test.go index 7a98e22ea..7a98e22ea 100644 --- a/internal/middleware/sentryhandler/sentryhandler_test.go +++ b/internal/grpc/middleware/sentryhandler/sentryhandler_test.go diff --git a/internal/middleware/statushandler/statushandler.go b/internal/grpc/middleware/statushandler/statushandler.go index 07b736d97..07b736d97 100644 --- a/internal/middleware/statushandler/statushandler.go +++ b/internal/grpc/middleware/statushandler/statushandler.go diff --git a/internal/middleware/statushandler/statushandler_test.go b/internal/grpc/middleware/statushandler/statushandler_test.go index 077806ba5..077806ba5 100644 --- a/internal/middleware/statushandler/statushandler_test.go +++ b/internal/grpc/middleware/statushandler/statushandler_test.go diff --git a/internal/praefect/grpc-proxy/LICENSE.txt b/internal/grpc/proxy/LICENSE.txt index cbfdef8c5..cbfdef8c5 100644 --- a/internal/praefect/grpc-proxy/LICENSE.txt +++ b/internal/grpc/proxy/LICENSE.txt diff --git a/internal/praefect/grpc-proxy/proxy/codec.go b/internal/grpc/proxy/codec.go index 65501a418..65501a418 100644 --- a/internal/praefect/grpc-proxy/proxy/codec.go +++ b/internal/grpc/proxy/codec.go diff --git a/internal/praefect/grpc-proxy/proxy/codec_test.go b/internal/grpc/proxy/codec_test.go index cfa6a05b7..cfa6a05b7 100644 --- a/internal/praefect/grpc-proxy/proxy/codec_test.go +++ b/internal/grpc/proxy/codec_test.go diff --git a/internal/praefect/grpc-proxy/proxy/director.go b/internal/grpc/proxy/director.go index b7aa280ab..b7aa280ab 100644 --- a/internal/praefect/grpc-proxy/proxy/director.go +++ b/internal/grpc/proxy/director.go diff --git a/internal/praefect/grpc-proxy/proxy/doc.go b/internal/grpc/proxy/doc.go index 01328f332..01328f332 100644 --- a/internal/praefect/grpc-proxy/proxy/doc.go +++ b/internal/grpc/proxy/doc.go diff --git a/internal/praefect/grpc-proxy/proxy/handler.go b/internal/grpc/proxy/handler.go index 25dbed58e..eccaf00f5 100644 --- a/internal/praefect/grpc-proxy/proxy/handler.go +++ b/internal/grpc/proxy/handler.go @@ -9,7 +9,7 @@ import ( "fmt" "io" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/sentryhandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/sentryhandler" "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/codes" diff --git a/internal/praefect/grpc-proxy/proxy/handler_ext_test.go b/internal/grpc/proxy/handler_ext_test.go index ec7a8685a..bfa1236c5 100644 --- a/internal/praefect/grpc-proxy/proxy/handler_ext_test.go +++ b/internal/grpc/proxy/handler_ext_test.go @@ -19,8 +19,8 @@ import ( "github.com/getsentry/sentry-go" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "google.golang.org/grpc" diff --git a/internal/praefect/grpc-proxy/proxy/handler_test.go b/internal/grpc/proxy/handler_test.go index b88501eb2..b88501eb2 100644 --- a/internal/praefect/grpc-proxy/proxy/handler_test.go +++ b/internal/grpc/proxy/handler_test.go diff --git a/internal/praefect/grpc-proxy/proxy/peeker.go b/internal/grpc/proxy/peeker.go index 6825d7f37..6825d7f37 100644 --- a/internal/praefect/grpc-proxy/proxy/peeker.go +++ b/internal/grpc/proxy/peeker.go diff --git a/internal/praefect/grpc-proxy/proxy/peeker_test.go b/internal/grpc/proxy/peeker_test.go index a38629db1..aea17bba2 100644 --- a/internal/praefect/grpc-proxy/proxy/peeker_test.go +++ b/internal/grpc/proxy/peeker_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/protobuf/proto" diff --git a/internal/praefect/grpc-proxy/proxy/testhelper_test.go b/internal/grpc/proxy/testhelper_test.go index 929e8e4f3..5efcfa265 100644 --- a/internal/praefect/grpc-proxy/proxy/testhelper_test.go +++ b/internal/grpc/proxy/testhelper_test.go @@ -7,9 +7,9 @@ import ( grpcmwtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/sentryhandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/fieldextractors" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/sentryhandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/sidechannel/conn.go b/internal/grpc/sidechannel/conn.go index f77bbcbe3..f77bbcbe3 100644 --- a/internal/sidechannel/conn.go +++ b/internal/grpc/sidechannel/conn.go diff --git a/internal/sidechannel/conn_test.go b/internal/grpc/sidechannel/conn_test.go index 257fd4273..257fd4273 100644 --- a/internal/sidechannel/conn_test.go +++ b/internal/grpc/sidechannel/conn_test.go diff --git a/internal/sidechannel/proxy.go b/internal/grpc/sidechannel/proxy.go index 30bebbae1..e7b750b1e 100644 --- a/internal/sidechannel/proxy.go +++ b/internal/grpc/sidechannel/proxy.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "google.golang.org/grpc" grpcMetadata "google.golang.org/grpc/metadata" ) diff --git a/internal/sidechannel/proxy_test.go b/internal/grpc/sidechannel/proxy_test.go index 8191d1b99..7ab65bb1f 100644 --- a/internal/sidechannel/proxy_test.go +++ b/internal/grpc/sidechannel/proxy_test.go @@ -9,9 +9,9 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc" diff --git a/internal/sidechannel/registry.go b/internal/grpc/sidechannel/registry.go index 3148cb05d..3148cb05d 100644 --- a/internal/sidechannel/registry.go +++ b/internal/grpc/sidechannel/registry.go diff --git a/internal/sidechannel/registry_test.go b/internal/grpc/sidechannel/registry_test.go index a0b55c2ac..a0b55c2ac 100644 --- a/internal/sidechannel/registry_test.go +++ b/internal/grpc/sidechannel/registry_test.go diff --git a/internal/sidechannel/sidechannel.go b/internal/grpc/sidechannel/sidechannel.go index 1eff50c99..1cc197b8b 100644 --- a/internal/sidechannel/sidechannel.go +++ b/internal/grpc/sidechannel/sidechannel.go @@ -10,9 +10,9 @@ import ( "time" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" "google.golang.org/grpc" "google.golang.org/grpc/credentials" diff --git a/internal/sidechannel/sidechannel_test.go b/internal/grpc/sidechannel/sidechannel_test.go index d274f8d8c..ddf6a640c 100644 --- a/internal/sidechannel/sidechannel_test.go +++ b/internal/grpc/sidechannel/sidechannel_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/log/log_test.go b/internal/log/log_test.go index dc927ae15..cd3063490 100644 --- a/internal/log/log_test.go +++ b/internal/log/log_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/grpcstats" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/grpcstats" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/interop/grpc_testing" diff --git a/internal/metadata/featureflag/ff_revlist_for_connectivity.go b/internal/metadata/featureflag/ff_revlist_for_connectivity.go deleted file mode 100644 index 09c667fb4..000000000 --- a/internal/metadata/featureflag/ff_revlist_for_connectivity.go +++ /dev/null @@ -1,10 +0,0 @@ -package featureflag - -// RevlistForConnectivity causes the connectivity check when removing alternates -// files to use git-rev-list instead of git-fsck -var RevlistForConnectivity = NewFeatureFlag( - "revlist_for_connectivity", - "v15.5.0", - "https://gitlab.com/gitlab-org/gitaly/-/issues/4489", - false, -) diff --git a/internal/middleware/limithandler/monitor_test.go b/internal/middleware/limithandler/monitor_test.go deleted file mode 100644 index 1c5eb9dbf..000000000 --- a/internal/middleware/limithandler/monitor_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package limithandler - -import ( - "bytes" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" - promconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/prometheus" - "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" -) - -func TestNewPerRPCPromMonitor(t *testing.T) { - system := "gitaly" - fullMethod := "fullMethod" - acquiringSecondsVec := prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "acquiring_seconds", - Help: "seconds to acquire", - Buckets: promconfig.DefaultConfig().GRPCLatencyBuckets, - }, - []string{"system", "grpc_service", "grpc_method"}, - ) - inProgressMetric := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "in_progress", - Help: "requests in progress", - }, - []string{"system", "grpc_service", "grpc_method"}, - ) - queuedMetric := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "queued", - Help: "number of queued requests", - }, - []string{"system", "grpc_service", "grpc_method"}, - ) - requestsDroppedMetric := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "dropped", - Help: "number of dropped requests", - }, - []string{ - "system", - "grpc_service", - "grpc_method", - "reason", - }, - ) - - rpcMonitor := newPerRPCPromMonitor( - system, - fullMethod, - queuedMetric, - inProgressMetric, - acquiringSecondsVec, - requestsDroppedMetric, - ) - - ctx := InitLimitStats(testhelper.Context(t)) - - rpcMonitor.Queued(ctx, fullMethod, 5) - rpcMonitor.Enter(ctx, time.Second) - rpcMonitor.Dropped(ctx, fullMethod, 5, "load") - - expectedMetrics := `# HELP acquiring_seconds seconds to acquire -# TYPE acquiring_seconds histogram -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.001"} 0 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.005"} 0 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.025"} 0 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.1"} 0 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="0.5"} 0 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="10"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="30"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="60"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="300"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="1500"} 1 -acquiring_seconds_bucket{grpc_method="unknown",grpc_service="unknown",system="gitaly",le="+Inf"} 1 -acquiring_seconds_sum{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 -acquiring_seconds_count{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 -# HELP dropped number of dropped requests -# TYPE dropped counter -dropped{grpc_method="unknown",grpc_service="unknown",reason="load",system="gitaly"} 1 -# HELP in_progress requests in progress -# TYPE in_progress gauge -in_progress{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 -# HELP queued number of queued requests -# TYPE queued gauge -queued{grpc_method="unknown",grpc_service="unknown",system="gitaly"} 1 - -` - require.NoError(t, testutil.CollectAndCompare( - rpcMonitor, - bytes.NewBufferString(expectedMetrics), - "in_progress", - "queued", - "dropped", - "acquiring_seconds", - )) - - stats := limitStatsFromContext(ctx) - require.NotNil(t, stats) - require.Equal(t, logrus.Fields{ - "limit.limiting_type": TypePerRPC, - "limit.limiting_key": fullMethod, - "limit.concurrency_queue_ms": int64(1000), - "limit.concurrency_queue_length": 5, - "limit.concurrency_dropped": "load", - }, stats.Fields()) -} - -func TestNewPackObjectsConcurrencyMonitor(t *testing.T) { - ctx := InitLimitStats(testhelper.Context(t)) - - m := NewPackObjectsConcurrencyMonitor( - promconfig.DefaultConfig().GRPCLatencyBuckets, - ) - - m.Queued(ctx, "1234", 5) - m.Enter(ctx, time.Second) - m.Dropped(ctx, "1234", 5, "load") - - expectedMetrics := `# HELP gitaly_pack_objects_acquiring_seconds Histogram of time calls are rate limited (in seconds) -# TYPE gitaly_pack_objects_acquiring_seconds histogram -gitaly_pack_objects_acquiring_seconds_bucket{le="0.001"} 0 -gitaly_pack_objects_acquiring_seconds_bucket{le="0.005"} 0 -gitaly_pack_objects_acquiring_seconds_bucket{le="0.025"} 0 -gitaly_pack_objects_acquiring_seconds_bucket{le="0.1"} 0 -gitaly_pack_objects_acquiring_seconds_bucket{le="0.5"} 0 -gitaly_pack_objects_acquiring_seconds_bucket{le="1"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="10"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="30"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="60"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="300"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="1500"} 1 -gitaly_pack_objects_acquiring_seconds_bucket{le="+Inf"} 1 -gitaly_pack_objects_acquiring_seconds_sum 1 -gitaly_pack_objects_acquiring_seconds_count 1 -# HELP gitaly_pack_objects_dropped_total Number of requests dropped from the queue -# TYPE gitaly_pack_objects_dropped_total counter -gitaly_pack_objects_dropped_total{reason="load"} 1 -# HELP gitaly_pack_objects_queued Gauge of number of queued calls -# TYPE gitaly_pack_objects_queued gauge -gitaly_pack_objects_queued 1 - -` - require.NoError(t, testutil.CollectAndCompare( - m, - bytes.NewBufferString(expectedMetrics), - "gitaly_pack_objects_acquiring_seconds", - "gitaly_pack_objecfts_in_progress", - "gitaly_pack_objects_queued", - "gitaly_pack_objects_dropped_total", - )) - - stats := limitStatsFromContext(ctx) - require.NotNil(t, stats) - require.Equal(t, logrus.Fields{ - "limit.limiting_type": TypePackObjects, - "limit.limiting_key": "1234", - "limit.concurrency_queue_ms": int64(1000), - "limit.concurrency_queue_length": 5, - "limit.concurrency_dropped": "load", - }, stats.Fields()) -} diff --git a/internal/praefect/coordinator.go b/internal/praefect/coordinator.go index 2431fa058..24eb35825 100644 --- a/internal/praefect/coordinator.go +++ b/internal/praefect/coordinator.go @@ -11,14 +11,14 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" gitalyerrors "gitlab.com/gitlab-org/gitaly/v16/internal/errors" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/metrics" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" diff --git a/internal/praefect/coordinator_test.go b/internal/praefect/coordinator_test.go index 946ed5161..9106176c9 100644 --- a/internal/praefect/coordinator_test.go +++ b/internal/praefect/coordinator_test.go @@ -23,16 +23,16 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/client" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/datastructure" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" - gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" + gitaly_metadata "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" diff --git a/internal/praefect/grpc-proxy/README.md b/internal/praefect/grpc-proxy/README.md deleted file mode 100644 index b5f1a3168..000000000 --- a/internal/praefect/grpc-proxy/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# gRPC Proxy - -[](https://travis-ci.org/mwitkow/grpc-proxy) -[](https://goreportcard.com/report/github.com/mwitkow/grpc-proxy) -[](https://godoc.org/github.com/mwitkow/grpc-proxy) -[](LICENSE) - -[gRPC Go](https://github.com/grpc/grpc-go) Proxy server - -## Project Goal - -Build a transparent reverse proxy for gRPC targets that will make it easy to expose gRPC services -over the internet. This includes: - * no needed knowledge of the semantics of requests exchanged in the call (independent rollouts) - * easy, declarative definition of backends and their mappings to frontends - * simple round-robin load balancing of inbound requests from a single connection to multiple backends - -The project now exists as a **proof of concept**, with the key piece being the `proxy` package that -is a generic gRPC reverse proxy handler. - -## Proxy Handler - -The package [`proxy`](proxy/) contains a generic gRPC reverse proxy handler that allows a gRPC server to -not know about registered handlers or their data types. Please consult the docs, here's an example usage. - -Defining a `StreamDirector` that decides where (if at all) to send the request -```go -director = func(ctx context.Context, fullMethodName string) (*grpc.ClientConn, error) { - // Make sure we never forward internal services. - if strings.HasPrefix(fullMethodName, "/com.example.internal.") { - return nil, grpc.Errorf(codes.Unimplemented, "Unknown method") - } - md, ok := metadata.FromContext(ctx) - if ok { - // Decide on which backend to dial - if val, exists := md[":authority"]; exists && val[0] == "staging.api.example.com" { - // Make sure we use DialContext so the dialing can be cancelled/time out together with the context. - return grpc.DialContext(ctx, "api-service.staging.svc.local", grpc.WithCodec(proxy.NewCodec())) - } else if val, exists := md[":authority"]; exists && val[0] == "api.example.com" { - return grpc.DialContext(ctx, "api-service.prod.svc.local", grpc.WithCodec(proxy.NewCodec())) - } - } - return nil, grpc.Errorf(codes.Unimplemented, "Unknown method") -} -``` -Then you need to register it with a `grpc.Server`. The server may have other handlers that will be served -locally: - -```go -server := grpc.NewServer( - grpc.CustomCodec(proxy.NewCodec()), - grpc.UnknownServiceHandler(proxy.TransparentHandler(director))) -pb_test.RegisterTestServiceServer(server, &testImpl{}) -``` - -## License - -`grpc-proxy` is released under the Apache 2.0 license. See [LICENSE.txt](LICENSE.txt). - diff --git a/internal/praefect/grpc-proxy/checkup.sh b/internal/praefect/grpc-proxy/checkup.sh deleted file mode 100755 index 37e9aac6e..000000000 --- a/internal/praefect/grpc-proxy/checkup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Script that checks up code (govet). - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -function print_real_go_files { - grep --files-without-match 'DO NOT EDIT!' $(find . -iname '*.go') -} - -function govet_all { - ret=0 - for i in $(print_real_go_files); do - output=$(go tool vet -all=true -tests=false ${i}) - ret=$(($ret | $?)) - echo -n ${output} - done; - return ${ret} -} - -govet_all -echo "returning $?"
\ No newline at end of file diff --git a/internal/praefect/grpc-proxy/fixup.sh b/internal/praefect/grpc-proxy/fixup.sh deleted file mode 100755 index 5b4a66ade..000000000 --- a/internal/praefect/grpc-proxy/fixup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# Script that checks the code for errors. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -function print_real_go_files { - grep --files-without-match 'DO NOT EDIT!' $(find . -iname '*.go') -} - -function generate_markdown { - echo "Generating markdown" - oldpwd=$(pwd) - for i in $(find . -iname 'doc.go'); do - dir=${i%/*} - echo "$dir" - cd ${dir} - ${GOPATH}/bin/godocdown -heading=Title -o DOC.md - ln -s DOC.md README.md 2> /dev/null # can fail - cd ${oldpwd} - done; -} - -function goimports_all { - echo "Running goimports" - goimports -l -w $(print_real_go_files) - return $? -} - -generate_markdown -goimports_all -echo "returning $?"
\ No newline at end of file diff --git a/internal/praefect/grpc-proxy/proxy/DOC.md b/internal/praefect/grpc-proxy/proxy/DOC.md deleted file mode 100644 index 55fc8c810..000000000 --- a/internal/praefect/grpc-proxy/proxy/DOC.md +++ /dev/null @@ -1,83 +0,0 @@ -# proxy --- - import "github.com/mwitkow/grpc-proxy/proxy" - -Package proxy provides a reverse proxy handler for gRPC. - -The implementation allows a `grpc.Server` to pass a received ServerStream to a -ClientStream without understanding the semantics of the messages exchanged. It -basically provides a transparent reverse-proxy. - -This package is intentionally generic, exposing a `StreamDirector` function that -allows users of this package to implement whatever logic of backend-picking, -dialing and service verification to perform. - -See examples on documented functions. - -## Usage - -#### func Codec - -```go -func NewCodec() Codec -``` -Codec returns a proxying Codec with the default protobuf codec as parent. - -See CodecWithParent. - -#### func CodecWithParent - -```go -func CodecWithParent(fallback grpc.Codec) Codec -``` -CodecWithParent returns a proxying grpc.Codec with a user provided codec as -parent. - -This codec is *crucial* to the functioning of the proxy. It allows the proxy -server to be oblivious to the schema of the forwarded messages. It basically -treats a gRPC message frame as raw bytes. However, if the server handler, or the -client caller are not proxy-internal functions it will fall back to trying to -decode the message using a fallback codec. - -#### func RegisterService - -```go -func RegisterService(server *grpc.Server, director StreamDirector, serviceName string, methodNames ...string) -``` -RegisterService sets up a proxy handler for a particular gRPC service and -method. The behaviour is the same as if you were registering a handler method, -e.g. from a codegenerated pb.go file. - -This can *only* be used if the `server` also uses grpcproxy.CodecForServer() -ServerOption. - -#### func TransparentHandler - -```go -func TransparentHandler(director StreamDirector) grpc.StreamHandler -``` -TransparentHandler returns a handler that attempts to proxy all requests that -are not registered in the server. The indented use here is as a transparent -proxy, where the server doesn't know about the services implemented by the -backends. It should be used as a `grpc.UnknownServiceHandler`. - -This can *only* be used if the `server` also uses grpcproxy.CodecForServer() -ServerOption. - -#### type StreamDirector - -```go -type StreamDirector func(ctx context.Context, fullMethodName string) (*grpc.ClientConn, error) -``` - -StreamDirector returns a gRPC ClientConn to be used to forward the call to. - -The presence of the `Context` allows for rich filtering, e.g. based on Metadata -(headers). If no handling is meant to be done, a `codes.NotImplemented` gRPC -error should be returned. - -It is worth noting that the StreamDirector will be fired *after* all server-side -stream interceptors are invoked. So decisions around authorization, monitoring -etc. are better to be handled there. - -See the rather rich example. diff --git a/internal/praefect/grpc-proxy/proxy/README.md b/internal/praefect/grpc-proxy/proxy/README.md deleted file mode 120000 index 71bfc07c9..000000000 --- a/internal/praefect/grpc-proxy/proxy/README.md +++ /dev/null @@ -1 +0,0 @@ -DOC.md
\ No newline at end of file diff --git a/internal/praefect/grpc-proxy/proxy/examples_test.go b/internal/praefect/grpc-proxy/proxy/examples_test.go deleted file mode 100644 index 7de70230f..000000000 --- a/internal/praefect/grpc-proxy/proxy/examples_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Michal Witkowski. All Rights Reserved. -// See LICENSE for licensing terms. - -package proxy_test - -import ( - "context" - "strings" - - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - grpc_metadata "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" -) - -var director proxy.StreamDirector - -func ExampleRegisterService() { - // A gRPC server with the proxying codec enabled. - server := grpc.NewServer(grpc.ForceServerCodec(proxy.NewCodec())) - // Register a TestService with 4 of its methods explicitly. - proxy.RegisterService(server, director, - "mwitkow.testproto.TestService", - "PingEmpty", "Ping", "PingError", "PingList") -} - -func ExampleTransparentHandler() { - grpc.NewServer( - grpc.ForceServerCodec(proxy.NewCodec()), - grpc.UnknownServiceHandler(proxy.TransparentHandler(director))) -} - -// Provide sa simple example of a director that shields internal services and dials a staging or production backend. -// This is a *very naive* implementation that creates a new connection on every request. Consider using pooling. -func ExampleStreamDirector() { - director = func(ctx context.Context, fullMethodName string, _ proxy.StreamPeeker) (*proxy.StreamParameters, error) { - // Make sure we never forward internal services. - if strings.HasPrefix(fullMethodName, "/com.example.internal.") { - return nil, status.Errorf(codes.Unimplemented, "Unknown method") - } - md, ok := grpc_metadata.FromIncomingContext(ctx) - if ok { - // Decide on which backend to dial - if val, exists := md[":authority"]; exists && val[0] == "staging.api.example.com" { - // Make sure we use DialContext so the dialing can be cancelled/time out together with the context. - conn, err := grpc.DialContext(ctx, "api-service.staging.svc.local", grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.NewCodec()))) - return proxy.NewStreamParameters(proxy.Destination{ - Conn: conn, - Ctx: metadata.IncomingToOutgoing(ctx), - }, nil, nil, nil), err - } else if val, exists := md[":authority"]; exists && val[0] == "api.example.com" { - conn, err := grpc.DialContext(ctx, "api-service.prod.svc.local", grpc.WithDefaultCallOptions(grpc.ForceCodec(proxy.NewCodec()))) - return proxy.NewStreamParameters(proxy.Destination{ - Conn: conn, - Ctx: metadata.IncomingToOutgoing(ctx), - }, nil, nil, nil), err - } - } - return nil, status.Errorf(codes.Unimplemented, "Unknown method") - } -} diff --git a/internal/praefect/info_service_test.go b/internal/praefect/info_service_test.go index 334b294ab..e1f989149 100644 --- a/internal/praefect/info_service_test.go +++ b/internal/praefect/info_service_test.go @@ -7,18 +7,18 @@ import ( "testing" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" diff --git a/internal/praefect/middleware/errorhandler_test.go b/internal/praefect/middleware/errorhandler_test.go index da157cca1..18e689ecf 100644 --- a/internal/praefect/middleware/errorhandler_test.go +++ b/internal/praefect/middleware/errorhandler_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes/tracker" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" diff --git a/internal/praefect/node.go b/internal/praefect/node.go index a4b9ec688..0f5785918 100644 --- a/internal/praefect/node.go +++ b/internal/praefect/node.go @@ -5,11 +5,11 @@ import ( "fmt" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes/tracker" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "google.golang.org/grpc" "google.golang.org/grpc/health/grpc_health_v1" ) diff --git a/internal/praefect/nodes/manager.go b/internal/praefect/nodes/manager.go index 876d5bb5a..ab8fb5fe6 100644 --- a/internal/praefect/nodes/manager.go +++ b/internal/praefect/nodes/manager.go @@ -14,16 +14,16 @@ import ( gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/internal/datastructure" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/client" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/metrics" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/middleware" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes/tracker" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" prommetrics "gitlab.com/gitlab-org/gitaly/v16/internal/prometheus/metrics" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "google.golang.org/grpc" "google.golang.org/grpc/backoff" healthpb "google.golang.org/grpc/health/grpc_health_v1" diff --git a/internal/praefect/nodes/sql_elector_test.go b/internal/praefect/nodes/sql_elector_test.go index 047c0cb6d..65fa7271e 100644 --- a/internal/praefect/nodes/sql_elector_test.go +++ b/internal/praefect/nodes/sql_elector_test.go @@ -11,8 +11,8 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore/glsql" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" diff --git a/internal/praefect/reconciler/reconciler_test.go b/internal/praefect/reconciler/reconciler_test.go index 66ae97307..bf68e4898 100644 --- a/internal/praefect/reconciler/reconciler_test.go +++ b/internal/praefect/reconciler/reconciler_test.go @@ -9,8 +9,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" diff --git a/internal/praefect/remove_all_test.go b/internal/praefect/remove_all_test.go index ecec560d7..58686559b 100644 --- a/internal/praefect/remove_all_test.go +++ b/internal/praefect/remove_all_test.go @@ -9,9 +9,9 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/praefect/remove_repository_test.go b/internal/praefect/remove_repository_test.go index d5dc7d917..fb91f2efc 100644 --- a/internal/praefect/remove_repository_test.go +++ b/internal/praefect/remove_repository_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/praefect/replicator.go b/internal/praefect/replicator.go index bbbf69269..841871741 100644 --- a/internal/praefect/replicator.go +++ b/internal/praefect/replicator.go @@ -11,8 +11,8 @@ import ( "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" prommetrics "gitlab.com/gitlab-org/gitaly/v16/internal/prometheus/metrics" diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go index 288937e0a..3114e3d05 100644 --- a/internal/praefect/replicator_test.go +++ b/internal/praefect/replicator_test.go @@ -20,8 +20,8 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" diff --git a/internal/praefect/repocleaner/repository_test.go b/internal/praefect/repocleaner/repository_test.go index fef4cbf69..4cd415900 100644 --- a/internal/praefect/repocleaner/repository_test.go +++ b/internal/praefect/repocleaner/repository_test.go @@ -13,9 +13,9 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" diff --git a/internal/praefect/repository_exists_test.go b/internal/praefect/repository_exists_test.go index 9106d85a2..84f4df527 100644 --- a/internal/praefect/repository_exists_test.go +++ b/internal/praefect/repository_exists_test.go @@ -7,9 +7,9 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" diff --git a/internal/praefect/router_per_repository.go b/internal/praefect/router_per_repository.go index 59cdbcff1..63070ecfb 100644 --- a/internal/praefect/router_per_repository.go +++ b/internal/praefect/router_per_repository.go @@ -5,8 +5,8 @@ import ( "errors" "fmt" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/praefectutil" diff --git a/internal/praefect/router_per_repository_test.go b/internal/praefect/router_per_repository_test.go index b6396c9b0..e4c032aa9 100644 --- a/internal/praefect/router_per_repository_test.go +++ b/internal/praefect/router_per_repository_test.go @@ -7,8 +7,8 @@ import ( "testing" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/commonerr" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" diff --git a/internal/praefect/server.go b/internal/praefect/server.go index 3be311a8c..0b5696a97 100644 --- a/internal/praefect/server.go +++ b/internal/praefect/server.go @@ -11,18 +11,19 @@ import ( grpcmwtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/server/auth" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/metadatahandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/panichandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/sentryhandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/statushandler" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/fieldextractors" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/log" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/metadatahandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/panichandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/sentryhandler" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/statushandler" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/middleware" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service" @@ -30,7 +31,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/server" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" grpccorrelation "gitlab.com/gitlab-org/labkit/correlation/grpc" grpctracing "gitlab.com/gitlab-org/labkit/tracing/grpc" diff --git a/internal/praefect/server_factory.go b/internal/praefect/server_factory.go index 55ca84de7..637be14ff 100644 --- a/internal/praefect/server_factory.go +++ b/internal/praefect/server_factory.go @@ -7,9 +7,9 @@ import ( "sync" "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service" diff --git a/internal/praefect/server_factory_test.go b/internal/praefect/server_factory_test.go index c0614ae67..2a5641d9b 100644 --- a/internal/praefect/server_factory_test.go +++ b/internal/praefect/server_factory_test.go @@ -16,20 +16,20 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/bootstrap/starter" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/promtest" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" diff --git a/internal/praefect/server_test.go b/internal/praefect/server_test.go index 2ff7e26f2..b9eb1544a 100644 --- a/internal/praefect/server_test.go +++ b/internal/praefect/server_test.go @@ -19,17 +19,17 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/datastructure" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" - "gitlab.com/gitlab-org/gitaly/v16/internal/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/listenmux" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes/tracker" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" diff --git a/internal/praefect/service/server/info.go b/internal/praefect/service/server/info.go index 753b29ecf..9481ecb8d 100644 --- a/internal/praefect/service/server/info.go +++ b/internal/praefect/service/server/info.go @@ -6,7 +6,7 @@ import ( "github.com/google/uuid" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/metadata" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" "google.golang.org/grpc" ) diff --git a/internal/praefect/testserver.go b/internal/praefect/testserver.go index ace4e9f36..ddfd7bb90 100644 --- a/internal/praefect/testserver.go +++ b/internal/praefect/testserver.go @@ -12,10 +12,10 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/client" gitalycfgauth "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config/auth" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/server/auth" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" - "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/grpc-proxy/proxy" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/nodes" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service" diff --git a/internal/praefect/verifier_test.go b/internal/praefect/verifier_test.go index eaed7d039..f7938c34a 100644 --- a/internal/praefect/verifier_test.go +++ b/internal/praefect/verifier_test.go @@ -13,12 +13,13 @@ import ( "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" gitalyconfig "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/datastore" @@ -27,7 +28,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/protoregistry" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/service/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/transactions" - "gitlab.com/gitlab-org/gitaly/v16/internal/sidechannel" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testdb" diff --git a/internal/testhelper/featureset.go b/internal/testhelper/featureset.go index f6bbb08fa..54ad33ba7 100644 --- a/internal/testhelper/featureset.go +++ b/internal/testhelper/featureset.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" ) // FeatureSet is a representation of a set of features that should be disabled. diff --git a/internal/testhelper/featureset_test.go b/internal/testhelper/featureset_test.go index 71a0dc7f7..956345adc 100644 --- a/internal/testhelper/featureset_test.go +++ b/internal/testhelper/featureset_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/require" - ff "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" + ff "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "google.golang.org/grpc/metadata" ) diff --git a/internal/testhelper/leakage.go b/internal/testhelper/leakage.go index c0bfcb4a2..5fa5a387b 100644 --- a/internal/testhelper/leakage.go +++ b/internal/testhelper/leakage.go @@ -29,7 +29,7 @@ func mustHaveNoGoroutines() { goleak.IgnoreTopFunction("gitlab.com/gitlab-org/labkit/log.listenForSignalHangup"), // The backchannel code is somehow stock on closing its connections. I have no clue // why that is, but we should investigate. - goleak.IgnoreTopFunction(PkgPath("internal/backchannel.clientHandshake.serve.func4")), + goleak.IgnoreTopFunction(PkgPath("internal/grpc/backchannel.clientHandshake.serve.func4")), ); err != nil { panic(fmt.Errorf("goroutines running: %w", err)) } diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index 642b4aea7..327288a56 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -28,8 +28,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v16/internal/featureflag" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/metadata/featureflag" "golang.org/x/exp/slices" ) diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index b9a62e363..0f7349d7c 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/require" gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth" "gitlab.com/gitlab-org/gitaly/v16/client" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/cache" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" @@ -29,8 +28,9 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/middleware/limithandler" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" - "gitlab.com/gitlab-org/gitaly/v16/internal/middleware/limithandler" praefectconfig "gitlab.com/gitlab-org/gitaly/v16/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/v16/internal/streamcache" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" diff --git a/internal/transaction/txinfo/transaction.go b/internal/transaction/txinfo/transaction.go index ae0a2ce75..3f21bbd92 100644 --- a/internal/transaction/txinfo/transaction.go +++ b/internal/transaction/txinfo/transaction.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" - "gitlab.com/gitlab-org/gitaly/v16/internal/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "google.golang.org/grpc/metadata" ) diff --git a/proto/go/gitalypb/internal.pb.go b/proto/go/gitalypb/internal.pb.go index 024bd48d4..abac28f7c 100644 --- a/proto/go/gitalypb/internal.pb.go +++ b/proto/go/gitalypb/internal.pb.go @@ -130,6 +130,165 @@ func (x *WalkReposResponse) GetModificationTime() *timestamppb.Timestamp { return nil } +// BackupReposRequest contains request parameters for the BackupRepos RPC. +type BackupReposRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // For each request stream there must be first a request with a header + // containing details about the backups to perform. + Header *BackupReposRequest_Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + // Repositories to have a backups triggered. + Repositories []*Repository `protobuf:"bytes,2,rep,name=repositories,proto3" json:"repositories,omitempty"` +} + +func (x *BackupReposRequest) Reset() { + *x = BackupReposRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackupReposRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupReposRequest) ProtoMessage() {} + +func (x *BackupReposRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupReposRequest.ProtoReflect.Descriptor instead. +func (*BackupReposRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +func (x *BackupReposRequest) GetHeader() *BackupReposRequest_Header { + if x != nil { + return x.Header + } + return nil +} + +func (x *BackupReposRequest) GetRepositories() []*Repository { + if x != nil { + return x.Repositories + } + return nil +} + +// BackupReposResponse contains response parameters for the BackupRepos RPC. +type BackupReposResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *BackupReposResponse) Reset() { + *x = BackupReposResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackupReposResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupReposResponse) ProtoMessage() {} + +func (x *BackupReposResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupReposResponse.ProtoReflect.Descriptor instead. +func (*BackupReposResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + +// Header contains information to create the backups and must be sent in the +// first message. +type BackupReposRequest_Header struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // BackupId determines which collective backup these repository backups will + // be part of. Must be sent in first message. + BackupId string `protobuf:"bytes,1,opt,name=backup_id,json=backupId,proto3" json:"backup_id,omitempty"` + // StorageUrl is the object-storage URL where the backups will be stored. + // See https://gocloud.dev/howto/blob/ and https://gocloud.dev/concepts/urls/ + StorageUrl string `protobuf:"bytes,2,opt,name=storage_url,json=storageUrl,proto3" json:"storage_url,omitempty"` +} + +func (x *BackupReposRequest_Header) Reset() { + *x = BackupReposRequest_Header{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackupReposRequest_Header) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupReposRequest_Header) ProtoMessage() {} + +func (x *BackupReposRequest_Header) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupReposRequest_Header.ProtoReflect.Descriptor instead. +func (*BackupReposRequest_Header) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2, 0} +} + +func (x *BackupReposRequest_Header) GetBackupId() string { + if x != nil { + return x.BackupId + } + return "" +} + +func (x *BackupReposRequest_Header) GetStorageUrl() string { + if x != nil { + return x.StorageUrl + } + return "" +} + var File_internal_proto protoreflect.FileDescriptor var file_internal_proto_rawDesc = []byte{ @@ -137,25 +296,46 @@ var file_internal_proto_rawDesc = []byte{ 0x12, 0x06, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0a, 0x6c, 0x69, 0x6e, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, - 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x04, 0x88, 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, - 0x11, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x32, 0x5e, 0x0a, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x57, 0x61, 0x6c, 0x6b, - 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, - 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x19, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, - 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, - 0x08, 0x02, 0x10, 0x02, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0c, 0x73, 0x68, 0x61, 0x72, 0x65, 0x64, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0x3b, 0x0a, 0x10, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, + 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x04, 0x88, + 0xc6, 0x2c, 0x01, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, + 0x22, 0x81, 0x01, 0x0a, 0x11, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x6c, 0x61, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, + 0x65, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x47, 0x0a, 0x11, 0x6d, + 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x10, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x69, 0x6d, 0x65, 0x22, 0xd5, 0x01, 0x0a, 0x12, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x39, 0x0a, 0x06, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x69, + 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x52, 0x06, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, + 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, + 0x42, 0x04, 0x98, 0xc6, 0x2c, 0x01, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, + 0x72, 0x69, 0x65, 0x73, 0x1a, 0x46, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x1b, + 0x0a, 0x09, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x22, 0x15, 0x0a, 0x13, + 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x32, 0xb0, 0x01, 0x0a, 0x0e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x47, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x12, 0x4c, 0x0a, 0x09, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, + 0x70, 0x6f, 0x73, 0x12, 0x18, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, + 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x57, 0x61, 0x6c, 0x6b, 0x52, 0x65, 0x70, 0x6f, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x02, + 0x10, 0x02, 0x30, 0x01, 0x12, 0x50, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, + 0x70, 0x6f, 0x73, 0x12, 0x1a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1b, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x70, 0x6f, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, + 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, @@ -174,21 +354,29 @@ func file_internal_proto_rawDescGZIP() []byte { return file_internal_proto_rawDescData } -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_internal_proto_goTypes = []interface{}{ - (*WalkReposRequest)(nil), // 0: gitaly.WalkReposRequest - (*WalkReposResponse)(nil), // 1: gitaly.WalkReposResponse - (*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp + (*WalkReposRequest)(nil), // 0: gitaly.WalkReposRequest + (*WalkReposResponse)(nil), // 1: gitaly.WalkReposResponse + (*BackupReposRequest)(nil), // 2: gitaly.BackupReposRequest + (*BackupReposResponse)(nil), // 3: gitaly.BackupReposResponse + (*BackupReposRequest_Header)(nil), // 4: gitaly.BackupReposRequest.Header + (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*Repository)(nil), // 6: gitaly.Repository } var file_internal_proto_depIdxs = []int32{ - 2, // 0: gitaly.WalkReposResponse.modification_time:type_name -> google.protobuf.Timestamp - 0, // 1: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest - 1, // 2: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 5, // 0: gitaly.WalkReposResponse.modification_time:type_name -> google.protobuf.Timestamp + 4, // 1: gitaly.BackupReposRequest.header:type_name -> gitaly.BackupReposRequest.Header + 6, // 2: gitaly.BackupReposRequest.repositories:type_name -> gitaly.Repository + 0, // 3: gitaly.InternalGitaly.WalkRepos:input_type -> gitaly.WalkReposRequest + 2, // 4: gitaly.InternalGitaly.BackupRepos:input_type -> gitaly.BackupReposRequest + 1, // 5: gitaly.InternalGitaly.WalkRepos:output_type -> gitaly.WalkReposResponse + 3, // 6: gitaly.InternalGitaly.BackupRepos:output_type -> gitaly.BackupReposResponse + 5, // [5:7] is the sub-list for method output_type + 3, // [3:5] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -197,6 +385,7 @@ func file_internal_proto_init() { return } file_lint_proto_init() + file_shared_proto_init() if !protoimpl.UnsafeEnabled { file_internal_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*WalkReposRequest); i { @@ -222,6 +411,42 @@ func file_internal_proto_init() { return nil } } + file_internal_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BackupReposRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BackupReposResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BackupReposRequest_Header); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -229,7 +454,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_internal_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/go/gitalypb/internal_grpc.pb.go b/proto/go/gitalypb/internal_grpc.pb.go index 3ff76c214..ca1784197 100644 --- a/proto/go/gitalypb/internal_grpc.pb.go +++ b/proto/go/gitalypb/internal_grpc.pb.go @@ -25,6 +25,9 @@ type InternalGitalyClient interface { // WalkRepos walks the storage and streams back all known git repos on the // requested storage WalkRepos(ctx context.Context, in *WalkReposRequest, opts ...grpc.CallOption) (InternalGitaly_WalkReposClient, error) + // BackupRepos triggers a backup for each repository in the stream. This RPC + // is intended to be used to coordinate backups within the gitaly cluster. + BackupRepos(ctx context.Context, opts ...grpc.CallOption) (InternalGitaly_BackupReposClient, error) } type internalGitalyClient struct { @@ -67,6 +70,40 @@ func (x *internalGitalyWalkReposClient) Recv() (*WalkReposResponse, error) { return m, nil } +func (c *internalGitalyClient) BackupRepos(ctx context.Context, opts ...grpc.CallOption) (InternalGitaly_BackupReposClient, error) { + stream, err := c.cc.NewStream(ctx, &InternalGitaly_ServiceDesc.Streams[1], "/gitaly.InternalGitaly/BackupRepos", opts...) + if err != nil { + return nil, err + } + x := &internalGitalyBackupReposClient{stream} + return x, nil +} + +type InternalGitaly_BackupReposClient interface { + Send(*BackupReposRequest) error + CloseAndRecv() (*BackupReposResponse, error) + grpc.ClientStream +} + +type internalGitalyBackupReposClient struct { + grpc.ClientStream +} + +func (x *internalGitalyBackupReposClient) Send(m *BackupReposRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *internalGitalyBackupReposClient) CloseAndRecv() (*BackupReposResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(BackupReposResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // InternalGitalyServer is the server API for InternalGitaly service. // All implementations must embed UnimplementedInternalGitalyServer // for forward compatibility @@ -74,6 +111,9 @@ type InternalGitalyServer interface { // WalkRepos walks the storage and streams back all known git repos on the // requested storage WalkRepos(*WalkReposRequest, InternalGitaly_WalkReposServer) error + // BackupRepos triggers a backup for each repository in the stream. This RPC + // is intended to be used to coordinate backups within the gitaly cluster. + BackupRepos(InternalGitaly_BackupReposServer) error mustEmbedUnimplementedInternalGitalyServer() } @@ -84,6 +124,9 @@ type UnimplementedInternalGitalyServer struct { func (UnimplementedInternalGitalyServer) WalkRepos(*WalkReposRequest, InternalGitaly_WalkReposServer) error { return status.Errorf(codes.Unimplemented, "method WalkRepos not implemented") } +func (UnimplementedInternalGitalyServer) BackupRepos(InternalGitaly_BackupReposServer) error { + return status.Errorf(codes.Unimplemented, "method BackupRepos not implemented") +} func (UnimplementedInternalGitalyServer) mustEmbedUnimplementedInternalGitalyServer() {} // UnsafeInternalGitalyServer may be embedded to opt out of forward compatibility for this service. @@ -118,6 +161,32 @@ func (x *internalGitalyWalkReposServer) Send(m *WalkReposResponse) error { return x.ServerStream.SendMsg(m) } +func _InternalGitaly_BackupRepos_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(InternalGitalyServer).BackupRepos(&internalGitalyBackupReposServer{stream}) +} + +type InternalGitaly_BackupReposServer interface { + SendAndClose(*BackupReposResponse) error + Recv() (*BackupReposRequest, error) + grpc.ServerStream +} + +type internalGitalyBackupReposServer struct { + grpc.ServerStream +} + +func (x *internalGitalyBackupReposServer) SendAndClose(m *BackupReposResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *internalGitalyBackupReposServer) Recv() (*BackupReposRequest, error) { + m := new(BackupReposRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // InternalGitaly_ServiceDesc is the grpc.ServiceDesc for InternalGitaly service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -131,6 +200,11 @@ var InternalGitaly_ServiceDesc = grpc.ServiceDesc{ Handler: _InternalGitaly_WalkRepos_Handler, ServerStreams: true, }, + { + StreamName: "BackupRepos", + Handler: _InternalGitaly_BackupRepos_Handler, + ClientStreams: true, + }, }, Metadata: "internal.proto", } diff --git a/proto/go/gitalypb/testproto/error_metadata.pb.go b/proto/go/gitalypb/testproto/error_metadata.pb.go index 74caf99b1..b45c2ccfd 100644 --- a/proto/go/gitalypb/testproto/error_metadata.pb.go +++ b/proto/go/gitalypb/testproto/error_metadata.pb.go @@ -89,10 +89,11 @@ var file_testproto_error_metadata_proto_rawDesc = []byte{ 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, + 0x61, 0x6c, 0x75, 0x65, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, - 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, - 0x65, 0x73, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, + 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/go/gitalypb/testproto/invalid.pb.go b/proto/go/gitalypb/testproto/invalid.pb.go index 299b75ce7..ec3de26a4 100644 --- a/proto/go/gitalypb/testproto/invalid.pb.go +++ b/proto/go/gitalypb/testproto/invalid.pb.go @@ -976,10 +976,11 @@ var file_testproto_invalid_proto_rawDesc = []byte{ 0x52, 0x65, 0x70, 0x6f, 0x1a, 0x20, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x08, 0xfa, 0x97, 0x28, 0x04, 0x08, 0x03, 0x10, 0x02, - 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, + 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, - 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, + 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/go/gitalypb/testproto/valid.pb.go b/proto/go/gitalypb/testproto/valid.pb.go index 60d0ec725..50eb1021a 100644 --- a/proto/go/gitalypb/testproto/valid.pb.go +++ b/proto/go/gitalypb/testproto/valid.pb.go @@ -659,10 +659,11 @@ var file_testproto_valid_proto_rawDesc = []byte{ 0x6e, 0x6e, 0x65, 0x72, 0x4e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, - 0x02, 0x08, 0x03, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, + 0x02, 0x08, 0x03, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x74, 0x65, - 0x73, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, + 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/internal.proto b/proto/internal.proto index 44f18d609..1e3f40882 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -4,6 +4,7 @@ package gitaly; import "google/protobuf/timestamp.proto"; import "lint.proto"; +import "shared.proto"; option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"; @@ -18,6 +19,14 @@ service InternalGitaly { scope_level: STORAGE }; } + + // BackupRepos triggers a backup for each repository in the stream. This RPC + // is intended to be used to coordinate backups within the gitaly cluster. + rpc BackupRepos (stream BackupReposRequest) returns (BackupReposResponse) { + option (op_type) = { + op: ACCESSOR + }; + } } // This comment is left unintentionally blank. @@ -35,3 +44,29 @@ message WalkReposResponse { // modified. google.protobuf.Timestamp modification_time = 2; } + +// BackupReposRequest contains request parameters for the BackupRepos RPC. +message BackupReposRequest { + // Header contains information to create the backups and must be sent in the + // first message. + message Header { + // BackupId determines which collective backup these repository backups will + // be part of. Must be sent in first message. + string backup_id = 1; + + // StorageUrl is the object-storage URL where the backups will be stored. + // See https://gocloud.dev/howto/blob/ and https://gocloud.dev/concepts/urls/ + string storage_url = 2; + } + + // For each request stream there must be first a request with a header + // containing details about the backups to perform. + Header header = 1; + + // Repositories to have a backups triggered. + repeated Repository repositories = 2 [(target_repository)=true]; +} + +// BackupReposResponse contains response parameters for the BackupRepos RPC. +message BackupReposResponse { +} diff --git a/proto/testproto/error_metadata.proto b/proto/testproto/error_metadata.proto index d0ddcd86f..325ccc8ca 100644 --- a/proto/testproto/error_metadata.proto +++ b/proto/testproto/error_metadata.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package testproto; -option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/testproto"; +option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb/testproto"; // ErrorMetadata is a key-value metadata item that may be attached to errors. We only use this // infrastructure for testing purposes to assert that we add error metadata as expected that would diff --git a/proto/testproto/invalid.proto b/proto/testproto/invalid.proto index ffa0ca3d2..33db4d2fb 100644 --- a/proto/testproto/invalid.proto +++ b/proto/testproto/invalid.proto @@ -5,7 +5,7 @@ package testproto; import "lint.proto"; import "shared.proto"; -option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/testproto"; +option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb/testproto"; message InvalidMethodRequest { } diff --git a/proto/testproto/valid.proto b/proto/testproto/valid.proto index 2d009ded4..82a51e924 100644 --- a/proto/testproto/valid.proto +++ b/proto/testproto/valid.proto @@ -5,7 +5,7 @@ package testproto; import "lint.proto"; import "shared.proto"; -option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/testproto"; +option go_package = "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb/testproto"; message ValidRequest { gitaly.Repository destination = 1 [(gitaly.target_repository)=true]; |