diff options
author | Will Chandler <wchandler@gitlab.com> | 2023-07-27 16:05:27 +0300 |
---|---|---|
committer | Will Chandler <wchandler@gitlab.com> | 2023-07-28 20:39:08 +0300 |
commit | d6615e048843869771ac3da934e0ee54d289adb2 (patch) | |
tree | 3b411bbc993b5debacf8902be547c4174a904955 | |
parent | 9374616416692d67ccf342d060d7bfec1800fc01 (diff) |
counter: Wire up RepositoryCounter to repoutil
All repository creation and removal is done via `repoutil`, so
hooking `RepositoryCounter` in here will allow us to track changes
without dealing with the idiosyncracies of the many RPCs that may
create repositories.
Add a `RepositoryCounter` as a parameter to the `repoutil.Create` and
`repoutil.Remove` functions and wire it up to necessary services.
The exception to this is the `RemoveAll` RPC which deletes an entire
storage, we use the counter directly here.
43 files changed, 109 insertions, 16 deletions
diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 24843c534..cb02b31ca 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -12,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -153,6 +154,7 @@ func NewManagerLocal( gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, txManager transaction.Manager, + repoCounter *counter.RepositoryCounter, ) *Manager { return &Manager{ sink: sink, @@ -161,7 +163,7 @@ func NewManagerLocal( repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) { localRepo := localrepo.New(storageLocator, gitCmdFactory, catfileCache, repo) - return newLocalRepository(storageLocator, gitCmdFactory, txManager, localRepo), nil + return newLocalRepository(storageLocator, gitCmdFactory, txManager, repoCounter, localRepo), nil }, } } diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index b6c0e6ab8..30184d771 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -20,6 +20,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" @@ -69,6 +70,7 @@ func TestManager_Create(t *testing.T) { cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll) ctx := testhelper.Context(t) + repoCounter := counter.NewRepositoryCounter() for _, managerTC := range []struct { desc string @@ -98,7 +100,7 @@ func TestManager_Create(t *testing.T) { tb.Cleanup(catfileCache.Stop) txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, repoCounter) }, }, } { @@ -228,6 +230,7 @@ func TestManager_Create_incremental(t *testing.T) { cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll) ctx := testhelper.Context(t) + repoCounter := counter.NewRepositoryCounter() for _, managerTC := range []struct { desc string @@ -257,7 +260,7 @@ func TestManager_Create_incremental(t *testing.T) { tb.Cleanup(catfileCache.Stop) txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, repoCounter) }, }, } { @@ -374,6 +377,7 @@ func TestManager_Restore_latest(t *testing.T) { cfg := testcfg.Build(t) testcfg.BuildGitalyHooks(t, cfg) cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll) + repoCounter := counter.NewRepositoryCounter() for _, managerTC := range []struct { desc string @@ -403,7 +407,7 @@ func TestManager_Restore_latest(t *testing.T) { tb.Cleanup(catfileCache.Stop) txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, repoCounter) }, }, } { @@ -652,6 +656,7 @@ func TestManager_Restore_specific(t *testing.T) { cfg := testcfg.Build(t) testcfg.BuildGitalyHooks(t, cfg) cfg.SocketPath = testserver.RunGitalyServer(t, cfg, setup.RegisterAll) + repoCounter := counter.NewRepositoryCounter() for _, managerTC := range []struct { desc string @@ -681,7 +686,7 @@ func TestManager_Restore_specific(t *testing.T) { tb.Cleanup(catfileCache.Stop) txManager := transaction.NewTrackingManager() - return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager) + return backup.NewManagerLocal(sink, locator, storageLocator, gitCmdFactory, catfileCache, txManager, repoCounter) }, }, } { diff --git a/internal/backup/repository.go b/internal/backup/repository.go index 942e5b5dd..e93eaa00f 100644 --- a/internal/backup/repository.go +++ b/internal/backup/repository.go @@ -12,6 +12,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/chunk" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" @@ -267,6 +268,7 @@ type localRepository struct { locator storage.Locator gitCmdFactory git.CommandFactory txManager transaction.Manager + repoCounter *counter.RepositoryCounter repo *localrepo.Repo } @@ -274,12 +276,14 @@ func newLocalRepository( locator storage.Locator, gitCmdFactory git.CommandFactory, txManager transaction.Manager, + repoCounter *counter.RepositoryCounter, repo *localrepo.Repo, ) *localRepository { return &localRepository{ locator: locator, gitCmdFactory: gitCmdFactory, txManager: txManager, + repoCounter: repoCounter, repo: repo, } } @@ -348,7 +352,7 @@ func (r *localRepository) CreateBundle(ctx context.Context, out io.Writer, patte // Remove removes the repository. Does not return an error if the repository // cannot be found. func (r *localRepository) Remove(ctx context.Context) error { - err := repoutil.Remove(ctx, r.locator, r.txManager, r.repo) + err := repoutil.Remove(ctx, r.locator, r.txManager, r.repoCounter, r.repo) switch { case status.Code(err) == codes.NotFound: return nil @@ -365,6 +369,7 @@ func (r *localRepository) Create(ctx context.Context) error { r.locator, r.gitCmdFactory, r.txManager, + r.repoCounter, r.repo, func(repository *gitalypb.Repository) error { return nil }, ); err != nil { diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index d89d3d7ba..890195750 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -33,6 +33,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/setup" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" @@ -222,6 +223,9 @@ func run(cfg config.Cfg) error { locator := config.NewLocator(cfg) + repoCounter := counter.NewRepositoryCounter() + prometheus.MustRegister(repoCounter) + tempdir.StartCleaning(locator, cfg.Storages, time.Hour) prometheus.MustRegister(gitCmdFactory) @@ -356,6 +360,7 @@ func run(cfg config.Cfg) error { DiskCache: diskCache, PackObjectsCache: streamCache, PackObjectsLimiter: packObjectsLimiter, + RepositoryCounter: repoCounter, Git2goExecutor: git2goExecutor, UpdaterWithHooks: updaterWithHooks, HousekeepingManager: housekeepingManager, diff --git a/internal/git/localrepo/remote_extra_test.go b/internal/git/localrepo/remote_extra_test.go index 799d22157..0dcd2188e 100644 --- a/internal/git/localrepo/remote_extra_test.go +++ b/internal/git/localrepo/remote_extra_test.go @@ -52,6 +52,7 @@ func TestRepo_FetchInternal(t *testing.T) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, testserver.WithGitCommandFactory(protocolDetectingFactory)) diff --git a/internal/git/remoterepo/testhelper_test.go b/internal/git/remoterepo/testhelper_test.go index 32b95f78e..ebac4880c 100644 --- a/internal/git/remoterepo/testhelper_test.go +++ b/internal/git/remoterepo/testhelper_test.go @@ -34,6 +34,7 @@ func setupGitalyServer(t *testing.T) config.Cfg { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterCommitServiceServer(srv, commit.NewServer( deps.GetCfg(), diff --git a/internal/gitaly/repoutil/create.go b/internal/gitaly/repoutil/create.go index 650996a75..1b6101f98 100644 --- a/internal/gitaly/repoutil/create.go +++ b/internal/gitaly/repoutil/create.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" @@ -66,6 +67,7 @@ func Create( locator storage.Locator, gitCmdFactory git.CommandFactory, txManager transaction.Manager, + repoCounter *counter.RepositoryCounter, repository storage.Repository, seedRepository func(repository *gitalypb.Repository) error, options ...CreateOption, @@ -244,6 +246,8 @@ func Create( return structerr.NewFailedPrecondition("committing vote: %w", err) } + repoCounter.Increment(repository) + // We unlock the repository implicitly via the deferred `Close()` call. return nil } diff --git a/internal/gitaly/repoutil/create_test.go b/internal/gitaly/repoutil/create_test.go index 1728caac2..614782342 100644 --- a/internal/gitaly/repoutil/create_test.go +++ b/internal/gitaly/repoutil/create_test.go @@ -15,6 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" @@ -321,6 +322,7 @@ func TestCreate(t *testing.T) { // manager's data. ctx := ctx *txManager = transaction.MockManager{} + repoCounter := counter.NewRepositoryCounter() repo := &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, @@ -342,7 +344,7 @@ func TestCreate(t *testing.T) { } var tempRepo *gitalypb.Repository - require.Equal(t, tc.expectedErr, Create(ctx, locator, gitCmdFactory, txManager, repo, func(tr *gitalypb.Repository) error { + require.Equal(t, tc.expectedErr, Create(ctx, locator, gitCmdFactory, txManager, repoCounter, repo, func(tr *gitalypb.Repository) error { tempRepo = tr // The temporary repository must have been created in Gitaly's diff --git a/internal/gitaly/repoutil/remove.go b/internal/gitaly/repoutil/remove.go index 0ce2cd8c9..eb702d742 100644 --- a/internal/gitaly/repoutil/remove.go +++ b/internal/gitaly/repoutil/remove.go @@ -9,6 +9,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" @@ -22,9 +23,16 @@ func Remove( ctx context.Context, locator storage.Locator, txManager transaction.Manager, + repoCounter *counter.RepositoryCounter, repository storage.Repository, ) error { - return remove(ctx, locator, txManager, repository, os.RemoveAll) + if err := remove(ctx, locator, txManager, repository, os.RemoveAll); err != nil { + return err + } + + repoCounter.Decrement(repository) + + return nil } func remove( diff --git a/internal/gitaly/repoutil/remove_test.go b/internal/gitaly/repoutil/remove_test.go index 19a61ab82..7d59b78c2 100644 --- a/internal/gitaly/repoutil/remove_test.go +++ b/internal/gitaly/repoutil/remove_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" @@ -93,6 +94,7 @@ func TestRemove(t *testing.T) { cfg := testcfg.Build(t) locator := config.NewLocator(cfg) txManager := transaction.NewTrackingManager() + repoCounter := counter.NewRepositoryCounter() repo, repoPath := tc.createRepo(t, ctx, cfg) @@ -100,7 +102,7 @@ func TestRemove(t *testing.T) { require.DirExists(t, repoPath) } - err := Remove(ctx, locator, txManager, repo) + err := Remove(ctx, locator, txManager, repoCounter, repo) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr, err) diff --git a/internal/gitaly/service/blob/testhelper_test.go b/internal/gitaly/service/blob/testhelper_test.go index aeb43915f..bf76a55cc 100644 --- a/internal/gitaly/service/blob/testhelper_test.go +++ b/internal/gitaly/service/blob/testhelper_test.go @@ -40,6 +40,7 @@ func setup(tb testing.TB, ctx context.Context) (config.Cfg, gitalypb.BlobService deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }) cfg.SocketPath = addr diff --git a/internal/gitaly/service/cleanup/testhelper_test.go b/internal/gitaly/service/cleanup/testhelper_test.go index 3f3ff807c..074a9052a 100644 --- a/internal/gitaly/service/cleanup/testhelper_test.go +++ b/internal/gitaly/service/cleanup/testhelper_test.go @@ -58,6 +58,7 @@ func runCleanupServiceServer(t *testing.T, cfg config.Cfg) string { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }) } diff --git a/internal/gitaly/service/commit/testhelper_test.go b/internal/gitaly/service/commit/testhelper_test.go index bb6204b34..77756a318 100644 --- a/internal/gitaly/service/commit/testhelper_test.go +++ b/internal/gitaly/service/commit/testhelper_test.go @@ -71,6 +71,7 @@ func startTestServices(tb testing.TB, cfg config.Cfg, opts ...testserver.GitalyS deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, opts...) } diff --git a/internal/gitaly/service/conflicts/testhelper_test.go b/internal/gitaly/service/conflicts/testhelper_test.go index bec508e0c..35dfda869 100644 --- a/internal/gitaly/service/conflicts/testhelper_test.go +++ b/internal/gitaly/service/conflicts/testhelper_test.go @@ -60,6 +60,7 @@ func runConflictsServer(tb testing.TB, cfg config.Cfg, hookManager hook.Manager) deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterSSHServiceServer(srv, ssh.NewServer( deps.GetLocator(), diff --git a/internal/gitaly/service/dependencies.go b/internal/gitaly/service/dependencies.go index 7dd1f3dd8..664aeca96 100644 --- a/internal/gitaly/service/dependencies.go +++ b/internal/gitaly/service/dependencies.go @@ -12,6 +12,7 @@ import ( gitalyhook "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/hook/updateref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" @@ -36,6 +37,7 @@ type Dependencies struct { PackObjectsCache streamcache.Cache PackObjectsLimiter limiter.Limiter LimitHandler *limithandler.LimiterMiddleware + RepositoryCounter *counter.RepositoryCounter Git2goExecutor *git2go.Executor UpdaterWithHooks *updateref.UpdaterWithHooks HousekeepingManager housekeeping.Manager @@ -104,6 +106,11 @@ func (dc *Dependencies) GetLimitHandler() *limithandler.LimiterMiddleware { return dc.LimitHandler } +// GetRepositoryCounter returns the repository counter. +func (dc *Dependencies) GetRepositoryCounter() *counter.RepositoryCounter { + return dc.RepositoryCounter +} + // GetGit2goExecutor returns the git2go executor. func (dc *Dependencies) GetGit2goExecutor() *git2go.Executor { return dc.Git2goExecutor diff --git a/internal/gitaly/service/diff/testhelper_test.go b/internal/gitaly/service/diff/testhelper_test.go index c13fa72ef..d337142e8 100644 --- a/internal/gitaly/service/diff/testhelper_test.go +++ b/internal/gitaly/service/diff/testhelper_test.go @@ -51,6 +51,7 @@ func setupDiffServiceWithoutRepo(tb testing.TB, opt ...testserver.GitalyServerOp deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, opt...) cfg.SocketPath = addr diff --git a/internal/gitaly/service/hook/testhelper_test.go b/internal/gitaly/service/hook/testhelper_test.go index 9dbb2786f..51404398e 100644 --- a/internal/gitaly/service/hook/testhelper_test.go +++ b/internal/gitaly/service/hook/testhelper_test.go @@ -77,6 +77,7 @@ func runHooksServer(tb testing.TB, cfg config.Cfg, opts []serverOption, serverOp deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, serverOpts...) } diff --git a/internal/gitaly/service/objectpool/create.go b/internal/gitaly/service/objectpool/create.go index ccafefd1c..fd994f15a 100644 --- a/internal/gitaly/service/objectpool/create.go +++ b/internal/gitaly/service/objectpool/create.go @@ -29,7 +29,7 @@ func (s *server) CreateObjectPool(ctx context.Context, in *gitalypb.CreateObject return nil, errInvalidPoolDir } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, poolRepo, func(poolRepo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, poolRepo, func(poolRepo *gitalypb.Repository) error { if _, err := objectpool.Create( ctx, s.locator, diff --git a/internal/gitaly/service/objectpool/server.go b/internal/gitaly/service/objectpool/server.go index fa8c0e1a9..abdc8b870 100644 --- a/internal/gitaly/service/objectpool/server.go +++ b/internal/gitaly/service/objectpool/server.go @@ -6,6 +6,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -17,6 +18,7 @@ type server struct { catfileCache catfile.Cache txManager transaction.Manager housekeepingManager housekeeping.Manager + repositoryCounter *counter.RepositoryCounter } // NewServer creates a new instance of a gRPC repo server @@ -26,6 +28,7 @@ func NewServer( catfileCache catfile.Cache, txManager transaction.Manager, housekeepingManager housekeeping.Manager, + repositoryCounter *counter.RepositoryCounter, ) gitalypb.ObjectPoolServiceServer { return &server{ locator: locator, @@ -33,6 +36,7 @@ func NewServer( catfileCache: catfileCache, txManager: txManager, housekeepingManager: housekeepingManager, + repositoryCounter: repositoryCounter, } } diff --git a/internal/gitaly/service/objectpool/testhelper_test.go b/internal/gitaly/service/objectpool/testhelper_test.go index 7a95c21a6..836ae3332 100644 --- a/internal/gitaly/service/objectpool/testhelper_test.go +++ b/internal/gitaly/service/objectpool/testhelper_test.go @@ -69,6 +69,7 @@ func runObjectPoolServer(t *testing.T, cfg config.Cfg, locator storage.Locator, deps.GetCatfileCache(), deps.GetTxManager(), deps.GetHousekeepingManager(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer( deps.GetHookManager(), @@ -88,6 +89,7 @@ func runObjectPoolServer(t *testing.T, cfg config.Cfg, locator storage.Locator, deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, append(opts, testserver.WithLocator(locator), testserver.WithLogger(logger))...) } diff --git a/internal/gitaly/service/operations/testhelper_test.go b/internal/gitaly/service/operations/testhelper_test.go index b2896c899..46fd62b51 100644 --- a/internal/gitaly/service/operations/testhelper_test.go +++ b/internal/gitaly/service/operations/testhelper_test.go @@ -125,6 +125,7 @@ func runOperationServiceServer(tb testing.TB, cfg config.Cfg, options ...testser deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterRefServiceServer(srv, ref.NewServer( deps.GetLocator(), diff --git a/internal/gitaly/service/ref/delete_refs_test.go b/internal/gitaly/service/ref/delete_refs_test.go index 4d87a8c90..b5f085fb5 100644 --- a/internal/gitaly/service/ref/delete_refs_test.go +++ b/internal/gitaly/service/ref/delete_refs_test.go @@ -123,6 +123,7 @@ func TestDeleteRefs_transaction(t *testing.T) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer( deps.GetHookManager(), diff --git a/internal/gitaly/service/ref/testhelper_test.go b/internal/gitaly/service/ref/testhelper_test.go index fdb21ecbe..cf5615ddf 100644 --- a/internal/gitaly/service/ref/testhelper_test.go +++ b/internal/gitaly/service/ref/testhelper_test.go @@ -83,6 +83,7 @@ func runRefServiceServer(tb testing.TB, cfg config.Cfg) string { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }) } diff --git a/internal/gitaly/service/remote/testhelper_test.go b/internal/gitaly/service/remote/testhelper_test.go index da5681147..863d7f4d2 100644 --- a/internal/gitaly/service/remote/testhelper_test.go +++ b/internal/gitaly/service/remote/testhelper_test.go @@ -43,6 +43,7 @@ func setupRemoteService(t *testing.T, ctx context.Context, opts ...testserver.Gi deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, opts...) cfg.SocketPath = addr diff --git a/internal/gitaly/service/remote/update_remote_mirror_test.go b/internal/gitaly/service/remote/update_remote_mirror_test.go index f3007de17..d295fbe6e 100644 --- a/internal/gitaly/service/remote/update_remote_mirror_test.go +++ b/internal/gitaly/service/remote/update_remote_mirror_test.go @@ -591,6 +591,7 @@ func TestUpdateRemoteMirror(t *testing.T) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }) cfg.SocketPath = addr diff --git a/internal/gitaly/service/repository/backup_repository.go b/internal/gitaly/service/repository/backup_repository.go index fa2ad5baa..723bd3681 100644 --- a/internal/gitaly/service/repository/backup_repository.go +++ b/internal/gitaly/service/repository/backup_repository.go @@ -26,6 +26,7 @@ func (s *server) BackupRepository(ctx context.Context, in *gitalypb.BackupReposi s.gitCmdFactory, s.catfileCache, s.txManager, + s.repositoryCounter, ) err := manager.Create(ctx, &backup.CreateRequest{ diff --git a/internal/gitaly/service/repository/create_fork.go b/internal/gitaly/service/repository/create_fork.go index a5b8894db..28507cbd9 100644 --- a/internal/gitaly/service/repository/create_fork.go +++ b/internal/gitaly/service/repository/create_fork.go @@ -28,7 +28,7 @@ func (s *server) CreateFork(ctx context.Context, req *gitalypb.CreateForkRequest targetRepository := req.Repository sourceRepository := req.SourceRepository - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, targetRepository, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, targetRepository, func(repo *gitalypb.Repository) error { targetPath, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return err diff --git a/internal/gitaly/service/repository/create_repository.go b/internal/gitaly/service/repository/create_repository.go index a57970a3a..39195d3ea 100644 --- a/internal/gitaly/service/repository/create_repository.go +++ b/internal/gitaly/service/repository/create_repository.go @@ -26,6 +26,7 @@ func (s *server) CreateRepository(ctx context.Context, req *gitalypb.CreateRepos s.locator, s.gitCmdFactory, s.txManager, + s.repositoryCounter, repository, func(repo *gitalypb.Repository) error { // We do not want to seed the repository with any contents, so we just diff --git a/internal/gitaly/service/repository/create_repository_from_bundle.go b/internal/gitaly/service/repository/create_repository_from_bundle.go index 4e97ccb92..83376c947 100644 --- a/internal/gitaly/service/repository/create_repository_from_bundle.go +++ b/internal/gitaly/service/repository/create_repository_from_bundle.go @@ -36,7 +36,7 @@ func (s *server) CreateRepositoryFromBundle(stream gitalypb.RepositoryService_Cr return request.GetData(), err }) - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, repo, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repo, func(repo *gitalypb.Repository) error { opts := &localrepo.FetchBundleOpts{ UpdateHead: true, } diff --git a/internal/gitaly/service/repository/create_repository_from_snapshot.go b/internal/gitaly/service/repository/create_repository_from_snapshot.go index 460e83891..c5574fb18 100644 --- a/internal/gitaly/service/repository/create_repository_from_snapshot.go +++ b/internal/gitaly/service/repository/create_repository_from_snapshot.go @@ -126,7 +126,7 @@ func (s *server) CreateRepositoryFromSnapshot(ctx context.Context, in *gitalypb. return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, repository, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repository, func(repo *gitalypb.Repository) error { path, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return structerr.NewInternal("getting repo path: %w", err) diff --git a/internal/gitaly/service/repository/create_repository_from_url.go b/internal/gitaly/service/repository/create_repository_from_url.go index 4ed9daed5..be1c7c72d 100644 --- a/internal/gitaly/service/repository/create_repository_from_url.go +++ b/internal/gitaly/service/repository/create_repository_from_url.go @@ -81,7 +81,7 @@ func (s *server) CreateRepositoryFromURL(ctx context.Context, req *gitalypb.Crea return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, req.GetRepository(), func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, req.GetRepository(), func(repo *gitalypb.Repository) error { targetPath, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return fmt.Errorf("getting temporary repository path: %w", err) diff --git a/internal/gitaly/service/repository/remove.go b/internal/gitaly/service/repository/remove.go index 4dfa4711a..0abac2e6a 100644 --- a/internal/gitaly/service/repository/remove.go +++ b/internal/gitaly/service/repository/remove.go @@ -15,7 +15,7 @@ func (s *server) RemoveRepository(ctx context.Context, in *gitalypb.RemoveReposi return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Remove(ctx, s.locator, s.txManager, repository); err != nil { + if err := repoutil.Remove(ctx, s.locator, s.txManager, s.repositoryCounter, repository); err != nil { return nil, err } diff --git a/internal/gitaly/service/repository/remove_all.go b/internal/gitaly/service/repository/remove_all.go index 3a0d24a1f..e5f62de36 100644 --- a/internal/gitaly/service/repository/remove_all.go +++ b/internal/gitaly/service/repository/remove_all.go @@ -21,6 +21,8 @@ func (s *server) RemoveAll(ctx context.Context, in *gitalypb.RemoveAllRequest) ( return nil, structerr.NewInternal("remove all: %w", err).WithMetadata("storage", in.StorageName) } + s.repositoryCounter.DeleteStorage(in.StorageName) + return &gitalypb.RemoveAllResponse{}, nil } diff --git a/internal/gitaly/service/repository/replicate.go b/internal/gitaly/service/repository/replicate.go index 4f1be41f9..29a23d0a0 100644 --- a/internal/gitaly/service/repository/replicate.go +++ b/internal/gitaly/service/repository/replicate.go @@ -162,7 +162,7 @@ func (s *server) create(ctx context.Context, in *gitalypb.ReplicateRepositoryReq } func (s *server) createFromSnapshot(ctx context.Context, source, target *gitalypb.Repository) error { - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, target, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, target, func(repo *gitalypb.Repository) error { if err := s.extractSnapshot(ctx, source, repo); err != nil { return fmt.Errorf("extracting snapshot: %w", err) } diff --git a/internal/gitaly/service/repository/restore_repository.go b/internal/gitaly/service/repository/restore_repository.go index b2ae8e247..9818fec86 100644 --- a/internal/gitaly/service/repository/restore_repository.go +++ b/internal/gitaly/service/repository/restore_repository.go @@ -26,6 +26,7 @@ func (s *server) RestoreRepository(ctx context.Context, in *gitalypb.RestoreRepo s.gitCmdFactory, s.catfileCache, s.txManager, + s.repositoryCounter, ) err := manager.Restore(ctx, &backup.RestoreRequest{ diff --git a/internal/gitaly/service/repository/server.go b/internal/gitaly/service/repository/server.go index 0265fadb6..aafcf2a9f 100644 --- a/internal/gitaly/service/repository/server.go +++ b/internal/gitaly/service/repository/server.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git2go" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/unarycache" @@ -32,6 +33,7 @@ type server struct { housekeepingManager housekeeping.Manager backupSink backup.Sink backupLocator backup.Locator + repositoryCounter *counter.RepositoryCounter licenseCache *unarycache.Cache[git.ObjectID, *gitalypb.FindLicenseResponse] } @@ -48,6 +50,7 @@ func NewServer( housekeepingManager housekeeping.Manager, backupSink backup.Sink, backupLocator backup.Locator, + repositoryCounter *counter.RepositoryCounter, ) gitalypb.RepositoryServiceServer { return &server{ locator: locator, @@ -61,6 +64,7 @@ func NewServer( housekeepingManager: housekeepingManager, backupSink: backupSink, backupLocator: backupLocator, + repositoryCounter: repositoryCounter, licenseCache: newLicenseCache(), } diff --git a/internal/gitaly/service/repository/testhelper_test.go b/internal/gitaly/service/repository/testhelper_test.go index 8891207e1..98d65e99a 100644 --- a/internal/gitaly/service/repository/testhelper_test.go +++ b/internal/gitaly/service/repository/testhelper_test.go @@ -68,6 +68,7 @@ func runRepositoryService(tb testing.TB, cfg config.Cfg, opts ...testserver.Gita deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer( deps.GetHookManager(), @@ -106,6 +107,7 @@ func runRepositoryService(tb testing.TB, cfg config.Cfg, opts ...testserver.Gita deps.GetCatfileCache(), deps.GetTxManager(), deps.GetHousekeepingManager(), + deps.GetRepositoryCounter(), )) }, opts...) diff --git a/internal/gitaly/service/setup/register.go b/internal/gitaly/service/setup/register.go index 1708124e7..919f5e99a 100644 --- a/internal/gitaly/service/setup/register.go +++ b/internal/gitaly/service/setup/register.go @@ -102,6 +102,7 @@ func RegisterAll(srv *grpc.Server, deps *service.Dependencies) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterSSHServiceServer(srv, ssh.NewServer( deps.GetLocator(), @@ -139,6 +140,7 @@ func RegisterAll(srv *grpc.Server, deps *service.Dependencies) { deps.GetCatfileCache(), deps.GetTxManager(), deps.GetHousekeepingManager(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterHookServiceServer(srv, hook.NewServer( deps.GetHookManager(), diff --git a/internal/gitaly/service/smarthttp/testhelper_test.go b/internal/gitaly/service/smarthttp/testhelper_test.go index aece0ff37..4dc4c0640 100644 --- a/internal/gitaly/service/smarthttp/testhelper_test.go +++ b/internal/gitaly/service/smarthttp/testhelper_test.go @@ -44,6 +44,7 @@ func startSmartHTTPServerWithOptions(t *testing.T, cfg config.Cfg, opts []Server deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterObjectPoolServiceServer(srv, objectpool.NewServer( deps.GetLocator(), @@ -51,6 +52,7 @@ func startSmartHTTPServerWithOptions(t *testing.T, cfg config.Cfg, opts []Server deps.GetCatfileCache(), deps.GetTxManager(), deps.GetHousekeepingManager(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer( deps.GetHookManager(), diff --git a/internal/gitaly/service/ssh/testhelper_test.go b/internal/gitaly/service/ssh/testhelper_test.go index 8c3cb7796..a1620b6f3 100644 --- a/internal/gitaly/service/ssh/testhelper_test.go +++ b/internal/gitaly/service/ssh/testhelper_test.go @@ -53,6 +53,7 @@ func startSSHServerWithOptions(t *testing.T, cfg config.Cfg, opts []ServerOpt, s deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) gitalypb.RegisterObjectPoolServiceServer(srv, objectpool.NewServer( deps.GetLocator(), @@ -60,6 +61,7 @@ func startSSHServerWithOptions(t *testing.T, cfg config.Cfg, opts []ServerOpt, s deps.GetCatfileCache(), deps.GetTxManager(), deps.GetHousekeepingManager(), + deps.GetRepositoryCounter(), )) }, serverOpts...) } diff --git a/internal/praefect/info_service_test.go b/internal/praefect/info_service_test.go index a5f291e09..12c7bbfbf 100644 --- a/internal/praefect/info_service_test.go +++ b/internal/praefect/info_service_test.go @@ -47,6 +47,7 @@ func TestInfoService_RepositoryReplicas(t *testing.T) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )) }, testserver.WithDisablePraefect()) cfgNodes = append(cfgNodes, &config.Node{ diff --git a/internal/praefect/verifier_test.go b/internal/praefect/verifier_test.go index 089b551dd..476810642 100644 --- a/internal/praefect/verifier_test.go +++ b/internal/praefect/verifier_test.go @@ -488,6 +488,7 @@ func TestVerifier(t *testing.T) { deps.GetHousekeepingManager(), deps.GetBackupSink(), deps.GetBackupLocator(), + deps.GetRepositoryCounter(), )}) } } diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index aca958d2d..8e44b7839 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -27,6 +27,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/server" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/storagemgr" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/gitlab" @@ -266,6 +267,7 @@ type gitalyServerDeps struct { packObjectsCache streamcache.Cache packObjectsLimiter limiter.Limiter limitHandler *limithandler.LimiterMiddleware + repositoryCounter *counter.RepositoryCounter git2goExecutor *git2go.Executor updaterWithHooks *updateref.UpdaterWithHooks housekeepingManager housekeeping.Manager @@ -337,6 +339,10 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, cfg config.Cfg) * gsd.limitHandler = limithandler.New(cfg, limithandler.LimitConcurrencyByRepo, limithandler.WithConcurrencyLimiters) } + if gsd.repositoryCounter == nil { + gsd.repositoryCounter = counter.NewRepositoryCounter() + } + if gsd.git2goExecutor == nil { gsd.git2goExecutor = git2go.NewExecutor(cfg, gsd.gitCmdFactory, gsd.locator) } @@ -381,6 +387,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, cfg config.Cfg) * PackObjectsCache: gsd.packObjectsCache, PackObjectsLimiter: gsd.packObjectsLimiter, LimitHandler: gsd.limitHandler, + RepositoryCounter: gsd.repositoryCounter, Git2goExecutor: gsd.git2goExecutor, UpdaterWithHooks: gsd.updaterWithHooks, HousekeepingManager: gsd.housekeepingManager, @@ -466,6 +473,14 @@ func WithDiskCache(diskCache cache.Cache) GitalyServerOpt { } } +// WithRepositoryCounter sets the counter.RepositoryCounter instance that will be used for gitaly services initialisation. +func WithRepositoryCounter(repositoryCounter *counter.RepositoryCounter) GitalyServerOpt { + return func(deps gitalyServerDeps) gitalyServerDeps { + deps.repositoryCounter = repositoryCounter + return deps + } +} + // WithPackObjectsLimiter sets the PackObjectsLimiter that will be // used for gitaly services initialization. func WithPackObjectsLimiter(limiter *limiter.ConcurrencyLimiter) GitalyServerOpt { |