diff options
author | Karthik Nayak <knayak@gitlab.com> | 2023-05-09 10:49:16 +0300 |
---|---|---|
committer | Karthik Nayak <knayak@gitlab.com> | 2023-05-30 11:00:45 +0300 |
commit | 3ee7c6661f34477c3fce660e6580c80ffb1204fe (patch) | |
tree | 58f7d0a793681eb8c0efabf5cf4ef2710f512e8a | |
parent | 1a557c8c7a2dd3bab5d76f8a1587313e902becd1 (diff) |
gitaly: Add `RepositoryManager` to `TransactionManager`
In the `TransactionManager` we want to cleanup stale lock files if we
run into them (but not when housekeeping is running). To do this we need
the `housekeeping.RepositoryManager`, let's add this field to the
TransactionManager.
The following commit[s] will utilize the field.
-rw-r--r-- | internal/gitaly/partition_manager.go | 16 | ||||
-rw-r--r-- | internal/gitaly/partition_manager_test.go | 9 | ||||
-rw-r--r-- | internal/gitaly/transaction_manager.go | 15 | ||||
-rw-r--r-- | internal/gitaly/transaction_manager_test.go | 17 | ||||
-rw-r--r-- | internal/testhelper/testserver/gitaly.go | 1 |
5 files changed, 50 insertions, 8 deletions
diff --git a/internal/gitaly/partition_manager.go b/internal/gitaly/partition_manager.go index 14a115a28..deaeb84e5 100644 --- a/internal/gitaly/partition_manager.go +++ b/internal/gitaly/partition_manager.go @@ -12,6 +12,7 @@ import ( "github.com/dgraph-io/badger/v3" "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" repo "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" @@ -30,6 +31,8 @@ type PartitionManager struct { storages map[string]*storageManager // commandFactory is passed as a dependency to the constructed TransactionManagers. commandFactory git.CommandFactory + // housekeepingManager access to the housekeeping.Manager. + housekeepingManager housekeeping.Manager } // storageManager represents a single storage. @@ -111,7 +114,13 @@ func (ptn *partition) stop() { } // NewPartitionManager returns a new PartitionManager. -func NewPartitionManager(configuredStorages []config.Storage, cmdFactory git.CommandFactory, localRepoFactory localrepo.Factory, logger logrus.FieldLogger) (*PartitionManager, error) { +func NewPartitionManager( + configuredStorages []config.Storage, + cmdFactory git.CommandFactory, + housekeepingManager housekeeping.Manager, + localRepoFactory localrepo.Factory, + logger logrus.FieldLogger, +) (*PartitionManager, error) { storages := make(map[string]*storageManager, len(configuredStorages)) for _, storage := range configuredStorages { repoFactory, err := localRepoFactory.ScopeByStorage(storage.Name) @@ -154,7 +163,7 @@ func NewPartitionManager(configuredStorages []config.Storage, cmdFactory git.Com } } - return &PartitionManager{storages: storages, commandFactory: cmdFactory}, nil + return &PartitionManager{storages: storages, commandFactory: cmdFactory, housekeepingManager: housekeepingManager}, nil } func stagingDirectoryPath(storagePath string) string { @@ -194,7 +203,8 @@ func (pm *PartitionManager) Begin(ctx context.Context, repo repo.GitRepo) (*Tran return nil, fmt.Errorf("create staging directory: %w", err) } - mgr := NewTransactionManager(storageMgr.database, storageMgr.path, relativePath, stagingDir, storageMgr.repoFactory, pm.commandFactory, storageMgr.transactionFinalizerFactory(ptn)) + mgr := NewTransactionManager(storageMgr.database, storageMgr.path, relativePath, stagingDir, pm.commandFactory, pm.housekeepingManager, storageMgr.repoFactory, storageMgr.transactionFinalizerFactory(ptn)) + ptn.transactionManager = mgr storageMgr.partitions[relativePath] = ptn diff --git a/internal/gitaly/partition_manager_test.go b/internal/gitaly/partition_manager_test.go index 0c8056474..623a34b14 100644 --- a/internal/gitaly/partition_manager_test.go +++ b/internal/gitaly/partition_manager_test.go @@ -12,9 +12,12 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" repo "gitlab.com/gitlab-org/gitaly/v16/internal/git/repository" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" @@ -563,8 +566,12 @@ func TestPartitionManager(t *testing.T) { ) } - partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, localRepoFactory, logrus.StandardLogger()) + txManager := transaction.NewManager(cfg, backchannel.NewRegistry()) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) + + partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, housekeepingManager, localRepoFactory, logrus.StandardLogger()) require.NoError(t, err) + defer func() { partitionManager.Stop() for _, storage := range cfg.Storages { diff --git a/internal/gitaly/transaction_manager.go b/internal/gitaly/transaction_manager.go index 8b3919136..939cd4ca7 100644 --- a/internal/gitaly/transaction_manager.go +++ b/internal/gitaly/transaction_manager.go @@ -18,6 +18,7 @@ import ( "github.com/dgraph-io/badger/v3" "gitlab.com/gitlab-org/gitaly/v16/internal/git" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/updateref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" @@ -455,6 +456,8 @@ type TransactionManager struct { appliedLogIndex LogIndex // customHookIndex stores the log index of the latest committed custom custom hooks in the repository. customHookIndex LogIndex + // housekeepingManager access to the housekeeping.Manager. + housekeepingManager housekeeping.Manager // transactionFinalizer executes when a transaction is completed. transactionFinalizer func() @@ -466,7 +469,16 @@ type TransactionManager struct { } // NewTransactionManager returns a new TransactionManager for the given repository. -func NewTransactionManager(db *badger.DB, storagePath, relativePath, stagingDir string, repositoryFactory localrepo.StorageScopedFactory, cmdFactory git.CommandFactory, transactionFinalizer func()) *TransactionManager { +func NewTransactionManager( + db *badger.DB, + storagePath, + relativePath, + stagingDir string, + cmdFactory git.CommandFactory, + housekeepingManager housekeeping.Manager, + repositoryFactory localrepo.StorageScopedFactory, + transactionFinalizer func(), +) *TransactionManager { ctx, cancel := context.WithCancel(context.Background()) return &TransactionManager{ ctx: ctx, @@ -483,6 +495,7 @@ func NewTransactionManager(db *badger.DB, storagePath, relativePath, stagingDir initialized: make(chan struct{}), applyNotifications: make(map[LogIndex]chan struct{}), stagingDirectory: stagingDir, + housekeepingManager: housekeepingManager, transactionFinalizer: transactionFinalizer, awaitingTransactions: make(map[LogIndex]resultChannel), } diff --git a/internal/gitaly/transaction_manager_test.go b/internal/gitaly/transaction_manager_test.go index db1c8eb20..83eae522b 100644 --- a/internal/gitaly/transaction_manager_test.go +++ b/internal/gitaly/transaction_manager_test.go @@ -21,10 +21,13 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/git/updateref" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/repoutil" + "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" @@ -2864,11 +2867,14 @@ func TestTransactionManager(t *testing.T) { stagingDir := t.TempDir() storagePath := setup.Config.Storages[0].Path + txManager := transaction.NewManager(setup.Config, backchannel.NewRegistry()) + housekeepingManager := housekeeping.NewManager(setup.Config.Prometheus, txManager) + var ( // managerRunning tracks whether the manager is running or stopped. managerRunning bool // transactionManager is the current TransactionManager instance. - transactionManager = NewTransactionManager(database, storagePath, relativePath, stagingDir, setup.RepositoryFactory, setup.CommandFactory, noopTransactionFinalizer) + transactionManager = NewTransactionManager(database, storagePath, relativePath, stagingDir, setup.CommandFactory, housekeepingManager, setup.RepositoryFactory, noopTransactionFinalizer) // managerErr is used for synchronizing manager stopping and returning // the error from Run. managerErr chan error @@ -2909,7 +2915,8 @@ func TestTransactionManager(t *testing.T) { managerRunning = true managerErr = make(chan error) - transactionManager = NewTransactionManager(database, storagePath, relativePath, stagingDir, setup.RepositoryFactory, setup.CommandFactory, noopTransactionFinalizer) + transactionManager = NewTransactionManager(database, storagePath, relativePath, stagingDir, setup.CommandFactory, housekeepingManager, setup.RepositoryFactory, noopTransactionFinalizer) + installHooks(t, transactionManager, database, hooks{ beforeReadLogEntry: step.Hooks.BeforeApplyLogEntry, beforeStoreLogEntry: step.Hooks.BeforeAppendLogEntry, @@ -3215,6 +3222,9 @@ func BenchmarkTransactionManager(b *testing.B) { require.NoError(b, err) defer testhelper.MustClose(b, database) + txManager := transaction.NewManager(cfg, backchannel.NewRegistry()) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) + var ( // managerWG records the running TransactionManager.Run goroutines. managerWG sync.WaitGroup @@ -3256,7 +3266,8 @@ func BenchmarkTransactionManager(b *testing.B) { commit1 = gittest.WriteCommit(b, cfg, repoPath, gittest.WithParents()) commit2 = gittest.WriteCommit(b, cfg, repoPath, gittest.WithParents(commit1)) - manager := NewTransactionManager(database, cfg.Storages[0].Path, repo.RelativePath, b.TempDir(), repositoryFactory, cmdFactory, noopTransactionFinalizer) + manager := NewTransactionManager(database, cfg.Storages[0].Path, repo.RelativePath, b.TempDir(), cmdFactory, housekeepingManager, repositoryFactory, noopTransactionFinalizer) + managers = append(managers, manager) managerWG.Add(1) diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index 43ac10b0c..d46061101 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -355,6 +355,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, cfg config.Cfg) * partitionManager, err = gitaly.NewPartitionManager( cfg.Storages, gsd.gitCmdFactory, + gsd.housekeepingManager, localrepo.NewFactory(gsd.locator, gsd.gitCmdFactory, gsd.catfileCache), gsd.logger, ) |