diff options
author | Justin Tobler <jtobler@gitlab.com> | 2023-10-12 04:50:08 +0300 |
---|---|---|
committer | Justin Tobler <jtobler@gitlab.com> | 2023-10-12 04:50:08 +0300 |
commit | 8f7a79ea157b7005fb7c59fb95772af70afaf28e (patch) | |
tree | 0a1492f5d8df113f3909cfce4e8320d1808c683a | |
parent | b6a68035a08fd6b020781582315d1c6877c48b82 (diff) | |
parent | 695b57bdf033a5e6cb1ddbac471aa26d00d2186c (diff) |
Merge branch 'pks-log-replace-ctxlogrus-pt4' into 'master'
global: Replace use of ctxlogrus with injected logger (pt.4)
See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/6464
Merged-by: Justin Tobler <jtobler@gitlab.com>
Approved-by: Justin Tobler <jtobler@gitlab.com>
Reviewed-by: karthik nayak <knayak@gitlab.com>
Co-authored-by: Patrick Steinhardt <psteinhardt@gitlab.com>
88 files changed, 337 insertions, 262 deletions
@@ -191,11 +191,13 @@ Thus, gRPC handlers should avoid using `Unavailable` status code. ### Use context-based logging -The `ctxlogrus` package allows to extract a logger from the current -`context.Context` structure. This should be the default logging facility, as it -may carry additional context-sensitive information like the `correlation_id` -that makes it easy to correlate a log entry with other entries of the same -event. +The context may contain additional information about the current calling context, like for example the correlation ID. +This structured data can be added to the context via calls to `log.AddFields()` and is injected by default via our +requestinfo gRPC middleware. + +This structured data can be extracted from the context into generated log message by using context-aware logging +functions like `log.DebugContext()` and related functions. These functions should thus be used instead of the +non-context-aware logging functions like `log.Debug()` whenever a context is available. ### Errors diff --git a/internal/backup/backup.go b/internal/backup/backup.go index d730c2c93..42094222a 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -177,7 +177,7 @@ func NewManagerLocal( conns: nil, // Will be removed once the restore operations are part of the Repository interface. locator: locator, repositoryFactory: func(ctx context.Context, repo *gitalypb.Repository, server storage.ServerInfo) (Repository, error) { - localRepo := localrepo.New(storageLocator, gitCmdFactory, catfileCache, repo) + localRepo := localrepo.New(logger, storageLocator, gitCmdFactory, catfileCache, repo) return newLocalRepository(logger, storageLocator, gitCmdFactory, txManager, repoCounter, localRepo), nil }, diff --git a/internal/backup/repository.go b/internal/backup/repository.go index 2ef51046a..ca28a8ef3 100644 --- a/internal/backup/repository.go +++ b/internal/backup/repository.go @@ -363,7 +363,7 @@ func (r *localRepository) CreateBundle(ctx context.Context, out io.Writer, patte // Remove removes the repository. Does not return an error if the repository // cannot be found. func (r *localRepository) Remove(ctx context.Context) error { - err := repoutil.Remove(ctx, r.locator, r.txManager, r.repoCounter, r.repo) + err := repoutil.Remove(ctx, r.logger, r.locator, r.txManager, r.repoCounter, r.repo) switch { case status.Code(err) == codes.NotFound: return nil @@ -377,6 +377,7 @@ func (r *localRepository) Remove(ctx context.Context) error { func (r *localRepository) Create(ctx context.Context, hash git.ObjectHash) error { if err := repoutil.Create( ctx, + r.logger, r.locator, r.gitCmdFactory, r.txManager, diff --git a/internal/cli/gitaly/serve.go b/internal/cli/gitaly/serve.go index 7a94a63b8..c9d067fcc 100644 --- a/internal/cli/gitaly/serve.go +++ b/internal/cli/gitaly/serve.go @@ -241,7 +241,7 @@ func run(cfg config.Cfg, logger log.Logger) error { transactionManager := transaction.NewManager(cfg, logger, registry) prometheus.MustRegister(transactionManager) - housekeepingManager := housekeeping.NewManager(cfg.Prometheus, transactionManager) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, logger, transactionManager) prometheus.MustRegister(housekeepingManager) hookManager := hook.Manager(hook.DisabledManager{}) @@ -483,7 +483,7 @@ func run(cfg config.Cfg, logger log.Logger) error { ctx, logger, maintenance.DailyOptimizationWorker(cfg, maintenance.OptimizerFunc(func(ctx context.Context, logger log.Logger, repo storage.Repository) error { - return housekeepingManager.OptimizeRepository(ctx, logger, localrepo.New(locator, gitCmdFactory, catfileCache, repo)) + return housekeepingManager.OptimizeRepository(ctx, localrepo.New(logger, locator, gitCmdFactory, catfileCache, repo)) })), ) if err != nil { diff --git a/internal/git/housekeeping/clean_stale_data.go b/internal/git/housekeeping/clean_stale_data.go index 58c5c821c..6cffe2cc9 100644 --- a/internal/git/housekeeping/clean_stale_data.go +++ b/internal/git/housekeeping/clean_stale_data.go @@ -14,7 +14,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tracing" @@ -90,13 +89,13 @@ func DefaultStaleDataCleanup() CleanStaleDataConfig { } // CleanStaleData removes any stale data in the repository as per the provided configuration. -func (m *RepositoryManager) CleanStaleData(ctx context.Context, logger log.Logger, repo *localrepo.Repo, cfg CleanStaleDataConfig) error { +func (m *RepositoryManager) CleanStaleData(ctx context.Context, repo *localrepo.Repo, cfg CleanStaleDataConfig) error { span, ctx := tracing.StartSpanIfHasParent(ctx, "housekeeping.CleanStaleData", nil) defer span.Finish() repoPath, err := repo.Path() if err != nil { - myLogger(logger).WithError(err).Warn("housekeeping failed to get repo path") + m.logger.WithError(err).WarnContext(ctx, "housekeeping failed to get repo path") if structerr.GRPCCode(err) == codes.NotFound { return nil } @@ -109,12 +108,12 @@ func (m *RepositoryManager) CleanStaleData(ctx context.Context, logger log.Logge return } - logEntry := myLogger(logger) + logEntry := m.logger for staleDataType, count := range staleDataByType { logEntry = logEntry.WithField(fmt.Sprintf("stale_data.%s", staleDataType), count) m.prunedFilesTotal.WithLabelValues(staleDataType).Add(float64(count)) } - logEntry.Info("removed files") + logEntry.InfoContext(ctx, "removed files") }() var filesToPrune []string @@ -134,7 +133,7 @@ func (m *RepositoryManager) CleanStaleData(ctx context.Context, logger log.Logge continue } staleDataByType["failures"]++ - myLogger(logger).WithError(err).WithField("path", path).Warn("unable to remove stale file") + m.logger.WithError(err).WithField("path", path).WarnContext(ctx, "unable to remove stale file") } } @@ -655,7 +654,3 @@ func removeEmptyDirs(ctx context.Context, target string) (int, error) { return prunedDirsTotal + 1, nil } - -func myLogger(logger log.Logger) log.Logger { - return logger.WithField("system", "housekeeping") -} diff --git a/internal/git/housekeeping/clean_stale_data_test.go b/internal/git/housekeeping/clean_stale_data_test.go index 8ca4a0e83..96e6752bd 100644 --- a/internal/git/housekeeping/clean_stale_data_test.go +++ b/internal/git/housekeeping/clean_stale_data_test.go @@ -399,9 +399,9 @@ func TestRepositoryManager_CleanStaleData(t *testing.T) { e.create(t, repoPath) } - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, e := range tc.entries { e.validate(t, repoPath) @@ -504,9 +504,9 @@ func TestRepositoryManager_CleanStaleData_references(t *testing.T) { require.NoError(t, os.Chtimes(path, filetime, filetime)) } - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) var actual []string require.NoError(t, filepath.Walk(filepath.Join(repoPath, "refs"), func(path string, info os.FileInfo, _ error) error { @@ -631,9 +631,9 @@ func TestRepositoryManager_CleanStaleData_emptyRefDirs(t *testing.T) { e.create(t, repoPath) } - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, e := range tc.entries { e.validate(t, repoPath) @@ -767,9 +767,9 @@ func TestRepositoryManager_CleanStaleData_withSpecificFile(t *testing.T) { SkipCreationViaService: true, }) repo := localrepo.NewTestRepo(t, cfg, repoProto) - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, subcase := range []struct { desc string entry entry @@ -813,7 +813,7 @@ func TestRepositoryManager_CleanStaleData_withSpecificFile(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, subcase.expectedFiles, staleFiles) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) entry.validate(t, repoPath) }) @@ -865,9 +865,9 @@ func TestRepositoryManager_CleanStaleData_serverInfo(t *testing.T) { filepath.Join(repoPath, "objects/info/packs_123456"), }, staleFiles) - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) for _, entry := range entries { entry.validate(t, repoPath) @@ -995,9 +995,9 @@ func TestRepositoryManager_CleanStaleData_referenceLocks(t *testing.T) { require.NoError(t, err) require.ElementsMatch(t, expectedReferenceLocks, staleLockfiles) - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, tc.cfg)) + require.NoError(t, mgr.CleanStaleData(ctx, repo, tc.cfg)) for _, e := range tc.entries { e.validate(t, repoPath) @@ -1110,7 +1110,7 @@ func TestRepositoryManager_CleanStaleData_missingRepo(t *testing.T) { require.NoError(t, os.RemoveAll(repoPath)) - require.NoError(t, NewManager(cfg.Prometheus, nil).CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil).CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) } func TestRepositoryManager_CleanStaleData_unsetConfiguration(t *testing.T) { @@ -1149,9 +1149,9 @@ func TestRepositoryManager_CleanStaleData_unsetConfiguration(t *testing.T) { unrelated = untouched `), perm.SharedFile)) - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, `[core] repositoryformatversion = 0 @@ -1190,7 +1190,7 @@ func TestRepositoryManager_CleanStaleData_unsetConfigurationTransactional(t *tes AuthInfo: backchannel.WithID(nil, 1234), }) - require.NoError(t, NewManager(cfg.Prometheus, txManager).CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, NewManager(cfg.Prometheus, testhelper.SharedLogger(t), txManager).CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, 2, len(txManager.Votes())) configKeys := gittest.Exec(t, cfg, "-C", repoPath, "config", "--list", "--local", "--name-only") @@ -1240,9 +1240,9 @@ func TestRepositoryManager_CleanStaleData_pruneEmptyConfigSections(t *testing.T) [remote "tmp-8c948ca94832c2725733e48cb2902287"] `), perm.SharedFile)) - mgr := NewManager(cfg.Prometheus, nil) + mgr := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), nil) - require.NoError(t, mgr.CleanStaleData(ctx, testhelper.SharedLogger(t), repo, DefaultStaleDataCleanup())) + require.NoError(t, mgr.CleanStaleData(ctx, repo, DefaultStaleDataCleanup())) require.Equal(t, `[core] repositoryformatversion = 0 filemode = true diff --git a/internal/git/housekeeping/manager.go b/internal/git/housekeeping/manager.go index f07cbe393..4caab5568 100644 --- a/internal/git/housekeeping/manager.go +++ b/internal/git/housekeeping/manager.go @@ -16,10 +16,10 @@ import ( // such as the cleanup of unneeded files and optimizations for the repository's data structures. type Manager interface { // CleanStaleData removes any stale data in the repository as per the provided configuration. - CleanStaleData(context.Context, log.Logger, *localrepo.Repo, CleanStaleDataConfig) error + CleanStaleData(context.Context, *localrepo.Repo, CleanStaleDataConfig) error // OptimizeRepository optimizes the repository's data structures such that it can be more // efficiently served. - OptimizeRepository(context.Context, log.Logger, *localrepo.Repo, ...OptimizeRepositoryOption) error + OptimizeRepository(context.Context, *localrepo.Repo, ...OptimizeRepositoryOption) error // AddPackRefsInhibitor allows clients to block housekeeping from running git-pack-refs(1). AddPackRefsInhibitor(ctx context.Context, repoPath string) (bool, func(), error) } @@ -205,6 +205,7 @@ func (s *repositoryStates) tryRunningPackRefs(repoPath string) (successful bool, // RepositoryManager is an implementation of the Manager interface. type RepositoryManager struct { + logger log.Logger txManager transaction.Manager tasksTotal *prometheus.CounterVec @@ -219,8 +220,9 @@ type RepositoryManager struct { } // NewManager creates a new RepositoryManager. -func NewManager(promCfg gitalycfgprom.Config, txManager transaction.Manager) *RepositoryManager { +func NewManager(promCfg gitalycfgprom.Config, logger log.Logger, txManager transaction.Manager) *RepositoryManager { return &RepositoryManager{ + logger: logger.WithField("system", "housekeeping"), txManager: txManager, tasksTotal: prometheus.NewCounterVec( diff --git a/internal/git/housekeeping/optimize_repository.go b/internal/git/housekeeping/optimize_repository.go index 3bd9a7ff7..1dc0d9337 100644 --- a/internal/git/housekeeping/optimize_repository.go +++ b/internal/git/housekeeping/optimize_repository.go @@ -41,7 +41,6 @@ func WithOptimizationStrategyConstructor(strategyConstructor OptimizationStrateg // or not depends on a set of heuristics. func (m *RepositoryManager) OptimizeRepository( ctx context.Context, - logger log.Logger, repo *localrepo.Repo, opts ...OptimizeRepositoryOption, ) error { @@ -75,7 +74,7 @@ func (m *RepositoryManager) OptimizeRepository( if err != nil { return fmt.Errorf("deriving repository info: %w", err) } - m.reportRepositoryInfo(ctx, logger, repositoryInfo) + m.reportRepositoryInfo(ctx, m.logger, repositoryInfo) var strategy OptimizationStrategy if cfg.StrategyConstructor == nil { @@ -84,7 +83,7 @@ func (m *RepositoryManager) OptimizeRepository( strategy = cfg.StrategyConstructor(repositoryInfo) } - return m.optimizeFunc(ctx, m, logger, repo, strategy) + return m.optimizeFunc(ctx, m, m.logger, repo, strategy) } func (m *RepositoryManager) reportRepositoryInfo(ctx context.Context, logger log.Logger, info stats.RepositoryInfo) { @@ -160,7 +159,7 @@ func optimizeRepository( }() timer := prometheus.NewTimer(m.tasksLatency.WithLabelValues("clean-stale-data")) - if err := m.CleanStaleData(ctx, logger, repo, DefaultStaleDataCleanup()); err != nil { + if err := m.CleanStaleData(ctx, repo, DefaultStaleDataCleanup()); err != nil { return fmt.Errorf("could not execute houskeeping: %w", err) } timer.ObserveDuration() diff --git a/internal/git/housekeeping/optimize_repository_ext_test.go b/internal/git/housekeeping/optimize_repository_ext_test.go index 374c9afeb..d12804bc7 100644 --- a/internal/git/housekeeping/optimize_repository_ext_test.go +++ b/internal/git/housekeeping/optimize_repository_ext_test.go @@ -172,7 +172,7 @@ func TestPruneIfNeeded(t *testing.T) { logger := testhelper.NewLogger(t) hook := testhelper.AddLoggerHook(logger) - require.NoError(t, housekeeping.NewManager(cfg.Prometheus, nil).OptimizeRepository(ctx, logger, repo)) + require.NoError(t, housekeeping.NewManager(cfg.Prometheus, logger, nil).OptimizeRepository(ctx, repo)) require.Equal(t, tc.expectedLogEntries, hook.LastEntry().Data["optimizations"]) }) } diff --git a/internal/git/housekeeping/optimize_repository_test.go b/internal/git/housekeeping/optimize_repository_test.go index 874b3c830..2df92f33b 100644 --- a/internal/git/housekeeping/optimize_repository_test.go +++ b/internal/git/housekeeping/optimize_repository_test.go @@ -234,7 +234,7 @@ func TestRepackIfNeeded(t *testing.T) { }, } - repo := localrepo.New(config.NewLocator(cfg), gitCmdFactory, nil, repoProto) + repo := localrepo.New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, nil, repoProto) expectedCfg := RepackObjectsConfig{ Strategy: RepackObjectsStrategyFullWithCruft, @@ -404,6 +404,7 @@ func TestPackRefsIfNeeded(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) gitCmdFactory := blockingCommandFactory{ CommandFactory: gittest.NewCommandFactory(t, cfg), @@ -413,7 +414,7 @@ func TestPackRefsIfNeeded(t *testing.T) { repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := localrepo.New(config.NewLocator(cfg), &gitCmdFactory, nil, repoProto) + repo := localrepo.New(logger, config.NewLocator(cfg), &gitCmdFactory, nil, repoProto) // Write an empty commit such that we can create valid refs. gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main")) @@ -421,7 +422,7 @@ func TestPackRefsIfNeeded(t *testing.T) { packedRefsPath := filepath.Join(repoPath, "packed-refs") looseRefPath := filepath.Join(repoPath, "refs", "heads", "main") - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, logger, nil) data := tc.setup(t, ctx, manager, repoPath, &gitCmdFactory) didRepack, err := manager.packRefsIfNeeded(ctx, repo, mockOptimizationStrategy{ @@ -1007,7 +1008,7 @@ func TestOptimizeRepository(t *testing.T) { } return setupData{ - repo: localrepo.New(config.NewLocator(cfg), gitCmdFactory, nil, repo), + repo: localrepo.New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, nil, repo), expectedMetrics: []metric{ {name: "packed_objects_geometric", status: "failure", count: 1}, {name: "written_bitmap", status: "failure", count: 1}, @@ -1026,9 +1027,9 @@ func TestOptimizeRepository(t *testing.T) { setup := tc.setup(t, relativePath) - manager := NewManager(cfg.Prometheus, txManager) + manager := NewManager(cfg.Prometheus, testhelper.SharedLogger(t), txManager) - err := manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), setup.repo, setup.options...) + err := manager.OptimizeRepository(ctx, setup.repo, setup.options...) require.Equal(t, setup.expectedErr, err) expectedMetrics := setup.expectedMetrics @@ -1075,7 +1076,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { }) repo := localrepo.NewTestRepo(t, cfg, repoProto) - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, testhelper.NewLogger(t), nil) manager.optimizeFunc = func(context.Context, *RepositoryManager, log.Logger, *localrepo.Repo, OptimizationStrategy) error { reqReceivedCh <- struct{}{} ch <- struct{}{} @@ -1084,14 +1085,14 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { } go func() { - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) }() <-reqReceivedCh // When repository optimizations are performed for a specific repository already, // then any subsequent calls to the same repository should just return immediately // without doing any optimizations at all. - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) <-ch }) @@ -1107,7 +1108,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { }) repo := localrepo.NewTestRepo(t, cfg, repoProto) - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, testhelper.SharedLogger(t), nil) manager.optimizeFunc = func(context.Context, *RepositoryManager, log.Logger, *localrepo.Repo, OptimizationStrategy) error { // This should only happen if housekeeping is running successfully. // So by sending data on this channel we can notify the test that this @@ -1127,7 +1128,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { } go func() { - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) }() // Only if optimizeFunc is run, we shall receive data here, this acts as test that @@ -1141,7 +1142,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { }) repo := localrepo.NewTestRepo(t, cfg, repoProto) - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, testhelper.SharedLogger(t), nil) manager.optimizeFunc = func(context.Context, *RepositoryManager, log.Logger, *localrepo.Repo, OptimizationStrategy) error { require.FailNow(t, "housekeeping run should have been skipped") return nil @@ -1153,7 +1154,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { // check that the state actually exists. require.Contains(t, manager.repositoryStates.values, repoPath) - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) // After running the cleanup, the state should be removed. cleanup() @@ -1174,7 +1175,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { reposOptimized := make(map[string]struct{}) - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, testhelper.SharedLogger(t), nil) manager.optimizeFunc = func(_ context.Context, _ *RepositoryManager, _ log.Logger, repo *localrepo.Repo, _ OptimizationStrategy) error { reposOptimized[repo.GetRelativePath()] = struct{}{} @@ -1189,13 +1190,13 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { // We block in the first call so that we can assert that a second call // to a different repository performs the optimization regardless without blocking. go func() { - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repoFirst)) + require.NoError(t, manager.OptimizeRepository(ctx, repoFirst)) }() <-reqReceivedCh // Because this optimizes a different repository this call shouldn't block. - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repoSecond)) + require.NoError(t, manager.OptimizeRepository(ctx, repoSecond)) <-ch @@ -1211,7 +1212,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { repo := localrepo.NewTestRepo(t, cfg, repoProto) var optimizations int - manager := NewManager(gitalycfgprom.Config{}, nil) + manager := NewManager(gitalycfgprom.Config{}, testhelper.SharedLogger(t), nil) manager.optimizeFunc = func(context.Context, *RepositoryManager, log.Logger, *localrepo.Repo, OptimizationStrategy) error { optimizations++ @@ -1227,7 +1228,7 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) }() <-reqReceivedCh @@ -1236,9 +1237,9 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { // that all subsequent calls which try to optimize the same repository return immediately. // Furthermore, we expect to see only a single call to the optimizing function because we // don't want to optimize the same repository concurrently. - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) assert.Equal(t, 1, optimizations) <-ch @@ -1247,9 +1248,9 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) { // When performing optimizations sequentially though the repository // should be unlocked after every call, and consequentially we should // also see multiple calls to the optimizing function. - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) - require.NoError(t, manager.OptimizeRepository(ctx, testhelper.SharedLogger(t), repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) + require.NoError(t, manager.OptimizeRepository(ctx, repo)) assert.Equal(t, 4, optimizations) }) } diff --git a/internal/git/housekeeping/worktrees_test.go b/internal/git/housekeeping/worktrees_test.go index d09b0c2b9..97fa927ce 100644 --- a/internal/git/housekeeping/worktrees_test.go +++ b/internal/git/housekeeping/worktrees_test.go @@ -29,7 +29,7 @@ func TestCleanupDisconnectedWorktrees_doesNothingWithoutWorktrees(t *testing.T) countingGitCmdFactory := gittest.NewCountingCommandFactory(t, cfg) - repo := localrepo.New(config.NewLocator(cfg), countingGitCmdFactory, nil, repoProto) + repo := localrepo.New(testhelper.NewLogger(t), config.NewLocator(cfg), countingGitCmdFactory, nil, repoProto) // If this command did spawn git-worktree(1) we'd see an error. It doesn't though because it // detects that there aren't any worktrees at all. diff --git a/internal/git/localrepo/bundle.go b/internal/git/localrepo/bundle.go index f15dc219a..53a02a9b9 100644 --- a/internal/git/localrepo/bundle.go +++ b/internal/git/localrepo/bundle.go @@ -171,7 +171,7 @@ func (repo *Repo) FetchBundle(ctx context.Context, txManager transaction.Manager // createTempBundle copies reader onto the filesystem so that a path can be // passed to git. git-fetch does not support streaming a bundle over a pipe. func (repo *Repo) createTempBundle(ctx context.Context, reader io.Reader) (bundlPath string, returnErr error) { - tmpDir, err := tempdir.New(ctx, repo.GetStorageName(), repo.locator) + tmpDir, err := tempdir.New(ctx, repo.GetStorageName(), repo.logger, repo.locator) if err != nil { return "", fmt.Errorf("create temp bundle: %w", err) } diff --git a/internal/git/localrepo/bundle_test.go b/internal/git/localrepo/bundle_test.go index 6de4941aa..12fe03173 100644 --- a/internal/git/localrepo/bundle_test.go +++ b/internal/git/localrepo/bundle_test.go @@ -207,7 +207,7 @@ func TestRepo_CloneBundle(t *testing.T) { } data := tc.setup(t, ctx, cfg) - repo := New(config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto) + repo := New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto) err := repo.CloneBundle(ctx, data.reader) if data.expectedErr != nil { diff --git a/internal/git/localrepo/factory.go b/internal/git/localrepo/factory.go index 93429ef1a..b79b9e144 100644 --- a/internal/git/localrepo/factory.go +++ b/internal/git/localrepo/factory.go @@ -6,11 +6,13 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) // Factory builds Repo instances. type Factory struct { + logger log.Logger locator storage.Locator cmdFactory git.CommandFactory catfileCache catfile.Cache @@ -25,8 +27,9 @@ type StorageScopedFactory struct { // NewFactory returns a factory type that builds Repo instances. It helps avoid having to drill down Repo // dependencies to all call sites that need to build a Repo. -func NewFactory(locator storage.Locator, cmdFactory git.CommandFactory, catfileCache catfile.Cache) Factory { +func NewFactory(logger log.Logger, locator storage.Locator, cmdFactory git.CommandFactory, catfileCache catfile.Cache) Factory { return Factory{ + logger: logger, locator: locator, cmdFactory: cmdFactory, catfileCache: catfileCache, @@ -35,7 +38,7 @@ func NewFactory(locator storage.Locator, cmdFactory git.CommandFactory, catfileC // Build returns a Repo for the given repository. func (f Factory) Build(repo *gitalypb.Repository) *Repo { - return New(f.locator, f.cmdFactory, f.catfileCache, repo) + return New(f.logger, f.locator, f.cmdFactory, f.catfileCache, repo) } // ScopeByStorage returns a StorageScopedFactory that builds Repo instances for a given storage diff --git a/internal/git/localrepo/factory_test.go b/internal/git/localrepo/factory_test.go index b2fcdfe4f..b99da7da9 100644 --- a/internal/git/localrepo/factory_test.go +++ b/internal/git/localrepo/factory_test.go @@ -9,6 +9,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" + "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -22,7 +23,7 @@ func TestFactory(t *testing.T) { catfileCache := catfile.NewCache(cfg) defer catfileCache.Stop() - factory := NewFactory(locator, cmdFactory, catfileCache) + factory := NewFactory(testhelper.NewLogger(t), locator, cmdFactory, catfileCache) t.Run("Build", func(t *testing.T) { t.Run("parameters are passthrough", func(t *testing.T) { diff --git a/internal/git/localrepo/objects_test.go b/internal/git/localrepo/objects_test.go index fad157f32..eb59b0926 100644 --- a/internal/git/localrepo/objects_test.go +++ b/internal/git/localrepo/objects_test.go @@ -108,7 +108,7 @@ func TestRepo_ReadObject_catfileCount(t *testing.T) { repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := New(config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto) + repo := New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto) blobID := gittest.WriteBlob(t, cfg, repoPath, []byte("content")) diff --git a/internal/git/localrepo/paths_test.go b/internal/git/localrepo/paths_test.go index b33c1444a..80d13f76d 100644 --- a/internal/git/localrepo/paths_test.go +++ b/internal/git/localrepo/paths_test.go @@ -73,7 +73,7 @@ func TestRepo_ObjectDirectoryPath(t *testing.T) { }) locator := config.NewLocator(cfg) - quarantine, err := quarantine.New(ctx, repoProto, locator) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.NewLogger(t), locator) require.NoError(t, err) quarantinedRepo := quarantine.QuarantinedRepo() diff --git a/internal/git/localrepo/refs.go b/internal/git/localrepo/refs.go index 984f916b7..c0be1ebdf 100644 --- a/internal/git/localrepo/refs.go +++ b/internal/git/localrepo/refs.go @@ -13,7 +13,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/command" "gitlab.com/gitlab-org/gitaly/v16/internal/git" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" - "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" ) @@ -164,7 +163,7 @@ func (repo *Repo) setDefaultBranchWithTransaction(ctx context.Context, txManager } defer func() { if err := lockingFileWriter.Close(); err != nil { - log.FromContext(ctx).WithError(err).Error("closing locked HEAD failed") + repo.logger.WithError(err).ErrorContext(ctx, "closing locked HEAD failed") } }() diff --git a/internal/git/localrepo/refs_test.go b/internal/git/localrepo/refs_test.go index b4ba959c7..8ec5aced5 100644 --- a/internal/git/localrepo/refs_test.go +++ b/internal/git/localrepo/refs_test.go @@ -255,6 +255,7 @@ func TestRepo_GetRemoteReferences(t *testing.T) { defer catfileCache.Stop() repo := New( + testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, catfileCache, @@ -478,7 +479,7 @@ func TestRepo_UpdateRef(t *testing.T) { repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := New(repo.locator, repo.gitCmdFactory, repo.catfileCache, repoProto) + repo := New(repo.logger, repo.locator, repo.gitCmdFactory, repo.catfileCache, repoProto) seedRepo(t, repoPath) err := repo.UpdateRef(ctx, git.ReferenceName(tc.ref), tc.newValue, tc.oldValue) diff --git a/internal/git/localrepo/remote_test.go b/internal/git/localrepo/remote_test.go index b36a2c33d..c27c56fef 100644 --- a/internal/git/localrepo/remote_test.go +++ b/internal/git/localrepo/remote_test.go @@ -27,6 +27,7 @@ func TestRepo_FetchRemote(t *testing.T) { catfileCache := catfile.NewCache(cfg) defer catfileCache.Stop() locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) _, remoteRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, @@ -49,11 +50,11 @@ func TestRepo_FetchRemote(t *testing.T) { require.NoError(t, err) } - return New(locator, gitCmdFactory, catfileCache, clientRepo), clientRepoPath + return New(logger, locator, gitCmdFactory, catfileCache, clientRepo), clientRepoPath } t.Run("invalid name", func(t *testing.T) { - repo := New(locator, gitCmdFactory, catfileCache, nil) + repo := New(logger, locator, gitCmdFactory, catfileCache, nil) err := repo.FetchRemote(ctx, " ", FetchOpts{}) require.True(t, errors.Is(err, git.ErrInvalidArg)) @@ -65,7 +66,7 @@ func TestRepo_FetchRemote(t *testing.T) { SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, repoProto) + repo := New(logger, locator, gitCmdFactory, catfileCache, repoProto) var stderr bytes.Buffer err := repo.FetchRemote(ctx, "stub", FetchOpts{Stderr: &stderr}) require.Error(t, err) @@ -97,7 +98,7 @@ func TestRepo_FetchRemote(t *testing.T) { SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, testRepo) + repo := New(logger, locator, gitCmdFactory, catfileCache, testRepo) gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", remoteRepoPath) var stderr bytes.Buffer @@ -110,7 +111,7 @@ func TestRepo_FetchRemote(t *testing.T) { SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, testRepo) + repo := New(logger, locator, gitCmdFactory, catfileCache, testRepo) gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", remoteRepoPath) var stderr bytes.Buffer @@ -127,7 +128,7 @@ func TestRepo_FetchRemote(t *testing.T) { SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, testRepo) + repo := New(logger, locator, gitCmdFactory, catfileCache, testRepo) gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", remoteRepoPath) require.NoError(t, repo.FetchRemote(ctx, "source", FetchOpts{})) @@ -156,7 +157,7 @@ func TestRepo_FetchRemote(t *testing.T) { SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, testRepo) + repo := New(logger, locator, gitCmdFactory, catfileCache, testRepo) gittest.Exec(t, cfg, "-C", testRepoPath, "remote", "add", "source", remoteRepoPath) require.NoError(t, repo.FetchRemote(ctx, "source", FetchOpts{})) @@ -260,11 +261,12 @@ func TestRepo_Push(t *testing.T) { catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) sourceRepoProto, sourceRepoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - sourceRepo := New(locator, gitCmdFactory, catfileCache, sourceRepoProto) + sourceRepo := New(logger, locator, gitCmdFactory, catfileCache, sourceRepoProto) gittest.WriteCommit(t, cfg, sourceRepoPath, gittest.WithBranch("master")) gittest.WriteCommit(t, cfg, sourceRepoPath, gittest.WithBranch("feature")) @@ -272,14 +274,14 @@ func TestRepo_Push(t *testing.T) { repoProto, repopath := gittest.CreateRepository(tb, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - return New(locator, gitCmdFactory, catfileCache, repoProto), repopath, nil + return New(logger, locator, gitCmdFactory, catfileCache, repoProto), repopath, nil } setupDivergedRepo := func(tb testing.TB) (*Repo, string, []git.ConfigPair) { repoProto, repoPath := gittest.CreateRepository(tb, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, repoProto) + repo := New(logger, locator, gitCmdFactory, catfileCache, repoProto) // set up master as a diverging ref in push repo sourceMaster, err := sourceRepo.GetReference(ctx, "refs/heads/master") @@ -355,7 +357,7 @@ func TestRepo_Push(t *testing.T) { repoProto, _ := gittest.CreateRepository(tb, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - return New(locator, gitCmdFactory, catfileCache, repoProto), "", nil + return New(logger, locator, gitCmdFactory, catfileCache, repoProto), "", nil }, refspecs: []string{"refs/heads/master"}, errorMessage: `git push: exit status 128, stderr: "fatal: no path specified; see 'git help pull' for valid url syntax\n"`, @@ -366,7 +368,7 @@ func TestRepo_Push(t *testing.T) { repoProto, repoPath := gittest.CreateRepository(tb, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - return New(locator, gitCmdFactory, catfileCache, repoProto), "inmemory", []git.ConfigPair{ + return New(logger, locator, gitCmdFactory, catfileCache, repoProto), "inmemory", []git.ConfigPair{ {Key: "remote.inmemory.url", Value: repoPath}, } }, diff --git a/internal/git/localrepo/repo.go b/internal/git/localrepo/repo.go index 8be4adb99..8bbf40237 100644 --- a/internal/git/localrepo/repo.go +++ b/internal/git/localrepo/repo.go @@ -27,6 +27,7 @@ import ( // Repo represents a local Git repository. type Repo struct { storage.Repository + logger log.Logger locator storage.Locator gitCmdFactory git.CommandFactory catfileCache catfile.Cache @@ -37,9 +38,10 @@ type Repo struct { } // New creates a new Repo from its protobuf representation. -func New(locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, repo storage.Repository) *Repo { +func New(logger log.Logger, locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, repo storage.Repository) *Repo { return &Repo{ Repository: repo, + logger: logger, locator: locator, gitCmdFactory: gitCmdFactory, catfileCache: catfileCache, @@ -65,6 +67,7 @@ func (repo *Repo) Quarantine(quarantineDirectory string) (*Repo, error) { } return New( + repo.logger, repo.locator, repo.gitCmdFactory, repo.catfileCache, @@ -88,10 +91,11 @@ func NewTestRepo(tb testing.TB, cfg config.Cfg, repo storage.Repository, factory //nolint:forbidigo // We can't use the testhelper package here given that this is production code, so we can't //use `teshelper.NewDiscardingLogEntry()`. - logger := logrus.New() - logger.Out = io.Discard + logrusLogger := logrus.New() + logrusLogger.Out = io.Discard + logger := log.FromLogrusEntry(logrus.NewEntry(logrusLogger)) - gitCmdFactory, cleanup, err := git.NewExecCommandFactory(cfg, log.FromLogrusEntry(logrus.NewEntry(logger)), factoryOpts...) + gitCmdFactory, cleanup, err := git.NewExecCommandFactory(cfg, logger, factoryOpts...) tb.Cleanup(cleanup) require.NoError(tb, err) @@ -100,7 +104,7 @@ func NewTestRepo(tb testing.TB, cfg config.Cfg, repo storage.Repository, factory locator := config.NewLocator(cfg) - return New(locator, gitCmdFactory, catfileCache, repo) + return New(logger, locator, gitCmdFactory, catfileCache, repo) } // Exec creates a git command with the given args and Repo, executed in the diff --git a/internal/git/localrepo/repo_test.go b/internal/git/localrepo/repo_test.go index 425798965..41862391d 100644 --- a/internal/git/localrepo/repo_test.go +++ b/internal/git/localrepo/repo_test.go @@ -31,7 +31,7 @@ func TestRepo(t *testing.T) { gitCmdFactory := gittest.NewCommandFactory(tb, cfg) catfileCache := catfile.NewCache(cfg) tb.Cleanup(catfileCache.Stop) - return New(config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto), repoPath + return New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto), repoPath }) } @@ -48,6 +48,7 @@ func TestRepo_Quarantine(t *testing.T) { }) unquarantinedRepo := New( + testhelper.NewLogger(t), config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg), catfileCache, @@ -151,6 +152,7 @@ func TestRepo_Quarantine_nonExistentRepository(t *testing.T) { defer catfileCache.Stop() repo := New( + testhelper.NewLogger(t), config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg), catfileCache, @@ -181,7 +183,7 @@ func TestRepo_StorageTempDir(t *testing.T) { repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, repoProto) + repo := New(testhelper.NewLogger(t), locator, gitCmdFactory, catfileCache, repoProto) expected, err := locator.TempDir(cfg.Storages[0].Name) require.NoError(t, err) @@ -217,7 +219,7 @@ func TestRepo_ObjectHash(t *testing.T) { repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, }) - repo := New(locator, gitCmdFactory, catfileCache, repoProto) + repo := New(testhelper.NewLogger(t), locator, gitCmdFactory, catfileCache, repoProto) objectHash, err := repo.ObjectHash(ctx) require.NoError(t, err) diff --git a/internal/git/localrepo/testhelper_test.go b/internal/git/localrepo/testhelper_test.go index 85e5f88c8..6ee2539b8 100644 --- a/internal/git/localrepo/testhelper_test.go +++ b/internal/git/localrepo/testhelper_test.go @@ -51,5 +51,5 @@ func setupRepo(t *testing.T, opts ...setupRepoOption) (config.Cfg, *Repo, string gitCmdFactory := gittest.NewCommandFactory(t, cfg, commandFactoryOpts...) catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) - return cfg, New(config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto), repoPath + return cfg, New(testhelper.NewLogger(t), config.NewLocator(cfg), gitCmdFactory, catfileCache, repoProto), repoPath } diff --git a/internal/git/objectpool/create.go b/internal/git/objectpool/create.go index 4f4787b1a..cc8b29b27 100644 --- a/internal/git/objectpool/create.go +++ b/internal/git/objectpool/create.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -24,6 +25,7 @@ import ( // The source repository will not join the object pool. Thus, its objects won't get deduplicated. func Create( ctx context.Context, + logger log.Logger, locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, @@ -82,7 +84,7 @@ func Create( return nil, fmt.Errorf("cloning to pool: %w, stderr: %q", err, stderr.String()) } - objectPool, err := FromProto(locator, gitCmdFactory, catfileCache, txManager, housekeepingManager, proto) + objectPool, err := FromProto(logger, locator, gitCmdFactory, catfileCache, txManager, housekeepingManager, proto) if err != nil { return nil, err } diff --git a/internal/git/objectpool/create_test.go b/internal/git/objectpool/create_test.go index c962afd92..2744078fd 100644 --- a/internal/git/objectpool/create_test.go +++ b/internal/git/objectpool/create_test.go @@ -28,6 +28,7 @@ func TestCreate(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.SharedLogger(t) repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, @@ -38,15 +39,16 @@ func TestCreate(t *testing.T) { createPool := func(t *testing.T, poolProto *gitalypb.ObjectPool) (*ObjectPool, string, error) { catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(t), backchannel.NewRegistry()) + txManager := transaction.NewManager(cfg, logger, backchannel.NewRegistry()) pool, err := Create( ctx, + logger, config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg, git.WithSkipHooks()), catfileCache, txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), + housekeeping.NewManager(cfg.Prometheus, logger, txManager), poolProto, repo, ) diff --git a/internal/git/objectpool/disconnect.go b/internal/git/objectpool/disconnect.go index 1f1fcb9ea..050086832 100644 --- a/internal/git/objectpool/disconnect.go +++ b/internal/git/objectpool/disconnect.go @@ -32,7 +32,7 @@ import ( // until after the connectivity check completes. If Gitaly crashes before the backup is restored, // the repository may be in a broken state until an administrator intervenes and restores the backed // up copy of objects/info/alternates. -func Disconnect(ctx context.Context, repo *localrepo.Repo, txManager transaction.Manager) error { +func Disconnect(ctx context.Context, repo *localrepo.Repo, logger log.Logger, txManager transaction.Manager) error { repoPath, err := repo.Path() if err != nil { return err @@ -116,7 +116,7 @@ func Disconnect(ctx context.Context, repo *localrepo.Repo, txManager transaction return err } - return removeAlternatesIfOk(ctx, repo, altFile, backupFile, txManager) + return removeAlternatesIfOk(ctx, repo, altFile, backupFile, logger, txManager) } func findObjectFiles(altDir string) ([]string, error) { @@ -195,7 +195,7 @@ func newBackupFile(altFile string) (string, error) { // middle of this function, the repo is left in a broken state. We do // take care to leave a copy of the alternates file, so that it can be // manually restored by an administrator if needed. -func removeAlternatesIfOk(ctx context.Context, repo *localrepo.Repo, altFile, backupFile string, txManager transaction.Manager) error { +func removeAlternatesIfOk(ctx context.Context, repo *localrepo.Repo, altFile, backupFile string, logger log.Logger, txManager transaction.Manager) error { if err := transaction.VoteOnContext(ctx, txManager, voting.VoteFromData([]byte("disconnect alternate")), voting.Prepared); err != nil { return fmt.Errorf("preparatory vote for disconnecting alternate: %w", err) } @@ -210,8 +210,6 @@ func removeAlternatesIfOk(ctx context.Context, repo *localrepo.Repo, altFile, ba return } - logger := log.FromContext(ctx) - // If we would do a os.Rename, and then someone else comes and clobbers // our file, it's gone forever. This trick with os.Link and os.Rename // is equivalent to "cp $backupFile $altFile", meaning backupFile is @@ -219,12 +217,12 @@ func removeAlternatesIfOk(ctx context.Context, repo *localrepo.Repo, altFile, ba tmp := backupFile + ".2" if err := os.Link(backupFile, tmp); err != nil { - logger.WithError(err).Error("copy backup alternates file") + logger.WithError(err).ErrorContext(ctx, "copy backup alternates file") return } if err := os.Rename(tmp, altFile); err != nil { - logger.WithError(err).Error("restore backup alternates file") + logger.WithError(err).ErrorContext(ctx, "restore backup alternates file") } }() diff --git a/internal/git/objectpool/disconnect_test.go b/internal/git/objectpool/disconnect_test.go index 3e37ff395..ff1b0f0d0 100644 --- a/internal/git/objectpool/disconnect_test.go +++ b/internal/git/objectpool/disconnect_test.go @@ -36,6 +36,7 @@ func TestDisconnect(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.SharedLogger(t) type setupData struct { repository *localrepo.Repo @@ -52,17 +53,18 @@ func TestDisconnect(t *testing.T) { }) repo := localrepo.NewTestRepo(t, cfg, repoProto) - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(t), nil) + txManager := transaction.NewManager(cfg, logger, nil) catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) pool, err := Create( ctx, + logger, config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg, git.WithSkipHooks()), catfileCache, txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), + housekeeping.NewManager(cfg.Prometheus, logger, txManager), &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, @@ -317,7 +319,7 @@ func TestDisconnect(t *testing.T) { ctx = testhelper.MergeOutgoingMetadata(ctx, testcfg.GitalyServersMetadataFromCfg(t, cfg)) } - disconnectErr := Disconnect(ctx, setup.repository, setup.txManager) + disconnectErr := Disconnect(ctx, setup.repository, logger, setup.txManager) altInfoAfter, err := stats.AlternatesInfoForRepository(repoPath) require.NoError(t, err) @@ -367,6 +369,7 @@ func TestRemoveAlternatesIfOk(t *testing.T) { t.Run("pack files are missing", func(t *testing.T) { cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true}) repo := localrepo.NewTestRepo(t, cfg, repoProto) @@ -388,7 +391,7 @@ func TestRemoveAlternatesIfOk(t *testing.T) { // Now we try to remove the alternates file. This is expected to fail due to the // consistency check. altBackup := altPath + ".backup" - err = removeAlternatesIfOk(ctx, repo, altPath, altBackup, nil) + err = removeAlternatesIfOk(ctx, repo, altPath, altBackup, logger, nil) require.Error(t, err, "removeAlternatesIfOk should fail") require.IsType(t, &connectivityError{}, err, "error must be because of fsck") @@ -401,6 +404,7 @@ func TestRemoveAlternatesIfOk(t *testing.T) { t.Run("commit graph exists but object is missing from odb", func(t *testing.T) { cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{SkipCreationViaService: true}) repo := localrepo.NewTestRepo(t, cfg, repoProto) @@ -425,7 +429,7 @@ func TestRemoveAlternatesIfOk(t *testing.T) { // Now when we try to remove the alternates file we should notice the corruption and // abort. altBackup := altPath + ".backup" - err = removeAlternatesIfOk(ctx, repo, altPath, altBackup, nil) + err = removeAlternatesIfOk(ctx, repo, altPath, altBackup, logger, nil) require.Error(t, err, "removeAlternatesIfOk should fail") require.IsType(t, &connectivityError{}, err, "error must be because of connectivity check") connectivityErr := err.(*connectivityError) diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go index 803a13ea4..09eb8b91a 100644 --- a/internal/git/objectpool/fetch.go +++ b/internal/git/objectpool/fetch.go @@ -23,8 +23,6 @@ var objectPoolRefspec = fmt.Sprintf("+refs/*:%s/*", git.ObjectPoolRefNamespace) // FetchFromOrigin initializes the pool and fetches the objects from its origin repository func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *localrepo.Repo) error { - logger := log.FromContext(ctx) - if !o.Exists() { return structerr.NewInvalidArgument("object pool does not exist") } @@ -34,11 +32,11 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *localrepo.Repo return fmt.Errorf("computing origin repo's path: %w", err) } - if err := o.housekeepingManager.CleanStaleData(ctx, logger, o.Repo, housekeeping.DefaultStaleDataCleanup()); err != nil { + if err := o.housekeepingManager.CleanStaleData(ctx, o.Repo, housekeeping.DefaultStaleDataCleanup()); err != nil { return fmt.Errorf("cleaning stale data: %w", err) } - if err := o.logStats(ctx, logger.WithField("when", "before fetch")); err != nil { + if err := o.logStats(ctx, "before fetch"); err != nil { return fmt.Errorf("computing stats before fetch: %w", err) } @@ -100,11 +98,11 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *localrepo.Repo return fmt.Errorf("rescuing dangling objects: %w", err) } - if err := o.logStats(ctx, logger.WithField("when", "after fetch")); err != nil { + if err := o.logStats(ctx, "after fetch"); err != nil { return fmt.Errorf("computing stats after fetch: %w", err) } - if err := o.housekeepingManager.OptimizeRepository(ctx, logger, o.Repo); err != nil { + if err := o.housekeepingManager.OptimizeRepository(ctx, o.Repo); err != nil { return fmt.Errorf("optimizing pool repo: %w", err) } @@ -319,8 +317,10 @@ type referencedObjectTypes struct { Trees uint64 `json:"trees"` } -func (o *ObjectPool) logStats(ctx context.Context, logger log.Logger) error { - fields := log.Fields{} +func (o *ObjectPool) logStats(ctx context.Context, when string) error { + fields := log.Fields{ + "when": when, + } repoInfo, err := stats.RepositoryInfoForRepository(o.Repo) if err != nil { @@ -372,7 +372,7 @@ func (o *ObjectPool) logStats(ctx context.Context, logger log.Logger) error { fields["references.dangling"] = danglingTypes fields["references.normal"] = normalTypes - logger.WithFields(fields).Info("pool dangling ref stats") + o.logger.WithFields(fields).InfoContext(ctx, "pool dangling ref stats") return nil } diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go index 67ce72eca..01766ca9b 100644 --- a/internal/git/objectpool/fetch_test.go +++ b/internal/git/objectpool/fetch_test.go @@ -261,16 +261,17 @@ func TestObjectPool_logStats(t *testing.T) { for _, tc := range []struct { desc string - setup func(t *testing.T) *ObjectPool + setup func(t *testing.T, logger log.Logger) *ObjectPool expectedFields log.Fields }{ { desc: "empty object pool", - setup: func(t *testing.T) *ObjectPool { - _, pool, _ := setupObjectPool(t, ctx) + setup: func(t *testing.T, logger log.Logger) *ObjectPool { + _, pool, _ := setupObjectPool(t, ctx, withLogger(logger)) return pool }, expectedFields: log.Fields{ + "when": "now", "references.dangling": referencedObjectTypes{}, "references.normal": referencedObjectTypes{}, "repository_info": stats.RepositoryInfo{ @@ -280,12 +281,13 @@ func TestObjectPool_logStats(t *testing.T) { }, { desc: "normal reference", - setup: func(t *testing.T) *ObjectPool { - cfg, pool, _ := setupObjectPool(t, ctx) + setup: func(t *testing.T, logger log.Logger) *ObjectPool { + cfg, pool, _ := setupObjectPool(t, ctx, withLogger(logger)) gittest.WriteCommit(t, cfg, gittest.RepositoryPath(t, pool), gittest.WithBranch("main")) return pool }, expectedFields: log.Fields{ + "when": "now", "references.dangling": referencedObjectTypes{}, "references.normal": referencedObjectTypes{ Commits: 1, @@ -304,12 +306,13 @@ func TestObjectPool_logStats(t *testing.T) { }, { desc: "dangling reference", - setup: func(t *testing.T) *ObjectPool { - cfg, pool, _ := setupObjectPool(t, ctx) + setup: func(t *testing.T, logger log.Logger) *ObjectPool { + cfg, pool, _ := setupObjectPool(t, ctx, withLogger(logger)) gittest.WriteCommit(t, cfg, gittest.RepositoryPath(t, pool), gittest.WithReference("refs/dangling/commit")) return pool }, expectedFields: log.Fields{ + "when": "now", "references.dangling": referencedObjectTypes{ Commits: 1, }, @@ -330,9 +333,9 @@ func TestObjectPool_logStats(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { logger := testhelper.NewLogger(t) hook := testhelper.AddLoggerHook(logger) - pool := tc.setup(t) + pool := tc.setup(t, logger) - require.NoError(t, pool.logStats(ctx, logger)) + require.NoError(t, pool.logStats(ctx, "now")) logEntries := hook.AllEntries() require.Len(t, logEntries, 1) diff --git a/internal/git/objectpool/pool.go b/internal/git/objectpool/pool.go index c924014e9..5749d727d 100644 --- a/internal/git/objectpool/pool.go +++ b/internal/git/objectpool/pool.go @@ -17,6 +17,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -38,6 +39,7 @@ var ( type ObjectPool struct { *localrepo.Repo + logger log.Logger locator storage.Locator gitCmdFactory git.CommandFactory txManager transaction.Manager @@ -47,6 +49,7 @@ type ObjectPool struct { // FromProto returns an object pool object from its Protobuf representation. This function verifies // that the object pool exists and is a valid pool repository. func FromProto( + logger log.Logger, locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, @@ -74,7 +77,8 @@ func FromProto( } pool := &ObjectPool{ - Repo: localrepo.New(locator, gitCmdFactory, catfileCache, proto.GetRepository()), + Repo: localrepo.New(logger, locator, gitCmdFactory, catfileCache, proto.GetRepository()), + logger: logger, locator: locator, gitCmdFactory: gitCmdFactory, txManager: txManager, @@ -141,6 +145,7 @@ func (o *ObjectPool) Remove(ctx context.Context) (err error) { // FromRepo returns an instance of ObjectPool that the repository points to func FromRepo( + logger log.Logger, locator storage.Locator, gitCmdFactory git.CommandFactory, catfileCache catfile.Cache, @@ -183,7 +188,7 @@ func FromRepo( return nil, ErrInvalidPoolRepository } - return FromProto(locator, gitCmdFactory, catfileCache, txManager, housekeepingManager, objectPoolProto) + return FromProto(logger, locator, gitCmdFactory, catfileCache, txManager, housekeepingManager, objectPoolProto) } // getAlternateObjectDir returns the entry in the objects/info/attributes file if it exists diff --git a/internal/git/objectpool/pool_test.go b/internal/git/objectpool/pool_test.go index 8e8a649d3..9426ed135 100644 --- a/internal/git/objectpool/pool_test.go +++ b/internal/git/objectpool/pool_test.go @@ -20,18 +20,19 @@ func TestFromProto(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) t.Run("successful", func(t *testing.T) { cfg, pool, _ := setupObjectPool(t, ctx) locator := config.NewLocator(cfg) - _, err := FromProto(locator, nil, nil, nil, nil, pool.ToProto()) + _, err := FromProto(logger, locator, nil, nil, nil, nil, pool.ToProto()) require.NoError(t, err) }) t.Run("nonexistent", func(t *testing.T) { - _, err := FromProto(locator, nil, nil, nil, nil, &gitalypb.ObjectPool{ + _, err := FromProto(logger, locator, nil, nil, nil, nil, &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, RelativePath: gittest.NewObjectPoolName(t), @@ -41,7 +42,7 @@ func TestFromProto(t *testing.T) { }) t.Run("unknown storage", func(t *testing.T) { - _, err := FromProto(locator, nil, nil, nil, nil, &gitalypb.ObjectPool{ + _, err := FromProto(logger, locator, nil, nil, nil, nil, &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: "mepmep", RelativePath: gittest.NewObjectPoolName(t), @@ -57,11 +58,12 @@ func TestFromRepo_successful(t *testing.T) { ctx := testhelper.Context(t) cfg, pool, repo := setupObjectPool(t, ctx) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) require.NoError(t, pool.Link(ctx, repo)) - poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, nil, repo) + poolFromRepo, err := FromRepo(logger, locator, pool.gitCmdFactory, nil, nil, nil, repo) require.NoError(t, err) require.Equal(t, pool.GetRelativePath(), poolFromRepo.GetRelativePath()) require.Equal(t, pool.GetStorageName(), poolFromRepo.GetStorageName()) @@ -74,9 +76,10 @@ func TestFromRepo_failures(t *testing.T) { t.Run("without alternates file", func(t *testing.T) { cfg, pool, repo := setupObjectPool(t, ctx) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) - poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, nil, repo) + poolFromRepo, err := FromRepo(logger, locator, pool.gitCmdFactory, nil, nil, nil, repo) require.Equal(t, ErrAlternateObjectDirNotExist, err) require.Nil(t, poolFromRepo) }) @@ -104,6 +107,7 @@ func TestFromRepo_failures(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { cfg, pool, repo := setupObjectPool(t, ctx) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) repoPath, err := repo.Path() require.NoError(t, err) @@ -111,7 +115,7 @@ func TestFromRepo_failures(t *testing.T) { require.NoError(t, os.MkdirAll(filepath.Join(repoPath, "objects", "info"), perm.SharedDir)) alternateFilePath := filepath.Join(repoPath, "objects", "info", "alternates") require.NoError(t, os.WriteFile(alternateFilePath, tc.fileContent, perm.SharedFile)) - poolFromRepo, err := FromRepo(locator, pool.gitCmdFactory, nil, nil, nil, repo) + poolFromRepo, err := FromRepo(logger, locator, pool.gitCmdFactory, nil, nil, nil, repo) require.Equal(t, tc.expectedErr, err) require.Nil(t, poolFromRepo) diff --git a/internal/git/objectpool/testhelper_test.go b/internal/git/objectpool/testhelper_test.go index d41590162..3d861d063 100644 --- a/internal/git/objectpool/testhelper_test.go +++ b/internal/git/objectpool/testhelper_test.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" "gitlab.com/gitlab-org/gitaly/v16/internal/grpc/backchannel" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" @@ -22,9 +23,29 @@ func TestMain(m *testing.M) { testhelper.Run(m) } -func setupObjectPool(t *testing.T, ctx context.Context) (config.Cfg, *ObjectPool, *localrepo.Repo) { +type setupObjectPoolConfig struct { + logger log.Logger +} + +type setupObjectPoolOption func(*setupObjectPoolConfig) + +func withLogger(logger log.Logger) setupObjectPoolOption { + return func(cfg *setupObjectPoolConfig) { + cfg.logger = logger + } +} + +func setupObjectPool(t *testing.T, ctx context.Context, opts ...setupObjectPoolOption) (config.Cfg, *ObjectPool, *localrepo.Repo) { t.Helper() + var setupCfg setupObjectPoolConfig + for _, opt := range opts { + opt(&setupCfg) + } + if setupCfg.logger == nil { + setupCfg.logger = testhelper.NewLogger(t) + } + cfg := testcfg.Build(t) repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ SkipCreationViaService: true, @@ -35,15 +56,16 @@ func setupObjectPool(t *testing.T, ctx context.Context) (config.Cfg, *ObjectPool catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(t), backchannel.NewRegistry()) + txManager := transaction.NewManager(cfg, setupCfg.logger, backchannel.NewRegistry()) pool, err := Create( ctx, + setupCfg.logger, config.NewLocator(cfg), gitCommandFactory, catfileCache, txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), + housekeeping.NewManager(cfg.Prometheus, setupCfg.logger, txManager), &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: repo.GetStorageName(), diff --git a/internal/git/quarantine/quarantine.go b/internal/git/quarantine/quarantine.go index 14811a93a..d726822e3 100644 --- a/internal/git/quarantine/quarantine.go +++ b/internal/git/quarantine/quarantine.go @@ -11,6 +11,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tempdir" @@ -33,14 +34,14 @@ type Dir struct { // New creates a new quarantine directory and returns the directory. The repository is cleaned // up when the user invokes the Migrate() functionality on the Dir. -func New(ctx context.Context, repo *gitalypb.Repository, locator storage.Locator) (*Dir, error) { +func New(ctx context.Context, repo *gitalypb.Repository, logger log.Logger, locator storage.Locator) (*Dir, error) { repoPath, err := locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return nil, structerr.NewInternal("getting repo path: %w", err) } quarantineDir, err := tempdir.NewWithPrefix(ctx, repo.GetStorageName(), - storage.QuarantineDirectoryPrefix(repo), locator) + storage.QuarantineDirectoryPrefix(repo), logger, locator) if err != nil { return nil, fmt.Errorf("creating quarantine: %w", err) } diff --git a/internal/git/quarantine/quarantine_ext_test.go b/internal/git/quarantine/quarantine_ext_test.go index b3aff0548..6da12de47 100644 --- a/internal/git/quarantine/quarantine_ext_test.go +++ b/internal/git/quarantine/quarantine_ext_test.go @@ -28,7 +28,7 @@ func TestQuarantine_localrepo(t *testing.T) { locator := config.NewLocator(cfg) - quarantine, err := quarantine.New(ctx, repoProto, locator) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.NewLogger(t), locator) require.NoError(t, err) quarantined := localrepo.NewTestRepo(t, cfg, quarantine.QuarantinedRepo()) diff --git a/internal/git/quarantine/quarantine_test.go b/internal/git/quarantine/quarantine_test.go index fff85d10c..99fe12d78 100644 --- a/internal/git/quarantine/quarantine_test.go +++ b/internal/git/quarantine/quarantine_test.go @@ -47,9 +47,10 @@ func TestQuarantine_lifecycle(t *testing.T) { SkipCreationViaService: true, }) locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) t.Run("quarantine directory gets created", func(t *testing.T) { - quarantine, err := New(ctx, repo, locator) + quarantine, err := New(ctx, repo, logger, locator) require.NoError(t, err) relativeQuarantinePath, err := filepath.Rel(repoPath, quarantine.dir.Path()) @@ -74,7 +75,7 @@ func TestQuarantine_lifecycle(t *testing.T) { t.Run("context cancellation cleans up quarantine directory", func(t *testing.T) { ctx, cancel := context.WithCancel(ctx) - quarantine, err := New(ctx, repo, locator) + quarantine, err := New(ctx, repo, logger, locator) require.NoError(t, err) require.DirExists(t, quarantine.dir.Path()) @@ -89,6 +90,7 @@ func TestQuarantine_Migrate(t *testing.T) { cfg := testcfg.Build(t) locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) t.Run("no changes", func(t *testing.T) { ctx := testhelper.Context(t) @@ -100,7 +102,7 @@ func TestQuarantine_Migrate(t *testing.T) { oldContents := listEntries(t, repoPath) - quarantine, err := New(ctx, repo, locator) + quarantine, err := New(ctx, repo, logger, locator) require.NoError(t, err) require.NoError(t, quarantine.Migrate()) @@ -118,7 +120,7 @@ func TestQuarantine_Migrate(t *testing.T) { oldContents := listEntries(t, repoPath) require.NotContains(t, oldContents, "objects/file") - quarantine, err := New(ctx, repo, locator) + quarantine, err := New(ctx, repo, logger, locator) require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(quarantine.dir.Path(), "file"), []byte("foobar"), perm.PublicFile)) @@ -141,7 +143,7 @@ func TestQuarantine_Migrate(t *testing.T) { repoContents := listEntries(t, repoPath) require.NotContains(t, repoContents, "objects/file") - quarantine, err := New(ctx, repo, locator) + quarantine, err := New(ctx, repo, logger, locator) require.NoError(t, err) require.Empty(t, listEntries(t, quarantine.dir.Path())) @@ -149,7 +151,7 @@ func TestQuarantine_Migrate(t *testing.T) { // Quarantine the already quarantined repository and write the object there. We expect the // object to be migrated from the second level quarantine to the first level quarantine. The // main repository should stay untouched. - recursiveQuarantine, err := New(ctx, quarantine.QuarantinedRepo(), locator) + recursiveQuarantine, err := New(ctx, quarantine.QuarantinedRepo(), logger, locator) require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(recursiveQuarantine.dir.Path(), "file"), []byte("foobar"), perm.PublicFile)) diff --git a/internal/gitaly/hook/postreceive_test.go b/internal/gitaly/hook/postreceive_test.go index f5b40ffac..808035831 100644 --- a/internal/gitaly/hook/postreceive_test.go +++ b/internal/gitaly/hook/postreceive_test.go @@ -410,7 +410,7 @@ func TestPostReceive_quarantine(t *testing.T) { SkipCreationViaService: true, }) - quarantine, err := quarantine.New(ctx, repoProto, config.NewLocator(cfg)) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.SharedLogger(t), config.NewLocator(cfg)) require.NoError(t, err) quarantinedRepo := localrepo.NewTestRepo(t, cfg, quarantine.QuarantinedRepo()) diff --git a/internal/gitaly/hook/prereceive_test.go b/internal/gitaly/hook/prereceive_test.go index e02599fce..2f276448b 100644 --- a/internal/gitaly/hook/prereceive_test.go +++ b/internal/gitaly/hook/prereceive_test.go @@ -219,7 +219,7 @@ func TestPrereceive_quarantine(t *testing.T) { SkipCreationViaService: true, }) - quarantine, err := quarantine.New(ctx, repoProto, config.NewLocator(cfg)) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.SharedLogger(t), config.NewLocator(cfg)) require.NoError(t, err) quarantinedRepo := localrepo.NewTestRepo(t, cfg, quarantine.QuarantinedRepo()) diff --git a/internal/gitaly/hook/update_test.go b/internal/gitaly/hook/update_test.go index 9342a2f6e..8d8ab8e30 100644 --- a/internal/gitaly/hook/update_test.go +++ b/internal/gitaly/hook/update_test.go @@ -249,7 +249,7 @@ func TestUpdate_quarantine(t *testing.T) { SkipCreationViaService: true, }) - quarantine, err := quarantine.New(ctx, repoProto, config.NewLocator(cfg)) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.SharedLogger(t), config.NewLocator(cfg)) require.NoError(t, err) quarantinedRepo := localrepo.NewTestRepo(t, cfg, quarantine.QuarantinedRepo()) diff --git a/internal/gitaly/hook/updateref/update_with_hooks.go b/internal/gitaly/hook/updateref/update_with_hooks.go index 680c3ef91..a1c2771a5 100644 --- a/internal/gitaly/hook/updateref/update_with_hooks.go +++ b/internal/gitaly/hook/updateref/update_with_hooks.go @@ -316,5 +316,5 @@ func (u *UpdaterWithHooks) UpdateReference( } func (u *UpdaterWithHooks) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(u.locator, u.gitCmdFactory, u.catfileCache, repo) + return localrepo.New(u.logger, u.locator, u.gitCmdFactory, u.catfileCache, repo) } diff --git a/internal/gitaly/hook/updateref/update_with_hooks_test.go b/internal/gitaly/hook/updateref/update_with_hooks_test.go index 2443cb2e8..36bfbe658 100644 --- a/internal/gitaly/hook/updateref/update_with_hooks_test.go +++ b/internal/gitaly/hook/updateref/update_with_hooks_test.go @@ -316,7 +316,7 @@ func TestUpdaterWithHooks_quarantine(t *testing.T) { unquarantinedRepo := localrepo.NewTestRepo(t, cfg, repoProto) - quarantine, err := quarantine.New(ctx, repoProto, locator) + quarantine, err := quarantine.New(ctx, repoProto, testhelper.NewLogger(t), locator) require.NoError(t, err) quarantinedRepo := localrepo.NewTestRepo(t, cfg, quarantine.QuarantinedRepo()) blobID, err := quarantinedRepo.WriteBlob(ctx, strings.NewReader("1834298812398123"), localrepo.WriteBlobConfig{}) diff --git a/internal/gitaly/linguist/linguist_test.go b/internal/gitaly/linguist/linguist_test.go index be9d2f424..73a8dcece 100644 --- a/internal/gitaly/linguist/linguist_test.go +++ b/internal/gitaly/linguist/linguist_test.go @@ -543,6 +543,7 @@ func TestInstance_Stats_failureGitattributes(t *testing.T) { cfg := testcfg.Build(t) ctx := testhelper.Context(t) locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) @@ -565,9 +566,9 @@ func TestInstance_Stats_failureGitattributes(t *testing.T) { gittest.TreeEntry{Path: ".gitattributes", Mode: "100644", Content: "*.rb -linguist-vendored"}, )) - repo := localrepo.New(locator, gitCmdFactory, catfileCache, repoProto) + repo := localrepo.New(logger, locator, gitCmdFactory, catfileCache, repoProto) - linguist := New(cfg, testhelper.NewLogger(t), catfileCache, repo) + linguist := New(cfg, logger, catfileCache, repo) _, err := linguist.Stats(ctx, commitID) expectedErr := `linguist object iterator: ls-tree skip: new file instance: checking attribute:` diff --git a/internal/gitaly/maintenance/optimize_test.go b/internal/gitaly/maintenance/optimize_test.go index 26d63479f..2b0d6cd2e 100644 --- a/internal/gitaly/maintenance/optimize_test.go +++ b/internal/gitaly/maintenance/optimize_test.go @@ -34,9 +34,9 @@ func (mo *mockOptimizer) OptimizeRepository(ctx context.Context, logger log.Logg catfileCache := catfile.NewCache(mo.cfg) mo.t.Cleanup(catfileCache.Stop) txManager := transaction.NewManager(mo.cfg, logger, backchannel.NewRegistry()) - housekeepingManager := housekeeping.NewManager(mo.cfg.Prometheus, txManager) + housekeepingManager := housekeeping.NewManager(mo.cfg.Prometheus, logger, txManager) - return housekeepingManager.OptimizeRepository(ctx, logger, localrepo.New(l, gitCmdFactory, catfileCache, repository)) + return housekeepingManager.OptimizeRepository(ctx, localrepo.New(logger, l, gitCmdFactory, catfileCache, repository)) } func TestOptimizeReposRandomly(t *testing.T) { diff --git a/internal/gitaly/repoutil/create.go b/internal/gitaly/repoutil/create.go index 1b6101f98..83eb2d847 100644 --- a/internal/gitaly/repoutil/create.go +++ b/internal/gitaly/repoutil/create.go @@ -15,6 +15,7 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage/counter" "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction" + "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/safe" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/tempdir" @@ -64,6 +65,7 @@ func WithSkipInit() CreateOption { // The repository can optionally be seeded with contents func Create( ctx context.Context, + logger log.Logger, locator storage.Locator, gitCmdFactory git.CommandFactory, txManager transaction.Manager, @@ -83,7 +85,7 @@ func Create( return structerr.NewAlreadyExists("repository exists already") } - newRepo, newRepoDir, err := tempdir.NewRepository(ctx, repository.GetStorageName(), locator) + newRepo, newRepoDir, err := tempdir.NewRepository(ctx, repository.GetStorageName(), logger, locator) if err != nil { return fmt.Errorf("creating temporary repository: %w", err) } @@ -203,7 +205,7 @@ func Create( // This sequence guarantees that the change is atomic and can trivially be rolled // back in case we fail to either lock the repository or reach quorum in the initial // vote. - unlock, err := Lock(ctx, locator, repository) + unlock, err := Lock(ctx, logger, locator, repository) if err != nil { return fmt.Errorf("locking repository: %w", err) } diff --git a/internal/gitaly/repoutil/create_test.go b/internal/gitaly/repoutil/create_test.go index 08d2d43bc..e449ff448 100644 --- a/internal/gitaly/repoutil/create_test.go +++ b/internal/gitaly/repoutil/create_test.go @@ -35,6 +35,7 @@ func TestCreate(t *testing.T) { cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) txManager := &transaction.MockManager{} locator := config.NewLocator(cfg) gitCmdFactory := gittest.NewCommandFactory(t, cfg) @@ -344,7 +345,7 @@ func TestCreate(t *testing.T) { } var tempRepo *gitalypb.Repository - require.Equal(t, tc.expectedErr, Create(ctx, locator, gitCmdFactory, txManager, repoCounter, repo, func(tr *gitalypb.Repository) error { + require.Equal(t, tc.expectedErr, Create(ctx, logger, locator, gitCmdFactory, txManager, repoCounter, repo, func(tr *gitalypb.Repository) error { tempRepo = tr // The temporary repository must have been created in Gitaly's diff --git a/internal/gitaly/repoutil/custom_hooks.go b/internal/gitaly/repoutil/custom_hooks.go index b21634aad..f81d827e8 100644 --- a/internal/gitaly/repoutil/custom_hooks.go +++ b/internal/gitaly/repoutil/custom_hooks.go @@ -125,7 +125,7 @@ func SetCustomHooks( // future modifications to the repository's hooks will be prevented. If // this occurs, the `.lock` file will have to be manually removed. if err := hooksLock.Unlock(); err != nil { - log.FromContext(ctx).WithError(err).Error("failed to unlock hooks") + logger.WithError(err).ErrorContext(ctx, "failed to unlock hooks") } }() @@ -133,14 +133,14 @@ func SetCustomHooks( // temporarily store the current repository hooks. This enables "atomic" // directory swapping by acting as an intermediary storage location between // moves. - tmpDir, err := tempdir.NewWithoutContext(repo.GetStorageName(), locator) + tmpDir, err := tempdir.NewWithoutContext(repo.GetStorageName(), logger, locator) if err != nil { return fmt.Errorf("creating temp directory: %w", err) } defer func() { if err := os.RemoveAll(tmpDir.Path()); err != nil { - log.FromContext(ctx).WithError(err).Warn("failed to remove temporary directory") + logger.WithError(err).WarnContext(ctx, "failed to remove temporary directory") } }() diff --git a/internal/gitaly/repoutil/lock.go b/internal/gitaly/repoutil/lock.go index f0c13d29a..131ed7a48 100644 --- a/internal/gitaly/repoutil/lock.go +++ b/internal/gitaly/repoutil/lock.go @@ -19,7 +19,7 @@ import ( // // Returns the error safe.ErrFileAlreadyLocked if the repository is already // locked. -func Lock(ctx context.Context, locator storage.Locator, repository storage.Repository) (func(), error) { +func Lock(ctx context.Context, logger log.Logger, locator storage.Locator, repository storage.Repository) (func(), error) { path, err := locator.GetRepoPath(repository, storage.WithRepositoryVerificationSkipped()) if err != nil { return nil, err @@ -41,7 +41,7 @@ func Lock(ctx context.Context, locator storage.Locator, repository storage.Repos unlock := func() { if err := locker.Close(); err != nil { - log.FromContext(ctx).WithError(err).Error("closing repository locker failed") + logger.WithError(err).ErrorContext(ctx, "closing repository locker failed") } } diff --git a/internal/gitaly/repoutil/lock_test.go b/internal/gitaly/repoutil/lock_test.go index 956b50e58..5b7b4a1e3 100644 --- a/internal/gitaly/repoutil/lock_test.go +++ b/internal/gitaly/repoutil/lock_test.go @@ -18,6 +18,7 @@ func TestLock(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) repo := &gitalypb.Repository{ @@ -28,12 +29,12 @@ func TestLock(t *testing.T) { repoPath, err := locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) require.NoError(t, err) - unlock, err := Lock(ctx, locator, repo) + unlock, err := Lock(ctx, logger, locator, repo) require.NoError(t, err) require.FileExists(t, repoPath+".lock") - _, err = Lock(ctx, locator, repo) + _, err = Lock(ctx, logger, locator, repo) require.ErrorIs(t, err, safe.ErrFileAlreadyLocked) unlock() diff --git a/internal/gitaly/repoutil/remove.go b/internal/gitaly/repoutil/remove.go index 721be1ff4..487f55f0d 100644 --- a/internal/gitaly/repoutil/remove.go +++ b/internal/gitaly/repoutil/remove.go @@ -21,12 +21,13 @@ import ( // Remove will remove a repository in a race-free way with proper transactional semantics. func Remove( ctx context.Context, + logger log.Logger, locator storage.Locator, txManager transaction.Manager, repoCounter *counter.RepositoryCounter, repository storage.Repository, ) error { - if err := remove(ctx, locator, txManager, repository, os.RemoveAll); err != nil { + if err := remove(ctx, logger, locator, txManager, repository, os.RemoveAll); err != nil { return err } @@ -37,6 +38,7 @@ func Remove( func remove( ctx context.Context, + logger log.Logger, locator storage.Locator, txManager transaction.Manager, repository storage.Repository, @@ -70,7 +72,7 @@ func remove( // Lock the repository such that it cannot be created or removed by any concurrent // RPC call. - unlock, err := Lock(ctx, locator, repository) + unlock, err := Lock(ctx, logger, locator, repository) if err != nil { if errors.Is(err, safe.ErrFileAlreadyLocked) { return structerr.NewFailedPrecondition("repository is already locked") @@ -100,7 +102,7 @@ func remove( defer func() { if err := removeAll(destDir); err != nil { - log.FromContext(ctx).WithError(err).Error("failed removing repository from temporary directory") + logger.WithError(err).ErrorContext(ctx, "failed removing repository from temporary directory") } }() diff --git a/internal/gitaly/repoutil/remove_test.go b/internal/gitaly/repoutil/remove_test.go index ae46b4d25..087cdc8a3 100644 --- a/internal/gitaly/repoutil/remove_test.go +++ b/internal/gitaly/repoutil/remove_test.go @@ -69,6 +69,7 @@ func TestRemove(t *testing.T) { require.NoError(t, remove( ctx, + testhelper.SharedLogger(t), config.NewLocator(cfg), transaction.NewTrackingManager(), repo, @@ -92,6 +93,7 @@ func TestRemove(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.SharedLogger(t) locator := config.NewLocator(cfg) txManager := transaction.NewTrackingManager() repoCounter := counter.NewRepositoryCounter(cfg.Storages) @@ -102,7 +104,7 @@ func TestRemove(t *testing.T) { require.DirExists(t, repoPath) } - err := Remove(ctx, locator, txManager, repoCounter, repo) + err := Remove(ctx, logger, locator, txManager, repoCounter, repo) if tc.expectedErr != nil { require.Equal(t, tc.expectedErr, err) diff --git a/internal/gitaly/service/blob/blobs_test.go b/internal/gitaly/service/blob/blobs_test.go index 78dcd2bf5..f1ccd5e5e 100644 --- a/internal/gitaly/service/blob/blobs_test.go +++ b/internal/gitaly/service/blob/blobs_test.go @@ -311,7 +311,7 @@ func TestListAllBlobs(t *testing.T) { repo, _, _ := setupRepoWithLFS(t, ctx, cfg) - quarantine, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), config.NewLocator(cfg)) + quarantine, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), testhelper.NewLogger(t), config.NewLocator(cfg)) require.NoError(t, err) // quarantine.New in Gitaly would receive an already rewritten repository. Gitaly would then calculate diff --git a/internal/gitaly/service/blob/lfs_pointers_test.go b/internal/gitaly/service/blob/lfs_pointers_test.go index adc7dff08..a61336005 100644 --- a/internal/gitaly/service/blob/lfs_pointers_test.go +++ b/internal/gitaly/service/blob/lfs_pointers_test.go @@ -217,7 +217,7 @@ size 12345` setup: func(t *testing.T) setupData { repo, _, _ := setupRepoWithLFS(t, ctx, cfg) - quarantineDir, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), config.NewLocator(cfg)) + quarantineDir, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), testhelper.NewLogger(t), config.NewLocator(cfg)) require.NoError(t, err) repo.GitObjectDirectory = quarantineDir.QuarantinedRepo().GitObjectDirectory @@ -240,7 +240,7 @@ size 12345` // this case, LFS pointer checks may want to inspect all newly // pushed objects, denoted by a repository proto message which only // has its object directory set to the quarantine directory. - quarantineDir, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), config.NewLocator(cfg)) + quarantineDir, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), testhelper.NewLogger(t), config.NewLocator(cfg)) require.NoError(t, err) // Note that we need to continue using the non-rewritten repository diff --git a/internal/gitaly/service/blob/server.go b/internal/gitaly/service/blob/server.go index 05b5b5ced..ab4e3f4a3 100644 --- a/internal/gitaly/service/blob/server.go +++ b/internal/gitaly/service/blob/server.go @@ -29,5 +29,5 @@ func NewServer(deps *service.Dependencies) gitalypb.BlobServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/cleanup/cleaner.go b/internal/gitaly/service/cleanup/cleaner.go index 81ae71a29..574933f7f 100644 --- a/internal/gitaly/service/cleanup/cleaner.go +++ b/internal/gitaly/service/cleanup/cleaner.go @@ -24,6 +24,7 @@ type forEachFunc func(ctx context.Context, oldOID, newOID string, isInternalRef type cleaner struct { ctx context.Context forEach forEachFunc + logger log.Logger // Map of SHA -> reference names table map[string][]git.ReferenceName @@ -42,7 +43,7 @@ func newCleaner(ctx context.Context, logger log.Logger, repo git.RepositoryExecu return nil, err } - return &cleaner{ctx: ctx, table: table, repo: repo, forEach: forEach}, nil + return &cleaner{ctx: ctx, logger: logger, table: table, repo: repo, forEach: forEach}, nil } // applyObjectMap processes an object map file generated by git filter-repo, or @@ -119,10 +120,10 @@ func (c *cleaner) processEntry(ctx context.Context, updater *updateref.Updater, return nil } - log.FromContext(ctx).WithFields(log.Fields{ + c.logger.WithFields(log.Fields{ "sha": oldSHA, "refs": refs, - }).Info("removing internal references") + }).InfoContext(ctx, "removing internal references") // Remove the internal refs pointing to oldSHA for _, ref := range refs { diff --git a/internal/gitaly/service/cleanup/server.go b/internal/gitaly/service/cleanup/server.go index bccf6700b..3fd2d06f6 100644 --- a/internal/gitaly/service/cleanup/server.go +++ b/internal/gitaly/service/cleanup/server.go @@ -29,5 +29,5 @@ func NewServer(deps *service.Dependencies) gitalypb.CleanupServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/commit/server.go b/internal/gitaly/service/commit/server.go index d6d69e400..d56c1ec46 100644 --- a/internal/gitaly/service/commit/server.go +++ b/internal/gitaly/service/commit/server.go @@ -32,5 +32,5 @@ func NewServer(deps *service.Dependencies) gitalypb.CommitServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/conflicts/server.go b/internal/gitaly/service/conflicts/server.go index 17cfa52f8..745380d78 100644 --- a/internal/gitaly/service/conflicts/server.go +++ b/internal/gitaly/service/conflicts/server.go @@ -42,13 +42,13 @@ func NewServer(deps *service.Dependencies) gitalypb.ConflictsServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } func (s *server) quarantinedRepo( ctx context.Context, repo *gitalypb.Repository, ) (*quarantine.Dir, *localrepo.Repo, error) { - quarantineDir, err := quarantine.New(ctx, repo, s.locator) + quarantineDir, err := quarantine.New(ctx, repo, s.logger, s.locator) if err != nil { return nil, nil, structerr.NewInternal("creating object quarantine: %w", err) } diff --git a/internal/gitaly/service/diff/server.go b/internal/gitaly/service/diff/server.go index 39ad05449..d92cca842 100644 --- a/internal/gitaly/service/diff/server.go +++ b/internal/gitaly/service/diff/server.go @@ -33,5 +33,5 @@ func NewServer(deps *service.Dependencies) gitalypb.DiffServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/objectpool/alternates.go b/internal/gitaly/service/objectpool/alternates.go index 21afa428a..ba85c720d 100644 --- a/internal/gitaly/service/objectpool/alternates.go +++ b/internal/gitaly/service/objectpool/alternates.go @@ -22,7 +22,7 @@ func (s *server) DisconnectGitAlternates(ctx context.Context, req *gitalypb.Disc repo := s.localrepo(repository) - if err := objectpool.Disconnect(ctx, repo, s.txManager); err != nil { + if err := objectpool.Disconnect(ctx, repo, s.logger, s.txManager); err != nil { return nil, structerr.NewInternal("%w", err) } diff --git a/internal/gitaly/service/objectpool/create.go b/internal/gitaly/service/objectpool/create.go index fd994f15a..c60398285 100644 --- a/internal/gitaly/service/objectpool/create.go +++ b/internal/gitaly/service/objectpool/create.go @@ -29,9 +29,10 @@ func (s *server) CreateObjectPool(ctx context.Context, in *gitalypb.CreateObject return nil, errInvalidPoolDir } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, poolRepo, func(poolRepo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, poolRepo, func(poolRepo *gitalypb.Repository) error { if _, err := objectpool.Create( ctx, + s.logger, s.locator, s.gitCmdFactory, s.catfileCache, diff --git a/internal/gitaly/service/objectpool/create_test.go b/internal/gitaly/service/objectpool/create_test.go index abbaefb99..f7449c8a8 100644 --- a/internal/gitaly/service/objectpool/create_test.go +++ b/internal/gitaly/service/objectpool/create_test.go @@ -52,11 +52,12 @@ func TestCreate(t *testing.T) { require.NoError(t, err) pool, err := objectpool.FromProto( + logger, config.NewLocator(cfg), gittest.NewCommandFactory(t, cfg), catfileCache, txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), + housekeeping.NewManager(cfg.Prometheus, logger, txManager), &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, diff --git a/internal/gitaly/service/objectpool/fetch_into_object_pool.go b/internal/gitaly/service/objectpool/fetch_into_object_pool.go index 1b279a90f..3e65497d8 100644 --- a/internal/gitaly/service/objectpool/fetch_into_object_pool.go +++ b/internal/gitaly/service/objectpool/fetch_into_object_pool.go @@ -15,7 +15,7 @@ func (s *server) FetchIntoObjectPool(ctx context.Context, req *gitalypb.FetchInt return nil, structerr.NewInvalidArgument("%w", err) } - objectPool, err := objectpool.FromProto(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, req.GetObjectPool()) + objectPool, err := objectpool.FromProto(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, req.GetObjectPool()) if err != nil { return nil, structerr.NewInvalidArgument("object pool invalid: %w", err) } diff --git a/internal/gitaly/service/objectpool/get.go b/internal/gitaly/service/objectpool/get.go index 73bf2ae5d..4308690ea 100644 --- a/internal/gitaly/service/objectpool/get.go +++ b/internal/gitaly/service/objectpool/get.go @@ -16,7 +16,7 @@ func (s *server) GetObjectPool(ctx context.Context, in *gitalypb.GetObjectPoolRe repo := s.localrepo(repository) - objectPool, err := objectpool.FromRepo(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, repo) + objectPool, err := objectpool.FromRepo(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, repo) if err != nil { s.logger. WithError(err). diff --git a/internal/gitaly/service/objectpool/server.go b/internal/gitaly/service/objectpool/server.go index 54a1a9cb2..8aeb77bac 100644 --- a/internal/gitaly/service/objectpool/server.go +++ b/internal/gitaly/service/objectpool/server.go @@ -38,5 +38,5 @@ func NewServer(deps *service.Dependencies) gitalypb.ObjectPoolServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/objectpool/testhelper_test.go b/internal/gitaly/service/objectpool/testhelper_test.go index 158d2825d..d1a083f45 100644 --- a/internal/gitaly/service/objectpool/testhelper_test.go +++ b/internal/gitaly/service/objectpool/testhelper_test.go @@ -81,16 +81,18 @@ func createObjectPool( poolProto, poolProtoPath := gittest.CreateObjectPool(tb, ctx, cfg, source) - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(tb), nil) + logger := testhelper.SharedLogger(tb) + txManager := transaction.NewManager(cfg, logger, nil) catfileCache := catfile.NewCache(cfg) tb.Cleanup(catfileCache.Stop) pool, err := objectpool.FromProto( + logger, config.NewLocator(cfg), gittest.NewCommandFactory(tb, cfg), catfileCache, txManager, - housekeeping.NewManager(cfg.Prometheus, txManager), + housekeeping.NewManager(cfg.Prometheus, logger, txManager), &gitalypb.ObjectPool{ Repository: &gitalypb.Repository{ StorageName: cfg.Storages[0].Name, diff --git a/internal/gitaly/service/objectpool/util.go b/internal/gitaly/service/objectpool/util.go index da6844928..0de4bc753 100644 --- a/internal/gitaly/service/objectpool/util.go +++ b/internal/gitaly/service/objectpool/util.go @@ -32,7 +32,7 @@ func ExtractPool(req PoolRequest) (*gitalypb.Repository, error) { } func (s *server) poolForRequest(req PoolRequest) (*objectpool.ObjectPool, error) { - pool, err := objectpool.FromProto(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, req.GetObjectPool()) + pool, err := objectpool.FromProto(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, req.GetObjectPool()) if err != nil { if err == objectpool.ErrInvalidPoolDir { return nil, errInvalidPoolDir diff --git a/internal/gitaly/service/operations/server.go b/internal/gitaly/service/operations/server.go index 21c97cd94..9673bb2da 100644 --- a/internal/gitaly/service/operations/server.go +++ b/internal/gitaly/service/operations/server.go @@ -48,13 +48,13 @@ func NewServer(deps *service.Dependencies) *Server { } func (s *Server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } func (s *Server) quarantinedRepo( ctx context.Context, repo *gitalypb.Repository, ) (*quarantine.Dir, *localrepo.Repo, error) { - quarantineDir, err := quarantine.New(ctx, repo, s.locator) + quarantineDir, err := quarantine.New(ctx, repo, s.logger, s.locator) if err != nil { return nil, nil, structerr.NewInternal("creating object quarantine: %w", err) } diff --git a/internal/gitaly/service/ref/server.go b/internal/gitaly/service/ref/server.go index ca8816d23..8564dcd32 100644 --- a/internal/gitaly/service/ref/server.go +++ b/internal/gitaly/service/ref/server.go @@ -32,5 +32,5 @@ func NewServer(deps *service.Dependencies) gitalypb.RefServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/remote/server.go b/internal/gitaly/service/remote/server.go index c2ee956e6..b86946436 100644 --- a/internal/gitaly/service/remote/server.go +++ b/internal/gitaly/service/remote/server.go @@ -36,5 +36,5 @@ func NewServer(deps *service.Dependencies) gitalypb.RemoteServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } diff --git a/internal/gitaly/service/repository/create_fork.go b/internal/gitaly/service/repository/create_fork.go index e24c7370f..35f4c8ea2 100644 --- a/internal/gitaly/service/repository/create_fork.go +++ b/internal/gitaly/service/repository/create_fork.go @@ -28,7 +28,7 @@ func (s *server) CreateFork(ctx context.Context, req *gitalypb.CreateForkRequest targetRepository := req.Repository sourceRepository := req.SourceRepository - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, targetRepository, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, targetRepository, func(repo *gitalypb.Repository) error { targetPath, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return err diff --git a/internal/gitaly/service/repository/create_repository.go b/internal/gitaly/service/repository/create_repository.go index 39195d3ea..40e27538a 100644 --- a/internal/gitaly/service/repository/create_repository.go +++ b/internal/gitaly/service/repository/create_repository.go @@ -23,6 +23,7 @@ func (s *server) CreateRepository(ctx context.Context, req *gitalypb.CreateRepos if err := repoutil.Create( ctx, + s.logger, s.locator, s.gitCmdFactory, s.txManager, diff --git a/internal/gitaly/service/repository/create_repository_from_bundle.go b/internal/gitaly/service/repository/create_repository_from_bundle.go index e91b4ca83..5fcb729b3 100644 --- a/internal/gitaly/service/repository/create_repository_from_bundle.go +++ b/internal/gitaly/service/repository/create_repository_from_bundle.go @@ -34,7 +34,7 @@ func (s *server) CreateRepositoryFromBundle(stream gitalypb.RepositoryService_Cr return request.GetData(), err }) - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repo, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repo, func(repo *gitalypb.Repository) error { if err := s.localrepo(repo).CloneBundle(ctx, bundleReader); err != nil { return structerr.NewInternal("cloning bundle: %w", err) } diff --git a/internal/gitaly/service/repository/create_repository_from_snapshot.go b/internal/gitaly/service/repository/create_repository_from_snapshot.go index e784a6fd2..52a4b474a 100644 --- a/internal/gitaly/service/repository/create_repository_from_snapshot.go +++ b/internal/gitaly/service/repository/create_repository_from_snapshot.go @@ -126,7 +126,7 @@ func (s *server) CreateRepositoryFromSnapshot(ctx context.Context, in *gitalypb. return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repository, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, repository, func(repo *gitalypb.Repository) error { path, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return structerr.NewInternal("getting repo path: %w", err) diff --git a/internal/gitaly/service/repository/create_repository_from_url.go b/internal/gitaly/service/repository/create_repository_from_url.go index be1c7c72d..1afb76a0e 100644 --- a/internal/gitaly/service/repository/create_repository_from_url.go +++ b/internal/gitaly/service/repository/create_repository_from_url.go @@ -81,7 +81,7 @@ func (s *server) CreateRepositoryFromURL(ctx context.Context, req *gitalypb.Crea return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, req.GetRepository(), func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, req.GetRepository(), func(repo *gitalypb.Repository) error { targetPath, err := s.locator.GetRepoPath(repo, storage.WithRepositoryVerificationSkipped()) if err != nil { return fmt.Errorf("getting temporary repository path: %w", err) diff --git a/internal/gitaly/service/repository/license.go b/internal/gitaly/service/repository/license.go index 0bc2047f3..dcbf50aa9 100644 --- a/internal/gitaly/service/repository/license.go +++ b/internal/gitaly/service/repository/license.go @@ -51,7 +51,7 @@ func (s *server) FindLicense(ctx context.Context, req *gitalypb.FindLicenseReque if err := s.locator.ValidateRepository(repository); err != nil { return nil, structerr.NewInvalidArgument("%w", err) } - repo := localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repository) + repo := localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repository) headOID, err := repo.ResolveRevision(ctx, "HEAD") if err != nil { diff --git a/internal/gitaly/service/repository/optimize.go b/internal/gitaly/service/repository/optimize.go index 04392959d..02c8b9fa2 100644 --- a/internal/gitaly/service/repository/optimize.go +++ b/internal/gitaly/service/repository/optimize.go @@ -6,7 +6,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/git/housekeeping" "gitlab.com/gitlab-org/gitaly/v16/internal/git/stats" - "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb" ) @@ -37,7 +36,7 @@ func (s *server) OptimizeRepository(ctx context.Context, in *gitalypb.OptimizeRe return nil, structerr.NewInvalidArgument("unsupported optimization strategy %d", in.GetStrategy()) } - if err := s.housekeepingManager.OptimizeRepository(ctx, log.FromContext(ctx), repo, + if err := s.housekeepingManager.OptimizeRepository(ctx, repo, housekeeping.WithOptimizationStrategyConstructor(strategyConstructor), ); err != nil { return nil, structerr.NewInternal("%w", err) diff --git a/internal/gitaly/service/repository/optimize_test.go b/internal/gitaly/service/repository/optimize_test.go index 2264bae59..cfdcf82da 100644 --- a/internal/gitaly/service/repository/optimize_test.go +++ b/internal/gitaly/service/repository/optimize_test.go @@ -18,7 +18,6 @@ import ( "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/perm" "gitlab.com/gitlab-org/gitaly/v16/internal/helper/text" - "gitlab.com/gitlab-org/gitaly/v16/internal/log" "gitlab.com/gitlab-org/gitaly/v16/internal/structerr" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testserver" @@ -258,7 +257,7 @@ type mockHousekeepingManager struct { strategyCh chan housekeeping.OptimizationStrategy } -func (m mockHousekeepingManager) OptimizeRepository(_ context.Context, _ log.Logger, _ *localrepo.Repo, opts ...housekeeping.OptimizeRepositoryOption) error { +func (m mockHousekeepingManager) OptimizeRepository(_ context.Context, _ *localrepo.Repo, opts ...housekeeping.OptimizeRepositoryOption) error { var cfg housekeeping.OptimizeRepositoryConfig for _, opt := range opts { opt(&cfg) diff --git a/internal/gitaly/service/repository/remove.go b/internal/gitaly/service/repository/remove.go index 0abac2e6a..e1281a2f6 100644 --- a/internal/gitaly/service/repository/remove.go +++ b/internal/gitaly/service/repository/remove.go @@ -15,7 +15,7 @@ func (s *server) RemoveRepository(ctx context.Context, in *gitalypb.RemoveReposi return nil, structerr.NewInvalidArgument("%w", err) } - if err := repoutil.Remove(ctx, s.locator, s.txManager, s.repositoryCounter, repository); err != nil { + if err := repoutil.Remove(ctx, s.logger, s.locator, s.txManager, s.repositoryCounter, repository); err != nil { return nil, err } diff --git a/internal/gitaly/service/repository/replicate.go b/internal/gitaly/service/repository/replicate.go index 37e78345f..74c7f7b61 100644 --- a/internal/gitaly/service/repository/replicate.go +++ b/internal/gitaly/service/repository/replicate.go @@ -141,7 +141,7 @@ func validateReplicateRepository(locator storage.Locator, in *gitalypb.Replicate func (s *server) create(ctx context.Context, in *gitalypb.ReplicateRepositoryRequest, repoPath string) error { // if the directory exists, remove it if _, err := os.Stat(repoPath); err == nil { - tempDir, err := tempdir.NewWithoutContext(in.GetRepository().GetStorageName(), s.locator) + tempDir, err := tempdir.NewWithoutContext(in.GetRepository().GetStorageName(), s.logger, s.locator) if err != nil { return err } @@ -161,7 +161,7 @@ func (s *server) create(ctx context.Context, in *gitalypb.ReplicateRepositoryReq } func (s *server) createFromSnapshot(ctx context.Context, source, target *gitalypb.Repository) error { - if err := repoutil.Create(ctx, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, target, func(repo *gitalypb.Repository) error { + if err := repoutil.Create(ctx, s.logger, s.locator, s.gitCmdFactory, s.txManager, s.repositoryCounter, target, func(repo *gitalypb.Repository) error { if err := s.extractSnapshot(ctx, source, repo); err != nil { return fmt.Errorf("extracting snapshot: %w", err) } @@ -415,7 +415,7 @@ func (s *server) syncObjectPool(ctx context.Context, sourceRepoProto, targetRepo // In the case where the source repository does not have any Git alternates, but the // existing target repository does, the target repository should have its alternates // disconnected to match the current state of the source repository. - if err := objectpool.Disconnect(ctx, targetRepo, s.txManager); err != nil { + if err := objectpool.Disconnect(ctx, targetRepo, s.logger, s.txManager); err != nil { return fmt.Errorf("disconnect target from object pool: %w", err) } @@ -423,7 +423,7 @@ func (s *server) syncObjectPool(ctx context.Context, sourceRepoProto, targetRepo } // Check the target repository for an existing object pool link. - targetPool, err := objectpool.FromRepo(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetRepo) + targetPool, err := objectpool.FromRepo(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetRepo) if err != nil && !errors.Is(err, objectpool.ErrAlternateObjectDirNotExist) { return fmt.Errorf("get target object pool: %w", err) } @@ -452,7 +452,7 @@ func (s *server) syncObjectPool(ctx context.Context, sourceRepoProto, targetRepo targetPoolProto.GetRepository().StorageName = targetRepoProto.GetStorageName() // Check if object pool required for target repository already exists on the current node. - targetPool, err = objectpool.FromProto(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetPoolProto) + targetPool, err = objectpool.FromProto(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetPoolProto) switch { case errors.Is(err, objectpool.ErrInvalidPoolRepository): // In the case where the source repository does link to an object pool, but the object pool @@ -463,7 +463,7 @@ func (s *server) syncObjectPool(ctx context.Context, sourceRepoProto, targetRepo return fmt.Errorf("replicate object pool: %w", err) } - targetPool, err = objectpool.FromProto(s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetPoolProto) + targetPool, err = objectpool.FromProto(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, s.txManager, s.housekeepingManager, targetPoolProto) if err != nil { return fmt.Errorf("get replicated object pool: %w", err) } diff --git a/internal/gitaly/service/repository/server.go b/internal/gitaly/service/repository/server.go index 9b7ea0fea..230fb4160 100644 --- a/internal/gitaly/service/repository/server.go +++ b/internal/gitaly/service/repository/server.go @@ -60,13 +60,13 @@ func NewServer(deps *service.Dependencies) gitalypb.RepositoryServiceServer { } func (s *server) localrepo(repo storage.Repository) *localrepo.Repo { - return localrepo.New(s.locator, s.gitCmdFactory, s.catfileCache, repo) + return localrepo.New(s.logger, s.locator, s.gitCmdFactory, s.catfileCache, repo) } func (s *server) quarantinedRepo( ctx context.Context, repo *gitalypb.Repository, ) (*quarantine.Dir, *localrepo.Repo, error) { - quarantineDir, err := quarantine.New(ctx, repo, s.locator) + quarantineDir, err := quarantine.New(ctx, repo, s.logger, s.locator) if err != nil { return nil, nil, structerr.NewInternal("creating object quarantine: %w", err) } diff --git a/internal/gitaly/service/repository/size_test.go b/internal/gitaly/service/repository/size_test.go index 07039a724..f41cb2186 100644 --- a/internal/gitaly/service/repository/size_test.go +++ b/internal/gitaly/service/repository/size_test.go @@ -166,6 +166,7 @@ func TestGetObjectDirectorySize_quarantine(t *testing.T) { ctx := testhelper.Context(t) cfg, client := setupRepositoryService(t) locator := config.NewLocator(cfg) + logger := testhelper.NewLogger(t) t.Run("quarantined repo", func(t *testing.T) { repo, repoPath := gittest.CreateRepository(t, ctx, cfg) @@ -173,7 +174,7 @@ func TestGetObjectDirectorySize_quarantine(t *testing.T) { gittest.WriteBlob(t, cfg, repoPath, uncompressibleData(16*1024)) requireObjectDirectorySize(t, ctx, client, repo, 16) - quarantine, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), locator) + quarantine, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo), logger, locator) require.NoError(t, err) // quarantine.New in Gitaly would receive an already rewritten repository. Gitaly would then calculate @@ -191,11 +192,11 @@ func TestGetObjectDirectorySize_quarantine(t *testing.T) { t.Run("quarantined repo with different relative path", func(t *testing.T) { repo1, _ := gittest.CreateRepository(t, ctx, cfg) - quarantine1, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo1), locator) + quarantine1, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo1), logger, locator) require.NoError(t, err) repo2, _ := gittest.CreateRepository(t, ctx, cfg) - quarantine2, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo2), locator) + quarantine2, err := quarantine.New(ctx, gittest.RewrittenRepository(t, ctx, cfg, repo2), logger, locator) require.NoError(t, err) // We swap out the the object directories of both quarantines. So while both are diff --git a/internal/gitaly/storage/storagemgr/partition_manager_test.go b/internal/gitaly/storage/storagemgr/partition_manager_test.go index a80907445..d13a7f703 100644 --- a/internal/gitaly/storage/storagemgr/partition_manager_test.go +++ b/internal/gitaly/storage/storagemgr/partition_manager_test.go @@ -654,12 +654,13 @@ func TestPartitionManager(t *testing.T) { t.Parallel() cfg := testcfg.Build(t, testcfg.WithStorages("default", "other-storage")) + logger := testhelper.SharedLogger(t) cmdFactory := gittest.NewCommandFactory(t, cfg) catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) - localRepoFactory := localrepo.NewFactory(config.NewLocator(cfg), cmdFactory, catfileCache) + localRepoFactory := localrepo.NewFactory(logger, config.NewLocator(cfg), cmdFactory, catfileCache) setup := tc.setup(t, cfg) @@ -674,10 +675,10 @@ func TestPartitionManager(t *testing.T) { ) } - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(t), backchannel.NewRegistry()) - housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) + txManager := transaction.NewManager(cfg, logger, backchannel.NewRegistry()) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, logger, txManager) - partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, housekeepingManager, localRepoFactory, testhelper.SharedLogger(t)) + partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, housekeepingManager, localRepoFactory, logger) require.NoError(t, err) if setup.transactionManagerFactory != nil { @@ -795,17 +796,18 @@ func TestPartitionManager_concurrentClose(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) + logger := testhelper.SharedLogger(t) cmdFactory := gittest.NewCommandFactory(t, cfg) catfileCache := catfile.NewCache(cfg) defer catfileCache.Stop() - localRepoFactory := localrepo.NewFactory(config.NewLocator(cfg), cmdFactory, catfileCache) + localRepoFactory := localrepo.NewFactory(logger, config.NewLocator(cfg), cmdFactory, catfileCache) - txManager := transaction.NewManager(cfg, testhelper.SharedLogger(t), backchannel.NewRegistry()) - housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) + txManager := transaction.NewManager(cfg, logger, backchannel.NewRegistry()) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, logger, txManager) - partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, housekeepingManager, localRepoFactory, testhelper.SharedLogger(t)) + partitionManager, err := NewPartitionManager(cfg.Storages, cmdFactory, housekeepingManager, localRepoFactory, logger) require.NoError(t, err) defer partitionManager.Close() diff --git a/internal/gitaly/storage/storagemgr/transaction_manager.go b/internal/gitaly/storage/storagemgr/transaction_manager.go index 7f273123d..df0b7909c 100644 --- a/internal/gitaly/storage/storagemgr/transaction_manager.go +++ b/internal/gitaly/storage/storagemgr/transaction_manager.go @@ -1478,7 +1478,7 @@ func (mgr *TransactionManager) prepareReferenceTransaction(ctx context.Context, // We ask housekeeping to cleanup stale reference locks. We don't add a grace period, because // transaction manager is the only process which writes into the repository, so it is safe // to delete these locks. - if err := mgr.housekeepingManager.CleanStaleData(ctx, mgr.logger, repository, housekeeping.OnlyStaleReferenceLockCleanup(0)); err != nil { + if err := mgr.housekeepingManager.CleanStaleData(ctx, repository, housekeeping.OnlyStaleReferenceLockCleanup(0)); err != nil { return nil, fmt.Errorf("running reflock cleanup: %w", err) } diff --git a/internal/gitaly/storage/storagemgr/transaction_manager_test.go b/internal/gitaly/storage/storagemgr/transaction_manager_test.go index 6e28528c4..85ac09a8b 100644 --- a/internal/gitaly/storage/storagemgr/transaction_manager_test.go +++ b/internal/gitaly/storage/storagemgr/transaction_manager_test.go @@ -195,8 +195,10 @@ func TestTransactionManager(t *testing.T) { catfileCache := catfile.NewCache(cfg) t.Cleanup(catfileCache.Stop) + logger := testhelper.NewLogger(t) locator := config.NewLocator(cfg) localRepo := localrepo.New( + logger, locator, cmdFactory, catfileCache, @@ -227,7 +229,7 @@ func TestTransactionManager(t *testing.T) { Config: cfg, ObjectHash: objectHash, CommandFactory: cmdFactory, - RepositoryFactory: localrepo.NewFactory(locator, cmdFactory, catfileCache), + RepositoryFactory: localrepo.NewFactory(logger, locator, cmdFactory, catfileCache), NonExistentOID: nonExistentOID, Commits: testCommits{ First: testCommit{ @@ -4157,7 +4159,7 @@ func TestTransactionManager(t *testing.T) { defer testhelper.MustClose(t, database) txManager := transaction.NewManager(setup.Config, logger, backchannel.NewRegistry()) - housekeepingManager := housekeeping.NewManager(setup.Config.Prometheus, txManager) + housekeepingManager := housekeeping.NewManager(setup.Config.Prometheus, logger, txManager) storagePath := setup.Config.Storages[0].Path stateDir := filepath.Join(storagePath, "state") @@ -4557,7 +4559,7 @@ func BenchmarkTransactionManager(b *testing.B) { defer testhelper.MustClose(b, database) txManager := transaction.NewManager(cfg, logger, backchannel.NewRegistry()) - housekeepingManager := housekeeping.NewManager(cfg.Prometheus, txManager) + housekeepingManager := housekeeping.NewManager(cfg.Prometheus, logger, txManager) var ( // managerWG records the running TransactionManager.Run goroutines. @@ -4583,7 +4585,7 @@ func BenchmarkTransactionManager(b *testing.B) { } repositoryFactory, err := localrepo.NewFactory( - config.NewLocator(cfg), cmdFactory, cache, + logger, config.NewLocator(cfg), cmdFactory, cache, ).ScopeByStorage(cfg.Storages[0].Name) require.NoError(b, err) diff --git a/internal/log/logger.go b/internal/log/logger.go index e0e855f63..03dd0f799 100644 --- a/internal/log/logger.go +++ b/internal/log/logger.go @@ -86,8 +86,8 @@ func (l LogrusLogger) Error(msg string) { l.entry.Error(msg) } -// ToContext injects the logger into the given context so that it can be retrieved via `FromContext()`. -func (l LogrusLogger) ToContext(ctx context.Context) context.Context { +// toContext injects the logger into the given context so that it can be retrieved via `FromContext()`. +func (l LogrusLogger) toContext(ctx context.Context) context.Context { return ctxlogrus.ToContext(ctx, l.entry) } @@ -125,16 +125,16 @@ func (l LogrusLogger) ErrorContext(ctx context.Context, msg string) { l.log(ctx, logrus.ErrorLevel, msg) } -// FromContext extracts the logger from the context. If no logger has been injected then this will return a discarding +// fromContext extracts the logger from the context. If no logger has been injected then this will return a discarding // logger. -func FromContext(ctx context.Context) LogrusLogger { +func fromContext(ctx context.Context) LogrusLogger { return LogrusLogger{ entry: ctxlogrus.Extract(ctx), } } -// AddFields adds the given log fields to the context so that it will be used by any context logger extracted via -// `FromContext()`. +// AddFields adds the given log fields to the context so that it will be used by any logging function like +// `InfoContext()` that receives a context as input. func AddFields(ctx context.Context, fields Fields) { ctxlogrus.AddFields(ctx, fields) } diff --git a/internal/log/middleware.go b/internal/log/middleware.go index 4cb82541a..14c53c8fb 100644 --- a/internal/log/middleware.go +++ b/internal/log/middleware.go @@ -112,7 +112,7 @@ func PropagationMessageProducer(actual grpcmwlogrus.MessageProducer) grpcmwlogru return } *mpp = messageProducerHolder{ - logger: FromContext(ctx), + logger: fromContext(ctx), actual: actual, format: format, level: level, @@ -169,7 +169,7 @@ func (lh PerRPCLogHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { // a logger we need to set logger manually into the context. // It's needed because github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus.DefaultMessageProducer // extracts logger from the context and use it to write the logs. - ctx = mpp.logger.ToContext(ctx) + ctx = mpp.logger.toContext(ctx) mpp.actual(ctx, mpp.format, mpp.level, mpp.code, mpp.err, mpp.fields) return } @@ -190,7 +190,7 @@ func UnaryLogDataCatcherServerInterceptor() grpc.UnaryServerInterceptor { return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { mpp := messageProducerPropagationFrom(ctx) if mpp != nil { - mpp.fields = FromContext(ctx).entry.Data + mpp.fields = fromContext(ctx).entry.Data } return handler(ctx, req) } @@ -203,7 +203,7 @@ func StreamLogDataCatcherServerInterceptor() grpc.StreamServerInterceptor { ctx := ss.Context() mpp := messageProducerPropagationFrom(ctx) if mpp != nil { - mpp.fields = FromContext(ctx).entry.Data + mpp.fields = fromContext(ctx).entry.Data } return handler(srv, ss) } diff --git a/internal/tempdir/tempdir.go b/internal/tempdir/tempdir.go index 349938dd8..7ad88c469 100644 --- a/internal/tempdir/tempdir.go +++ b/internal/tempdir/tempdir.go @@ -15,6 +15,7 @@ import ( // Dir is a storage-scoped temporary directory. type Dir struct { + logger log.Logger path string doneCh chan struct{} } @@ -26,15 +27,15 @@ func (d Dir) Path() string { // New returns the path of a new temporary directory for the given storage. The directory is removed // asynchronously with os.RemoveAll when the context expires. -func New(ctx context.Context, storageName string, locator storage.Locator) (Dir, error) { - return NewWithPrefix(ctx, storageName, "repo", locator) +func New(ctx context.Context, storageName string, logger log.Logger, locator storage.Locator) (Dir, error) { + return NewWithPrefix(ctx, storageName, "repo", logger, locator) } // NewWithPrefix returns the path of a new temporary directory for the given storage with a specific // prefix used to create the temporary directory's name. The directory is removed asynchronously // with os.RemoveAll when the context expires. -func NewWithPrefix(ctx context.Context, storageName, prefix string, locator storage.Locator) (Dir, error) { - dir, err := newDirectory(ctx, storageName, prefix, locator) +func NewWithPrefix(ctx context.Context, storageName, prefix string, logger log.Logger, locator storage.Locator) (Dir, error) { + dir, err := newDirectory(ctx, storageName, prefix, logger, locator) if err != nil { return Dir{}, err } @@ -47,20 +48,20 @@ func NewWithPrefix(ctx context.Context, storageName, prefix string, locator stor // NewWithoutContext returns a temporary directory for the given storage suitable which is not // storage scoped. The temporary directory will thus not get cleaned up when the context expires, // but instead when the temporary directory is older than MaxAge. -func NewWithoutContext(storageName string, locator storage.Locator) (Dir, error) { +func NewWithoutContext(storageName string, logger log.Logger, locator storage.Locator) (Dir, error) { prefix := fmt.Sprintf("%s-repositories.old.%d.", storageName, time.Now().Unix()) - return newDirectory(context.Background(), storageName, prefix, locator) + return newDirectory(context.Background(), storageName, prefix, logger, locator) } // NewRepository is the same as New, but it returns a *gitalypb.Repository for the created directory // as well as the bare path as a string. -func NewRepository(ctx context.Context, storageName string, locator storage.Locator) (*gitalypb.Repository, Dir, error) { +func NewRepository(ctx context.Context, storageName string, logger log.Logger, locator storage.Locator) (*gitalypb.Repository, Dir, error) { storagePath, err := locator.GetStorageByName(storageName) if err != nil { return nil, Dir{}, err } - dir, err := New(ctx, storageName, locator) + dir, err := New(ctx, storageName, logger, locator) if err != nil { return nil, Dir{}, err } @@ -74,7 +75,7 @@ func NewRepository(ctx context.Context, storageName string, locator storage.Loca return newRepo, dir, nil } -func newDirectory(ctx context.Context, storageName string, prefix string, loc storage.Locator) (Dir, error) { +func newDirectory(ctx context.Context, storageName string, prefix string, logger log.Logger, loc storage.Locator) (Dir, error) { root, err := loc.TempDir(storageName) if err != nil { return Dir{}, fmt.Errorf("temp directory: %w", err) @@ -90,6 +91,7 @@ func newDirectory(ctx context.Context, storageName string, prefix string, loc st } return Dir{ + logger: logger, path: tempDir, doneCh: make(chan struct{}), }, err @@ -98,7 +100,7 @@ func newDirectory(ctx context.Context, storageName string, prefix string, loc st func (d Dir) cleanupOnDone(ctx context.Context) { <-ctx.Done() if err := os.RemoveAll(d.Path()); err != nil { - log.FromContext(ctx).WithError(err).WithField("temporary_directory", d.Path).Error("failed to cleanup temp dir") + d.logger.WithError(err).WithField("temporary_directory", d.Path).ErrorContext(ctx, "failed to cleanup temp dir") } close(d.doneCh) } diff --git a/internal/tempdir/tempdir_test.go b/internal/tempdir/tempdir_test.go index 61aaba1eb..c625b7d4b 100644 --- a/internal/tempdir/tempdir_test.go +++ b/internal/tempdir/tempdir_test.go @@ -20,7 +20,7 @@ func TestNewRepositorySuccess(t *testing.T) { cfg := testcfg.Build(t) locator := config.NewLocator(cfg) - repo, tempDir, err := NewRepository(ctx, cfg.Storages[0].Name, locator) + repo, tempDir, err := NewRepository(ctx, cfg.Storages[0].Name, testhelper.NewLogger(t), locator) require.NoError(t, err) require.Equal(t, cfg.Storages[0].Name, repo.StorageName) require.Contains(t, repo.RelativePath, tmpRootPrefix) @@ -44,7 +44,7 @@ func TestNewWithPrefix(t *testing.T) { locator := config.NewLocator(cfg) ctx := testhelper.Context(t) - dir, err := NewWithPrefix(ctx, cfg.Storages[0].Name, "foobar-", locator) + dir, err := NewWithPrefix(ctx, cfg.Storages[0].Name, "foobar-", testhelper.NewLogger(t), locator) require.NoError(t, err) require.Contains(t, dir.Path(), "/foobar-") @@ -52,6 +52,6 @@ func TestNewWithPrefix(t *testing.T) { func TestNewAsRepositoryFailStorageUnknown(t *testing.T) { ctx := testhelper.Context(t) - _, err := New(ctx, "does-not-exist", config.NewLocator(config.Cfg{})) + _, err := New(ctx, "does-not-exist", testhelper.NewLogger(t), config.NewLocator(config.Cfg{})) require.Error(t, err) } diff --git a/internal/testhelper/testserver/gitaly.go b/internal/testhelper/testserver/gitaly.go index 3632a9943..1c7b48bd3 100644 --- a/internal/testhelper/testserver/gitaly.go +++ b/internal/testhelper/testserver/gitaly.go @@ -346,7 +346,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, cfg config.Cfg) * } if gsd.housekeepingManager == nil { - gsd.housekeepingManager = housekeeping.NewManager(cfg.Prometheus, gsd.txMgr) + gsd.housekeepingManager = housekeeping.NewManager(cfg.Prometheus, gsd.logger, gsd.txMgr) } var partitionManager *storagemgr.PartitionManager @@ -356,7 +356,7 @@ func (gsd *gitalyServerDeps) createDependencies(tb testing.TB, cfg config.Cfg) * cfg.Storages, gsd.gitCmdFactory, gsd.housekeepingManager, - localrepo.NewFactory(gsd.locator, gsd.gitCmdFactory, gsd.catfileCache), + localrepo.NewFactory(gsd.logger, gsd.locator, gsd.gitCmdFactory, gsd.catfileCache), gsd.logger, ) require.NoError(tb, err) |