Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJustin Tobler <jtobler@gitlab.com>2023-01-26 19:13:30 +0300
committerJustin Tobler <jtobler@gitlab.com>2023-01-26 19:13:30 +0300
commit4a014f5227b3defe785d9cec776abe2b1a944826 (patch)
treec5104c626712334f6fd1aa8f4b1d0c768081970e
parent6d228addeac5e159eaf89ce54aa523f71e336a29 (diff)
parent512aed451d7e2c0c45629a13c1b12d86dece3131 (diff)
Merge branch 'pks-housekeeping-repository-metrics' into 'master'
housekeeping: Add Prometheus metrics for on-disk repository states See merge request https://gitlab.com/gitlab-org/gitaly/-/merge_requests/5276 Merged-by: Justin Tobler <jtobler@gitlab.com> Approved-by: karthik nayak <knayak@gitlab.com> Approved-by: Justin Tobler <jtobler@gitlab.com> Co-authored-by: Patrick Steinhardt <psteinhardt@gitlab.com>
-rw-r--r--internal/git/housekeeping/manager.go39
-rw-r--r--internal/git/housekeeping/objects.go3
-rw-r--r--internal/git/housekeeping/objects_test.go3
-rw-r--r--internal/git/housekeeping/optimization_strategy.go50
-rw-r--r--internal/git/housekeeping/optimization_strategy_test.go400
-rw-r--r--internal/git/housekeeping/optimize_repository.go80
-rw-r--r--internal/git/housekeeping/optimize_repository_test.go48
-rw-r--r--internal/git/objectpool/fetch_test.go6
-rw-r--r--internal/git/objectpool/pool.go3
-rw-r--r--internal/git/stats/object_pool.go (renamed from internal/git/housekeeping/object_pool.go)2
-rw-r--r--internal/git/stats/object_pool_test.go (renamed from internal/git/housekeeping/object_pool_test.go)2
-rw-r--r--internal/git/stats/repository_info.go4
-rw-r--r--internal/gitaly/service/objectpool/create.go4
-rw-r--r--internal/gitaly/service/repository/optimize.go21
-rw-r--r--internal/gitaly/service/repository/optimize_test.go26
-rw-r--r--internal/praefect/delete_object_pool.go4
-rw-r--r--internal/praefect/router_per_repository.go4
17 files changed, 188 insertions, 511 deletions
diff --git a/internal/git/housekeeping/manager.go b/internal/git/housekeeping/manager.go
index 19bfca2de..55f1f24c3 100644
--- a/internal/git/housekeeping/manager.go
+++ b/internal/git/housekeeping/manager.go
@@ -24,11 +24,14 @@ type Manager interface {
type RepositoryManager struct {
txManager transaction.Manager
- tasksTotal *prometheus.CounterVec
- tasksLatency *prometheus.HistogramVec
- prunedFilesTotal *prometheus.CounterVec
- optimizeFunc func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizeRepositoryConfig) error
- reposInProgress sync.Map
+ tasksTotal *prometheus.CounterVec
+ tasksLatency *prometheus.HistogramVec
+ prunedFilesTotal *prometheus.CounterVec
+ dataStructureExistence *prometheus.CounterVec
+ dataStructureCount *prometheus.HistogramVec
+ dataStructureSize *prometheus.HistogramVec
+ optimizeFunc func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizationStrategy) error
+ reposInProgress sync.Map
}
// NewManager creates a new RepositoryManager.
@@ -58,6 +61,29 @@ func NewManager(promCfg gitalycfgprom.Config, txManager transaction.Manager) *Re
},
[]string{"filetype"},
),
+ dataStructureExistence: prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "gitaly_housekeeping_data_structure_existence_total",
+ Help: "Total number of data structures that exist in the repository",
+ },
+ []string{"data_structure", "exists"},
+ ),
+ dataStructureCount: prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "gitaly_housekeeping_data_structure_wcount",
+ Help: "Total count of the data structures that exist in the repository",
+ Buckets: prometheus.ExponentialBucketsRange(1, 10_000_000, 16),
+ },
+ []string{"data_structure"},
+ ),
+ dataStructureSize: prometheus.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "gitaly_housekeeping_data_structure_size",
+ Help: "Total size of the data structures that exist in the repository",
+ Buckets: prometheus.ExponentialBucketsRange(1, 50_000_000_000, 16),
+ },
+ []string{"data_structure"},
+ ),
optimizeFunc: optimizeRepository,
}
}
@@ -72,4 +98,7 @@ func (m *RepositoryManager) Collect(metrics chan<- prometheus.Metric) {
m.tasksTotal.Collect(metrics)
m.tasksLatency.Collect(metrics)
m.prunedFilesTotal.Collect(metrics)
+ m.dataStructureExistence.Collect(metrics)
+ m.dataStructureCount.Collect(metrics)
+ m.dataStructureSize.Collect(metrics)
}
diff --git a/internal/git/housekeeping/objects.go b/internal/git/housekeeping/objects.go
index e66aaf475..3e4fbaeab 100644
--- a/internal/git/housekeeping/objects.go
+++ b/internal/git/housekeeping/objects.go
@@ -7,6 +7,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v15/internal/git"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/repository"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
)
@@ -71,7 +72,7 @@ func GetRepackGitConfig(ctx context.Context, repo repository.GitRepo, bitmap boo
{Key: "pack.writeBitmapLookupTable", Value: strconv.FormatBool(featureflag.WriteBitmapLookupTable.IsEnabled(ctx))},
}
- if IsPoolRepository(repo) {
+ if stats.IsPoolRepository(repo) {
config = append(config,
git.ConfigPair{Key: "pack.island", Value: git.ObjectPoolRefNamespace + "/he(a)ds"},
git.ConfigPair{Key: "pack.island", Value: git.ObjectPoolRefNamespace + "/t(a)gs"},
diff --git a/internal/git/housekeeping/objects_test.go b/internal/git/housekeeping/objects_test.go
index 93a8f4f88..c8005e911 100644
--- a/internal/git/housekeeping/objects_test.go
+++ b/internal/git/housekeeping/objects_test.go
@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
@@ -301,7 +302,7 @@ func testRepackObjects(t *testing.T, ctx context.Context) {
})
repo := localrepo.NewTestRepo(t, cfg, repoProto)
- gittest.TestDeltaIslands(t, cfg, repoPath, repoPath, IsPoolRepository(repoProto), func() error {
+ gittest.TestDeltaIslands(t, cfg, repoPath, repoPath, stats.IsPoolRepository(repoProto), func() error {
return RepackObjects(ctx, repo, RepackObjectsConfig{
FullRepack: true,
})
diff --git a/internal/git/housekeeping/optimization_strategy.go b/internal/git/housekeeping/optimization_strategy.go
index 39aa004ca..e1c7df86b 100644
--- a/internal/git/housekeeping/optimization_strategy.go
+++ b/internal/git/housekeeping/optimization_strategy.go
@@ -2,11 +2,8 @@ package housekeeping
import (
"context"
- "fmt"
"math"
- "os"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
)
@@ -30,25 +27,16 @@ type OptimizationStrategy interface {
// HeuristicalOptimizationStrategy is an optimization strategy that is based on a set of
// heuristics.
type HeuristicalOptimizationStrategy struct {
- info stats.RepositoryInfo
- isObjectPool bool
+ info stats.RepositoryInfo
}
// NewHeuristicalOptimizationStrategy constructs a heuristicalOptimizationStrategy for the given
-// repository. It derives all data from the repository so that the heuristics used by this
+// repository info. It derives all data from the repository so that the heuristics used by this
// repository can be decided without further disk reads.
-func NewHeuristicalOptimizationStrategy(ctx context.Context, repo *localrepo.Repo) (HeuristicalOptimizationStrategy, error) {
- var strategy HeuristicalOptimizationStrategy
- var err error
-
- strategy.isObjectPool = IsPoolRepository(repo)
- strategy.info, err = stats.RepositoryInfoForRepository(repo)
- if err != nil {
- return HeuristicalOptimizationStrategy{}, fmt.Errorf("deriving repository info: %w", err)
+func NewHeuristicalOptimizationStrategy(info stats.RepositoryInfo) HeuristicalOptimizationStrategy {
+ return HeuristicalOptimizationStrategy{
+ info: info,
}
- strategy.info.Log(ctx)
-
- return strategy, nil
}
// ShouldRepackObjects checks whether the repository's objects need to be repacked. This uses a
@@ -112,7 +100,7 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackObjects(context.Context) (b
// This is a heuristic and thus imperfect by necessity. We may tune it as we gain experience
// with the way it behaves.
lowerLimit, log := 5.0, 1.3
- if s.isObjectPool {
+ if s.info.IsObjectPool {
lowerLimit, log = 2.0, 10.0
}
@@ -192,7 +180,7 @@ func (s HeuristicalOptimizationStrategy) ShouldWriteCommitGraph(ctx context.Cont
func (s HeuristicalOptimizationStrategy) ShouldPruneObjects(context.Context) bool {
// Pool repositories must never prune any objects, or otherwise we may corrupt members of
// that pool if they still refer to that object.
- if s.isObjectPool {
+ if s.info.IsObjectPool {
return false
}
@@ -245,26 +233,14 @@ func (s HeuristicalOptimizationStrategy) ShouldRepackReferences(context.Context)
// EagerOptimizationStrategy is a strategy that will eagerly perform optimizations. All of the data
// structures will be optimized regardless of whether they already are in an optimal state or not.
type EagerOptimizationStrategy struct {
- hasAlternate bool
- isObjectPool bool
+ info stats.RepositoryInfo
}
// NewEagerOptimizationStrategy creates a new EagerOptimizationStrategy.
-func NewEagerOptimizationStrategy(ctx context.Context, repo *localrepo.Repo) (EagerOptimizationStrategy, error) {
- altFile, err := repo.InfoAlternatesPath()
- if err != nil {
- return EagerOptimizationStrategy{}, fmt.Errorf("getting alternates path: %w", err)
- }
-
- hasAlternate := true
- if _, err := os.Stat(altFile); os.IsNotExist(err) {
- hasAlternate = false
- }
-
+func NewEagerOptimizationStrategy(info stats.RepositoryInfo) EagerOptimizationStrategy {
return EagerOptimizationStrategy{
- hasAlternate: hasAlternate,
- isObjectPool: IsPoolRepository(repo),
- }, nil
+ info: info,
+ }
}
// ShouldRepackObjects always instructs the caller to repack objects. The strategy will always be to
@@ -273,7 +249,7 @@ func NewEagerOptimizationStrategy(ctx context.Context, repo *localrepo.Repo) (Ea
func (s EagerOptimizationStrategy) ShouldRepackObjects(context.Context) (bool, RepackObjectsConfig) {
return true, RepackObjectsConfig{
FullRepack: true,
- WriteBitmap: !s.hasAlternate,
+ WriteBitmap: len(s.info.Alternates) == 0,
}
}
@@ -288,7 +264,7 @@ func (s EagerOptimizationStrategy) ShouldWriteCommitGraph(context.Context) (bool
// ShouldPruneObjects always instructs the caller to prune objects, unless the repository is an
// object pool.
func (s EagerOptimizationStrategy) ShouldPruneObjects(context.Context) bool {
- return !s.isObjectPool
+ return !s.info.IsObjectPool
}
// ShouldRepackReferences always instructs the caller to repack references.
diff --git a/internal/git/housekeeping/optimization_strategy_test.go b/internal/git/housekeeping/optimization_strategy_test.go
index 093e68336..405b3de84 100644
--- a/internal/git/housekeeping/optimization_strategy_test.go
+++ b/internal/git/housekeeping/optimization_strategy_test.go
@@ -3,328 +3,13 @@ package housekeeping
import (
"context"
"fmt"
- "os"
- "path/filepath"
"testing"
- "time"
"github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
- "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg"
- "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
)
-func TestNewHeuristicalOptimizationStrategy_variousParameters(t *testing.T) {
- t.Parallel()
-
- ctx := testhelper.Context(t)
- cfg := testcfg.Build(t)
-
- disabledGenerationVersion := "commitGraph.generationVersion=1"
- enabledGenerationVersion := "commitGraph.generationVersion=2"
-
- for _, tc := range []struct {
- desc string
- setup func(t *testing.T, relativePath string) *gitalypb.Repository
- expectedStrategy HeuristicalOptimizationStrategy
- }{
- {
- desc: "empty repo",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{},
- },
- {
- desc: "object in 17 shard",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
-
- looseObjectPath := filepath.Join(repoPath, "objects", "17", "1234")
- require.NoError(t, os.MkdirAll(filepath.Dir(looseObjectPath), 0o755))
- require.NoError(t, os.WriteFile(looseObjectPath, nil, 0o644))
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 1,
- },
- },
- },
- },
- {
- desc: "old object in 17 shard",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
-
- shard := filepath.Join(repoPath, "objects", "17")
- require.NoError(t, os.MkdirAll(shard, 0o755))
-
- // We write one recent...
- require.NoError(t, os.WriteFile(filepath.Join(shard, "1234"), nil, 0o644))
-
- // ... and one stale object.
- staleObjectPath := filepath.Join(shard, "5678")
- require.NoError(t, os.WriteFile(staleObjectPath, nil, 0o644))
- twoWeeksAgo := time.Now().Add(stats.StaleObjectsGracePeriod)
- require.NoError(t, os.Chtimes(staleObjectPath, twoWeeksAgo, twoWeeksAgo))
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 2,
- StaleCount: 1,
- },
- },
- },
- },
- {
- desc: "packfile",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
-
- require.NoError(t, os.WriteFile(filepath.Join(repoPath, "objects", "pack", "pack-foobar.pack"), nil, 0o644))
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- Packfiles: stats.PackfilesInfo{
- Count: 1,
- },
- },
- },
- },
- {
- desc: "alternate",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
-
- require.NoError(t, os.WriteFile(filepath.Join(repoPath, "objects", "info", "alternates"), []byte("/path/to/alternate"), 0o644))
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- Alternates: []string{"/path/to/alternate"},
- },
- },
- },
- {
- desc: "bitmap",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
- gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
-
- gittest.Exec(t, cfg, "-C", repoPath, "repack", "-Adb")
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- References: stats.ReferencesInfo{
- LooseReferencesCount: 1,
- },
- Packfiles: stats.PackfilesInfo{
- Count: 1,
- Size: hashDependentObjectSize(163, 189),
- Bitmap: stats.BitmapInfo{
- Exists: true,
- Version: 1,
- HasHashCache: true,
- },
- },
- },
- },
- },
- {
- desc: "existing unsplit commit-graph with bloom filters and no generation data",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
- gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
-
- // Write a non-split commit-graph with bloom filters. We should
- // always rewrite the commit-graphs when we're not using a split
- // commit-graph. We make sure to add bloom filters via
- // `--changed-paths` given that it would otherwise cause us to
- // rewrite the graph regardless of whether the graph is split or not
- // if they were missing.
- gittest.Exec(t, cfg, "-C", repoPath, "-c", disabledGenerationVersion,
- "commit-graph", "write", "--reachable", "--changed-paths",
- )
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 2,
- Size: hashDependentObjectSize(142, 158),
- },
- References: stats.ReferencesInfo{
- LooseReferencesCount: 1,
- },
- CommitGraph: stats.CommitGraphInfo{
- Exists: true,
- HasBloomFilters: true,
- HasGenerationData: false,
- },
- },
- },
- },
- {
- desc: "existing unsplit commit-graph with bloom filters with generation data",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
- gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
-
- // Write a non-split commit-graph with bloom filters. We should
- // always rewrite the commit-graphs when we're not using a split
- // commit-graph. We make sure to add bloom filters via
- // `--changed-paths` given that it would otherwise cause us to
- // rewrite the graph regardless of whether the graph is split or not
- // if they were missing.
- gittest.Exec(t, cfg, "-C", repoPath, "-c", enabledGenerationVersion,
- "commit-graph", "write", "--reachable", "--changed-paths",
- )
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 2,
- Size: hashDependentObjectSize(142, 158),
- },
- References: stats.ReferencesInfo{
- LooseReferencesCount: 1,
- },
- CommitGraph: stats.CommitGraphInfo{
- Exists: true,
- HasBloomFilters: true,
- HasGenerationData: true,
- },
- },
- },
- },
- {
- desc: "existing split commit-graph without bloom filters and generation data",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
- gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
-
- // Generate a split commit-graph, but don't enable computation of
- // changed paths. This should trigger a rewrite so that we can
- // recompute all graphs and generate the changed paths.
- gittest.Exec(t, cfg, "-C", repoPath, "-c", disabledGenerationVersion,
- "commit-graph", "write", "--reachable", "--split",
- )
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 2,
- Size: hashDependentObjectSize(142, 158),
- },
- References: stats.ReferencesInfo{
- LooseReferencesCount: 1,
- },
- CommitGraph: stats.CommitGraphInfo{
- Exists: true,
- CommitGraphChainLength: 1,
- },
- },
- },
- },
- {
- desc: "existing split commit-graph with bloom filters and generation data",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
- gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"))
-
- // Write a split commit-graph with bitmaps. This is the state we
- // want to be in, so there is no write required if we didn't also
- // repack objects.
- gittest.Exec(t, cfg, "-C", repoPath, "-c", enabledGenerationVersion,
- "commit-graph", "write", "--reachable", "--split", "--changed-paths",
- )
-
- return repoProto
- },
- expectedStrategy: HeuristicalOptimizationStrategy{
- info: stats.RepositoryInfo{
- LooseObjects: stats.LooseObjectsInfo{
- Count: 2,
- Size: hashDependentObjectSize(142, 158),
- },
- References: stats.ReferencesInfo{
- LooseReferencesCount: 1,
- },
- CommitGraph: stats.CommitGraphInfo{
- Exists: true,
- CommitGraphChainLength: 1,
- HasBloomFilters: true,
- HasGenerationData: true,
- },
- },
- },
- },
- } {
- tc := tc
-
- t.Run(tc.desc, func(t *testing.T) {
- t.Parallel()
-
- testRepoAndPool(t, tc.desc, func(t *testing.T, relativePath string) {
- repoProto := tc.setup(t, relativePath)
- repo := localrepo.NewTestRepo(t, cfg, repoProto)
-
- tc.expectedStrategy.isObjectPool = IsPoolRepository(repo)
-
- strategy, err := NewHeuristicalOptimizationStrategy(ctx, repo)
- require.NoError(t, err)
- require.Equal(t, tc.expectedStrategy, strategy)
- })
- })
- }
-}
-
func TestHeuristicalOptimizationStrategy_ShouldRepackObjects(t *testing.T) {
t.Parallel()
@@ -467,6 +152,7 @@ func TestHeuristicalOptimizationStrategy_ShouldRepackObjects(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
strategy := HeuristicalOptimizationStrategy{
info: stats.RepositoryInfo{
+ IsObjectPool: tc.isPool,
Packfiles: stats.PackfilesInfo{
Size: outerTC.packfileSizeInMB * 1024 * 1024,
Count: tc.requiredPackfiles - 1,
@@ -476,7 +162,6 @@ func TestHeuristicalOptimizationStrategy_ShouldRepackObjects(t *testing.T) {
},
Alternates: tc.alternates,
},
- isObjectPool: tc.isPool,
}
repackNeeded, _ := strategy.ShouldRepackObjects(ctx)
@@ -539,6 +224,7 @@ func TestHeuristicalOptimizationStrategy_ShouldRepackObjects(t *testing.T) {
t.Run(tc.desc, func(t *testing.T) {
strategy := HeuristicalOptimizationStrategy{
info: stats.RepositoryInfo{
+ IsObjectPool: tc.isPool,
LooseObjects: stats.LooseObjectsInfo{
Count: outerTC.looseObjects,
},
@@ -550,7 +236,6 @@ func TestHeuristicalOptimizationStrategy_ShouldRepackObjects(t *testing.T) {
},
},
},
- isObjectPool: tc.isPool,
}
repackNeeded, repackCfg := strategy.ShouldRepackObjects(ctx)
@@ -620,7 +305,7 @@ func TestHeuristicalOptimizationStrategy_ShouldPruneObjects(t *testing.T) {
t.Run("object pool", func(t *testing.T) {
strategy := tc.strategy
- strategy.isObjectPool = true
+ strategy.info.IsObjectPool = true
require.False(t, strategy.ShouldPruneObjects(ctx))
})
})
@@ -840,64 +525,6 @@ func TestHeuristicalOptimizationStrategy_NeedsWriteCommitGraph(t *testing.T) {
}
}
-func TestNewEagerOptimizationStrategy(t *testing.T) {
- t.Parallel()
-
- ctx := testhelper.Context(t)
- cfg := testcfg.Build(t)
-
- for _, tc := range []struct {
- desc string
- setup func(t *testing.T, relativePath string) *gitalypb.Repository
- expectedStrategy EagerOptimizationStrategy
- }{
- {
- desc: "empty repo",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, _ := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
- return repoProto
- },
- expectedStrategy: EagerOptimizationStrategy{},
- },
- {
- desc: "alternate",
- setup: func(t *testing.T, relativePath string) *gitalypb.Repository {
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- RelativePath: relativePath,
- })
-
- require.NoError(t, os.WriteFile(filepath.Join(repoPath, "objects", "info", "alternates"), nil, 0o644))
-
- return repoProto
- },
- expectedStrategy: EagerOptimizationStrategy{
- hasAlternate: true,
- },
- },
- } {
- tc := tc
-
- t.Run(tc.desc, func(t *testing.T) {
- t.Parallel()
-
- testRepoAndPool(t, tc.desc, func(t *testing.T, relativePath string) {
- repoProto := tc.setup(t, relativePath)
- repo := localrepo.NewTestRepo(t, cfg, repoProto)
-
- tc.expectedStrategy.isObjectPool = IsPoolRepository(repo)
-
- strategy, err := NewEagerOptimizationStrategy(ctx, repo)
- require.NoError(t, err)
- require.Equal(t, tc.expectedStrategy, strategy)
- })
- })
- }
-}
-
func TestEagerOptimizationStrategy(t *testing.T) {
t.Parallel()
@@ -917,22 +544,28 @@ func TestEagerOptimizationStrategy(t *testing.T) {
{
desc: "alternate",
strategy: EagerOptimizationStrategy{
- hasAlternate: true,
+ info: stats.RepositoryInfo{
+ Alternates: []string{"path/to/alternate"},
+ },
},
expectShouldPruneObjects: true,
},
{
desc: "object pool",
strategy: EagerOptimizationStrategy{
- isObjectPool: true,
+ info: stats.RepositoryInfo{
+ IsObjectPool: true,
+ },
},
expectWriteBitmap: true,
},
{
desc: "object pool with alternate",
strategy: EagerOptimizationStrategy{
- hasAlternate: true,
- isObjectPool: true,
+ info: stats.RepositoryInfo{
+ IsObjectPool: true,
+ Alternates: []string{"path/to/alternate"},
+ },
},
},
} {
@@ -981,10 +614,3 @@ func (m mockOptimizationStrategy) ShouldRepackReferences(context.Context) bool {
func (m mockOptimizationStrategy) ShouldWriteCommitGraph(context.Context) (bool, WriteCommitGraphConfig) {
return m.shouldWriteCommitGraph, m.writeCommitGraphCfg
}
-
-func hashDependentObjectSize(sha1Size, sha256Size uint64) uint64 {
- if gittest.ObjectHashIsSHA256() {
- return sha256Size
- }
- return sha1Size
-}
diff --git a/internal/git/housekeeping/optimize_repository.go b/internal/git/housekeeping/optimize_repository.go
index b4827259c..74f70d1b5 100644
--- a/internal/git/housekeeping/optimize_repository.go
+++ b/internal/git/housekeeping/optimize_repository.go
@@ -4,27 +4,34 @@ import (
"bytes"
"context"
"fmt"
+ "strconv"
"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/prometheus/client_golang/prometheus"
"gitlab.com/gitlab-org/gitaly/v15/internal/git"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
)
// OptimizeRepositoryConfig is the configuration used by OptimizeRepository that is computed by
// applying all the OptimizeRepositoryOption modifiers.
type OptimizeRepositoryConfig struct {
- Strategy OptimizationStrategy
+ StrategyConstructor OptimizationStrategyConstructor
}
// OptimizeRepositoryOption is an option that can be passed to OptimizeRepository.
type OptimizeRepositoryOption func(cfg *OptimizeRepositoryConfig)
-// WithOptimizationStrategy changes the strategy used to determine which parts of the repository
-// will be optimized. By default the HeuristicalOptimizationStrategy is used.
-func WithOptimizationStrategy(strategy OptimizationStrategy) OptimizeRepositoryOption {
+// OptimizationStrategyConstructor is a constructor for an OptimizationStrategy that is being
+// informed by the passed-in RepositoryInfo.
+type OptimizationStrategyConstructor func(stats.RepositoryInfo) OptimizationStrategy
+
+// WithOptimizationStrategyConstructor changes the constructor for the optimization strategy.that is
+// used to determine which parts of the repository will be optimized. By default the
+// HeuristicalOptimizationStrategy is used.
+func WithOptimizationStrategyConstructor(strategyConstructor OptimizationStrategyConstructor) OptimizeRepositoryOption {
return func(cfg *OptimizeRepositoryConfig) {
- cfg.Strategy = strategy
+ cfg.StrategyConstructor = strategyConstructor
}
}
@@ -53,23 +60,62 @@ func (m *RepositoryManager) OptimizeRepository(
opt(&cfg)
}
- if cfg.Strategy == nil {
- strategy, err := NewHeuristicalOptimizationStrategy(ctx, repo)
- if err != nil {
- return fmt.Errorf("creating default optimization strategy: %w", err)
- }
+ repositoryInfo, err := stats.RepositoryInfoForRepository(repo)
+ if err != nil {
+ return fmt.Errorf("deriving repository info: %w", err)
+ }
+ m.reportRepositoryInfo(ctx, repositoryInfo)
- cfg.Strategy = strategy
+ var strategy OptimizationStrategy
+ if cfg.StrategyConstructor == nil {
+ strategy = NewHeuristicalOptimizationStrategy(repositoryInfo)
+ } else {
+ strategy = cfg.StrategyConstructor(repositoryInfo)
}
- return m.optimizeFunc(ctx, m, repo, cfg)
+ return m.optimizeFunc(ctx, m, repo, strategy)
+}
+
+func (m *RepositoryManager) reportRepositoryInfo(ctx context.Context, info stats.RepositoryInfo) {
+ info.Log(ctx)
+
+ m.reportDataStructureExistence("commit_graph", info.CommitGraph.Exists)
+ m.reportDataStructureExistence("commit_graph_bloom_filters", info.CommitGraph.HasBloomFilters)
+ m.reportDataStructureExistence("commit_graph_generation_data", info.CommitGraph.HasGenerationData)
+ m.reportDataStructureExistence("commit_graph_generation_data_overflow", info.CommitGraph.HasGenerationDataOverflow)
+ m.reportDataStructureExistence("bitmap", info.Packfiles.Bitmap.Exists)
+ m.reportDataStructureExistence("multi_pack_index", info.Packfiles.HasMultiPackIndex)
+ m.reportDataStructureExistence("multi_pack_index_bitmap", info.Packfiles.MultiPackIndexBitmap.Exists)
+
+ m.reportDataStructureCount("loose_objects", info.LooseObjects.Count)
+ m.reportDataStructureCount("loose_objects_stale_count", info.LooseObjects.StaleCount)
+ m.reportDataStructureCount("commit_graph_chain", info.CommitGraph.CommitGraphChainLength)
+ m.reportDataStructureCount("packfiles", info.Packfiles.Count)
+ m.reportDataStructureCount("loose_references", info.References.LooseReferencesCount)
+
+ m.reportDataStructureSize("loose_objects", info.LooseObjects.Size)
+ m.reportDataStructureSize("loose_objects_stale", info.LooseObjects.StaleSize)
+ m.reportDataStructureSize("packfiles", info.Packfiles.Size)
+ m.reportDataStructureSize("packed_references", info.References.PackedReferencesSize)
+}
+
+func (m *RepositoryManager) reportDataStructureExistence(dataStructure string, exists bool) {
+ m.dataStructureExistence.WithLabelValues(dataStructure, strconv.FormatBool(exists)).Inc()
+}
+
+func (m *RepositoryManager) reportDataStructureCount(dataStructure string, count uint64) {
+ m.dataStructureCount.WithLabelValues(dataStructure).Observe(float64(count))
+}
+
+func (m *RepositoryManager) reportDataStructureSize(dataStructure string, size uint64) {
+ m.dataStructureSize.WithLabelValues(dataStructure).Observe(float64(size))
}
func optimizeRepository(
ctx context.Context,
m *RepositoryManager,
repo *localrepo.Repo,
- cfg OptimizeRepositoryConfig,
+ strategy OptimizationStrategy,
) error {
totalTimer := prometheus.NewTimer(m.tasksLatency.WithLabelValues("total"))
totalStatus := "failure"
@@ -99,7 +145,7 @@ func optimizeRepository(
timer.ObserveDuration()
timer = prometheus.NewTimer(m.tasksLatency.WithLabelValues("repack"))
- didRepack, repackCfg, err := repackIfNeeded(ctx, repo, cfg.Strategy)
+ didRepack, repackCfg, err := repackIfNeeded(ctx, repo, strategy)
if err != nil {
optimizations["packed_objects_full"] = "failure"
optimizations["packed_objects_incremental"] = "failure"
@@ -119,7 +165,7 @@ func optimizeRepository(
timer.ObserveDuration()
timer = prometheus.NewTimer(m.tasksLatency.WithLabelValues("prune"))
- didPrune, err := pruneIfNeeded(ctx, repo, cfg.Strategy)
+ didPrune, err := pruneIfNeeded(ctx, repo, strategy)
if err != nil {
optimizations["pruned_objects"] = "failure"
return fmt.Errorf("could not prune: %w", err)
@@ -129,7 +175,7 @@ func optimizeRepository(
timer.ObserveDuration()
timer = prometheus.NewTimer(m.tasksLatency.WithLabelValues("pack-refs"))
- didPackRefs, err := packRefsIfNeeded(ctx, repo, cfg.Strategy)
+ didPackRefs, err := packRefsIfNeeded(ctx, repo, strategy)
if err != nil {
optimizations["packed_refs"] = "failure"
return fmt.Errorf("could not pack refs: %w", err)
@@ -139,7 +185,7 @@ func optimizeRepository(
timer.ObserveDuration()
timer = prometheus.NewTimer(m.tasksLatency.WithLabelValues("commit-graph"))
- if didWriteCommitGraph, writeCommitGraphCfg, err := writeCommitGraphIfNeeded(ctx, repo, cfg.Strategy); err != nil {
+ if didWriteCommitGraph, writeCommitGraphCfg, err := writeCommitGraphIfNeeded(ctx, repo, strategy); err != nil {
optimizations["written_commit_graph_full"] = "failure"
optimizations["written_commit_graph_incremental"] = "failure"
return fmt.Errorf("could not write commit-graph: %w", err)
diff --git a/internal/git/housekeeping/optimize_repository_test.go b/internal/git/housekeeping/optimize_repository_test.go
index 78e0a790b..caddf732b 100644
--- a/internal/git/housekeeping/optimize_repository_test.go
+++ b/internal/git/housekeeping/optimize_repository_test.go
@@ -18,6 +18,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
+ gitalycfgprom "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config/prometheus"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag"
"gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
@@ -398,7 +399,7 @@ gitaly_housekeeping_tasks_total{housekeeping_task="total", status="success"} 1
require.Equal(t, tc.expectedErr, err)
expectedMetrics := tc.expectedMetrics
- if IsPoolRepository(repoProto) && tc.expectedMetricsForPool != "" {
+ if stats.IsPoolRepository(repoProto) && tc.expectedMetricsForPool != "" {
expectedMetrics = tc.expectedMetricsForPool
}
@@ -424,13 +425,12 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) {
})
repo := localrepo.NewTestRepo(t, cfg, repoProto)
- manager := &RepositoryManager{
- optimizeFunc: func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizeRepositoryConfig) error {
- reqReceivedCh <- struct{}{}
- ch <- struct{}{}
+ manager := NewManager(gitalycfgprom.Config{}, nil)
+ manager.optimizeFunc = func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizationStrategy) error {
+ reqReceivedCh <- struct{}{}
+ ch <- struct{}{}
- return nil
- },
+ return nil
}
go func() {
@@ -460,17 +460,16 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) {
reposOptimized := make(map[string]struct{})
- manager := &RepositoryManager{
- optimizeFunc: func(_ context.Context, _ *RepositoryManager, repo *localrepo.Repo, _ OptimizeRepositoryConfig) error {
- reposOptimized[repo.GetRelativePath()] = struct{}{}
+ manager := NewManager(gitalycfgprom.Config{}, nil)
+ manager.optimizeFunc = func(_ context.Context, _ *RepositoryManager, repo *localrepo.Repo, _ OptimizationStrategy) error {
+ reposOptimized[repo.GetRelativePath()] = struct{}{}
- if repo.GitRepo.GetRelativePath() == repoFirst.GetRelativePath() {
- reqReceivedCh <- struct{}{}
- ch <- struct{}{}
- }
+ if repo.GitRepo.GetRelativePath() == repoFirst.GetRelativePath() {
+ reqReceivedCh <- struct{}{}
+ ch <- struct{}{}
+ }
- return nil
- },
+ return nil
}
// We block in the first call so that we can assert that a second call
@@ -498,17 +497,16 @@ func TestOptimizeRepository_ConcurrencyLimit(t *testing.T) {
repo := localrepo.NewTestRepo(t, cfg, repoProto)
var optimizations int
- manager := &RepositoryManager{
- optimizeFunc: func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizeRepositoryConfig) error {
- optimizations++
+ manager := NewManager(gitalycfgprom.Config{}, nil)
+ manager.optimizeFunc = func(context.Context, *RepositoryManager, *localrepo.Repo, OptimizationStrategy) error {
+ optimizations++
- if optimizations == 1 {
- reqReceivedCh <- struct{}{}
- ch <- struct{}{}
- }
+ if optimizations == 1 {
+ reqReceivedCh <- struct{}{}
+ ch <- struct{}{}
+ }
- return nil
- },
+ return nil
}
var wg sync.WaitGroup
diff --git a/internal/git/objectpool/fetch_test.go b/internal/git/objectpool/fetch_test.go
index 517b91d9f..06e097a36 100644
--- a/internal/git/objectpool/fetch_test.go
+++ b/internal/git/objectpool/fetch_test.go
@@ -291,7 +291,9 @@ func TestObjectPool_logStats(t *testing.T) {
expectedFields: logrus.Fields{
"references.dangling": referencedObjectTypes{},
"references.normal": referencedObjectTypes{},
- "repository_info": stats.RepositoryInfo{},
+ "repository_info": stats.RepositoryInfo{
+ IsObjectPool: true,
+ },
},
},
{
@@ -307,6 +309,7 @@ func TestObjectPool_logStats(t *testing.T) {
Commits: 1,
},
"repository_info": stats.RepositoryInfo{
+ IsObjectPool: true,
LooseObjects: stats.LooseObjectsInfo{
Count: 2,
Size: 142,
@@ -330,6 +333,7 @@ func TestObjectPool_logStats(t *testing.T) {
},
"references.normal": referencedObjectTypes{},
"repository_info": stats.RepositoryInfo{
+ IsObjectPool: true,
LooseObjects: stats.LooseObjectsInfo{
Count: 2,
Size: 142,
diff --git a/internal/git/objectpool/pool.go b/internal/git/objectpool/pool.go
index bdde64629..2243a76f7 100644
--- a/internal/git/objectpool/pool.go
+++ b/internal/git/objectpool/pool.go
@@ -15,6 +15,7 @@ import (
"gitlab.com/gitlab-org/gitaly/v15/internal/git/catfile"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/transaction"
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
@@ -57,7 +58,7 @@ func FromProto(
return nil, err
}
- if !housekeeping.IsPoolRepository(proto.GetRepository()) {
+ if !stats.IsPoolRepository(proto.GetRepository()) {
// When creating repositories in the ObjectPool service we will first create the
// repository in a temporary directory. So we need to check whether the path we see
// here is in such a temporary directory and let it pass.
diff --git a/internal/git/housekeeping/object_pool.go b/internal/git/stats/object_pool.go
index 732045e09..3235f6d06 100644
--- a/internal/git/housekeeping/object_pool.go
+++ b/internal/git/stats/object_pool.go
@@ -1,4 +1,4 @@
-package housekeeping
+package stats
import (
"regexp"
diff --git a/internal/git/housekeeping/object_pool_test.go b/internal/git/stats/object_pool_test.go
index f534ab7ee..1737c5a80 100644
--- a/internal/git/housekeeping/object_pool_test.go
+++ b/internal/git/stats/object_pool_test.go
@@ -1,4 +1,4 @@
-package housekeeping
+package stats
import (
"testing"
diff --git a/internal/git/stats/repository_info.go b/internal/git/stats/repository_info.go
index a0083f88a..87e7fb8b9 100644
--- a/internal/git/stats/repository_info.go
+++ b/internal/git/stats/repository_info.go
@@ -55,6 +55,8 @@ func LogRepositoryInfo(ctx context.Context, repo *localrepo.Repo) {
// RepositoryInfo contains information about the repository.
type RepositoryInfo struct {
+ // IsObjectPool determines whether the repository is an object pool or a normal repository.
+ IsObjectPool bool `json:"is_object_pool"`
// LooseObjects contains information about loose objects.
LooseObjects LooseObjectsInfo `json:"loose_objects"`
// Packfiles contains information about packfiles.
@@ -78,6 +80,8 @@ func RepositoryInfoForRepository(repo *localrepo.Repo) (RepositoryInfo, error) {
return RepositoryInfo{}, err
}
+ info.IsObjectPool = IsPoolRepository(repo)
+
info.LooseObjects, err = LooseObjectsInfoForRepository(repo, time.Now().Add(StaleObjectsGracePeriod))
if err != nil {
return RepositoryInfo{}, fmt.Errorf("counting loose objects: %w", err)
diff --git a/internal/gitaly/service/objectpool/create.go b/internal/gitaly/service/objectpool/create.go
index 94a614a62..7f432c901 100644
--- a/internal/gitaly/service/objectpool/create.go
+++ b/internal/gitaly/service/objectpool/create.go
@@ -3,8 +3,8 @@ package objectpool
import (
"context"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/objectpool"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/repoutil"
"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
@@ -25,7 +25,7 @@ func (s *server) CreateObjectPool(ctx context.Context, in *gitalypb.CreateObject
return nil, errMissingPool
}
- if !housekeeping.IsPoolRepository(poolRepo) {
+ if !stats.IsPoolRepository(poolRepo) {
return nil, errInvalidPoolDir
}
diff --git a/internal/gitaly/service/repository/optimize.go b/internal/gitaly/service/repository/optimize.go
index 32bffb60a..9fe0e4f95 100644
--- a/internal/gitaly/service/repository/optimize.go
+++ b/internal/gitaly/service/repository/optimize.go
@@ -4,6 +4,7 @@ import (
"context"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service"
"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
@@ -16,27 +17,23 @@ func (s *server) OptimizeRepository(ctx context.Context, in *gitalypb.OptimizeRe
repo := s.localrepo(in.GetRepository())
- var strategyOpt housekeeping.OptimizeRepositoryOption
+ var strategyConstructor housekeeping.OptimizationStrategyConstructor
switch in.GetStrategy() {
case gitalypb.OptimizeRepositoryRequest_STRATEGY_UNSPECIFIED, gitalypb.OptimizeRepositoryRequest_STRATEGY_HEURISTICAL:
- strategy, err := housekeeping.NewHeuristicalOptimizationStrategy(ctx, repo)
- if err != nil {
- return nil, structerr.NewInternal("creating heuristical optimization strategy: %w", err)
+ strategyConstructor = func(info stats.RepositoryInfo) housekeeping.OptimizationStrategy {
+ return housekeeping.NewHeuristicalOptimizationStrategy(info)
}
-
- strategyOpt = housekeeping.WithOptimizationStrategy(strategy)
case gitalypb.OptimizeRepositoryRequest_STRATEGY_EAGER:
- strategy, err := housekeeping.NewEagerOptimizationStrategy(ctx, repo)
- if err != nil {
- return nil, structerr.NewInternal("creating eager optimization strategy: %w", err)
+ strategyConstructor = func(info stats.RepositoryInfo) housekeeping.OptimizationStrategy {
+ return housekeeping.NewEagerOptimizationStrategy(info)
}
-
- strategyOpt = housekeeping.WithOptimizationStrategy(strategy)
default:
return nil, structerr.NewInvalidArgument("unsupported optimization strategy %d", in.GetStrategy())
}
- if err := s.housekeepingManager.OptimizeRepository(ctx, repo, strategyOpt); err != nil {
+ if err := s.housekeepingManager.OptimizeRepository(ctx, repo,
+ housekeeping.WithOptimizationStrategyConstructor(strategyConstructor),
+ ); err != nil {
return nil, structerr.NewInternal("%w", err)
}
diff --git a/internal/gitaly/service/repository/optimize_test.go b/internal/gitaly/service/repository/optimize_test.go
index 8d58e4313..f713de3c2 100644
--- a/internal/gitaly/service/repository/optimize_test.go
+++ b/internal/gitaly/service/repository/optimize_test.go
@@ -260,7 +260,7 @@ func testOptimizeRepository(t *testing.T, ctx context.Context) {
type mockHousekeepingManager struct {
housekeeping.Manager
- cfgCh chan housekeeping.OptimizeRepositoryConfig
+ strategyCh chan housekeeping.OptimizationStrategy
}
func (m mockHousekeepingManager) OptimizeRepository(_ context.Context, _ *localrepo.Repo, opts ...housekeeping.OptimizeRepositoryOption) error {
@@ -269,7 +269,7 @@ func (m mockHousekeepingManager) OptimizeRepository(_ context.Context, _ *localr
opt(&cfg)
}
- m.cfgCh <- cfg
+ m.strategyCh <- cfg.StrategyConstructor(stats.RepositoryInfo{})
return nil
}
@@ -277,7 +277,7 @@ func TestOptimizeRepository_strategy(t *testing.T) {
t.Parallel()
housekeepingManager := mockHousekeepingManager{
- cfgCh: make(chan housekeeping.OptimizeRepositoryConfig, 1),
+ strategyCh: make(chan housekeeping.OptimizationStrategy, 1),
}
ctx := testhelper.Context(t)
@@ -286,18 +286,16 @@ func TestOptimizeRepository_strategy(t *testing.T) {
repoProto, _ := gittest.CreateRepository(t, ctx, cfg)
for _, tc := range []struct {
- desc string
- request *gitalypb.OptimizeRepositoryRequest
- expectedCfg housekeeping.OptimizeRepositoryConfig
+ desc string
+ request *gitalypb.OptimizeRepositoryRequest
+ expectedStrategy housekeeping.OptimizationStrategy
}{
{
desc: "no strategy",
request: &gitalypb.OptimizeRepositoryRequest{
Repository: repoProto,
},
- expectedCfg: housekeeping.OptimizeRepositoryConfig{
- Strategy: housekeeping.HeuristicalOptimizationStrategy{},
- },
+ expectedStrategy: housekeeping.HeuristicalOptimizationStrategy{},
},
{
desc: "heuristical strategy",
@@ -305,9 +303,7 @@ func TestOptimizeRepository_strategy(t *testing.T) {
Repository: repoProto,
Strategy: gitalypb.OptimizeRepositoryRequest_STRATEGY_HEURISTICAL,
},
- expectedCfg: housekeeping.OptimizeRepositoryConfig{
- Strategy: housekeeping.HeuristicalOptimizationStrategy{},
- },
+ expectedStrategy: housekeeping.HeuristicalOptimizationStrategy{},
},
{
desc: "eager strategy",
@@ -315,9 +311,7 @@ func TestOptimizeRepository_strategy(t *testing.T) {
Repository: repoProto,
Strategy: gitalypb.OptimizeRepositoryRequest_STRATEGY_EAGER,
},
- expectedCfg: housekeeping.OptimizeRepositoryConfig{
- Strategy: housekeeping.EagerOptimizationStrategy{},
- },
+ expectedStrategy: housekeeping.EagerOptimizationStrategy{},
},
} {
t.Run(tc.desc, func(t *testing.T) {
@@ -325,7 +319,7 @@ func TestOptimizeRepository_strategy(t *testing.T) {
require.NoError(t, err)
testhelper.ProtoEqual(t, &gitalypb.OptimizeRepositoryResponse{}, response)
- require.Equal(t, tc.expectedCfg, <-housekeepingManager.cfgCh)
+ require.Equal(t, tc.expectedStrategy, <-housekeepingManager.strategyCh)
})
}
}
diff --git a/internal/praefect/delete_object_pool.go b/internal/praefect/delete_object_pool.go
index bc45dfcf5..e30f07d53 100644
--- a/internal/praefect/delete_object_pool.go
+++ b/internal/praefect/delete_object_pool.go
@@ -4,8 +4,8 @@ import (
"context"
"fmt"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/objectpool"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
objectpoolsvc "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/objectpool"
"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore"
"gitlab.com/gitlab-org/gitaly/v15/internal/structerr"
@@ -29,7 +29,7 @@ func DeleteObjectPoolHandler(rs datastore.RepositoryStore, conns Connections) gr
return nil, err
}
- if !housekeeping.IsRailsPoolRepository(repo) {
+ if !stats.IsRailsPoolRepository(repo) {
return nil, structerr.NewInvalidArgument("%w", objectpool.ErrInvalidPoolDir)
}
diff --git a/internal/praefect/router_per_repository.go b/internal/praefect/router_per_repository.go
index d9cc071fa..c64da8f7a 100644
--- a/internal/praefect/router_per_repository.go
+++ b/internal/praefect/router_per_repository.go
@@ -5,7 +5,7 @@ import (
"errors"
"fmt"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/housekeeping"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git/stats"
"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore"
"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/nodes"
"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/praefectutil"
@@ -311,7 +311,7 @@ func (r *PerRepositoryRouter) RouteRepositoryCreation(ctx context.Context, virtu
}
replicaPath := praefectutil.DeriveReplicaPath(id)
- if housekeeping.IsRailsPoolRepository(&gitalypb.Repository{
+ if stats.IsRailsPoolRepository(&gitalypb.Repository{
StorageName: virtualStorage,
RelativePath: relativePath,
}) {