diff options
author | Pavlo Strokov <pstrokov@gitlab.com> | 2020-01-28 11:18:32 +0300 |
---|---|---|
committer | Pavlo Strokov <pstrokov@gitlab.com> | 2020-01-28 11:18:32 +0300 |
commit | fb0413be2ec0606a6fb5be79aa514e69bd5ddc2d (patch) | |
tree | ce332eeb0b37119ee7888b6613240cacbc1f6114 | |
parent | cdb513169e7c062f99aa831f827660f869465454 (diff) | |
parent | 6dd32f2add388eddd2c056e1e523e33d90cc5f02 (diff) |
Merge branch 'ps-use-core-delta-islands' into 'master'
Feature flag: Use core delta islands to increase opportunity of pack reuse
See merge request gitlab-org/gitaly!1775
-rw-r--r-- | changelogs/unreleased/ps-use-core-delta-islands.yml | 5 | ||||
-rw-r--r-- | internal/git/objectpool/fetch.go | 38 | ||||
-rw-r--r-- | internal/metadata/featureflag/feature_flags.go | 2 | ||||
-rw-r--r-- | internal/service/repository/repack.go | 22 |
4 files changed, 59 insertions, 8 deletions
diff --git a/changelogs/unreleased/ps-use-core-delta-islands.yml b/changelogs/unreleased/ps-use-core-delta-islands.yml new file mode 100644 index 000000000..a6dfb3122 --- /dev/null +++ b/changelogs/unreleased/ps-use-core-delta-islands.yml @@ -0,0 +1,5 @@ +--- +title: Use core delta islands to increase opportunity of pack reuse +merge_request: 1775 +author: +type: added diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go index b042706a0..37ad27746 100644 --- a/internal/git/objectpool/fetch.go +++ b/internal/git/objectpool/fetch.go @@ -12,12 +12,14 @@ import ( "strings" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/internal/command" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/git/repository" "gitlab.com/gitlab-org/gitaly/internal/git/updateref" "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" ) @@ -26,6 +28,21 @@ const ( sourceRefNamespace = "refs/remotes/" + sourceRemote ) +var ( + // FullRepackCounter is a counter used to track full repacks (with/without core delta islands). + FullRepackCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitaly_full_repack_total", + Help: "Counter of repack commands run with/without core delta island use", + }, + []string{"core_island"}, + ) +) + +func init() { + prometheus.Register(FullRepackCounter) +} + // FetchFromOrigin initializes the pool and fetches the objects from its origin repository func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repository) error { if err := o.Init(ctx); err != nil { @@ -171,11 +188,24 @@ func rescueDanglingObjects(ctx context.Context, repo repository.GitRepo) error { } func repackPool(ctx context.Context, pool repository.GitRepo) error { - repackArgs := []git.Option{ - git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/heads"}, - git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/tags"}, - git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"}, + var repackArgs []git.Option + if featureflag.IsEnabled(ctx, featureflag.UseCoreDeltaIslands) { + FullRepackCounter.WithLabelValues("yes").Inc() + repackArgs = []git.Option{ + git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/he(a)ds"}, + git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/t(a)gs"}, + git.ValueFlag{"-c", "pack.islandCore=a"}, + } + } else { + FullRepackCounter.WithLabelValues("no").Inc() + repackArgs = []git.Option{ + git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/heads"}, + git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/tags"}, + } } + + repackArgs = append(repackArgs, git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"}) + repackCmd, err := git.SafeCmd(ctx, pool, repackArgs, git.SubCmd{ Name: "repack", Flags: []git.Option{git.Flag{"-aidb"}}, diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go index a11d2cbee..f9555a85d 100644 --- a/internal/metadata/featureflag/feature_flags.go +++ b/internal/metadata/featureflag/feature_flags.go @@ -16,6 +16,8 @@ const ( // The old one with query fot Info before fetching info about Commit // or the new one that skips Info call and checks object type in Commit method call. CommitWithoutBatchCheck = "commit_without_batch_check" + // UseCoreDeltaIslands enables support of core delta islands for 'repack'. + UseCoreDeltaIslands = "use_core_delta_islands" ) const ( diff --git a/internal/service/repository/repack.go b/internal/service/repository/repack.go index 6a72b1933..ca23c039e 100644 --- a/internal/service/repository/repack.go +++ b/internal/service/repository/repack.go @@ -6,7 +6,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/objectpool" "gitlab.com/gitlab-org/gitaly/internal/git/repository" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -68,12 +70,24 @@ func repackCommand(ctx context.Context, repo repository.GitRepo, bitmap bool, ar } func repackConfig(ctx context.Context, bitmap bool) []git.Option { - args := []git.Option{ - git.ValueFlag{"-c", "pack.island=refs/heads"}, - git.ValueFlag{"-c", "pack.island=refs/tags"}, - git.ValueFlag{"-c", "repack.useDeltaIslands=true"}, + var args []git.Option + if featureflag.IsEnabled(ctx, featureflag.UseCoreDeltaIslands) { + objectpool.FullRepackCounter.WithLabelValues("yes").Inc() + args = []git.Option{ + git.ValueFlag{"-c", "pack.island=r(e)fs/heads"}, + git.ValueFlag{"-c", "pack.island=r(e)fs/tags"}, + git.ValueFlag{"-c", "pack.islandCore=e"}, + } + } else { + objectpool.FullRepackCounter.WithLabelValues("no").Inc() + args = []git.Option{ + git.ValueFlag{"-c", "pack.island=refs/heads"}, + git.ValueFlag{"-c", "pack.island=refs/tags"}, + } } + args = append(args, git.ValueFlag{"-c", "repack.useDeltaIslands=true"}) + if bitmap { args = append(args, git.ValueFlag{"-c", "repack.writeBitmaps=true"}) args = append(args, git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"}) |