Use core delta islands to increase opportunity of pack reuse

Including counter to track how often repack happens.
Adding feature toggle to enable usage of delta core islands.

Part of: https://gitlab.com/gitlab-org/gitaly/issues/1519

4 files changed, 59 insertions, 8 deletions

diff --git a/changelogs/unreleased/ps-use-core-delta-islands.yml b/changelogs/unreleased/ps-use-core-delta-islands.yml
new file mode 100644
index 000000000..a6dfb3122
--- /dev/null
+++ b/changelogs/unreleased/ps-use-core-delta-islands.yml
@@ -0,0 +1,5 @@
+---
+title: Use core delta islands to increase opportunity of pack reuse
+merge_request: 1775
+author:
+type: added
diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go
index b042706a0..37ad27746 100644
--- a/internal/git/objectpool/fetch.go
+++ b/internal/git/objectpool/fetch.go
@@ -12,12 +12,14 @@ import (
 	"strings"
 
 	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/gitlab-org/gitaly/internal/command"
 	"gitlab.com/gitlab-org/gitaly/internal/git"
 	"gitlab.com/gitlab-org/gitaly/internal/git/repository"
 	"gitlab.com/gitlab-org/gitaly/internal/git/updateref"
 	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 )
 
@@ -26,6 +28,21 @@ const (
 	sourceRefNamespace = "refs/remotes/" + sourceRemote
 )
 
+var (
+	// FullRepackCounter is a counter used to track full repacks (with/without core delta islands).
+	FullRepackCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gitaly_full_repack_total",
+			Help: "Counter of repack commands run with/without core delta island use",
+		},
+		[]string{"core_island"},
+	)
+)
+
+func init() {
+	prometheus.Register(FullRepackCounter)
+}
+
 // FetchFromOrigin initializes the pool and fetches the objects from its origin repository
 func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repository) error {
 	if err := o.Init(ctx); err != nil {
@@ -171,11 +188,24 @@ func rescueDanglingObjects(ctx context.Context, repo repository.GitRepo) error {
 }
 
 func repackPool(ctx context.Context, pool repository.GitRepo) error {
-	repackArgs := []git.Option{
-		git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/heads"},
-		git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/tags"},
-		git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"},
+	var repackArgs []git.Option
+	if featureflag.IsEnabled(ctx, featureflag.UseCoreDeltaIslands) {
+		FullRepackCounter.WithLabelValues("yes").Inc()
+		repackArgs = []git.Option{
+			git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/he(a)ds"},
+			git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/t(a)gs"},
+			git.ValueFlag{"-c", "pack.islandCore=a"},
+		}
+	} else {
+		FullRepackCounter.WithLabelValues("no").Inc()
+		repackArgs = []git.Option{
+			git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/heads"},
+			git.ValueFlag{"-c", "pack.island=" + sourceRefNamespace + "/tags"},
+		}
 	}
+
+	repackArgs = append(repackArgs, git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"})
+
 	repackCmd, err := git.SafeCmd(ctx, pool, repackArgs, git.SubCmd{
 		Name:  "repack",
 		Flags: []git.Option{git.Flag{"-aidb"}},
diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go
index a11d2cbee..f9555a85d 100644
--- a/internal/metadata/featureflag/feature_flags.go
+++ b/internal/metadata/featureflag/feature_flags.go
@@ -16,6 +16,8 @@ const (
 	// The old one with query fot Info before fetching info about Commit
 	// or the new one that skips Info call and checks object type in Commit method call.
 	CommitWithoutBatchCheck = "commit_without_batch_check"
+	// UseCoreDeltaIslands enables support of core delta islands for 'repack'.
+	UseCoreDeltaIslands = "use_core_delta_islands"
 )
 
 const (
diff --git a/internal/service/repository/repack.go b/internal/service/repository/repack.go
index 6a72b1933..ca23c039e 100644
--- a/internal/service/repository/repack.go
+++ b/internal/service/repository/repack.go
@@ -6,7 +6,9 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"gitlab.com/gitlab-org/gitaly/internal/git"
+	"gitlab.com/gitlab-org/gitaly/internal/git/objectpool"
 	"gitlab.com/gitlab-org/gitaly/internal/git/repository"
+	"gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -68,12 +70,24 @@ func repackCommand(ctx context.Context, repo repository.GitRepo, bitmap bool, ar
 }
 
 func repackConfig(ctx context.Context, bitmap bool) []git.Option {
-	args := []git.Option{
-		git.ValueFlag{"-c", "pack.island=refs/heads"},
-		git.ValueFlag{"-c", "pack.island=refs/tags"},
-		git.ValueFlag{"-c", "repack.useDeltaIslands=true"},
+	var args []git.Option
+	if featureflag.IsEnabled(ctx, featureflag.UseCoreDeltaIslands) {
+		objectpool.FullRepackCounter.WithLabelValues("yes").Inc()
+		args = []git.Option{
+			git.ValueFlag{"-c", "pack.island=r(e)fs/heads"},
+			git.ValueFlag{"-c", "pack.island=r(e)fs/tags"},
+			git.ValueFlag{"-c", "pack.islandCore=e"},
+		}
+	} else {
+		objectpool.FullRepackCounter.WithLabelValues("no").Inc()
+		args = []git.Option{
+			git.ValueFlag{"-c", "pack.island=refs/heads"},
+			git.ValueFlag{"-c", "pack.island=refs/tags"},
+		}
 	}
 
+	args = append(args, git.ValueFlag{"-c", "repack.useDeltaIslands=true"})
+
 	if bitmap {
 		args = append(args, git.ValueFlag{"-c", "repack.writeBitmaps=true"})
 		args = append(args, git.ValueFlag{"-c", "pack.writeBitmapHashCache=true"})