Merge branch 'jv-count-dangling' into 'master'

Count dangling refs before/after FetchIntoObjectPool

See merge request gitlab-org/gitaly!1585

3 files changed, 83 insertions, 0 deletions

diff --git a/changelogs/unreleased/jv-count-dangling.yml b/changelogs/unreleased/jv-count-dangling.yml
new file mode 100644
index 000000000..9d2ad00cd
--- /dev/null
+++ b/changelogs/unreleased/jv-count-dangling.yml
@@ -0,0 +1,5 @@
+---
+title: Count dangling refs before/after FetchIntoObjectPool
+merge_request: 1585
+author:
+type: other
diff --git a/internal/git/objectpool/fetch.go b/internal/git/objectpool/fetch.go
index a5240fbfd..6230bb049 100644
--- a/internal/git/objectpool/fetch.go
+++ b/internal/git/objectpool/fetch.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"strings"
 
+	grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
 	"gitlab.com/gitlab-org/gitaly/internal/command"
 	"gitlab.com/gitlab-org/gitaly/internal/git"
 	"gitlab.com/gitlab-org/gitaly/internal/git/repository"
@@ -65,6 +66,10 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repos
 		return err
 	}
 
+	if err := logDanglingRefs(ctx, o, "before fetch"); err != nil {
+		return err
+	}
+
 	refSpec := fmt.Sprintf("+refs/*:%s/*", sourceRefNamespace)
 	fetchCmd, err := git.Command(ctx, o, "fetch", "--quiet", sourceRemote, refSpec)
 	if err != nil {
@@ -79,6 +84,10 @@ func (o *ObjectPool) FetchFromOrigin(ctx context.Context, origin *gitalypb.Repos
 		return err
 	}
 
+	if err := logDanglingRefs(ctx, o, "after fetch"); err != nil {
+		return err
+	}
+
 	packRefs, err := git.Command(ctx, o, "pack-refs", "--all")
 	if err != nil {
 		return err
@@ -157,3 +166,37 @@ func repackPool(ctx context.Context, pool repository.GitRepo) error {
 
 	return nil
 }
+
+func logDanglingRefs(ctx context.Context, pool repository.GitRepo, when string) error {
+	forEachRef, err := git.SafeCmd(ctx, pool, nil, git.SubCmd{
+		Name:  "for-each-ref",
+		Flags: []git.Option{git.Flag{"--format=%(objecttype)"}},
+		Args:  []string{danglingObjectNamespace},
+	})
+	if err != nil {
+		return err
+	}
+
+	counts := make(map[string]int)
+	scanner := bufio.NewScanner(forEachRef)
+	for scanner.Scan() {
+		counts[scanner.Text()]++
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+	if err := forEachRef.Wait(); err != nil {
+		return err
+	}
+
+	entry := grpc_logrus.Extract(ctx).WithField("when", when)
+	for _, field := range []string{"blob", "commit", "tag", "tree"} {
+		key := "dangling." + field + ".ref"
+		entry = entry.WithField(key, counts[field])
+	}
+
+	entry.Info("pool dangling ref stats")
+
+	return nil
+}
diff --git a/internal/helper/fieldextractors/fieldextractor.go b/internal/helper/fieldextractors/fieldextractor.go
index 82c889ebe..4318eef5c 100644
--- a/internal/helper/fieldextractors/fieldextractor.go
+++ b/internal/helper/fieldextractors/fieldextractor.go
@@ -94,7 +94,42 @@ func FieldExtractor(fullMethod string, req interface{}) map[string]interface{} {
 	if result == nil {
 		result = make(map[string]interface{})
 	}
+
+	switch {
+	case strings.HasPrefix(fullMethod, "/gitaly.ObjectPoolService/"):
+		addObjectPool(req, result)
+	}
+
 	result["fullMethod"] = fullMethod
 
 	return result
 }
+
+type objectPoolRequest interface {
+	GetObjectPool() *gitalypb.ObjectPool
+}
+
+func addObjectPool(req interface{}, tags map[string]interface{}) {
+	oReq, ok := req.(objectPoolRequest)
+	if !ok {
+		return
+	}
+
+	pool := oReq.GetObjectPool()
+	if pool == nil {
+		return
+	}
+
+	repo := pool.GetRepository()
+	if repo == nil {
+		return
+	}
+
+	for k, v := range map[string]string{
+		"pool.storage":           repo.StorageName,
+		"pool.relativePath":      repo.RelativePath,
+		"pool.sourceProjectPath": repo.GlProjectPath,
+	} {
+		tags[k] = v
+	}
+}