Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPatrick Steinhardt <psteinhardt@gitlab.com>2022-10-19 15:11:36 +0300
committerPatrick Steinhardt <psteinhardt@gitlab.com>2022-10-19 17:11:26 +0300
commit2e8976dddc947445f33e71ba15bfd6ad6b172138 (patch)
treea32e1ff638b7854980f3c6b4cdbe9fe181803702
parentfdcad2a1e5c6b8803c6b42a0c253f5b27aab307a (diff)
git/stats: Merge files that implement objects stats
We have two different code files that implement statistics specific to the object database. Merge them into a single file "objects_info.go" to make them easier to discover.
-rw-r--r--internal/git/stats/objects_info.go (renamed from internal/git/stats/git.go)64
-rw-r--r--internal/git/stats/objects_info_test.go (renamed from internal/git/stats/git_test.go)58
-rw-r--r--internal/git/stats/profile.go71
-rw-r--r--internal/git/stats/profile_test.go73
4 files changed, 122 insertions, 144 deletions
diff --git a/internal/git/stats/git.go b/internal/git/stats/objects_info.go
index ce9794c72..12a8e4643 100644
--- a/internal/git/stats/git.go
+++ b/internal/git/stats/objects_info.go
@@ -3,7 +3,11 @@ package stats
import (
"bufio"
"context"
+ "errors"
"io"
+ "io/fs"
+ "os"
+ "path/filepath"
"strconv"
"strings"
@@ -11,6 +15,66 @@ import (
"gitlab.com/gitlab-org/gitaly/v15/internal/git"
)
+// HasBitmap returns whether or not the repository contains an object bitmap.
+func HasBitmap(repoPath string) (bool, error) {
+ bitmaps, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "*.bitmap"))
+ if err != nil {
+ return false, err
+ }
+
+ return len(bitmaps) > 0, nil
+}
+
+// PackfilesCount returns the number of packfiles a repository has.
+func PackfilesCount(repoPath string) (int, error) {
+ packFiles, err := GetPackfiles(repoPath)
+ if err != nil {
+ return 0, err
+ }
+
+ return len(packFiles), nil
+}
+
+// GetPackfiles returns the FileInfo of packfiles inside a repository.
+func GetPackfiles(repoPath string) ([]fs.DirEntry, error) {
+ files, err := os.ReadDir(filepath.Join(repoPath, "objects/pack/"))
+ if err != nil {
+ return nil, err
+ }
+
+ var packFiles []fs.DirEntry
+ for _, f := range files {
+ if filepath.Ext(f.Name()) == ".pack" {
+ packFiles = append(packFiles, f)
+ }
+ }
+
+ return packFiles, nil
+}
+
+// LooseObjects returns the number of loose objects that are not in a packfile.
+func LooseObjects(ctx context.Context, repo git.RepositoryExecutor) (int64, error) {
+ cmd, err := repo.Exec(ctx, git.SubCmd{
+ Name: "count-objects",
+ Flags: []git.Option{git.Flag{Name: "--verbose"}},
+ })
+ if err != nil {
+ return 0, err
+ }
+
+ objectStats, err := readObjectInfoStatistic(cmd)
+ if err != nil {
+ return 0, err
+ }
+
+ count, ok := objectStats["count"].(int64)
+ if !ok {
+ return 0, errors.New("could not get object count")
+ }
+
+ return count, nil
+}
+
// LogObjectsInfo read statistics of the git repo objects
// and logs it under 'count-objects' key as structured entry.
func LogObjectsInfo(ctx context.Context, repo git.RepositoryExecutor) {
diff --git a/internal/git/stats/git_test.go b/internal/git/stats/objects_info_test.go
index a07e82471..7d30a86d5 100644
--- a/internal/git/stats/git_test.go
+++ b/internal/git/stats/objects_info_test.go
@@ -7,11 +7,13 @@ import (
"path/filepath"
"strings"
"testing"
+ "time"
"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
"github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitaly/v15/internal/git"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
"gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config"
@@ -20,6 +22,62 @@ import (
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
)
+func TestRepositoryProfile(t *testing.T) {
+ ctx := testhelper.Context(t)
+ cfg := testcfg.Build(t)
+
+ repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
+ SkipCreationViaService: true,
+ })
+ repo := localrepo.NewTestRepo(t, cfg, repoProto)
+
+ hasBitmap, err := HasBitmap(repoPath)
+ require.NoError(t, err)
+ require.False(t, hasBitmap, "repository should not have a bitmap initially")
+ packfiles, err := GetPackfiles(repoPath)
+ require.NoError(t, err)
+ require.Empty(t, packfiles)
+ packfilesCount, err := PackfilesCount(repoPath)
+ require.NoError(t, err)
+ require.Zero(t, packfilesCount)
+
+ blobs := 10
+ blobIDs := gittest.WriteBlobs(t, cfg, repoPath, blobs)
+
+ looseObjects, err := LooseObjects(ctx, repo)
+ require.NoError(t, err)
+ require.Equal(t, int64(blobs), looseObjects)
+
+ for _, blobID := range blobIDs {
+ commitID := gittest.WriteCommit(t, cfg, repoPath,
+ gittest.WithTreeEntries(gittest.TreeEntry{
+ Mode: "100644", Path: "blob", OID: git.ObjectID(blobID),
+ }),
+ )
+ gittest.Exec(t, cfg, "-C", repoPath, "update-ref", "refs/heads/"+blobID, commitID.String())
+ }
+
+ // write a loose object
+ gittest.WriteBlobs(t, cfg, repoPath, 1)
+
+ gittest.Exec(t, cfg, "-C", repoPath, "repack", "-A", "-b", "-d")
+
+ looseObjects, err = LooseObjects(ctx, repo)
+ require.NoError(t, err)
+ require.Equal(t, int64(1), looseObjects)
+
+ // write another loose object
+ blobID := gittest.WriteBlobs(t, cfg, repoPath, 1)[0]
+
+ // due to OS semantics, ensure that the blob has a timestamp that is after the packfile
+ theFuture := time.Now().Add(10 * time.Minute)
+ require.NoError(t, os.Chtimes(filepath.Join(repoPath, "objects", blobID[0:2], blobID[2:]), theFuture, theFuture))
+
+ looseObjects, err = LooseObjects(ctx, repo)
+ require.NoError(t, err)
+ require.Equal(t, int64(2), looseObjects)
+}
+
func TestLogObjectInfo(t *testing.T) {
ctx := testhelper.Context(t)
cfg := testcfg.Build(t)
diff --git a/internal/git/stats/profile.go b/internal/git/stats/profile.go
deleted file mode 100644
index 44cb007d7..000000000
--- a/internal/git/stats/profile.go
+++ /dev/null
@@ -1,71 +0,0 @@
-package stats
-
-import (
- "context"
- "errors"
- "io/fs"
- "os"
- "path/filepath"
-
- "gitlab.com/gitlab-org/gitaly/v15/internal/git"
-)
-
-// HasBitmap returns whether or not the repository contains an object bitmap.
-func HasBitmap(repoPath string) (bool, error) {
- bitmaps, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "*.bitmap"))
- if err != nil {
- return false, err
- }
-
- return len(bitmaps) > 0, nil
-}
-
-// PackfilesCount returns the number of packfiles a repository has.
-func PackfilesCount(repoPath string) (int, error) {
- packFiles, err := GetPackfiles(repoPath)
- if err != nil {
- return 0, err
- }
-
- return len(packFiles), nil
-}
-
-// GetPackfiles returns the FileInfo of packfiles inside a repository.
-func GetPackfiles(repoPath string) ([]fs.DirEntry, error) {
- files, err := os.ReadDir(filepath.Join(repoPath, "objects/pack/"))
- if err != nil {
- return nil, err
- }
-
- var packFiles []fs.DirEntry
- for _, f := range files {
- if filepath.Ext(f.Name()) == ".pack" {
- packFiles = append(packFiles, f)
- }
- }
-
- return packFiles, nil
-}
-
-// LooseObjects returns the number of loose objects that are not in a packfile.
-func LooseObjects(ctx context.Context, repo git.RepositoryExecutor) (int64, error) {
- cmd, err := repo.Exec(ctx, git.SubCmd{
- Name: "count-objects",
- Flags: []git.Option{git.Flag{Name: "--verbose"}},
- })
- if err != nil {
- return 0, err
- }
-
- objectStats, err := readObjectInfoStatistic(cmd)
- if err != nil {
- return 0, err
- }
-
- count, ok := objectStats["count"].(int64)
- if !ok {
- return 0, errors.New("could not get object count")
- }
-
- return count, nil
-}
diff --git a/internal/git/stats/profile_test.go b/internal/git/stats/profile_test.go
deleted file mode 100644
index 566941522..000000000
--- a/internal/git/stats/profile_test.go
+++ /dev/null
@@ -1,73 +0,0 @@
-//go:build !gitaly_test_sha256
-
-package stats
-
-import (
- "os"
- "path/filepath"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
- "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo"
- "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper"
- "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg"
-)
-
-func TestRepositoryProfile(t *testing.T) {
- ctx := testhelper.Context(t)
- cfg := testcfg.Build(t)
-
- repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{
- SkipCreationViaService: true,
- })
- repo := localrepo.NewTestRepo(t, cfg, repoProto)
-
- hasBitmap, err := HasBitmap(repoPath)
- require.NoError(t, err)
- require.False(t, hasBitmap, "repository should not have a bitmap initially")
- packfiles, err := GetPackfiles(repoPath)
- require.NoError(t, err)
- require.Empty(t, packfiles)
- packfilesCount, err := PackfilesCount(repoPath)
- require.NoError(t, err)
- require.Zero(t, packfilesCount)
-
- blobs := 10
- blobIDs := gittest.WriteBlobs(t, cfg, repoPath, blobs)
-
- looseObjects, err := LooseObjects(ctx, repo)
- require.NoError(t, err)
- require.Equal(t, int64(blobs), looseObjects)
-
- for _, blobID := range blobIDs {
- commitID := gittest.WriteCommit(t, cfg, repoPath,
- gittest.WithTreeEntries(gittest.TreeEntry{
- Mode: "100644", Path: "blob", OID: git.ObjectID(blobID),
- }),
- )
- gittest.Exec(t, cfg, "-C", repoPath, "update-ref", "refs/heads/"+blobID, commitID.String())
- }
-
- // write a loose object
- gittest.WriteBlobs(t, cfg, repoPath, 1)
-
- gittest.Exec(t, cfg, "-C", repoPath, "repack", "-A", "-b", "-d")
-
- looseObjects, err = LooseObjects(ctx, repo)
- require.NoError(t, err)
- require.Equal(t, int64(1), looseObjects)
-
- // write another loose object
- blobID := gittest.WriteBlobs(t, cfg, repoPath, 1)[0]
-
- // due to OS semantics, ensure that the blob has a timestamp that is after the packfile
- theFuture := time.Now().Add(10 * time.Minute)
- require.NoError(t, os.Chtimes(filepath.Join(repoPath, "objects", blobID[0:2], blobID[2:]), theFuture, theFuture))
-
- looseObjects, err = LooseObjects(ctx, repo)
- require.NoError(t, err)
- require.Equal(t, int64(2), looseObjects)
-}