diff options
author | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-10-19 15:11:36 +0300 |
---|---|---|
committer | Patrick Steinhardt <psteinhardt@gitlab.com> | 2022-10-19 17:11:26 +0300 |
commit | 2e8976dddc947445f33e71ba15bfd6ad6b172138 (patch) | |
tree | a32e1ff638b7854980f3c6b4cdbe9fe181803702 | |
parent | fdcad2a1e5c6b8803c6b42a0c253f5b27aab307a (diff) |
git/stats: Merge files that implement objects stats
We have two different code files that implement statistics specific to
the object database. Merge them into a single file "objects_info.go" to
make them easier to discover.
-rw-r--r-- | internal/git/stats/objects_info.go (renamed from internal/git/stats/git.go) | 64 | ||||
-rw-r--r-- | internal/git/stats/objects_info_test.go (renamed from internal/git/stats/git_test.go) | 58 | ||||
-rw-r--r-- | internal/git/stats/profile.go | 71 | ||||
-rw-r--r-- | internal/git/stats/profile_test.go | 73 |
4 files changed, 122 insertions, 144 deletions
diff --git a/internal/git/stats/git.go b/internal/git/stats/objects_info.go index ce9794c72..12a8e4643 100644 --- a/internal/git/stats/git.go +++ b/internal/git/stats/objects_info.go @@ -3,7 +3,11 @@ package stats import ( "bufio" "context" + "errors" "io" + "io/fs" + "os" + "path/filepath" "strconv" "strings" @@ -11,6 +15,66 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/git" ) +// HasBitmap returns whether or not the repository contains an object bitmap. +func HasBitmap(repoPath string) (bool, error) { + bitmaps, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "*.bitmap")) + if err != nil { + return false, err + } + + return len(bitmaps) > 0, nil +} + +// PackfilesCount returns the number of packfiles a repository has. +func PackfilesCount(repoPath string) (int, error) { + packFiles, err := GetPackfiles(repoPath) + if err != nil { + return 0, err + } + + return len(packFiles), nil +} + +// GetPackfiles returns the FileInfo of packfiles inside a repository. +func GetPackfiles(repoPath string) ([]fs.DirEntry, error) { + files, err := os.ReadDir(filepath.Join(repoPath, "objects/pack/")) + if err != nil { + return nil, err + } + + var packFiles []fs.DirEntry + for _, f := range files { + if filepath.Ext(f.Name()) == ".pack" { + packFiles = append(packFiles, f) + } + } + + return packFiles, nil +} + +// LooseObjects returns the number of loose objects that are not in a packfile. +func LooseObjects(ctx context.Context, repo git.RepositoryExecutor) (int64, error) { + cmd, err := repo.Exec(ctx, git.SubCmd{ + Name: "count-objects", + Flags: []git.Option{git.Flag{Name: "--verbose"}}, + }) + if err != nil { + return 0, err + } + + objectStats, err := readObjectInfoStatistic(cmd) + if err != nil { + return 0, err + } + + count, ok := objectStats["count"].(int64) + if !ok { + return 0, errors.New("could not get object count") + } + + return count, nil +} + // LogObjectsInfo read statistics of the git repo objects // and logs it under 'count-objects' key as structured entry. func LogObjectsInfo(ctx context.Context, repo git.RepositoryExecutor) { diff --git a/internal/git/stats/git_test.go b/internal/git/stats/objects_info_test.go index a07e82471..7d30a86d5 100644 --- a/internal/git/stats/git_test.go +++ b/internal/git/stats/objects_info_test.go @@ -7,11 +7,13 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v15/internal/git" "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config" @@ -20,6 +22,62 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" ) +func TestRepositoryProfile(t *testing.T) { + ctx := testhelper.Context(t) + cfg := testcfg.Build(t) + + repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ + SkipCreationViaService: true, + }) + repo := localrepo.NewTestRepo(t, cfg, repoProto) + + hasBitmap, err := HasBitmap(repoPath) + require.NoError(t, err) + require.False(t, hasBitmap, "repository should not have a bitmap initially") + packfiles, err := GetPackfiles(repoPath) + require.NoError(t, err) + require.Empty(t, packfiles) + packfilesCount, err := PackfilesCount(repoPath) + require.NoError(t, err) + require.Zero(t, packfilesCount) + + blobs := 10 + blobIDs := gittest.WriteBlobs(t, cfg, repoPath, blobs) + + looseObjects, err := LooseObjects(ctx, repo) + require.NoError(t, err) + require.Equal(t, int64(blobs), looseObjects) + + for _, blobID := range blobIDs { + commitID := gittest.WriteCommit(t, cfg, repoPath, + gittest.WithTreeEntries(gittest.TreeEntry{ + Mode: "100644", Path: "blob", OID: git.ObjectID(blobID), + }), + ) + gittest.Exec(t, cfg, "-C", repoPath, "update-ref", "refs/heads/"+blobID, commitID.String()) + } + + // write a loose object + gittest.WriteBlobs(t, cfg, repoPath, 1) + + gittest.Exec(t, cfg, "-C", repoPath, "repack", "-A", "-b", "-d") + + looseObjects, err = LooseObjects(ctx, repo) + require.NoError(t, err) + require.Equal(t, int64(1), looseObjects) + + // write another loose object + blobID := gittest.WriteBlobs(t, cfg, repoPath, 1)[0] + + // due to OS semantics, ensure that the blob has a timestamp that is after the packfile + theFuture := time.Now().Add(10 * time.Minute) + require.NoError(t, os.Chtimes(filepath.Join(repoPath, "objects", blobID[0:2], blobID[2:]), theFuture, theFuture)) + + looseObjects, err = LooseObjects(ctx, repo) + require.NoError(t, err) + require.Equal(t, int64(2), looseObjects) +} + func TestLogObjectInfo(t *testing.T) { ctx := testhelper.Context(t) cfg := testcfg.Build(t) diff --git a/internal/git/stats/profile.go b/internal/git/stats/profile.go deleted file mode 100644 index 44cb007d7..000000000 --- a/internal/git/stats/profile.go +++ /dev/null @@ -1,71 +0,0 @@ -package stats - -import ( - "context" - "errors" - "io/fs" - "os" - "path/filepath" - - "gitlab.com/gitlab-org/gitaly/v15/internal/git" -) - -// HasBitmap returns whether or not the repository contains an object bitmap. -func HasBitmap(repoPath string) (bool, error) { - bitmaps, err := filepath.Glob(filepath.Join(repoPath, "objects", "pack", "*.bitmap")) - if err != nil { - return false, err - } - - return len(bitmaps) > 0, nil -} - -// PackfilesCount returns the number of packfiles a repository has. -func PackfilesCount(repoPath string) (int, error) { - packFiles, err := GetPackfiles(repoPath) - if err != nil { - return 0, err - } - - return len(packFiles), nil -} - -// GetPackfiles returns the FileInfo of packfiles inside a repository. -func GetPackfiles(repoPath string) ([]fs.DirEntry, error) { - files, err := os.ReadDir(filepath.Join(repoPath, "objects/pack/")) - if err != nil { - return nil, err - } - - var packFiles []fs.DirEntry - for _, f := range files { - if filepath.Ext(f.Name()) == ".pack" { - packFiles = append(packFiles, f) - } - } - - return packFiles, nil -} - -// LooseObjects returns the number of loose objects that are not in a packfile. -func LooseObjects(ctx context.Context, repo git.RepositoryExecutor) (int64, error) { - cmd, err := repo.Exec(ctx, git.SubCmd{ - Name: "count-objects", - Flags: []git.Option{git.Flag{Name: "--verbose"}}, - }) - if err != nil { - return 0, err - } - - objectStats, err := readObjectInfoStatistic(cmd) - if err != nil { - return 0, err - } - - count, ok := objectStats["count"].(int64) - if !ok { - return 0, errors.New("could not get object count") - } - - return count, nil -} diff --git a/internal/git/stats/profile_test.go b/internal/git/stats/profile_test.go deleted file mode 100644 index 566941522..000000000 --- a/internal/git/stats/profile_test.go +++ /dev/null @@ -1,73 +0,0 @@ -//go:build !gitaly_test_sha256 - -package stats - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" - "gitlab.com/gitlab-org/gitaly/v15/internal/git" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest" - "gitlab.com/gitlab-org/gitaly/v15/internal/git/localrepo" - "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper" - "gitlab.com/gitlab-org/gitaly/v15/internal/testhelper/testcfg" -) - -func TestRepositoryProfile(t *testing.T) { - ctx := testhelper.Context(t) - cfg := testcfg.Build(t) - - repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg, gittest.CreateRepositoryConfig{ - SkipCreationViaService: true, - }) - repo := localrepo.NewTestRepo(t, cfg, repoProto) - - hasBitmap, err := HasBitmap(repoPath) - require.NoError(t, err) - require.False(t, hasBitmap, "repository should not have a bitmap initially") - packfiles, err := GetPackfiles(repoPath) - require.NoError(t, err) - require.Empty(t, packfiles) - packfilesCount, err := PackfilesCount(repoPath) - require.NoError(t, err) - require.Zero(t, packfilesCount) - - blobs := 10 - blobIDs := gittest.WriteBlobs(t, cfg, repoPath, blobs) - - looseObjects, err := LooseObjects(ctx, repo) - require.NoError(t, err) - require.Equal(t, int64(blobs), looseObjects) - - for _, blobID := range blobIDs { - commitID := gittest.WriteCommit(t, cfg, repoPath, - gittest.WithTreeEntries(gittest.TreeEntry{ - Mode: "100644", Path: "blob", OID: git.ObjectID(blobID), - }), - ) - gittest.Exec(t, cfg, "-C", repoPath, "update-ref", "refs/heads/"+blobID, commitID.String()) - } - - // write a loose object - gittest.WriteBlobs(t, cfg, repoPath, 1) - - gittest.Exec(t, cfg, "-C", repoPath, "repack", "-A", "-b", "-d") - - looseObjects, err = LooseObjects(ctx, repo) - require.NoError(t, err) - require.Equal(t, int64(1), looseObjects) - - // write another loose object - blobID := gittest.WriteBlobs(t, cfg, repoPath, 1)[0] - - // due to OS semantics, ensure that the blob has a timestamp that is after the packfile - theFuture := time.Now().Add(10 * time.Minute) - require.NoError(t, os.Chtimes(filepath.Join(repoPath, "objects", blobID[0:2], blobID[2:]), theFuture, theFuture)) - - looseObjects, err = LooseObjects(ctx, repo) - require.NoError(t, err) - require.Equal(t, int64(2), looseObjects) -} |