diff options
author | John Cai <jcai@gitlab.com> | 2022-04-04 18:08:28 +0300 |
---|---|---|
committer | John Cai <jcai@gitlab.com> | 2022-04-04 18:08:28 +0300 |
commit | a8ca1e263382d60f0263a90005876163805f44e6 (patch) | |
tree | 0b8022925d3415190b5fed76f2b159d3b13ce259 /internal/git | |
parent | c83fc8eb3e2fc2a4840e249ed35910621e5a4bc6 (diff) | |
parent | 0d28358d259724bc71b1833ffb877f73852b197c (diff) |
Merge branch 'jc-accurate-calculation-of-repo-size' into 'master'
Use rev-list --all --objects --disk-usage to calculate repository usage
See merge request gitlab-org/gitaly!4430
Diffstat (limited to 'internal/git')
-rw-r--r-- | internal/git/localrepo/repo.go | 29 | ||||
-rw-r--r-- | internal/git/localrepo/repo_test.go | 136 |
2 files changed, 165 insertions, 0 deletions
diff --git a/internal/git/localrepo/repo.go b/internal/git/localrepo/repo.go index 806b61553..7d7f3e409 100644 --- a/internal/git/localrepo/repo.go +++ b/internal/git/localrepo/repo.go @@ -1,8 +1,11 @@ package localrepo import ( + "bytes" "context" "fmt" + "strconv" + "strings" "testing" "github.com/stretchr/testify/require" @@ -90,3 +93,29 @@ func errorWithStderr(err error, stderr []byte) error { } return fmt.Errorf("%w, stderr: %q", err, stderr) } + +// Size calculates the size of all reachable objects in bytes +func (repo *Repo) Size(ctx context.Context) (int64, error) { + var stdout bytes.Buffer + if err := repo.ExecAndWait(ctx, + git.SubCmd{ + Name: "rev-list", + Flags: []git.Option{ + git.Flag{Name: "--all"}, + git.Flag{Name: "--objects"}, + git.Flag{Name: "--use-bitmap-index"}, + git.Flag{Name: "--disk-usage"}, + }, + }, + git.WithStdout(&stdout), + ); err != nil { + return -1, err + } + + size, err := strconv.ParseInt(strings.TrimSuffix(stdout.String(), "\n"), 10, 64) + if err != nil { + return -1, err + } + + return size, nil +} diff --git a/internal/git/localrepo/repo_test.go b/internal/git/localrepo/repo_test.go index 356ce2ce5..c305519c8 100644 --- a/internal/git/localrepo/repo_test.go +++ b/internal/git/localrepo/repo_test.go @@ -1,13 +1,19 @@ package localrepo import ( + "bytes" "context" + "os" + "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/catfile" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" + "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testcfg" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" ) @@ -35,3 +41,133 @@ func TestRepo(t *testing.T) { return New(config.NewLocator(cfg), gitCmdFactory, catfileCache, pbRepo), repoPath }) } + +func TestSize(t *testing.T) { + cfg := testcfg.Build(t) + gitCmdFactory := gittest.NewCommandFactory(t, cfg) + catfileCache := catfile.NewCache(cfg) + t.Cleanup(catfileCache.Stop) + + testCases := []struct { + desc string + setup func(repoPath string, t *testing.T) + expectedSize int64 + }{ + { + desc: "empty repository", + expectedSize: 0, + }, + { + desc: "one committed file", + setup: func(repoPath string, t *testing.T) { + require.NoError(t, os.WriteFile( + filepath.Join(repoPath, "file"), + bytes.Repeat([]byte("a"), 1000), + 0o644, + )) + + cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "add", "file") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-m", "initial") + require.NoError(t, cmd.Run()) + }, + expectedSize: 202, + }, + { + desc: "one large loose blob", + setup: func(repoPath string, t *testing.T) { + require.NoError(t, os.WriteFile( + filepath.Join(repoPath, "file"), + bytes.Repeat([]byte("a"), 1000), + 0o644, + )) + + cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "checkout", "-b", "branch-a") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "add", "file") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-m", "initial") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "update-ref", "-d", "refs/heads/branch-a") + require.NoError(t, cmd.Run()) + }, + expectedSize: 0, + }, + { + desc: "modification to blob without repack", + setup: func(repoPath string, t *testing.T) { + require.NoError(t, os.WriteFile( + filepath.Join(repoPath, "file"), + bytes.Repeat([]byte("a"), 1000), + 0o644, + )) + + cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "add", "file") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-m", "initial") + require.NoError(t, cmd.Run()) + + f, err := os.OpenFile( + filepath.Join(repoPath, "file"), + os.O_APPEND|os.O_WRONLY, + 0o644) + require.NoError(t, err) + defer f.Close() + _, err = f.WriteString("a") + assert.NoError(t, err) + + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-am", "modification") + require.NoError(t, cmd.Run()) + }, + expectedSize: 437, + }, + { + desc: "modification to blob after repack", + setup: func(repoPath string, t *testing.T) { + require.NoError(t, os.WriteFile( + filepath.Join(repoPath, "file"), + bytes.Repeat([]byte("a"), 1000), + 0o644, + )) + + cmd := gittest.NewCommand(t, cfg, "-C", repoPath, "add", "file") + require.NoError(t, cmd.Run()) + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-m", "initial") + require.NoError(t, cmd.Run()) + + f, err := os.OpenFile( + filepath.Join(repoPath, "file"), + os.O_APPEND|os.O_WRONLY, + 0o644) + require.NoError(t, err) + defer f.Close() + _, err = f.WriteString("a") + assert.NoError(t, err) + + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "commit", "-am", "modification") + require.NoError(t, cmd.Run()) + + cmd = gittest.NewCommand(t, cfg, "-C", repoPath, "repack", "-a", "-d") + require.NoError(t, cmd.Run()) + }, + expectedSize: 391, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + pbRepo, repoPath := gittest.InitRepo(t, cfg, cfg.Storages[0], gittest.InitRepoOpts{ + WithWorktree: true, + }) + repo := New(config.NewLocator(cfg), gitCmdFactory, catfileCache, pbRepo) + if tc.setup != nil { + tc.setup(repoPath, t) + } + + ctx := testhelper.Context(t) + size, err := repo.Size(ctx) + require.NoError(t, err) + assert.Equal(t, tc.expectedSize, size) + }) + } +} |