diff options
author | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2019-07-18 21:20:38 +0300 |
---|---|---|
committer | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2019-07-18 21:20:38 +0300 |
commit | 84c3ffcdc62b5bf4118429f7232744cae7ac36db (patch) | |
tree | 87ce1e4fcb1168a6e3abdcd9b8da996bfdbd4241 | |
parent | 0406874d095732f75de94a63495b5b45599b770e (diff) | |
parent | 495a384d41e8b4ed5a6e3e45375eda6aecd4f4fc (diff) |
Merge branch 'jc-fast-fork' into 'master'
Add CloneFromPoolInternal and CloneFromPool RPCs
Closes #1347
See merge request gitlab-org/gitaly!1301
-rw-r--r-- | changelogs/unreleased/jc-fast-fork.yml | 5 | ||||
-rw-r--r-- | internal/git/gittest/http_server.go | 57 | ||||
-rw-r--r-- | internal/git/objectpool/link.go | 11 | ||||
-rw-r--r-- | internal/git/objectpool/link_test.go | 48 | ||||
-rw-r--r-- | internal/git/objectpool/testhelper_test.go | 17 | ||||
-rw-r--r-- | internal/helper/error.go | 11 | ||||
-rw-r--r-- | internal/service/repository/clone_from_pool.go | 83 | ||||
-rw-r--r-- | internal/service/repository/clone_from_pool_internal.go | 142 | ||||
-rw-r--r-- | internal/service/repository/clone_from_pool_internal_test.go | 110 | ||||
-rw-r--r-- | internal/service/repository/clone_from_pool_test.go | 76 | ||||
-rw-r--r-- | internal/service/repository/server.go | 8 | ||||
-rw-r--r-- | internal/testhelper/commit.go | 13 | ||||
-rw-r--r-- | internal/testhelper/testhelper.go | 2 |
13 files changed, 560 insertions, 23 deletions
diff --git a/changelogs/unreleased/jc-fast-fork.yml b/changelogs/unreleased/jc-fast-fork.yml new file mode 100644 index 000000000..597e8eb34 --- /dev/null +++ b/changelogs/unreleased/jc-fast-fork.yml @@ -0,0 +1,5 @@ +--- +title: Add CloneFromPool RPC +merge_request: 1301 +author: +type: added diff --git a/internal/git/gittest/http_server.go b/internal/git/gittest/http_server.go new file mode 100644 index 000000000..4e5d1916a --- /dev/null +++ b/internal/git/gittest/http_server.go @@ -0,0 +1,57 @@ +package gittest + +import ( + "compress/gzip" + "context" + "fmt" + "net/http" + "net/http/httptest" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/command" +) + +// RemoteUploadPackServer implements two HTTP routes for git-upload-pack by copying stdin and stdout into and out of the git upload-pack command +func RemoteUploadPackServer(ctx context.Context, t *testing.T, repoName, httpToken, repoPath string) (*httptest.Server, string) { + s := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.String() { + case fmt.Sprintf("/%s.git/git-upload-pack", repoName): + w.WriteHeader(http.StatusOK) + + var err error + reader := r.Body + + if r.Header.Get("Content-Encoding") == "gzip" { + reader, err = gzip.NewReader(r.Body) + require.NoError(t, err) + } + defer r.Body.Close() + + cmd, err := command.New(ctx, exec.Command("git", "-C", repoPath, "upload-pack", "--stateless-rpc", "."), reader, w, nil) + require.NoError(t, err) + require.NoError(t, cmd.Wait()) + case fmt.Sprintf("/%s.git/info/refs?service=git-upload-pack", repoName): + if httpToken != "" && r.Header.Get("Authorization") != httpToken { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement") + w.WriteHeader(http.StatusOK) + + w.Write([]byte("001e# service=git-upload-pack\n")) + w.Write([]byte("0000")) + + cmd, err := command.New(ctx, exec.Command("git", "-C", repoPath, "upload-pack", "--advertise-refs", "."), nil, w, nil) + require.NoError(t, err) + require.NoError(t, cmd.Wait()) + default: + w.WriteHeader(http.StatusNotFound) + } + }), + ) + + return s, fmt.Sprintf("%s/%s.git", s.URL, repoName) +} diff --git a/internal/git/objectpool/link.go b/internal/git/objectpool/link.go index ad3cfbc5f..0bc9e75e0 100644 --- a/internal/git/objectpool/link.go +++ b/internal/git/objectpool/link.go @@ -36,16 +36,19 @@ func (o *ObjectPool) Link(ctx context.Context, repo *gitalypb.Repository) error expectedContent := filepath.Join(relPath, "objects") - actualContent, err := ioutil.ReadFile(altPath) + actualContentBytes, err := ioutil.ReadFile(altPath) if err == nil { - if text.ChompBytes(actualContent) == expectedContent { + actualContent := text.ChompBytes(actualContentBytes) + if actualContent == expectedContent { return nil } - return fmt.Errorf("unexpected alternates content: %q", actualContent) + if filepath.Clean(actualContent) != filepath.Join(o.FullPath(), "objects") { + return fmt.Errorf("unexpected alternates content: %q", actualContent) + } } - if !os.IsNotExist(err) { + if err != nil && !os.IsNotExist(err) { return err } diff --git a/internal/git/objectpool/link_test.go b/internal/git/objectpool/link_test.go index 84f6d0d0a..c3a7318af 100644 --- a/internal/git/objectpool/link_test.go +++ b/internal/git/objectpool/link_test.go @@ -19,8 +19,8 @@ func TestLink(t *testing.T) { testRepo, _, cleanupFn := testhelper.NewTestRepo(t) defer cleanupFn() - pool, err := NewObjectPool(testRepo.GetStorageName(), testhelper.NewTestObjectPoolName(t)) - require.NoError(t, err) + pool, poolCleanup := NewTestObjectPool(ctx, t, testRepo.GetStorageName()) + defer poolCleanup() require.NoError(t, pool.Remove(ctx), "make sure pool does not exist prior to creation") require.NoError(t, pool.Create(ctx, testRepo), "create pool") @@ -56,8 +56,8 @@ func TestLinkRemoveBitmap(t *testing.T) { testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) defer cleanupFn() - pool, err := NewObjectPool(testRepo.GetStorageName(), testhelper.NewTestObjectPoolName(t)) - require.NoError(t, err) + pool, poolCleanup := NewTestObjectPool(ctx, t, testRepo.GetStorageName()) + defer poolCleanup() require.NoError(t, pool.Init(ctx)) @@ -104,9 +104,8 @@ func TestUnlink(t *testing.T) { testRepo, _, cleanupFn := testhelper.NewTestRepo(t) defer cleanupFn() - pool, err := NewObjectPool(testRepo.GetStorageName(), t.Name()) - require.NoError(t, err) - defer pool.Remove(ctx) + pool, poolCleanup := NewTestObjectPool(ctx, t, testRepo.GetStorageName()) + defer poolCleanup() require.Error(t, pool.Unlink(ctx, testRepo), "removing a non-existing pool should be an error") @@ -118,3 +117,38 @@ func TestUnlink(t *testing.T) { require.NoError(t, pool.Unlink(ctx, testRepo), "unlink repo") require.False(t, testhelper.RemoteExists(t, pool.FullPath(), testRepo.GetGlRepository()), "pool remotes should no longer include %v", testRepo) } + +func TestLinkAbsoluteLinkExists(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + pool, poolCleanup := NewTestObjectPool(ctx, t, testRepo.GetStorageName()) + defer poolCleanup() + + require.NoError(t, pool.Remove(ctx), "make sure pool does not exist prior to creation") + require.NoError(t, pool.Create(ctx, testRepo), "create pool") + + altPath, err := git.InfoAlternatesPath(testRepo) + require.NoError(t, err) + + fullPath := filepath.Join(pool.FullPath(), "objects") + + require.NoError(t, ioutil.WriteFile(altPath, []byte(fullPath), 0644)) + + require.NoError(t, pool.Link(ctx, testRepo), "we expect this call to change the absolute link to a relative link") + + require.FileExists(t, altPath, "alternates file must exist after Link") + + content, err := ioutil.ReadFile(altPath) + require.NoError(t, err) + + require.False(t, filepath.IsAbs(string(content)), "expected %q to be relative path", content) + + testRepoObjectsPath := filepath.Join(testRepoPath, "objects") + require.Equal(t, fullPath, filepath.Join(testRepoObjectsPath, string(content)), "the content of the alternates file should be the relative version of the absolute pat") + + require.True(t, testhelper.RemoteExists(t, pool.FullPath(), "origin"), "pool remotes should include %v", testRepo) +} diff --git a/internal/git/objectpool/testhelper_test.go b/internal/git/objectpool/testhelper_test.go new file mode 100644 index 000000000..f616fdeae --- /dev/null +++ b/internal/git/objectpool/testhelper_test.go @@ -0,0 +1,17 @@ +package objectpool + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/testhelper" +) + +func NewTestObjectPool(ctx context.Context, t *testing.T, storageName string) (*ObjectPool, func()) { + pool, err := NewObjectPool(storageName, testhelper.NewTestObjectPoolName(t)) + require.NoError(t, err) + return pool, func() { + require.NoError(t, pool.Remove(ctx)) + } +} diff --git a/internal/helper/error.go b/internal/helper/error.go index 41bb907b2..db0f9703c 100644 --- a/internal/helper/error.go +++ b/internal/helper/error.go @@ -19,13 +19,18 @@ func DecorateError(code codes.Code, err error) error { return err } -// ErrInternal wrappes err with codes.Internal, unless err is already a grpc error +// ErrInternal wraps err with codes.Internal, unless err is already a grpc error func ErrInternal(err error) error { return DecorateError(codes.Internal, err) } +// ErrInternalf wrapps a formatted error with codes.Internal, unless err is already a grpc error +func ErrInternalf(format string, a ...interface{}) error { + return DecorateError(codes.Internal, fmt.Errorf(format, a...)) +} + // ErrInvalidArgument wraps err with codes.InvalidArgument, unless err is already a grpc error func ErrInvalidArgument(err error) error { return DecorateError(codes.InvalidArgument, err) } -// ErrInvalidArgumentf wraps err with codes.InvalidArgument, unless err is already a grpc error +// ErrInvalidArgumentf wraps a formatted error with codes.InvalidArgument, unless err is already a grpc error func ErrInvalidArgumentf(format string, a ...interface{}) error { return DecorateError(codes.InvalidArgument, fmt.Errorf(format, a...)) } @@ -33,7 +38,7 @@ func ErrInvalidArgumentf(format string, a ...interface{}) error { // ErrPreconditionFailed wraps err with codes.FailedPrecondition, unless err is already a grpc error func ErrPreconditionFailed(err error) error { return DecorateError(codes.FailedPrecondition, err) } -// ErrPreconditionFailedf wraps err with codes.FailedPrecondition, unless err is already a grpc error +// ErrPreconditionFailedf wraps a formatted error with codes.FailedPrecondition, unless err is already a grpc error func ErrPreconditionFailedf(format string, a ...interface{}) error { return DecorateError(codes.FailedPrecondition, fmt.Errorf(format, a...)) } diff --git a/internal/service/repository/clone_from_pool.go b/internal/service/repository/clone_from_pool.go new file mode 100644 index 000000000..00f346e0f --- /dev/null +++ b/internal/service/repository/clone_from_pool.go @@ -0,0 +1,83 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "os" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git/objectpool" + "gitlab.com/gitlab-org/gitaly/internal/helper" +) + +func (s *server) CloneFromPool(ctx context.Context, req *gitalypb.CloneFromPoolRequest) (*gitalypb.CloneFromPoolResponse, error) { + if err := validateCloneFromPoolRequestArgs(req); err != nil { + return nil, helper.ErrInvalidArgument(err) + } + + if err := validateCloneFromPoolRequestRepositoryState(req); err != nil { + return nil, helper.ErrInternal(err) + } + + if err := cloneFromPool(ctx, req.GetPool(), req.GetRepository()); err != nil { + return nil, helper.ErrInternal(err) + } + + if _, err := s.FetchRemote(ctx, &gitalypb.FetchRemoteRequest{ + Repository: req.GetRepository(), + RemoteParams: req.GetRemote(), + Timeout: 1000, + }); err != nil { + return nil, helper.ErrInternalf("fetch http remote: %v", err) + } + + objectPool, err := objectpool.FromProto(req.GetPool()) + if err != nil { + return nil, helper.ErrInternalf("get object pool from request: %v", err) + } + + if err = objectPool.Link(ctx, req.GetRepository()); err != nil { + return nil, helper.ErrInternalf("change hard link to relative: %v", err) + } + + return &gitalypb.CloneFromPoolResponse{}, nil +} + +func validateCloneFromPoolRequestRepositoryState(req *gitalypb.CloneFromPoolRequest) error { + targetRepositoryFullPath, err := helper.GetPath(req.GetRepository()) + if err != nil { + return fmt.Errorf("getting target repository path: %v", err) + } + + if _, err := os.Stat(targetRepositoryFullPath); !os.IsNotExist(err) { + return errors.New("target reopsitory already exists") + } + + objectPool, err := objectpool.FromProto(req.GetPool()) + if err != nil { + return fmt.Errorf("getting object pool from repository: %v", err) + } + + if !objectPool.IsValid() { + return errors.New("object pool is not valid") + } + + return nil +} + +func validateCloneFromPoolRequestArgs(req *gitalypb.CloneFromPoolRequest) error { + if req.GetRepository() == nil { + return errors.New("repository required") + } + + if req.GetRemote() == nil { + return errors.New("remote required") + } + + if req.GetPool() == nil { + return errors.New("pool is empty") + } + + return nil +} diff --git a/internal/service/repository/clone_from_pool_internal.go b/internal/service/repository/clone_from_pool_internal.go new file mode 100644 index 000000000..deadf6736 --- /dev/null +++ b/internal/service/repository/clone_from_pool_internal.go @@ -0,0 +1,142 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "os" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/objectpool" + "gitlab.com/gitlab-org/gitaly/internal/git/repository" + "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/rubyserver" +) + +func (s *server) CloneFromPoolInternal(ctx context.Context, req *gitalypb.CloneFromPoolInternalRequest) (*gitalypb.CloneFromPoolInternalResponse, error) { + if err := validateCloneFromPoolInternalRequestArgs(req); err != nil { + return nil, helper.ErrInvalidArgument(err) + } + + if err := validateCloneFromPoolInternalRequestRepositoryState(req); err != nil { + return nil, helper.ErrInternal(err) + } + + if err := cloneFromPool(ctx, req.GetPool(), req.GetRepository()); err != nil { + return nil, helper.ErrInternal(err) + } + + client, err := s.RemoteServiceClient(ctx) + if err != nil { + return nil, helper.ErrInternalf("getting remote service client: %v", err) + } + + fetchInternalReq := &gitalypb.FetchInternalRemoteRequest{ + Repository: req.GetRepository(), + RemoteRepository: req.GetSourceRepository(), + } + + clientCtx, err := rubyserver.SetHeaders(ctx, fetchInternalReq.GetRepository()) + if err != nil { + return nil, err + } + + if _, err = client.FetchInternalRemote(clientCtx, fetchInternalReq); err != nil { + return nil, helper.ErrInternalf("fetch internal remote: %v", err) + } + + objectPool, err := objectpool.FromProto(req.GetPool()) + if err != nil { + return nil, helper.ErrInternalf("get object pool from request: %v", err) + } + + if err = objectPool.Link(ctx, req.GetRepository()); err != nil { + return nil, helper.ErrInternalf("change hard link to relative: %v", err) + } + + return &gitalypb.CloneFromPoolInternalResponse{}, nil +} + +func validateCloneFromPoolInternalRequestRepositoryState(req *gitalypb.CloneFromPoolInternalRequest) error { + targetRepositoryFullPath, err := helper.GetPath(req.GetRepository()) + if err != nil { + return fmt.Errorf("getting target repository path: %v", err) + } + + if _, err := os.Stat(targetRepositoryFullPath); !os.IsNotExist(err) { + return errors.New("target reopsitory already exists") + } + + objectPool, err := objectpool.FromProto(req.GetPool()) + if err != nil { + return fmt.Errorf("getting object pool from repository: %v", err) + } + + if !objectPool.IsValid() { + return errors.New("object pool is not valid") + } + + linked, err := objectPool.LinkedToRepository(req.GetSourceRepository()) + if err != nil { + return fmt.Errorf("error when testing if source repository is linked to pool repository: %v", err) + } + + if !linked { + return errors.New("source repository is not linked to pool repository") + } + + return nil +} + +func validateCloneFromPoolInternalRequestArgs(req *gitalypb.CloneFromPoolInternalRequest) error { + if req.GetRepository() == nil { + return errors.New("repository required") + } + + if req.GetSourceRepository() == nil { + return errors.New("source repository required") + } + + if req.GetPool() == nil { + return errors.New("pool is empty") + } + + if req.GetSourceRepository().GetStorageName() != req.GetRepository().GetStorageName() { + return errors.New("source repository and target repository are not on the same storage") + } + + return nil +} + +func cloneFromPool(ctx context.Context, objectPoolRepo *gitalypb.ObjectPool, repo repository.GitRepo) error { + + objectPoolPath, err := helper.GetPath(objectPoolRepo.GetRepository()) + if err != nil { + return fmt.Errorf("could not get object pool path: %v", err) + } + repositoryPath, err := helper.GetPath(repo) + if err != nil { + return fmt.Errorf("could not get object pool path: %v", err) + } + + args := []string{ + "clone", + "--bare", + "--shared", + "--", + objectPoolPath, + repositoryPath, + } + + cmd, err := git.BareCommand(ctx, nil, nil, nil, nil, args...) + if err != nil { + return fmt.Errorf("clone with object pool start: %v", err) + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("clone with object pool wait %v", err) + } + + return nil +} diff --git a/internal/service/repository/clone_from_pool_internal_test.go b/internal/service/repository/clone_from_pool_internal_test.go new file mode 100644 index 000000000..4e4dba45c --- /dev/null +++ b/internal/service/repository/clone_from_pool_internal_test.go @@ -0,0 +1,110 @@ +package repository_test + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git/objectpool" + "gitlab.com/gitlab-org/gitaly/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/internal/service/repository" + "gitlab.com/gitlab-org/gitaly/internal/testhelper" +) + +func NewTestObjectPool(t *testing.T) (*objectpool.ObjectPool, *gitalypb.Repository) { + repo, _, relativePath := testhelper.CreateRepo(t, testhelper.GitlabTestStoragePath()) + + pool, err := objectpool.NewObjectPool(repo.GetStorageName(), relativePath) + require.NoError(t, err) + + return pool, repo +} + +// getForkDestination creates a repo struct and path, but does not actually create the directory +func getForkDestination(t *testing.T) (*gitalypb.Repository, string, func()) { + folder, err := text.RandomHex(20) + require.NoError(t, err) + forkRepoPath := filepath.Join(testhelper.GitlabTestStoragePath(), folder) + forkedRepo := &gitalypb.Repository{StorageName: "default", RelativePath: folder, GlRepository: "project-1"} + + return forkedRepo, forkRepoPath, func() { os.RemoveAll(forkRepoPath) } +} + +// getGitObjectDirSize gets the number of 1k blocks of a git object directory +func getGitObjectDirSize(t *testing.T, repoPath string) int64 { + output := testhelper.MustRunCommand(t, nil, "du", "-s", "-k", filepath.Join(repoPath, "objects")) + if len(output) < 2 { + t.Error("invalid output of du -s -k") + } + + outputSplit := strings.SplitN(string(output), "\t", 2) + blocks, err := strconv.ParseInt(outputSplit[0], 10, 64) + require.NoError(t, err) + + return blocks +} + +func TestCloneFromPoolInternal(t *testing.T) { + server, serverSocketPath := runFullServer(t) + defer server.Stop() + + ctxOuter, cancel := testhelper.Context() + defer cancel() + + md := testhelper.GitalyServersMetadata(t, serverSocketPath) + ctx := metadata.NewOutgoingContext(ctxOuter, md) + + client, conn := repository.NewRepositoryClient(t, serverSocketPath) + defer conn.Close() + + testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + pool, poolRepo := NewTestObjectPool(t) + defer pool.Remove(ctx) + + require.NoError(t, pool.Create(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, testRepo)) + + fullRepack(t, testRepoPath) + + _, newBranch := testhelper.CreateCommitOnNewBranch(t, testRepoPath) + + forkedRepo, forkRepoPath, forkRepoCleanup := getForkDestination(t) + defer forkRepoCleanup() + + req := &gitalypb.CloneFromPoolInternalRequest{ + Repository: forkedRepo, + SourceRepository: testRepo, + Pool: &gitalypb.ObjectPool{ + Repository: poolRepo, + }, + } + + _, err := client.CloneFromPoolInternal(ctx, req) + require.NoError(t, err) + + assert.True(t, getGitObjectDirSize(t, forkRepoPath) < 100) + + isLinked, err := pool.LinkedToRepository(testRepo) + require.NoError(t, err) + require.True(t, isLinked) + + // feature is a branch known to exist in the source repository. By looking it up in the target + // we establish that the target has branches, even though (as we saw above) it has no objects. + testhelper.MustRunCommand(t, nil, "git", "-C", forkRepoPath, "show-ref", "--verify", "refs/heads/feature") + testhelper.MustRunCommand(t, nil, "git", "-C", forkRepoPath, "show-ref", "--verify", fmt.Sprintf("refs/heads/%s", newBranch)) +} + +// fullRepack does a full repack on the repository, which means if it has a pool repository linked, it will get rid of redundant objects that are reachable in the pool +func fullRepack(t *testing.T, repoPath string) { + testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "repack", "-A", "-l", "-d") +} diff --git a/internal/service/repository/clone_from_pool_test.go b/internal/service/repository/clone_from_pool_test.go new file mode 100644 index 000000000..3a9ef4ae3 --- /dev/null +++ b/internal/service/repository/clone_from_pool_test.go @@ -0,0 +1,76 @@ +package repository_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/internal/service/repository" + "gitlab.com/gitlab-org/gitaly/internal/testhelper" +) + +func TestCloneFromPoolHTTP(t *testing.T) { + server, serverSocketPath := runFullServer(t) + defer server.Stop() + + ctxOuter, cancel := testhelper.Context() + defer cancel() + + md := testhelper.GitalyServersMetadata(t, serverSocketPath) + ctx := metadata.NewOutgoingContext(ctxOuter, md) + + client, conn := repository.NewRepositoryClient(t, serverSocketPath) + defer conn.Close() + + testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + pool, poolRepo := NewTestObjectPool(t) + defer pool.Remove(ctx) + + require.NoError(t, pool.Create(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, testRepo)) + + fullRepack(t, testRepoPath) + + _, newBranch := testhelper.CreateCommitOnNewBranch(t, testRepoPath) + + forkedRepo, forkRepoPath, forkRepoCleanup := getForkDestination(t) + defer forkRepoCleanup() + + authorizationHeader := "ABCefg0999182" + _, remoteURL := gittest.RemoteUploadPackServer(ctx, t, "my-repo", authorizationHeader, testRepoPath) + + req := &gitalypb.CloneFromPoolRequest{ + Repository: forkedRepo, + Remote: &gitalypb.Remote{ + Url: remoteURL, + Name: "geo", + HttpAuthorizationHeader: authorizationHeader, + MirrorRefmaps: []string{"all_refs"}, + }, + Pool: &gitalypb.ObjectPool{ + Repository: poolRepo, + }, + } + + _, err := client.CloneFromPool(ctx, req) + require.NoError(t, err) + + isLinked, err := pool.LinkedToRepository(testRepo) + require.NoError(t, err) + require.True(t, isLinked, "repository is not linked to the pool repository") + + assert.True(t, getGitObjectDirSize(t, forkRepoPath) < 100, "expect a small object directory size") + + // feature is a branch known to exist in the source repository. By looking it up in the target + // we establish that the target has branches, even though (as we saw above) it has no objects. + testhelper.MustRunCommand(t, nil, "git", "-C", forkRepoPath, "show-ref", "--verify", "refs/heads/feature") + testhelper.MustRunCommand(t, nil, "git", "-C", forkRepoPath, "show-ref", "--verify", fmt.Sprintf("refs/heads/%s", newBranch)) + +} diff --git a/internal/service/repository/server.go b/internal/service/repository/server.go index dd8092af8..a4a0355e9 100644 --- a/internal/service/repository/server.go +++ b/internal/service/repository/server.go @@ -20,11 +20,3 @@ func NewServer(rs *rubyserver.Server) gitalypb.RepositoryServiceServer { func (*server) FetchHTTPRemote(context.Context, *gitalypb.FetchHTTPRemoteRequest) (*gitalypb.FetchHTTPRemoteResponse, error) { return nil, helper.Unimplemented } - -func (*server) CloneFromPool(context.Context, *gitalypb.CloneFromPoolRequest) (*gitalypb.CloneFromPoolResponse, error) { - return nil, helper.Unimplemented -} - -func (*server) CloneFromPoolInternal(context.Context, *gitalypb.CloneFromPoolInternalRequest) (*gitalypb.CloneFromPoolInternalResponse, error) { - return nil, helper.Unimplemented -} diff --git a/internal/testhelper/commit.go b/internal/testhelper/commit.go index 51aa9ca9e..f31a38ce2 100644 --- a/internal/testhelper/commit.go +++ b/internal/testhelper/commit.go @@ -95,3 +95,16 @@ func CommitBlobWithName(t *testing.T, testRepoPath, blobID, fileName, commitMess treeID := text.ChompBytes(MustRunCommand(t, mktreeIn, "git", "-C", testRepoPath, "mktree")) return text.ChompBytes(MustRunCommand(t, nil, "git", "-C", testRepoPath, "commit-tree", treeID, "-m", commitMessage)) } + +// CreateCommitOnNewBranch creates a branch and a commit, returning the commit sha and the branch name respectivelyi +func CreateCommitOnNewBranch(t *testing.T, repoPath string) (string, string) { + nonce, err := text.RandomHex(4) + require.NoError(t, err) + newBranch := "branch-" + nonce + + sha := CreateCommit(t, repoPath, newBranch, &CreateCommitOpts{ + Message: "a new branch and commit " + nonce, + }) + + return sha, newBranch +} diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index fcd7db736..a4b81cdd1 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -367,7 +367,7 @@ func Context() (context.Context, func()) { return context.WithCancel(context.Background()) } -// CreateRepo creates an temporary directory for a repo, without initializing it +// CreateRepo creates a temporary directory for a repo, without initializing it func CreateRepo(t testing.TB, storagePath string) (repo *gitalypb.Repository, repoPath, relativePath string) { normalizedPrefix := strings.Replace(t.Name(), "/", "-", -1) //TempDir doesn't like a prefix containing slashes |