diff options
author | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2018-12-13 12:55:46 +0300 |
---|---|---|
committer | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2018-12-18 17:12:14 +0300 |
commit | 7721690a4e5a6de9b793adc92e021d0d51cfa605 (patch) | |
tree | 7a9885583249748709ebae5e1c0ad589657e7361 | |
parent | 8a9876c5606a6bf615af51d3d77faa96718bd85d (diff) |
Allow repositories to be reduplicated
When a fork of a GitLab project changes visibility level from public to
anything else, the objects shouldn't be shared anymore. This means that
the repository needs to be reduplicated in terms of git objects.
By executing a repack with all objects, git will make all objects local
again to the repository. Afterwards the alternates can be unset.
Given the diff was small, this commit includes the protobuf upgrade.
Part of: https://gitlab.com/gitlab-org/gitaly/issues/1348
-rw-r--r-- | changelogs/unreleased/zj-reduplicate.yml | 5 | ||||
-rw-r--r-- | internal/service/objectpool/reduplicate.go | 26 | ||||
-rw-r--r-- | internal/service/objectpool/reduplicate_test.go | 47 | ||||
-rw-r--r-- | vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go | 2 | ||||
-rw-r--r-- | vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go | 105 | ||||
-rw-r--r-- | vendor/vendor.json | 10 |
6 files changed, 168 insertions, 27 deletions
diff --git a/changelogs/unreleased/zj-reduplicate.yml b/changelogs/unreleased/zj-reduplicate.yml new file mode 100644 index 000000000..6ba277dba --- /dev/null +++ b/changelogs/unreleased/zj-reduplicate.yml @@ -0,0 +1,5 @@ +--- +title: Allow repositories to be reduplicated +merge_request: 1003 +author: +type: added diff --git a/internal/service/objectpool/reduplicate.go b/internal/service/objectpool/reduplicate.go new file mode 100644 index 000000000..908a7c866 --- /dev/null +++ b/internal/service/objectpool/reduplicate.go @@ -0,0 +1,26 @@ +package objectpool + +import ( + "context" + + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *server) ReduplicateRepository(ctx context.Context, req *gitalypb.ReduplicateRepositoryRequest) (*gitalypb.ReduplicateRepositoryResponse, error) { + if req.GetRepository() == nil { + return nil, status.Errorf(codes.InvalidArgument, "ReduplicateRepository: no repository") + } + + cmd, err := git.Command(ctx, req.GetRepository(), "repack", "--quiet", "-a") + if err != nil { + return nil, err + } + if err := cmd.Wait(); err != nil { + return nil, err + } + + return &gitalypb.ReduplicateRepositoryResponse{}, nil +} diff --git a/internal/service/objectpool/reduplicate_test.go b/internal/service/objectpool/reduplicate_test.go new file mode 100644 index 000000000..e5119bc11 --- /dev/null +++ b/internal/service/objectpool/reduplicate_test.go @@ -0,0 +1,47 @@ +package objectpool + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/git/objectpool" + "gitlab.com/gitlab-org/gitaly/internal/testhelper" +) + +func TestReduplicate(t *testing.T) { + server, serverSocketPath := runObjectPoolServer(t) + defer server.Stop() + + client, conn := newObjectPoolClient(t, serverSocketPath) + defer conn.Close() + + ctx, cancel := testhelper.Context() + defer cancel() + + testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + pool, err := objectpool.NewObjectPool(testRepo.GetStorageName(), t.Name()) + require.NoError(t, err) + defer pool.Remove(ctx) + require.NoError(t, pool.Create(ctx, testRepo)) + require.NoError(t, pool.Link(ctx, testRepo)) + testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "gc") + + existingObjectID := "55bc176024cfa3baaceb71db584c7e5df900ea65" + // Corrupt the repository to check if the object can't be found + require.NoError(t, pool.Unlink(ctx, testRepo)) + cmd, err := git.Command(ctx, testRepo, "cat-file", "-e", existingObjectID) + require.NoError(t, err) + require.Error(t, cmd.Wait()) + + // Reduplicate and check if the objects appear again + require.NoError(t, pool.Link(ctx, testRepo)) + _, err = client.ReduplicateRepository(ctx, &gitalypb.ReduplicateRepositoryRequest{Repository: testRepo}) + require.NoError(t, err) + + require.NoError(t, pool.Unlink(ctx, testRepo)) + testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "cat-file", "-e", existingObjectID) +} diff --git a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go index 60b087151..bdf29ba86 100644 --- a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go +++ b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go @@ -120,6 +120,8 @@ It has these top-level messages: LinkRepositoryToObjectPoolResponse UnlinkRepositoryFromObjectPoolRequest UnlinkRepositoryFromObjectPoolResponse + ReduplicateRepositoryRequest + ReduplicateRepositoryResponse UserCreateBranchRequest UserCreateBranchResponse UserUpdateBranchRequest diff --git a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go index 0731d078c..4cae3ccbc 100644 --- a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go +++ b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go @@ -169,6 +169,30 @@ func (*UnlinkRepositoryFromObjectPoolResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{8} } +type ReduplicateRepositoryRequest struct { + Repository *Repository `protobuf:"bytes,1,opt,name=repository" json:"repository,omitempty"` +} + +func (m *ReduplicateRepositoryRequest) Reset() { *m = ReduplicateRepositoryRequest{} } +func (m *ReduplicateRepositoryRequest) String() string { return proto.CompactTextString(m) } +func (*ReduplicateRepositoryRequest) ProtoMessage() {} +func (*ReduplicateRepositoryRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{9} } + +func (m *ReduplicateRepositoryRequest) GetRepository() *Repository { + if m != nil { + return m.Repository + } + return nil +} + +type ReduplicateRepositoryResponse struct { +} + +func (m *ReduplicateRepositoryResponse) Reset() { *m = ReduplicateRepositoryResponse{} } +func (m *ReduplicateRepositoryResponse) String() string { return proto.CompactTextString(m) } +func (*ReduplicateRepositoryResponse) ProtoMessage() {} +func (*ReduplicateRepositoryResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{10} } + func init() { proto.RegisterType((*ObjectPool)(nil), "gitaly.ObjectPool") proto.RegisterType((*CreateObjectPoolRequest)(nil), "gitaly.CreateObjectPoolRequest") @@ -179,6 +203,8 @@ func init() { proto.RegisterType((*LinkRepositoryToObjectPoolResponse)(nil), "gitaly.LinkRepositoryToObjectPoolResponse") proto.RegisterType((*UnlinkRepositoryFromObjectPoolRequest)(nil), "gitaly.UnlinkRepositoryFromObjectPoolRequest") proto.RegisterType((*UnlinkRepositoryFromObjectPoolResponse)(nil), "gitaly.UnlinkRepositoryFromObjectPoolResponse") + proto.RegisterType((*ReduplicateRepositoryRequest)(nil), "gitaly.ReduplicateRepositoryRequest") + proto.RegisterType((*ReduplicateRepositoryResponse)(nil), "gitaly.ReduplicateRepositoryResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -197,6 +223,7 @@ type ObjectPoolServiceClient interface { // Repositories are assumed to be stored on the same disk LinkRepositoryToObjectPool(ctx context.Context, in *LinkRepositoryToObjectPoolRequest, opts ...grpc.CallOption) (*LinkRepositoryToObjectPoolResponse, error) UnlinkRepositoryFromObjectPool(ctx context.Context, in *UnlinkRepositoryFromObjectPoolRequest, opts ...grpc.CallOption) (*UnlinkRepositoryFromObjectPoolResponse, error) + ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error) } type objectPoolServiceClient struct { @@ -243,6 +270,15 @@ func (c *objectPoolServiceClient) UnlinkRepositoryFromObjectPool(ctx context.Con return out, nil } +func (c *objectPoolServiceClient) ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error) { + out := new(ReduplicateRepositoryResponse) + err := grpc.Invoke(ctx, "/gitaly.ObjectPoolService/ReduplicateRepository", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for ObjectPoolService service type ObjectPoolServiceServer interface { @@ -251,6 +287,7 @@ type ObjectPoolServiceServer interface { // Repositories are assumed to be stored on the same disk LinkRepositoryToObjectPool(context.Context, *LinkRepositoryToObjectPoolRequest) (*LinkRepositoryToObjectPoolResponse, error) UnlinkRepositoryFromObjectPool(context.Context, *UnlinkRepositoryFromObjectPoolRequest) (*UnlinkRepositoryFromObjectPoolResponse, error) + ReduplicateRepository(context.Context, *ReduplicateRepositoryRequest) (*ReduplicateRepositoryResponse, error) } func RegisterObjectPoolServiceServer(s *grpc.Server, srv ObjectPoolServiceServer) { @@ -329,6 +366,24 @@ func _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler(srv interface{}, return interceptor(ctx, in, info, handler) } +func _ObjectPoolService_ReduplicateRepository_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReduplicateRepositoryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ObjectPoolServiceServer).ReduplicateRepository(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/gitaly.ObjectPoolService/ReduplicateRepository", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ObjectPoolServiceServer).ReduplicateRepository(ctx, req.(*ReduplicateRepositoryRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{ ServiceName: "gitaly.ObjectPoolService", HandlerType: (*ObjectPoolServiceServer)(nil), @@ -349,6 +404,10 @@ var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{ MethodName: "UnlinkRepositoryFromObjectPool", Handler: _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler, }, + { + MethodName: "ReduplicateRepository", + Handler: _ObjectPoolService_ReduplicateRepository_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "objectpool.proto", @@ -357,27 +416,29 @@ var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("objectpool.proto", fileDescriptor7) } var fileDescriptor7 = []byte{ - // 338 bytes of a gzipped FileDescriptorProto + // 377 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x94, 0x41, 0x4f, 0xc2, 0x40, - 0x10, 0x85, 0x29, 0x26, 0x1c, 0x06, 0x0f, 0xb8, 0x17, 0xc8, 0x1e, 0x14, 0x1b, 0x35, 0x48, 0x62, - 0x0f, 0xe5, 0x0f, 0x98, 0x68, 0x3c, 0x19, 0x35, 0x55, 0xe3, 0xd1, 0x14, 0x9c, 0xe0, 0x6a, 0xed, - 0xd4, 0xed, 0x62, 0x82, 0x37, 0xef, 0x1e, 0xfc, 0x49, 0xfe, 0x34, 0x53, 0xda, 0xb2, 0xd0, 0x66, - 0xa1, 0x31, 0x5c, 0xa7, 0xaf, 0xef, 0x7d, 0x9d, 0x79, 0x29, 0xb4, 0x68, 0xf8, 0x82, 0x23, 0x15, - 0x11, 0x05, 0x4e, 0x24, 0x49, 0x11, 0x6b, 0x8c, 0x85, 0xf2, 0x83, 0x29, 0xdf, 0x8e, 0x9f, 0x7d, - 0x89, 0x4f, 0xe9, 0xd4, 0x3e, 0x05, 0xb8, 0x9e, 0x29, 0x6f, 0x88, 0x02, 0xe6, 0x02, 0x48, 0x8c, - 0x28, 0x16, 0x8a, 0xe4, 0xb4, 0x63, 0x75, 0xad, 0x5e, 0xd3, 0x65, 0x4e, 0xfa, 0xa2, 0xe3, 0xcd, - 0x9f, 0x78, 0x0b, 0x2a, 0xfb, 0x13, 0xda, 0x67, 0x12, 0x7d, 0x85, 0xda, 0xc7, 0xc3, 0xf7, 0x09, - 0xc6, 0x8a, 0x0d, 0xa0, 0x99, 0x62, 0x3c, 0x26, 0x1c, 0x45, 0xbf, 0x05, 0x3d, 0x90, 0x66, 0xe8, - 0x43, 0x83, 0xa4, 0x18, 0x8b, 0xb0, 0x53, 0x37, 0xe6, 0x67, 0x0a, 0x9b, 0x43, 0xa7, 0x9c, 0x1d, - 0x47, 0x14, 0xc6, 0x68, 0x5f, 0x41, 0xfb, 0x1c, 0x03, 0xdc, 0x14, 0x57, 0x92, 0x55, 0xf6, 0xcb, - 0xb2, 0xbe, 0x2d, 0xd8, 0xbf, 0x14, 0xe1, 0xab, 0x46, 0xbc, 0xa3, 0x0d, 0xad, 0x63, 0xf9, 0x24, - 0xf5, 0x4a, 0x27, 0x39, 0x00, 0x7b, 0x15, 0x4d, 0x06, 0xfd, 0x63, 0xc1, 0xe1, 0x7d, 0x18, 0x2c, - 0x09, 0x2f, 0x24, 0xbd, 0x95, 0xc1, 0xff, 0x51, 0x8b, 0xe2, 0xc7, 0xd6, 0x2b, 0xed, 0xb8, 0x07, - 0x47, 0xeb, 0x88, 0x52, 0x78, 0xf7, 0x77, 0x0b, 0x76, 0xf4, 0xf8, 0x16, 0xe5, 0x87, 0x18, 0x21, - 0x7b, 0x80, 0x56, 0xb1, 0x0f, 0x6c, 0x2f, 0xcf, 0x34, 0xb4, 0x94, 0x77, 0xcd, 0x82, 0x6c, 0x53, - 0xb5, 0xc4, 0xb8, 0x78, 0x7c, 0x6d, 0x6c, 0xa8, 0x99, 0x36, 0x36, 0xf6, 0xa6, 0xc6, 0x26, 0xc0, - 0xcd, 0xa7, 0x62, 0xc7, 0xb9, 0xc3, 0xda, 0x72, 0xf1, 0x7e, 0x15, 0xe9, 0x3c, 0xf6, 0xcb, 0x82, - 0xdd, 0xd5, 0x9b, 0x66, 0x27, 0xb9, 0x61, 0xa5, 0x8e, 0x70, 0xa7, 0xaa, 0x3c, 0x67, 0x18, 0x36, - 0x66, 0x7f, 0xa0, 0xc1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4c, 0x09, 0x9c, 0xf6, 0xab, 0x04, - 0x00, 0x00, + 0x10, 0x85, 0x29, 0x31, 0x1c, 0x06, 0x0f, 0xb8, 0x89, 0x81, 0x34, 0x2a, 0xd8, 0x80, 0x41, 0x12, + 0x7b, 0x80, 0x3f, 0x60, 0xa2, 0xf1, 0x64, 0xd4, 0x54, 0x8d, 0x47, 0x53, 0x60, 0xc4, 0xd5, 0xda, + 0xa9, 0xdb, 0xc5, 0x04, 0x6f, 0xde, 0x3d, 0xf8, 0x33, 0xfc, 0x99, 0x06, 0xda, 0xb2, 0x50, 0x5c, + 0x68, 0x08, 0xd7, 0xf6, 0xf5, 0xbd, 0x6f, 0x67, 0x5e, 0x17, 0x4a, 0xd4, 0x7d, 0xc1, 0x9e, 0x0c, + 0x88, 0x3c, 0x3b, 0x10, 0x24, 0x89, 0x15, 0x06, 0x5c, 0xba, 0xde, 0xc8, 0xdc, 0x0e, 0x9f, 0x5d, + 0x81, 0xfd, 0xe8, 0xa9, 0x75, 0x0a, 0x70, 0x3d, 0x51, 0xde, 0x10, 0x79, 0xac, 0x0d, 0x20, 0x30, + 0xa0, 0x90, 0x4b, 0x12, 0xa3, 0x8a, 0x51, 0x33, 0x9a, 0xc5, 0x36, 0xb3, 0xa3, 0x0f, 0x6d, 0x67, + 0xfa, 0xc6, 0x99, 0x51, 0x59, 0x9f, 0x50, 0x3e, 0x13, 0xe8, 0x4a, 0x54, 0x3e, 0x0e, 0xbe, 0x0f, + 0x31, 0x94, 0xac, 0x03, 0xc5, 0x08, 0xe3, 0x71, 0xcc, 0x91, 0xf6, 0x9b, 0xd1, 0x03, 0x29, 0x86, + 0x16, 0x14, 0x48, 0xf0, 0x01, 0xf7, 0x2b, 0x79, 0x6d, 0x7e, 0xac, 0xb0, 0x4c, 0xa8, 0x2c, 0x66, + 0x87, 0x01, 0xf9, 0x21, 0x5a, 0x57, 0x50, 0x3e, 0x47, 0x0f, 0x37, 0xc5, 0x35, 0xce, 0x5a, 0xf4, + 0x8b, 0xb3, 0xbe, 0x0d, 0x38, 0xbc, 0xe4, 0xfe, 0xab, 0x42, 0xbc, 0xa3, 0x0d, 0x8d, 0x63, 0x7e, + 0x25, 0xf9, 0x4c, 0x2b, 0xa9, 0x83, 0xb5, 0x8c, 0x26, 0x86, 0xfe, 0x31, 0xa0, 0x71, 0xef, 0x7b, + 0x73, 0xc2, 0x0b, 0x41, 0x6f, 0x8b, 0xe0, 0x6b, 0xd4, 0x22, 0x7d, 0xd8, 0x7c, 0xa6, 0x19, 0x37, + 0xe1, 0x68, 0x15, 0x51, 0x0c, 0xef, 0xc0, 0x9e, 0x83, 0xfd, 0x61, 0xe0, 0xf1, 0x9e, 0x2b, 0x71, + 0x86, 0x61, 0x7d, 0x64, 0xab, 0x0a, 0xfb, 0x1a, 0xcf, 0x28, 0xb4, 0xfd, 0xbb, 0x05, 0x3b, 0x8a, + 0xe5, 0x16, 0xc5, 0x07, 0xef, 0x21, 0x7b, 0x80, 0x52, 0xba, 0x84, 0xac, 0x9a, 0x44, 0x69, 0x7e, + 0x0d, 0xb3, 0xa6, 0x17, 0xc4, 0x27, 0xcc, 0x8d, 0x8d, 0xd3, 0x8d, 0x53, 0xc6, 0x9a, 0x6e, 0x2b, + 0x63, 0x6d, 0x59, 0x73, 0x6c, 0x08, 0xa6, 0xbe, 0x1f, 0xec, 0x38, 0x71, 0x58, 0xd9, 0x68, 0xb3, + 0x95, 0x45, 0x3a, 0x8d, 0xfd, 0x32, 0xe0, 0x60, 0xf9, 0x7a, 0xd9, 0x49, 0x62, 0x98, 0xa9, 0x98, + 0xa6, 0x9d, 0x55, 0x3e, 0x65, 0x78, 0x82, 0xdd, 0x7f, 0x77, 0xcc, 0xea, 0xaa, 0x1c, 0xfa, 0x5a, + 0x99, 0x8d, 0x15, 0xaa, 0x24, 0xa7, 0x5b, 0x98, 0x5c, 0xaf, 0x9d, 0xbf, 0x00, 0x00, 0x00, 0xff, + 0xff, 0x9b, 0x40, 0xdd, 0xa1, 0x88, 0x05, 0x00, 0x00, } diff --git a/vendor/vendor.json b/vendor/vendor.json index 47bad3ba1..7f09cf4f2 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -315,12 +315,12 @@ "versionExact": "v1.2.2" }, { - "checksumSHA1": "5t/1h2cvv7Nbho0apE2ghr2K7L0=", + "checksumSHA1": "VpLdojyLYx9d/bwnqFkEy12sNd8=", "path": "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb", - "revision": "8eb6120caacdbe12b0abd0c9125e2e319a7689dc", - "revisionTime": "2018-12-10T09:05:58Z", - "version": "v1.4.0", - "versionExact": "v1.4.0" + "revision": "20e398e887e239ba4ddaa44cf075e99841e43f8a", + "revisionTime": "2018-12-17T08:03:04Z", + "version": "v1.5.0", + "versionExact": "v1.5.0" }, { "checksumSHA1": "S9x46Eq79I1EkoXI8woeIU6w/wY=", |