Allow repositories to be reduplicated

When a fork of a GitLab project changes visibility level from public to
anything else, the objects shouldn't be shared anymore. This means that
the repository needs to be reduplicated in terms of git objects.

By executing a repack with all objects, git will make all objects local
again to the repository. Afterwards the alternates can be unset.

Given the diff was small, this commit includes the protobuf upgrade.

Part of: https://gitlab.com/gitlab-org/gitaly/issues/1348

6 files changed, 168 insertions, 27 deletions

diff --git a/changelogs/unreleased/zj-reduplicate.yml b/changelogs/unreleased/zj-reduplicate.yml
new file mode 100644
index 000000000..6ba277dba
--- /dev/null
+++ b/changelogs/unreleased/zj-reduplicate.yml
@@ -0,0 +1,5 @@
+---
+title: Allow repositories to be reduplicated
+merge_request: 1003
+author:
+type: added
diff --git a/internal/service/objectpool/reduplicate.go b/internal/service/objectpool/reduplicate.go
new file mode 100644
index 000000000..908a7c866
--- /dev/null
+++ b/internal/service/objectpool/reduplicate.go
@@ -0,0 +1,26 @@
+package objectpool
+
+import (
+	"context"
+
+	"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitaly/internal/git"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func (s *server) ReduplicateRepository(ctx context.Context, req *gitalypb.ReduplicateRepositoryRequest) (*gitalypb.ReduplicateRepositoryResponse, error) {
+	if req.GetRepository() == nil {
+		return nil, status.Errorf(codes.InvalidArgument, "ReduplicateRepository: no repository")
+	}
+
+	cmd, err := git.Command(ctx, req.GetRepository(), "repack", "--quiet", "-a")
+	if err != nil {
+		return nil, err
+	}
+	if err := cmd.Wait(); err != nil {
+		return nil, err
+	}
+
+	return &gitalypb.ReduplicateRepositoryResponse{}, nil
+}
diff --git a/internal/service/objectpool/reduplicate_test.go b/internal/service/objectpool/reduplicate_test.go
new file mode 100644
index 000000000..e5119bc11
--- /dev/null
+++ b/internal/service/objectpool/reduplicate_test.go
@@ -0,0 +1,47 @@
+package objectpool
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly-proto/go/gitalypb"
+	"gitlab.com/gitlab-org/gitaly/internal/git"
+	"gitlab.com/gitlab-org/gitaly/internal/git/objectpool"
+	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
+)
+
+func TestReduplicate(t *testing.T) {
+	server, serverSocketPath := runObjectPoolServer(t)
+	defer server.Stop()
+
+	client, conn := newObjectPoolClient(t, serverSocketPath)
+	defer conn.Close()
+
+	ctx, cancel := testhelper.Context()
+	defer cancel()
+
+	testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t)
+	defer cleanupFn()
+
+	pool, err := objectpool.NewObjectPool(testRepo.GetStorageName(), t.Name())
+	require.NoError(t, err)
+	defer pool.Remove(ctx)
+	require.NoError(t, pool.Create(ctx, testRepo))
+	require.NoError(t, pool.Link(ctx, testRepo))
+	testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "gc")
+
+	existingObjectID := "55bc176024cfa3baaceb71db584c7e5df900ea65"
+	// Corrupt the repository to check if the object can't be found
+	require.NoError(t, pool.Unlink(ctx, testRepo))
+	cmd, err := git.Command(ctx, testRepo, "cat-file", "-e", existingObjectID)
+	require.NoError(t, err)
+	require.Error(t, cmd.Wait())
+
+	// Reduplicate and check if the objects appear again
+	require.NoError(t, pool.Link(ctx, testRepo))
+	_, err = client.ReduplicateRepository(ctx, &gitalypb.ReduplicateRepositoryRequest{Repository: testRepo})
+	require.NoError(t, err)
+
+	require.NoError(t, pool.Unlink(ctx, testRepo))
+	testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "cat-file", "-e", existingObjectID)
+}
diff --git a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go
index 60b087151..bdf29ba86 100644
--- a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go
+++ b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/blob.pb.go
@@ -120,6 +120,8 @@ It has these top-level messages:
 	LinkRepositoryToObjectPoolResponse
 	UnlinkRepositoryFromObjectPoolRequest
 	UnlinkRepositoryFromObjectPoolResponse
+	ReduplicateRepositoryRequest
+	ReduplicateRepositoryResponse
 	UserCreateBranchRequest
 	UserCreateBranchResponse
 	UserUpdateBranchRequest
diff --git a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go
index 0731d078c..4cae3ccbc 100644
--- a/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go
+++ b/vendor/gitlab.com/gitlab-org/gitaly-proto/go/gitalypb/objectpool.pb.go
@@ -169,6 +169,30 @@ func (*UnlinkRepositoryFromObjectPoolResponse) Descriptor() ([]byte, []int) {
 	return fileDescriptor7, []int{8}
 }
 
+type ReduplicateRepositoryRequest struct {
+	Repository *Repository `protobuf:"bytes,1,opt,name=repository" json:"repository,omitempty"`
+}
+
+func (m *ReduplicateRepositoryRequest) Reset()                    { *m = ReduplicateRepositoryRequest{} }
+func (m *ReduplicateRepositoryRequest) String() string            { return proto.CompactTextString(m) }
+func (*ReduplicateRepositoryRequest) ProtoMessage()               {}
+func (*ReduplicateRepositoryRequest) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{9} }
+
+func (m *ReduplicateRepositoryRequest) GetRepository() *Repository {
+	if m != nil {
+		return m.Repository
+	}
+	return nil
+}
+
+type ReduplicateRepositoryResponse struct {
+}
+
+func (m *ReduplicateRepositoryResponse) Reset()                    { *m = ReduplicateRepositoryResponse{} }
+func (m *ReduplicateRepositoryResponse) String() string            { return proto.CompactTextString(m) }
+func (*ReduplicateRepositoryResponse) ProtoMessage()               {}
+func (*ReduplicateRepositoryResponse) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{10} }
+
 func init() {
 	proto.RegisterType((*ObjectPool)(nil), "gitaly.ObjectPool")
 	proto.RegisterType((*CreateObjectPoolRequest)(nil), "gitaly.CreateObjectPoolRequest")
@@ -179,6 +203,8 @@ func init() {
 	proto.RegisterType((*LinkRepositoryToObjectPoolResponse)(nil), "gitaly.LinkRepositoryToObjectPoolResponse")
 	proto.RegisterType((*UnlinkRepositoryFromObjectPoolRequest)(nil), "gitaly.UnlinkRepositoryFromObjectPoolRequest")
 	proto.RegisterType((*UnlinkRepositoryFromObjectPoolResponse)(nil), "gitaly.UnlinkRepositoryFromObjectPoolResponse")
+	proto.RegisterType((*ReduplicateRepositoryRequest)(nil), "gitaly.ReduplicateRepositoryRequest")
+	proto.RegisterType((*ReduplicateRepositoryResponse)(nil), "gitaly.ReduplicateRepositoryResponse")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -197,6 +223,7 @@ type ObjectPoolServiceClient interface {
 	// Repositories are assumed to be stored on the same disk
 	LinkRepositoryToObjectPool(ctx context.Context, in *LinkRepositoryToObjectPoolRequest, opts ...grpc.CallOption) (*LinkRepositoryToObjectPoolResponse, error)
 	UnlinkRepositoryFromObjectPool(ctx context.Context, in *UnlinkRepositoryFromObjectPoolRequest, opts ...grpc.CallOption) (*UnlinkRepositoryFromObjectPoolResponse, error)
+	ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error)
 }
 
 type objectPoolServiceClient struct {
@@ -243,6 +270,15 @@ func (c *objectPoolServiceClient) UnlinkRepositoryFromObjectPool(ctx context.Con
 	return out, nil
 }
 
+func (c *objectPoolServiceClient) ReduplicateRepository(ctx context.Context, in *ReduplicateRepositoryRequest, opts ...grpc.CallOption) (*ReduplicateRepositoryResponse, error) {
+	out := new(ReduplicateRepositoryResponse)
+	err := grpc.Invoke(ctx, "/gitaly.ObjectPoolService/ReduplicateRepository", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for ObjectPoolService service
 
 type ObjectPoolServiceServer interface {
@@ -251,6 +287,7 @@ type ObjectPoolServiceServer interface {
 	// Repositories are assumed to be stored on the same disk
 	LinkRepositoryToObjectPool(context.Context, *LinkRepositoryToObjectPoolRequest) (*LinkRepositoryToObjectPoolResponse, error)
 	UnlinkRepositoryFromObjectPool(context.Context, *UnlinkRepositoryFromObjectPoolRequest) (*UnlinkRepositoryFromObjectPoolResponse, error)
+	ReduplicateRepository(context.Context, *ReduplicateRepositoryRequest) (*ReduplicateRepositoryResponse, error)
 }
 
 func RegisterObjectPoolServiceServer(s *grpc.Server, srv ObjectPoolServiceServer) {
@@ -329,6 +366,24 @@ func _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler(srv interface{},
 	return interceptor(ctx, in, info, handler)
 }
 
+func _ObjectPoolService_ReduplicateRepository_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReduplicateRepositoryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ObjectPoolServiceServer).ReduplicateRepository(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/gitaly.ObjectPoolService/ReduplicateRepository",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ObjectPoolServiceServer).ReduplicateRepository(ctx, req.(*ReduplicateRepositoryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "gitaly.ObjectPoolService",
 	HandlerType: (*ObjectPoolServiceServer)(nil),
@@ -349,6 +404,10 @@ var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{
 			MethodName: "UnlinkRepositoryFromObjectPool",
 			Handler:    _ObjectPoolService_UnlinkRepositoryFromObjectPool_Handler,
 		},
+		{
+			MethodName: "ReduplicateRepository",
+			Handler:    _ObjectPoolService_ReduplicateRepository_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "objectpool.proto",
@@ -357,27 +416,29 @@ var _ObjectPoolService_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("objectpool.proto", fileDescriptor7) }
 
 var fileDescriptor7 = []byte{
-	// 338 bytes of a gzipped FileDescriptorProto
+	// 377 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x94, 0x41, 0x4f, 0xc2, 0x40,
-	0x10, 0x85, 0x29, 0x26, 0x1c, 0x06, 0x0f, 0xb8, 0x17, 0xc8, 0x1e, 0x14, 0x1b, 0x35, 0x48, 0x62,
-	0x0f, 0xe5, 0x0f, 0x98, 0x68, 0x3c, 0x19, 0x35, 0x55, 0xe3, 0xd1, 0x14, 0x9c, 0xe0, 0x6a, 0xed,
-	0xd4, 0xed, 0x62, 0x82, 0x37, 0xef, 0x1e, 0xfc, 0x49, 0xfe, 0x34, 0x53, 0xda, 0xb2, 0xd0, 0x66,
-	0xa1, 0x31, 0x5c, 0xa7, 0xaf, 0xef, 0x7d, 0x9d, 0x79, 0x29, 0xb4, 0x68, 0xf8, 0x82, 0x23, 0x15,
-	0x11, 0x05, 0x4e, 0x24, 0x49, 0x11, 0x6b, 0x8c, 0x85, 0xf2, 0x83, 0x29, 0xdf, 0x8e, 0x9f, 0x7d,
-	0x89, 0x4f, 0xe9, 0xd4, 0x3e, 0x05, 0xb8, 0x9e, 0x29, 0x6f, 0x88, 0x02, 0xe6, 0x02, 0x48, 0x8c,
-	0x28, 0x16, 0x8a, 0xe4, 0xb4, 0x63, 0x75, 0xad, 0x5e, 0xd3, 0x65, 0x4e, 0xfa, 0xa2, 0xe3, 0xcd,
-	0x9f, 0x78, 0x0b, 0x2a, 0xfb, 0x13, 0xda, 0x67, 0x12, 0x7d, 0x85, 0xda, 0xc7, 0xc3, 0xf7, 0x09,
-	0xc6, 0x8a, 0x0d, 0xa0, 0x99, 0x62, 0x3c, 0x26, 0x1c, 0x45, 0xbf, 0x05, 0x3d, 0x90, 0x66, 0xe8,
-	0x43, 0x83, 0xa4, 0x18, 0x8b, 0xb0, 0x53, 0x37, 0xe6, 0x67, 0x0a, 0x9b, 0x43, 0xa7, 0x9c, 0x1d,
-	0x47, 0x14, 0xc6, 0x68, 0x5f, 0x41, 0xfb, 0x1c, 0x03, 0xdc, 0x14, 0x57, 0x92, 0x55, 0xf6, 0xcb,
-	0xb2, 0xbe, 0x2d, 0xd8, 0xbf, 0x14, 0xe1, 0xab, 0x46, 0xbc, 0xa3, 0x0d, 0xad, 0x63, 0xf9, 0x24,
-	0xf5, 0x4a, 0x27, 0x39, 0x00, 0x7b, 0x15, 0x4d, 0x06, 0xfd, 0x63, 0xc1, 0xe1, 0x7d, 0x18, 0x2c,
-	0x09, 0x2f, 0x24, 0xbd, 0x95, 0xc1, 0xff, 0x51, 0x8b, 0xe2, 0xc7, 0xd6, 0x2b, 0xed, 0xb8, 0x07,
-	0x47, 0xeb, 0x88, 0x52, 0x78, 0xf7, 0x77, 0x0b, 0x76, 0xf4, 0xf8, 0x16, 0xe5, 0x87, 0x18, 0x21,
-	0x7b, 0x80, 0x56, 0xb1, 0x0f, 0x6c, 0x2f, 0xcf, 0x34, 0xb4, 0x94, 0x77, 0xcd, 0x82, 0x6c, 0x53,
-	0xb5, 0xc4, 0xb8, 0x78, 0x7c, 0x6d, 0x6c, 0xa8, 0x99, 0x36, 0x36, 0xf6, 0xa6, 0xc6, 0x26, 0xc0,
-	0xcd, 0xa7, 0x62, 0xc7, 0xb9, 0xc3, 0xda, 0x72, 0xf1, 0x7e, 0x15, 0xe9, 0x3c, 0xf6, 0xcb, 0x82,
-	0xdd, 0xd5, 0x9b, 0x66, 0x27, 0xb9, 0x61, 0xa5, 0x8e, 0x70, 0xa7, 0xaa, 0x3c, 0x67, 0x18, 0x36,
-	0x66, 0x7f, 0xa0, 0xc1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4c, 0x09, 0x9c, 0xf6, 0xab, 0x04,
-	0x00, 0x00,
+	0x10, 0x85, 0x29, 0x31, 0x1c, 0x06, 0x0f, 0xb8, 0x89, 0x81, 0x34, 0x2a, 0xd8, 0x80, 0x41, 0x12,
+	0x7b, 0x80, 0x3f, 0x60, 0xa2, 0xf1, 0x64, 0xd4, 0x54, 0x8d, 0x47, 0x53, 0x60, 0xc4, 0xd5, 0xda,
+	0xa9, 0xdb, 0xc5, 0x04, 0x6f, 0xde, 0x3d, 0xf8, 0x33, 0xfc, 0x99, 0x06, 0xda, 0xb2, 0x50, 0x5c,
+	0x68, 0x08, 0xd7, 0xf6, 0xf5, 0xbd, 0x6f, 0x67, 0x5e, 0x17, 0x4a, 0xd4, 0x7d, 0xc1, 0x9e, 0x0c,
+	0x88, 0x3c, 0x3b, 0x10, 0x24, 0x89, 0x15, 0x06, 0x5c, 0xba, 0xde, 0xc8, 0xdc, 0x0e, 0x9f, 0x5d,
+	0x81, 0xfd, 0xe8, 0xa9, 0x75, 0x0a, 0x70, 0x3d, 0x51, 0xde, 0x10, 0x79, 0xac, 0x0d, 0x20, 0x30,
+	0xa0, 0x90, 0x4b, 0x12, 0xa3, 0x8a, 0x51, 0x33, 0x9a, 0xc5, 0x36, 0xb3, 0xa3, 0x0f, 0x6d, 0x67,
+	0xfa, 0xc6, 0x99, 0x51, 0x59, 0x9f, 0x50, 0x3e, 0x13, 0xe8, 0x4a, 0x54, 0x3e, 0x0e, 0xbe, 0x0f,
+	0x31, 0x94, 0xac, 0x03, 0xc5, 0x08, 0xe3, 0x71, 0xcc, 0x91, 0xf6, 0x9b, 0xd1, 0x03, 0x29, 0x86,
+	0x16, 0x14, 0x48, 0xf0, 0x01, 0xf7, 0x2b, 0x79, 0x6d, 0x7e, 0xac, 0xb0, 0x4c, 0xa8, 0x2c, 0x66,
+	0x87, 0x01, 0xf9, 0x21, 0x5a, 0x57, 0x50, 0x3e, 0x47, 0x0f, 0x37, 0xc5, 0x35, 0xce, 0x5a, 0xf4,
+	0x8b, 0xb3, 0xbe, 0x0d, 0x38, 0xbc, 0xe4, 0xfe, 0xab, 0x42, 0xbc, 0xa3, 0x0d, 0x8d, 0x63, 0x7e,
+	0x25, 0xf9, 0x4c, 0x2b, 0xa9, 0x83, 0xb5, 0x8c, 0x26, 0x86, 0xfe, 0x31, 0xa0, 0x71, 0xef, 0x7b,
+	0x73, 0xc2, 0x0b, 0x41, 0x6f, 0x8b, 0xe0, 0x6b, 0xd4, 0x22, 0x7d, 0xd8, 0x7c, 0xa6, 0x19, 0x37,
+	0xe1, 0x68, 0x15, 0x51, 0x0c, 0xef, 0xc0, 0x9e, 0x83, 0xfd, 0x61, 0xe0, 0xf1, 0x9e, 0x2b, 0x71,
+	0x86, 0x61, 0x7d, 0x64, 0xab, 0x0a, 0xfb, 0x1a, 0xcf, 0x28, 0xb4, 0xfd, 0xbb, 0x05, 0x3b, 0x8a,
+	0xe5, 0x16, 0xc5, 0x07, 0xef, 0x21, 0x7b, 0x80, 0x52, 0xba, 0x84, 0xac, 0x9a, 0x44, 0x69, 0x7e,
+	0x0d, 0xb3, 0xa6, 0x17, 0xc4, 0x27, 0xcc, 0x8d, 0x8d, 0xd3, 0x8d, 0x53, 0xc6, 0x9a, 0x6e, 0x2b,
+	0x63, 0x6d, 0x59, 0x73, 0x6c, 0x08, 0xa6, 0xbe, 0x1f, 0xec, 0x38, 0x71, 0x58, 0xd9, 0x68, 0xb3,
+	0x95, 0x45, 0x3a, 0x8d, 0xfd, 0x32, 0xe0, 0x60, 0xf9, 0x7a, 0xd9, 0x49, 0x62, 0x98, 0xa9, 0x98,
+	0xa6, 0x9d, 0x55, 0x3e, 0x65, 0x78, 0x82, 0xdd, 0x7f, 0x77, 0xcc, 0xea, 0xaa, 0x1c, 0xfa, 0x5a,
+	0x99, 0x8d, 0x15, 0xaa, 0x24, 0xa7, 0x5b, 0x98, 0x5c, 0xaf, 0x9d, 0xbf, 0x00, 0x00, 0x00, 0xff,
+	0xff, 0x9b, 0x40, 0xdd, 0xa1, 0x88, 0x05, 0x00, 0x00,
 }
diff --git a/vendor/vendor.json b/vendor/vendor.json
index 47bad3ba1..7f09cf4f2 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -315,12 +315,12 @@
 			"versionExact": "v1.2.2"
 		},
 		{
-			"checksumSHA1": "5t/1h2cvv7Nbho0apE2ghr2K7L0=",
+			"checksumSHA1": "VpLdojyLYx9d/bwnqFkEy12sNd8=",
 			"path": "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb",
-			"revision": "8eb6120caacdbe12b0abd0c9125e2e319a7689dc",
-			"revisionTime": "2018-12-10T09:05:58Z",
-			"version": "v1.4.0",
-			"versionExact": "v1.4.0"
+			"revision": "20e398e887e239ba4ddaa44cf075e99841e43f8a",
+			"revisionTime": "2018-12-17T08:03:04Z",
+			"version": "v1.5.0",
+			"versionExact": "v1.5.0"
 		},
 		{
 			"checksumSHA1": "S9x46Eq79I1EkoXI8woeIU6w/wY=",