praefect: Support object pool disconnection in Praefect replicator

When a secondary repository replica in Praefect is out of sync with the
primary replica, Praefect creates a replication job that performs the
`ReplicateRepository` RPC to replicate the up-to-date primary onto the
secondary. As part of this operation the replicator also checks if the
source repository is linked to an object pool and if so links the target
to the matching object pool on the secondary.

If the primary is not linked to an object pool and the secondary is
linked to an object pool, no change is made to the target repository
object pool relationship and the replication job continues normally.
This is problematic because it allows secondary repositories to diverge
from the primary. Also if the repository attempts to link to a new
object pool via the `LinkRepositoryToObjectPool` RPC, the secondary
repositories will always fail due to already being linked to an object
pool.

To prevent this from occurring, the target repository should also
perform the `DisconnectGitAlternates` RPC if it is determined that the
source repository is not linked to an object pool. This change updates
the Praefect replicator to perform object pool disconnects to ensure the
target repository remains consistent with the source.

2 files changed, 39 insertions, 2 deletions

diff --git a/internal/praefect/replicator.go b/internal/praefect/replicator.go
index 42b3eebee..b19d17481 100644
--- a/internal/praefect/replicator.go
+++ b/internal/praefect/replicator.go
@@ -102,8 +102,19 @@ func (dr defaultReplicator) Replicate(ctx context.Context, event datastore.Repli
 
 	sourceObjectPool := resp.GetObjectPool()
 
-	if sourceObjectPool != nil {
-		targetObjectPoolClient := gitalypb.NewObjectPoolServiceClient(targetCC)
+	targetObjectPoolClient := gitalypb.NewObjectPoolServiceClient(targetCC)
+
+	if sourceObjectPool == nil {
+		// If the source repository is not linked to an object pool, the target repository
+		// should also not be linked to any object pool to ensure consistency.
+		if _, err := targetObjectPoolClient.DisconnectGitAlternates(ctx, &gitalypb.DisconnectGitAlternatesRequest{
+			Repository: targetRepository,
+		}); err != nil {
+			return err
+		}
+	} else {
+		// If the source repository is linked to an object pool, the target repository
+		// should link to the same object pool.
 		targetObjectPool := proto.Clone(sourceObjectPool).(*gitalypb.ObjectPool)
 		targetObjectPool.GetRepository().StorageName = targetRepository.GetStorageName()
 		if _, err := targetObjectPoolClient.LinkRepositoryToObjectPool(ctx, &gitalypb.LinkRepositoryToObjectPoolRequest{
diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go
index da06bb09d..369064ee6 100644
--- a/internal/praefect/replicator_test.go
+++ b/internal/praefect/replicator_test.go
@@ -313,6 +313,32 @@ func testDefaultReplicatorReplicate(t *testing.T, ctx context.Context) {
 				}
 			},
 		},
+		{
+			desc: "target disconnected from alternate to match source",
+			setup: func(t *testing.T) setupData {
+				// Create repository on source Gitaly that does not have a link to
+				// an object pool.
+				sourceRepo, _ := gittest.CreateRepository(t, ctx, sourceCfg)
+
+				// Create repository on target Gitaly with the same relative path as
+				// the source repository and link it to an object pool.
+				targetRepo, _ := gittest.CreateRepository(t, ctx, targetCfg, gittest.CreateRepositoryConfig{
+					RelativePath: sourceRepo.RelativePath,
+				})
+				gittest.CreateObjectPool(t, ctx, targetCfg, targetRepo, gittest.CreateObjectPoolConfig{
+					LinkRepositoryToObjectPool: true,
+				})
+
+				return setupData{
+					job: datastore.ReplicationJob{
+						ReplicaPath:       sourceRepo.RelativePath,
+						TargetNodeStorage: targetStorage,
+						SourceNodeStorage: sourceStorage,
+					},
+					expectedPool: nil,
+				}
+			},
+		},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
 			testSetup := tc.setup(t)