praefect: Derive replica path from repository IDsmh-generate-unique-replica-paths

4 files changed, 28 insertions, 12 deletions

diff --git a/internal/praefect/coordinator.go b/internal/praefect/coordinator.go
index 13aa771d2..7a1a0ad8a 100644
--- a/internal/praefect/coordinator.go
+++ b/internal/praefect/coordinator.go
@@ -586,6 +586,7 @@ func (c *Coordinator) mutatorStreamParameters(ctx context.Context, call grpcCall
 				route.RepositoryID,
 				virtualStorage,
 				targetRepo,
+				route.ReplicaPath,
 				route.Primary.Storage,
 				nil,
 				append(routerNodesToStorages(route.Secondaries), route.ReplicationTargets...),
@@ -864,7 +865,7 @@ func (c *Coordinator) createTransactionFinalizer(
 		}
 
 		return c.newRequestFinalizer(
-			ctx, route.RepositoryID, virtualStorage, targetRepo, route.Primary.Storage,
+			ctx, route.RepositoryID, virtualStorage, targetRepo, route.ReplicaPath, route.Primary.Storage,
 			updated, outdated, change, params, cause)()
 	}
 }
@@ -999,6 +1000,7 @@ func (c *Coordinator) newRequestFinalizer(
 	repositoryID int64,
 	virtualStorage string,
 	targetRepo *gitalypb.Repository,
+	replicaPath string,
 	primary string,
 	updatedSecondaries []string,
 	outdatedSecondaries []string,
@@ -1055,7 +1057,7 @@ func (c *Coordinator) newRequestFinalizer(
 				repositoryID,
 				virtualStorage,
 				targetRepo.GetRelativePath(),
-				targetRepo.GetRelativePath(),
+				replicaPath,
 				primary,
 				updatedSecondaries,
 				outdatedSecondaries,
diff --git a/internal/praefect/coordinator_test.go b/internal/praefect/coordinator_test.go
index 816405c08..e5d52e196 100644
--- a/internal/praefect/coordinator_test.go
+++ b/internal/praefect/coordinator_test.go
@@ -2095,6 +2095,7 @@ func TestNewRequestFinalizer_contextIsDisjointedFromTheRPC(t *testing.T) {
 					0,
 					"virtual storage",
 					&gitalypb.Repository{},
+					"replica-path",
 					"primary",
 					[]string{},
 					[]string{"secondary"},
diff --git a/internal/praefect/router_per_repository.go b/internal/praefect/router_per_repository.go
index f6d3f89ed..55c10faae 100644
--- a/internal/praefect/router_per_repository.go
+++ b/internal/praefect/router_per_repository.go
@@ -2,8 +2,10 @@ package praefect
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
+	"strconv"
 
 	"gitlab.com/gitlab-org/gitaly/v14/internal/praefect/datastore"
 	"gitlab.com/gitlab-org/gitaly/v14/internal/praefect/nodes"
@@ -277,6 +279,16 @@ func (r *PerRepositoryRouter) RouteRepositoryMutator(ctx context.Context, virtua
 	return route, nil
 }
 
+// deriveReplicaPath derives a repository's disk storage path from its repository ID. The repository ID
+// is SHA256 and the first four hex digits of the hash are used as the two subdirectories in the path.
+// The format is @repositories/ab/cd/<repository-id>.
+func deriveReplicaPath(repositoryID int64) string {
+	hasher := sha256.New()
+	hasher.Write([]byte(strconv.FormatInt(repositoryID, 10)))
+	hash := hasher.Sum(nil)
+	return fmt.Sprintf("@repositories/%x/%x/%d", hash[0:1], hash[1:2], repositoryID)
+}
+
 // RouteRepositoryCreation picks a random healthy node to act as the primary node and selects the secondary nodes
 // if assignments are enabled. Healthy secondaries take part in the transaction, unhealthy secondaries are set as
 // replication targets.
@@ -300,7 +312,7 @@ func (r *PerRepositoryRouter) RouteRepositoryCreation(ctx context.Context, virtu
 	if replicationFactor == 1 {
 		return RepositoryMutatorRoute{
 			RepositoryID: id,
-			ReplicaPath:  relativePath,
+			ReplicaPath:  deriveReplicaPath(id),
 			Primary:      primary,
 		}, nil
 	}
@@ -349,7 +361,7 @@ func (r *PerRepositoryRouter) RouteRepositoryCreation(ctx context.Context, virtu
 
 	return RepositoryMutatorRoute{
 		RepositoryID:       id,
-		ReplicaPath:        relativePath,
+		ReplicaPath:        deriveReplicaPath(id),
 		Primary:            primary,
 		Secondaries:        secondaries,
 		ReplicationTargets: replicationTargets,
diff --git a/internal/praefect/router_per_repository_test.go b/internal/praefect/router_per_repository_test.go
index 8372422d4..4eadb54f1 100644
--- a/internal/praefect/router_per_repository_test.go
+++ b/internal/praefect/router_per_repository_test.go
@@ -485,6 +485,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 	db := glsql.NewDB(t)
 
 	const relativePath = "relative-path"
+	const replicaPath = "@repositories/6b/86/1"
 
 	for _, tc := range []struct {
 		desc                string
@@ -518,7 +519,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID:       1,
-					ReplicaPath:        relativePath,
+					ReplicaPath:        replicaPath,
 					Primary:            RouterNode{Storage: "primary", Connection: primaryConn},
 					ReplicationTargets: []string{"secondary-1", "secondary-2"},
 				},
@@ -533,7 +534,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID: 1,
-					ReplicaPath:  relativePath,
+					ReplicaPath:  replicaPath,
 					Primary:      RouterNode{Storage: "primary", Connection: primaryConn},
 					Secondaries: []RouterNode{
 						{Storage: "secondary-1", Connection: secondary1Conn},
@@ -551,7 +552,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID: 1,
-					ReplicaPath:  relativePath,
+					ReplicaPath:  replicaPath,
 					Primary:      RouterNode{Storage: "primary", Connection: primaryConn},
 					Secondaries: []RouterNode{
 						{Storage: "secondary-1", Connection: secondary1Conn},
@@ -570,7 +571,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID: 1,
-					ReplicaPath:  relativePath,
+					ReplicaPath:  replicaPath,
 					Primary:      RouterNode{Storage: "primary", Connection: primaryConn},
 				},
 			),
@@ -586,13 +587,13 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID: 1,
-					ReplicaPath:  relativePath,
+					ReplicaPath:  replicaPath,
 					Primary:      RouterNode{Storage: "primary", Connection: primaryConn},
 					Secondaries:  []RouterNode{{Storage: "secondary-1", Connection: secondary1Conn}},
 				},
 				RepositoryMutatorRoute{
 					RepositoryID: 1,
-					ReplicaPath:  relativePath,
+					ReplicaPath:  replicaPath,
 					Primary:      RouterNode{Storage: "primary", Connection: primaryConn},
 					Secondaries:  []RouterNode{{Storage: "secondary-2", Connection: secondary1Conn}},
 				},
@@ -609,7 +610,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			matchRoute: requireOneOf(
 				RepositoryMutatorRoute{
 					RepositoryID:       1,
-					ReplicaPath:        relativePath,
+					ReplicaPath:        replicaPath,
 					Primary:            RouterNode{Storage: "primary", Connection: primaryConn},
 					Secondaries:        []RouterNode{{Storage: "secondary-1", Connection: secondary1Conn}},
 					ReplicationTargets: []string{"secondary-2"},
@@ -635,7 +636,7 @@ func TestPerRepositoryRouter_RouteRepositoryCreation(t *testing.T) {
 			rs := datastore.NewPostgresRepositoryStore(db, nil)
 			if tc.repositoryExists {
 				require.NoError(t,
-					rs.CreateRepository(ctx, 1, "virtual-storage-1", relativePath, relativePath, "primary", nil, nil, true, true),
+					rs.CreateRepository(ctx, 1, "virtual-storage-1", relativePath, replicaPath, "primary", nil, nil, true, true),
 				)
 			}