Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitaly.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSami Hiltunen <shiltunen@gitlab.com>2021-05-26 14:49:41 +0300
committerSami Hiltunen <shiltunen@gitlab.com>2021-07-12 12:03:18 +0300
commit4d07d9b0fc94482ddd88a700624040a0005f031a (patch)
treefe7ffa963004c9622a9c2132040dd9366907a218
parentfe6d52572ed4619191dbe73ade0b7baa4fceaafe (diff)
Remove support for virtual storage primaries in `praefect dataloss`
Starting from 14.0, Praefect only supports repository-specific primaries. This commit removes support for virtual storage scoped primaries in `praefect dataloss` to make future changes easier. Changelog: removed
-rw-r--r--cmd/praefect/subcmd_dataloss_test.go211
-rw-r--r--internal/praefect/datastore/repository_store.go15
-rw-r--r--internal/praefect/datastore/repository_store_mock.go6
-rw-r--r--internal/praefect/datastore/repository_store_test.go347
-rw-r--r--internal/praefect/service/info/dataloss.go4
5 files changed, 266 insertions, 317 deletions
diff --git a/cmd/praefect/subcmd_dataloss_test.go b/cmd/praefect/subcmd_dataloss_test.go
index e52cf0f42..0a717f531 100644
--- a/cmd/praefect/subcmd_dataloss_test.go
+++ b/cmd/praefect/subcmd_dataloss_test.go
@@ -4,7 +4,6 @@ package main
import (
"bytes"
- "fmt"
"testing"
"github.com/stretchr/testify/require"
@@ -28,63 +27,39 @@ func registerPraefectInfoServer(impl gitalypb.PraefectInfoServiceServer) svcRegi
}
func TestDatalossSubcommand(t *testing.T) {
- for _, scope := range []struct {
- desc string
- electionStrategy config.ElectionStrategy
- primaries map[string]string
- }{
- {
- desc: "sql elector",
- electionStrategy: config.ElectionStrategySQL,
- primaries: map[string]string{
- "repository-1": "gitaly-1",
- "repository-2": "gitaly-1",
+ cfg := config.Config{
+ VirtualStorages: []*config.VirtualStorage{
+ {
+ Name: "virtual-storage-1",
+ Nodes: []*config.Node{
+ {Storage: "gitaly-1"},
+ {Storage: "gitaly-2"},
+ {Storage: "gitaly-3"},
+ },
},
- },
- {
- desc: "per_repository elector",
- electionStrategy: config.ElectionStrategyPerRepository,
- primaries: map[string]string{
- "repository-1": "gitaly-1",
- "repository-2": "gitaly-3",
+ {
+ Name: "virtual-storage-2",
+ Nodes: []*config.Node{
+ {Storage: "gitaly-4"},
+ },
},
},
- } {
- t.Run(scope.desc, func(t *testing.T) {
- cfg := config.Config{
- Failover: config.Failover{ElectionStrategy: scope.electionStrategy},
- VirtualStorages: []*config.VirtualStorage{
- {
- Name: "virtual-storage-1",
- Nodes: []*config.Node{
- {Storage: "gitaly-1"},
- {Storage: "gitaly-2"},
- {Storage: "gitaly-3"},
- },
- },
- {
- Name: "virtual-storage-2",
- Nodes: []*config.Node{
- {Storage: "gitaly-4"},
- },
- },
- },
- }
+ }
- db := getDB(t)
- gs := datastore.NewPostgresRepositoryStore(db, cfg.StorageNames())
+ db := getDB(t)
+ gs := datastore.NewPostgresRepositoryStore(db, cfg.StorageNames())
- ctx, cancel := testhelper.Context()
- defer cancel()
+ ctx, cancel := testhelper.Context()
+ defer cancel()
- for _, q := range []string{
- `
+ for _, q := range []string{
+ `
INSERT INTO repositories (virtual_storage, relative_path, "primary")
VALUES
('virtual-storage-1', 'repository-1', 'gitaly-1'),
('virtual-storage-1', 'repository-2', 'gitaly-3')
`,
- `
+ `
INSERT INTO repository_assignments (virtual_storage, relative_path, storage)
VALUES
('virtual-storage-1', 'repository-1', 'gitaly-1'),
@@ -92,79 +67,75 @@ func TestDatalossSubcommand(t *testing.T) {
('virtual-storage-1', 'repository-2', 'gitaly-1'),
('virtual-storage-1', 'repository-2', 'gitaly-3')
`,
- `
- INSERT INTO shard_primaries (shard_name, node_name, elected_by_praefect, elected_at)
- VALUES ('virtual-storage-1', 'gitaly-1', 'ignored', now())
- `,
- } {
- _, err := db.ExecContext(ctx, q)
- require.NoError(t, err)
- }
+ } {
+ _, err := db.ExecContext(ctx, q)
+ require.NoError(t, err)
+ }
- require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-1", 1))
- require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-2", 0))
- require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-3", 0))
+ require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-1", 1))
+ require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-2", 0))
+ require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-1", "gitaly-3", 0))
- require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-2", "gitaly-2", 1))
- require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-2", "gitaly-3", 0))
+ require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-2", "gitaly-2", 1))
+ require.NoError(t, gs.SetGeneration(ctx, "virtual-storage-1", "repository-2", "gitaly-3", 0))
- ln, clean := listenAndServe(t, []svcRegistrar{
- registerPraefectInfoServer(info.NewServer(nil, cfg, nil, gs, nil, nil, nil))})
- defer clean()
- for _, tc := range []struct {
- desc string
- args []string
- virtualStorages []*config.VirtualStorage
- output string
- error error
- }{
- {
- desc: "positional arguments",
- args: []string{"-virtual-storage=virtual-storage-1", "positional-arg"},
- error: unexpectedPositionalArgsError{Command: "dataloss"},
- },
- {
- desc: "data loss with read-only repositories",
- args: []string{"-virtual-storage=virtual-storage-1"},
- output: fmt.Sprintf(`Virtual storage: virtual-storage-1
+ ln, clean := listenAndServe(t, []svcRegistrar{
+ registerPraefectInfoServer(info.NewServer(nil, cfg, nil, gs, nil, nil, nil))})
+ defer clean()
+ for _, tc := range []struct {
+ desc string
+ args []string
+ virtualStorages []*config.VirtualStorage
+ output string
+ error error
+ }{
+ {
+ desc: "positional arguments",
+ args: []string{"-virtual-storage=virtual-storage-1", "positional-arg"},
+ error: unexpectedPositionalArgsError{Command: "dataloss"},
+ },
+ {
+ desc: "data loss with read-only repositories",
+ args: []string{"-virtual-storage=virtual-storage-1"},
+ output: `Virtual storage: virtual-storage-1
Outdated repositories:
repository-2 (read-only):
- Primary: %s
+ Primary: gitaly-3
In-Sync Storages:
gitaly-2
Outdated Storages:
gitaly-1 is behind by 2 changes or less, assigned host
gitaly-3 is behind by 1 change or less, assigned host
-`, scope.primaries["repository-2"]),
- },
- {
- desc: "data loss with partially replicated repositories",
- args: []string{"-virtual-storage=virtual-storage-1", "-partially-replicated"},
- output: fmt.Sprintf(`Virtual storage: virtual-storage-1
+`,
+ },
+ {
+ desc: "data loss with partially replicated repositories",
+ args: []string{"-virtual-storage=virtual-storage-1", "-partially-replicated"},
+ output: `Virtual storage: virtual-storage-1
Outdated repositories:
repository-1 (writable):
- Primary: %s
+ Primary: gitaly-1
In-Sync Storages:
gitaly-1, assigned host
Outdated Storages:
gitaly-2 is behind by 1 change or less, assigned host
gitaly-3 is behind by 1 change or less
repository-2 (read-only):
- Primary: %s
+ Primary: gitaly-3
In-Sync Storages:
gitaly-2
Outdated Storages:
gitaly-1 is behind by 2 changes or less, assigned host
gitaly-3 is behind by 1 change or less, assigned host
-`, scope.primaries["repository-1"], scope.primaries["repository-2"]),
- },
- {
- desc: "multiple virtual storages with read-only repositories",
- virtualStorages: []*config.VirtualStorage{{Name: "virtual-storage-2"}, {Name: "virtual-storage-1"}},
- output: fmt.Sprintf(`Virtual storage: virtual-storage-1
+`,
+ },
+ {
+ desc: "multiple virtual storages with read-only repositories",
+ virtualStorages: []*config.VirtualStorage{{Name: "virtual-storage-2"}, {Name: "virtual-storage-1"}},
+ output: `Virtual storage: virtual-storage-1
Outdated repositories:
repository-2 (read-only):
- Primary: %s
+ Primary: gitaly-3
In-Sync Storages:
gitaly-2
Outdated Storages:
@@ -172,23 +143,23 @@ func TestDatalossSubcommand(t *testing.T) {
gitaly-3 is behind by 1 change or less, assigned host
Virtual storage: virtual-storage-2
All repositories are writable!
-`, scope.primaries["repository-2"]),
- },
- {
- desc: "multiple virtual storages with partially replicated repositories",
- args: []string{"-partially-replicated"},
- virtualStorages: []*config.VirtualStorage{{Name: "virtual-storage-2"}, {Name: "virtual-storage-1"}},
- output: fmt.Sprintf(`Virtual storage: virtual-storage-1
+`,
+ },
+ {
+ desc: "multiple virtual storages with partially replicated repositories",
+ args: []string{"-partially-replicated"},
+ virtualStorages: []*config.VirtualStorage{{Name: "virtual-storage-2"}, {Name: "virtual-storage-1"}},
+ output: `Virtual storage: virtual-storage-1
Outdated repositories:
repository-1 (writable):
- Primary: %s
+ Primary: gitaly-1
In-Sync Storages:
gitaly-1, assigned host
Outdated Storages:
gitaly-2 is behind by 1 change or less, assigned host
gitaly-3 is behind by 1 change or less
repository-2 (read-only):
- Primary: %s
+ Primary: gitaly-3
In-Sync Storages:
gitaly-2
Outdated Storages:
@@ -196,24 +167,22 @@ Virtual storage: virtual-storage-2
gitaly-3 is behind by 1 change or less, assigned host
Virtual storage: virtual-storage-2
All repositories are up to date!
-`, scope.primaries["repository-1"], scope.primaries["repository-2"]),
- },
- } {
- t.Run(tc.desc, func(t *testing.T) {
- cmd := newDatalossSubcommand()
- output := &bytes.Buffer{}
- cmd.output = output
+`,
+ },
+ } {
+ t.Run(tc.desc, func(t *testing.T) {
+ cmd := newDatalossSubcommand()
+ output := &bytes.Buffer{}
+ cmd.output = output
- fs := cmd.FlagSet()
- require.NoError(t, fs.Parse(tc.args))
- err := cmd.Exec(fs, config.Config{
- VirtualStorages: tc.virtualStorages,
- SocketPath: ln.Addr().String(),
- })
- require.Equal(t, tc.error, err, err)
- require.Equal(t, tc.output, output.String())
- })
- }
+ fs := cmd.FlagSet()
+ require.NoError(t, fs.Parse(tc.args))
+ err := cmd.Exec(fs, config.Config{
+ VirtualStorages: tc.virtualStorages,
+ SocketPath: ln.Addr().String(),
+ })
+ require.Equal(t, tc.error, err, err)
+ require.Equal(t, tc.output, output.String())
})
}
}
diff --git a/internal/praefect/datastore/repository_store.go b/internal/praefect/datastore/repository_store.go
index 7a1702b6d..b686617e4 100644
--- a/internal/praefect/datastore/repository_store.go
+++ b/internal/praefect/datastore/repository_store.go
@@ -117,9 +117,7 @@ type RepositoryStore interface {
// RepositoryExists returns whether the repository exists on a virtual storage.
RepositoryExists(ctx context.Context, virtualStorage, relativePath string) (bool, error)
// GetPartiallyReplicatedRepositories returns information on repositories which have an outdated copy on an assigned storage.
- // By default, repository specific primaries are returned in the results. If useVirtualStoragePrimaries is set, virtual storage's
- // primary is returned instead for each repository.
- GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string, virtualStoragePrimaries bool) ([]OutdatedRepository, error)
+ GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string) ([]OutdatedRepository, error)
// DeleteInvalidRepository is a method for deleting records of invalid repositories. It deletes a storage's
// record of the invalid repository. If the storage was the only storage with the repository, the repository's
// record on the virtual storage is also deleted.
@@ -541,7 +539,7 @@ type OutdatedRepository struct {
Storages []OutdatedRepositoryStorageDetails
}
-func (rs *PostgresRepositoryStore) GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string, useVirtualStoragePrimaries bool) ([]OutdatedRepository, error) {
+func (rs *PostgresRepositoryStore) GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string) ([]OutdatedRepository, error) {
configuredStorages, ok := rs.storages[virtualStorage]
if !ok {
return nil, fmt.Errorf("unknown virtual storage: %q", virtualStorage)
@@ -575,7 +573,6 @@ func (rs *PostgresRepositoryStore) GetPartiallyReplicatedRepositories(ctx contex
// is reached. Status of unassigned storages does not matter as long as they don't contain a later generation
// than the assigned ones.
//
- // If virtual storage scoped primaries are used, the primary is instead selected from the `shard_primaries` table.
rows, err := rs.db.QueryContext(ctx, `
SELECT
json_build_object (
@@ -592,10 +589,7 @@ SELECT
FROM (
SELECT
relative_path,
- CASE WHEN $3
- THEN shard_primaries.node_name
- ELSE repositories."primary"
- END AS "primary",
+ repositories.primary,
storage,
max(storage_repositories.generation) OVER (PARTITION BY virtual_storage, relative_path) - COALESCE(storage_repositories.generation, -1) AS behind_by,
repository_assignments.storage IS NOT NULL AS assigned
@@ -613,14 +607,13 @@ FROM (
)
) AS repository_assignments USING (virtual_storage, relative_path, storage)
JOIN repositories USING (virtual_storage, relative_path)
- LEFT JOIN shard_primaries ON $3 AND shard_name = virtual_storage AND NOT demoted
WHERE virtual_storage = $1
ORDER BY relative_path, "primary", storage
) AS outdated_repositories
GROUP BY relative_path, "primary"
HAVING max(behind_by) FILTER(WHERE assigned) > 0
ORDER BY relative_path, "primary"
- `, virtualStorage, pq.StringArray(configuredStorages), useVirtualStoragePrimaries)
+ `, virtualStorage, pq.StringArray(configuredStorages))
if err != nil {
return nil, fmt.Errorf("query: %w", err)
}
diff --git a/internal/praefect/datastore/repository_store_mock.go b/internal/praefect/datastore/repository_store_mock.go
index 3c5b5f81c..88ec52177 100644
--- a/internal/praefect/datastore/repository_store_mock.go
+++ b/internal/praefect/datastore/repository_store_mock.go
@@ -14,7 +14,7 @@ type MockRepositoryStore struct {
DeleteReplicaFunc func(ctx context.Context, virtualStorage, relativePath, storage string) error
RenameRepositoryFunc func(ctx context.Context, virtualStorage, relativePath, storage, newRelativePath string) error
GetConsistentStoragesFunc func(ctx context.Context, virtualStorage, relativePath string) (map[string]struct{}, error)
- GetPartiallyReplicatedRepositoriesFunc func(ctx context.Context, virtualStorage string, virtualStorageScopedPrimaries bool) ([]OutdatedRepository, error)
+ GetPartiallyReplicatedRepositoriesFunc func(ctx context.Context, virtualStorage string) ([]OutdatedRepository, error)
DeleteInvalidRepositoryFunc func(ctx context.Context, virtualStorage, relativePath, storage string) error
RepositoryExistsFunc func(ctx context.Context, virtualStorage, relativePath string) (bool, error)
}
@@ -95,12 +95,12 @@ func (m MockRepositoryStore) GetConsistentStorages(ctx context.Context, virtualS
return m.GetConsistentStoragesFunc(ctx, virtualStorage, relativePath)
}
-func (m MockRepositoryStore) GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string, virtualStorageScopedPrimaries bool) ([]OutdatedRepository, error) {
+func (m MockRepositoryStore) GetPartiallyReplicatedRepositories(ctx context.Context, virtualStorage string) ([]OutdatedRepository, error) {
if m.GetPartiallyReplicatedRepositoriesFunc == nil {
return nil, nil
}
- return m.GetPartiallyReplicatedRepositoriesFunc(ctx, virtualStorage, virtualStorageScopedPrimaries)
+ return m.GetPartiallyReplicatedRepositoriesFunc(ctx, virtualStorage)
}
func (m MockRepositoryStore) DeleteInvalidRepository(ctx context.Context, virtualStorage, relativePath, storage string) error {
diff --git a/internal/praefect/datastore/repository_store_test.go b/internal/praefect/datastore/repository_store_test.go
index 57e61dac0..7d6a9855d 100644
--- a/internal/praefect/datastore/repository_store_test.go
+++ b/internal/praefect/datastore/repository_store_test.go
@@ -800,207 +800,196 @@ func testRepositoryStore(t *testing.T, newStore repositoryStoreFactory) {
}
func TestPostgresRepositoryStore_GetPartiallyReplicatedRepositories(t *testing.T) {
- for _, scope := range []struct {
- desc string
- useVirtualStoragePrimaries bool
- primary string
+ for _, tc := range []struct {
+ desc string
+ nonExistentRepository bool
+ existingGenerations map[string]int
+ existingAssignments []string
+ storageDetails []OutdatedRepositoryStorageDetails
}{
- {desc: "virtual storage primaries", useVirtualStoragePrimaries: true, primary: "virtual-storage-primary"},
- {desc: "repository primaries", useVirtualStoragePrimaries: false, primary: "repository-primary"},
+ {
+ desc: "all up to date without assignments",
+ existingGenerations: map[string]int{"primary": 0, "secondary-1": 0},
+ },
+ {
+ desc: "unconfigured node outdated without assignments",
+ existingGenerations: map[string]int{"primary": 1, "secondary-1": 1, "unconfigured": 0},
+ },
+ {
+ desc: "unconfigured node contains the latest",
+ existingGenerations: map[string]int{"primary": 0, "secondary-1": 0, "unconfigured": 1},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 1, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ {Name: "unconfigured", BehindBy: 0, Assigned: false},
+ },
+ },
+ {
+ desc: "node has no repository without assignments",
+ existingGenerations: map[string]int{"primary": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ },
+ },
+ {
+ desc: "node has outdated repository without assignments",
+ existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ },
+ },
+ {
+ desc: "node with no repository heavily outdated",
+ existingGenerations: map[string]int{"primary": 10},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 11, Assigned: true},
+ },
+ },
+ {
+ desc: "node with a heavily outdated repository",
+ existingGenerations: map[string]int{"primary": 10, "secondary-1": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 10, Assigned: true},
+ },
+ },
+ {
+ desc: "outdated nodes ignored when repository should not exist",
+ nonExistentRepository: true,
+ existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
+ },
+ {
+ desc: "unassigned node has no repository",
+ existingAssignments: []string{"primary"},
+ existingGenerations: map[string]int{"primary": 0},
+ },
+ {
+ desc: "unassigned node has an outdated repository",
+ existingAssignments: []string{"primary"},
+ existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
+ },
+ {
+ desc: "assigned node has no repository",
+ existingAssignments: []string{"primary", "secondary-1"},
+ existingGenerations: map[string]int{"primary": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ },
+ },
+ {
+ desc: "assigned node has outdated repository",
+ existingAssignments: []string{"primary", "secondary-1"},
+ existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 0, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ },
+ },
+ {
+ desc: "unassigned node contains the latest repository",
+ existingAssignments: []string{"primary"},
+ existingGenerations: map[string]int{"primary": 0, "secondary-1": 1},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 1, Assigned: true},
+ {Name: "secondary-1", BehindBy: 0, Assigned: false},
+ },
+ },
+ {
+ desc: "unassigned node contains the only repository",
+ existingAssignments: []string{"primary"},
+ existingGenerations: map[string]int{"secondary-1": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 1, Assigned: true},
+ {Name: "secondary-1", BehindBy: 0, Assigned: false},
+ },
+ },
+ {
+ desc: "unassigned unconfigured node contains the only repository",
+ existingAssignments: []string{"primary"},
+ existingGenerations: map[string]int{"unconfigured": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 1, Assigned: true},
+ {Name: "unconfigured", BehindBy: 0, Assigned: false},
+ },
+ },
+ {
+ desc: "assigned unconfigured node has no repository",
+ existingAssignments: []string{"primary", "unconfigured"},
+ existingGenerations: map[string]int{"primary": 1},
+ },
+ {
+ desc: "assigned unconfigured node is outdated",
+ existingAssignments: []string{"primary", "unconfigured"},
+ existingGenerations: map[string]int{"primary": 1, "unconfigured": 0},
+ },
+ {
+ desc: "unconfigured node is the only assigned node",
+ existingAssignments: []string{"unconfigured"},
+ existingGenerations: map[string]int{"unconfigured": 0},
+ storageDetails: []OutdatedRepositoryStorageDetails{
+ {Name: "primary", BehindBy: 1, Assigned: true},
+ {Name: "secondary-1", BehindBy: 1, Assigned: true},
+ {Name: "unconfigured", BehindBy: 0, Assigned: false},
+ },
+ },
} {
- t.Run(scope.desc, func(t *testing.T) {
- for _, tc := range []struct {
- desc string
- nonExistentRepository bool
- existingGenerations map[string]int
- existingAssignments []string
- storageDetails []OutdatedRepositoryStorageDetails
- }{
- {
- desc: "all up to date without assignments",
- existingGenerations: map[string]int{"primary": 0, "secondary-1": 0},
- },
- {
- desc: "unconfigured node outdated without assignments",
- existingGenerations: map[string]int{"primary": 1, "secondary-1": 1, "unconfigured": 0},
- },
- {
- desc: "unconfigured node contains the latest",
- existingGenerations: map[string]int{"primary": 0, "secondary-1": 0, "unconfigured": 1},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 1, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- {Name: "unconfigured", BehindBy: 0, Assigned: false},
- },
- },
- {
- desc: "node has no repository without assignments",
- existingGenerations: map[string]int{"primary": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- },
- },
- {
- desc: "node has outdated repository without assignments",
- existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- },
- },
- {
- desc: "node with no repository heavily outdated",
- existingGenerations: map[string]int{"primary": 10},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 11, Assigned: true},
- },
- },
- {
- desc: "node with a heavily outdated repository",
- existingGenerations: map[string]int{"primary": 10, "secondary-1": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 10, Assigned: true},
- },
- },
- {
- desc: "outdated nodes ignored when repository should not exist",
- nonExistentRepository: true,
- existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
- },
- {
- desc: "unassigned node has no repository",
- existingAssignments: []string{"primary"},
- existingGenerations: map[string]int{"primary": 0},
- },
- {
- desc: "unassigned node has an outdated repository",
- existingAssignments: []string{"primary"},
- existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
- },
- {
- desc: "assigned node has no repository",
- existingAssignments: []string{"primary", "secondary-1"},
- existingGenerations: map[string]int{"primary": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- },
- },
- {
- desc: "assigned node has outdated repository",
- existingAssignments: []string{"primary", "secondary-1"},
- existingGenerations: map[string]int{"primary": 1, "secondary-1": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 0, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- },
- },
- {
- desc: "unassigned node contains the latest repository",
- existingAssignments: []string{"primary"},
- existingGenerations: map[string]int{"primary": 0, "secondary-1": 1},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 1, Assigned: true},
- {Name: "secondary-1", BehindBy: 0, Assigned: false},
- },
- },
- {
- desc: "unassigned node contains the only repository",
- existingAssignments: []string{"primary"},
- existingGenerations: map[string]int{"secondary-1": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 1, Assigned: true},
- {Name: "secondary-1", BehindBy: 0, Assigned: false},
- },
- },
- {
- desc: "unassigned unconfigured node contains the only repository",
- existingAssignments: []string{"primary"},
- existingGenerations: map[string]int{"unconfigured": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 1, Assigned: true},
- {Name: "unconfigured", BehindBy: 0, Assigned: false},
- },
- },
- {
- desc: "assigned unconfigured node has no repository",
- existingAssignments: []string{"primary", "unconfigured"},
- existingGenerations: map[string]int{"primary": 1},
- },
- {
- desc: "assigned unconfigured node is outdated",
- existingAssignments: []string{"primary", "unconfigured"},
- existingGenerations: map[string]int{"primary": 1, "unconfigured": 0},
- },
- {
- desc: "unconfigured node is the only assigned node",
- existingAssignments: []string{"unconfigured"},
- existingGenerations: map[string]int{"unconfigured": 0},
- storageDetails: []OutdatedRepositoryStorageDetails{
- {Name: "primary", BehindBy: 1, Assigned: true},
- {Name: "secondary-1", BehindBy: 1, Assigned: true},
- {Name: "unconfigured", BehindBy: 0, Assigned: false},
- },
- },
- } {
- t.Run(tc.desc, func(t *testing.T) {
- ctx, cancel := testhelper.Context()
- defer cancel()
+ t.Run(tc.desc, func(t *testing.T) {
+ ctx, cancel := testhelper.Context()
+ defer cancel()
- db := getDB(t)
+ db := getDB(t)
- configuredStorages := map[string][]string{"virtual-storage": {"primary", "secondary-1"}}
+ configuredStorages := map[string][]string{"virtual-storage": {"primary", "secondary-1"}}
- if !tc.nonExistentRepository {
- _, err := db.ExecContext(ctx, `
+ if !tc.nonExistentRepository {
+ _, err := db.ExecContext(ctx, `
INSERT INTO repositories (virtual_storage, relative_path, "primary")
VALUES ('virtual-storage', 'relative-path', 'repository-primary')
`)
- require.NoError(t, err)
- }
+ require.NoError(t, err)
+ }
- for storage, generation := range tc.existingGenerations {
- _, err := db.ExecContext(ctx, `
+ for storage, generation := range tc.existingGenerations {
+ _, err := db.ExecContext(ctx, `
INSERT INTO storage_repositories VALUES ('virtual-storage', 'relative-path', $1, $2)
`, storage, generation)
- require.NoError(t, err)
- }
+ require.NoError(t, err)
+ }
- for _, storage := range tc.existingAssignments {
- _, err := db.ExecContext(ctx, `
+ for _, storage := range tc.existingAssignments {
+ _, err := db.ExecContext(ctx, `
INSERT INTO repository_assignments VALUES ('virtual-storage', 'relative-path', $1)
`, storage)
- require.NoError(t, err)
- }
+ require.NoError(t, err)
+ }
- _, err := db.ExecContext(ctx, `
+ _, err := db.ExecContext(ctx, `
INSERT INTO shard_primaries (shard_name, node_name, elected_by_praefect, elected_at)
VALUES ('virtual-storage', 'virtual-storage-primary', 'ignored', now())
`)
- require.NoError(t, err)
-
- store := NewPostgresRepositoryStore(db, configuredStorages)
- outdated, err := store.GetPartiallyReplicatedRepositories(ctx, "virtual-storage", scope.useVirtualStoragePrimaries)
- require.NoError(t, err)
+ require.NoError(t, err)
- expected := []OutdatedRepository{
- {
- RelativePath: "relative-path",
- Primary: scope.primary,
- Storages: tc.storageDetails,
- },
- }
+ store := NewPostgresRepositoryStore(db, configuredStorages)
+ outdated, err := store.GetPartiallyReplicatedRepositories(ctx, "virtual-storage")
+ require.NoError(t, err)
- if tc.storageDetails == nil {
- expected = nil
- }
+ expected := []OutdatedRepository{
+ {
+ RelativePath: "relative-path",
+ Primary: "repository-primary",
+ Storages: tc.storageDetails,
+ },
+ }
- require.Equal(t, expected, outdated)
- })
+ if tc.storageDetails == nil {
+ expected = nil
}
+
+ require.Equal(t, expected, outdated)
})
}
}
diff --git a/internal/praefect/service/info/dataloss.go b/internal/praefect/service/info/dataloss.go
index 200f2bffb..9b6ad7da2 100644
--- a/internal/praefect/service/info/dataloss.go
+++ b/internal/praefect/service/info/dataloss.go
@@ -3,13 +3,11 @@ package info
import (
"context"
- "gitlab.com/gitlab-org/gitaly/v14/internal/praefect/config"
"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
)
func (s *Server) DatalossCheck(ctx context.Context, req *gitalypb.DatalossCheckRequest) (*gitalypb.DatalossCheckResponse, error) {
- outdatedRepos, err := s.rs.GetPartiallyReplicatedRepositories(
- ctx, req.GetVirtualStorage(), s.conf.Failover.ElectionStrategy != config.ElectionStrategyPerRepository)
+ outdatedRepos, err := s.rs.GetPartiallyReplicatedRepositories(ctx, req.GetVirtualStorage())
if err != nil {
return nil, err
}