praefect: Add -db-only flag to remove repositorywc-remove-db-only

In some situations, most notably Geo replication, a large number of
repos may be present in the DB but not on disk. It's possible to remove
these entries using the `remove-repository` sub-command, but this
creates a risk of accidentally deleting a repo that's present on disk
and triggering permanent dataloss.

This commit adds a new `-db-only` flag to `remove-repository` allowing
admins to be confident that even if an on-disk repo is passed in it
won't be deleted from disk and can be re-tracked easily.

Changelog: added

2 files changed, 60 insertions, 6 deletions

diff --git a/cmd/praefect/subcmd_remove_repository.go b/cmd/praefect/subcmd_remove_repository.go
index fe991082d..9c966de6d 100644
--- a/cmd/praefect/subcmd_remove_repository.go
+++ b/cmd/praefect/subcmd_remove_repository.go
@@ -23,6 +23,7 @@ import (
 const (
 	removeRepositoryCmdName = "remove-repository"
 	paramApply              = "apply"
+	paramDBOnly             = "db-only"
 )
 
 type writer struct {
@@ -41,6 +42,7 @@ type removeRepository struct {
 	virtualStorage string
 	relativePath   string
 	apply          bool
+	dbOnly         bool
 	dialTimeout    time.Duration
 	w              io.Writer
 }
@@ -53,16 +55,19 @@ func (cmd *removeRepository) FlagSet() *flag.FlagSet {
 	fs := flag.NewFlagSet(removeRepositoryCmdName, flag.ExitOnError)
 	fs.StringVar(&cmd.virtualStorage, paramVirtualStorage, "", "name of the repository's virtual storage")
 	fs.BoolVar(&cmd.apply, paramApply, false, "physically remove the repository from disk and the database")
+	fs.BoolVar(&cmd.dbOnly, paramDBOnly, false, "remove the repository records from the database only")
 	fs.StringVar(&cmd.relativePath, paramRelativePath, "", "relative path to the repository")
 	fs.Usage = func() {
 		printfErr("Description:\n" +
 			"	This command removes all state associated with a given repository from the Gitaly Cluster.\n" +
 			"	This includes both on-disk repositories on all relevant Gitaly nodes as well as any potential\n" +
-			"	database state as tracked by Praefect.\n" +
+			"	database state as tracked by Praefect, or optionally only database state.\n" +
 			"	Runs in dry-run mode by default checks whether the repository exists" +
 			"	without actually removing it from the database and disk.\n" +
 			"	When -apply is used, the repository will be removed from the database as well as\n" +
-			"	the individual gitaly nodes on which they exist.\n")
+			"	the individual gitaly nodes on which they exist.\n" +
+			"   When -apply and -db-only are used, the repository will be removed from the\n" +
+			"   database but left in-place on the gitaly nodes.")
 		fs.PrintDefaults()
 	}
 	return fs
@@ -115,7 +120,24 @@ func (cmd *removeRepository) exec(ctx context.Context, logger logrus.FieldLogger
 	}
 
 	if !cmd.apply {
-		fmt.Fprintln(cmd.w, "Re-run the command with -apply to remove repositories from the database and disk.")
+		fmt.Fprintln(cmd.w, "Re-run the command with -apply to remove repositories from the database"+
+			" and disk or -apply and -db-only to remove from database only.")
+		return nil
+	}
+
+	ticker := helper.NewTimerTicker(time.Second)
+	defer ticker.Stop()
+
+	if cmd.dbOnly {
+		fmt.Fprintf(cmd.w, "Attempting to remove %s from the database...\n", cmd.relativePath)
+		if _, _, err := rs.DeleteRepository(ctx, cmd.virtualStorage, cmd.relativePath); err != nil {
+			return fmt.Errorf("remove repository from database: %w", err)
+		}
+		fmt.Fprintln(cmd.w, "Repository removal from database completed.")
+
+		if err := cmd.removeReplicationEvents(ctx, logger, db, ticker); err != nil {
+			return fmt.Errorf("remove scheduled replication events: %w", err)
+		}
 		return nil
 	}
 
@@ -137,8 +159,6 @@ func (cmd *removeRepository) exec(ctx context.Context, logger logrus.FieldLogger
 	fmt.Fprintln(cmd.w, "Repository removal completed.")
 
 	fmt.Fprintln(cmd.w, "Removing replication events...")
-	ticker := helper.NewTimerTicker(time.Second)
-	defer ticker.Stop()
 	if err := cmd.removeReplicationEvents(ctx, logger, db, ticker); err != nil {
 		return fmt.Errorf("remove scheduled replication events: %w", err)
 	}
diff --git a/cmd/praefect/subcmd_remove_repository_test.go b/cmd/praefect/subcmd_remove_repository_test.go
index 7811f87d0..19e53a231 100644
--- a/cmd/praefect/subcmd_remove_repository_test.go
+++ b/cmd/praefect/subcmd_remove_repository_test.go
@@ -19,6 +19,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v15/internal/git/gittest"
 	gitalycfg "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/config"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/service/setup"
+	"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/storage"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/helper"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/config"
 	"gitlab.com/gitlab-org/gitaly/v15/internal/praefect/datastore"
@@ -127,6 +128,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 			relativePath:   repo.RelativePath,
 			dialTimeout:    time.Second,
 			apply:          false,
+			dbOnly:         false,
 			w:              &writer{w: &out},
 		}
 		require.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf))
@@ -134,7 +136,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 		require.DirExists(t, filepath.Join(g1Cfg.Storages[0].Path, replicaPath))
 		require.DirExists(t, filepath.Join(g2Cfg.Storages[0].Path, replicaPath))
 		assert.Contains(t, out.String(), "Repository found in the database.\n")
-		assert.Contains(t, out.String(), "Re-run the command with -apply to remove repositories from the database and disk.\n")
+		assert.Contains(t, out.String(), "Re-run the command with -apply to remove repositories from the database and disk or -apply and -db-only to remove from database only.")
 		require.True(t, repositoryExists(t, repo))
 	})
 
@@ -151,6 +153,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 			relativePath:   repo.RelativePath,
 			dialTimeout:    time.Second,
 			apply:          true,
+			dbOnly:         false,
 			w:              &writer{w: &out},
 		}
 		require.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf))
@@ -163,6 +166,34 @@ func TestRemoveRepository_Exec(t *testing.T) {
 		require.False(t, repositoryExists(t, repo))
 	})
 
+	t.Run("db only", func(t *testing.T) {
+		var out bytes.Buffer
+		repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name())
+		replicaPath := gittest.GetReplicaPath(ctx, t, gitalycfg.Cfg{SocketPath: praefectServer.Address()}, repo)
+
+		cmd := &removeRepository{
+			logger:         testhelper.NewDiscardingLogger(t),
+			virtualStorage: repo.StorageName,
+			relativePath:   repo.RelativePath,
+			dialTimeout:    time.Second,
+			apply:          true,
+			dbOnly:         true,
+			w:              &writer{w: &out},
+		}
+		require.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf))
+
+		// Repo is no longer present in the Praefect DB.
+		require.False(t, repositoryExists(t, repo))
+
+		// Repo is still present on-disk on the Gitaly nodes.
+		require.True(t, storage.IsGitDirectory(filepath.Join(g1Cfg.Storages[0].Path, replicaPath)))
+		require.True(t, storage.IsGitDirectory(filepath.Join(g2Cfg.Storages[0].Path, replicaPath)))
+
+		assert.Contains(t, out.String(), "Repository found in the database.\n")
+		assert.Contains(t, out.String(), fmt.Sprintf("Attempting to remove %s from the database...\n", repo.RelativePath))
+		assert.Contains(t, out.String(), "Repository removal from database completed.")
+	})
+
 	t.Run("repository doesnt exist on one gitaly", func(t *testing.T) {
 		var out bytes.Buffer
 		repo := createRepo(t, ctx, repoClient, praefectStorage, t.Name())
@@ -178,6 +209,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 			relativePath:   repo.RelativePath,
 			dialTimeout:    time.Second,
 			apply:          true,
+			dbOnly:         false,
 			w:              &writer{w: &out},
 		}
 		require.NoError(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf))
@@ -205,6 +237,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 			relativePath:   repo.RelativePath,
 			dialTimeout:    time.Second,
 			apply:          true,
+			dbOnly:         false,
 			w:              &writer{w: &out},
 		}
 		require.Error(t, cmd.Exec(flag.NewFlagSet("", flag.PanicOnError), conf), "repository is not being tracked in Praefect")
@@ -228,6 +261,7 @@ func TestRemoveRepository_Exec(t *testing.T) {
 			relativePath:   repo.RelativePath,
 			dialTimeout:    100 * time.Millisecond,
 			apply:          true,
+			dbOnly:         false,
 			w:              &writer{w: &out},
 		}