Merge branch 'restore_incremental' into 'master'

Incremental restores

See merge request gitlab-org/gitaly!3915

5 files changed, 359 insertions, 170 deletions

diff --git a/internal/backup/backup.go b/internal/backup/backup.go
index 70a9b6d91..c59591b59 100644
--- a/internal/backup/backup.go
+++ b/internal/backup/backup.go
@@ -38,10 +38,20 @@ type Sink interface {
 	GetReader(ctx context.Context, relativePath string) (io.ReadCloser, error)
 }
 
-// Full represents all paths required for a full backup
-type Full struct {
+// Backup represents all the information needed to restore a backup for a repository
+type Backup struct {
+	// Steps are the ordered list of steps required to restore this backup
+	Steps []Step
+}
+
+// Step represents an incremental step that makes up a complete backup for a repository
+type Step struct {
 	// BundlePath is the path of the bundle
 	BundlePath string
+	// SkippableOnNotFound defines if the bundle can be skipped when it does
+	// not exist. This allows us to maintain legacy behaviour where we always
+	// check a specific location for a bundle without knowing if it exists.
+	SkippableOnNotFound bool
 	// RefPath is the path of the ref file
 	RefPath string
 	// CustomHooksPath is the path of the custom hooks archive
@@ -50,14 +60,14 @@ type Full struct {
 
 // Locator finds sink backup paths for repositories
 type Locator interface {
-	// BeginFull returns paths for a new full backup
-	BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Full
+	// BeginFull returns a tentative first step needed to create a new full backup.
+	BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Step
 
-	// CommitFull persists the paths for a new backup so that it can be looked up by FindLatestFull
-	CommitFull(ctx context.Context, full *Full) error
+	// CommitFull marks the step returned by `BeginFull` as the latest backup.
+	CommitFull(ctx context.Context, step *Step) error
 
-	// FindLatestFull returns the paths committed by the latest call to CommitFull
-	FindLatestFull(ctx context.Context, repo *gitalypb.Repository) (*Full, error)
+	// FindLatest returns the latest backup that was written by CommitFull
+	FindLatest(ctx context.Context, repo *gitalypb.Repository) (*Backup, error)
 }
 
 // ResolveSink returns a sink implementation based on the provided path.
@@ -172,38 +182,40 @@ func (mgr *Manager) Restore(ctx context.Context, req *RestoreRequest) error {
 		return fmt.Errorf("manager: %w", err)
 	}
 
-	full, err := mgr.locator.FindLatestFull(ctx, req.Repository)
+	backup, err := mgr.locator.FindLatest(ctx, req.Repository)
 	if err != nil {
-		return mgr.checkRestoreSkip(ctx, err, req)
+		return fmt.Errorf("manager: %w", err)
 	}
 
-	if err := mgr.restoreBundle(ctx, full.BundlePath, req.Server, req.Repository); err != nil {
-		return mgr.checkRestoreSkip(ctx, err, req)
-	}
-	if err := mgr.restoreCustomHooks(ctx, full.CustomHooksPath, req.Server, req.Repository); err != nil {
+	if err := mgr.createRepository(ctx, req.Server, req.Repository); err != nil {
 		return fmt.Errorf("manager: %w", err)
 	}
-	return nil
-}
 
-func (mgr *Manager) checkRestoreSkip(ctx context.Context, err error, req *RestoreRequest) error {
-	if errors.Is(err, ErrDoesntExist) {
-		// For compatibility with existing backups we need to always create the
-		// repository even if there's no bundle for project repositories
-		// (not wiki or snippet repositories).  Gitaly does not know which
-		// repository is which type so here we accept a parameter to tell us
-		// to employ this behaviour.
-		if req.AlwaysCreate {
-			if err := mgr.createRepository(ctx, req.Server, req.Repository); err != nil {
-				return fmt.Errorf("manager: %w", err)
+	for _, step := range backup.Steps {
+		if err := mgr.restoreBundle(ctx, step.BundlePath, req.Server, req.Repository); err != nil {
+			if step.SkippableOnNotFound && errors.Is(err, ErrDoesntExist) {
+				// For compatibility with existing backups we need to make sure the
+				// repository exists even if there's no bundle for project
+				// repositories (not wiki or snippet repositories).  Gitaly does
+				// not know which repository is which type so here we accept a
+				// parameter to tell us to employ this behaviour. Since the
+				// repository has already been created, we simply skip cleaning up.
+				if req.AlwaysCreate {
+					return nil
+				}
+
+				if err := mgr.removeRepository(ctx, req.Server, req.Repository); err != nil {
+					return fmt.Errorf("manager: remove on skipped: %w", err)
+				}
+
+				return fmt.Errorf("manager: %w: %s", ErrSkipped, err.Error())
 			}
-			return nil
 		}
-
-		return fmt.Errorf("manager: %w: %s", ErrSkipped, err.Error())
+		if err := mgr.restoreCustomHooks(ctx, step.CustomHooksPath, req.Server, req.Repository); err != nil {
+			return fmt.Errorf("manager: %w", err)
+		}
 	}
-
-	return fmt.Errorf("manager: %w", err)
+	return nil
 }
 
 func (mgr *Manager) isEmpty(ctx context.Context, server storage.ServerInfo, repo *gitalypb.Repository) (bool, error) {
@@ -317,11 +329,11 @@ func (mgr *Manager) restoreBundle(ctx context.Context, path string, server stora
 	if err != nil {
 		return fmt.Errorf("restore bundle: %q: %w", path, err)
 	}
-	stream, err := repoClient.CreateRepositoryFromBundle(ctx)
+	stream, err := repoClient.FetchBundle(ctx)
 	if err != nil {
 		return fmt.Errorf("restore bundle: %q: %w", path, err)
 	}
-	request := &gitalypb.CreateRepositoryFromBundleRequest{Repository: repo}
+	request := &gitalypb.FetchBundleRequest{Repository: repo}
 	bundle := streamio.NewWriter(func(p []byte) error {
 		request.Data = p
 		if err := stream.Send(request); err != nil {
@@ -329,7 +341,7 @@ func (mgr *Manager) restoreBundle(ctx context.Context, path string, server stora
 		}
 
 		// Only set `Repository` on the first `Send` of the stream
-		request = &gitalypb.CreateRepositoryFromBundleRequest{}
+		request = &gitalypb.FetchBundleRequest{}
 
 		return nil
 	})
diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go
index fd1dd04d8..ab8304965 100644
--- a/internal/backup/backup_test.go
+++ b/internal/backup/backup_test.go
@@ -2,7 +2,6 @@ package backup
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -203,7 +202,8 @@ func testManagerRestore(t *testing.T, cfg config.Cfg, gitalyAddr string) {
 
 	cc, err := client.Dial(gitalyAddr, nil)
 	require.NoError(t, err)
-	defer func() { require.NoError(t, cc.Close()) }()
+	defer testhelper.MustClose(t, cc)
+
 	repoClient := gitalypb.NewRepositoryServiceClient(cc)
 
 	createRepo := func(t testing.TB, relativePath string) *gitalypb.Repository {
@@ -211,7 +211,7 @@ func testManagerRestore(t *testing.T, cfg config.Cfg, gitalyAddr string) {
 
 		repo := &gitalypb.Repository{
 			StorageName:  "default",
-			RelativePath: relativePath,
+			RelativePath: gittest.NewRepositoryName(t, false) + relativePath,
 		}
 
 		for i := 0; true; i++ {
@@ -230,90 +230,158 @@ func testManagerRestore(t *testing.T, cfg config.Cfg, gitalyAddr string) {
 
 	path := testhelper.TempDir(t)
 
-	existingRepo := createRepo(t, "existing")
-	require.NoError(t, os.MkdirAll(filepath.Join(path, existingRepo.RelativePath), os.ModePerm))
-	existingRepoBundlePath := filepath.Join(path, existingRepo.RelativePath+".bundle")
-	gittest.BundleTestRepo(t, cfg, "gitlab-test.git", existingRepoBundlePath)
-
-	existingRepoHooks := createRepo(t, "existing_hooks")
-	existingRepoHooksBundlePath := filepath.Join(path, existingRepoHooks.RelativePath+".bundle")
-	existingRepoHooksCustomHooksPath := filepath.Join(path, existingRepoHooks.RelativePath, "custom_hooks.tar")
-	require.NoError(t, os.MkdirAll(filepath.Join(path, existingRepoHooks.RelativePath), os.ModePerm))
-	testhelper.CopyFile(t, existingRepoBundlePath, existingRepoHooksBundlePath)
-	testhelper.CopyFile(t, "../gitaly/service/repository/testdata/custom_hooks.tar", existingRepoHooksCustomHooksPath)
-
-	missingBundleRepo := createRepo(t, "missing_bundle")
-	missingBundleRepoAlwaysCreate := createRepo(t, "missing_bundle_always_create")
-
-	nonexistentRepo := &gitalypb.Repository{
-		StorageName:  "default",
-		RelativePath: "nonexistent",
-	}
-	nonexistentRepoBundlePath := filepath.Join(path, nonexistentRepo.RelativePath+".bundle")
-	testhelper.CopyFile(t, existingRepoBundlePath, nonexistentRepoBundlePath)
-
 	for _, tc := range []struct {
 		desc          string
-		repo          *gitalypb.Repository
+		locators      []string
+		setup         func(t testing.TB) (repo *gitalypb.Repository, bundles []string)
 		alwaysCreate  bool
+		expectExists  bool
 		expectedPaths []string
 		expectedErrAs error
-		expectVerify  bool
 	}{
 		{
-			desc:         "existing repo, without hooks",
-			repo:         existingRepo,
-			expectVerify: true,
+			desc:     "existing repo, without hooks",
+			locators: []string{"legacy", "pointer"},
+			setup: func(t testing.TB) (repo *gitalypb.Repository, bundles []string) {
+				existingRepo := createRepo(t, "existing")
+				require.NoError(t, os.MkdirAll(filepath.Join(path, existingRepo.RelativePath), os.ModePerm))
+				existingRepoBundlePath := filepath.Join(path, existingRepo.RelativePath+".bundle")
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", existingRepoBundlePath)
+				return existingRepo, []string{existingRepoBundlePath}
+			},
+			expectExists: true,
 		},
 		{
-			desc: "existing repo, with hooks",
-			repo: existingRepoHooks,
+			desc:     "existing repo, with hooks",
+			locators: []string{"legacy", "pointer"},
+			setup: func(t testing.TB) (repo *gitalypb.Repository, bundles []string) {
+				existingRepoHooks := createRepo(t, "existing_hooks")
+				existingRepoHooksBundlePath := filepath.Join(path, existingRepoHooks.RelativePath+".bundle")
+				existingRepoHooksCustomHooksPath := filepath.Join(path, existingRepoHooks.RelativePath, "custom_hooks.tar")
+				require.NoError(t, os.MkdirAll(filepath.Join(path, existingRepoHooks.RelativePath), os.ModePerm))
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", existingRepoHooksBundlePath)
+				testhelper.CopyFile(t, "../gitaly/service/repository/testdata/custom_hooks.tar", existingRepoHooksCustomHooksPath)
+				return existingRepoHooks, []string{existingRepoHooksBundlePath}
+			},
 			expectedPaths: []string{
 				"custom_hooks/pre-commit.sample",
 				"custom_hooks/prepare-commit-msg.sample",
 				"custom_hooks/pre-push.sample",
 			},
-			expectVerify: true,
+			expectExists: true,
 		},
 		{
-			desc:          "missing bundle",
-			repo:          missingBundleRepo,
+			desc:     "missing bundle",
+			locators: []string{"legacy", "pointer"},
+			setup: func(t testing.TB) (repo *gitalypb.Repository, bundles []string) {
+				missingBundleRepo := createRepo(t, "missing_bundle")
+				return missingBundleRepo, nil
+			},
 			expectedErrAs: ErrSkipped,
 		},
 		{
-			desc:         "missing bundle, always create",
-			repo:         missingBundleRepoAlwaysCreate,
+			desc:     "missing bundle, always create",
+			locators: []string{"legacy", "pointer"},
+			setup: func(t testing.TB) (repo *gitalypb.Repository, bundles []string) {
+				missingBundleRepoAlwaysCreate := createRepo(t, "missing_bundle_always_create")
+				return missingBundleRepoAlwaysCreate, nil
+			},
 			alwaysCreate: true,
+			expectExists: true,
+		},
+		{
+			desc:     "nonexistent repo",
+			locators: []string{"legacy", "pointer"},
+			setup: func(t testing.TB) (repo *gitalypb.Repository, bundles []string) {
+				nonexistentRepo := &gitalypb.Repository{
+					StorageName:  "default",
+					RelativePath: "nonexistent",
+				}
+				nonexistentRepoBundlePath := filepath.Join(path, nonexistentRepo.RelativePath+".bundle")
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", nonexistentRepoBundlePath)
+				return nonexistentRepo, []string{nonexistentRepoBundlePath}
+			},
+			expectExists: true,
+		},
+		{
+			desc:     "single incremental",
+			locators: []string{"pointer"},
+			setup: func(t testing.TB) (*gitalypb.Repository, []string) {
+				const backupID = "abc123"
+				repo := createRepo(t, "incremental")
+				repoBackupPath := filepath.Join(path, repo.RelativePath)
+				backupPath := filepath.Join(repoBackupPath, backupID)
+				require.NoError(t, os.MkdirAll(backupPath, os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("001"), os.ModePerm))
+				bundlePath := filepath.Join(backupPath, "001.bundle")
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", bundlePath)
+				return repo, []string{bundlePath}
+			},
+			expectExists: true,
 		},
 		{
-			desc:         "nonexistent repo",
-			repo:         nonexistentRepo,
-			expectVerify: true,
+			desc:     "many incrementals",
+			locators: []string{"pointer"},
+			setup: func(t testing.TB) (*gitalypb.Repository, []string) {
+				const backupID = "abc123"
+				repo := createRepo(t, "incremental")
+				repoBackupPath := filepath.Join(path, repo.RelativePath)
+				backupPath := filepath.Join(repoBackupPath, backupID)
+				require.NoError(t, os.MkdirAll(backupPath, os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(repoBackupPath, "LATEST"), []byte(backupID), os.ModePerm))
+				require.NoError(t, os.WriteFile(filepath.Join(backupPath, "LATEST"), []byte("002"), os.ModePerm))
+
+				bundlePath1 := filepath.Join(backupPath, "001.bundle")
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", bundlePath1, "master")
+
+				bundlePath2 := filepath.Join(backupPath, "002.bundle")
+				gittest.BundleTestRepo(t, cfg, "gitlab-test.git", bundlePath2, "feature")
+
+				return repo, []string{bundlePath1, bundlePath2}
+			},
+			expectExists: true,
 		},
 	} {
 		t.Run(tc.desc, func(t *testing.T) {
-			repoPath := filepath.Join(cfg.Storages[0].Path, tc.repo.RelativePath)
-			bundlePath := filepath.Join(path, tc.repo.RelativePath+".bundle")
-
-			fsBackup := NewManager(NewFilesystemSink(path), LegacyLocator{})
-			err := fsBackup.Restore(ctx, &RestoreRequest{
-				Server:       storage.ServerInfo{Address: gitalyAddr, Token: cfg.Auth.Token},
-				Repository:   tc.repo,
-				AlwaysCreate: tc.alwaysCreate,
-			})
-			if tc.expectedErrAs != nil {
-				require.True(t, errors.Is(err, tc.expectedErrAs), err.Error())
-			} else {
-				require.NoError(t, err)
-			}
-
-			if tc.expectVerify {
-				output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
-				require.Contains(t, string(output), "The bundle records a complete history")
-			}
-
-			for _, p := range tc.expectedPaths {
-				require.FileExists(t, filepath.Join(repoPath, p))
+			require.GreaterOrEqual(t, len(tc.locators), 1, "each test case must specify a locator")
+
+			for _, locatorName := range tc.locators {
+				t.Run(locatorName, func(t *testing.T) {
+					repo, bundles := tc.setup(t)
+					repoPath := filepath.Join(cfg.Storages[0].Path, repo.RelativePath)
+
+					sink := NewFilesystemSink(path)
+					locator, err := ResolveLocator(locatorName, sink)
+					require.NoError(t, err)
+
+					fsBackup := NewManager(sink, locator)
+					err = fsBackup.Restore(ctx, &RestoreRequest{
+						Server:       storage.ServerInfo{Address: gitalyAddr, Token: cfg.Auth.Token},
+						Repository:   repo,
+						AlwaysCreate: tc.alwaysCreate,
+					})
+					if tc.expectedErrAs != nil {
+						require.ErrorAs(t, err, &tc.expectedErrAs)
+					} else {
+						require.NoError(t, err)
+					}
+
+					exists, err := repoClient.RepositoryExists(ctx, &gitalypb.RepositoryExistsRequest{
+						Repository: repo,
+					})
+					require.NoError(t, err)
+					require.Equal(t, tc.expectExists, exists.Exists, "repository exists")
+
+					for _, bundlePath := range bundles {
+						output := gittest.Exec(t, cfg, "-C", repoPath, "bundle", "verify", bundlePath)
+						require.Contains(t, string(output), "The bundle records a complete history")
+					}
+
+					for _, p := range tc.expectedPaths {
+						require.FileExists(t, filepath.Join(repoPath, p))
+					}
+				})
 			}
 		})
 	}
diff --git a/internal/backup/locator.go b/internal/backup/locator.go
index 6110240b9..81e53dcb1 100644
--- a/internal/backup/locator.go
+++ b/internal/backup/locator.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"gitlab.com/gitlab-org/gitaly/v14/internal/helper/text"
@@ -25,27 +26,32 @@ import (
 type LegacyLocator struct{}
 
 // BeginFull returns the static paths for a legacy repository backup
-func (l LegacyLocator) BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Full {
+func (l LegacyLocator) BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Step {
 	return l.newFull(repo)
 }
 
 // CommitFull is unused as the locations are static
-func (l LegacyLocator) CommitFull(ctx context.Context, full *Full) error {
+func (l LegacyLocator) CommitFull(ctx context.Context, full *Step) error {
 	return nil
 }
 
-// FindLatestFull returns the static paths for a legacy repository backup
-func (l LegacyLocator) FindLatestFull(ctx context.Context, repo *gitalypb.Repository) (*Full, error) {
-	return l.newFull(repo), nil
+// FindLatest returns the static paths for a legacy repository backup
+func (l LegacyLocator) FindLatest(ctx context.Context, repo *gitalypb.Repository) (*Backup, error) {
+	return &Backup{
+		Steps: []Step{
+			*l.newFull(repo),
+		},
+	}, nil
 }
 
-func (l LegacyLocator) newFull(repo *gitalypb.Repository) *Full {
+func (l LegacyLocator) newFull(repo *gitalypb.Repository) *Step {
 	backupPath := strings.TrimSuffix(repo.RelativePath, ".git")
 
-	return &Full{
-		BundlePath:      backupPath + ".bundle",
-		RefPath:         backupPath + ".refs",
-		CustomHooksPath: filepath.Join(backupPath, "custom_hooks.tar"),
+	return &Step{
+		SkippableOnNotFound: true,
+		BundlePath:          backupPath + ".bundle",
+		RefPath:             backupPath + ".refs",
+		CustomHooksPath:     filepath.Join(backupPath, "custom_hooks.tar"),
 	}
 }
 
@@ -54,53 +60,78 @@ func (l LegacyLocator) newFull(repo *gitalypb.Repository) *Full {
 // file named LATEST.
 //
 // Structure:
-//   <repo relative path>/<backup id>/full.bundle
-//   <repo relative path>/<backup id>/full.refs
-//   <repo relative path>/<backup id>/custom_hooks.tar
 //   <repo relative path>/LATEST
+//   <repo relative path>/<backup id>/LATEST
+//   <repo relative path>/<backup id>/<nnn>.bundle
+//   <repo relative path>/<backup id>/<nnn>.refs
+//   <repo relative path>/<backup id>/<nnn>.custom_hooks.tar
 type PointerLocator struct {
 	Sink     Sink
 	Fallback Locator
 }
 
 // BeginFull returns paths for a new full backup
-func (l PointerLocator) BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Full {
+func (l PointerLocator) BeginFull(ctx context.Context, repo *gitalypb.Repository, backupID string) *Step {
 	backupPath := strings.TrimSuffix(repo.RelativePath, ".git")
 
-	return &Full{
-		BundlePath:      filepath.Join(backupPath, backupID, "full.bundle"),
-		RefPath:         filepath.Join(backupPath, backupID, "full.refs"),
-		CustomHooksPath: filepath.Join(backupPath, backupID, "custom_hooks.tar"),
+	return &Step{
+		BundlePath:      filepath.Join(backupPath, backupID, "001.bundle"),
+		RefPath:         filepath.Join(backupPath, backupID, "001.refs"),
+		CustomHooksPath: filepath.Join(backupPath, backupID, "001.custom_hooks.tar"),
 	}
 }
 
-// CommitFull persists the paths for a new backup so that it can be looked up by FindLatestFull
-func (l PointerLocator) CommitFull(ctx context.Context, full *Full) error {
+// CommitFull persists the paths for a new backup so that it can be looked up by FindLatest
+func (l PointerLocator) CommitFull(ctx context.Context, full *Step) error {
 	bundleDir := filepath.Dir(full.BundlePath)
 	backupID := filepath.Base(bundleDir)
 	backupPath := filepath.Dir(bundleDir)
-	return l.commitLatestID(ctx, backupPath, backupID)
+	if err := l.writeLatest(ctx, bundleDir, "001"); err != nil {
+		return err
+	}
+	if err := l.writeLatest(ctx, backupPath, backupID); err != nil {
+		return err
+	}
+	return nil
 }
 
-// FindLatestFull returns the paths committed by the latest call to CommitFull.
+// FindLatest returns the paths committed by the latest call to CommitFull.
 //
 // If there is no `LATEST` file, the result of the `Fallback` is used.
-func (l PointerLocator) FindLatestFull(ctx context.Context, repo *gitalypb.Repository) (*Full, error) {
-	backupPath := strings.TrimSuffix(repo.RelativePath, ".git")
+func (l PointerLocator) FindLatest(ctx context.Context, repo *gitalypb.Repository) (*Backup, error) {
+	repoPath := strings.TrimSuffix(repo.RelativePath, ".git")
 
-	latest, err := l.findLatestID(ctx, backupPath)
+	backupID, err := l.findLatestID(ctx, repoPath)
 	if err != nil {
 		if l.Fallback != nil && errors.Is(err, ErrDoesntExist) {
-			return l.Fallback.FindLatestFull(ctx, repo)
+			return l.Fallback.FindLatest(ctx, repo)
 		}
-		return nil, fmt.Errorf("pointer locator: %w", err)
+		return nil, fmt.Errorf("pointer locator: backup: %w", err)
 	}
 
-	return &Full{
-		BundlePath:      filepath.Join(backupPath, latest, "full.bundle"),
-		RefPath:         filepath.Join(backupPath, latest, "full.refs"),
-		CustomHooksPath: filepath.Join(backupPath, latest, "custom_hooks.tar"),
-	}, nil
+	backupPath := filepath.Join(repoPath, backupID)
+
+	latestIncrementID, err := l.findLatestID(ctx, backupPath)
+	if err != nil {
+		return nil, fmt.Errorf("pointer locator: latest incremental: %w", err)
+	}
+
+	max, err := strconv.Atoi(latestIncrementID)
+	if err != nil {
+		return nil, fmt.Errorf("pointer locator: latest incremental: %w", err)
+	}
+
+	var backup Backup
+
+	for i := 1; i <= max; i++ {
+		backup.Steps = append(backup.Steps, Step{
+			BundlePath:      filepath.Join(backupPath, fmt.Sprintf("%03d.bundle", i)),
+			RefPath:         filepath.Join(backupPath, fmt.Sprintf("%03d.refs", i)),
+			CustomHooksPath: filepath.Join(backupPath, fmt.Sprintf("%03d.custom_hooks.tar", i)),
+		})
+	}
+
+	return &backup, nil
 }
 
 func (l PointerLocator) findLatestID(ctx context.Context, backupPath string) (string, error) {
@@ -118,10 +149,10 @@ func (l PointerLocator) findLatestID(ctx context.Context, backupPath string) (st
 	return text.ChompBytes(latest), nil
 }
 
-func (l PointerLocator) commitLatestID(ctx context.Context, backupPath, backupID string) error {
-	latest := strings.NewReader(backupID)
-	if err := l.Sink.Write(ctx, filepath.Join(backupPath, "LATEST"), latest); err != nil {
-		return fmt.Errorf("commit latest ID: %w", err)
+func (l PointerLocator) writeLatest(ctx context.Context, path, target string) error {
+	latest := strings.NewReader(target)
+	if err := l.Sink.Write(ctx, filepath.Join(path, "LATEST"), latest); err != nil {
+		return fmt.Errorf("write latest: %w", err)
 	}
 	return nil
 }
diff --git a/internal/backup/locator_test.go b/internal/backup/locator_test.go
index f8e97a2e5..7bd3b48d7 100644
--- a/internal/backup/locator_test.go
+++ b/internal/backup/locator_test.go
@@ -19,10 +19,11 @@ func TestLegacyLocator(t *testing.T) {
 		ctx, cancel := testhelper.Context()
 		defer cancel()
 
-		expected := &Full{
-			BundlePath:      repo.RelativePath + ".bundle",
-			RefPath:         repo.RelativePath + ".refs",
-			CustomHooksPath: filepath.Join(repo.RelativePath, "custom_hooks.tar"),
+		expected := &Step{
+			SkippableOnNotFound: true,
+			BundlePath:          repo.RelativePath + ".bundle",
+			RefPath:             repo.RelativePath + ".refs",
+			CustomHooksPath:     filepath.Join(repo.RelativePath, "custom_hooks.tar"),
 		}
 
 		full := l.BeginFull(ctx, repo, "abc123")
@@ -31,17 +32,22 @@ func TestLegacyLocator(t *testing.T) {
 		require.NoError(t, l.CommitFull(ctx, full))
 	})
 
-	t.Run("FindLatestFull", func(t *testing.T) {
+	t.Run("FindLatest", func(t *testing.T) {
 		ctx, cancel := testhelper.Context()
 		defer cancel()
 
-		expected := &Full{
-			BundlePath:      repo.RelativePath + ".bundle",
-			RefPath:         repo.RelativePath + ".refs",
-			CustomHooksPath: filepath.Join(repo.RelativePath, "custom_hooks.tar"),
+		expected := &Backup{
+			Steps: []Step{
+				{
+					SkippableOnNotFound: true,
+					BundlePath:          repo.RelativePath + ".bundle",
+					RefPath:             repo.RelativePath + ".refs",
+					CustomHooksPath:     filepath.Join(repo.RelativePath, "custom_hooks.tar"),
+				},
+			},
 		}
 
-		full, err := l.FindLatestFull(ctx, repo)
+		full, err := l.FindLatest(ctx, repo)
 		require.NoError(t, err)
 
 		assert.Equal(t, expected, full)
@@ -62,10 +68,11 @@ func TestPointerLocator(t *testing.T) {
 		ctx, cancel := testhelper.Context()
 		defer cancel()
 
-		expected := &Full{
-			BundlePath:      filepath.Join(repo.RelativePath, backupID, "full.bundle"),
-			RefPath:         filepath.Join(repo.RelativePath, backupID, "full.refs"),
-			CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "custom_hooks.tar"),
+		const expectedIncrement = "001"
+		expected := &Step{
+			BundlePath:      filepath.Join(repo.RelativePath, backupID, expectedIncrement+".bundle"),
+			RefPath:         filepath.Join(repo.RelativePath, backupID, expectedIncrement+".refs"),
+			CustomHooksPath: filepath.Join(repo.RelativePath, backupID, expectedIncrement+".custom_hooks.tar"),
 		}
 
 		full := l.BeginFull(ctx, repo, backupID)
@@ -73,11 +80,14 @@ func TestPointerLocator(t *testing.T) {
 
 		require.NoError(t, l.CommitFull(ctx, full))
 
-		pointer := testhelper.MustReadFile(t, filepath.Join(backupPath, repo.RelativePath, "LATEST"))
-		require.Equal(t, backupID, string(pointer))
+		backupPointer := testhelper.MustReadFile(t, filepath.Join(backupPath, repo.RelativePath, "LATEST"))
+		require.Equal(t, backupID, string(backupPointer))
+
+		incrementPointer := testhelper.MustReadFile(t, filepath.Join(backupPath, repo.RelativePath, backupID, "LATEST"))
+		require.Equal(t, expectedIncrement, string(incrementPointer))
 	})
 
-	t.Run("FindLatestFull", func(t *testing.T) {
+	t.Run("FindLatest", func(t *testing.T) {
 		t.Run("no fallback", func(t *testing.T) {
 			backupPath := testhelper.TempDir(t)
 			var l Locator = PointerLocator{
@@ -87,18 +97,33 @@ func TestPointerLocator(t *testing.T) {
 			ctx, cancel := testhelper.Context()
 			defer cancel()
 
-			_, err := l.FindLatestFull(ctx, repo)
+			_, err := l.FindLatest(ctx, repo)
 			require.ErrorIs(t, err, ErrDoesntExist)
 
-			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath), 0o755))
+			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath, backupID), 0o755))
 			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, "LATEST"), []byte(backupID), 0o644))
-			expected := &Full{
-				BundlePath:      filepath.Join(repo.RelativePath, backupID, "full.bundle"),
-				RefPath:         filepath.Join(repo.RelativePath, backupID, "full.refs"),
-				CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "custom_hooks.tar"),
+			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, backupID, "LATEST"), []byte("003"), 0o644))
+			expected := &Backup{
+				Steps: []Step{
+					{
+						BundlePath:      filepath.Join(repo.RelativePath, backupID, "001.bundle"),
+						RefPath:         filepath.Join(repo.RelativePath, backupID, "001.refs"),
+						CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "001.custom_hooks.tar"),
+					},
+					{
+						BundlePath:      filepath.Join(repo.RelativePath, backupID, "002.bundle"),
+						RefPath:         filepath.Join(repo.RelativePath, backupID, "002.refs"),
+						CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "002.custom_hooks.tar"),
+					},
+					{
+						BundlePath:      filepath.Join(repo.RelativePath, backupID, "003.bundle"),
+						RefPath:         filepath.Join(repo.RelativePath, backupID, "003.refs"),
+						CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "003.custom_hooks.tar"),
+					},
+				},
 			}
 
-			full, err := l.FindLatestFull(ctx, repo)
+			full, err := l.FindLatest(ctx, repo)
 			require.NoError(t, err)
 			require.Equal(t, expected, full)
 		})
@@ -113,27 +138,75 @@ func TestPointerLocator(t *testing.T) {
 			ctx, cancel := testhelper.Context()
 			defer cancel()
 
-			expectedFallback := &Full{
-				BundlePath:      repo.RelativePath + ".bundle",
-				RefPath:         repo.RelativePath + ".refs",
-				CustomHooksPath: filepath.Join(repo.RelativePath, "custom_hooks.tar"),
+			expectedFallback := &Backup{
+				Steps: []Step{
+					{
+						SkippableOnNotFound: true,
+						BundlePath:          repo.RelativePath + ".bundle",
+						RefPath:             repo.RelativePath + ".refs",
+						CustomHooksPath:     filepath.Join(repo.RelativePath, "custom_hooks.tar"),
+					},
+				},
 			}
 
-			fallbackFull, err := l.FindLatestFull(ctx, repo)
+			fallbackFull, err := l.FindLatest(ctx, repo)
 			require.NoError(t, err)
 			require.Equal(t, expectedFallback, fallbackFull)
 
-			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath), 0o755))
+			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath, backupID), 0o755))
 			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, "LATEST"), []byte(backupID), 0o644))
-			expected := &Full{
-				BundlePath:      filepath.Join(repo.RelativePath, backupID, "full.bundle"),
-				RefPath:         filepath.Join(repo.RelativePath, backupID, "full.refs"),
-				CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "custom_hooks.tar"),
+			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, backupID, "LATEST"), []byte("001"), 0o644))
+			expected := &Backup{
+				Steps: []Step{
+					{
+						BundlePath:      filepath.Join(repo.RelativePath, backupID, "001.bundle"),
+						RefPath:         filepath.Join(repo.RelativePath, backupID, "001.refs"),
+						CustomHooksPath: filepath.Join(repo.RelativePath, backupID, "001.custom_hooks.tar"),
+					},
+				},
 			}
 
-			full, err := l.FindLatestFull(ctx, repo)
+			full, err := l.FindLatest(ctx, repo)
 			require.NoError(t, err)
 			require.Equal(t, expected, full)
 		})
+
+		t.Run("invalid backup LATEST", func(t *testing.T) {
+			backupPath := testhelper.TempDir(t)
+			var l Locator = PointerLocator{
+				Sink: NewFilesystemSink(backupPath),
+			}
+
+			ctx, cancel := testhelper.Context()
+			defer cancel()
+
+			_, err := l.FindLatest(ctx, repo)
+			require.ErrorIs(t, err, ErrDoesntExist)
+
+			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath), 0o755))
+			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, "LATEST"), []byte("invalid"), 0o644))
+			_, err = l.FindLatest(ctx, repo)
+			require.EqualError(t, err, "pointer locator: latest incremental: find latest ID: filesystem sink: get reader for \"TestPointerLocator/invalid/LATEST\": doesn't exist")
+		})
+
+		t.Run("invalid incremental LATEST", func(t *testing.T) {
+			backupPath := testhelper.TempDir(t)
+			var l Locator = PointerLocator{
+				Sink: NewFilesystemSink(backupPath),
+			}
+
+			ctx, cancel := testhelper.Context()
+			defer cancel()
+
+			_, err := l.FindLatest(ctx, repo)
+			require.ErrorIs(t, err, ErrDoesntExist)
+
+			require.NoError(t, os.MkdirAll(filepath.Join(backupPath, repo.RelativePath, backupID), 0o755))
+			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, "LATEST"), []byte(backupID), 0o644))
+			require.NoError(t, os.WriteFile(filepath.Join(backupPath, repo.RelativePath, backupID, "LATEST"), []byte("invalid"), 0o644))
+
+			_, err = l.FindLatest(ctx, repo)
+			require.EqualError(t, err, "pointer locator: latest incremental: strconv.Atoi: parsing \"invalid\": invalid syntax")
+		})
 	})
 }
diff --git a/internal/git/gittest/repo.go b/internal/git/gittest/repo.go
index 6821a38fb..771549c6c 100644
--- a/internal/git/gittest/repo.go
+++ b/internal/git/gittest/repo.go
@@ -155,10 +155,15 @@ func CloneRepo(t testing.TB, cfg config.Cfg, storage config.Storage, opts ...Clo
 	return repo, absolutePath
 }
 
-// BundleTestRepo creates a bundle of a local test repo. E.g. `gitlab-test.git`
-func BundleTestRepo(t testing.TB, cfg config.Cfg, sourceRepo, bundlePath string) {
+// BundleTestRepo creates a bundle of a local test repo. E.g.
+// `gitlab-test.git`. `patterns` define the bundle contents as per
+// `git-rev-list-args`. If there are no patterns then `--all` is assumed.
+func BundleTestRepo(t testing.TB, cfg config.Cfg, sourceRepo, bundlePath string, patterns ...string) {
+	if len(patterns) == 0 {
+		patterns = []string{"--all"}
+	}
 	repoPath := testRepositoryPath(t, sourceRepo)
-	Exec(t, cfg, "-C", repoPath, "bundle", "create", bundlePath, "--all")
+	Exec(t, cfg, append([]string{"-C", repoPath, "bundle", "create", bundlePath}, patterns...)...)
 }
 
 // testRepositoryPath returns the absolute path of local 'gitlab-org/gitlab-test.git' clone.