From 0a3c32cb49864bf5a045aaf3e32d034117f6ab0e Mon Sep 17 00:00:00 2001 From: James Fargher Date: Mon, 15 Jan 2024 14:10:20 +1300 Subject: backup: Try read latest manifest when available Now that a latest manifest is written, we can use this manifest when loading the "latest" backup. --- internal/backup/locator.go | 32 ++++++++--- internal/backup/locator_test.go | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 6 deletions(-) diff --git a/internal/backup/locator.go b/internal/backup/locator.go index 216037aea..c0ccca3b3 100644 --- a/internal/backup/locator.go +++ b/internal/backup/locator.go @@ -267,6 +267,8 @@ func (l PointerLocator) writeLatest(ctx context.Context, path, target string) (r return nil } +const latestManifestName = "+latest" + // ManifestLocator locates backup paths based on manifest files that are // written to a predetermined path: // @@ -297,34 +299,52 @@ func (l ManifestLocator) Commit(ctx context.Context, backup *Backup) (returnErr if err := l.writeManifest(ctx, backup, backup.ID); err != nil { return fmt.Errorf("manifest: commit: %w", err) } - if err := l.writeManifest(ctx, backup, "+latest"); err != nil { - return fmt.Errorf("manifest: commit: %w", err) + if err := l.writeManifest(ctx, backup, latestManifestName); err != nil { + return fmt.Errorf("manifest: commit latest: %w", err) } return nil } -// FindLatest passes through to Fallback +// FindLatest loads the manifest called +latest. If this manifest does not +// exist, the Fallback is used. func (l ManifestLocator) FindLatest(ctx context.Context, repo storage.Repository) (*Backup, error) { - return l.Fallback.FindLatest(ctx, repo) + backup, err := l.readManifest(ctx, repo, latestManifestName) + switch { + case errors.Is(err, ErrDoesntExist): + return l.Fallback.FindLatest(ctx, repo) + case err != nil: + return nil, fmt.Errorf("manifest: find latest: %w", err) + } + + return backup, nil } // Find loads the manifest for the provided repo and backupID. If this manifest // does not exist, the fallback is used. func (l ManifestLocator) Find(ctx context.Context, repo storage.Repository, backupID string) (*Backup, error) { - f, err := l.Sink.GetReader(ctx, manifestPath(repo, backupID)) + backup, err := l.readManifest(ctx, repo, backupID) switch { case errors.Is(err, ErrDoesntExist): return l.Fallback.Find(ctx, repo, backupID) case err != nil: return nil, fmt.Errorf("manifest: find: %w", err) } + + return backup, nil +} + +func (l ManifestLocator) readManifest(ctx context.Context, repo storage.Repository, backupID string) (*Backup, error) { + f, err := l.Sink.GetReader(ctx, manifestPath(repo, backupID)) + if err != nil { + return nil, fmt.Errorf("read manifest: %w", err) + } defer f.Close() var backup Backup if err := toml.NewDecoder(f).Decode(&backup); err != nil { - return nil, fmt.Errorf("manifest: find: %w", err) + return nil, fmt.Errorf("read manifest: %w", err) } backup.ID = backupID diff --git a/internal/backup/locator_test.go b/internal/backup/locator_test.go index cca9ee8e8..1d6010554 100644 --- a/internal/backup/locator_test.go +++ b/internal/backup/locator_test.go @@ -610,3 +610,120 @@ custom_hooks_path = 'path/to/002.custom_hooks.tar' }) } } + +func TestManifestLocator_FindLatest(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + desc string + repo storage.Repository + setup func(t *testing.T, ctx context.Context, backupPath string) + expectedBackup *Backup + }{ + { + desc: "finds manifest", + repo: &gitalypb.Repository{ + StorageName: "default", + RelativePath: "vanity/repo.git", + }, + setup: func(t *testing.T, ctx context.Context, backupPath string) { + testhelper.WriteFiles(t, backupPath, map[string]any{ + "vanity/repo/LATEST": "abc123", + "vanity/repo/abc123/LATEST": "002", + "manifests/default/vanity/repo.git/+latest.toml": `object_format = 'sha1' + +[[steps]] +bundle_path = 'manifest-path/to/001.bundle' +ref_path = 'manifest-path/to/001.refs' +custom_hooks_path = 'manifest-path/to/001.custom_hooks.tar' + +[[steps]] +bundle_path = 'manifest-path/to/002.bundle' +ref_path = 'manifest-path/to/002.refs' +previous_ref_path = 'manifest-path/to/001.refs' +custom_hooks_path = 'manifest-path/to/002.custom_hooks.tar' +`, + }) + }, + expectedBackup: &Backup{ + ID: "+latest", + Repository: &gitalypb.Repository{ + StorageName: "default", + RelativePath: "vanity/repo.git", + }, + ObjectFormat: "sha1", + Steps: []Step{ + { + BundlePath: "manifest-path/to/001.bundle", + RefPath: "manifest-path/to/001.refs", + CustomHooksPath: "manifest-path/to/001.custom_hooks.tar", + }, + { + BundlePath: "manifest-path/to/002.bundle", + RefPath: "manifest-path/to/002.refs", + PreviousRefPath: "manifest-path/to/001.refs", + CustomHooksPath: "manifest-path/to/002.custom_hooks.tar", + }, + }, + }, + }, + { + desc: "fallback", + repo: &gitalypb.Repository{ + StorageName: "default", + RelativePath: "vanity/repo.git", + }, + setup: func(t *testing.T, ctx context.Context, backupPath string) { + testhelper.WriteFiles(t, backupPath, map[string]any{ + "vanity/repo/LATEST": "abc123", + "vanity/repo/abc123/LATEST": "002", + }) + }, + expectedBackup: &Backup{ + ID: "abc123", + Repository: &gitalypb.Repository{ + StorageName: "default", + RelativePath: "vanity/repo.git", + }, + ObjectFormat: "sha1", + Steps: []Step{ + { + BundlePath: "vanity/repo/abc123/001.bundle", + RefPath: "vanity/repo/abc123/001.refs", + CustomHooksPath: "vanity/repo/abc123/001.custom_hooks.tar", + }, + { + BundlePath: "vanity/repo/abc123/002.bundle", + RefPath: "vanity/repo/abc123/002.refs", + PreviousRefPath: "vanity/repo/abc123/001.refs", + CustomHooksPath: "vanity/repo/abc123/002.custom_hooks.tar", + }, + }, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + ctx := testhelper.Context(t) + backupPath := testhelper.TempDir(t) + + tc.setup(t, ctx, backupPath) + + sink := NewFilesystemSink(backupPath) + var l Locator = PointerLocator{ + Sink: sink, + } + l = ManifestLocator{ + Sink: sink, + Fallback: l, + } + + backup, err := l.FindLatest(ctx, tc.repo) + require.NoError(t, err) + + require.Equal(t, tc.expectedBackup, backup) + }) + } +} -- cgit v1.2.3