From 59af1f680333ab8240cfe0f51b0ac408818b2e95 Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Thu, 11 Jan 2024 11:27:59 -0500
Subject: Makefile: Add git-filter-repo for testing

We will shortly begin using git-filter-repo(1) to rewrite repository
history. This is a Python script that requires Python 3.5+, but it has
no external dependencies and can be downloaded as a single file.
We have added Python3 and installed git-filter-repo in Omnibus and CNG
GitLab, but still need it to be present for local testing and CI.

Add a target to clone filter-repo as a dependency and copy the script
into ${BUILD_DIR}/bin which is added to PATH for our tests.

We add a '.version' file for filter-repo to be consistent with the other
dependencies, but this isn't really required as we're not running a
build process in its source directory.
---
 Makefile | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index ba51a9acc..23f6f6206 100644
--- a/Makefile
+++ b/Makefile
@@ -173,6 +173,12 @@ ifdef GIT_FIPS_BUILD_OPTIONS
 	GIT_BUILD_OPTIONS += ${GIT_FIPS_BUILD_OPTIONS}
 endif
 
+# git-filter-repo target
+GIT_FILTER_REPO                      ?= ${BUILD_DIR}/bin/git-filter-repo
+GIT_FILTER_REPO_VERSION              ?= v2.38.0
+GIT_FILTER_REPO_REPO_URL             ?= https://github.com/newren/git-filter-repo
+GIT_FILTER_REPO_SOURCE_DIR           ?= ${DEPENDENCY_DIR}/git-filter-repo
+
 # These variables control test options and artifacts
 ## List of Go packages which shall be tested.
 ## Go packages to test when using the test-go target.
@@ -333,7 +339,7 @@ run_go_tests += \
 endif
 
 .PHONY: prepare-tests
-prepare-tests: ${GOTESTSUM} ${GITALY_PACKED_EXECUTABLES}
+prepare-tests: ${GOTESTSUM} ${GITALY_PACKED_EXECUTABLES} ${GIT_FILTER_REPO}
 	${Q}mkdir -p "$(dir ${TEST_JUNIT_REPORT})"
 
 .PHONY: prepare-debug
@@ -588,7 +594,8 @@ ${DEPENDENCY_DIR}/git-%.version: dependency-version | ${DEPENDENCY_DIR}
 	${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${GIT_VERSION} ${GIT_BUILD_OPTIONS}" ] || >$@ echo -n "${GIT_VERSION} ${GIT_BUILD_OPTIONS}"
 ${DEPENDENCY_DIR}/protoc.version: dependency-version | ${DEPENDENCY_DIR}
 	${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${PROTOC_VERSION} ${PROTOC_BUILD_OPTIONS}" ] || >$@ echo -n "${PROTOC_VERSION} ${PROTOC_BUILD_OPTIONS}"
-
+${DEPENDENCY_DIR}/git-filter-repo.version: dependency-version | ${DEPENDENCY_DIR}
+	${Q}[ x"$$(cat "$@" 2>/dev/null)" = x"${GIT_FILTER_REPO_VERSION}" ] || >$@ echo -n "${GIT_FILTER_REPO_VERSION}"
 # This target is responsible for checking out Git sources. In theory, we'd only
 # need to depend on the source directory. But given that the source directory
 # always changes when anything inside of it changes, like when we for example
@@ -638,6 +645,15 @@ ${PROTOC_GEN_GITALY_LINT}: proto | ${TOOLS_DIR}
 ${PROTOC_GEN_GITALY_PROTOLIST}: | ${TOOLS_DIR}
 	${Q}go build -o $@ ${SOURCE_DIR}/tools/protoc-gen-gitaly-protolist
 
+${GIT_FILTER_REPO}: ${DEPENDENCY_DIR}/git-filter-repo.version | ${BUILD_DIR}/bin
+	${Q}${GIT} -c init.defaultBranch=master init ${GIT_QUIET} "${GIT_FILTER_REPO_SOURCE_DIR}"
+	${Q}${GIT} -C "${GIT_FILTER_REPO_SOURCE_DIR}" config remote.origin.url ${GIT_FILTER_REPO_REPO_URL}
+	${Q}${GIT} -C "${GIT_FILTER_REPO_SOURCE_DIR}" config remote.origin.tagOpt --no-tags
+	${Q}${GIT} -C "${GIT_FILTER_REPO_SOURCE_DIR}" fetch --depth 1 ${GIT_QUIET} origin ${GIT_FILTER_REPO_VERSION}
+	${Q}${GIT} -C "${GIT_FILTER_REPO_SOURCE_DIR}" reset --hard
+	${Q}${GIT} -C "${GIT_FILTER_REPO_SOURCE_DIR}" checkout ${GIT_QUIET} --detach FETCH_HEAD
+	${Q}cp "${GIT_FILTER_REPO_SOURCE_DIR}/git-filter-repo" ${GIT_FILTER_REPO}
+
 # External tools
 ${TOOLS_DIR}/%: ${SOURCE_DIR}/tools/%/tool.go ${SOURCE_DIR}/tools/%/go.mod ${SOURCE_DIR}/tools/%/go.sum | ${TOOLS_DIR}
 	${Q}GOBIN=${TOOLS_DIR} go install -modfile ${SOURCE_DIR}/tools/$*/go.mod ${TOOL_PACKAGE}
-- 
cgit v1.2.3


From 41d2c45b64120f025404c71ca966f1930787d4ce Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Thu, 11 Jan 2024 11:36:32 -0500
Subject: ci: Install Python3 for FIPS jobs

The ubi image used for FIPS testing does not have Python3 installed by
default. Install it as part of the `before_script` steps.
---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b4935523..4b7b47489 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -319,6 +319,7 @@ test:fips:
     CACHE_PREFIX: ubi-${UBI_VERSION}
   before_script:
     - *test_before_script
+    - dnf install -y python3
     - test "$(cat /proc/sys/crypto/fips_enabled)" = "1" || (echo "System is not running in FIPS mode" && exit 1)
   parallel:
     matrix:
-- 
cgit v1.2.3


From 203cf24398c04a9d955bcf979a89eaf230603974 Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Sun, 14 Jan 2024 21:56:31 -0500
Subject: git: Add filter-repo subcommand

Add git-filter-repo(1) as a recognized subcommand.
---
 internal/git/command_description.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/internal/git/command_description.go b/internal/git/command_description.go
index 262402e3b..0432bcc38 100644
--- a/internal/git/command_description.go
+++ b/internal/git/command_description.go
@@ -131,6 +131,9 @@ var commandDescriptions = map[string]commandDescription{
 			}, fetchFsckConfiguration(ctx)...), packConfiguration(ctx)...)
 		},
 	},
+	"filter-repo": {
+		flags: scNoEndOfOptions,
+	},
 	"for-each-ref": {
 		flags: scNoRefUpdates,
 	},
-- 
cgit v1.2.3


From 7630f4b49669114fa8190e23f39978da902c6192 Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Thu, 11 Jan 2024 12:39:01 -0500
Subject: cleanup: Add RewriteHistory RPC

Historically we have advised users who need to rewrite history to do so
locally and force push their change to Gitlab. However, upcoming changes
may prevent a user from pushing in scenarios where they need to remove a
large blob from their repository's history.

To handle this scenario, we introduce a new `RewriteHistory` RPC which
will invoke git-filer-repo(1) on the target repository. filter-repo
has a large number of options, but we will support only two:

 --strip-blogs-with-ids
   Given a file containing a list of newline-delimited object ids,
   rewrite history to remove them from all commits.
 --replace-text
   Given a file of literals and patterns, replace all matching
   instances in history with '***REMOVED***'.

filter-repo works by fetching the repository contents via
git-fast-export(1), making the requested changes, and writing the
changes back via git-fast-import(1). As filter-repo uses the '--force'
flag[0] the repository must be made read-only before calling this RPC.

filter-repo is currently incompatible with SHA256 repositories.

[0] https://git-scm.com/docs/git-fast-import#_parallel_operation

Changelog: added
---
 internal/gitaly/service/cleanup/rewrite_history.go | 184 ++++++++
 .../gitaly/service/cleanup/rewrite_history_test.go | 518 +++++++++++++++++++++
 internal/gitaly/service/cleanup/server.go          |   3 +
 internal/gitaly/service/cleanup/testhelper_test.go |   2 +
 proto/cleanup.proto                                |  39 ++
 proto/go/gitalypb/cleanup.pb.go                    | 218 +++++++--
 proto/go/gitalypb/cleanup_grpc.pb.go               |  98 ++++
 7 files changed, 1031 insertions(+), 31 deletions(-)
 create mode 100644 internal/gitaly/service/cleanup/rewrite_history.go
 create mode 100644 internal/gitaly/service/cleanup/rewrite_history_test.go

diff --git a/internal/gitaly/service/cleanup/rewrite_history.go b/internal/gitaly/service/cleanup/rewrite_history.go
new file mode 100644
index 000000000..7bbdd258e
--- /dev/null
+++ b/internal/gitaly/service/cleanup/rewrite_history.go
@@ -0,0 +1,184 @@
+package cleanup
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/tempdir"
+	"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+)
+
+// RewriteHistory uses git-filter-repo(1) to remove specified blobs from commit history and
+// replace blobs to redact specified text patterns. This does not delete the removed blobs from
+// the object database, they must be garbage collected separately.
+func (s *server) RewriteHistory(server gitalypb.CleanupService_RewriteHistoryServer) error {
+	ctx := server.Context()
+
+	request, err := server.Recv()
+	if err != nil {
+		return fmt.Errorf("receiving initial request: %w", err)
+	}
+
+	repoProto := request.GetRepository()
+	if err := s.locator.ValidateRepository(repoProto); err != nil {
+		return structerr.NewInvalidArgument("%w", err)
+	}
+
+	repo := s.localrepo(repoProto)
+
+	objectHash, err := repo.ObjectHash(ctx)
+	if err != nil {
+		return fmt.Errorf("detecting object hash: %w", err)
+	}
+
+	if objectHash.Format == "sha256" {
+		return structerr.NewInvalidArgument("git-filter-repo does not support repositories using the SHA256 object format")
+	}
+
+	// Unset repository so that we can validate that repository is not sent on subsequent requests.
+	request.Repository = nil
+
+	blobsToRemove := make([]string, 0, len(request.GetBlobs()))
+	redactions := make([][]byte, 0, len(request.GetRedactions()))
+
+	for {
+		if request.GetRepository() != nil {
+			return structerr.NewInvalidArgument("subsequent requests must not contain repository")
+		}
+
+		if len(request.GetBlobs()) == 0 && len(request.GetRedactions()) == 0 {
+			return structerr.NewInvalidArgument("no object IDs or text replacements specified")
+		}
+
+		for _, oid := range request.GetBlobs() {
+			if err := objectHash.ValidateHex(oid); err != nil {
+				return structerr.NewInvalidArgument("validating object ID: %w", err).WithMetadata("oid", oid)
+			}
+			blobsToRemove = append(blobsToRemove, oid)
+		}
+
+		for _, pattern := range request.GetRedactions() {
+			if strings.Contains(string(pattern), "\n") {
+				// We deliberately do not log the invalid pattern as this is
+				// likely to contain sensitive information.
+				return structerr.NewInvalidArgument("redaction pattern contains newline")
+			}
+			redactions = append(redactions, pattern)
+		}
+
+		request, err = server.Recv()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return fmt.Errorf("receiving next request: %w", err)
+		}
+	}
+
+	if err := s.runFilterRepo(ctx, repo, repoProto, blobsToRemove, redactions); err != nil {
+		return fmt.Errorf("rewriting repository history: %w", err)
+	}
+
+	if err := server.SendAndClose(&gitalypb.RewriteHistoryResponse{}); err != nil {
+		return fmt.Errorf("sending RewriteHistoryResponse: %w", err)
+	}
+
+	return nil
+}
+
+func (s *server) runFilterRepo(
+	ctx context.Context,
+	repo *localrepo.Repo,
+	repoProto *gitalypb.Repository,
+	blobsToRemove []string,
+	redactions [][]byte,
+) error {
+	// Place argument files in a tempdir so that cleanup is handled automatically.
+	tmpDir, err := tempdir.New(ctx, repo.GetStorageName(), s.logger, s.locator)
+	if err != nil {
+		return fmt.Errorf("create tempdir: %w", err)
+	}
+
+	flags := make([]git.Option, 0, 2)
+
+	if len(blobsToRemove) > 0 {
+		blobPath, err := writeArgFile("strip-blobs", tmpDir.Path(), []byte(strings.Join(blobsToRemove, "\n")))
+		if err != nil {
+			return err
+		}
+
+		flags = append(flags, git.Flag{Name: "--strip-blobs-with-ids=" + blobPath})
+	}
+
+	if len(redactions) > 0 {
+		replacePath, err := writeArgFile("replace-text", tmpDir.Path(), bytes.Join(redactions, []byte("\n")))
+		if err != nil {
+			return err
+		}
+
+		flags = append(flags, git.Flag{Name: "--replace-text=" + replacePath})
+	}
+
+	var stdout, stderr strings.Builder
+	if err := repo.ExecAndWait(ctx,
+		git.Command{
+			Name: "filter-repo",
+			Flags: append([]git.Option{
+				// Prevent automatic cleanup tasks like deleting 'origin' and running git-gc(1).
+				git.Flag{Name: "--partial"},
+				// Bypass check that repository is not a fresh clone.
+				git.Flag{Name: "--force"},
+				// filter-repo will by default create 'replace' refs for refs it rewrites, but Gitaly
+				// disables this feature. This option will update any existing user-created replace refs,
+				// while preventing the creation of new ones.
+				git.Flag{Name: "--replace-refs=update-no-add"},
+				// Pass '--quiet' to child git processes.
+				git.Flag{Name: "--quiet"},
+			}, flags...),
+		},
+		git.WithRefTxHook(repo),
+		git.WithStdout(&stdout),
+		git.WithStderr(&stderr),
+	); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return structerr.New("git-filter-repo failed with exit code %d", exitErr.ExitCode()).WithMetadataItems(
+				structerr.MetadataItem{Key: "stdout", Value: stdout.String()},
+				structerr.MetadataItem{Key: "stderr", Value: stderr.String()},
+			)
+		}
+		return fmt.Errorf("running git-filter-repo: %w", err)
+	}
+
+	return nil
+}
+
+func writeArgFile(name string, dir string, input []byte) (string, error) {
+	f, err := os.CreateTemp(dir, name)
+	if err != nil {
+		return "", fmt.Errorf("creating %q file: %w", name, err)
+	}
+
+	path := f.Name()
+
+	_, err = f.Write(input)
+	if err != nil {
+		return "", fmt.Errorf("writing %q file: %w", name, err)
+	}
+
+	if err := f.Close(); err != nil {
+		return "", fmt.Errorf("closing %q file: %w", name, err)
+	}
+
+	return path, nil
+}
diff --git a/internal/gitaly/service/cleanup/rewrite_history_test.go b/internal/gitaly/service/cleanup/rewrite_history_test.go
new file mode 100644
index 000000000..0d2c4363c
--- /dev/null
+++ b/internal/gitaly/service/cleanup/rewrite_history_test.go
@@ -0,0 +1,518 @@
+package cleanup
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/git/gittest"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/structerr"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
+	"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+)
+
+func TestRewriteHistory(t *testing.T) {
+	t.Parallel()
+	gittest.SkipWithSHA256(t)
+
+	ctx := testhelper.Context(t)
+	cfg := testcfg.Build(t)
+	testcfg.BuildGitalyHooks(t, cfg)
+	cfg.SocketPath = runCleanupServiceServer(t, cfg)
+
+	client, conn := newCleanupServiceClient(t, cfg.SocketPath)
+	t.Cleanup(func() { conn.Close() })
+
+	addUnmodifiedRefs := func(repoPath string) []git.Reference {
+		unmodifiedCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("unmodified"), gittest.WithTreeEntries(
+			gittest.TreeEntry{Path: "unmodified-file", Mode: "100644", Content: "can't touch this"},
+		))
+		unmodifiedTag := gittest.WriteTag(t, cfg, repoPath, "unmodified-tag", unmodifiedCommit.Revision(), gittest.WriteTagConfig{
+			Message: "annotated",
+		})
+		keepRef := git.ReferenceName("refs/keeparound/" + unmodifiedCommit.String())
+		gittest.WriteRef(t, cfg, repoPath, keepRef, unmodifiedCommit)
+
+		return []git.Reference{
+			{
+				Name:   "refs/heads/unmodified",
+				Target: unmodifiedCommit.String(),
+			},
+			{
+				Name:   "refs/tags/unmodified-tag",
+				Target: unmodifiedTag.String(),
+			},
+			{
+				Name:   keepRef,
+				Target: unmodifiedCommit.String(),
+			},
+		}
+	}
+
+	type setupData struct {
+		requests         []*gitalypb.RewriteHistoryRequest
+		repoPath         string
+		expectedRefs     []git.Reference
+		expectedErr      error
+		expectedResponse *gitalypb.RewriteHistoryResponse
+	}
+
+	for _, tc := range []struct {
+		desc  string
+		setup func(t *testing.T) setupData
+	}{
+		{
+			desc: "no requests",
+			setup: func(t *testing.T) setupData {
+				return setupData{
+					requests: nil,
+					expectedErr: testhelper.GitalyOrPraefect(
+						structerr.NewInternal("receiving initial request: EOF"),
+						structerr.NewInternal("EOF"),
+					),
+				}
+			},
+		},
+		{
+			desc: "missing repository",
+			setup: func(t *testing.T) setupData {
+				repoPath := gittest.NewRepositoryName(t)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: &gitalypb.Repository{
+								StorageName:  cfg.Storages[0].Name,
+								RelativePath: repoPath,
+							},
+							Redactions: [][]byte{[]byte("hunter2")},
+						},
+					},
+					expectedErr: testhelper.ToInterceptedMetadata(
+						structerr.New("%w", storage.NewRepositoryNotFoundError(cfg.Storages[0].Name, repoPath)),
+					),
+				}
+			},
+		},
+		{
+			desc: "repository in subsequent request",
+			setup: func(t *testing.T) setupData {
+				repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Redactions: [][]byte{[]byte("hunter2")},
+						},
+						{
+							Repository: repo,
+							Redactions: [][]byte{[]byte("secretpassword")},
+						},
+					},
+					expectedErr: structerr.NewInvalidArgument("subsequent requests must not contain repository"),
+				}
+			},
+		},
+		{
+			desc: "empty request",
+			setup: func(t *testing.T) setupData {
+				repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+						},
+					},
+					expectedErr: structerr.NewInvalidArgument("no object IDs or text replacements specified"),
+				}
+			},
+		},
+		{
+			desc: "remove invalid oid",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{"invalid oid"},
+						},
+					},
+					repoPath:     repoPath,
+					expectedRefs: unmodifiedRefs,
+					expectedErr: testhelper.WithInterceptedMetadata(
+						structerr.NewInvalidArgument("validating object ID: invalid object ID: \"invalid oid\", expected length %v, got 11", gittest.DefaultObjectHash.EncodedLen()),
+						"oid", "invalid oid",
+					),
+				}
+			},
+		},
+		{
+			desc: "redaction pattern with newline",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Redactions: [][]byte{[]byte("hunter\n2")},
+						},
+					},
+					repoPath:     repoPath,
+					expectedRefs: unmodifiedRefs,
+					expectedErr:  structerr.NewInvalidArgument("redaction pattern contains newline"),
+				}
+			},
+		},
+		{
+			desc: "redaction pattern with escaped newline",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Redactions: [][]byte{[]byte("hunter\\n2")},
+						},
+					},
+					repoPath:         repoPath,
+					expectedRefs:     unmodifiedRefs,
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "remove non-existent oid",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{strings.Repeat("a", gittest.DefaultObjectHash.EncodedLen())},
+						},
+					},
+					repoPath:         repoPath,
+					expectedRefs:     unmodifiedRefs,
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "remove blobs",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				blobToRemove := gittest.WriteBlob(t, cfg, repoPath, []byte("big blob"))
+				_ = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: blobToRemove},
+					gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "foobar"},
+				))
+				updatedCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "foobar"},
+				))
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{blobToRemove.String()},
+						},
+					},
+					repoPath: repoPath,
+					expectedRefs: append(unmodifiedRefs, []git.Reference{
+						{
+							Name:   "refs/heads/main",
+							Target: updatedCommit.String(),
+						},
+					}...),
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "redact blobs",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				_ = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "redact-me", Mode: "100644", Content: "my password is hunter2"},
+				))
+				updatedCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "redact-me", Mode: "100644", Content: "my password is ***REMOVED***"},
+				))
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Redactions: [][]byte{
+								[]byte("hunter2"),
+							},
+						},
+					},
+					repoPath: repoPath,
+					expectedRefs: append(unmodifiedRefs, []git.Reference{
+						{
+							Name:   "refs/heads/main",
+							Target: updatedCommit.String(),
+						},
+					}...),
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "multiple requests",
+			setup: func(t *testing.T) setupData {
+				repoProto, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				blobToRemove := gittest.WriteBlob(t, cfg, repoPath, []byte("big blob"))
+				_ = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: blobToRemove},
+					gittest.TreeEntry{Path: "redact-me", Mode: "100644", Content: "my password is hunter2"},
+				))
+				updatedCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "redact-me", Mode: "100644", Content: "my password is ***REMOVED***"},
+				))
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repoProto,
+							Blobs:      []string{blobToRemove.String()},
+						},
+						{
+							Redactions: [][]byte{[]byte("hunter2")},
+						},
+					},
+					repoPath: repoPath,
+					expectedRefs: append(unmodifiedRefs, []git.Reference{
+						{
+							Name:   "refs/heads/main",
+							Target: updatedCommit.String(),
+						},
+					}...),
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "empty branch deleted",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				blobToRemove := gittest.WriteBlob(t, cfg, repoPath, []byte("big blob"))
+				_ = gittest.WriteCommit(t, cfg, repoPath, gittest.WithBranch("main"), gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: blobToRemove},
+				))
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{blobToRemove.String()},
+						},
+					},
+					repoPath:         repoPath,
+					expectedRefs:     unmodifiedRefs,
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "tag updated",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				blobToRemove := gittest.WriteBlob(t, cfg, repoPath, []byte("big blob"))
+				commit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: blobToRemove},
+					gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "foobar"},
+				))
+				_ = gittest.WriteTag(t, cfg, repoPath, "updated-tag", commit.Revision())
+
+				updatedCommit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "foobar"},
+				))
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{blobToRemove.String()},
+						},
+					},
+					repoPath: repoPath,
+					expectedRefs: append(unmodifiedRefs, []git.Reference{
+						{
+							Name:   "refs/tags/updated-tag",
+							Target: updatedCommit.String(),
+						},
+					}...),
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "empty tag deleted",
+			setup: func(t *testing.T) setupData {
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				blobToRemove := gittest.WriteBlob(t, cfg, repoPath, []byte("big blob"))
+				commit := gittest.WriteCommit(t, cfg, repoPath, gittest.WithTreeEntries(
+					gittest.TreeEntry{Path: "a-file", Mode: "100644", OID: blobToRemove},
+				))
+				_ = gittest.WriteTag(t, cfg, repoPath, "deleted-tag", commit.Revision(), gittest.WriteTagConfig{
+					Message: "annotated",
+				})
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{blobToRemove.String()},
+						},
+					},
+					repoPath:         repoPath,
+					expectedRefs:     unmodifiedRefs,
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+		{
+			desc: "remove blob in pool repo",
+			setup: func(t *testing.T) setupData {
+				testhelper.SkipWithWAL(t, `
+Object pools are not yet supported with transaction management.`)
+
+				repo, repoPath := gittest.CreateRepository(t, ctx, cfg)
+				unmodifiedRefs := addUnmodifiedRefs(repoPath)
+
+				poolBlob := gittest.WriteBlob(t, cfg, repoPath, []byte("pool blob"))
+				parentCommit := gittest.WriteCommit(t, cfg, repoPath,
+					gittest.WithTreeEntries(
+						gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: poolBlob},
+						gittest.TreeEntry{Path: "other-pool-file", Mode: "100644", Content: "pool blob to retain"},
+					),
+					gittest.WithBranch("main"),
+				)
+
+				gittest.CreateObjectPool(t, ctx, cfg, repo, gittest.CreateObjectPoolConfig{
+					LinkRepositoryToObjectPool: true,
+				})
+
+				_ = gittest.WriteCommit(t, cfg, repoPath,
+					gittest.WithTreeEntries(
+						gittest.TreeEntry{Path: "remove-me", Mode: "100644", OID: poolBlob},
+						gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "local blob"},
+					),
+					gittest.WithParents(parentCommit),
+					gittest.WithBranch("main"),
+				)
+
+				updatedParent := gittest.WriteCommit(t, cfg, repoPath,
+					gittest.WithTreeEntries(
+						gittest.TreeEntry{Path: "other-pool-file", Mode: "100644", Content: "pool blob to retain"},
+					),
+				)
+				updatedCommit := gittest.WriteCommit(t, cfg, repoPath,
+					gittest.WithTreeEntries(
+						gittest.TreeEntry{Path: "a-file", Mode: "100644", Content: "local blob"},
+					),
+					gittest.WithParents(updatedParent),
+				)
+
+				return setupData{
+					requests: []*gitalypb.RewriteHistoryRequest{
+						{
+							Repository: repo,
+							Blobs:      []string{poolBlob.String()},
+						},
+					},
+					repoPath: repoPath,
+					expectedRefs: append(unmodifiedRefs, []git.Reference{
+						{
+							Name:   "refs/heads/main",
+							Target: updatedCommit.String(),
+						},
+					}...),
+					expectedResponse: &gitalypb.RewriteHistoryResponse{},
+				}
+			},
+		},
+	} {
+		tc := tc
+
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+
+			setup := tc.setup(t)
+
+			stream, err := client.RewriteHistory(ctx)
+			require.NoError(t, err)
+
+			for _, request := range setup.requests {
+				require.NoError(t, stream.Send(request))
+			}
+
+			response, err := stream.CloseAndRecv()
+			testhelper.RequireGrpcError(t, setup.expectedErr, err)
+			testhelper.ProtoEqual(t, setup.expectedResponse, response)
+
+			if setup.repoPath != "" {
+				refs := gittest.GetReferences(t, cfg, setup.repoPath)
+				require.ElementsMatch(t, refs, setup.expectedRefs)
+			}
+		})
+	}
+}
+
+func TestRewriteHistory_SHA256(t *testing.T) {
+	t.Parallel()
+
+	if !gittest.ObjectHashIsSHA256() {
+		t.Skip("test is not compatible with SHA1")
+	}
+
+	ctx := testhelper.Context(t)
+	cfg := testcfg.Build(t)
+	testcfg.BuildGitalyHooks(t, cfg)
+	cfg.SocketPath = runCleanupServiceServer(t, cfg)
+
+	client, conn := newCleanupServiceClient(t, cfg.SocketPath)
+	t.Cleanup(func() { conn.Close() })
+
+	repo, _ := gittest.CreateRepository(t, ctx, cfg)
+
+	stream, err := client.RewriteHistory(ctx)
+	require.NoError(t, err)
+
+	require.NoError(t, stream.Send(&gitalypb.RewriteHistoryRequest{
+		Repository: repo,
+		Redactions: [][]byte{[]byte("hunter2")},
+	}))
+
+	response, err := stream.CloseAndRecv()
+	require.Nil(t, response)
+	testhelper.RequireGrpcError(t, structerr.NewInvalidArgument("git-filter-repo does not support repositories using the SHA256 object format"), err)
+}
diff --git a/internal/gitaly/service/cleanup/server.go b/internal/gitaly/service/cleanup/server.go
index 3fd2d06f6..33058e5a6 100644
--- a/internal/gitaly/service/cleanup/server.go
+++ b/internal/gitaly/service/cleanup/server.go
@@ -6,6 +6,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/storage"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/transaction"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/log"
 	"gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
 )
@@ -16,6 +17,7 @@ type server struct {
 	locator       storage.Locator
 	gitCmdFactory git.CommandFactory
 	catfileCache  catfile.Cache
+	txManager     transaction.Manager
 }
 
 // NewServer creates a new instance of a grpc CleanupServer
@@ -25,6 +27,7 @@ func NewServer(deps *service.Dependencies) gitalypb.CleanupServiceServer {
 		locator:       deps.GetLocator(),
 		gitCmdFactory: deps.GetGitCmdFactory(),
 		catfileCache:  deps.GetCatfileCache(),
+		txManager:     deps.GetTxManager(),
 	}
 }
 
diff --git a/internal/gitaly/service/cleanup/testhelper_test.go b/internal/gitaly/service/cleanup/testhelper_test.go
index 3f488eba8..8dd085f22 100644
--- a/internal/gitaly/service/cleanup/testhelper_test.go
+++ b/internal/gitaly/service/cleanup/testhelper_test.go
@@ -8,6 +8,7 @@ import (
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/config"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service"
 	hookservice "gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/hook"
+	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/objectpool"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/gitaly/service/repository"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper"
 	"gitlab.com/gitlab-org/gitaly/v16/internal/testhelper/testcfg"
@@ -38,6 +39,7 @@ func runCleanupServiceServer(t *testing.T, cfg config.Cfg) string {
 		gitalypb.RegisterCleanupServiceServer(srv, NewServer(deps))
 		gitalypb.RegisterHookServiceServer(srv, hookservice.NewServer(deps))
 		gitalypb.RegisterRepositoryServiceServer(srv, repository.NewServer(deps))
+		gitalypb.RegisterObjectPoolServiceServer(srv, objectpool.NewServer(deps))
 	})
 }
 
diff --git a/proto/cleanup.proto b/proto/cleanup.proto
index 511bbff46..77cf6c6a3 100644
--- a/proto/cleanup.proto
+++ b/proto/cleanup.proto
@@ -17,6 +17,26 @@ service CleanupService {
     };
   }
 
+  // RewriteHistory redacts targeted strings and deletes requested blobs in a
+  // repository and updates all references to point to the rewritten commit
+  // history. This is useful for removing inadvertently pushed secrets from your
+  // repository and purging large blobs. This is a dangerous operation.
+  //
+  // The following known error conditions may happen:
+  //
+  // - `InvalidArgument` in the following situations:
+  //    - The provided repository can't be validated.
+  //    - The repository field is set on any request other than the initial one.
+  //    - Any request, including the initial one, does not contain either blobs to
+  //      remove or redaction patterns to redact.
+  //    - A blob object ID is invalid.
+  //    - A redaction pattern contains a newline character.
+  rpc RewriteHistory(stream RewriteHistoryRequest) returns (RewriteHistoryResponse) {
+    option (op_type) = {
+      op : MUTATOR
+    };
+  }
+
 }
 
 // ApplyBfgObjectMapStreamRequest ...
@@ -47,3 +67,22 @@ message ApplyBfgObjectMapStreamResponse {
   // entries ...
   repeated Entry entries = 1;
 }
+
+// RewriteHistoryRequest is a request for the RewriteHistory RPC.
+// Each request must contain blobs, redactions, or both.
+message RewriteHistoryRequest {
+  // repository is the repository that shall be rewritten.
+  // Must be sent on the first request only.
+  Repository repository = 1 [(target_repository)=true];
+  // blobs is the list of blob object ids that will be removed from history.
+  repeated string blobs = 2;
+  // redactions is the list of literals or patterns that will be replaced
+  // with "***REMOVED***". Items cannot contain newline characters.
+  // See https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html
+  // for a full explanation of what patterns are supported.
+  repeated bytes redactions = 3;
+}
+
+// RewriteHistoryResponse a response for the RewriteHistory RPC.
+message RewriteHistoryResponse {
+}
diff --git a/proto/go/gitalypb/cleanup.pb.go b/proto/go/gitalypb/cleanup.pb.go
index b54d5d1e2..028f0f8cc 100644
--- a/proto/go/gitalypb/cleanup.pb.go
+++ b/proto/go/gitalypb/cleanup.pb.go
@@ -130,6 +130,117 @@ func (x *ApplyBfgObjectMapStreamResponse) GetEntries() []*ApplyBfgObjectMapStrea
 	return nil
 }
 
+// RewriteHistoryRequest is a request for the RewriteHistory RPC.
+// Each request must contain blobs, redactions, or both.
+type RewriteHistoryRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// repository is the repository that shall be rewritten.
+	// Must be sent on the first request only.
+	Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"`
+	// blobs is the list of blob object ids that will be removed from history.
+	Blobs []string `protobuf:"bytes,2,rep,name=blobs,proto3" json:"blobs,omitempty"`
+	// redactions is the list of literals or patterns that will be replaced
+	// with "***REMOVED***". Items cannot contain newline characters.
+	// See https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html
+	// for a full explanation of what patterns are supported.
+	Redactions [][]byte `protobuf:"bytes,3,rep,name=redactions,proto3" json:"redactions,omitempty"`
+}
+
+func (x *RewriteHistoryRequest) Reset() {
+	*x = RewriteHistoryRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_cleanup_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RewriteHistoryRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RewriteHistoryRequest) ProtoMessage() {}
+
+func (x *RewriteHistoryRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_cleanup_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RewriteHistoryRequest.ProtoReflect.Descriptor instead.
+func (*RewriteHistoryRequest) Descriptor() ([]byte, []int) {
+	return file_cleanup_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *RewriteHistoryRequest) GetRepository() *Repository {
+	if x != nil {
+		return x.Repository
+	}
+	return nil
+}
+
+func (x *RewriteHistoryRequest) GetBlobs() []string {
+	if x != nil {
+		return x.Blobs
+	}
+	return nil
+}
+
+func (x *RewriteHistoryRequest) GetRedactions() [][]byte {
+	if x != nil {
+		return x.Redactions
+	}
+	return nil
+}
+
+// RewriteHistoryResponse a response for the RewriteHistory RPC.
+type RewriteHistoryResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *RewriteHistoryResponse) Reset() {
+	*x = RewriteHistoryResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_cleanup_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RewriteHistoryResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RewriteHistoryResponse) ProtoMessage() {}
+
+func (x *RewriteHistoryResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_cleanup_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RewriteHistoryResponse.ProtoReflect.Descriptor instead.
+func (*RewriteHistoryResponse) Descriptor() ([]byte, []int) {
+	return file_cleanup_proto_rawDescGZIP(), []int{3}
+}
+
 // Entry refers to each parsed entry in the request's object map so the client
 // can take action.
 type ApplyBfgObjectMapStreamResponse_Entry struct {
@@ -148,7 +259,7 @@ type ApplyBfgObjectMapStreamResponse_Entry struct {
 func (x *ApplyBfgObjectMapStreamResponse_Entry) Reset() {
 	*x = ApplyBfgObjectMapStreamResponse_Entry{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_cleanup_proto_msgTypes[2]
+		mi := &file_cleanup_proto_msgTypes[4]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -161,7 +272,7 @@ func (x *ApplyBfgObjectMapStreamResponse_Entry) String() string {
 func (*ApplyBfgObjectMapStreamResponse_Entry) ProtoMessage() {}
 
 func (x *ApplyBfgObjectMapStreamResponse_Entry) ProtoReflect() protoreflect.Message {
-	mi := &file_cleanup_proto_msgTypes[2]
+	mi := &file_cleanup_proto_msgTypes[4]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -224,20 +335,36 @@ var file_cleanup_proto_rawDesc = []byte{
 	0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x6f, 0x6c,
 	0x64, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x6c, 0x64,
 	0x4f, 0x69, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x65, 0x77, 0x5f, 0x6f, 0x69, 0x64, 0x18, 0x03,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x4f, 0x69, 0x64, 0x32, 0x88, 0x01, 0x0a,
-	0x0e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12,
-	0x76, 0x0a, 0x17, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x42, 0x66, 0x67, 0x4f, 0x62, 0x6a, 0x65, 0x63,
-	0x74, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x26, 0x2e, 0x67, 0x69, 0x74,
-	0x61, 0x6c, 0x79, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x42, 0x66, 0x67, 0x4f, 0x62, 0x6a, 0x65,
-	0x63, 0x74, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x41, 0x70, 0x70, 0x6c,
-	0x79, 0x42, 0x66, 0x67, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72,
-	0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28,
-	0x02, 0x08, 0x01, 0x28, 0x01, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61,
-	0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67,
-	0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74,
-	0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x4f, 0x69, 0x64, 0x22, 0x87, 0x01, 0x0a,
+	0x15, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69,
+	0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x69, 0x74,
+	0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x42, 0x04,
+	0x98, 0xc6, 0x2c, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79,
+	0x12, 0x14, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x05, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x64, 0x61, 0x63, 0x74,
+	0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x65, 0x64, 0x61,
+	0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x18, 0x0a, 0x16, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74,
+	0x65, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x32, 0xe3, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76,
+	0x69, 0x63, 0x65, 0x12, 0x76, 0x0a, 0x17, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x42, 0x66, 0x67, 0x4f,
+	0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x26,
+	0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x42, 0x66, 0x67,
+	0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e,
+	0x41, 0x70, 0x70, 0x6c, 0x79, 0x42, 0x66, 0x67, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4d, 0x61,
+	0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
+	0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x01, 0x28, 0x01, 0x30, 0x01, 0x12, 0x59, 0x0a, 0x0e, 0x52,
+	0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1d, 0x2e,
+	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x48, 0x69,
+	0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67,
+	0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x77, 0x72, 0x69, 0x74, 0x65, 0x48, 0x69, 0x73,
+	0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97,
+	0x28, 0x02, 0x08, 0x01, 0x28, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f,
+	0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x36, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -252,25 +379,30 @@ func file_cleanup_proto_rawDescGZIP() []byte {
 	return file_cleanup_proto_rawDescData
 }
 
-var file_cleanup_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_cleanup_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
 var file_cleanup_proto_goTypes = []interface{}{
 	(*ApplyBfgObjectMapStreamRequest)(nil),        // 0: gitaly.ApplyBfgObjectMapStreamRequest
 	(*ApplyBfgObjectMapStreamResponse)(nil),       // 1: gitaly.ApplyBfgObjectMapStreamResponse
-	(*ApplyBfgObjectMapStreamResponse_Entry)(nil), // 2: gitaly.ApplyBfgObjectMapStreamResponse.Entry
-	(*Repository)(nil),                            // 3: gitaly.Repository
-	(ObjectType)(0),                               // 4: gitaly.ObjectType
+	(*RewriteHistoryRequest)(nil),                 // 2: gitaly.RewriteHistoryRequest
+	(*RewriteHistoryResponse)(nil),                // 3: gitaly.RewriteHistoryResponse
+	(*ApplyBfgObjectMapStreamResponse_Entry)(nil), // 4: gitaly.ApplyBfgObjectMapStreamResponse.Entry
+	(*Repository)(nil),                            // 5: gitaly.Repository
+	(ObjectType)(0),                               // 6: gitaly.ObjectType
 }
 var file_cleanup_proto_depIdxs = []int32{
-	3, // 0: gitaly.ApplyBfgObjectMapStreamRequest.repository:type_name -> gitaly.Repository
-	2, // 1: gitaly.ApplyBfgObjectMapStreamResponse.entries:type_name -> gitaly.ApplyBfgObjectMapStreamResponse.Entry
-	4, // 2: gitaly.ApplyBfgObjectMapStreamResponse.Entry.type:type_name -> gitaly.ObjectType
-	0, // 3: gitaly.CleanupService.ApplyBfgObjectMapStream:input_type -> gitaly.ApplyBfgObjectMapStreamRequest
-	1, // 4: gitaly.CleanupService.ApplyBfgObjectMapStream:output_type -> gitaly.ApplyBfgObjectMapStreamResponse
-	4, // [4:5] is the sub-list for method output_type
-	3, // [3:4] is the sub-list for method input_type
-	3, // [3:3] is the sub-list for extension type_name
-	3, // [3:3] is the sub-list for extension extendee
-	0, // [0:3] is the sub-list for field type_name
+	5, // 0: gitaly.ApplyBfgObjectMapStreamRequest.repository:type_name -> gitaly.Repository
+	4, // 1: gitaly.ApplyBfgObjectMapStreamResponse.entries:type_name -> gitaly.ApplyBfgObjectMapStreamResponse.Entry
+	5, // 2: gitaly.RewriteHistoryRequest.repository:type_name -> gitaly.Repository
+	6, // 3: gitaly.ApplyBfgObjectMapStreamResponse.Entry.type:type_name -> gitaly.ObjectType
+	0, // 4: gitaly.CleanupService.ApplyBfgObjectMapStream:input_type -> gitaly.ApplyBfgObjectMapStreamRequest
+	2, // 5: gitaly.CleanupService.RewriteHistory:input_type -> gitaly.RewriteHistoryRequest
+	1, // 6: gitaly.CleanupService.ApplyBfgObjectMapStream:output_type -> gitaly.ApplyBfgObjectMapStreamResponse
+	3, // 7: gitaly.CleanupService.RewriteHistory:output_type -> gitaly.RewriteHistoryResponse
+	6, // [6:8] is the sub-list for method output_type
+	4, // [4:6] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
 }
 
 func init() { file_cleanup_proto_init() }
@@ -306,6 +438,30 @@ func file_cleanup_proto_init() {
 			}
 		}
 		file_cleanup_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RewriteHistoryRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_cleanup_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RewriteHistoryResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_cleanup_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*ApplyBfgObjectMapStreamResponse_Entry); i {
 			case 0:
 				return &v.state
@@ -324,7 +480,7 @@ func file_cleanup_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_cleanup_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   3,
+			NumMessages:   5,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
diff --git a/proto/go/gitalypb/cleanup_grpc.pb.go b/proto/go/gitalypb/cleanup_grpc.pb.go
index 4d42e293e..a9b1d2c62 100644
--- a/proto/go/gitalypb/cleanup_grpc.pb.go
+++ b/proto/go/gitalypb/cleanup_grpc.pb.go
@@ -24,6 +24,21 @@ const _ = grpc.SupportPackageIsVersion7
 type CleanupServiceClient interface {
 	// ApplyBfgObjectMapStream ...
 	ApplyBfgObjectMapStream(ctx context.Context, opts ...grpc.CallOption) (CleanupService_ApplyBfgObjectMapStreamClient, error)
+	// RewriteHistory redacts targeted strings and deletes requested blobs in a
+	// repository and updates all references to point to the rewritten commit
+	// history. This is useful for removing inadvertently pushed secrets from your
+	// repository and purging large blobs. This is a dangerous operation.
+	//
+	// The following known error conditions may happen:
+	//
+	// - `InvalidArgument` in the following situations:
+	//   - The provided repository can't be validated.
+	//   - The repository field is set on any request other than the initial one.
+	//   - Any request, including the initial one, does not contain either blobs to
+	//     remove or redaction patterns to redact.
+	//   - A blob object ID is invalid.
+	//   - A redaction pattern contains a newline character.
+	RewriteHistory(ctx context.Context, opts ...grpc.CallOption) (CleanupService_RewriteHistoryClient, error)
 }
 
 type cleanupServiceClient struct {
@@ -65,12 +80,61 @@ func (x *cleanupServiceApplyBfgObjectMapStreamClient) Recv() (*ApplyBfgObjectMap
 	return m, nil
 }
 
+func (c *cleanupServiceClient) RewriteHistory(ctx context.Context, opts ...grpc.CallOption) (CleanupService_RewriteHistoryClient, error) {
+	stream, err := c.cc.NewStream(ctx, &CleanupService_ServiceDesc.Streams[1], "/gitaly.CleanupService/RewriteHistory", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &cleanupServiceRewriteHistoryClient{stream}
+	return x, nil
+}
+
+type CleanupService_RewriteHistoryClient interface {
+	Send(*RewriteHistoryRequest) error
+	CloseAndRecv() (*RewriteHistoryResponse, error)
+	grpc.ClientStream
+}
+
+type cleanupServiceRewriteHistoryClient struct {
+	grpc.ClientStream
+}
+
+func (x *cleanupServiceRewriteHistoryClient) Send(m *RewriteHistoryRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *cleanupServiceRewriteHistoryClient) CloseAndRecv() (*RewriteHistoryResponse, error) {
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	m := new(RewriteHistoryResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // CleanupServiceServer is the server API for CleanupService service.
 // All implementations must embed UnimplementedCleanupServiceServer
 // for forward compatibility
 type CleanupServiceServer interface {
 	// ApplyBfgObjectMapStream ...
 	ApplyBfgObjectMapStream(CleanupService_ApplyBfgObjectMapStreamServer) error
+	// RewriteHistory redacts targeted strings and deletes requested blobs in a
+	// repository and updates all references to point to the rewritten commit
+	// history. This is useful for removing inadvertently pushed secrets from your
+	// repository and purging large blobs. This is a dangerous operation.
+	//
+	// The following known error conditions may happen:
+	//
+	// - `InvalidArgument` in the following situations:
+	//   - The provided repository can't be validated.
+	//   - The repository field is set on any request other than the initial one.
+	//   - Any request, including the initial one, does not contain either blobs to
+	//     remove or redaction patterns to redact.
+	//   - A blob object ID is invalid.
+	//   - A redaction pattern contains a newline character.
+	RewriteHistory(CleanupService_RewriteHistoryServer) error
 	mustEmbedUnimplementedCleanupServiceServer()
 }
 
@@ -81,6 +145,9 @@ type UnimplementedCleanupServiceServer struct {
 func (UnimplementedCleanupServiceServer) ApplyBfgObjectMapStream(CleanupService_ApplyBfgObjectMapStreamServer) error {
 	return status.Errorf(codes.Unimplemented, "method ApplyBfgObjectMapStream not implemented")
 }
+func (UnimplementedCleanupServiceServer) RewriteHistory(CleanupService_RewriteHistoryServer) error {
+	return status.Errorf(codes.Unimplemented, "method RewriteHistory not implemented")
+}
 func (UnimplementedCleanupServiceServer) mustEmbedUnimplementedCleanupServiceServer() {}
 
 // UnsafeCleanupServiceServer may be embedded to opt out of forward compatibility for this service.
@@ -120,6 +187,32 @@ func (x *cleanupServiceApplyBfgObjectMapStreamServer) Recv() (*ApplyBfgObjectMap
 	return m, nil
 }
 
+func _CleanupService_RewriteHistory_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(CleanupServiceServer).RewriteHistory(&cleanupServiceRewriteHistoryServer{stream})
+}
+
+type CleanupService_RewriteHistoryServer interface {
+	SendAndClose(*RewriteHistoryResponse) error
+	Recv() (*RewriteHistoryRequest, error)
+	grpc.ServerStream
+}
+
+type cleanupServiceRewriteHistoryServer struct {
+	grpc.ServerStream
+}
+
+func (x *cleanupServiceRewriteHistoryServer) SendAndClose(m *RewriteHistoryResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *cleanupServiceRewriteHistoryServer) Recv() (*RewriteHistoryRequest, error) {
+	m := new(RewriteHistoryRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // CleanupService_ServiceDesc is the grpc.ServiceDesc for CleanupService service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -134,6 +227,11 @@ var CleanupService_ServiceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 			ClientStreams: true,
 		},
+		{
+			StreamName:    "RewriteHistory",
+			Handler:       _CleanupService_RewriteHistory_Handler,
+			ClientStreams: true,
+		},
 	},
 	Metadata: "cleanup.proto",
 }
-- 
cgit v1.2.3


From a103d1d7e84585f627d9cee8ed6a0117a77c5698 Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Fri, 12 Jan 2024 15:13:26 -0500
Subject: cleanup: Don't run filter-repo in-place

git-filter-repo(1) uses git-fast-import(1) to import the rewritten
repository history. This will unpack the new objects, then iterate over
references serially and update them using reference transactions. This
does not atomically update the references[0], so an interruption during
this final stage will result in partially applied changes.

To mitigate this risk, create a temporary staging repository to write
the updated history into, then atomically force fetch that into the
original repo.

This has the downside of being slower than modifying the repository
in-place[1], but improving safety for a high-risk operation like this
is a greater priority.

[0] https://gitlab.com/gitlab-org/git/-/blob/d4dbce1db5cd227a57074bcfc7ec9f0655961bba/builtin/fast-import.c#L1659-1668
[1] https://github.com/newren/git-filter-repo/issues/66#issuecomment-602100316
---
 internal/gitaly/service/cleanup/rewrite_history.go | 132 +++++++++++++++++++--
 1 file changed, 125 insertions(+), 7 deletions(-)

diff --git a/internal/gitaly/service/cleanup/rewrite_history.go b/internal/gitaly/service/cleanup/rewrite_history.go
index 7bbdd258e..85e45ab55 100644
--- a/internal/gitaly/service/cleanup/rewrite_history.go
+++ b/internal/gitaly/service/cleanup/rewrite_history.go
@@ -85,8 +85,8 @@ func (s *server) RewriteHistory(server gitalypb.CleanupService_RewriteHistorySer
 		}
 	}
 
-	if err := s.runFilterRepo(ctx, repo, repoProto, blobsToRemove, redactions); err != nil {
-		return fmt.Errorf("rewriting repository history: %w", err)
+	if err := s.rewriteHistory(ctx, repo, repoProto, blobsToRemove, redactions); err != nil {
+		return err
 	}
 
 	if err := server.SendAndClose(&gitalypb.RewriteHistoryResponse{}); err != nil {
@@ -96,15 +96,110 @@ func (s *server) RewriteHistory(server gitalypb.CleanupService_RewriteHistorySer
 	return nil
 }
 
-func (s *server) runFilterRepo(
+func (s *server) rewriteHistory(
 	ctx context.Context,
 	repo *localrepo.Repo,
 	repoProto *gitalypb.Repository,
 	blobsToRemove []string,
 	redactions [][]byte,
+) error {
+	defaultBranch, err := repo.HeadReference(ctx)
+	if err != nil {
+		return fmt.Errorf("finding HEAD reference: %w", err)
+	}
+
+	stagingRepo, stagingRepoPath, err := s.initStagingRepo(ctx, repoProto, defaultBranch)
+	if err != nil {
+		return fmt.Errorf("setting up staging repo: %w", err)
+	}
+
+	if err := s.runFilterRepo(ctx, repo, stagingRepo, blobsToRemove, redactions); err != nil {
+		return fmt.Errorf("rewriting repository history: %w", err)
+	}
+
+	var stderr strings.Builder
+	if err := repo.ExecAndWait(ctx,
+		git.Command{
+			Name: "fetch",
+			Flags: []git.Option{
+				// Delete any refs that were removed by filter-repo.
+				git.Flag{Name: "--prune"},
+				// The mirror refspec includes tags, don't fetch them again.
+				git.Flag{Name: "--no-tags"},
+				// New history will be disjoint from the original repo.
+				git.Flag{Name: "--force"},
+				// Ensure we don't partially apply the rewritten history.
+				// We don't expect file / directory conflicts as all refs
+				// in the staging repo are from the original.
+				git.Flag{Name: "--atomic"},
+				// We're going to have a lot of these, don't waste
+				// time displaying them.
+				git.Flag{Name: "--no-show-forced-updates"},
+				// No need for FETCH_HEAD when mirroring.
+				git.Flag{Name: "--no-write-fetch-head"},
+				git.Flag{Name: "--quiet"},
+			},
+			Args: append(
+				[]string{"file://" + stagingRepoPath},
+				git.MirrorRefSpec,
+			),
+		},
+		git.WithRefTxHook(repo),
+		git.WithStderr(&stderr),
+		git.WithConfig(git.ConfigPair{
+			Key: "advice.fetchShowForcedUpdates", Value: "false",
+		}),
+	); err != nil {
+		return structerr.New("fetching rewritten history: %w", err).WithMetadata("stderr", &stderr)
+	}
+
+	return nil
+}
+
+// initStagingRepo creates a new bare repository to write the rewritten history into
+// with default branch is set to match the source repo.
+func (s *server) initStagingRepo(ctx context.Context, repo *gitalypb.Repository, defaultBranch git.ReferenceName) (*localrepo.Repo, string, error) {
+	stagingRepoProto, stagingRepoDir, err := tempdir.NewRepository(ctx, repo.GetStorageName(), s.logger, s.locator)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var stderr strings.Builder
+	cmd, err := s.gitCmdFactory.NewWithoutRepo(ctx, git.Command{
+		Name: "init",
+		Flags: []git.Option{
+			git.Flag{Name: "--bare"},
+			git.Flag{Name: "--quiet"},
+		},
+		Args: []string{stagingRepoDir.Path()},
+	}, git.WithStderr(&stderr))
+	if err != nil {
+		return nil, "", fmt.Errorf("spawning git-init: %w", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return nil, "", structerr.New("creating repository: %w", err).WithMetadata("stderr", &stderr)
+	}
+
+	stagingRepo := s.localrepo(stagingRepoProto)
+
+	// Ensure HEAD matches the source repository. In practice a mismatch doesn't cause problems,
+	// but out of an abundance of caution let's keep the two repos as similar as possible.
+	if err := stagingRepo.SetDefaultBranch(ctx, s.txManager, defaultBranch); err != nil {
+		return nil, "", fmt.Errorf("setting default branch: %w", err)
+	}
+
+	return stagingRepo, stagingRepoDir.Path(), nil
+}
+
+func (s *server) runFilterRepo(
+	ctx context.Context,
+	srcRepo, stagingRepo *localrepo.Repo,
+	blobsToRemove []string,
+	redactions [][]byte,
 ) error {
 	// Place argument files in a tempdir so that cleanup is handled automatically.
-	tmpDir, err := tempdir.New(ctx, repo.GetStorageName(), s.logger, s.locator)
+	tmpDir, err := tempdir.New(ctx, srcRepo.GetStorageName(), s.logger, s.locator)
 	if err != nil {
 		return fmt.Errorf("create tempdir: %w", err)
 	}
@@ -129,11 +224,29 @@ func (s *server) runFilterRepo(
 		flags = append(flags, git.Flag{Name: "--replace-text=" + replacePath})
 	}
 
+	srcPath, err := srcRepo.Path()
+	if err != nil {
+		return fmt.Errorf("getting source repo path: %w", err)
+	}
+
+	stagingPath, err := stagingRepo.Path()
+	if err != nil {
+		return fmt.Errorf("getting target repo path: %w", err)
+	}
+
+	// We must run this using 'NewWithoutRepo' because setting '--git-dir',
+	// as 'repo.ExecAndWait' does, will override the '--target' flag and
+	// write the updates directly to the original repository.
 	var stdout, stderr strings.Builder
-	if err := repo.ExecAndWait(ctx,
+	cmd, err := s.gitCmdFactory.NewWithoutRepo(ctx,
 		git.Command{
 			Name: "filter-repo",
 			Flags: append([]git.Option{
+				// Repository to write filtered history into.
+				git.Flag{Name: "--target=" + stagingPath},
+				// Repository to read from.
+				git.Flag{Name: "--source=" + srcPath},
+				// git.Flag{Name: "--refs=refs/*"},
 				// Prevent automatic cleanup tasks like deleting 'origin' and running git-gc(1).
 				git.Flag{Name: "--partial"},
 				// Bypass check that repository is not a fresh clone.
@@ -146,10 +259,15 @@ func (s *server) runFilterRepo(
 				git.Flag{Name: "--quiet"},
 			}, flags...),
 		},
-		git.WithRefTxHook(repo),
+		git.WithDisabledHooks(),
 		git.WithStdout(&stdout),
 		git.WithStderr(&stderr),
-	); err != nil {
+	)
+	if err != nil {
+		return fmt.Errorf("spawning git-filter-repo: %w", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
 			return structerr.New("git-filter-repo failed with exit code %d", exitErr.ExitCode()).WithMetadataItems(
-- 
cgit v1.2.3


From c23a2ab30f27b449061ca85c8d79321def12f3c1 Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Fri, 19 Jan 2024 00:20:51 -0500
Subject: cleanup: Validate repo was not modified before fetching

On large repositories git-filter-repo(1) make take a significant amount
of time to run. Should a write occur after the git-fast-export(1)
portion of the task has completed, it is possible that the repository
history will not be fully rewritten.

To guard against this condition, we checksum the repository before and
after running filter-repo. If the checksums do not match we abort and do
not fetch the updated history into the repository.
---
 internal/gitaly/service/cleanup/rewrite_history.go | 56 ++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/internal/gitaly/service/cleanup/rewrite_history.go b/internal/gitaly/service/cleanup/rewrite_history.go
index 85e45ab55..e6fceefc5 100644
--- a/internal/gitaly/service/cleanup/rewrite_history.go
+++ b/internal/gitaly/service/cleanup/rewrite_history.go
@@ -1,6 +1,7 @@
 package cleanup
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"errors"
@@ -113,10 +114,31 @@ func (s *server) rewriteHistory(
 		return fmt.Errorf("setting up staging repo: %w", err)
 	}
 
+	// Check state of source repository prior to running filter-repo.
+	initialChecksum, err := checksumRepo(ctx, s.gitCmdFactory, repo)
+	if err != nil {
+		return fmt.Errorf("calculate initial checksum: %w", err)
+	}
+
 	if err := s.runFilterRepo(ctx, repo, stagingRepo, blobsToRemove, redactions); err != nil {
 		return fmt.Errorf("rewriting repository history: %w", err)
 	}
 
+	// Recheck repository state to confirm no changes occurred while filter-repo ran. The
+	// repository may not be fully rewritten if it was modified after git-fast-export(1)
+	// completed.
+	validationChecksum, err := checksumRepo(ctx, s.gitCmdFactory, repo)
+	if err != nil {
+		return fmt.Errorf("recalculate checksum: %w", err)
+	}
+
+	if initialChecksum != validationChecksum {
+		return structerr.NewAborted("source repository checksum altered").WithMetadataItems(
+			structerr.MetadataItem{Key: "initial checksum", Value: initialChecksum},
+			structerr.MetadataItem{Key: "validation checksum", Value: validationChecksum},
+		)
+	}
+
 	var stderr strings.Builder
 	if err := repo.ExecAndWait(ctx,
 		git.Command{
@@ -300,3 +322,37 @@ func writeArgFile(name string, dir string, input []byte) (string, error) {
 
 	return path, nil
 }
+
+func checksumRepo(ctx context.Context, cmdFactory git.CommandFactory, repo *localrepo.Repo) (string, error) {
+	var stderr strings.Builder
+	cmd, err := cmdFactory.New(ctx, repo, git.Command{
+		Name: "show-ref",
+		Flags: []git.Option{
+			git.Flag{Name: "--head"},
+		},
+	}, git.WithSetupStdout(), git.WithStderr(&stderr))
+	if err != nil {
+		return "", fmt.Errorf("spawning git-show-ref: %w", err)
+	}
+
+	var checksum git.Checksum
+
+	scanner := bufio.NewScanner(cmd)
+	for scanner.Scan() {
+		checksum.AddBytes(scanner.Bytes())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return "", structerr.New("git-show-ref failed with exit code %d", exitErr.ExitCode()).WithMetadata("stderr", stderr.String())
+		}
+		return "", fmt.Errorf("running git-show-ref: %w", err)
+	}
+
+	return checksum.String(), nil
+}
-- 
cgit v1.2.3


From 992753dca25886085bc3c4815a0bd760e5dc4e0f Mon Sep 17 00:00:00 2001
From: Will Chandler <wchandler@gitlab.com>
Date: Sun, 14 Jan 2024 23:11:20 -0500
Subject: coordinator: Enable transactions for new RewriteHistory RPC

Enable transactions for the new RewriteHistory RPC.
---
 internal/praefect/coordinator.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal/praefect/coordinator.go b/internal/praefect/coordinator.go
index 03f9e0df2..170826299 100644
--- a/internal/praefect/coordinator.go
+++ b/internal/praefect/coordinator.go
@@ -44,6 +44,7 @@ func transactionsDisabled(context.Context) bool { return false }
 // behaviour. If none is given, it's always enabled.
 var transactionRPCs = map[string]transactionsCondition{
 	"/gitaly.CleanupService/ApplyBfgObjectMapStream":         transactionsEnabled,
+	"/gitaly.CleanupService/RewriteHistory":                  transactionsEnabled,
 	"/gitaly.ConflictsService/ResolveConflicts":              transactionsEnabled,
 	"/gitaly.ObjectPoolService/DisconnectGitAlternates":      transactionsEnabled,
 	"/gitaly.ObjectPoolService/FetchIntoObjectPool":          transactionsEnabled,
-- 
cgit v1.2.3