Log a pack file index along side transaction's pack file

The objects a transaction makes reachable are collected into a pack
file that then ultimately gets logged. On log applicaton that pack
file is currently unpacked into the repository. A more efficient way
to apply the pack file would be to hard link it into the repository
directly from the log to avoid copying. In order for the objects to
be available for reading, the pack file needs an index which we're
currently not computing. To prepare for linking the pack files into
place, this commit computes an index for the transaction's pack file
prior to logging the transaction. The index is already logged along
side the pack file but we don't yet make use of it.

1 files changed, 28 insertions, 18 deletions

diff --git a/internal/gitaly/transaction_manager.go b/internal/gitaly/transaction_manager.go
index 97ad016fc..e98ba7a92 100644
--- a/internal/gitaly/transaction_manager.go
+++ b/internal/gitaly/transaction_manager.go
@@ -601,34 +601,44 @@ func (mgr *TransactionManager) packObjects(ctx context.Context, transaction *Tra
 		return nil
 	})
 
+	packReader, packWriter := io.Pipe()
 	group.Go(func() (returnedErr error) {
-		defer func() { objectsReader.CloseWithError(returnedErr) }()
+		defer func() {
+			objectsReader.CloseWithError(returnedErr)
+			packWriter.CloseWithError(returnedErr)
+		}()
 
-		if err := os.Mkdir(transaction.walFilesPath(), perm.PrivateDir); err != nil {
-			return fmt.Errorf("create wal files directory: %w", err)
+		if err := transaction.stagingRepository.PackObjects(ctx, objectsReader, packWriter); err != nil {
+			return fmt.Errorf("pack objects: %w", err)
 		}
 
-		destinationFile, err := os.OpenFile(
-			packFilePath(transaction.walFilesPath()),
-			os.O_WRONLY|os.O_CREATE|os.O_EXCL,
-			perm.PrivateFile,
-		)
-		if err != nil {
-			return fmt.Errorf("open file: %w", err)
+		return nil
+	})
+
+	group.Go(func() (returnedErr error) {
+		defer packReader.CloseWithError(returnedErr)
+
+		if err := os.Mkdir(transaction.walFilesPath(), perm.PrivateDir); err != nil {
+			return fmt.Errorf("create wal files directory: %w", err)
 		}
-		defer destinationFile.Close()
 
-		if err := transaction.stagingRepository.PackObjects(ctx, objectsReader, destinationFile); err != nil {
-			return fmt.Errorf("pack objects: %w", err)
+		var stderr bytes.Buffer
+		if err := transaction.stagingRepository.ExecAndWait(ctx, git.Command{
+			Name:  "index-pack",
+			Flags: []git.Option{git.Flag{Name: "--stdin"}},
+			Args:  []string{packFilePath(transaction.walFilesPath())},
+		}, git.WithStdin(packReader), git.WithStderr(&stderr)); err != nil {
+			return structerr.New("index pack: %w", err).WithMetadata("stderr", stderr.String())
 		}
 
-		// Sync the contents of the pack so they are flushed to disk prior to the transaction
-		// being admitted for processing.
-		if err := destinationFile.Sync(); err != nil {
-			return fmt.Errorf("sync pack: %w", err)
+		// Sync the files and the directory entries so everything is flushed to the disk prior
+		// to moving on to committing the log entry. This way we only have to flush the directory
+		// move when we move the staged files into the log.
+		if err := safe.NewSyncer().SyncRecursive(transaction.walFilesPath()); err != nil {
+			return fmt.Errorf("sync recursive: %w", err)
 		}
 
-		return destinationFile.Close()
+		return nil
 	})
 
 	return group.Wait()