1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
package repository
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus"
gitalyerrors "gitlab.com/gitlab-org/gitaly/v15/internal/errors"
"gitlab.com/gitlab-org/gitaly/v15/internal/git"
"gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/archive"
"gitlab.com/gitlab-org/gitaly/v15/internal/helper"
"gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"
"gitlab.com/gitlab-org/gitaly/v15/streamio"
)
var objectFiles = []*regexp.Regexp{
regexp.MustCompile(`/[[:xdigit:]]{2}/[[:xdigit:]]{38}\z`),
regexp.MustCompile(`/pack/pack\-[[:xdigit:]]{40}\.(pack|idx)\z`),
}
func (s *server) GetSnapshot(in *gitalypb.GetSnapshotRequest, stream gitalypb.RepositoryService_GetSnapshotServer) error {
if in.GetRepository() == nil {
return helper.ErrInvalidArgument(gitalyerrors.ErrEmptyRepository)
}
path, err := s.locator.GetRepoPath(in.Repository)
if err != nil {
return err
}
writer := streamio.NewWriter(func(p []byte) error {
return stream.Send(&gitalypb.GetSnapshotResponse{Data: p})
})
// Building a raw archive may race with `git push`, but GitLab can enforce
// concurrency control if necessary. Using `TarBuilder` means we can keep
// going even if some files are added or removed during the operation.
builder := archive.NewTarBuilder(path, writer)
// Pick files directly by filename so we can get a snapshot even if the
// repository is corrupted. https://gitirc.eu/gitrepository-layout.html
// documents the various files and directories. We exclude the following
// on purpose:
//
// * branches - legacy, not replicated by git fetch
// * commondir - may differ between sites
// * config - may contain credentials, and cannot be managed by client
// * custom-hooks - GitLab-specific, no supported in Geo, may differ between sites
// * hooks - symlink, may differ between sites
// * {shared,}index[.*] - not found in bare repositories
// * info/{attributes,exclude,grafts} - not replicated by git fetch
// * info/refs - dumb protocol only
// * logs/* - not replicated by git fetch
// * modules/* - not replicated by git fetch
// * objects/info/* - unneeded (dumb protocol) or to do with alternates
// * worktrees/* - not replicated by git fetch
// References
_ = builder.FileIfExist("HEAD")
_ = builder.FileIfExist("packed-refs")
_ = builder.RecursiveDirIfExist("refs")
_ = builder.RecursiveDirIfExist("branches")
// The packfiles + any loose objects.
_ = builder.RecursiveDirIfExist("objects", objectFiles...)
// In case this repository is a shallow clone. Seems unlikely, but better
// safe than sorry.
_ = builder.FileIfExist("shallow")
if err := s.addAlternateFiles(stream.Context(), in.GetRepository(), builder); err != nil {
return helper.ErrInternalf("add alternates: %w", err)
}
if err := builder.Close(); err != nil {
return helper.ErrInternal(fmt.Errorf("building snapshot failed: %v", err))
}
return nil
}
func (s *server) addAlternateFiles(ctx context.Context, repository *gitalypb.Repository, builder *archive.TarBuilder) error {
storageRoot, err := s.locator.GetStorageByName(repository.GetStorageName())
if err != nil {
return fmt.Errorf("get storage path: %w", err)
}
repoPath, err := s.locator.GetRepoPath(repository)
if err != nil {
return err
}
altObjDirs, err := git.AlternateObjectDirectories(ctx, storageRoot, repoPath)
if err != nil {
ctxlogrus.Extract(ctx).WithField("error", err).Warn("error getting alternate object directories")
return nil
}
for _, altObjDir := range altObjDirs {
if err := walkAndAddToBuilder(altObjDir, builder); err != nil {
return fmt.Errorf("walking alternates file: %v", err)
}
}
return nil
}
func walkAndAddToBuilder(alternateObjDir string, builder *archive.TarBuilder) error {
matchWalker := archive.NewMatchWalker(objectFiles, func(path string, info os.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("error walking %v: %v", path, err)
}
relPath, err := filepath.Rel(alternateObjDir, path)
if err != nil {
return fmt.Errorf("alternative object directory path: %w", err)
}
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("opening file %s: %v", path, err)
}
defer file.Close()
objectPath := filepath.Join("objects", relPath)
if err := builder.VirtualFileWithContents(objectPath, file); err != nil {
return fmt.Errorf("expected file %v to exist: %v", path, err)
}
return nil
})
if err := filepath.Walk(alternateObjDir, matchWalker.Walk); err != nil {
return fmt.Errorf("error when traversing: %v", err)
}
return nil
}
|
