1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
package linguist
import (
"compress/zlib"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"gitlab.com/gitlab-org/gitaly/v16/internal/git/localrepo"
)
const (
// languageStatsFilename is the name of the file in the repo that stores
// a cached version of the language statistics. The name is
// intentionally different from what the linguist gem uses.
languageStatsFilename = "gitaly-language.stats"
languageStatsVersion = "v3:gitaly"
)
// languageStats takes care of accumulating and caching language statistics for
// a repository.
type languageStats struct {
// Version holds the file format version
Version string `json:"version"`
// CommitID holds the commit ID for the cached Totals
CommitID string `json:"commit_id"`
// m will protect concurrent writes to Totals & ByFile maps
m *sync.Mutex
// Totals contains the total statistics for the CommitID
Totals ByteCountPerLanguage `json:"totals"`
// ByFile contains the statistics for a single file, where the filename
// is its key.
ByFile map[string]ByteCountPerLanguage `json:"by_file"`
}
func newLanguageStats() languageStats {
return languageStats{
Totals: ByteCountPerLanguage{},
ByFile: make(map[string]ByteCountPerLanguage),
m: &sync.Mutex{},
}
}
// initLanguageStats tries to load the optionally available stats from file or
// returns a blank languageStats struct.
func initLanguageStats(repo *localrepo.Repo) (languageStats, error) {
objPath, err := repo.Path()
if err != nil {
return newLanguageStats(), fmt.Errorf("new language stats get repo path: %w", err)
}
file, err := os.Open(filepath.Join(objPath, languageStatsFilename))
if err != nil {
if os.IsNotExist(err) {
return newLanguageStats(), nil
}
return newLanguageStats(), fmt.Errorf("new language stats open: %w", err)
}
defer file.Close()
r, err := zlib.NewReader(file)
if err != nil {
return newLanguageStats(), fmt.Errorf("new language stats zlib reader: %w", err)
}
var loaded languageStats
if err = json.NewDecoder(r).Decode(&loaded); err != nil {
return newLanguageStats(), fmt.Errorf("new language stats json decode: %w", err)
}
if loaded.Version != languageStatsVersion {
return newLanguageStats(), fmt.Errorf("new language stats version mismatch %s vs %s", languageStatsVersion, loaded.Version)
}
loaded.m = &sync.Mutex{}
return loaded, nil
}
// add the statistics for the given filename
func (c *languageStats) add(filename, language string, size uint64) {
c.m.Lock()
defer c.m.Unlock()
for k, v := range c.ByFile[filename] {
c.Totals[k] -= v
if c.Totals[k] <= 0 {
delete(c.Totals, k)
}
}
c.ByFile[filename] = ByteCountPerLanguage{language: size}
if size > 0 {
c.Totals[language] += size
}
}
// drop statistics for the given files
func (c *languageStats) drop(filenames ...string) {
c.m.Lock()
defer c.m.Unlock()
for _, f := range filenames {
for k, v := range c.ByFile[f] {
c.Totals[k] -= v
if c.Totals[k] <= 0 {
delete(c.Totals, k)
}
}
delete(c.ByFile, f)
}
}
// save the language stats to file in the repository
func (c *languageStats) save(repo *localrepo.Repo, commitID string) error {
c.CommitID = commitID
c.Version = languageStatsVersion
repoPath, err := repo.Path()
if err != nil {
return fmt.Errorf("languageStats save get repo path: %w", err)
}
tempPath, err := repo.StorageTempDir()
if err != nil {
return fmt.Errorf("languageStats locate temp dir: %w", err)
}
file, err := os.CreateTemp(tempPath, languageStatsFilename)
if err != nil {
return fmt.Errorf("languageStats create temp file: %w", err)
}
defer func() {
file.Close()
_ = os.Remove(file.Name())
}()
w := zlib.NewWriter(file)
defer func() {
// We already check the error further down.
_ = w.Close()
}()
if err = json.NewEncoder(w).Encode(c); err != nil {
return fmt.Errorf("languageStats encode json: %w", err)
}
if err = w.Close(); err != nil {
return fmt.Errorf("languageStats zlib write: %w", err)
}
if err = file.Sync(); err != nil {
return fmt.Errorf("languageStats flush: %w", err)
}
if err = file.Close(); err != nil {
return fmt.Errorf("languageStats close: %w", err)
}
if err = os.Rename(file.Name(), filepath.Join(repoPath, languageStatsFilename)); err != nil {
return fmt.Errorf("languageStats rename: %w", err)
}
return nil
}
|
