diff options
author | Toon Claes <toon@gitlab.com> | 2022-09-08 21:37:46 +0300 |
---|---|---|
committer | Toon Claes <toon@gitlab.com> | 2022-10-03 16:26:16 +0300 |
commit | 7b49c61f7b3042de9596a73e321b5c936c9117f9 (patch) | |
tree | d006cd07e1b7b69e6849d91e648abc7defc8bf0e | |
parent | 1c491ca68bfb4b28453befa0b562c275bc18c2f8 (diff) |
linguist: Use the colors from go-enry
We're about to go all-in on go-enry for the language detection. Which
means we'll soon no longer can load the language colors from a json file
provided by the linguist gem. We could copy the file, but go-enry
provides colors as well. So use those instead.
This also made it possible so turn the Color() function into a normal
function, instead of a receiver function.
This has a little disadvantage: it's no longer possible to load a custom
set of colors. We believe this isn't used, so we're willing to give up
on that.
-rw-r--r-- | internal/gitaly/linguist/linguist.go | 67 | ||||
-rw-r--r-- | internal/gitaly/linguist/linguist_test.go | 75 | ||||
-rw-r--r-- | internal/gitaly/service/commit/languages.go | 3 |
3 files changed, 9 insertions, 136 deletions
diff --git a/internal/gitaly/linguist/linguist.go b/internal/gitaly/linguist/linguist.go index 9e4fa648c..e22b583ab 100644 --- a/internal/gitaly/linguist/linguist.go +++ b/internal/gitaly/linguist/linguist.go @@ -8,7 +8,6 @@ import ( "io" "os" "os/exec" - "path/filepath" "github.com/go-enry/go-enry/v2" "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus/ctxlogrus" @@ -22,37 +21,19 @@ import ( "gitlab.com/gitlab-org/gitaly/v15/internal/metadata/featureflag" ) -// Language is used to parse Linguist's language.json file. -type Language struct { - Color string `json:"color"` -} - // ByteCountPerLanguage represents a counter value (bytes) per language. type ByteCountPerLanguage map[string]uint64 // Instance is a holder of the defined in the system language settings. type Instance struct { - cfg config.Cfg - colorMap map[string]Language + cfg config.Cfg } // New loads the name->color map from the Linguist gem and returns initialized // instance to use back to the caller or an error. func New(cfg config.Cfg) (*Instance, error) { - jsonReader, err := openLanguagesJSON(cfg) - if err != nil { - return nil, err - } - defer jsonReader.Close() - - var colorMap map[string]Language - if err := json.NewDecoder(jsonReader).Decode(&colorMap); err != nil { - return nil, err - } - return &Instance{ - cfg: cfg, - colorMap: colorMap, + cfg: cfg, }, nil } @@ -90,8 +71,8 @@ func (inst *Instance) Stats(ctx context.Context, repo *localrepo.Repo, commitID } // Color returns the color Linguist has assigned to language. -func (inst *Instance) Color(language string) string { - if color := inst.colorMap[language].Color; color != "" { +func Color(language string) string { + if color := enry.GetColor(language); color != "#cccccc" { return color } @@ -119,46 +100,6 @@ func (inst *Instance) startGitLinguist(ctx context.Context, repoPath string, com return internalCmd, nil } -func openLanguagesJSON(cfg config.Cfg) (io.ReadCloser, error) { - if jsonPath := cfg.Ruby.LinguistLanguagesPath; jsonPath != "" { - // This is a fallback for environments where dynamic discovery of the - // linguist path via Bundler is not working for some reason, for example - // https://gitlab.com/gitlab-org/gitaly/issues/1119. - return os.Open(jsonPath) - } - - linguistPathSymlink, err := os.CreateTemp("", "gitaly-linguist-path") - if err != nil { - return nil, err - } - defer func() { _ = os.Remove(linguistPathSymlink.Name()) }() - - if err := linguistPathSymlink.Close(); err != nil { - return nil, err - } - - // We use a symlink because we cannot trust Bundler to not print garbage - // on its stdout. - rubyScript := `FileUtils.ln_sf(Bundler.rubygems.find_name('github-linguist').first.full_gem_path, ARGV.first)` - cmd := exec.Command("bundle", "exec", "ruby", "-rfileutils", "-e", rubyScript, linguistPathSymlink.Name()) - cmd.Dir = cfg.Ruby.Dir - - // We have learned that in practice the command we are about to run is a - // canary for Ruby/Bundler configuration problems. Including stderr and - // stdout in the gitaly log is useful for debugging such problems. - cmd.Stderr = os.Stderr - cmd.Stdout = os.Stdout - - if err := cmd.Run(); err != nil { - if exitError, ok := err.(*exec.ExitError); ok { - err = fmt.Errorf("%v; stderr: %q", exitError, exitError.Stderr) - } - return nil, err - } - - return os.Open(filepath.Join(linguistPathSymlink.Name(), "lib", "linguist", "languages.json")) -} - func (inst *Instance) enryStats(ctx context.Context, repo *localrepo.Repo, commitID string, catfileCache catfile.Cache) (ByteCountPerLanguage, error) { stats, err := newLanguageStats(repo) if err != nil { diff --git a/internal/gitaly/linguist/linguist_test.go b/internal/gitaly/linguist/linguist_test.go index 6e9887074..a3b98859a 100644 --- a/internal/gitaly/linguist/linguist_test.go +++ b/internal/gitaly/linguist/linguist_test.go @@ -10,8 +10,6 @@ import ( "strings" "testing" - "github.com/go-enry/go-enry/v2" - enrydata "github.com/go-enry/go-enry/v2/data" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v15/internal/git" "gitlab.com/gitlab-org/gitaly/v15/internal/git/catfile" @@ -28,56 +26,6 @@ func TestMain(m *testing.M) { testhelper.Run(m) } -// TestNew_knownLanguages tests the compatibility between the Ruby and the Go -// implementation. This test will be removed together with the Ruby implementation. -func TestNew_knownLanguages(t *testing.T) { - t.Parallel() - - cfg := testcfg.Build(t, testcfg.WithRealLinguist()) - - linguist, err := New(cfg) - require.NoError(t, err) - - t.Run("by name", func(t *testing.T) { - linguistLanguages := make([]string, 0, len(linguist.colorMap)) - for language := range linguist.colorMap { - linguistLanguages = append(linguistLanguages, language) - } - - enryLanguages := make([]string, 0, len(enrydata.IDByLanguage)) - for language := range enrydata.IDByLanguage { - enryLanguages = append(enryLanguages, language) - } - - require.ElementsMatch(t, linguistLanguages, enryLanguages) - }) - - t.Run("with their color", func(t *testing.T) { - exclude := map[string]struct{}{} - - linguistLanguages := make(map[string]string, len(linguist.colorMap)) - for language, color := range linguist.colorMap { - if color.Color == "" { - exclude[language] = struct{}{} - continue - } - linguistLanguages[language] = color.Color - } - - enryLanguages := make(map[string]string, len(enrydata.IDByLanguage)) - for language := range enrydata.IDByLanguage { - if _, excluded := exclude[language]; excluded { - continue - } - - color := enry.GetColor(language) - enryLanguages[language] = color - } - - require.Equal(t, linguistLanguages, enryLanguages) - }) -} - func TestInstance_Stats(t *testing.T) { testhelper.NewFeatureSets(featureflag.GoLanguageStats). Run(t, testInstanceStats) @@ -290,14 +238,9 @@ func TestInstance_Stats_unmarshalJSONError(t *testing.T) { require.False(t, ok, "expected the error not be a json Syntax Error") } -func TestInstance_Color(t *testing.T) { +func TestColor(t *testing.T) { t.Parallel() - cfg := testcfg.Build(t, testcfg.WithRealLinguist()) - - ling, err := New(cfg) - require.NoError(t, err) - for _, tc := range []struct { language string expectedColor string @@ -307,27 +250,15 @@ func TestInstance_Color(t *testing.T) { {language: "HTML", expectedColor: "#e34c26"}, {language: "Markdown", expectedColor: "#083fa1"}, {language: "Javascript", expectedColor: "#75712c"}, - {language: "SSH Config", expectedColor: "#2d519e"}, + {language: "SSH Config", expectedColor: "#d1dbe0"}, // grouped into INI by go-enry {language: "Wozzle Wuzzle", expectedColor: "#3adbcf"}, // non-existing language } { t.Run(tc.language, func(t *testing.T) { - require.Equal(t, tc.expectedColor, ling.Color(tc.language), "color value for '%v'", tc.language) + require.Equal(t, tc.expectedColor, Color(tc.language), "color value for '%v'", tc.language) }) } } -func TestNew_loadLanguagesCustomPath(t *testing.T) { - jsonPath, err := filepath.Abs("testdata/fake-languages.json") - require.NoError(t, err) - - cfg := testcfg.Build(t, testcfg.WithBase(config.Cfg{Ruby: config.Ruby{LinguistLanguagesPath: jsonPath}})) - - ling, err := New(cfg) - require.NoError(t, err) - - require.Equal(t, "foo color", ling.Color("FooBar")) -} - // filenameForCache returns the filename where the cache is stored, depending on // the feature flag. func filenameForCache(ctx context.Context) string { diff --git a/internal/gitaly/service/commit/languages.go b/internal/gitaly/service/commit/languages.go index 4363011c0..8fef28db8 100644 --- a/internal/gitaly/service/commit/languages.go +++ b/internal/gitaly/service/commit/languages.go @@ -10,6 +10,7 @@ import ( "strings" "gitlab.com/gitlab-org/gitaly/v15/internal/git" + "gitlab.com/gitlab-org/gitaly/v15/internal/gitaly/linguist" "gitlab.com/gitlab-org/gitaly/v15/internal/helper" "gitlab.com/gitlab-org/gitaly/v15/internal/helper/text" "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb" @@ -61,7 +62,7 @@ func (s *server) CommitLanguages(ctx context.Context, req *gitalypb.CommitLangua l := &gitalypb.CommitLanguagesResponse_Language{ Name: lang, Share: float32(100*count) / float32(total), - Color: s.linguist.Color(lang), + Color: linguist.Color(lang), Bytes: stats[lang], } resp.Languages = append(resp.Languages, l) |