diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2017-08-31 16:53:27 +0300 |
---|---|---|
committer | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2017-08-31 16:53:27 +0300 |
commit | ee47129ffd082384f465a4a6365d1ffeae7b7c53 (patch) | |
tree | 25dbfe3abbb9df5c765b7146efb11d671ffe9001 | |
parent | af66130f437865ef5a9a6dd0c1b1233bcaf50540 (diff) | |
parent | c78b640c2b33f047270594d40c38462557429670 (diff) |
Merge branch 'git-linguist' into 'master'
Use git-linguist to implement CommitLanguages
See merge request !316
-rw-r--r-- | CHANGELOG.md | 2 | ||||
-rw-r--r-- | cmd/gitaly/main.go | 7 | ||||
-rw-r--r-- | internal/linguist/linguist.go | 74 | ||||
-rw-r--r-- | internal/service/commit/languages.go | 77 | ||||
-rw-r--r-- | internal/service/commit/testhelper_test.go | 5 | ||||
-rwxr-xr-x | ruby/bin/ruby-cd | 6 | ||||
-rw-r--r-- | ruby/lib/gitaly_server/commit_service.rb | 15 | ||||
-rw-r--r-- | ruby/lib/gitlab/git.rb | 2 |
8 files changed, 168 insertions, 20 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cf30bbef..5c35e0a49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ UNRELEASED https://gitlab.com/gitlab-org/gitaly/merge_requests/318 - Implement {Create,Delete}Branch RPCs https://gitlab.com/gitlab-org/gitaly/merge_requests/311 +- Use git-linguist to implement CommitLanguages + https://gitlab.com/gitlab-org/gitaly/merge_requests/316 v0.35.0 diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go index 0615a0c78..f6c1221b9 100644 --- a/cmd/gitaly/main.go +++ b/cmd/gitaly/main.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/connectioncounter" + "gitlab.com/gitlab-org/gitaly/internal/linguist" "gitlab.com/gitlab-org/gitaly/internal/rubyserver" "gitlab.com/gitlab-org/gitaly/internal/server" "gitlab.com/gitlab-org/gitaly/internal/version" @@ -44,6 +45,12 @@ func loadConfig() { }).Warn("can not load configuration") } + if err := linguist.LoadColors(); err != nil { + log.WithFields(log.Fields{ + "ruby.dir": config.Config.Ruby.Dir, + "error": err, + }).Warn("can not load linguist languages") + } } func validateConfig() error { diff --git a/internal/linguist/linguist.go b/internal/linguist/linguist.go new file mode 100644 index 000000000..497f3b9a3 --- /dev/null +++ b/internal/linguist/linguist.go @@ -0,0 +1,74 @@ +package linguist + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + + "gitlab.com/gitlab-org/gitaly/internal/config" + "gitlab.com/gitlab-org/gitaly/internal/helper" +) + +var ( + colorMap = make(map[string]Language) +) + +// Language is used to parse Linguist's language.json file. +type Language struct { + Color string `json:"color"` +} + +// Stats returns the repository's language stats as reported by 'git-linguist'. +func Stats(ctx context.Context, repoPath string, commitID string) (map[string]int, error) { + cmd := exec.Command("bundle", "exec", "bin/ruby-cd", repoPath, "git-linguist", "--commit="+commitID, "stats") + cmd.Dir = config.Config.Ruby.Dir + reader, err := helper.NewCommand(ctx, cmd, nil, nil, nil, os.Environ()...) + if err != nil { + return nil, err + } + defer reader.Close() + + data, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + stats := make(map[string]int) + return stats, json.Unmarshal(data, &stats) +} + +// Color returns the color Linguist has assigned to language. +func Color(language string) string { + if color := colorMap[language].Color; color != "" { + return color + } + + colorSha := sha256.Sum256([]byte(language)) + return fmt.Sprintf("#%x", colorSha[0:3]) +} + +// LoadColors loads the name->color map from the Linguist gem. +func LoadColors() error { + cmd := exec.Command("bundle", "show", "linguist") + cmd.Dir = config.Config.Ruby.Dir + linguistPath, err := cmd.Output() + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + err = fmt.Errorf("%v; stderr: %q", exitError, exitError.Stderr) + } + return err + } + + languageJSON, err := ioutil.ReadFile(path.Join(strings.TrimSpace(string(linguistPath)), "lib/linguist/languages.json")) + if err != nil { + return err + } + + return json.Unmarshal(languageJSON, &colorMap) +} diff --git a/internal/service/commit/languages.go b/internal/service/commit/languages.go index 493f2ac0d..c23d8b8fb 100644 --- a/internal/service/commit/languages.go +++ b/internal/service/commit/languages.go @@ -1,7 +1,16 @@ package commit import ( - "gitlab.com/gitlab-org/gitaly/internal/rubyserver" + "io/ioutil" + "sort" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + + "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/linguist" + "gitlab.com/gitlab-org/gitaly/internal/service/ref" pb "gitlab.com/gitlab-org/gitaly-proto/go" @@ -9,15 +18,75 @@ import ( ) func (*server) CommitLanguages(ctx context.Context, req *pb.CommitLanguagesRequest) (*pb.CommitLanguagesResponse, error) { - client, err := rubyserver.CommitServiceClient(ctx) + repoPath, err := helper.GetRepoPath(req.Repository) + if err != nil { + return nil, err + } + + revision := string(req.Revision) + if revision == "" { + defaultBranch, err := ref.DefaultBranchName(ctx, repoPath) + if err != nil { + return nil, err + } + revision = string(defaultBranch) + } + + commitID, err := lookupRevision(ctx, repoPath, revision) if err != nil { return nil, err } - clientCtx, err := rubyserver.SetHeaders(ctx, req.GetRepository()) + stats, err := linguist.Stats(ctx, repoPath, commitID) if err != nil { return nil, err } - return client.CommitLanguages(clientCtx, req) + resp := &pb.CommitLanguagesResponse{} + if len(stats) == 0 { + return resp, nil + } + + total := 0 + for _, count := range stats { + total += count + } + + if total == 0 { + return nil, grpc.Errorf(codes.Internal, "linguist stats added up to zero: %v", stats) + } + + for lang, count := range stats { + l := &pb.CommitLanguagesResponse_Language{ + Name: lang, + Share: float32(100*count) / float32(total), + Color: linguist.Color(lang), + } + resp.Languages = append(resp.Languages, l) + } + + sort.Sort(languageSorter(resp.Languages)) + + return resp, nil +} + +type languageSorter []*pb.CommitLanguagesResponse_Language + +func (ls languageSorter) Len() int { return len(ls) } +func (ls languageSorter) Swap(i, j int) { ls[i], ls[j] = ls[j], ls[i] } +func (ls languageSorter) Less(i, j int) bool { return ls[i].Share > ls[j].Share } + +func lookupRevision(ctx context.Context, repoPath string, revision string) (string, error) { + revParse, err := helper.GitCommandReader(ctx, "--git-dir", repoPath, "rev-parse", revision) + if err != nil { + return "", err + } + defer revParse.Close() + + revParseBytes, err := ioutil.ReadAll(revParse) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(revParseBytes)), nil } diff --git a/internal/service/commit/testhelper_test.go b/internal/service/commit/testhelper_test.go index 0447c3d2e..33fbf3ddd 100644 --- a/internal/service/commit/testhelper_test.go +++ b/internal/service/commit/testhelper_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "gitlab.com/gitlab-org/gitaly/internal/linguist" "gitlab.com/gitlab-org/gitaly/internal/middleware/objectdirhandler" "gitlab.com/gitlab-org/gitaly/internal/rubyserver" "gitlab.com/gitlab-org/gitaly/internal/testhelper" @@ -30,6 +31,10 @@ func TestMain(m *testing.M) { func testMain(m *testing.M) int { testhelper.ConfigureRuby() + if err := linguist.LoadColors(); err != nil { + log.Fatal(err) + } + ruby, err := rubyserver.Start() if err != nil { log.Fatal(err) diff --git a/ruby/bin/ruby-cd b/ruby/bin/ruby-cd new file mode 100755 index 000000000..9dfb91e84 --- /dev/null +++ b/ruby/bin/ruby-cd @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +# This script lets you run `bundle exec` in one directory, and then changes into another. + +Dir.chdir(ARGV.shift) +exec(*ARGV) diff --git a/ruby/lib/gitaly_server/commit_service.rb b/ruby/lib/gitaly_server/commit_service.rb index 26dd9a577..07302de2c 100644 --- a/ruby/lib/gitaly_server/commit_service.rb +++ b/ruby/lib/gitaly_server/commit_service.rb @@ -2,21 +2,6 @@ module GitalyServer class CommitService < Gitaly::CommitService::Service include Utils - def commit_languages(request, _call) - repo = Gitlab::Git::Repository.from_call(_call) - revision = request.revision unless request.revision.empty? - - language_messages = repo.languages(revision).map do |language| - Gitaly::CommitLanguagesResponse::Language.new( - name: language[:label], - share: language[:value], - color: language[:color] - ) - end - - Gitaly::CommitLanguagesResponse.new(languages: language_messages) - end - def commit_stats(request, _call) repo = Gitlab::Git::Repository.from_call(_call) revision = request.revision unless request.revision.empty? diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb index 063d9c2c2..50935aa6c 100644 --- a/ruby/lib/gitlab/git.rb +++ b/ruby/lib/gitlab/git.rb @@ -1,6 +1,6 @@ # External dependencies of Gitlab::Git require 'rugged' -require 'linguist' +require 'linguist/blob_helper' # Ruby on Rails mix-ins that GitLab::Git code relies on require 'active_support/core_ext/object/blank' |