Merge branch 'git-linguist' into 'master'

Use git-linguist to implement CommitLanguages

See merge request !316

8 files changed, 168 insertions, 20 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7cf30bbef..5c35e0a49 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@ UNRELEASED
   https://gitlab.com/gitlab-org/gitaly/merge_requests/318
 - Implement {Create,Delete}Branch RPCs
   https://gitlab.com/gitlab-org/gitaly/merge_requests/311
+- Use git-linguist to implement CommitLanguages
+  https://gitlab.com/gitlab-org/gitaly/merge_requests/316
 
 v0.35.0
 
diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go
index 0615a0c78..f6c1221b9 100644
--- a/cmd/gitaly/main.go
+++ b/cmd/gitaly/main.go
@@ -13,6 +13,7 @@ import (
 
 	"gitlab.com/gitlab-org/gitaly/internal/config"
 	"gitlab.com/gitlab-org/gitaly/internal/connectioncounter"
+	"gitlab.com/gitlab-org/gitaly/internal/linguist"
 	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/server"
 	"gitlab.com/gitlab-org/gitaly/internal/version"
@@ -44,6 +45,12 @@ func loadConfig() {
 		}).Warn("can not load configuration")
 	}
 
+	if err := linguist.LoadColors(); err != nil {
+		log.WithFields(log.Fields{
+			"ruby.dir": config.Config.Ruby.Dir,
+			"error":    err,
+		}).Warn("can not load linguist languages")
+	}
 }
 
 func validateConfig() error {
diff --git a/internal/linguist/linguist.go b/internal/linguist/linguist.go
new file mode 100644
index 000000000..497f3b9a3
--- /dev/null
+++ b/internal/linguist/linguist.go
@@ -0,0 +1,74 @@
+package linguist
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitaly/internal/config"
+	"gitlab.com/gitlab-org/gitaly/internal/helper"
+)
+
+var (
+	colorMap = make(map[string]Language)
+)
+
+// Language is used to parse Linguist's language.json file.
+type Language struct {
+	Color string `json:"color"`
+}
+
+// Stats returns the repository's language stats as reported by 'git-linguist'.
+func Stats(ctx context.Context, repoPath string, commitID string) (map[string]int, error) {
+	cmd := exec.Command("bundle", "exec", "bin/ruby-cd", repoPath, "git-linguist", "--commit="+commitID, "stats")
+	cmd.Dir = config.Config.Ruby.Dir
+	reader, err := helper.NewCommand(ctx, cmd, nil, nil, nil, os.Environ()...)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	data, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	stats := make(map[string]int)
+	return stats, json.Unmarshal(data, &stats)
+}
+
+// Color returns the color Linguist has assigned to language.
+func Color(language string) string {
+	if color := colorMap[language].Color; color != "" {
+		return color
+	}
+
+	colorSha := sha256.Sum256([]byte(language))
+	return fmt.Sprintf("#%x", colorSha[0:3])
+}
+
+// LoadColors loads the name->color map from the Linguist gem.
+func LoadColors() error {
+	cmd := exec.Command("bundle", "show", "linguist")
+	cmd.Dir = config.Config.Ruby.Dir
+	linguistPath, err := cmd.Output()
+	if err != nil {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			err = fmt.Errorf("%v; stderr: %q", exitError, exitError.Stderr)
+		}
+		return err
+	}
+
+	languageJSON, err := ioutil.ReadFile(path.Join(strings.TrimSpace(string(linguistPath)), "lib/linguist/languages.json"))
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(languageJSON, &colorMap)
+}
diff --git a/internal/service/commit/languages.go b/internal/service/commit/languages.go
index 493f2ac0d..c23d8b8fb 100644
--- a/internal/service/commit/languages.go
+++ b/internal/service/commit/languages.go
@@ -1,7 +1,16 @@
 package commit
 
 import (
-	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
+	"io/ioutil"
+	"sort"
+	"strings"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+
+	"gitlab.com/gitlab-org/gitaly/internal/helper"
+	"gitlab.com/gitlab-org/gitaly/internal/linguist"
+	"gitlab.com/gitlab-org/gitaly/internal/service/ref"
 
 	pb "gitlab.com/gitlab-org/gitaly-proto/go"
 
@@ -9,15 +18,75 @@ import (
 )
 
 func (*server) CommitLanguages(ctx context.Context, req *pb.CommitLanguagesRequest) (*pb.CommitLanguagesResponse, error) {
-	client, err := rubyserver.CommitServiceClient(ctx)
+	repoPath, err := helper.GetRepoPath(req.Repository)
+	if err != nil {
+		return nil, err
+	}
+
+	revision := string(req.Revision)
+	if revision == "" {
+		defaultBranch, err := ref.DefaultBranchName(ctx, repoPath)
+		if err != nil {
+			return nil, err
+		}
+		revision = string(defaultBranch)
+	}
+
+	commitID, err := lookupRevision(ctx, repoPath, revision)
 	if err != nil {
 		return nil, err
 	}
 
-	clientCtx, err := rubyserver.SetHeaders(ctx, req.GetRepository())
+	stats, err := linguist.Stats(ctx, repoPath, commitID)
 	if err != nil {
 		return nil, err
 	}
 
-	return client.CommitLanguages(clientCtx, req)
+	resp := &pb.CommitLanguagesResponse{}
+	if len(stats) == 0 {
+		return resp, nil
+	}
+
+	total := 0
+	for _, count := range stats {
+		total += count
+	}
+
+	if total == 0 {
+		return nil, grpc.Errorf(codes.Internal, "linguist stats added up to zero: %v", stats)
+	}
+
+	for lang, count := range stats {
+		l := &pb.CommitLanguagesResponse_Language{
+			Name:  lang,
+			Share: float32(100*count) / float32(total),
+			Color: linguist.Color(lang),
+		}
+		resp.Languages = append(resp.Languages, l)
+	}
+
+	sort.Sort(languageSorter(resp.Languages))
+
+	return resp, nil
+}
+
+type languageSorter []*pb.CommitLanguagesResponse_Language
+
+func (ls languageSorter) Len() int           { return len(ls) }
+func (ls languageSorter) Swap(i, j int)      { ls[i], ls[j] = ls[j], ls[i] }
+func (ls languageSorter) Less(i, j int) bool { return ls[i].Share > ls[j].Share }
+
+func lookupRevision(ctx context.Context, repoPath string, revision string) (string, error) {
+	revParse, err := helper.GitCommandReader(ctx, "--git-dir", repoPath, "rev-parse", revision)
+	if err != nil {
+		return "", err
+	}
+	defer revParse.Close()
+
+	revParseBytes, err := ioutil.ReadAll(revParse)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(revParseBytes)), nil
 }
diff --git a/internal/service/commit/testhelper_test.go b/internal/service/commit/testhelper_test.go
index 0447c3d2e..33fbf3ddd 100644
--- a/internal/service/commit/testhelper_test.go
+++ b/internal/service/commit/testhelper_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"gitlab.com/gitlab-org/gitaly/internal/linguist"
 	"gitlab.com/gitlab-org/gitaly/internal/middleware/objectdirhandler"
 	"gitlab.com/gitlab-org/gitaly/internal/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
@@ -30,6 +31,10 @@ func TestMain(m *testing.M) {
 
 func testMain(m *testing.M) int {
 	testhelper.ConfigureRuby()
+	if err := linguist.LoadColors(); err != nil {
+		log.Fatal(err)
+	}
+
 	ruby, err := rubyserver.Start()
 	if err != nil {
 		log.Fatal(err)
diff --git a/ruby/bin/ruby-cd b/ruby/bin/ruby-cd
new file mode 100755
index 000000000..9dfb91e84
--- /dev/null
+++ b/ruby/bin/ruby-cd
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+# This script lets you run `bundle exec` in one directory, and then changes into another.
+
+Dir.chdir(ARGV.shift)
+exec(*ARGV)
diff --git a/ruby/lib/gitaly_server/commit_service.rb b/ruby/lib/gitaly_server/commit_service.rb
index 26dd9a577..07302de2c 100644
--- a/ruby/lib/gitaly_server/commit_service.rb
+++ b/ruby/lib/gitaly_server/commit_service.rb
@@ -2,21 +2,6 @@ module GitalyServer
   class CommitService < Gitaly::CommitService::Service
     include Utils
 
-    def commit_languages(request, _call)
-      repo = Gitlab::Git::Repository.from_call(_call)
-      revision = request.revision unless request.revision.empty?
-
-      language_messages = repo.languages(revision).map do |language|
-        Gitaly::CommitLanguagesResponse::Language.new(
-          name: language[:label],
-          share: language[:value],
-          color: language[:color]
-        )
-      end
-
-      Gitaly::CommitLanguagesResponse.new(languages: language_messages)
-    end
-
     def commit_stats(request, _call)
       repo = Gitlab::Git::Repository.from_call(_call)
       revision = request.revision unless request.revision.empty?
diff --git a/ruby/lib/gitlab/git.rb b/ruby/lib/gitlab/git.rb
index 063d9c2c2..50935aa6c 100644
--- a/ruby/lib/gitlab/git.rb
+++ b/ruby/lib/gitlab/git.rb
@@ -1,6 +1,6 @@
 # External dependencies of Gitlab::Git
 require 'rugged'
-require 'linguist'
+require 'linguist/blob_helper'
 
 # Ruby on Rails mix-ins that GitLab::Git code relies on
 require 'active_support/core_ext/object/blank'