diff options
author | Jacob Vosmaer (GitLab) <jacob@gitlab.com> | 2017-07-27 11:35:43 +0300 |
---|---|---|
committer | Andrew Newdigate <andrew@gitlab.com> | 2017-07-27 11:35:43 +0300 |
commit | 0c32842c37e5f41a1d427312f390963237ab57fa (patch) | |
tree | 43d6521277989e53ce3da795f241712b1a895c82 | |
parent | 0ffed4e45f9ced7aec0d15187432d85d68295f7d (diff) |
Implement CommitService.CommitLanguages
-rw-r--r-- | .gitlab-ci.yml | 1 | ||||
-rw-r--r-- | CHANGELOG.md | 5 | ||||
-rw-r--r-- | Makefile | 2 | ||||
-rw-r--r-- | README.md | 9 | ||||
-rw-r--r-- | cmd/gitaly/main.go | 44 | ||||
-rw-r--r-- | config.toml.example | 4 | ||||
-rw-r--r-- | internal/config/config.go | 1 | ||||
-rw-r--r-- | internal/config/ruby.go | 6 | ||||
-rw-r--r-- | internal/rubyserver/rubyserver.go | 99 | ||||
-rw-r--r-- | internal/service/commit/languages.go | 23 | ||||
-rw-r--r-- | internal/service/commit/languages_test.go | 68 | ||||
-rw-r--r-- | internal/service/commit/server.go | 5 | ||||
-rw-r--r-- | internal/service/commit/testhelper_test.go | 9 | ||||
-rw-r--r-- | internal/supervisor/supervisor.go | 39 | ||||
-rw-r--r-- | internal/testhelper/testhelper.go | 15 | ||||
-rw-r--r-- | ruby/Gemfile | 5 | ||||
-rw-r--r-- | ruby/Gemfile.lock | 58 | ||||
-rw-r--r-- | ruby/README.md | 29 | ||||
-rwxr-xr-x | ruby/bin/gitaly-ruby | 45 | ||||
-rw-r--r-- | ruby/lib/gitaly_server.rb | 15 | ||||
-rw-r--r-- | ruby/lib/gitaly_server/commit_service.rb | 28 |
21 files changed, 485 insertions, 25 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8647e9ef..9c422cad2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ verify: .test_template: &test_definition stage: build_test script: + - (cd ruby && bundle install) - go version - git version - make test diff --git a/CHANGELOG.md b/CHANGELOG.md index 94eb7cf39..4f84fa348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Gitaly changelog +UNRELEASED + +- Implement CommitService.CommitLanguages, add gitaly-ruby + https://gitlab.com/gitlab-org/gitaly/merge_requests/210 + v0.25.0 - Implement FindAllTags RPC @@ -9,6 +9,7 @@ export TEST_REPO_STORAGE_PATH := $(BUILD_DIR)/internal/testhelper/testdata/data TEST_REPO := $(TEST_REPO_STORAGE_PATH)/gitlab-test.git INSTALL_DEST_DIR := $(DESTDIR)$(PREFIX)/bin/ COVERAGE_DIR := $(TARGET_DIR)/cover +export GITALY_TEST_RUBY_DIR := $(BUILD_DIR)/ruby BUILDTIME = $(shell date -u +%Y%m%d.%H%M%S) VERSION_PREFIXED = $(shell git describe) @@ -44,6 +45,7 @@ $(TARGET_SETUP): .PHONY: build build: $(TARGET_SETUP) $(GOVENDOR) + cd ruby && bundle install go install $(LDFLAGS) $(COMMAND_PACKAGES) cp $(foreach cmd,$(COMMANDS),$(BIN_BUILD_DIR)/$(cmd)) $(BUILD_DIR)/ @@ -55,12 +55,9 @@ The progress of Gitaly's endpoint migrations is tracked via the [**Migration Boa ## Installation -Gitaly requires Go 1.8 or newer. To install into `/usr/local/bin`, -run: - -``` -make install -``` +Gitaly requires Go 1.8 or newer and Ruby 2.3. Run `make` to download +and compile Ruby dependencies, and to compile the Gitaly Go +executable. ## Configuration diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go index 1364c0777..c57aa2dcc 100644 --- a/cmd/gitaly/main.go +++ b/cmd/gitaly/main.go @@ -9,12 +9,14 @@ import ( log "github.com/Sirupsen/logrus" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/connectioncounter" + "gitlab.com/gitlab-org/gitaly/internal/rubyserver" "gitlab.com/gitlab-org/gitaly/internal/server" "gitlab.com/gitlab-org/gitaly/internal/version" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -121,17 +123,6 @@ func main() { listeners = append(listeners, connectioncounter.New("tcp", l)) } - server := server.New() - - serverError := make(chan error, len(listeners)) - for _, listener := range listeners { - // Must pass the listener as a function argument because there is a race - // between 'go' and 'for'. - go func(l net.Listener) { - serverError <- server.Serve(l) - }(listener) - } - if config.Config.PrometheusListenAddr != "" { log.WithField("address", config.Config.PrometheusListenAddr).Info("Starting prometheus listener") promMux := http.NewServeMux() @@ -141,7 +132,7 @@ func main() { }() } - log.Fatal(<-serverError) + log.WithError(run(listeners)).Fatal("shutting down") } func createUnixListener(socketPath string) (net.Listener, error) { @@ -151,3 +142,28 @@ func createUnixListener(socketPath string) (net.Listener, error) { l, err := net.Listen("unix", socketPath) return connectioncounter.New("unix", l), err } + +// Inside here we can use deferred functions. This is needed because +// log.Fatal bypasses deferred functions. +func run(listeners []net.Listener) error { + ruby, err := rubyserver.Start() + if err != nil { + // TODO: this will be a fatal error in the future + log.WithError(err).Warn("failed to start ruby service") + } else { + defer ruby.Stop() + } + + server := server.New() + + serverError := make(chan error, len(listeners)) + for _, listener := range listeners { + // Must pass the listener as a function argument because there is a race + // between 'go' and 'for'. + go func(l net.Listener) { + serverError <- server.Serve(l) + }(listener) + } + + return <-serverError +} diff --git a/config.toml.example b/config.toml.example index ff867cdec..7547c61e8 100644 --- a/config.toml.example +++ b/config.toml.example @@ -34,3 +34,7 @@ path = "/home/git/repositories" # # You can optionally configure Gitaly to record histogram latencies on GRPC method calls # [prometheus] # grpc_latency_buckets = [0.001, 0.005, 0.025, 0.1, 0.5, 1.0, 10.0, 30.0, 60.0, 300.0, 1500.0] + +[gitaly-ruby] +# The directory where gitaly-ruby is installed +dir = "/home/git/gitaly/ruby" diff --git a/internal/config/config.go b/internal/config/config.go index 113047e22..a7f1ee421 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,6 +25,7 @@ type config struct { Logging Logging `toml:"logging" envconfig:"logging"` Prometheus Prometheus `toml:"prometheus"` Auth Auth `toml:"auth"` + Ruby Ruby `toml:"gitaly-ruby"` } // Git contains the settings for the Git executable diff --git a/internal/config/ruby.go b/internal/config/ruby.go new file mode 100644 index 000000000..71ad92d56 --- /dev/null +++ b/internal/config/ruby.go @@ -0,0 +1,6 @@ +package config + +// Ruby contains setting for Ruby worker processes +type Ruby struct { + Dir string `toml:"dir"` +} diff --git a/internal/rubyserver/rubyserver.go b/internal/rubyserver/rubyserver.go new file mode 100644 index 000000000..6f79831ac --- /dev/null +++ b/internal/rubyserver/rubyserver.go @@ -0,0 +1,99 @@ +package rubyserver + +import ( + "io/ioutil" + "net" + "os" + "path" + "path/filepath" + "strconv" + "sync" + "time" + + "gitlab.com/gitlab-org/gitaly/internal/config" + "gitlab.com/gitlab-org/gitaly/internal/helper" + "gitlab.com/gitlab-org/gitaly/internal/supervisor" + + pb "gitlab.com/gitlab-org/gitaly-proto/go" + + log "github.com/Sirupsen/logrus" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +const ( + repoPathHeader = "gitaly-repo-path" +) + +var ( + socketPath string + + lazyInit sync.Once + + // ConnectTimeout is the timeout for establishing a connection to the gitaly-ruby process. + ConnectTimeout = 10 * time.Second +) + +func init() { + timeout64, err := strconv.ParseInt(os.Getenv("GITALY_RUBY_CONNECT_TIMEOUT"), 10, 32) + if err == nil && timeout64 > 0 { + ConnectTimeout = time.Duration(timeout64) * time.Second + } +} + +func prepareSocketPath() { + // The socket path must be short-ish because listen(2) fails on long + // socket paths. We hope/expect that ioutil.TempDir creates a directory + // that is not too deep. We need a directory, not a tempfile, because we + // will later want to set its permissions to 0700. The permission change + // is done in the Ruby child process. + socketDir, err := ioutil.TempDir("", "gitaly-ruby") + if err != nil { + log.Fatalf("create ruby server socket directory: %v", err) + } + socketPath = path.Join(filepath.Clean(socketDir), "socket") +} + +// Start spawns the Ruby server. +func Start() (*supervisor.Process, error) { + lazyInit.Do(prepareSocketPath) + + args := []string{"bundle", "exec", "bin/gitaly-ruby", socketPath} + return supervisor.New(nil, args, config.Config.Ruby.Dir) +} + +// CommitServiceClient returns a CommitServiceClient instance that is +// configured to connect to the running Ruby server. This assumes Start() +// has been called already. +func CommitServiceClient(ctx context.Context) (pb.CommitServiceClient, error) { + conn, err := newConnection(ctx) + return pb.NewCommitServiceClient(conn), err +} + +func newConnection(ctx context.Context) (*grpc.ClientConn, error) { + dialCtx, cancel := context.WithTimeout(ctx, ConnectTimeout) + defer cancel() + return grpc.DialContext(dialCtx, socketPath, dialOptions()...) +} + +func dialOptions() []grpc.DialOption { + return []grpc.DialOption{ + grpc.WithBlock(), // With this we get retries. Without, connections fail fast. + grpc.WithInsecure(), + grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { + return net.DialTimeout("unix", addr, timeout) + }), + } +} + +// SetHeaders adds headers that tell gitaly-ruby the full path to the repository. +func SetHeaders(ctx context.Context, repo *pb.Repository) (context.Context, error) { + repoPath, err := helper.GetRepoPath(repo) + if err != nil { + return nil, err + } + + newCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs(repoPathHeader, repoPath)) + return newCtx, nil +} diff --git a/internal/service/commit/languages.go b/internal/service/commit/languages.go new file mode 100644 index 000000000..493f2ac0d --- /dev/null +++ b/internal/service/commit/languages.go @@ -0,0 +1,23 @@ +package commit + +import ( + "gitlab.com/gitlab-org/gitaly/internal/rubyserver" + + pb "gitlab.com/gitlab-org/gitaly-proto/go" + + "golang.org/x/net/context" +) + +func (*server) CommitLanguages(ctx context.Context, req *pb.CommitLanguagesRequest) (*pb.CommitLanguagesResponse, error) { + client, err := rubyserver.CommitServiceClient(ctx) + if err != nil { + return nil, err + } + + clientCtx, err := rubyserver.SetHeaders(ctx, req.GetRepository()) + if err != nil { + return nil, err + } + + return client.CommitLanguages(clientCtx, req) +} diff --git a/internal/service/commit/languages_test.go b/internal/service/commit/languages_test.go new file mode 100644 index 000000000..d72af70b4 --- /dev/null +++ b/internal/service/commit/languages_test.go @@ -0,0 +1,68 @@ +package commit + +import ( + "context" + "testing" + + pb "gitlab.com/gitlab-org/gitaly-proto/go" + + "github.com/stretchr/testify/require" +) + +func TestLanguages(t *testing.T) { + client := newCommitServiceClient(t) + request := &pb.CommitLanguagesRequest{ + Repository: testRepo, + Revision: []byte("e63f41fe459e62e1228fcef60d7189127aeba95a"), + } + + resp, err := client.CommitLanguages(context.Background(), request) + require.NoError(t, err) + + require.NotZero(t, len(resp.Languages), "number of languages in response") + + expectedLanguages := []pb.CommitLanguagesResponse_Language{ + {Name: "Ruby", Share: 66, Color: "#701516"}, + {Name: "JavaScript", Share: 22, Color: "#f1e05a"}, + {Name: "HTML", Share: 7, Color: "#e44b23"}, + {Name: "CoffeeScript", Share: 2, Color: "#244776"}, + } + + for i, el := range expectedLanguages { + actualLanguage := resp.Languages[i] + require.True(t, languageEqual(&el, actualLanguage), "language %+v not equal to %+v", el, *actualLanguage) + } +} + +func languageEqual(expected, actual *pb.CommitLanguagesResponse_Language) bool { + if expected.Name != actual.Name { + return false + } + if expected.Color != actual.Color { + return false + } + if (expected.Share-actual.Share)*(expected.Share-actual.Share) >= 1.0 { + return false + } + return true +} + +func TestLanguagesEmptyRevision(t *testing.T) { + client := newCommitServiceClient(t) + request := &pb.CommitLanguagesRequest{ + Repository: testRepo, + } + + resp, err := client.CommitLanguages(context.Background(), request) + require.NoError(t, err) + + require.NotZero(t, len(resp.Languages), "number of languages in response") + + foundRuby := false + for _, l := range resp.Languages { + if l.Name == "Ruby" { + foundRuby = true + } + } + require.True(t, foundRuby, "expected to find Ruby as a language on HEAD") +} diff --git a/internal/service/commit/server.go b/internal/service/commit/server.go index 0a72c88be..8f47497a7 100644 --- a/internal/service/commit/server.go +++ b/internal/service/commit/server.go @@ -2,15 +2,10 @@ package commit import ( pb "gitlab.com/gitlab-org/gitaly-proto/go" - "golang.org/x/net/context" ) type server struct{} -func (s *server) CommitLanguages(ctx context.Context, r *pb.CommitLanguagesRequest) (*pb.CommitLanguagesResponse, error) { - return nil, nil -} - // NewServer creates a new instance of a grpc CommitServiceServer func NewServer() pb.CommitServiceServer { return &server{} diff --git a/internal/service/commit/testhelper_test.go b/internal/service/commit/testhelper_test.go index f7ddf491c..92ea9d8a0 100644 --- a/internal/service/commit/testhelper_test.go +++ b/internal/service/commit/testhelper_test.go @@ -9,6 +9,7 @@ import ( log "github.com/Sirupsen/logrus" + "gitlab.com/gitlab-org/gitaly/internal/rubyserver" "gitlab.com/gitlab-org/gitaly/internal/testhelper" "google.golang.org/grpc" @@ -27,7 +28,15 @@ var ( func TestMain(m *testing.M) { testRepo = testhelper.TestRepository() + testhelper.ConfigureRuby() + ruby, err := rubyserver.Start() + if err != nil { + log.WithError(err).Fatal("ruby spawn failed") + } + os.Exit(func() int { + defer ruby.Stop() + os.Remove(serverSocketPath) server := runCommitServer(m) defer func() { diff --git a/internal/supervisor/supervisor.go b/internal/supervisor/supervisor.go new file mode 100644 index 000000000..4f9bba63e --- /dev/null +++ b/internal/supervisor/supervisor.go @@ -0,0 +1,39 @@ +package supervisor + +import ( + "fmt" + "os" + "os/exec" +) + +// Process represents a running process. +type Process struct { + cmd *exec.Cmd +} + +// New creates a new proces instance. +func New(env []string, args []string, dir string) (*Process, error) { + if len(args) < 1 { + return nil, fmt.Errorf("need at least one argument") + } + + cmd := exec.Command(args[0], args[1:]...) + cmd.Env = env + cmd.Dir = dir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + // TODO: spawn goroutine that watches (restarts) this process. + return &Process{cmd: cmd}, cmd.Start() +} + +// Stop terminates the process. +func (p *Process) Stop() { + if p == nil || p.cmd == nil || p.cmd.Process == nil { + return + } + + process := p.cmd.Process + process.Kill() + process.Wait() +} diff --git a/internal/testhelper/testhelper.go b/internal/testhelper/testhelper.go index 091df89b9..869aee43d 100644 --- a/internal/testhelper/testhelper.go +++ b/internal/testhelper/testhelper.go @@ -176,3 +176,18 @@ func GetTemporaryGitalySocketFileName() string { return name } + +// ConfigureRuby configures Ruby settings for test purposes at run time. +func ConfigureRuby() { + if dir := os.Getenv("GITALY_TEST_RUBY_DIR"); len(dir) > 0 { + // Sometimes runtime.Caller is unreliable. This environment variable provides a bypass. + config.Config.Ruby.Dir = dir + return + } + + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + log.Fatal("Could not get caller info") + } + config.Config.Ruby.Dir = path.Join(path.Dir(currentFile), "../../ruby") +} diff --git a/ruby/Gemfile b/ruby/Gemfile new file mode 100644 index 000000000..cb634a49a --- /dev/null +++ b/ruby/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +gem 'github-linguist', '~> 4.7.0', require: 'linguist' + +gem 'gitaly', '~> 0.19.0' diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock new file mode 100644 index 000000000..6912ce0da --- /dev/null +++ b/ruby/Gemfile.lock @@ -0,0 +1,58 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.5.1) + public_suffix (~> 2.0, >= 2.0.2) + charlock_holmes (0.7.3) + escape_utils (1.1.1) + faraday (0.12.1) + multipart-post (>= 1.2, < 3) + gitaly (0.19.0) + google-protobuf (~> 3.1) + grpc (~> 1.0) + github-linguist (4.7.6) + charlock_holmes (~> 0.7.3) + escape_utils (~> 1.1.0) + mime-types (>= 1.19) + rugged (>= 0.23.0b) + google-protobuf (3.3.0) + googleauth (0.5.1) + faraday (~> 0.9) + jwt (~> 1.4) + logging (~> 2.0) + memoist (~> 0.12) + multi_json (~> 1.11) + os (~> 0.9) + signet (~> 0.7) + grpc (1.4.1) + google-protobuf (~> 3.1) + googleauth (~> 0.5.1) + jwt (1.5.6) + little-plugger (1.1.4) + logging (2.2.2) + little-plugger (~> 1.1) + multi_json (~> 1.10) + memoist (0.16.0) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) + multi_json (1.12.1) + multipart-post (2.0.0) + os (0.9.6) + public_suffix (2.0.5) + rugged (0.26.0) + signet (0.7.3) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (~> 1.5) + multi_json (~> 1.10) + +PLATFORMS + ruby + +DEPENDENCIES + gitaly (~> 0.19.0) + github-linguist (~> 4.7.0) + +BUNDLED WITH + 1.15.0 diff --git a/ruby/README.md b/ruby/README.md new file mode 100644 index 000000000..4adf74208 --- /dev/null +++ b/ruby/README.md @@ -0,0 +1,29 @@ +# `gitaly-ruby` + +`gitaly-ruby` is a 'sidecar' process for the main Gitaly service. It +allows us to run legacy Ruby application code for which it would be +too risky or even infeasible to port it to Go. It will also speed up +the Gitaly migration project. + +## Architecture + +Gitaly-ruby is a minimal Ruby gRPC service which should only receive +requests from its (Go) parent Gitaly process. The Gitaly parent +handles authentication, logging, metrics, configuration file parsing +etc. + +The Gitaly parent is also responsible for starting and (if necessary) +restarting Gitaly-ruby. + +## Authentication + +Gitaly-ruby listens on a Unix socket in a temporary directory with +mode 0700. It runs as the same user as the Gitaly parent process. + +## Testing + +All tests for code in Gitaly-ruby go through the parent Gitaly process +for two reasons. Firstly, testing through the parent proves that the +Ruby code under test is reachable. Secondly, testing through the +parent will make it easier to create a Go implementation in the parent +if we ever want to do that. diff --git a/ruby/bin/gitaly-ruby b/ruby/bin/gitaly-ruby new file mode 100755 index 000000000..184610e5b --- /dev/null +++ b/ruby/bin/gitaly-ruby @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +require 'grpc' + +require_relative '../lib/gitaly_server.rb' + +def main + if ARGV.length != 1 + abort "Usage: #{$0} /path/to/socket" + end + + start_parent_watcher + + socket_path = ARGV.first + FileUtils.rm_f(socket_path) + socket_dir = File.dirname(socket_path) + FileUtils.mkdir_p(socket_dir) + File.chmod(0700, socket_dir) + + s = GRPC::RpcServer.new + port = 'unix:' + socket_path + s.add_http2_port(port, :this_port_is_insecure) + GRPC.logger.info("... running insecurely on #{port}") + + GitalyServer.register_handlers(s) + + s.run_till_terminated +end + +def start_parent_watcher + Thread.new do + loop do + if Process.ppid == 1 + # If our parent is PID 1, our original parent is gone. Self-terminate. + Process.kill(9, Process.pid) + end + + sleep 1 + end + end +end + +main diff --git a/ruby/lib/gitaly_server.rb b/ruby/lib/gitaly_server.rb new file mode 100644 index 000000000..de8c2da3e --- /dev/null +++ b/ruby/lib/gitaly_server.rb @@ -0,0 +1,15 @@ +require 'gitaly' + +require_relative 'gitaly_server/commit_service.rb' + +module GitalyServer + REPO_PATH_HEADER = 'gitaly-repo-path'.freeze + + def self.repo_path(_call) + _call.metadata.fetch(REPO_PATH_HEADER) + end + + def self.register_handlers(server) + server.handle(CommitService.new) + end +end diff --git a/ruby/lib/gitaly_server/commit_service.rb b/ruby/lib/gitaly_server/commit_service.rb new file mode 100644 index 000000000..8811dc614 --- /dev/null +++ b/ruby/lib/gitaly_server/commit_service.rb @@ -0,0 +1,28 @@ +require 'linguist' +require 'rugged' + +module GitalyServer + class CommitService < Gitaly::CommitService::Service + def commit_languages(request, _call) + rugged_repo = Rugged::Repository.new(GitalyServer.repo_path(_call)) + revision = request.revision + revision = rugged_repo.head.target_id if revision.empty? + + languages = Linguist::Repository.new(rugged_repo, revision).languages + + total = languages.values.inject(0, :+) + language_messages = languages.map do |name, share| + Gitaly::CommitLanguagesResponse::Language.new( + name: name, + share: (share.to_f * 100 / total).round(2), + color: Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" + ) + end + language_messages.sort! do |x, y| + y.share <=> x.share + end + + Gitaly::CommitLanguagesResponse.new(languages: language_messages) + end + end +end |