diff options
author | John Cai <jcai@gitlab.com> | 2020-01-08 22:03:36 +0300 |
---|---|---|
committer | John Cai <jcai@gitlab.com> | 2020-01-31 01:03:58 +0300 |
commit | b094eb36cd08b2f258bec4e8a74b5f054e442434 (patch) | |
tree | cdda297143a33955032adfe84d69b3d8a55de166 | |
parent | 9c12e4566627e19ff33726bbd4da71b4eeac70bc (diff) |
Call Hook RPCs from gitaly-hooks binaryjc-call-hook-rpc-test
Migrate gitaly-hooks from calling the ruby hooks directly to calling the
hook RPCs through gitaly via an internal socket.
-rw-r--r-- | changelogs/unreleased/jc-call-hook-rpc.yml | 5 | ||||
-rw-r--r-- | client/receive_pack.go | 2 | ||||
-rw-r--r-- | client/std_stream.go | 11 | ||||
-rw-r--r-- | client/upload_archive.go | 2 | ||||
-rw-r--r-- | client/upload_pack.go | 2 | ||||
-rw-r--r-- | cmd/gitaly-hooks/hooks.go | 139 | ||||
-rw-r--r-- | cmd/gitaly-hooks/hooks_test.go | 93 | ||||
-rwxr-xr-x | cmd/gitaly-hooks/testdata/update | 2 | ||||
-rw-r--r-- | internal/git/receivepack.go | 6 | ||||
-rw-r--r-- | internal/gitlabshell/env.go | 4 |
10 files changed, 233 insertions, 33 deletions
diff --git a/changelogs/unreleased/jc-call-hook-rpc.yml b/changelogs/unreleased/jc-call-hook-rpc.yml new file mode 100644 index 000000000..093e1d2c1 --- /dev/null +++ b/changelogs/unreleased/jc-call-hook-rpc.yml @@ -0,0 +1,5 @@ +--- +title: Call Hook RPCs from gitaly-hooks binary +merge_request: 1740 +author: +type: performance diff --git a/client/receive_pack.go b/client/receive_pack.go index 0a92a8321..7047649fb 100644 --- a/client/receive_pack.go +++ b/client/receive_pack.go @@ -28,7 +28,7 @@ func ReceivePack(ctx context.Context, conn *grpc.ClientConn, stdin io.Reader, st return stream.Send(&gitalypb.SSHReceivePackRequest{Stdin: p}) }) - return streamHandler(func() (stdoutStderrResponse, error) { + return StreamHandler(func() (StdoutStderrResponse, error) { return stream.Recv() }, func(errC chan error) { _, errRecv := io.Copy(inWriter, stdin) diff --git a/client/std_stream.go b/client/std_stream.go index cf0d5e96e..730153c3e 100644 --- a/client/std_stream.go +++ b/client/std_stream.go @@ -7,17 +7,22 @@ import ( "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" ) -type stdoutStderrResponse interface { +// StdoutStderrResponse is an interface for RPC responses that need to stream stderr and stdout +type StdoutStderrResponse interface { GetExitStatus() *gitalypb.ExitStatus GetStderr() []byte GetStdout() []byte } -func streamHandler(recv func() (stdoutStderrResponse, error), send func(chan error), stdout, stderr io.Writer) (int32, error) { +// Sender is a function that sends input data to the stream +type Sender func(chan error) + +// StreamHandler takes care of sending and receiving to and from the stream +func StreamHandler(recv func() (StdoutStderrResponse, error), send Sender, stdout, stderr io.Writer) (int32, error) { var ( exitStatus int32 err error - resp stdoutStderrResponse + resp StdoutStderrResponse ) errC := make(chan error, 1) diff --git a/client/upload_archive.go b/client/upload_archive.go index 3dd5dc05c..d87709cf2 100644 --- a/client/upload_archive.go +++ b/client/upload_archive.go @@ -28,7 +28,7 @@ func UploadArchive(ctx context.Context, conn *grpc.ClientConn, stdin io.Reader, return stream.Send(&gitalypb.SSHUploadArchiveRequest{Stdin: p}) }) - return streamHandler(func() (stdoutStderrResponse, error) { + return StreamHandler(func() (StdoutStderrResponse, error) { return stream.Recv() }, func(errC chan error) { _, errRecv := io.Copy(inWriter, stdin) diff --git a/client/upload_pack.go b/client/upload_pack.go index dcd48b6b1..41aede96a 100644 --- a/client/upload_pack.go +++ b/client/upload_pack.go @@ -28,7 +28,7 @@ func UploadPack(ctx context.Context, conn *grpc.ClientConn, stdin io.Reader, std return stream.Send(&gitalypb.SSHUploadPackRequest{Stdin: p}) }) - return streamHandler(func() (stdoutStderrResponse, error) { + return StreamHandler(func() (StdoutStderrResponse, error) { return stream.Recv() }, func(errC chan error) { _, errRecv := io.Copy(inWriter, stdin) diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go index 9f616520b..77f6b6903 100644 --- a/cmd/gitaly-hooks/hooks.go +++ b/cmd/gitaly-hooks/hooks.go @@ -4,16 +4,23 @@ import ( "context" "errors" "fmt" + "io" "log" "os" "os/exec" "path/filepath" + "strconv" "github.com/BurntSushi/toml" + gitalyauth "gitlab.com/gitlab-org/gitaly/auth" + "gitlab.com/gitlab-org/gitaly/client" "gitlab.com/gitlab-org/gitaly/internal/command" "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/gitlabshell" gitalylog "gitlab.com/gitlab-org/gitaly/internal/log" + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/streamio" + "google.golang.org/grpc" ) func main() { @@ -36,17 +43,22 @@ func main() { os.Exit(status) } - gitalyRubyDir := os.Getenv("GITALY_RUBY_DIR") - if gitalyRubyDir == "" { - logger.Fatal(errors.New("GITALY_RUBY_DIR not set")) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gitalySocket, ok := os.LookupEnv("GITALY_SOCKET") + if !ok { + logger.Fatal(errors.New("GITALY_SOCKET not set")) } - rubyHookPath := filepath.Join(gitalyRubyDir, "gitlab-shell", "hooks", subCmd) + conn, err := client.Dial("unix://"+gitalySocket, dialOpts()) + if err != nil { + logger.Fatalf("error when dialing: %v", err) + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + client := gitalypb.NewHookServiceClient(conn) - var hookCmd *exec.Cmd + var hookStatus int32 switch subCmd { case "update": @@ -54,23 +66,120 @@ func main() { if len(args) != 3 { logger.Fatal(errors.New("update hook missing required arguments")) } + ref, oldValue, newValue := args[0], args[1], args[2] + + req := &gitalypb.UpdateHookRequest{ + Repository: &gitalypb.Repository{ + StorageName: os.Getenv("GL_REPO_STORAGE"), + RelativePath: os.Getenv("GL_REPO_RELATIVE_PATH"), + GlRepository: os.Getenv("GL_REPOSITORY"), + }, + KeyId: os.Getenv("GL_ID"), + Ref: []byte(ref), + OldValue: oldValue, + NewValue: newValue, + } - hookCmd = exec.Command(rubyHookPath, args...) - case "pre-receive", "post-receive": - hookCmd = exec.Command(rubyHookPath) + stream, err := client.UpdateHook(ctx, req) + if err != nil { + logger.Fatalf("error when starting command for %v: %v", subCmd, err) + } + if hookStatus, err = sendAndRecv(stream, new(gitalypb.UpdateHookResponse), nil, os.Stdout, os.Stderr); err != nil { + logger.Fatalf("error when receiving data for %v: %v", subCmd, err) + } + case "pre-receive": + stream, err := client.PreReceiveHook(ctx) + if err != nil { + logger.Fatalf("error when getting stream client for %v: %v", subCmd, err) + } + + if err := stream.Send(&gitalypb.PreReceiveHookRequest{ + Repository: &gitalypb.Repository{ + StorageName: os.Getenv("GL_REPO_STORAGE"), + RelativePath: os.Getenv("GL_REPO_RELATIVE_PATH"), + GlRepository: os.Getenv("GL_REPOSITORY"), + }, + KeyId: os.Getenv("GL_ID"), + Protocol: os.Getenv("GL_PROTOCOL"), + }); err != nil { + logger.Fatalf("error when sending request for %v: %v", subCmd, err) + } + + f := sendFunc(streamio.NewWriter(func(p []byte) error { + return stream.Send(&gitalypb.PreReceiveHookRequest{Stdin: p}) + }), stream, os.Stdin) + + if hookStatus, err = sendAndRecv(stream, new(gitalypb.PreReceiveHookResponse), f, os.Stdout, os.Stderr); err != nil { + logger.Fatalf("error when receiving data for %v: %v", subCmd, err) + } + case "post-receive": + stream, err := client.PostReceiveHook(ctx) + if err != nil { + logger.Fatalf("error when getting stream client for %v: %v", subCmd, err) + } + + if err := stream.Send(&gitalypb.PostReceiveHookRequest{ + Repository: &gitalypb.Repository{ + StorageName: os.Getenv("GL_REPO_STORAGE"), + RelativePath: os.Getenv("GL_REPO_RELATIVE_PATH"), + GlRepository: os.Getenv("GL_REPOSITORY"), + }, + KeyId: os.Getenv("GL_ID"), + GitPushOptions: gitPushOptions(), + }); err != nil { + logger.Fatalf("error when sending request for %v: %v", subCmd, err) + } + + f := sendFunc(streamio.NewWriter(func(p []byte) error { + return stream.Send(&gitalypb.PostReceiveHookRequest{Stdin: p}) + }), stream, os.Stdin) + + if hookStatus, err = sendAndRecv(stream, new(gitalypb.PostReceiveHookResponse), f, os.Stdout, os.Stderr); err != nil { + logger.Fatalf("error when receiving data for %v: %v", subCmd, err) + } default: - logger.Fatal(errors.New("hook name invalid")) + logger.Fatal(fmt.Errorf("subcommand name invalid: %v", subCmd)) } - cmd, err := command.New(ctx, hookCmd, os.Stdin, os.Stdout, os.Stderr, os.Environ()...) + os.Exit(int(hookStatus)) +} + +func gitPushOptions() []string { + gitPushOptionCount, err := strconv.Atoi(os.Getenv("GIT_PUSH_OPTION_COUNT")) if err != nil { - logger.Fatalf("error when starting command for %v: %v", rubyHookPath, err) + return []string{} + } + + var gitPushOptions []string + + for i := 0; i < gitPushOptionCount; i++ { + gitPushOptions = append(gitPushOptions, os.Getenv(fmt.Sprintf("GIT_PUSH_OPTION_%d", i))) } - if err = cmd.Wait(); err != nil { - os.Exit(1) + return gitPushOptions +} + +func dialOpts() []grpc.DialOption { + return append(client.DefaultDialOpts, grpc.WithPerRPCCredentials(gitalyauth.RPCCredentials(os.Getenv("GITALY_TOKEN")))) +} + +func sendFunc(reqWriter io.Writer, stream grpc.ClientStream, stdin io.Reader) func(errC chan error) { + return func(errC chan error) { + _, errSend := io.Copy(reqWriter, stdin) + stream.CloseSend() + errC <- errSend + } +} + +func sendAndRecv(stream grpc.ClientStream, resp client.StdoutStderrResponse, sender client.Sender, stdout, stderr io.Writer) (int32, error) { + if sender == nil { + sender = func(err chan error) {} } + return client.StreamHandler(func() (client.StdoutStderrResponse, error) { + err := stream.RecvMsg(resp) + return resp, err + }, sender, stdout, stderr) } // GitlabShellConfig contains a subset of gitlabshell's config.yml diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go index 1e2aef00f..00738b126 100644 --- a/cmd/gitaly-hooks/hooks_test.go +++ b/cmd/gitaly-hooks/hooks_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "log" + "net" "net/http" "net/http/httptest" "os" @@ -19,7 +20,9 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/command" "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git/hooks" + serverPkg "gitlab.com/gitlab-org/gitaly/internal/server" "gitlab.com/gitlab-org/gitaly/internal/testhelper" + "google.golang.org/grpc" "gopkg.in/yaml.v2" ) @@ -40,6 +43,9 @@ func TestHooksPrePostReceive(t *testing.T) { key := 1234 glRepository := "some_repo" + testRepo, _, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) defer cleanup() @@ -49,8 +55,17 @@ func TestHooksPrePostReceive(t *testing.T) { ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, changes, true, gitPushOptions...) defer ts.Close() + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) + srv, socket := runFullServer(t) + defer srv.Stop() + writeShellSecretFile(t, tempGitlabShellDir, secretToken) for _, hook := range []string{"pre-receive", "post-receive"} { @@ -65,6 +80,9 @@ func TestHooksPrePostReceive(t *testing.T) { t, glRepository, tempGitlabShellDir, + testRepo.GetStorageName(), + testRepo.GetRelativePath(), + socket, key, gitPushOptions..., ) @@ -84,16 +102,32 @@ func TestHooksUpdate(t *testing.T) { defer cleanup() writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: "http://www.example.com"}) + testRepo, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + os.Symlink(filepath.Join(config.Config.GitlabShell.Dir, "config.yml"), filepath.Join(tempGitlabShellDir, "config.yml")) + writeShellSecretFile(t, tempGitlabShellDir, "the wrong token") + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + + srv, socket := runFullServer(t) + defer srv.Stop() + require.NoError(t, os.MkdirAll(filepath.Join(tempGitlabShellDir, "hooks", "update.d"), 0755)) testhelper.MustRunCommand(t, nil, "cp", "testdata/update", filepath.Join(tempGitlabShellDir, "hooks", "update.d", "update")) + tempFilePath := filepath.Join(testRepoPath, "tempfile") refval, oldval, newval := "refval", "oldval", "newval" var stdout, stderr bytes.Buffer cmd := exec.Command("../../ruby/git-hooks/update", refval, oldval, newval) - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + cmd.Env = env(t, glRepository, tempGitlabShellDir, testRepo.GetStorageName(), testRepo.GetRelativePath(), socket, key) cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -101,9 +135,11 @@ func TestHooksUpdate(t *testing.T) { require.Empty(t, stdout.String()) require.Empty(t, stderr.String()) + require.FileExists(t, tempFilePath) + var inputs []string - f, err := os.Open("testdata/tempfile") + f, err := os.Open(tempFilePath) require.NoError(t, err) require.NoError(t, json.NewDecoder(f).Decode(&inputs)) require.Equal(t, []string{refval, oldval, newval}, inputs) @@ -118,6 +154,9 @@ func TestHooksPostReceiveFailed(t *testing.T) { tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) defer cleanup() + testRepo, _, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + // By setting the last parameter to false, the post-receive API call will // send back {"reference_counter_increased": false}, indicating something went wrong // with the call @@ -128,10 +167,20 @@ func TestHooksPostReceiveFailed(t *testing.T) { writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) writeShellSecretFile(t, tempGitlabShellDir, secretToken) + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + + srv, socket := runFullServer(t) + defer srv.Stop() + var stdout, stderr bytes.Buffer cmd := exec.Command(fmt.Sprintf("../../ruby/git-hooks/%s", "post-receive")) - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + cmd.Env = env(t, glRepository, tempGitlabShellDir, testRepo.GetStorageName(), testRepo.GetRelativePath(), socket, key) cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -153,17 +202,29 @@ func TestHooksNotAllowed(t *testing.T) { defer cleanup() ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", true) + testRepo, _, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + defer ts.Close() writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) writeShellSecretFile(t, tempGitlabShellDir, "the wrong token") + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + srv, socket := runFullServer(t) + defer srv.Stop() + var stderr, stdout bytes.Buffer cmd := exec.Command(fmt.Sprintf("../../ruby/git-hooks/%s", "pre-receive")) cmd.Stderr = &stderr cmd.Stdout = &stdout - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + cmd.Env = env(t, glRepository, tempGitlabShellDir, testRepo.GetStorageName(), testRepo.GetRelativePath(), socket, key) require.Error(t, cmd.Run()) require.Equal(t, "GitLab: 401 Unauthorized\n", stderr.String()) @@ -245,6 +306,20 @@ func TestCheckBadCreds(t *testing.T) { require.Equal(t, "FAILED. code: 401\n", stderr.String()) } +func runFullServer(t *testing.T) (*grpc.Server, string) { + server := serverPkg.NewInsecure(nil) + serverSocketPath := testhelper.GetTemporaryGitalySocketFileName() + + listener, err := net.Listen("unix", serverSocketPath) + if err != nil { + t.Fatal(err) + } + + go server.Serve(listener) + + return server, serverSocketPath +} + func handleAllowed(t *testing.T, secretToken string, key int, glRepository, changes string) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { require.NoError(t, r.ParseForm()) @@ -359,13 +434,12 @@ func writeTemporaryGitalyConfigFile(t *testing.T, tempDir string) (string, func( } } -func env(t *testing.T, glRepo, gitlabShellDir string, key int, gitPushOptions ...string) []string { - rubyDir, err := filepath.Abs("../../ruby") - require.NoError(t, err) - +func env(t *testing.T, glRepo, gitlabShellDir, glStorage, glRelativePath, gitalySocket string, key int, gitPushOptions ...string) []string { return append(append(oldEnv(t, glRepo, gitlabShellDir, key), []string{ "GITALY_BIN_DIR=testdata/gitaly-libexec", - fmt.Sprintf("GITALY_RUBY_DIR=%s", rubyDir), + fmt.Sprintf("GL_REPO_STORAGE=%s", glStorage), + fmt.Sprintf("GL_REPO_RELATIVE_PATH=%s", glRelativePath), + fmt.Sprintf("GITALY_SOCKET=%s", gitalySocket), }...), hooks.GitPushOptions(gitPushOptions)...) } @@ -378,7 +452,6 @@ func oldEnv(t *testing.T, glRepo, gitlabShellDir string, key int) []string { fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir), "GITALY_LOG_LEVEL=info", "GITALY_LOG_FORMAT=json", - fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir), }, os.Environ()...) } diff --git a/cmd/gitaly-hooks/testdata/update b/cmd/gitaly-hooks/testdata/update index a4076ec24..e982c4d17 100755 --- a/cmd/gitaly-hooks/testdata/update +++ b/cmd/gitaly-hooks/testdata/update @@ -1,4 +1,4 @@ #!/usr/bin/env ruby require 'json' -open('testdata/tempfile', 'w') { |f| f.puts(JSON.dump(ARGV)) } +open('tempfile', 'w') { |f| f.puts(JSON.dump(ARGV)) } diff --git a/internal/git/receivepack.go b/internal/git/receivepack.go index 5489ff9a5..177078523 100644 --- a/internal/git/receivepack.go +++ b/internal/git/receivepack.go @@ -3,8 +3,10 @@ package git import ( "fmt" + "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git/hooks" "gitlab.com/gitlab-org/gitaly/internal/gitlabshell" + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" ) // ReceivePackRequest abstracts away the different requests that end up @@ -13,6 +15,7 @@ type ReceivePackRequest interface { GetGlId() string GetGlUsername() string GetGlRepository() string + GetRepository() *gitalypb.Repository } // HookEnv is information we pass down to the Git hooks during @@ -22,6 +25,9 @@ func HookEnv(req ReceivePackRequest) []string { fmt.Sprintf("GL_ID=%s", req.GetGlId()), fmt.Sprintf("GL_USERNAME=%s", req.GetGlUsername()), fmt.Sprintf("GL_REPOSITORY=%s", req.GetGlRepository()), + fmt.Sprintf("GITALY_SOCKET=" + config.GitalyInternalSocketPath()), + fmt.Sprintf("GL_REPO_STORAGE=%s", req.GetRepository().GetStorageName()), + fmt.Sprintf("GL_REPO_RELATIVE_PATH=%s", req.GetRepository().GetRelativePath()), }, gitlabshell.Env()...) } diff --git a/internal/gitlabshell/env.go b/internal/gitlabshell/env.go index 79bae0646..4db3c4fd0 100644 --- a/internal/gitlabshell/env.go +++ b/internal/gitlabshell/env.go @@ -1,6 +1,8 @@ package gitlabshell -import "gitlab.com/gitlab-org/gitaly/internal/config" +import ( + "gitlab.com/gitlab-org/gitaly/internal/config" +) // Env is a helper that returns a slice with environment variables used by gitlab shell func Env() []string { |