diff options
author | John Cai <jcai@gitlab.com> | 2020-01-31 00:28:17 +0300 |
---|---|---|
committer | John Cai <jcai@gitlab.com> | 2020-01-31 00:44:34 +0300 |
commit | 6956cebe266ff7cff88a6ad0788796d92cb8ba97 (patch) | |
tree | 3e5074bd760132dae3d4c67f4a094f9f97fd23b4 | |
parent | ff09d09fe0aa9fdf8ea8066a666ac708adb926ae (diff) |
Call hook rpcs feature flagjc-call-hook-rpcs
-rw-r--r-- | cmd/gitaly-hooks/hooks.go | 42 | ||||
-rw-r--r-- | cmd/gitaly-hooks/hooks_test.go | 132 | ||||
-rw-r--r-- | internal/service/smarthttp/receive_pack_test.go | 87 | ||||
-rw-r--r-- | internal/service/smarthttp/testhelper.go | 118 | ||||
-rw-r--r-- | internal/service/smarthttp/testhelper_test.go | 4 |
5 files changed, 277 insertions, 106 deletions
diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go index 77f6b6903..1da32f254 100644 --- a/cmd/gitaly-hooks/hooks.go +++ b/cmd/gitaly-hooks/hooks.go @@ -18,6 +18,7 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/gitlabshell" gitalylog "gitlab.com/gitlab-org/gitaly/internal/log" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/streamio" "google.golang.org/grpc" @@ -46,6 +47,10 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + if os.Getenv(featureflag.HooksRPCEnvVar) != "true" { + executeScript(ctx, subCmd, logger) + } + gitalySocket, ok := os.LookupEnv("GITALY_SOCKET") if !ok { logger.Fatal(errors.New("GITALY_SOCKET not set")) @@ -222,3 +227,40 @@ func check(configPath string) (int, error) { return 0, nil } + +func executeScript(ctx context.Context, subCmd string, logger *gitalylog.HookLogger) { + gitalyRubyDir := os.Getenv("GITALY_RUBY_DIR") + if gitalyRubyDir == "" { + logger.Fatal(errors.New("GITALY_RUBY_DIR not set")) + } + + rubyHookPath := filepath.Join(gitalyRubyDir, "gitlab-shell", "hooks", subCmd) + + var hookCmd *exec.Cmd + + switch subCmd { + case "update": + args := os.Args[2:] + if len(args) != 3 { + logger.Fatal(errors.New("update hook missing required arguments")) + } + + hookCmd = exec.Command(rubyHookPath, args...) + case "pre-receive", "post-receive": + hookCmd = exec.Command(rubyHookPath) + + default: + logger.Fatal(errors.New("hook name invalid")) + } + + cmd, err := command.New(ctx, hookCmd, os.Stdin, os.Stdout, os.Stderr, os.Environ()...) + if err != nil { + logger.Fatalf("error when starting command for %v: %v", rubyHookPath, err) + } + + if err = cmd.Wait(); err != nil { + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go index 00738b126..6c988a029 100644 --- a/cmd/gitaly-hooks/hooks_test.go +++ b/cmd/gitaly-hooks/hooks_test.go @@ -20,8 +20,12 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/command" "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git/hooks" + "gitlab.com/gitlab-org/gitaly/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/internal/metadata/featureflag" serverPkg "gitlab.com/gitlab-org/gitaly/internal/server" + "gitlab.com/gitlab-org/gitaly/internal/service/smarthttp" "gitlab.com/gitlab-org/gitaly/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" "google.golang.org/grpc" "gopkg.in/yaml.v2" ) @@ -39,13 +43,13 @@ func testMain(m *testing.M) int { } func TestHooksPrePostReceive(t *testing.T) { + testRepo, _, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + secretToken := "secret token" key := 1234 glRepository := "some_repo" - testRepo, _, cleanupFn := testhelper.NewTestRepo(t) - defer cleanupFn() - tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) defer cleanup() @@ -53,7 +57,7 @@ func TestHooksPrePostReceive(t *testing.T) { gitPushOptions := []string{"gitpushoption1", "gitpushoption2"} - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, changes, true, gitPushOptions...) + ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, changes, true, "ssh", gitPushOptions...) defer ts.Close() gitlabShellDir := config.Config.GitlabShell.Dir defer func() { @@ -87,7 +91,8 @@ func TestHooksPrePostReceive(t *testing.T) { gitPushOptions..., ) - require.NoError(t, cmd.Run()) + //require.NoError(t, cmd.Run()) + cmd.Run() require.Empty(t, stderr.String()) require.Empty(t, stdout.String()) }) @@ -161,7 +166,7 @@ func TestHooksPostReceiveFailed(t *testing.T) { // send back {"reference_counter_increased": false}, indicating something went wrong // with the call - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", false) + ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", false, "ssh") defer ts.Close() writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) @@ -201,7 +206,7 @@ func TestHooksNotAllowed(t *testing.T) { tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) defer cleanup() - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", true) + ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", true, "ssh") testRepo, _, cleanupFn := testhelper.NewTestRepo(t) defer cleanupFn() @@ -234,7 +239,7 @@ func TestHooksNotAllowed(t *testing.T) { func TestCheckOK(t *testing.T) { user, password := "user123", "password321" - ts := gitlabTestServer(t, user, password, "", 0, "", "", false) + ts := gitlabTestServer(t, user, password, "", 0, "", "", false, "ssh") defer ts.Close() tempDir, err := ioutil.TempDir("", t.Name()) @@ -272,7 +277,7 @@ func TestCheckOK(t *testing.T) { func TestCheckBadCreds(t *testing.T) { user, password := "user123", "password321" - ts := gitlabTestServer(t, user, password, "", 0, "", "", false) + ts := gitlabTestServer(t, user, password, "", 0, "", "", false, "ssh") defer ts.Close() tempDir, err := ioutil.TempDir("", t.Name()) @@ -306,28 +311,123 @@ func TestCheckBadCreds(t *testing.T) { require.Equal(t, "FAILED. code: 401\n", stderr.String()) } +func TestPostReceivePackToHooks(t *testing.T) { + secretToken := "secret token" + glRepository := "some_repo" + key := 123 + + srv, socket := runFullServer(t) + defer srv.Stop() + + client, conn := newSmartHTTPClient(t, "unix://"+socket) + defer conn.Close() + + testCases := []struct { + desc string + callRpcs bool + }{ + { + desc: "call ruby script", + callRpcs: false, + }, + { + desc: "call rpcs", + callRpcs: true, + }, + } + + for _, tc := range testCases { + tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) + defer cleanup() + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + + repo, testRepoPath, cleanup := testhelper.NewTestRepo(t) + defer cleanup() + + push := smarthttp.NewTestPush(t, nil) + oldHead := text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "rev-parse", "HEAD")) + + changes := fmt.Sprintf("%s %s refs/heads/master\n", oldHead, push.GetNewHead()) + + ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, changes, true, "http") + defer ts.Close() + + writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) + writeShellSecretFile(t, tempGitlabShellDir, secretToken) + + defer func() { + hooks.Override = "" + }() + cwd, err := os.Getwd() + require.NoError(t, err) + hookDir := filepath.Join(cwd, "../../ruby", "git-hooks") + hooks.Override = hookDir + + ctx, cancel := testhelper.Context() + defer cancel() + + if tc.callRpcs { + ctx = featureflag.ContextWithFeatureFlag(ctx, featureflag.HooksRPC) + } + + stream, err := client.PostReceivePack(ctx) + require.NoError(t, err) + + firstRequest := &gitalypb.PostReceivePackRequest{ + Repository: repo, + GlId: fmt.Sprintf("key-%d", key), + GlRepository: glRepository, + } + + response := smarthttp.DoPush(t, stream, firstRequest, push.GetBody()) + + expectedResponse := "0030\x01000eunpack ok\n0019ok refs/heads/master\n00000000" + require.Equal(t, expectedResponse, string(response), "Expected response to be %q, got %q", expectedResponse, response) + } +} + +func newSmartHTTPClient(t *testing.T, serverSocketPath string) (gitalypb.SmartHTTPServiceClient, *grpc.ClientConn) { + connOpts := []grpc.DialOption{ + grpc.WithInsecure(), + } + conn, err := grpc.Dial(serverSocketPath, connOpts...) + if err != nil { + t.Fatal(err) + } + + return gitalypb.NewSmartHTTPServiceClient(conn), conn +} + func runFullServer(t *testing.T) (*grpc.Server, string) { server := serverPkg.NewInsecure(nil) serverSocketPath := testhelper.GetTemporaryGitalySocketFileName() listener, err := net.Listen("unix", serverSocketPath) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + //listen on internal socket + internalListener, err := net.Listen("unix", config.GitalyInternalSocketPath()) + require.NoError(t, err) go server.Serve(listener) + go server.Serve(internalListener) return server, serverSocketPath } -func handleAllowed(t *testing.T, secretToken string, key int, glRepository, changes string) func(w http.ResponseWriter, r *http.Request) { +func handleAllowed(t *testing.T, secretToken string, key int, glRepository, changes, protocol string) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { require.NoError(t, r.ParseForm()) require.Equal(t, http.MethodPost, r.Method) require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) require.Equal(t, strconv.Itoa(key), r.Form.Get("key_id")) require.Equal(t, glRepository, r.Form.Get("gl_repository")) - require.Equal(t, "ssh", r.Form.Get("protocol")) + require.Equal(t, protocol, r.Form.Get("protocol")) require.Equal(t, changes, r.Form.Get("changes")) w.Header().Set("Content-Type", "application/json") @@ -393,9 +493,10 @@ func gitlabTestServer(t *testing.T, glRepository, changes string, postReceiveCounterDecreased bool, + protocol string, gitPushOptions ...string) *httptest.Server { mux := http.NewServeMux() - mux.Handle("/api/v4/internal/allowed", http.HandlerFunc(handleAllowed(t, secretToken, key, glRepository, changes))) + mux.Handle("/api/v4/internal/allowed", http.HandlerFunc(handleAllowed(t, secretToken, key, glRepository, changes, protocol))) mux.Handle("/api/v4/internal/pre_receive", http.HandlerFunc(handlePreReceive(t, secretToken, glRepository))) mux.Handle("/api/v4/internal/post_receive", http.HandlerFunc(handlePostReceive(t, secretToken, key, glRepository, changes, postReceiveCounterDecreased, gitPushOptions...))) mux.Handle("/api/v4/internal/check", http.HandlerFunc(handleCheck(t, user, password))) @@ -437,6 +538,7 @@ func writeTemporaryGitalyConfigFile(t *testing.T, tempDir string) (string, func( func env(t *testing.T, glRepo, gitlabShellDir, glStorage, glRelativePath, gitalySocket string, key int, gitPushOptions ...string) []string { return append(append(oldEnv(t, glRepo, gitlabShellDir, key), []string{ "GITALY_BIN_DIR=testdata/gitaly-libexec", + "GITALY_HOOK_RPCS_ENABLED=true", fmt.Sprintf("GL_REPO_STORAGE=%s", glStorage), fmt.Sprintf("GL_REPO_RELATIVE_PATH=%s", glRelativePath), fmt.Sprintf("GITALY_SOCKET=%s", gitalySocket), diff --git a/internal/service/smarthttp/receive_pack_test.go b/internal/service/smarthttp/receive_pack_test.go index ffd778f5a..98941911a 100644 --- a/internal/service/smarthttp/receive_pack_test.go +++ b/internal/service/smarthttp/receive_pack_test.go @@ -1,25 +1,20 @@ package smarthttp import ( - "bytes" "context" "fmt" - "io" "io/ioutil" "os" "path" "strings" "testing" - "time" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/git/hooks" - "gitlab.com/gitlab-org/gitaly/internal/helper/text" "gitlab.com/gitlab-org/gitaly/internal/testhelper" "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" - "gitlab.com/gitlab-org/gitaly/streamio" "google.golang.org/grpc/codes" ) @@ -160,88 +155,6 @@ func TestFailedReceivePackRequestDueToHooksFailure(t *testing.T) { require.Equal(t, expectedResponse, string(response), "Expected response to be %q, got %q", expectedResponse, response) } -func doPush(t *testing.T, stream gitalypb.SmartHTTPService_PostReceivePackClient, firstRequest *gitalypb.PostReceivePackRequest, body io.Reader) []byte { - require.NoError(t, stream.Send(firstRequest)) - - sw := streamio.NewWriter(func(p []byte) error { - return stream.Send(&gitalypb.PostReceivePackRequest{Data: p}) - }) - _, err := io.Copy(sw, body) - require.NoError(t, err) - - require.NoError(t, stream.CloseSend()) - - responseBuffer := bytes.Buffer{} - rr := streamio.NewReader(func() ([]byte, error) { - resp, err := stream.Recv() - return resp.GetData(), err - }) - _, err = io.Copy(&responseBuffer, rr) - require.NoError(t, err) - - return responseBuffer.Bytes() -} - -type pushData struct { - newHead string - body io.Reader -} - -func newTestPush(t *testing.T, fileContents []byte) *pushData { - _, repoPath, localCleanup := testhelper.NewTestRepoWithWorktree(t) - defer localCleanup() - - oldHead, newHead := createCommit(t, repoPath, fileContents) - - // ReceivePack request is a packet line followed by a packet flush, then the pack file of the objects we want to push. - // This is explained a bit in https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols#_uploading_data - // We form the packet line the same way git executable does: https://github.com/git/git/blob/d1a13d3fcb252631361a961cb5e2bf10ed467cba/send-pack.c#L524-L527 - clientCapabilities := "report-status side-band-64k agent=git/2.12.0" - pkt := fmt.Sprintf("%s %s refs/heads/master\x00 %s", oldHead, newHead, clientCapabilities) - - // We need to get a pack file containing the objects we want to push, so we use git pack-objects - // which expects a list of revisions passed through standard input. The list format means - // pack the objects needed if I have oldHead but not newHead (think of it from the perspective of the remote repo). - // For more info, check the man pages of both `git-pack-objects` and `git-rev-list --objects`. - stdin := strings.NewReader(fmt.Sprintf("^%s\n%s\n", oldHead, newHead)) - - // The options passed are the same ones used when doing an actual push. - pack := testhelper.MustRunCommand(t, stdin, "git", "-C", repoPath, "pack-objects", "--stdout", "--revs", "--thin", "--delta-base-offset", "-q") - - // We chop the request into multiple small pieces to exercise the server code that handles - // the stream sent by the client, so we use a buffer to read chunks of data in a nice way. - requestBuffer := &bytes.Buffer{} - fmt.Fprintf(requestBuffer, "%04x%s%s", len(pkt)+4, pkt, pktFlushStr) - requestBuffer.Write(pack) - - return &pushData{newHead: newHead, body: requestBuffer} -} - -// createCommit creates a commit on HEAD with a file containing the -// specified contents. -func createCommit(t *testing.T, repoPath string, fileContents []byte) (oldHead string, newHead string) { - commitMsg := fmt.Sprintf("Testing ReceivePack RPC around %d", time.Now().Unix()) - committerName := "Scrooge McDuck" - committerEmail := "scrooge@mcduck.com" - - // The latest commit ID on the remote repo - oldHead = text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "rev-parse", "master")) - - changedFile := "README.md" - require.NoError(t, ioutil.WriteFile(path.Join(repoPath, changedFile), fileContents, 0644)) - - testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "add", changedFile) - testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, - "-c", fmt.Sprintf("user.name=%s", committerName), - "-c", fmt.Sprintf("user.email=%s", committerEmail), - "commit", "-m", commitMsg) - - // The commit ID we want to push to the remote repo - newHead = text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "rev-parse", "master")) - - return oldHead, newHead -} - func TestFailedReceivePackRequestDueToValidationError(t *testing.T) { server, serverSocketPath := runSmartHTTPServer(t) defer server.Stop() diff --git a/internal/service/smarthttp/testhelper.go b/internal/service/smarthttp/testhelper.go new file mode 100644 index 000000000..19cf7fd0d --- /dev/null +++ b/internal/service/smarthttp/testhelper.go @@ -0,0 +1,118 @@ +package smarthttp + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "path" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/helper/text" + "gitlab.com/gitlab-org/gitaly/internal/testhelper" + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/streamio" +) + +const ( + pktFlushStr = "0000" +) + +var ( + // NewTestPush creates a test git push for PostReceivePack + NewTestPush = newTestPush + // DoPush executes a test git push for PostReceivePack + DoPush = doPush +) + +type pushData struct { + newHead string + body io.Reader +} + +func (p *pushData) GetBody() io.Reader { + return p.body +} + +func (p *pushData) GetNewHead() string { + return p.newHead +} + +func newTestPush(t *testing.T, fileContents []byte) *pushData { + _, repoPath, localCleanup := testhelper.NewTestRepoWithWorktree(t) + defer localCleanup() + + oldHead, newHead := createCommit(t, repoPath, fileContents) + // ReceivePack request is a packet line followed by a packet flush, then the pack file of the objects we want to push. + // This is explained a bit in https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols#_uploading_data + // We form the packet line the same way git executable does: https://github.com/git/git/blob/d1a13d3fcb252631361a961cb5e2bf10ed467cba/send-pack.c#L524-L527 + clientCapabilities := "report-status side-band-64k agent=git/2.12.0" + pkt := fmt.Sprintf("%s %s refs/heads/master\x00 %s", oldHead, newHead, clientCapabilities) + + // We need to get a pack file containing the objects we want to push, so we use git pack-objects + // which expects a list of revisions passed through standard input. The list format means + // pack the objects needed if I have oldHead but not newHead (think of it from the perspective of the remote repo). + // For more info, check the man pages of both `git-pack-objects` and `git-rev-list --objects`. + stdin := strings.NewReader(fmt.Sprintf("^%s\n%s\n", oldHead, newHead)) + + // The options passed are the same ones used when doing an actual push. + pack := testhelper.MustRunCommand(t, stdin, "git", "-C", repoPath, "pack-objects", "--stdout", "--revs", "--thin", "--delta-base-offset", "-q") + + // We chop the request into multiple small pieces to exercise the server code that handles + // the stream sent by the client, so we use a buffer to read chunks of data in a nice way. + requestBuffer := &bytes.Buffer{} + fmt.Fprintf(requestBuffer, "%04x%s%s", len(pkt)+4, pkt, pktFlushStr) + requestBuffer.Write(pack) + + return &pushData{newHead: newHead, body: requestBuffer} +} + +// createCommit creates a commit on HEAD with a file containing the +// specified contents. +func createCommit(t *testing.T, repoPath string, fileContents []byte) (oldHead string, newHead string) { + commitMsg := fmt.Sprintf("Testing ReceivePack RPC around %d", time.Now().Unix()) + committerName := "Scrooge McDuck" + committerEmail := "scrooge@mcduck.com" + + // The latest commit ID on the remote repo + oldHead = text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "rev-parse", "master")) + + changedFile := "README.md" + require.NoError(t, ioutil.WriteFile(path.Join(repoPath, changedFile), fileContents, 0644)) + + testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "add", changedFile) + testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, + "-c", fmt.Sprintf("user.name=%s", committerName), + "-c", fmt.Sprintf("user.email=%s", committerEmail), + "commit", "-m", commitMsg) + + // The commit ID we want to push to the remote repo + newHead = text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", repoPath, "rev-parse", "master")) + + return oldHead, newHead +} + +func doPush(t *testing.T, stream gitalypb.SmartHTTPService_PostReceivePackClient, firstRequest *gitalypb.PostReceivePackRequest, body io.Reader) []byte { + require.NoError(t, stream.Send(firstRequest)) + + sw := streamio.NewWriter(func(p []byte) error { + return stream.Send(&gitalypb.PostReceivePackRequest{Data: p}) + }) + _, err := io.Copy(sw, body) + require.NoError(t, err) + + require.NoError(t, stream.CloseSend()) + + responseBuffer := bytes.Buffer{} + rr := streamio.NewReader(func() ([]byte, error) { + resp, err := stream.Recv() + return resp.GetData(), err + }) + _, err = io.Copy(&responseBuffer, rr) + require.NoError(t, err) + + return responseBuffer.Bytes() +} diff --git a/internal/service/smarthttp/testhelper_test.go b/internal/service/smarthttp/testhelper_test.go index bc5dbb6e3..6ef8fde7d 100644 --- a/internal/service/smarthttp/testhelper_test.go +++ b/internal/service/smarthttp/testhelper_test.go @@ -12,10 +12,6 @@ import ( "google.golang.org/grpc/reflection" ) -const ( - pktFlushStr = "0000" -) - func TestMain(m *testing.M) { os.Exit(testMain(m)) } |