diff options
author | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2020-02-11 11:25:01 +0300 |
---|---|---|
committer | Zeger-Jan van de Weg <git@zjvandeweg.nl> | 2020-02-11 11:25:01 +0300 |
commit | b34e98ed7d850c586b12153ae504bc0b24e6a7f1 (patch) | |
tree | 10b41e71806a5e884eabd44506eb9918613cc2a6 | |
parent | 8196d6f066f738707995f42e50e54f50b196be6f (diff) | |
parent | ba9f2b89a961cb60333f288562cf176be2a45b9d (diff) |
Merge branch 'jc-add-e2e-hook-test' into 'master'
Add test from PostReceivePack to hooks
See merge request gitlab-org/gitaly!1796
-rw-r--r-- | cmd/gitaly-hooks/hooks.go | 12 | ||||
-rw-r--r-- | cmd/gitaly-hooks/hooks_test.go | 343 | ||||
-rwxr-xr-x | cmd/gitaly-hooks/testdata/update | 2 | ||||
-rw-r--r-- | internal/service/smarthttp/receive_pack_test.go | 74 | ||||
-rw-r--r-- | internal/service/smarthttp/testhelper_test.go | 2 | ||||
-rw-r--r-- | internal/service/ssh/receive_pack_test.go | 126 | ||||
-rw-r--r-- | internal/service/ssh/testhelper_test.go | 1 | ||||
-rw-r--r-- | internal/testhelper/hook_env.go | 21 | ||||
-rw-r--r-- | internal/testhelper/testserver.go | 175 |
9 files changed, 528 insertions, 228 deletions
diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go index 9f616520b..1bc42aa59 100644 --- a/cmd/gitaly-hooks/hooks.go +++ b/cmd/gitaly-hooks/hooks.go @@ -73,18 +73,6 @@ func main() { } } -// GitlabShellConfig contains a subset of gitlabshell's config.yml -type GitlabShellConfig struct { - GitlabURL string `yaml:"gitlab_url"` - HTTPSettings HTTPSettings `yaml:"http_settings"` -} - -// HTTPSettings contains fields for http settings -type HTTPSettings struct { - User string `yaml:"user"` - Password string `yaml:"password"` -} - func check(configPath string) (int, error) { cfgFile, err := os.Open(configPath) if err != nil { diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go index 1e2aef00f..083b4d7bd 100644 --- a/cmd/gitaly-hooks/hooks_test.go +++ b/cmd/gitaly-hooks/hooks_test.go @@ -5,22 +5,15 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" - "net/http" - "net/http/httptest" "os" "os/exec" - "path" "path/filepath" - "strconv" "testing" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/internal/command" "gitlab.com/gitlab-org/gitaly/internal/config" - "gitlab.com/gitlab-org/gitaly/internal/git/hooks" "gitlab.com/gitlab-org/gitaly/internal/testhelper" - "gopkg.in/yaml.v2" ) func TestMain(m *testing.M) { @@ -30,44 +23,69 @@ func TestMain(m *testing.M) { func testMain(m *testing.M) int { defer testhelper.MustHaveNoChildProcess() - configureGitalyHooksBinary() + testhelper.ConfigureGitalyHooksBinary() + testhelper.ConfigureGitalySSH() return m.Run() } func TestHooksPrePostReceive(t *testing.T) { + _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + secretToken := "secret token" key := 1234 glRepository := "some_repo" - tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) defer cleanup() changes := "abc" gitPushOptions := []string{"gitpushoption1", "gitpushoption2"} - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, changes, true, gitPushOptions...) + c := testhelper.GitlabServerConfig{ + User: "", + Password: "", + SecretToken: secretToken, + Key: key, + GLRepository: glRepository, + Changes: changes, + PostReceiveCounterDecreased: true, + Protocol: "ssh", + GitPushOptions: gitPushOptions, + } + + ts := testhelper.NewGitlabTestServer(t, c) defer ts.Close() + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() - writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) - writeShellSecretFile(t, tempGitlabShellDir, secretToken) + config.Config.GitlabShell.Dir = tempGitlabShellDir + + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL}) + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, secretToken) for _, hook := range []string{"pre-receive", "post-receive"} { t.Run(hook, func(t *testing.T) { var stderr, stdout bytes.Buffer stdin := bytes.NewBuffer([]byte(changes)) - cmd := exec.Command(fmt.Sprintf("../../ruby/git-hooks/%s", hook)) + hookPath, err := filepath.Abs(fmt.Sprintf("../../ruby/git-hooks/%s", hook)) + require.NoError(t, err) + cmd := exec.Command(hookPath) cmd.Stderr = &stderr cmd.Stdout = &stdout cmd.Stdin = stdin - cmd.Env = env( + cmd.Env = testhelper.EnvForHooks( t, glRepository, tempGitlabShellDir, key, gitPushOptions..., ) + cmd.Dir = testRepoPath require.NoError(t, cmd.Run()) require.Empty(t, stderr.String()) @@ -80,30 +98,48 @@ func TestHooksUpdate(t *testing.T) { key := 1234 glRepository := "some_repo" - tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) defer cleanup() - writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: "http://www.example.com"}) - writeShellSecretFile(t, tempGitlabShellDir, "the wrong token") + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: "http://www.example.com"}) + _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + + os.Symlink(filepath.Join(config.Config.GitlabShell.Dir, "config.yml"), filepath.Join(tempGitlabShellDir, "config.yml")) + + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, "the wrong token") + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir require.NoError(t, os.MkdirAll(filepath.Join(tempGitlabShellDir, "hooks", "update.d"), 0755)) testhelper.MustRunCommand(t, nil, "cp", "testdata/update", filepath.Join(tempGitlabShellDir, "hooks", "update.d", "update")) + tempFilePath := filepath.Join(testRepoPath, "tempfile") refval, oldval, newval := "refval", "oldval", "newval" var stdout, stderr bytes.Buffer - cmd := exec.Command("../../ruby/git-hooks/update", refval, oldval, newval) - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + updateHookPath, err := filepath.Abs("../../ruby/git-hooks/update") + require.NoError(t, err) + cmd := exec.Command(updateHookPath, refval, oldval, newval) + cmd.Env = testhelper.EnvForHooks(t, glRepository, tempGitlabShellDir, key) cmd.Stdout = &stdout cmd.Stderr = &stderr + cmd.Dir = testRepoPath require.NoError(t, cmd.Run()) require.Empty(t, stdout.String()) require.Empty(t, stderr.String()) + require.FileExists(t, tempFilePath) + var inputs []string - f, err := os.Open("testdata/tempfile") + f, err := os.Open(tempFilePath) require.NoError(t, err) require.NoError(t, json.NewDecoder(f).Decode(&inputs)) require.Equal(t, []string{refval, oldval, newval}, inputs) @@ -115,27 +151,50 @@ func TestHooksPostReceiveFailed(t *testing.T) { key := 1234 glRepository := "some_repo" - tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) defer cleanup() + _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() + // By setting the last parameter to false, the post-receive API call will // send back {"reference_counter_increased": false}, indicating something went wrong // with the call - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", false) + c := testhelper.GitlabServerConfig{ + User: "", + Password: "", + SecretToken: secretToken, + Key: key, + GLRepository: glRepository, + Changes: "", + PostReceiveCounterDecreased: false, + Protocol: "ssh", + } + ts := testhelper.NewGitlabTestServer(t, c) defer ts.Close() - writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) - writeShellSecretFile(t, tempGitlabShellDir, secretToken) + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL}) + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, secretToken) + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir var stdout, stderr bytes.Buffer - cmd := exec.Command(fmt.Sprintf("../../ruby/git-hooks/%s", "post-receive")) - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + postReceiveHookPath, err := filepath.Abs("../../ruby/git-hooks/post-receive") + require.NoError(t, err) + cmd := exec.Command(postReceiveHookPath) + cmd.Env = testhelper.EnvForHooks(t, glRepository, tempGitlabShellDir, key) cmd.Stdout = &stdout cmd.Stderr = &stderr + cmd.Dir = testRepoPath - err := cmd.Run() + err = cmd.Run() code, ok := command.ExitStatus(err) require.True(t, ok, "expect exit status in %v", err) @@ -149,21 +208,43 @@ func TestHooksNotAllowed(t *testing.T) { key := 1234 glRepository := "some_repo" - tempGitlabShellDir, cleanup := createTempGitlabShellDir(t) + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) defer cleanup() - ts := gitlabTestServer(t, "", "", secretToken, key, glRepository, "", true) + c := testhelper.GitlabServerConfig{ + User: "", + Password: "", + SecretToken: secretToken, + Key: key, + GLRepository: glRepository, + Changes: "", + PostReceiveCounterDecreased: true, + Protocol: "ssh", + } + ts := testhelper.NewGitlabTestServer(t, c) defer ts.Close() + _, testRepoPath, cleanupFn := testhelper.NewTestRepo(t) + defer cleanupFn() - writeTemporaryConfigFile(t, tempGitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL}) - writeShellSecretFile(t, tempGitlabShellDir, "the wrong token") + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL}) + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, "the wrong token") + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir var stderr, stdout bytes.Buffer - cmd := exec.Command(fmt.Sprintf("../../ruby/git-hooks/%s", "pre-receive")) + preReceiveHookPath, err := filepath.Abs("../../ruby/git-hooks/pre-receive") + require.NoError(t, err) + cmd := exec.Command(preReceiveHookPath) cmd.Stderr = &stderr cmd.Stdout = &stdout - cmd.Env = env(t, glRepository, tempGitlabShellDir, key) + cmd.Env = testhelper.EnvForHooks(t, glRepository, tempGitlabShellDir, key) + cmd.Dir = testRepoPath require.Error(t, cmd.Run()) require.Equal(t, "GitLab: 401 Unauthorized\n", stderr.String()) @@ -173,7 +254,17 @@ func TestHooksNotAllowed(t *testing.T) { func TestCheckOK(t *testing.T) { user, password := "user123", "password321" - ts := gitlabTestServer(t, user, password, "", 0, "", "", false) + c := testhelper.GitlabServerConfig{ + User: user, + Password: password, + SecretToken: "", + Key: 0, + GLRepository: "", + Changes: "", + PostReceiveCounterDecreased: false, + Protocol: "ssh", + } + ts := testhelper.NewGitlabTestServer(t, c) defer ts.Close() tempDir, err := ioutil.TempDir("", t.Name()) @@ -190,10 +281,10 @@ func TestCheckOK(t *testing.T) { require.NoError(t, err) require.NoError(t, os.Symlink(filepath.Join(cwd, "../../ruby/gitlab-shell/bin/check"), filepath.Join(binDir, "check"))) - writeShellSecretFile(t, gitlabShellDir, "the secret") - writeTemporaryConfigFile(t, gitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL, HTTPSettings: HTTPSettings{User: user, Password: password}}) + testhelper.WriteShellSecretFile(t, gitlabShellDir, "the secret") + testhelper.WriteTemporaryGitlabShellConfigFile(t, gitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL, HTTPSettings: testhelper.HTTPSettings{User: user, Password: password}}) - configPath, cleanup := writeTemporaryGitalyConfigFile(t, tempDir) + configPath, cleanup := testhelper.WriteTemporaryGitalyConfigFile(t, tempDir) defer cleanup() cmd := exec.Command(fmt.Sprintf("%s/gitaly-hooks", config.Config.BinDir), "check", configPath) @@ -211,7 +302,18 @@ func TestCheckOK(t *testing.T) { func TestCheckBadCreds(t *testing.T) { user, password := "user123", "password321" - ts := gitlabTestServer(t, user, password, "", 0, "", "", false) + c := testhelper.GitlabServerConfig{ + User: user, + Password: password, + SecretToken: "", + Key: 0, + GLRepository: "", + Changes: "", + PostReceiveCounterDecreased: false, + Protocol: "ssh", + GitPushOptions: nil, + } + ts := testhelper.NewGitlabTestServer(t, c) defer ts.Close() tempDir, err := ioutil.TempDir("", t.Name()) @@ -228,10 +330,10 @@ func TestCheckBadCreds(t *testing.T) { require.NoError(t, err) require.NoError(t, os.Symlink(filepath.Join(cwd, "../../ruby/gitlab-shell/bin/check"), filepath.Join(binDir, "check"))) - writeTemporaryConfigFile(t, gitlabShellDir, GitlabShellConfig{GitlabURL: ts.URL, HTTPSettings: HTTPSettings{User: user + "wrong", Password: password}}) - writeShellSecretFile(t, gitlabShellDir, "the secret") + testhelper.WriteTemporaryGitlabShellConfigFile(t, gitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL, HTTPSettings: testhelper.HTTPSettings{User: user + "wrong", Password: password}}) + testhelper.WriteShellSecretFile(t, gitlabShellDir, "the secret") - configPath, cleanup := writeTemporaryGitalyConfigFile(t, tempDir) + configPath, cleanup := testhelper.WriteTemporaryGitalyConfigFile(t, tempDir) defer cleanup() cmd := exec.Command(fmt.Sprintf("%s/gitaly-hooks", config.Config.BinDir), "check", configPath) @@ -244,162 +346,3 @@ func TestCheckBadCreds(t *testing.T) { require.Equal(t, "Check GitLab API access: ", stdout.String()) require.Equal(t, "FAILED. code: 401\n", stderr.String()) } - -func handleAllowed(t *testing.T, secretToken string, key int, glRepository, changes string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, r.ParseForm()) - require.Equal(t, http.MethodPost, r.Method) - require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) - require.Equal(t, strconv.Itoa(key), r.Form.Get("key_id")) - require.Equal(t, glRepository, r.Form.Get("gl_repository")) - require.Equal(t, "ssh", r.Form.Get("protocol")) - require.Equal(t, changes, r.Form.Get("changes")) - - w.Header().Set("Content-Type", "application/json") - if r.Form.Get("secret_token") == secretToken { - w.Write([]byte(`{"status":true}`)) - return - } - w.WriteHeader(http.StatusUnauthorized) - w.Write([]byte(`{"message":"401 Unauthorized"}`)) - } -} - -func handlePreReceive(t *testing.T, secretToken, glRepository string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, r.ParseForm()) - require.Equal(t, http.MethodPost, r.Method) - require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) - require.Equal(t, glRepository, r.Form.Get("gl_repository")) - require.Equal(t, secretToken, r.Form.Get("secret_token")) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"reference_counter_increased": true}`)) - } -} - -func handlePostReceive(t *testing.T, secretToken string, key int, glRepository, changes string, counterDecreased bool, gitPushOptions ...string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - require.NoError(t, r.ParseForm()) - require.Equal(t, http.MethodPost, r.Method) - require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) - require.Equal(t, glRepository, r.Form.Get("gl_repository")) - require.Equal(t, secretToken, r.Form.Get("secret_token")) - require.Equal(t, fmt.Sprintf("key-%d", key), r.Form.Get("identifier")) - require.Equal(t, changes, r.Form.Get("changes")) - - if len(gitPushOptions) > 0 { - require.Equal(t, gitPushOptions, r.Form["push_options[]"]) - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(fmt.Sprintf(`{"reference_counter_decreased": %v}`, counterDecreased))) - } -} - -func handleCheck(t *testing.T, user, password string) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - u, p, ok := r.BasicAuth() - if !ok || u != user || p != password { - http.Error(w, "authorization failed", http.StatusUnauthorized) - return - } - - w.Write([]byte(`{"redis": true}`)) - w.WriteHeader(http.StatusOK) - } -} - -func gitlabTestServer(t *testing.T, - user, password, secretToken string, - key int, - glRepository, - changes string, - postReceiveCounterDecreased bool, - gitPushOptions ...string) *httptest.Server { - mux := http.NewServeMux() - mux.Handle("/api/v4/internal/allowed", http.HandlerFunc(handleAllowed(t, secretToken, key, glRepository, changes))) - mux.Handle("/api/v4/internal/pre_receive", http.HandlerFunc(handlePreReceive(t, secretToken, glRepository))) - mux.Handle("/api/v4/internal/post_receive", http.HandlerFunc(handlePostReceive(t, secretToken, key, glRepository, changes, postReceiveCounterDecreased, gitPushOptions...))) - mux.Handle("/api/v4/internal/check", http.HandlerFunc(handleCheck(t, user, password))) - - return httptest.NewServer(mux) -} - -func createTempGitlabShellDir(t *testing.T) (string, func()) { - tempDir, err := ioutil.TempDir("", "gitlab-shell") - require.NoError(t, err) - return tempDir, func() { - require.NoError(t, os.RemoveAll(tempDir)) - } -} - -func writeTemporaryConfigFile(t *testing.T, dir string, config GitlabShellConfig) string { - out, err := yaml.Marshal(&config) - require.NoError(t, err) - - path := filepath.Join(dir, "config.yml") - require.NoError(t, ioutil.WriteFile(path, out, 0644)) - - return path -} - -func writeTemporaryGitalyConfigFile(t *testing.T, tempDir string) (string, func()) { - path := filepath.Join(tempDir, "config.toml") - contents := fmt.Sprintf(` -[gitlab-shell] - dir = "%s/gitlab-shell" -`, tempDir) - require.NoError(t, ioutil.WriteFile(path, []byte(contents), 0644)) - - return path, func() { - os.RemoveAll(path) - } -} - -func env(t *testing.T, glRepo, gitlabShellDir string, key int, gitPushOptions ...string) []string { - rubyDir, err := filepath.Abs("../../ruby") - require.NoError(t, err) - - return append(append(oldEnv(t, glRepo, gitlabShellDir, key), []string{ - "GITALY_BIN_DIR=testdata/gitaly-libexec", - fmt.Sprintf("GITALY_RUBY_DIR=%s", rubyDir), - }...), hooks.GitPushOptions(gitPushOptions)...) -} - -func oldEnv(t *testing.T, glRepo, gitlabShellDir string, key int) []string { - return append([]string{ - fmt.Sprintf("GL_ID=key-%d", key), - fmt.Sprintf("GL_REPOSITORY=%s", glRepo), - "GL_PROTOCOL=ssh", - fmt.Sprintf("GITALY_GITLAB_SHELL_DIR=%s", gitlabShellDir), - fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir), - "GITALY_LOG_LEVEL=info", - "GITALY_LOG_FORMAT=json", - fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir), - }, os.Environ()...) -} - -func writeShellSecretFile(t *testing.T, dir, secretToken string) { - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, ".gitlab_shell_secret"), []byte(secretToken), 0644)) -} - -// configureGitalyHooksBinary builds gitaly-hooks command for tests -func configureGitalyHooksBinary() { - var err error - - config.Config.BinDir, err = filepath.Abs("testdata/gitaly-libexec") - if err != nil { - log.Fatal(err) - } - - goBuildArgs := []string{ - "build", - "-o", - path.Join(config.Config.BinDir, "gitaly-hooks"), - "gitlab.com/gitlab-org/gitaly/cmd/gitaly-hooks", - } - testhelper.MustRunCommand(nil, nil, "go", goBuildArgs...) -} diff --git a/cmd/gitaly-hooks/testdata/update b/cmd/gitaly-hooks/testdata/update index a4076ec24..e982c4d17 100755 --- a/cmd/gitaly-hooks/testdata/update +++ b/cmd/gitaly-hooks/testdata/update @@ -1,4 +1,4 @@ #!/usr/bin/env ruby require 'json' -open('testdata/tempfile', 'w') { |f| f.puts(JSON.dump(ARGV)) } +open('tempfile', 'w') { |f| f.puts(JSON.dump(ARGV)) } diff --git a/internal/service/smarthttp/receive_pack_test.go b/internal/service/smarthttp/receive_pack_test.go index 46b874034..ba9f658a7 100644 --- a/internal/service/smarthttp/receive_pack_test.go +++ b/internal/service/smarthttp/receive_pack_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -275,6 +276,79 @@ func TestFailedReceivePackRequestDueToValidationError(t *testing.T) { } } +func TestPostReceivePackToHooks(t *testing.T) { + secretToken := "secret token" + glRepository := "some_repo" + key := 123 + + server, socket := runSmartHTTPServer(t) + defer server.Stop() + + client, conn := newSmartHTTPClient(t, "unix://"+socket) + defer conn.Close() + + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) + defer cleanup() + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + + repo, testRepoPath, cleanup := testhelper.NewTestRepo(t) + defer cleanup() + + push := newTestPush(t, nil) + oldHead := text.ChompBytes(testhelper.MustRunCommand(t, nil, "git", "-C", testRepoPath, "rev-parse", "HEAD")) + + changes := fmt.Sprintf("%s %s refs/heads/master\n", oldHead, push.newHead) + + c := testhelper.GitlabServerConfig{ + User: "", + Password: "", + SecretToken: secretToken, + Key: key, + GLRepository: glRepository, + Changes: changes, + PostReceiveCounterDecreased: true, + Protocol: "http", + } + + ts := testhelper.NewGitlabTestServer(t, c) + defer ts.Close() + + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL}) + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, secretToken) + + defer func(override string) { + hooks.Override = override + }(hooks.Override) + + cwd, err := os.Getwd() + require.NoError(t, err) + hookDir := filepath.Join(cwd, "../../../ruby", "git-hooks") + hooks.Override = hookDir + + ctx, cancel := testhelper.Context() + defer cancel() + + stream, err := client.PostReceivePack(ctx) + require.NoError(t, err) + + firstRequest := &gitalypb.PostReceivePackRequest{ + Repository: repo, + GlId: fmt.Sprintf("key-%d", key), + GlRepository: glRepository, + } + + response := doPush(t, stream, firstRequest, push.body) + + expectedResponse := "0030\x01000eunpack ok\n0019ok refs/heads/master\n00000000" + require.Equal(t, expectedResponse, string(response), "Expected response to be %q, got %q", expectedResponse, response) +} + func drainPostReceivePackResponse(stream gitalypb.SmartHTTPService_PostReceivePackClient) error { var err error for err == nil { diff --git a/internal/service/smarthttp/testhelper_test.go b/internal/service/smarthttp/testhelper_test.go index bc5dbb6e3..ed5eff8d3 100644 --- a/internal/service/smarthttp/testhelper_test.go +++ b/internal/service/smarthttp/testhelper_test.go @@ -23,6 +23,8 @@ func TestMain(m *testing.M) { func testMain(m *testing.M) int { hooks.Override = "/" + testhelper.ConfigureGitalyHooksBinary() + return m.Run() } diff --git a/internal/service/ssh/receive_pack_test.go b/internal/service/ssh/receive_pack_test.go index 4eb1b1249..76ec4c82f 100644 --- a/internal/service/ssh/receive_pack_test.go +++ b/internal/service/ssh/receive_pack_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strings" "testing" "time" @@ -86,7 +87,9 @@ func TestReceivePackPushSuccess(t *testing.T) { server, serverSocketPath := runSSHServer(t) defer server.Stop() - lHead, rHead, err := testCloneAndPush(t, serverSocketPath, pushParams{storageName: testRepo.GetStorageName(), glID: "user-123"}) + glRepository := "project-456" + + lHead, rHead, err := testCloneAndPush(t, serverSocketPath, pushParams{storageName: testRepo.GetStorageName(), glID: "user-123", glRepository: glRepository}) if err != nil { t.Fatal(err) } @@ -97,7 +100,7 @@ func TestReceivePackPushSuccess(t *testing.T) { for _, env := range []string{ "GL_ID=user-123", - "GL_REPOSITORY=project-456", + fmt.Sprintf("GL_REPOSITORY=%s", glRepository), "GL_PROTOCOL=ssh", "GITALY_GITLAB_SHELL_DIR=" + "/foo/bar/gitlab-shell", } { @@ -202,7 +205,79 @@ func TestObjectPoolRefAdvertisementHidingSSH(t *testing.T) { require.NotContains(t, b.String(), commitID+" .have") } -func testCloneAndPush(t *testing.T, serverSocketPath string, params pushParams) (string, string, error) { +func TestSSHReceivePackToHooks(t *testing.T) { + secretToken := "secret token" + glRepository := "some_repo" + key := 123 + + restore := testhelper.EnableGitProtocolV2Support() + defer restore() + + server, serverSocketPath := runSSHServer(t) + defer server.Stop() + + tempGitlabShellDir, cleanup := testhelper.CreateTemporaryGitlabShellDir(t) + defer cleanup() + + gitlabShellDir := config.Config.GitlabShell.Dir + defer func() { + config.Config.GitlabShell.Dir = gitlabShellDir + }() + + config.Config.GitlabShell.Dir = tempGitlabShellDir + + cloneDetails, cleanup := setupSSHClone(t) + defer cleanup() + + c := testhelper.GitlabServerConfig{ + User: "", + Password: "", + SecretToken: secretToken, + Key: key, + GLRepository: glRepository, + Changes: fmt.Sprintf("%s %s refs/heads/master\n", string(cloneDetails.OldHead), string(cloneDetails.NewHead)), + PostReceiveCounterDecreased: true, + Protocol: "ssh", + } + ts := testhelper.NewGitlabTestServer(t, c) + defer ts.Close() + + testhelper.WriteTemporaryGitlabShellConfigFile(t, tempGitlabShellDir, testhelper.GitlabShellConfig{GitlabURL: ts.URL}) + testhelper.WriteShellSecretFile(t, tempGitlabShellDir, secretToken) + + defer func(override string) { + hooks.Override = override + }(hooks.Override) + + cwd, err := os.Getwd() + require.NoError(t, err) + hookDir := filepath.Join(cwd, "../../../ruby", "git-hooks") + hooks.Override = hookDir + + lHead, rHead, err := sshPush(t, cloneDetails, serverSocketPath, pushParams{ + storageName: testRepo.GetStorageName(), + glID: fmt.Sprintf("key-%d", key), + glRepository: glRepository, + gitProtocol: git.ProtocolV2, + }) + require.NoError(t, err) + require.Equal(t, lHead, rHead, "local and remote head not equal. push failed") + + envData, err := testhelper.GetGitEnvData() + + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("GIT_PROTOCOL=%s\n", git.ProtocolV2), envData) +} + +// SSHCloneDetails encapsulates values relevant for a test clone +type SSHCloneDetails struct { + LocalRepoPath, RemoteRepoPath, TempRepo string + OldHead []byte + NewHead []byte +} + +// setupSSHClone sets up a test clone +func setupSSHClone(t *testing.T) (SSHCloneDetails, func()) { storagePath := testhelper.GitlabTestStoragePath() tempRepo := "gitlab-test-ssh-receive-pack.git" testRepoPath := path.Join(storagePath, testRepo.GetRelativePath()) @@ -218,24 +293,36 @@ func testCloneAndPush(t *testing.T, serverSocketPath string, params pushParams) t.Fatal(err) } testhelper.MustRunCommand(t, nil, "git", "clone", remoteRepoPath, localRepoPath) - // We need git thinking we're pushing over SSH... - defer os.RemoveAll(remoteRepoPath) - defer os.RemoveAll(localRepoPath) - makeCommit(t, localRepoPath) + // We need git thinking we're pushing over SSH... + oldHead, newHead, success := makeCommit(t, localRepoPath) + require.True(t, success) + + return SSHCloneDetails{ + OldHead: oldHead, + NewHead: newHead, + LocalRepoPath: localRepoPath, + RemoteRepoPath: remoteRepoPath, + TempRepo: tempRepo, + }, func() { + os.RemoveAll(remoteRepoPath) + os.RemoveAll(localRepoPath) + } +} - pbTempRepo := &gitalypb.Repository{StorageName: params.storageName, RelativePath: tempRepo} +func sshPush(t *testing.T, cloneDetails SSHCloneDetails, serverSocketPath string, params pushParams) (string, string, error) { + pbTempRepo := &gitalypb.Repository{StorageName: params.storageName, RelativePath: cloneDetails.TempRepo} pbMarshaler := &jsonpb.Marshaler{} payload, err := pbMarshaler.MarshalToString(&gitalypb.SSHReceivePackRequest{ Repository: pbTempRepo, - GlRepository: "project-456", + GlRepository: params.glRepository, GlId: params.glID, GitConfigOptions: params.gitConfigOptions, GitProtocol: params.gitProtocol, }) require.NoError(t, err) - cmd := exec.Command("git", "-C", localRepoPath, "push", "-v", "git@localhost:test/test.git", "master") + cmd := exec.Command("git", "-C", cloneDetails.LocalRepoPath, "push", "-v", "git@localhost:test/test.git", "master") cmd.Env = []string{ fmt.Sprintf("GITALY_PAYLOAD=%s", payload), fmt.Sprintf("GITALY_ADDRESS=%s", serverSocketPath), @@ -244,18 +331,26 @@ func testCloneAndPush(t *testing.T, serverSocketPath string, params pushParams) } out, err := cmd.CombinedOutput() if err != nil { - return "", "", fmt.Errorf("Error pushing: %v: %q", err, out) + return "", "", fmt.Errorf("error pushing: %v: %q", err, out) } + if !cmd.ProcessState.Success() { - return "", "", fmt.Errorf("Failed to run `git push`: %q", out) + return "", "", fmt.Errorf("failed to run `git push`: %q", out) } - localHead := bytes.TrimSpace(testhelper.MustRunCommand(t, nil, "git", "-C", localRepoPath, "rev-parse", "master")) - remoteHead := bytes.TrimSpace(testhelper.MustRunCommand(t, nil, "git", "-C", remoteRepoPath, "rev-parse", "master")) + localHead := bytes.TrimSpace(testhelper.MustRunCommand(t, nil, "git", "-C", cloneDetails.LocalRepoPath, "rev-parse", "master")) + remoteHead := bytes.TrimSpace(testhelper.MustRunCommand(t, nil, "git", "-C", cloneDetails.RemoteRepoPath, "rev-parse", "master")) return string(localHead), string(remoteHead), nil } +func testCloneAndPush(t *testing.T, serverSocketPath string, params pushParams) (string, string, error) { + cloneDetails, cleanup := setupSSHClone(t) + defer cleanup() + + return sshPush(t, cloneDetails, serverSocketPath, params) +} + // makeCommit creates a new commit and returns oldHead, newHead, success func makeCommit(t *testing.T, localRepoPath string) ([]byte, []byte, bool) { commitMsg := fmt.Sprintf("Testing ReceivePack RPC around %d", time.Now().Unix()) @@ -281,7 +376,7 @@ func makeCommit(t *testing.T, localRepoPath string) ([]byte, []byte, bool) { // The commit ID we want to push to the remote repo newHead := bytes.TrimSpace(testhelper.MustRunCommand(t, nil, "git", "-C", localRepoPath, "rev-parse", "master")) - return oldHead, newHead, t.Failed() + return oldHead, newHead, true } func drainPostReceivePackResponse(stream gitalypb.SSHService_SSHReceivePackClient) error { @@ -295,6 +390,7 @@ func drainPostReceivePackResponse(stream gitalypb.SSHService_SSHReceivePackClien type pushParams struct { storageName string glID string + glRepository string gitConfigOptions []string gitProtocol string } diff --git a/internal/service/ssh/testhelper_test.go b/internal/service/ssh/testhelper_test.go index 171430590..e368ba877 100644 --- a/internal/service/ssh/testhelper_test.go +++ b/internal/service/ssh/testhelper_test.go @@ -43,6 +43,7 @@ func testMain(m *testing.M) int { testRepo = testhelper.TestRepository() + testhelper.ConfigureGitalyHooksBinary() testhelper.ConfigureGitalySSH() gitalySSHPath = path.Join(config.Config.BinDir, "gitaly-ssh") diff --git a/internal/testhelper/hook_env.go b/internal/testhelper/hook_env.go index 00691b4c2..3efdeb078 100644 --- a/internal/testhelper/hook_env.go +++ b/internal/testhelper/hook_env.go @@ -3,10 +3,13 @@ package testhelper import ( "io/ioutil" "os" + "path" "path/filepath" "testing" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git/hooks" ) @@ -32,3 +35,21 @@ env | grep -e ^GIT -e ^GL_ > `+hookOutputFile+"\n"), 0755)) hooks.Override = oldOverride } } + +// ConfigureGitalyHooksBinary builds gitaly-hooks command for tests +func ConfigureGitalyHooksBinary() { + var err error + + config.Config.BinDir, err = filepath.Abs("testdata/gitaly-libexec") + if err != nil { + log.Fatal(err) + } + + goBuildArgs := []string{ + "build", + "-o", + path.Join(config.Config.BinDir, "gitaly-hooks"), + "gitlab.com/gitlab-org/gitaly/cmd/gitaly-hooks", + } + MustRunCommand(nil, nil, "go", goBuildArgs...) +} diff --git a/internal/testhelper/testserver.go b/internal/testhelper/testserver.go index 4d69fbfc2..352a47194 100644 --- a/internal/testhelper/testserver.go +++ b/internal/testhelper/testserver.go @@ -6,9 +6,12 @@ import ( "fmt" "io/ioutil" "net" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" + "strconv" "testing" "time" @@ -17,11 +20,15 @@ import ( grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/config" + "gitlab.com/gitlab-org/gitaly/internal/git/hooks" "gitlab.com/gitlab-org/gitaly/internal/helper/fieldextractors" praefectconfig "gitlab.com/gitlab-org/gitaly/internal/praefect/config" "gitlab.com/gitlab-org/gitaly/internal/praefect/models" "google.golang.org/grpc" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "gopkg.in/yaml.v2" ) // NewTestServer instantiates a new TestServer @@ -176,3 +183,171 @@ func NewServer(tb testing.TB, streamInterceptors []grpc.StreamServerInterceptor, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unaryInterceptors...)), )) } + +func handleAllowed(t *testing.T, secretToken string, key int, glRepository, changes, protocol string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + require.Equal(t, http.MethodPost, r.Method) + require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) + require.Equal(t, strconv.Itoa(key), r.Form.Get("key_id")) + require.Equal(t, glRepository, r.Form.Get("gl_repository")) + require.Equal(t, protocol, r.Form.Get("protocol")) + require.Equal(t, changes, r.Form.Get("changes")) + + w.Header().Set("Content-Type", "application/json") + if r.Form.Get("secret_token") == secretToken { + w.Write([]byte(`{"status":true}`)) + return + } + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte(`{"message":"401 Unauthorized"}`)) + } +} + +func handlePreReceive(t *testing.T, secretToken, glRepository string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + require.Equal(t, http.MethodPost, r.Method) + require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) + require.Equal(t, glRepository, r.Form.Get("gl_repository")) + require.Equal(t, secretToken, r.Form.Get("secret_token")) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"reference_counter_increased": true}`)) + } +} + +func handlePostReceive(t *testing.T, secretToken string, key int, glRepository, changes string, counterDecreased bool, gitPushOptions ...string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + require.NoError(t, r.ParseForm()) + require.Equal(t, http.MethodPost, r.Method) + require.Equal(t, "application/x-www-form-urlencoded", r.Header.Get("Content-Type")) + require.Equal(t, glRepository, r.Form.Get("gl_repository")) + require.Equal(t, secretToken, r.Form.Get("secret_token")) + require.Equal(t, fmt.Sprintf("key-%d", key), r.Form.Get("identifier")) + require.Equal(t, changes, r.Form.Get("changes")) + + if len(gitPushOptions) > 0 { + require.Equal(t, gitPushOptions, r.Form["push_options[]"]) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf(`{"reference_counter_decreased": %v}`, counterDecreased))) + } +} + +func handleCheck(user, password string) func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || u != user || p != password { + http.Error(w, "authorization failed", http.StatusUnauthorized) + return + } + + w.Write([]byte(`{"redis": true}`)) + w.WriteHeader(http.StatusOK) + } +} + +// GitlabServerConfig is a config for a mock gitlab server +type GitlabServerConfig struct { + User, Password, SecretToken string + Key int + GLRepository string + Changes string + PostReceiveCounterDecreased bool + Protocol string + GitPushOptions []string +} + +// NewGitlabTestServer returns a mock gitlab server that responds to the hook api endpoints +func NewGitlabTestServer(t *testing.T, c GitlabServerConfig) *httptest.Server { + mux := http.NewServeMux() + mux.Handle("/api/v4/internal/allowed", http.HandlerFunc(handleAllowed(t, c.SecretToken, c.Key, c.GLRepository, c.Changes, c.Protocol))) + mux.Handle("/api/v4/internal/pre_receive", http.HandlerFunc(handlePreReceive(t, c.SecretToken, c.GLRepository))) + mux.Handle("/api/v4/internal/post_receive", http.HandlerFunc(handlePostReceive(t, c.SecretToken, c.Key, c.GLRepository, c.Changes, c.PostReceiveCounterDecreased, c.GitPushOptions...))) + mux.Handle("/api/v4/internal/check", http.HandlerFunc(handleCheck(c.User, c.Password))) + + return httptest.NewServer(mux) +} + +// CreateTemporaryGitlabShellDir creates a temporary gitlab shell directory. It returns the path to the directory +// and a cleanup function +func CreateTemporaryGitlabShellDir(t *testing.T) (string, func()) { + tempDir, err := ioutil.TempDir("", "gitlab-shell") + require.NoError(t, err) + return tempDir, func() { + require.NoError(t, os.RemoveAll(tempDir)) + } +} + +// WriteTemporaryGitlabShellConfigFile writes a gitlab shell config.yml in a temporary directory. It returns the path +// and a cleanup function +func WriteTemporaryGitlabShellConfigFile(t *testing.T, dir string, config GitlabShellConfig) (string, func()) { + out, err := yaml.Marshal(&config) + require.NoError(t, err) + + path := filepath.Join(dir, "config.yml") + require.NoError(t, ioutil.WriteFile(path, out, 0644)) + + return path, func() { + os.RemoveAll(path) + } +} + +// WriteTemporaryGitalyConfigFile writes a gitaly toml file into a temporary directory. It returns the path to +// the file as well as a cleanup function +func WriteTemporaryGitalyConfigFile(t *testing.T, tempDir string) (string, func()) { + path := filepath.Join(tempDir, "config.toml") + contents := fmt.Sprintf(` +[gitlab-shell] + dir = "%s/gitlab-shell" +`, tempDir) + require.NoError(t, ioutil.WriteFile(path, []byte(contents), 0644)) + + return path, func() { + os.RemoveAll(path) + } +} + +// EnvForHooks generates a set of environment variables for gitaly hooks +func EnvForHooks(t *testing.T, glRepo, gitlabShellDir string, key int, gitPushOptions ...string) []string { + rubyDir, err := filepath.Abs("../../ruby") + require.NoError(t, err) + + return append(append(oldEnv(t, glRepo, gitlabShellDir, key), []string{ + fmt.Sprintf("GITALY_BIN_DIR=%s", config.Config.BinDir), + fmt.Sprintf("GITALY_RUBY_DIR=%s", rubyDir), + }...), hooks.GitPushOptions(gitPushOptions)...) +} + +func oldEnv(t *testing.T, glRepo, gitlabShellDir string, key int) []string { + return append([]string{ + fmt.Sprintf("GL_ID=key-%d", key), + fmt.Sprintf("GL_REPOSITORY=%s", glRepo), + "GL_PROTOCOL=ssh", + fmt.Sprintf("GITALY_GITLAB_SHELL_DIR=%s", gitlabShellDir), + fmt.Sprintf("GITALY_LOG_DIR=%s", gitlabShellDir), + "GITALY_LOG_LEVEL=info", + "GITALY_LOG_FORMAT=json", + }, os.Environ()...) +} + +// WriteShellSecretFile writes a .gitlab_shell_secret file in the specified directory +func WriteShellSecretFile(t *testing.T, dir, secretToken string) { + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, ".gitlab_shell_secret"), []byte(secretToken), 0644)) +} + +// GitlabShellConfig contains a subset of gitlabshell's config.yml +type GitlabShellConfig struct { + GitlabURL string `yaml:"gitlab_url"` + HTTPSettings HTTPSettings `yaml:"http_settings"` +} + +// HTTPSettings contains fields for http settings +type HTTPSettings struct { + User string `yaml:"user"` + Password string `yaml:"password"` +} |