From 89c5b8e78463e091c88cb3d24c5e010738d21bfe Mon Sep 17 00:00:00 2001 From: John Cai Date: Thu, 9 Apr 2020 16:27:12 -0700 Subject: WIP: replace gitlab access --- internal/config/config.go | 19 +++- internal/log/hook.go | 1 + internal/service/hooks/api.go | 188 ++++++++++++++++++++++++++++++++++ internal/service/hooks/pre_receive.go | 31 +++++- internal/service/hooks/server.go | 26 ++++- internal/service/register.go | 8 +- ruby/gitlab-shell/hooks/pre-receive | 3 +- 7 files changed, 261 insertions(+), 15 deletions(-) create mode 100644 internal/service/hooks/api.go diff --git a/internal/config/config.go b/internal/config/config.go index 0f041f2a9..5c5270ea3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,17 +48,30 @@ type Cfg struct { GracefulRestartTimeout time.Duration GracefulRestartTimeoutToml duration `toml:"graceful_restart_timeout"` InternalSocketDir string `toml:"internal_socket_dir"` + GitlabURL string `toml:"gitlab_url"` + HTTP HTTP `toml:"http"` + SecretFile string `toml:"secret_file"` +} + +type HTTP struct { + User string `toml:"user"` + Password string `toml:"password"` + SelfSigned bool `toml:"self_signed"` + CAFile string `toml:"ca_file"` + CAPath string `toml:"ca_path"` } // TLS configuration type TLS struct { - CertPath string `toml:"certificate_path"` - KeyPath string `toml:"key_path"` + CertPath string `toml:"certificate_path"` + KeyPath string `toml:"key_path"` + SelfSigned bool `toml:"self_signed"` } // GitlabShell contains the settings required for executing `gitlab-shell` type GitlabShell struct { - Dir string `toml:"dir"` + Dir string `toml:"dir"` + SecretFile string `toml:"secret_file"` } // Git contains the settings for the Git executable diff --git a/internal/log/hook.go b/internal/log/hook.go index a8219b376..2d7275854 100644 --- a/internal/log/hook.go +++ b/internal/log/hook.go @@ -38,6 +38,7 @@ func (h *HookLogger) Fatal(err error) { // Fatalf logs a formatted error at the Fatal level func (h *HookLogger) Fatalf(format string, a ...interface{}) { fmt.Fprintf(os.Stderr, "error executing git hook") + fmt.Fprintf(os.Stderr, format, a...) h.logger.Fatalf(format, a...) } diff --git a/internal/service/hooks/api.go b/internal/service/hooks/api.go new file mode 100644 index 000000000..d242ee4ca --- /dev/null +++ b/internal/service/hooks/api.go @@ -0,0 +1,188 @@ +package hook + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" +) + +type internalClient struct { + client *http.Client + gitlabURL string + username, password string + secret string +} + +type gitlabAccessStatus struct { + StatusCode int `json:"status_code"` + Status bool `json:"status"` + Message string `json:"message"` +} + +func newInternalClient(gitlabURL, secretPath string, selfSigned bool, caFile, caDir string) (*internalClient, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: selfSigned, + } + + if caFile != "" { + os.Setenv("SSL_CERT_FILE", caFile) + } + + if caDir != "" { + os.Setenv("SSL_CERT_DIR", caDir) + } + + var secret string + if secretPath != "" { + secretBytes, err := ioutil.ReadFile(secretPath) + if err != nil { + return nil, err + } + secret = string(secretBytes) + } + + transport := &http.Transport{TLSClientConfig: tlsConfig} + + if strings.HasPrefix(gitlabURL, "http://unix") { + transport = &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + socket := strings.TrimPrefix(gitlabURL, "http://unix") + return net.Dial("unix", socket) + }, + } + } + return &internalClient{ + client: &http.Client{Transport: transport}, + gitlabURL: gitlabURL, + secret: secret, + }, nil +} + +const internalURL = "http://unix/api/v4/internal" + +func parseGitObjVars(repoPath string, env []string) (string, error) { + envMap := map[string]interface{}{} + + for _, v := range env { + kv := strings.SplitN(v, "=", 2) + k := kv[0] + v := kv[1] + + if k == "GIT_OBJECT_DIRECTORY" { + gitObjDirRel, err := filepath.Rel(repoPath, v) + if err != nil { + return "", err + } + envMap["GIT_OBJECT_DIRECTORY_RELATIVE"] = gitObjDirRel + continue + } + + if k == "GIT_ALTERNATE_OBJECT_DIRECTORIES" { + var gitAltObjRelDirs []string + + for _, gitAltObjDir := range strings.Split(v, ":") { + gitAltObjDirRel, err := filepath.Rel(repoPath, gitAltObjDir) + if err != nil { + return "", err + } + gitAltObjRelDirs = append(gitAltObjRelDirs, gitAltObjDirRel) + } + + envMap["GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE"] = gitAltObjRelDirs + continue + } + } + + output, err := json.Marshal(envMap) + + return string(output), err +} + +func getVar(vars []string, s string) string { + for _, v := range vars { + if strings.HasPrefix(v, s+"=") { + return strings.TrimPrefix(v, s+"=") + } + } + + return "" +} + +func (i *internalClient) checkAccess(repoPath, changes string, env []string) (bool, string, error) { + form := url.Values{} + form.Set("action", "git-receive-pack") + form.Set("changes", changes) + form.Set("gl_repository", getVar(env, "GL_REPOSITORY")) + form.Set("project", strings.Replace(repoPath, "'", "", -1)) + form.Set("protocol", getVar(env, "GL_PROTOCOL")) + form.Set("secret_token", i.secret) + + glIDKey, glIDValue, err := parseGLID(getVar(env, "GL_ID")) + if err != nil { + return false, "", err + } + form.Set(glIDKey, glIDValue) + + envVars, err := parseGitObjVars(repoPath, env) + form.Set("env", envVars) + + r, err := http.NewRequest(http.MethodPost, internalURL+"/allowed", strings.NewReader(form.Encode())) + if err != nil { + return false, "", err + } + + r.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := i.post(r) + if err != nil { + return false, "", err + } + + var allowedResponse gitlabAccessStatus + + if err = json.NewDecoder(resp.Body).Decode(&allowedResponse); err != nil { + return false, "", err + } + + switch resp.StatusCode { + case http.StatusOK, http.StatusMultipleChoices, http.StatusUnauthorized, http.StatusNotFound, http.StatusServiceUnavailable: + if resp.Header.Get("Content-Type") == "application/json" { + return allowedResponse.Status, allowedResponse.Message, nil + } + } + + return false, "API Inaccessible", nil +} + +func (i *internalClient) post(r *http.Request) (*http.Response, error) { + if i.username != "" && i.password != "" { + r.SetBasicAuth(i.username, i.password) + } + return i.client.Do(r) +} + +func parseGLID(glID string) (glIDKey string, glIDValue string, err error) { + switch { + case strings.HasPrefix(glID, "key-"): + glIDKey = "key_id" + glIDValue = strings.Replace(glID, "key-", "", 1) + case strings.HasPrefix(glID, "user-"): + glIDKey = "user_id" + glIDValue = strings.Replace(glID, "user-", "", 1) + case strings.HasPrefix(glID, "username-"): + glIDKey = "username" + glIDValue = strings.Replace(glID, "username-", "", 1) + default: + err = fmt.Errorf("gl_id='%s' is invalid") + } + + return +} diff --git a/internal/service/hooks/pre_receive.go b/internal/service/hooks/pre_receive.go index 0564bb71c..45eb77aaf 100644 --- a/internal/service/hooks/pre_receive.go +++ b/internal/service/hooks/pre_receive.go @@ -2,6 +2,7 @@ package hook import ( "errors" + "io/ioutil" "os/exec" "path/filepath" @@ -44,29 +45,49 @@ func (s *server) PreReceiveHook(stream gitalypb.HookService_PreReceiveHookServer return helper.ErrInvalidArgument(err) } + repoPath, err := helper.GetRepoPath(firstRequest.GetRepository()) + if err != nil { + return helper.ErrInternal(err) + } + stdin := streamio.NewReader(func() ([]byte, error) { req, err := stream.Recv() return req.GetStdin(), err }) + stdout := streamio.NewWriter(func(p []byte) error { return stream.Send(&gitalypb.PreReceiveHookResponse{Stdout: p}) }) stderr := streamio.NewWriter(func(p []byte) error { return stream.Send(&gitalypb.PreReceiveHookResponse{Stderr: p}) }) - repoPath, err := helper.GetRepoPath(firstRequest.GetRepository()) + env, err := preReceiveEnv(firstRequest) if err != nil { return helper.ErrInternal(err) } - c := exec.Command(gitlabShellHook("pre-receive")) - c.Dir = repoPath + changes, err := ioutil.ReadAll(stdin) + if err != nil { + return helper.ErrInternal(err) + } - env, err := preReceiveEnv(firstRequest) + ok, msg, err := s.internalClient.checkAccess(repoPath, string(changes), env) if err != nil { return helper.ErrInternal(err) } + if !ok { + if err := stream.SendMsg(&gitalypb.PreReceiveHookResponse{ + ExitStatus: &gitalypb.ExitStatus{Value: 1}, + Stderr: []byte(msg), + }); err != nil { + return helper.ErrInternal(err) + } + } + + c := exec.Command(gitlabShellHook("pre-receive")) + c.Dir = repoPath + status, err := streamCommandResponse( stream.Context(), - stdin, + nil, stdout, stderr, c, env, diff --git a/internal/service/hooks/server.go b/internal/service/hooks/server.go index 1977af1df..41526ba57 100644 --- a/internal/service/hooks/server.go +++ b/internal/service/hooks/server.go @@ -1,10 +1,28 @@ package hook -import "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" +import ( + "gitlab.com/gitlab-org/gitaly/internal/config" + "gitlab.com/gitlab-org/gitaly/proto/go/gitalypb" +) -type server struct{} +type server struct { + internalClient *internalClient +} // NewServer creates a new instance of a gRPC namespace server -func NewServer() gitalypb.HookServiceServer { - return &server{} +func NewServer(c config.Cfg) (gitalypb.HookServiceServer, error) { + client, err := newInternalClient( + c.GitlabURL, + c.GitlabShell.SecretFile, + c.HTTP.SelfSigned, + c.HTTP.CAFile, + c.HTTP.CAPath, + ) + if err != nil { + return nil, err + } + + return &server{ + internalClient: client, + }, nil } diff --git a/internal/service/register.go b/internal/service/register.go index 4015f99fc..6682a5d1d 100644 --- a/internal/service/register.go +++ b/internal/service/register.go @@ -1,6 +1,8 @@ package service import ( + "log" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "gitlab.com/gitlab-org/gitaly/internal/config" @@ -72,7 +74,11 @@ func RegisterAll(grpcServer *grpc.Server, cfg config.Cfg, rubyServer *rubyserver gitalypb.RegisterRemoteServiceServer(grpcServer, remote.NewServer(rubyServer)) gitalypb.RegisterServerServiceServer(grpcServer, server.NewServer(cfg.Storages)) gitalypb.RegisterObjectPoolServiceServer(grpcServer, objectpool.NewServer()) - gitalypb.RegisterHookServiceServer(grpcServer, hook.NewServer()) + hooksService, err := hook.NewServer(cfg) + if err != nil { + log.Fatal(err) + } + gitalypb.RegisterHookServiceServer(grpcServer, hooksService) gitalypb.RegisterInternalGitalyServer(grpcServer, internalgitaly.NewServer(config.Config.Storages)) healthpb.RegisterHealthServer(grpcServer, health.NewServer()) diff --git a/ruby/gitlab-shell/hooks/pre-receive b/ruby/gitlab-shell/hooks/pre-receive index 66c61d98c..728f2fa1b 100755 --- a/ruby/gitlab-shell/hooks/pre-receive +++ b/ruby/gitlab-shell/hooks/pre-receive @@ -23,8 +23,7 @@ require_relative '../lib/gitlab_net' # last so that it only runs if everything else succeeded. On post-receive on the # other hand, we run GitlabPostReceive first because the push is already done # and we don't want to skip it if the custom hook fails. -if GitlabAccess.new(gl_repository, repo_path, gl_id, refs, protocol).exec && - GitlabCustomHook.new(repo_path, gl_id).pre_receive(refs) && +if GitlabCustomHook.new(repo_path, gl_id).pre_receive(refs) && increase_reference_counter(gl_repository, repo_path) exit 0 else -- cgit v1.2.3