diff options
-rw-r--r-- | cmd/gitaly-hooks/hooks.go | 65 | ||||
-rw-r--r-- | cmd/gitaly-hooks/hooks_test.go | 65 | ||||
-rw-r--r-- | internal/gitaly/hook/sidechannel.go | 109 | ||||
-rw-r--r-- | internal/gitaly/hook/sidechannel_test.go | 60 | ||||
-rw-r--r-- | internal/gitaly/hook/testdata/.gitkeep | 0 | ||||
-rw-r--r-- | internal/gitaly/service/hook/pack_objects.go | 100 | ||||
-rw-r--r-- | internal/gitaly/service/hook/pack_objects_test.go | 152 | ||||
-rw-r--r-- | internal/metadata/featureflag/feature_flags.go | 2 | ||||
-rw-r--r-- | proto/go/gitalypb/hook.pb.go | 287 | ||||
-rw-r--r-- | proto/go/gitalypb/hook_grpc.pb.go | 43 | ||||
-rw-r--r-- | proto/hook.proto | 15 | ||||
-rw-r--r-- | ruby/proto/gitaly/hook_pb.rb | 8 | ||||
-rw-r--r-- | ruby/proto/gitaly/hook_services_pb.rb | 3 |
13 files changed, 778 insertions, 131 deletions
diff --git a/cmd/gitaly-hooks/hooks.go b/cmd/gitaly-hooks/hooks.go index af26438fb..9e177b03f 100644 --- a/cmd/gitaly-hooks/hooks.go +++ b/cmd/gitaly-hooks/hooks.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "net" "os" "strings" @@ -13,10 +14,12 @@ import ( gitalyauth "gitlab.com/gitlab-org/gitaly/v14/auth" "gitlab.com/gitlab-org/gitaly/v14/client" "gitlab.com/gitlab-org/gitaly/v14/internal/git" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/pktline" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config/prometheus" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v14/internal/gitlab" + "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/internal/helper/env" gitalylog "gitlab.com/gitlab-org/gitaly/v14/internal/log" "gitlab.com/gitlab-org/gitaly/v14/internal/metadata/featureflag" @@ -339,8 +342,20 @@ func packObjectsHook(ctx context.Context, payload git.HooksPayload, hookClient g fixedArgs = append(fixedArgs, fixFilterQuoteBug(a)) } - if err := handlePackObjects(ctx, hookClient, payload.Repo, fixedArgs); err != nil { - logger.Logger().WithFields(logrus.Fields{"args": args}).WithError(err).Error("PackObjectsHook RPC failed") + var rpc string + var err error + if featureflag.PackObjectsHookWithSidechannel.IsEnabled(helper.OutgoingToIncoming(ctx)) { + rpc = "PackObjectsHookWithSidechannel" + err = handlePackObjectsWithSidechannel(ctx, hookClient, payload.Repo, fixedArgs) + } else { + rpc = "PackObjectsHook" + err = handlePackObjects(ctx, hookClient, payload.Repo, fixedArgs) + } + if err != nil { + logger.Logger().WithFields(logrus.Fields{ + "args": args, + "rpc": rpc, + }).WithError(err).Error("RPC failed") return 1, nil } @@ -406,3 +421,49 @@ type nopExitStatus struct { } func (nopExitStatus) GetExitStatus() *gitalypb.ExitStatus { return nil } + +func handlePackObjectsWithSidechannel(ctx context.Context, hookClient gitalypb.HookServiceClient, repo *gitalypb.Repository, args []string) error { + ctx, wt, err := hook.SetupSidechannel(ctx, func(c *net.UnixConn) error { + // We don't have to worry about concurrent reads and writes and + // deadlocks, because we're connected to git-upload-pack which follows + // the sequence: (1) write to stdin of pack-objects, (2) close stdin of + // pack-objects, (3) concurrently read from stdout and stderr of + // pack-objects. + if _, err := io.Copy(c, os.Stdin); err != nil { + return fmt.Errorf("copy stdin: %w", err) + } + if err := c.CloseWrite(); err != nil { + return fmt.Errorf("close write: %w", err) + } + + if err := pktline.EachSidebandPacket(c, func(band byte, data []byte) error { + var err error + switch band { + case 1: + _, err = os.Stdout.Write(data) + case 2: + _, err = os.Stderr.Write(data) + default: + err = fmt.Errorf("unexpected side band: %d", band) + } + return err + }); err != nil { + return fmt.Errorf("demux response: %w", err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("SetupSidechannel: %w", err) + } + defer wt.Close() + + if _, err := hookClient.PackObjectsHookWithSidechannel( + ctx, + &gitalypb.PackObjectsHookWithSidechannelRequest{Repository: repo, Args: args}, + ); err != nil { + return fmt.Errorf("call PackObjectsHookWithSidechannel: %w", err) + } + + return wt.Wait() +} diff --git a/cmd/gitaly-hooks/hooks_test.go b/cmd/gitaly-hooks/hooks_test.go index 1218b6017..60f24f227 100644 --- a/cmd/gitaly-hooks/hooks_test.go +++ b/cmd/gitaly-hooks/hooks_test.go @@ -14,6 +14,7 @@ import ( "strings" "testing" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/command" @@ -48,19 +49,19 @@ type proxyValues struct { var enabledFeatureFlag = featureflag.FeatureFlag{Name: "enabled-feature-flag", OnByDefault: false} var disabledFeatureFlag = featureflag.FeatureFlag{Name: "disabled-feature-flag", OnByDefault: true} -func rawFeatureFlags() featureflag.Raw { - ctx := featureflag.IncomingCtxWithFeatureFlag(context.Background(), enabledFeatureFlag) +func rawFeatureFlags(ctx context.Context) featureflag.Raw { + ctx = featureflag.IncomingCtxWithFeatureFlag(ctx, enabledFeatureFlag) ctx = featureflag.IncomingCtxWithDisabledFeatureFlag(ctx, disabledFeatureFlag) return featureflag.RawFromContext(ctx) } // envForHooks generates a set of environment variables for gitaly hooks -func envForHooks(t testing.TB, cfg config.Cfg, repo *gitalypb.Repository, glHookValues glHookValues, proxyValues proxyValues, gitPushOptions ...string) []string { +func envForHooks(t testing.TB, ctx context.Context, cfg config.Cfg, repo *gitalypb.Repository, glHookValues glHookValues, proxyValues proxyValues, gitPushOptions ...string) []string { payload, err := git.NewHooksPayload(cfg, repo, nil, &git.ReceiveHooksPayload{ UserID: glHookValues.GLID, Username: glHookValues.GLUsername, Protocol: glHookValues.GLProtocol, - }, git.AllHooks, rawFeatureFlags()).Env() + }, git.AllHooks, rawFeatureFlags(ctx)).Env() require.NoError(t, err) env := append(os.Environ(), []string{ @@ -190,6 +191,7 @@ func testHooksPrePostReceive(t *testing.T, cfg config.Cfg, repo *gitalypb.Reposi cmd.Stdin = stdin cmd.Env = envForHooks( t, + context.Background(), cfg, repo, glHookValues{ @@ -280,7 +282,7 @@ func testHooksUpdate(t *testing.T, cfg config.Cfg, glValues glHookValues) { updateHookPath, err := filepath.Abs("../../ruby/git-hooks/update") require.NoError(t, err) cmd := exec.Command(updateHookPath, refval, oldval, newval) - cmd.Env = envForHooks(t, cfg, repo, glValues, proxyValues{}) + cmd.Env = envForHooks(t, context.Background(), cfg, repo, glValues, proxyValues{}) cmd.Dir = repoPath tempDir := testhelper.TempDir(t) @@ -415,11 +417,11 @@ func TestHooksPostReceiveFailed(t *testing.T) { Protocol: glProtocol, }, git.PostReceiveHook, - rawFeatureFlags(), + rawFeatureFlags(context.Background()), ).Env() require.NoError(t, err) - env := envForHooks(t, cfg, repo, glHookValues{}, proxyValues{}) + env := envForHooks(t, context.Background(), cfg, repo, glHookValues{}, proxyValues{}) env = append(env, hooksPayload) cmd := exec.Command(postReceiveHookPath) @@ -479,7 +481,7 @@ func TestHooksNotAllowed(t *testing.T) { cmd.Stderr = &stderr cmd.Stdout = &stdout cmd.Stdin = strings.NewReader(changes) - cmd.Env = envForHooks(t, cfg, repo, + cmd.Env = envForHooks(t, context.Background(), cfg, repo, glHookValues{ GLID: glID, GLUsername: glUsername, @@ -577,8 +579,8 @@ func TestCheckBadCreds(t *testing.T) { require.Regexp(t, `Checking GitLab API access: .* level=error msg="Internal API error" .* error="authorization failed" method=GET status=401 url="http://127.0.0.1:[0-9]+/api/v4/internal/check"\nFAIL`, stdout.String()) } -func runHookServiceServer(t *testing.T, cfg config.Cfg) { - runHookServiceWithGitlabClient(t, cfg, gitlab.NewMockClient()) +func runHookServiceServer(t *testing.T, cfg config.Cfg, serverOpts ...testserver.GitalyServerOpt) { + runHookServiceWithGitlabClient(t, cfg, gitlab.NewMockClient(), serverOpts...) } type featureFlagAsserter struct { @@ -617,12 +619,17 @@ func (svc featureFlagAsserter) PackObjectsHook(stream gitalypb.HookService_PackO return svc.wrapped.PackObjectsHook(stream) } -func runHookServiceWithGitlabClient(t *testing.T, cfg config.Cfg, gitlabClient gitlab.Client) { +func (svc featureFlagAsserter) PackObjectsHookWithSidechannel(ctx context.Context, req *gitalypb.PackObjectsHookWithSidechannelRequest) (*gitalypb.PackObjectsHookWithSidechannelResponse, error) { + svc.assertFlags(ctx) + return svc.wrapped.PackObjectsHookWithSidechannel(ctx, req) +} + +func runHookServiceWithGitlabClient(t *testing.T, cfg config.Cfg, gitlabClient gitlab.Client, serverOpts ...testserver.GitalyServerOpt) { testserver.RunGitalyServer(t, cfg, nil, func(srv *grpc.Server, deps *service.Dependencies) { gitalypb.RegisterHookServiceServer(srv, featureFlagAsserter{ t: t, wrapped: hook.NewServer(deps.GetCfg(), deps.GetHookManager(), deps.GetGitCmdFactory(), deps.GetPackObjectsCache()), }) - }, testserver.WithGitLabClient(gitlabClient)) + }, append(serverOpts, testserver.WithGitLabClient(gitlabClient))...) } func requireContainsOnce(t *testing.T, s string, contains string) { @@ -661,8 +668,6 @@ func TestGitalyHooksPackObjects(t *testing.T) { testhelper.BuildGitalyHooks(t, cfg) testhelper.BuildGitalySSH(t, cfg) - env := envForHooks(t, cfg, repo, glHookValues{}, proxyValues{}) - baseArgs := []string{ cfg.Git.BinPath, "clone", @@ -675,26 +680,48 @@ func TestGitalyHooksPackObjects(t *testing.T) { testCases := []struct { desc string + ctx context.Context extraArgs []string + method string }{ - {desc: "regular clone"}, - {desc: "shallow clone", extraArgs: []string{"--depth=1"}}, - {desc: "partial clone", extraArgs: []string{"--filter=blob:none"}}, + {desc: "regular clone", method: "PackObjectsHook"}, + {desc: "shallow clone", extraArgs: []string{"--depth=1"}, method: "PackObjectsHook"}, + {desc: "partial clone", extraArgs: []string{"--filter=blob:none"}, method: "PackObjectsHook"}, + { + desc: "regular clone PackObjectsHookWithSidechannel", + ctx: featureflag.IncomingCtxWithFeatureFlag(context.Background(), featureflag.PackObjectsHookWithSidechannel), + method: "PackObjectsHookWithSidechannel", + }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - runHookServiceServer(t, cfg) + ctx := context.Background() + if tc.ctx != nil { + ctx = tc.ctx + } + + logger, hook := test.NewNullLogger() + runHookServiceServer(t, cfg, testserver.WithLogger(logger)) tempDir := testhelper.TempDir(t) args := append(baseArgs[1:], tc.extraArgs...) args = append(args, repoPath, tempDir) cmd := exec.Command(baseArgs[0], args...) - cmd.Env = env + cmd.Env = envForHooks(t, ctx, cfg, repo, glHookValues{}, proxyValues{}) cmd.Stderr = os.Stderr require.NoError(t, cmd.Run()) + + foundMethod := false + for _, e := range hook.AllEntries() { + if e.Data["grpc.service"] == "gitaly.HookService" { + require.Equal(t, tc.method, e.Data["grpc.method"]) + foundMethod = true + } + } + require.True(t, foundMethod) }) } } diff --git a/internal/gitaly/hook/sidechannel.go b/internal/gitaly/hook/sidechannel.go new file mode 100644 index 000000000..18381237c --- /dev/null +++ b/internal/gitaly/hook/sidechannel.go @@ -0,0 +1,109 @@ +package hook + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "os" + "path" + "time" + + gitaly_metadata "gitlab.com/gitlab-org/gitaly/v14/internal/metadata" + "google.golang.org/grpc/metadata" +) + +const ( + sidechannelHeader = "gitaly-sidechannel-socket" + sidechannelSocket = "sidechannel" +) + +type errInvalidSidechannelAddress struct{ string } + +func (e *errInvalidSidechannelAddress) Error() string { + return fmt.Sprintf("invalid side channel address: %q", e.string) +} + +// GetSidechannel looks for a sidechannel address in an incoming context +// and establishes a connection if it finds an address. +func GetSidechannel(ctx context.Context) (net.Conn, error) { + address := gitaly_metadata.GetValue(ctx, sidechannelHeader) + if path.Base(address) != sidechannelSocket { + return nil, &errInvalidSidechannelAddress{address} + } + + return net.DialTimeout("unix", address, time.Second) +} + +// SetupSidechannel creates a sidechannel listener in a tempdir and +// launches a goroutine that will run the callback if the listener +// receives a connection. The address of the listener is stored in the +// returned context, so that the caller can propagate it to a server. The +// caller must Close the SidechannelWaiter to prevent resource leaks. +func SetupSidechannel(ctx context.Context, callback func(*net.UnixConn) error) (context.Context, *SidechannelWaiter, error) { + socketDir, err := ioutil.TempDir("", "gitaly") + if err != nil { + return nil, nil, err + } + + address := path.Join(socketDir, sidechannelSocket) + l, err := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: address}) + if err != nil { + return nil, nil, err + } + + wt := &SidechannelWaiter{ + errC: make(chan error), + socketDir: socketDir, + listener: l, + } + go wt.run(callback) + + ctx = metadata.AppendToOutgoingContext(ctx, sidechannelHeader, address) + return ctx, wt, nil +} + +// SidechannelWaiter provides cleanup and error propagation for a +// sidechannel callback. +type SidechannelWaiter struct { + errC chan error + socketDir string + listener *net.UnixListener +} + +func (wt *SidechannelWaiter) run(callback func(*net.UnixConn) error) { + defer close(wt.errC) + + wt.errC <- func() error { + c, err := wt.listener.AcceptUnix() + if err != nil { + return err + } + defer c.Close() + + return callback(c) + }() +} + +// Close cleans up sidechannel resources. If the callback is already +// running, Close will block until the callback is done. +func (wt *SidechannelWaiter) Close() error { + // Run all cleanup actions _before_ checking errors, so that we cannot + // forget one. + cleanupErrors := []error{ + wt.listener.Close(), + os.RemoveAll(wt.socketDir), + wt.Wait(), + } + + for _, err := range cleanupErrors { + if err != nil { + return err + } + } + + return nil +} + +// Wait waits for the callback to run and returns its error value. +func (wt *SidechannelWaiter) Wait() error { return <-wt.errC } diff --git a/internal/gitaly/hook/sidechannel_test.go b/internal/gitaly/hook/sidechannel_test.go new file mode 100644 index 000000000..36dbf326e --- /dev/null +++ b/internal/gitaly/hook/sidechannel_test.go @@ -0,0 +1,60 @@ +package hook + +import ( + "context" + "io" + "io/ioutil" + "net" + "testing" + + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/v14/internal/helper" + "google.golang.org/grpc/metadata" +) + +func TestSidechannel(t *testing.T) { + // Client side + ctxOut, wt, err := SetupSidechannel( + context.Background(), + func(c *net.UnixConn) error { + _, err := io.WriteString(c, "ping") + return err + }, + ) + require.NoError(t, err) + defer wt.Close() + + // Server side + ctxIn := helper.OutgoingToIncoming(ctxOut) + c, err := GetSidechannel(ctxIn) + require.NoError(t, err) + defer c.Close() + + buf, err := ioutil.ReadAll(c) + require.NoError(t, err) + require.Equal(t, "ping", string(buf)) + + // Client side + require.NoError(t, wt.Wait()) +} + +func TestGetSidechannel(t *testing.T) { + testCases := []string{ + "foobar", + "sc.foo/../../bar", + "foo/../../bar", + "/etc/passwd", + } + + for _, tc := range testCases { + t.Run(tc, func(t *testing.T) { + ctx := metadata.NewIncomingContext( + context.Background(), + map[string][]string{sidechannelHeader: []string{tc}}, + ) + _, err := GetSidechannel(ctx) + require.Error(t, err) + require.Equal(t, &errInvalidSidechannelAddress{tc}, err) + }) + } +} diff --git a/internal/gitaly/hook/testdata/.gitkeep b/internal/gitaly/hook/testdata/.gitkeep new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/internal/gitaly/hook/testdata/.gitkeep diff --git a/internal/gitaly/service/hook/pack_objects.go b/internal/gitaly/service/hook/pack_objects.go index 7d6cedb05..3272c0eea 100644 --- a/internal/gitaly/service/hook/pack_objects.go +++ b/internal/gitaly/service/hook/pack_objects.go @@ -19,10 +19,12 @@ import ( "github.com/sirupsen/logrus" "gitlab.com/gitlab-org/gitaly/v14/internal/git" "gitlab.com/gitlab-org/gitaly/v14/internal/git/pktline" + "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v14/internal/helper" "gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb" "gitlab.com/gitlab-org/gitaly/v14/streamio" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" ) var ( @@ -41,10 +43,6 @@ var ( ) func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServer) error { - if s.packObjectsCache == nil { - return helper.ErrInternalf("packObjectsCache is not available") - } - firstRequest, err := stream.Recv() if err != nil { return helper.ErrInternal(err) @@ -59,7 +57,32 @@ func (s *server) PackObjectsHook(stream gitalypb.HookService_PackObjectsHookServ return helper.ErrInvalidArgumentf("invalid pack-objects command: %v: %w", firstRequest.Args, err) } - if err := s.packObjectsHook(stream, firstRequest, args); err != nil { + stdin := streamio.NewReader(func() ([]byte, error) { + resp, err := stream.Recv() + return resp.GetStdin(), err + }) + + output := func(r io.Reader) (int64, error) { + var n int64 + err := pktline.EachSidebandPacket(r, func(band byte, data []byte) error { + resp := &gitalypb.PackObjectsHookResponse{} + + switch band { + case bandStdout: + resp.Stdout = data + case bandStderr: + resp.Stderr = data + default: + return fmt.Errorf("invalid side band: %d", band) + } + + n += int64(len(data)) + return stream.Send(resp) + }) + return n, err + } + + if err := s.packObjectsHook(stream.Context(), firstRequest.Repository, firstRequest, args, stdin, output); err != nil { return helper.ErrInternal(err) } @@ -71,10 +94,8 @@ const ( bandStderr = 2 ) -func (s *server) packObjectsHook(stream gitalypb.HookService_PackObjectsHookServer, firstRequest *gitalypb.PackObjectsHookRequest, args *packObjectsArgs) error { - ctx := stream.Context() - - data, err := protojson.Marshal(firstRequest) +func (s *server) packObjectsHook(ctx context.Context, repo *gitalypb.Repository, reqHash proto.Message, args *packObjectsArgs, stdinReader io.Reader, output func(io.Reader) (int64, error)) error { + data, err := protojson.Marshal(reqHash) if err != nil { return err } @@ -84,7 +105,7 @@ func (s *server) packObjectsHook(stream gitalypb.HookService_PackObjectsHookServ return err } - stdin, err := bufferStdin(stream, h) + stdin, err := bufferStdin(stdinReader, h) if err != nil { return err } @@ -104,7 +125,7 @@ func (s *server) packObjectsHook(stream gitalypb.HookService_PackObjectsHookServ key := hex.EncodeToString(h.Sum(nil)) r, created, err := s.packObjectsCache.FindOrCreate(key, func(w io.Writer) error { - return s.runPackObjects(ctx, w, firstRequest.Repository, args, stdin, key) + return s.runPackObjects(ctx, w, repo, args, stdin, key) }) if err != nil { return err @@ -127,21 +148,8 @@ func (s *server) packObjectsHook(stream gitalypb.HookService_PackObjectsHookServ packObjectsServedBytes.Add(float64(servedBytes)) }() - if err := pktline.EachSidebandPacket(r, func(band byte, data []byte) error { - resp := &gitalypb.PackObjectsHookResponse{} - - switch band { - case bandStdout: - resp.Stdout = data - case bandStderr: - resp.Stderr = data - default: - return fmt.Errorf("invalid side band: %d", band) - } - - servedBytes += int64(len(data)) - return stream.Send(resp) - }); err != nil { + servedBytes, err = output(r) + if err != nil { return err } @@ -298,7 +306,7 @@ func (p *packObjectsArgs) subcmd() git.SubCmd { return sc } -func bufferStdin(stream gitalypb.HookService_PackObjectsHookServer, h hash.Hash) (_ io.ReadCloser, err error) { +func bufferStdin(r io.Reader, h hash.Hash) (_ io.ReadCloser, err error) { f, err := ioutil.TempFile("", "PackObjectsHook-stdin") if err != nil { return nil, err @@ -313,15 +321,7 @@ func bufferStdin(stream gitalypb.HookService_PackObjectsHookServer, h hash.Hash) return nil, err } - stdin := io.TeeReader( - streamio.NewReader(func() ([]byte, error) { - resp, err := stream.Recv() - return resp.GetStdin(), err - }), - h, - ) - - _, err = io.Copy(f, stdin) + _, err = io.Copy(f, io.TeeReader(r, h)) if err != nil { return nil, err } @@ -343,3 +343,31 @@ func (cw *countingWriter) Write(p []byte) (int, error) { cw.N += int64(n) return n, err } + +func (s *server) PackObjectsHookWithSidechannel(ctx context.Context, req *gitalypb.PackObjectsHookWithSidechannelRequest) (*gitalypb.PackObjectsHookWithSidechannelResponse, error) { + if req.GetRepository() == nil { + return nil, helper.ErrInvalidArgument(errors.New("repository is empty")) + } + + args, err := parsePackObjectsArgs(req.Args) + if err != nil { + return nil, helper.ErrInvalidArgumentf("invalid pack-objects command: %v: %w", req.Args, err) + } + + c, err := hook.GetSidechannel(ctx) + if err != nil { + return nil, err + } + defer c.Close() + + output := func(r io.Reader) (int64, error) { return io.Copy(c, r) } + if err := s.packObjectsHook(ctx, req.Repository, req, args, c, output); err != nil { + return nil, err + } + + if err := c.Close(); err != nil { + return nil, err + } + + return &gitalypb.PackObjectsHookWithSidechannelResponse{}, nil +} diff --git a/internal/gitaly/service/hook/pack_objects_test.go b/internal/gitaly/service/hook/pack_objects_test.go index a10f1466e..b7587e086 100644 --- a/internal/gitaly/service/hook/pack_objects_test.go +++ b/internal/gitaly/service/hook/pack_objects_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "net" "strings" "testing" "time" @@ -12,7 +13,9 @@ import ( "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitaly/v14/internal/git/gittest" + "gitlab.com/gitlab-org/gitaly/v14/internal/git/pktline" "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/config" + hookPkg "gitlab.com/gitlab-org/gitaly/v14/internal/gitaly/hook" "gitlab.com/gitlab-org/gitaly/v14/internal/streamcache" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper" "gitlab.com/gitlab-org/gitaly/v14/internal/testhelper/testcfg" @@ -303,3 +306,152 @@ func TestServer_PackObjectsHook_usesCache(t *testing.T) { require.NoError(t, entries[i].Err) } } + +func TestServer_PackObjectsHookWithSidechannel(t *testing.T) { + cfg, repo, repoPath := cfgWithCache(t) + + testCases := []struct { + desc string + stdin string + args []string + }{ + { + desc: "clone 1 branch", + stdin: "3dd08961455abf80ef9115f4afdc1c6f968b503c\n--not\n\n", + args: []string{"pack-objects", "--revs", "--thin", "--stdout", "--progress", "--delta-base-offset"}, + }, + { + desc: "shallow clone 1 branch", + stdin: "--shallow 1e292f8fedd741b75372e19097c76d327140c312\n1e292f8fedd741b75372e19097c76d327140c312\n--not\n\n", + args: []string{"--shallow-file", "", "pack-objects", "--revs", "--thin", "--stdout", "--shallow", "--progress", "--delta-base-offset", "--include-tag"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + ctx, cancel := testhelper.Context() + defer cancel() + + logger, hook := test.NewNullLogger() + serverSocketPath := runHooksServer(t, cfg, nil, testserver.WithLogger(logger)) + + var packets []string + ctx, wt, err := hookPkg.SetupSidechannel( + ctx, + func(c *net.UnixConn) error { + if _, err := io.WriteString(c, tc.stdin); err != nil { + return err + } + if err := c.CloseWrite(); err != nil { + return err + } + + scanner := pktline.NewScanner(c) + for scanner.Scan() { + packets = append(packets, scanner.Text()) + } + return scanner.Err() + }, + ) + require.NoError(t, err) + defer wt.Close() + + client, conn := newHooksClient(t, serverSocketPath) + defer conn.Close() + + _, err = client.PackObjectsHookWithSidechannel(ctx, &gitalypb.PackObjectsHookWithSidechannelRequest{ + Repository: repo, + Args: tc.args, + }) + require.NoError(t, err) + + require.NoError(t, wt.Wait()) + require.NotEmpty(t, packets) + + var packdata []byte + for _, pkt := range packets { + require.Greater(t, len(pkt), 4) + + switch band := pkt[4]; band { + case 1: + packdata = append(packdata, pkt[5:]...) + case 2: + default: + t.Fatalf("unexpected band: %d", band) + } + } + + gittest.ExecStream( + t, + cfg, + bytes.NewReader(packdata), + "-C", repoPath, "index-pack", "--stdin", "--fix-thin", + ) + + for _, msg := range []string{"served bytes", "generated bytes"} { + t.Run(msg, func(t *testing.T) { + var entry *logrus.Entry + for _, e := range hook.AllEntries() { + if e.Message == msg { + entry = e + } + } + + require.NotNil(t, entry) + require.NotEmpty(t, entry.Data["cache_key"]) + require.Greater(t, entry.Data["bytes"], int64(0)) + }) + } + + t.Run("pack file compression statistic", func(t *testing.T) { + var entry *logrus.Entry + for _, e := range hook.AllEntries() { + if e.Message == "pack file compression statistic" { + entry = e + } + } + + require.NotNil(t, entry) + total := entry.Data["pack.stat"].(string) + require.True(t, strings.HasPrefix(total, "Total ")) + require.False(t, strings.Contains(total, "\n")) + }) + }) + } +} + +func TestServer_PackObjectsHookWithSidechannel_invalidArgument(t *testing.T) { + cfg, repo, _ := testcfg.BuildWithRepo(t) + serverSocketPath := runHooksServer(t, cfg, nil) + + ctx, cancel := testhelper.Context() + defer cancel() + + testCases := []struct { + desc string + req *gitalypb.PackObjectsHookWithSidechannelRequest + }{ + { + desc: "empty", + req: &gitalypb.PackObjectsHookWithSidechannelRequest{}, + }, + { + desc: "repo, no args", + req: &gitalypb.PackObjectsHookWithSidechannelRequest{Repository: repo}, + }, + { + desc: "repo, bad args", + req: &gitalypb.PackObjectsHookWithSidechannelRequest{Repository: repo, Args: []string{"rm", "-rf"}}, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + client, conn := newHooksClient(t, serverSocketPath) + defer conn.Close() + + _, err := client.PackObjectsHookWithSidechannel(ctx, tc.req) + testhelper.RequireGrpcError(t, err, codes.InvalidArgument) + }) + } +} diff --git a/internal/metadata/featureflag/feature_flags.go b/internal/metadata/featureflag/feature_flags.go index 3804ddf43..1f402581a 100644 --- a/internal/metadata/featureflag/feature_flags.go +++ b/internal/metadata/featureflag/feature_flags.go @@ -8,6 +8,8 @@ var ( GoSetConfig = FeatureFlag{Name: "go_set_config", OnByDefault: true} // GoUserApplyPatch enables the Go implementation of UserApplyPatch GoUserApplyPatch = FeatureFlag{Name: "go_user_apply_patch", OnByDefault: true} + // PackObjectsHookWithSidechannel enables Unix socket sidechannels in 'gitaly-hooks git pack-objects'. + PackObjectsHookWithSidechannel = FeatureFlag{Name: "pack_objects_hook_with_sidechannel", OnByDefault: false} ) // All includes all feature flags. diff --git a/proto/go/gitalypb/hook.pb.go b/proto/go/gitalypb/hook.pb.go index a25bf0254..51fbaf539 100644 --- a/proto/go/gitalypb/hook.pb.go +++ b/proto/go/gitalypb/hook.pb.go @@ -735,6 +735,100 @@ func (x *PackObjectsHookResponse) GetStderr() []byte { return nil } +type PackObjectsHookWithSidechannelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Repository *Repository `protobuf:"bytes,1,opt,name=repository,proto3" json:"repository,omitempty"` + // args contains the arguments passed to the pack-objects hook, without the leading "git" + Args []string `protobuf:"bytes,2,rep,name=args,proto3" json:"args,omitempty"` +} + +func (x *PackObjectsHookWithSidechannelRequest) Reset() { + *x = PackObjectsHookWithSidechannelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_hook_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PackObjectsHookWithSidechannelRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PackObjectsHookWithSidechannelRequest) ProtoMessage() {} + +func (x *PackObjectsHookWithSidechannelRequest) ProtoReflect() protoreflect.Message { + mi := &file_hook_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PackObjectsHookWithSidechannelRequest.ProtoReflect.Descriptor instead. +func (*PackObjectsHookWithSidechannelRequest) Descriptor() ([]byte, []int) { + return file_hook_proto_rawDescGZIP(), []int{10} +} + +func (x *PackObjectsHookWithSidechannelRequest) GetRepository() *Repository { + if x != nil { + return x.Repository + } + return nil +} + +func (x *PackObjectsHookWithSidechannelRequest) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +type PackObjectsHookWithSidechannelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *PackObjectsHookWithSidechannelResponse) Reset() { + *x = PackObjectsHookWithSidechannelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_hook_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PackObjectsHookWithSidechannelResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PackObjectsHookWithSidechannelResponse) ProtoMessage() {} + +func (x *PackObjectsHookWithSidechannelResponse) ProtoReflect() protoreflect.Message { + mi := &file_hook_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PackObjectsHookWithSidechannelResponse.ProtoReflect.Descriptor instead. +func (*PackObjectsHookWithSidechannelResponse) Descriptor() ([]byte, []int) { + return file_hook_proto_rawDescGZIP(), []int{11} +} + var File_hook_proto protoreflect.FileDescriptor var file_hook_proto_rawDesc = []byte{ @@ -842,42 +936,60 @@ var file_hook_proto_rawDesc = []byte{ 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, - 0x32, 0xf4, 0x03, 0x0a, 0x0b, 0x48, 0x6f, 0x6f, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x5b, 0x0a, 0x0e, 0x50, 0x72, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, - 0x6f, 0x6b, 0x12, 0x1d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x72, 0x65, 0x52, + 0x22, 0x75, 0x0a, 0x25, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, + 0x6f, 0x6f, 0x6b, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x64, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x38, 0x0a, 0x0a, 0x72, 0x65, 0x70, + 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, + 0x79, 0x42, 0x04, 0x98, 0xc6, 0x2c, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x6f, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x22, 0x28, 0x0a, 0x26, 0x50, 0x61, 0x63, 0x6b, 0x4f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x6f, 0x6f, 0x6b, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, + 0x64, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x32, 0xfe, 0x04, 0x0a, 0x0b, 0x48, 0x6f, 0x6f, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x5b, 0x0a, 0x0e, 0x50, 0x72, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, + 0x6f, 0x6f, 0x6b, 0x12, 0x1d, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x72, 0x65, + 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x72, 0x65, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5e, + 0x0a, 0x0f, 0x50, 0x6f, 0x73, 0x74, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, + 0x6b, 0x12, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x72, 0x65, 0x52, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5e, 0x0a, - 0x0f, 0x50, 0x6f, 0x73, 0x74, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, - 0x12, 0x1e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x52, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x52, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4d, 0x0a, - 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x19, 0x2e, 0x67, 0x69, - 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x18, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, - 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, - 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, - 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5e, 0x0a, 0x0f, 0x50, 0x61, 0x63, 0x6b, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x1e, 0x2e, 0x67, 0x69, 0x74, - 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, - 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x69, 0x74, + 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x6f, 0x73, 0x74, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4d, + 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x19, 0x2e, 0x67, + 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, + 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, 0x08, 0x02, 0x30, 0x01, 0x12, 0x79, 0x0a, + 0x18, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x27, 0x2e, 0x67, 0x69, 0x74, 0x61, + 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, + 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5e, 0x0a, 0x0f, 0x50, 0x61, 0x63, 0x6b, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x6f, 0x6f, 0x6b, 0x12, 0x1e, 0x2e, 0x67, 0x69, + 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x69, + 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, + 0x28, 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x12, 0x87, 0x01, 0x0a, 0x1e, 0x50, 0x61, 0x63, + 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, 0x6f, 0x6f, 0x6b, 0x57, 0x69, 0x74, 0x68, + 0x53, 0x69, 0x64, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x2d, 0x2e, 0x67, 0x69, + 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x48, 0x6f, 0x6f, 0x6b, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x64, 0x65, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x48, - 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, - 0x02, 0x08, 0x02, 0x28, 0x01, 0x30, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, - 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x34, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6f, 0x6b, 0x57, 0x69, 0x74, 0x68, 0x53, 0x69, 0x64, 0x65, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x06, 0xfa, 0x97, 0x28, 0x02, + 0x08, 0x02, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2d, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x69, 0x74, 0x61, + 0x6c, 0x79, 0x2f, 0x76, 0x31, 0x34, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, + 0x67, 0x69, 0x74, 0x61, 0x6c, 0x79, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -893,48 +1005,53 @@ func file_hook_proto_rawDescGZIP() []byte { } var file_hook_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_hook_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_hook_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_hook_proto_goTypes = []interface{}{ - (ReferenceTransactionHookRequest_State)(0), // 0: gitaly.ReferenceTransactionHookRequest.State - (*PreReceiveHookRequest)(nil), // 1: gitaly.PreReceiveHookRequest - (*PreReceiveHookResponse)(nil), // 2: gitaly.PreReceiveHookResponse - (*PostReceiveHookRequest)(nil), // 3: gitaly.PostReceiveHookRequest - (*PostReceiveHookResponse)(nil), // 4: gitaly.PostReceiveHookResponse - (*UpdateHookRequest)(nil), // 5: gitaly.UpdateHookRequest - (*UpdateHookResponse)(nil), // 6: gitaly.UpdateHookResponse - (*ReferenceTransactionHookRequest)(nil), // 7: gitaly.ReferenceTransactionHookRequest - (*ReferenceTransactionHookResponse)(nil), // 8: gitaly.ReferenceTransactionHookResponse - (*PackObjectsHookRequest)(nil), // 9: gitaly.PackObjectsHookRequest - (*PackObjectsHookResponse)(nil), // 10: gitaly.PackObjectsHookResponse - (*Repository)(nil), // 11: gitaly.Repository - (*ExitStatus)(nil), // 12: gitaly.ExitStatus + (ReferenceTransactionHookRequest_State)(0), // 0: gitaly.ReferenceTransactionHookRequest.State + (*PreReceiveHookRequest)(nil), // 1: gitaly.PreReceiveHookRequest + (*PreReceiveHookResponse)(nil), // 2: gitaly.PreReceiveHookResponse + (*PostReceiveHookRequest)(nil), // 3: gitaly.PostReceiveHookRequest + (*PostReceiveHookResponse)(nil), // 4: gitaly.PostReceiveHookResponse + (*UpdateHookRequest)(nil), // 5: gitaly.UpdateHookRequest + (*UpdateHookResponse)(nil), // 6: gitaly.UpdateHookResponse + (*ReferenceTransactionHookRequest)(nil), // 7: gitaly.ReferenceTransactionHookRequest + (*ReferenceTransactionHookResponse)(nil), // 8: gitaly.ReferenceTransactionHookResponse + (*PackObjectsHookRequest)(nil), // 9: gitaly.PackObjectsHookRequest + (*PackObjectsHookResponse)(nil), // 10: gitaly.PackObjectsHookResponse + (*PackObjectsHookWithSidechannelRequest)(nil), // 11: gitaly.PackObjectsHookWithSidechannelRequest + (*PackObjectsHookWithSidechannelResponse)(nil), // 12: gitaly.PackObjectsHookWithSidechannelResponse + (*Repository)(nil), // 13: gitaly.Repository + (*ExitStatus)(nil), // 14: gitaly.ExitStatus } var file_hook_proto_depIdxs = []int32{ - 11, // 0: gitaly.PreReceiveHookRequest.repository:type_name -> gitaly.Repository - 12, // 1: gitaly.PreReceiveHookResponse.exit_status:type_name -> gitaly.ExitStatus - 11, // 2: gitaly.PostReceiveHookRequest.repository:type_name -> gitaly.Repository - 12, // 3: gitaly.PostReceiveHookResponse.exit_status:type_name -> gitaly.ExitStatus - 11, // 4: gitaly.UpdateHookRequest.repository:type_name -> gitaly.Repository - 12, // 5: gitaly.UpdateHookResponse.exit_status:type_name -> gitaly.ExitStatus - 11, // 6: gitaly.ReferenceTransactionHookRequest.repository:type_name -> gitaly.Repository + 13, // 0: gitaly.PreReceiveHookRequest.repository:type_name -> gitaly.Repository + 14, // 1: gitaly.PreReceiveHookResponse.exit_status:type_name -> gitaly.ExitStatus + 13, // 2: gitaly.PostReceiveHookRequest.repository:type_name -> gitaly.Repository + 14, // 3: gitaly.PostReceiveHookResponse.exit_status:type_name -> gitaly.ExitStatus + 13, // 4: gitaly.UpdateHookRequest.repository:type_name -> gitaly.Repository + 14, // 5: gitaly.UpdateHookResponse.exit_status:type_name -> gitaly.ExitStatus + 13, // 6: gitaly.ReferenceTransactionHookRequest.repository:type_name -> gitaly.Repository 0, // 7: gitaly.ReferenceTransactionHookRequest.state:type_name -> gitaly.ReferenceTransactionHookRequest.State - 12, // 8: gitaly.ReferenceTransactionHookResponse.exit_status:type_name -> gitaly.ExitStatus - 11, // 9: gitaly.PackObjectsHookRequest.repository:type_name -> gitaly.Repository - 1, // 10: gitaly.HookService.PreReceiveHook:input_type -> gitaly.PreReceiveHookRequest - 3, // 11: gitaly.HookService.PostReceiveHook:input_type -> gitaly.PostReceiveHookRequest - 5, // 12: gitaly.HookService.UpdateHook:input_type -> gitaly.UpdateHookRequest - 7, // 13: gitaly.HookService.ReferenceTransactionHook:input_type -> gitaly.ReferenceTransactionHookRequest - 9, // 14: gitaly.HookService.PackObjectsHook:input_type -> gitaly.PackObjectsHookRequest - 2, // 15: gitaly.HookService.PreReceiveHook:output_type -> gitaly.PreReceiveHookResponse - 4, // 16: gitaly.HookService.PostReceiveHook:output_type -> gitaly.PostReceiveHookResponse - 6, // 17: gitaly.HookService.UpdateHook:output_type -> gitaly.UpdateHookResponse - 8, // 18: gitaly.HookService.ReferenceTransactionHook:output_type -> gitaly.ReferenceTransactionHookResponse - 10, // 19: gitaly.HookService.PackObjectsHook:output_type -> gitaly.PackObjectsHookResponse - 15, // [15:20] is the sub-list for method output_type - 10, // [10:15] is the sub-list for method input_type - 10, // [10:10] is the sub-list for extension type_name - 10, // [10:10] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 14, // 8: gitaly.ReferenceTransactionHookResponse.exit_status:type_name -> gitaly.ExitStatus + 13, // 9: gitaly.PackObjectsHookRequest.repository:type_name -> gitaly.Repository + 13, // 10: gitaly.PackObjectsHookWithSidechannelRequest.repository:type_name -> gitaly.Repository + 1, // 11: gitaly.HookService.PreReceiveHook:input_type -> gitaly.PreReceiveHookRequest + 3, // 12: gitaly.HookService.PostReceiveHook:input_type -> gitaly.PostReceiveHookRequest + 5, // 13: gitaly.HookService.UpdateHook:input_type -> gitaly.UpdateHookRequest + 7, // 14: gitaly.HookService.ReferenceTransactionHook:input_type -> gitaly.ReferenceTransactionHookRequest + 9, // 15: gitaly.HookService.PackObjectsHook:input_type -> gitaly.PackObjectsHookRequest + 11, // 16: gitaly.HookService.PackObjectsHookWithSidechannel:input_type -> gitaly.PackObjectsHookWithSidechannelRequest + 2, // 17: gitaly.HookService.PreReceiveHook:output_type -> gitaly.PreReceiveHookResponse + 4, // 18: gitaly.HookService.PostReceiveHook:output_type -> gitaly.PostReceiveHookResponse + 6, // 19: gitaly.HookService.UpdateHook:output_type -> gitaly.UpdateHookResponse + 8, // 20: gitaly.HookService.ReferenceTransactionHook:output_type -> gitaly.ReferenceTransactionHookResponse + 10, // 21: gitaly.HookService.PackObjectsHook:output_type -> gitaly.PackObjectsHookResponse + 12, // 22: gitaly.HookService.PackObjectsHookWithSidechannel:output_type -> gitaly.PackObjectsHookWithSidechannelResponse + 17, // [17:23] is the sub-list for method output_type + 11, // [11:17] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_hook_proto_init() } @@ -1065,6 +1182,30 @@ func file_hook_proto_init() { return nil } } + file_hook_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PackObjectsHookWithSidechannelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_hook_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PackObjectsHookWithSidechannelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1072,7 +1213,7 @@ func file_hook_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_hook_proto_rawDesc, NumEnums: 1, - NumMessages: 10, + NumMessages: 12, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/go/gitalypb/hook_grpc.pb.go b/proto/go/gitalypb/hook_grpc.pb.go index 7efe8249f..16324fac5 100644 --- a/proto/go/gitalypb/hook_grpc.pb.go +++ b/proto/go/gitalypb/hook_grpc.pb.go @@ -26,6 +26,9 @@ type HookServiceClient interface { // uploadpack.packObjectsHook mechanism. It generates a stream of packed // Git objects. PackObjectsHook(ctx context.Context, opts ...grpc.CallOption) (HookService_PackObjectsHookClient, error) + // PackObjectsHookWithSidechannel is an optimized version of PackObjectsHook that uses + // a unix socket side channel. + PackObjectsHookWithSidechannel(ctx context.Context, in *PackObjectsHookWithSidechannelRequest, opts ...grpc.CallOption) (*PackObjectsHookWithSidechannelResponse, error) } type hookServiceClient struct { @@ -192,6 +195,15 @@ func (x *hookServicePackObjectsHookClient) Recv() (*PackObjectsHookResponse, err return m, nil } +func (c *hookServiceClient) PackObjectsHookWithSidechannel(ctx context.Context, in *PackObjectsHookWithSidechannelRequest, opts ...grpc.CallOption) (*PackObjectsHookWithSidechannelResponse, error) { + out := new(PackObjectsHookWithSidechannelResponse) + err := c.cc.Invoke(ctx, "/gitaly.HookService/PackObjectsHookWithSidechannel", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // HookServiceServer is the server API for HookService service. // All implementations must embed UnimplementedHookServiceServer // for forward compatibility @@ -204,6 +216,9 @@ type HookServiceServer interface { // uploadpack.packObjectsHook mechanism. It generates a stream of packed // Git objects. PackObjectsHook(HookService_PackObjectsHookServer) error + // PackObjectsHookWithSidechannel is an optimized version of PackObjectsHook that uses + // a unix socket side channel. + PackObjectsHookWithSidechannel(context.Context, *PackObjectsHookWithSidechannelRequest) (*PackObjectsHookWithSidechannelResponse, error) mustEmbedUnimplementedHookServiceServer() } @@ -226,6 +241,9 @@ func (UnimplementedHookServiceServer) ReferenceTransactionHook(HookService_Refer func (UnimplementedHookServiceServer) PackObjectsHook(HookService_PackObjectsHookServer) error { return status.Errorf(codes.Unimplemented, "method PackObjectsHook not implemented") } +func (UnimplementedHookServiceServer) PackObjectsHookWithSidechannel(context.Context, *PackObjectsHookWithSidechannelRequest) (*PackObjectsHookWithSidechannelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PackObjectsHookWithSidechannel not implemented") +} func (UnimplementedHookServiceServer) mustEmbedUnimplementedHookServiceServer() {} // UnsafeHookServiceServer may be embedded to opt out of forward compatibility for this service. @@ -364,13 +382,36 @@ func (x *hookServicePackObjectsHookServer) Recv() (*PackObjectsHookRequest, erro return m, nil } +func _HookService_PackObjectsHookWithSidechannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PackObjectsHookWithSidechannelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServiceServer).PackObjectsHookWithSidechannel(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/gitaly.HookService/PackObjectsHookWithSidechannel", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServiceServer).PackObjectsHookWithSidechannel(ctx, req.(*PackObjectsHookWithSidechannelRequest)) + } + return interceptor(ctx, in, info, handler) +} + // HookService_ServiceDesc is the grpc.ServiceDesc for HookService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) var HookService_ServiceDesc = grpc.ServiceDesc{ ServiceName: "gitaly.HookService", HandlerType: (*HookServiceServer)(nil), - Methods: []grpc.MethodDesc{}, + Methods: []grpc.MethodDesc{ + { + MethodName: "PackObjectsHookWithSidechannel", + Handler: _HookService_PackObjectsHookWithSidechannel_Handler, + }, + }, Streams: []grpc.StreamDesc{ { StreamName: "PreReceiveHook", diff --git a/proto/hook.proto b/proto/hook.proto index 71dd75cd6..1c8a2001f 100644 --- a/proto/hook.proto +++ b/proto/hook.proto @@ -36,6 +36,13 @@ service HookService { op: ACCESSOR }; } + // PackObjectsHookWithSidechannel is an optimized version of PackObjectsHook that uses + // a unix socket side channel. + rpc PackObjectsHookWithSidechannel(PackObjectsHookWithSidechannelRequest) returns (PackObjectsHookWithSidechannelResponse) { + option (op_type) = { + op: ACCESSOR + }; + } } message PreReceiveHookRequest { @@ -110,3 +117,11 @@ message PackObjectsHookResponse { // stderr contains progress messages (such as "Enumerating objects ...") bytes stderr = 2; } + +message PackObjectsHookWithSidechannelRequest { + Repository repository = 1 [(target_repository)=true]; + // args contains the arguments passed to the pack-objects hook, without the leading "git" + repeated string args = 2; +} + +message PackObjectsHookWithSidechannelResponse {} diff --git a/ruby/proto/gitaly/hook_pb.rb b/ruby/proto/gitaly/hook_pb.rb index 20c8cb8b1..6ac768187 100644 --- a/ruby/proto/gitaly/hook_pb.rb +++ b/ruby/proto/gitaly/hook_pb.rb @@ -66,6 +66,12 @@ Google::Protobuf::DescriptorPool.generated_pool.build do optional :stdout, :bytes, 1 optional :stderr, :bytes, 2 end + add_message "gitaly.PackObjectsHookWithSidechannelRequest" do + optional :repository, :message, 1, "gitaly.Repository" + repeated :args, :string, 2 + end + add_message "gitaly.PackObjectsHookWithSidechannelResponse" do + end end end @@ -81,4 +87,6 @@ module Gitaly ReferenceTransactionHookResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.ReferenceTransactionHookResponse").msgclass PackObjectsHookRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookRequest").msgclass PackObjectsHookResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookResponse").msgclass + PackObjectsHookWithSidechannelRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookWithSidechannelRequest").msgclass + PackObjectsHookWithSidechannelResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("gitaly.PackObjectsHookWithSidechannelResponse").msgclass end diff --git a/ruby/proto/gitaly/hook_services_pb.rb b/ruby/proto/gitaly/hook_services_pb.rb index de4048201..1059d58bd 100644 --- a/ruby/proto/gitaly/hook_services_pb.rb +++ b/ruby/proto/gitaly/hook_services_pb.rb @@ -22,6 +22,9 @@ module Gitaly # uploadpack.packObjectsHook mechanism. It generates a stream of packed # Git objects. rpc :PackObjectsHook, stream(::Gitaly::PackObjectsHookRequest), stream(::Gitaly::PackObjectsHookResponse) + # PackObjectsHookWithSidechannel is an optimized version of PackObjectsHook that uses + # a unix socket side channel. + rpc :PackObjectsHookWithSidechannel, ::Gitaly::PackObjectsHookWithSidechannelRequest, ::Gitaly::PackObjectsHookWithSidechannelResponse end Stub = Service.rpc_stub_class |