Gitaly-ruby server should not depend on the global state

Configuration of the gitaly-ruby server depends on the
global config.Config variable. As we are on the road to
removing it this change break this dependency by introducing
a constructor function that accepts required configuration.
That change requires refactoring of the gitaly-ruby test
instance setup - now it should be created only after
configuration is properly set.

Part of: https://gitlab.com/gitlab-org/gitaly/-/issues/2699

16 files changed, 59 insertions, 59 deletions

diff --git a/internal/gitaly/rubyserver/concurrency_test.go b/internal/gitaly/rubyserver/concurrency_test.go
index 1472ec389..686269a18 100644
--- a/internal/gitaly/rubyserver/concurrency_test.go
+++ b/internal/gitaly/rubyserver/concurrency_test.go
@@ -30,7 +30,7 @@ func waitPing(s *Server) error {
 func BenchmarkConcurrency(b *testing.B) {
 	config.Config.Ruby.NumWorkers = 2
 
-	s := &Server{}
+	s := New(config.Config)
 	require.NoError(b, s.Start())
 	defer s.Stop()
 
diff --git a/internal/gitaly/rubyserver/health_test.go b/internal/gitaly/rubyserver/health_test.go
index c074435e8..1ef498f07 100644
--- a/internal/gitaly/rubyserver/health_test.go
+++ b/internal/gitaly/rubyserver/health_test.go
@@ -5,10 +5,11 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
 )
 
 func TestPingSuccess(t *testing.T) {
-	s := &Server{}
+	s := New(config.Config)
 	require.NoError(t, s.Start())
 	defer s.Stop()
 
diff --git a/internal/gitaly/rubyserver/rubyserver.go b/internal/gitaly/rubyserver/rubyserver.go
index 80f9617ec..8b588eb1c 100644
--- a/internal/gitaly/rubyserver/rubyserver.go
+++ b/internal/gitaly/rubyserver/rubyserver.go
@@ -67,6 +67,7 @@ func setupEnv(cfg config.Cfg) []string {
 
 // Server represents a gitaly-ruby helper process.
 type Server struct {
+	cfg          config.Cfg
 	startOnce    sync.Once
 	startErr     error
 	workers      []*worker
@@ -74,6 +75,11 @@ type Server struct {
 	clientConn   *grpc.ClientConn
 }
 
+// New returns a new instance of the server.
+func New(cfg config.Cfg) *Server {
+	return &Server{cfg: cfg}
+}
+
 // Stop shuts down the gitaly-ruby helper process and cleans up resources.
 func (s *Server) Stop() {
 	if s != nil {
@@ -102,7 +108,7 @@ func (s *Server) start() error {
 		return err
 	}
 
-	cfg := config.Config
+	cfg := s.cfg
 	env := setupEnv(cfg)
 
 	gitalyRuby := filepath.Join(cfg.Ruby.Dir, "bin", "gitaly-ruby")
diff --git a/internal/gitaly/rubyserver/rubyserver_test.go b/internal/gitaly/rubyserver/rubyserver_test.go
index 952d5959e..e6b96409e 100644
--- a/internal/gitaly/rubyserver/rubyserver_test.go
+++ b/internal/gitaly/rubyserver/rubyserver_test.go
@@ -20,7 +20,7 @@ import (
 func TestStopSafe(t *testing.T) {
 	badServers := []*Server{
 		nil,
-		&Server{},
+		New(config.Cfg{}),
 	}
 
 	for _, bs := range badServers {
diff --git a/internal/gitaly/server/auth_test.go b/internal/gitaly/server/auth_test.go
index 4d0f37c12..72baecd56 100644
--- a/internal/gitaly/server/auth_test.go
+++ b/internal/gitaly/server/auth_test.go
@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	gitalyauth "gitlab.com/gitlab-org/gitaly/auth"
@@ -30,6 +31,8 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 )
 
+var rubyServer *rubyserver.Server
+
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
 }
@@ -38,12 +41,19 @@ func testMain(m *testing.M) int {
 	defer testhelper.MustHaveNoChildProcess()
 	cleanup := testhelper.Configure()
 	defer cleanup()
-	testhelper.ConfigureGitalyHooksBinary()
+
+	rubyServer = rubyserver.New(config.Config)
+	if err := rubyServer.Start(); err != nil {
+		log.Error(err)
+		return 1
+	}
+	defer rubyServer.Stop()
+
 	return m.Run()
 }
 
 func TestSanity(t *testing.T) {
-	serverSocketPath, clean := runServer(t)
+	serverSocketPath, clean := runServer(t, config.Cfg{})
 	defer clean()
 
 	connOpts := []grpc.DialOption{
@@ -84,11 +94,6 @@ func TestTLSSanity(t *testing.T) {
 }
 
 func TestAuthFailures(t *testing.T) {
-	defer func(oldAuth auth.Config) {
-		config.Config.Auth = oldAuth
-	}(config.Config.Auth)
-	config.Config.Auth.Token = "quxbaz"
-
 	testCases := []struct {
 		desc string
 		opts []grpc.DialOption
@@ -108,7 +113,7 @@ func TestAuthFailures(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			serverSocketPath, clean := runServer(t)
+			serverSocketPath, clean := runServer(t, config.Cfg{Auth: auth.Config{Token: "quxbaz"}})
 			defer clean()
 
 			connOpts := append(tc.opts, grpc.WithInsecure())
@@ -156,7 +161,7 @@ func TestAuthSuccess(t *testing.T) {
 			config.Config.Auth.Token = tc.token
 			config.Config.Auth.Transitioning = !tc.required
 
-			serverSocketPath, clean := runServer(t)
+			serverSocketPath, clean := runServer(t, config.Config)
 			defer clean()
 
 			connOpts := append(tc.opts, grpc.WithInsecure())
@@ -201,14 +206,13 @@ func newOperationClient(t *testing.T, serverSocketPath string) (gitalypb.Operati
 	return gitalypb.NewOperationServiceClient(conn), conn
 }
 
-func runServerWithRuby(t *testing.T, ruby *rubyserver.Server) (string, func()) {
+func runServer(t *testing.T, cfg config.Cfg) (string, func()) {
 	conns := client.NewPool()
-	cfg := config.Config
 	locator := config.NewLocator(cfg)
 	txManager := transaction.NewManager(cfg)
 	hookManager := hook.NewManager(locator, txManager, hook.GitlabAPIStub, cfg)
 	gitCmdFactory := git.NewExecCommandFactory(cfg)
-	srv, err := New(false, ruby, hookManager, txManager, cfg, conns, locator, gitCmdFactory)
+	srv, err := New(false, rubyServer, hookManager, txManager, cfg, conns, locator, gitCmdFactory)
 	require.NoError(t, err)
 
 	serverSocketPath := testhelper.GetTemporaryGitalySocketFileName(t)
@@ -220,14 +224,9 @@ func runServerWithRuby(t *testing.T, ruby *rubyserver.Server) (string, func()) {
 	return "unix://" + serverSocketPath, func() {
 		conns.Close()
 		srv.Stop()
-		ruby.Stop()
 	}
 }
 
-func runServer(t *testing.T) (string, func()) {
-	return runServerWithRuby(t, nil)
-}
-
 //go:generate openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -out testdata/gitalycert.pem -keyout testdata/gitalykey.pem -subj "/C=US/ST=California/L=San Francisco/O=GitLab/OU=GitLab-Shell/CN=localhost" -addext "subjectAltName = IP:127.0.0.1, DNS:localhost"
 func runSecureServer(t *testing.T) (string, func()) {
 	t.Helper()
@@ -257,7 +256,7 @@ func TestUnaryNoAuth(t *testing.T) {
 		config.Config.Auth.Token = oldToken
 	}()
 
-	path, clean := runServer(t)
+	path, clean := runServer(t, config.Config)
 	defer clean()
 
 	connOpts := []grpc.DialOption{
@@ -285,7 +284,7 @@ func TestStreamingNoAuth(t *testing.T) {
 		config.Config.Auth.Token = oldToken
 	}()
 
-	path, clean := runServer(t)
+	path, clean := runServer(t, config.Config)
 	defer clean()
 
 	connOpts := []grpc.DialOption{
@@ -327,7 +326,7 @@ func TestAuthBeforeLimit(t *testing.T) {
 	defer cleanup()
 	config.Config.GitlabShell.Dir = gitlabShellDir
 
-	url, cleanup := testhelper.SetupAndStartGitlabServer(t, &testhelper.GitlabTestServerOptions{
+	url, cleanup := testhelper.SetupAndStartGitlabServer(t, config.Config.GitlabShell.Dir, &testhelper.GitlabTestServerOptions{
 		SecretToken:                 "secretToken",
 		GLID:                        testhelper.GlID,
 		GLRepository:                testRepo.GlRepository,
@@ -343,11 +342,8 @@ func TestAuthBeforeLimit(t *testing.T) {
 	config.ConfigureConcurrencyLimits(config.Config)
 
 	config.Config.Gitlab.URL = url
-	var RubyServer rubyserver.Server
-	if err := RubyServer.Start(); err != nil {
-		t.Fatal(err)
-	}
-	serverSocketPath, clean := runServerWithRuby(t, &RubyServer)
+
+	serverSocketPath, clean := runServer(t, config.Config)
 	defer clean()
 
 	client, conn := newOperationClient(t, serverSocketPath)
diff --git a/internal/gitaly/server/server_factory.go b/internal/gitaly/server/server_factory.go
index de4e61914..d610478a8 100644
--- a/internal/gitaly/server/server_factory.go
+++ b/internal/gitaly/server/server_factory.go
@@ -39,7 +39,7 @@ type GitalyServerFactory struct {
 func NewGitalyServerFactory(cfg config.Cfg, hookManager hook.Manager, txManager transaction.Manager, conns *client.Pool, locator storage.Locator, gitCmdFactory git.CommandFactory) *GitalyServerFactory {
 	return &GitalyServerFactory{
 		cfg:           cfg,
-		ruby:          &rubyserver.Server{},
+		ruby:          rubyserver.New(cfg),
 		hookManager:   hookManager,
 		txManager:     txManager,
 		conns:         conns,
diff --git a/internal/gitaly/service/blob/testhelper_test.go b/internal/gitaly/service/blob/testhelper_test.go
index 84450637c..ad0583813 100644
--- a/internal/gitaly/service/blob/testhelper_test.go
+++ b/internal/gitaly/service/blob/testhelper_test.go
@@ -15,7 +15,7 @@ import (
 	"google.golang.org/grpc/reflection"
 )
 
-var rubyServer = &rubyserver.Server{}
+var rubyServer *rubyserver.Server
 
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
@@ -32,6 +32,7 @@ func testMain(m *testing.M) int {
 		return 1
 	}
 
+	rubyServer = rubyserver.New(config.Config)
 	if err := rubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
diff --git a/internal/gitaly/service/conflicts/testhelper_test.go b/internal/gitaly/service/conflicts/testhelper_test.go
index 5e785db57..c56889f70 100644
--- a/internal/gitaly/service/conflicts/testhelper_test.go
+++ b/internal/gitaly/service/conflicts/testhelper_test.go
@@ -17,7 +17,7 @@ import (
 	"google.golang.org/grpc/reflection"
 )
 
-var RubyServer = &rubyserver.Server{}
+var RubyServer *rubyserver.Server
 
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
@@ -40,6 +40,7 @@ func testMain(m *testing.M) int {
 	// dir of the BinDir is a temp folder
 	hooks.Override = filepath.Join(filepath.Dir(config.Config.BinDir), "/hooks")
 
+	RubyServer = rubyserver.New(config.Config)
 	if err := RubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
diff --git a/internal/gitaly/service/operations/rebase_test.go b/internal/gitaly/service/operations/rebase_test.go
index f8db29d55..2271cfe3c 100644
--- a/internal/gitaly/service/operations/rebase_test.go
+++ b/internal/gitaly/service/operations/rebase_test.go
@@ -13,7 +13,6 @@ import (
 	"gitlab.com/gitlab-org/gitaly/internal/git"
 	"gitlab.com/gitlab-org/gitaly/internal/git/localrepo"
 	"gitlab.com/gitlab-org/gitaly/internal/gitaly/config"
-	"gitlab.com/gitlab-org/gitaly/internal/gitaly/rubyserver"
 	"gitlab.com/gitlab-org/gitaly/internal/testhelper"
 	"gitlab.com/gitlab-org/gitaly/proto/go/gitalypb"
 	"google.golang.org/grpc/codes"
@@ -25,16 +24,11 @@ var (
 )
 
 func TestSuccessfulUserRebaseConfirmableRequest(t *testing.T) {
-	var ruby rubyserver.Server
-
 	pushOptions := []string{"ci.skip", "test=value"}
 	cleanupSrv := setupAndStartGitlabServer(t, testhelper.GlID, "project-1", pushOptions...)
 	defer cleanupSrv()
 
-	require.NoError(t, ruby.Start())
-	defer ruby.Stop()
-
-	serverSocketPath, stop := runOperationServiceServerWithRubyServer(t, &ruby)
+	serverSocketPath, stop := runOperationServiceServer(t)
 	defer stop()
 
 	ctxOuter, cancel := testhelper.Context()
@@ -109,11 +103,7 @@ func TestUserRebaseConfirmable_stableCommitIDs(t *testing.T) {
 	cleanupSrv := setupAndStartGitlabServer(t, testhelper.GlID, "project-1")
 	defer cleanupSrv()
 
-	var ruby rubyserver.Server
-	require.NoError(t, ruby.Start())
-	defer ruby.Stop()
-
-	serverSocketPath, stop := runOperationServiceServerWithRubyServer(t, &ruby)
+	serverSocketPath, stop := runOperationServiceServer(t)
 	defer stop()
 
 	client, conn := newOperationClient(t, serverSocketPath)
diff --git a/internal/gitaly/service/operations/testhelper_test.go b/internal/gitaly/service/operations/testhelper_test.go
index 61c1cf663..467c1538d 100644
--- a/internal/gitaly/service/operations/testhelper_test.go
+++ b/internal/gitaly/service/operations/testhelper_test.go
@@ -29,7 +29,7 @@ var (
 	gitlabPostHooks = []string{"post-receive"}
 	GitlabPreHooks  = gitlabPreHooks
 	GitlabHooks     []string
-	RubyServer      = &rubyserver.Server{}
+	RubyServer      *rubyserver.Server
 )
 
 func init() {
@@ -64,6 +64,7 @@ func testMain(m *testing.M) int {
 	}(config.Config.Auth.Token)
 	config.Config.Auth.Token = testhelper.RepositoryAuthToken
 
+	RubyServer = rubyserver.New(config.Config)
 	if err := RubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
@@ -74,10 +75,6 @@ func testMain(m *testing.M) int {
 }
 
 func runOperationServiceServer(t *testing.T) (string, func()) {
-	return runOperationServiceServerWithRubyServer(t, RubyServer)
-}
-
-func runOperationServiceServerWithRubyServer(t *testing.T, ruby *rubyserver.Server) (string, func()) {
 	srv := testhelper.NewServerWithAuth(t, nil, nil, config.Config.Auth.Token, testhelper.WithInternalSocket(config.Config))
 
 	conns := client.NewPool()
@@ -86,11 +83,11 @@ func runOperationServiceServerWithRubyServer(t *testing.T, ruby *rubyserver.Serv
 	txManager := transaction.NewManager(config.Config)
 	hookManager := gitalyhook.NewManager(locator, txManager, gitalyhook.GitlabAPIStub, config.Config)
 	gitCmdFactory := git.NewExecCommandFactory(config.Config)
-	server := NewServer(config.Config, ruby, hookManager, locator, conns, gitCmdFactory)
+	server := NewServer(config.Config, RubyServer, hookManager, locator, conns, gitCmdFactory)
 
 	gitalypb.RegisterOperationServiceServer(srv.GrpcServer(), server)
 	gitalypb.RegisterHookServiceServer(srv.GrpcServer(), hook.NewServer(config.Config, hookManager, gitCmdFactory))
-	gitalypb.RegisterRepositoryServiceServer(srv.GrpcServer(), repository.NewServer(config.Config, ruby, locator, txManager, gitCmdFactory))
+	gitalypb.RegisterRepositoryServiceServer(srv.GrpcServer(), repository.NewServer(config.Config, RubyServer, locator, txManager, gitCmdFactory))
 	gitalypb.RegisterRefServiceServer(srv.GrpcServer(), ref.NewServer(config.Config, locator, gitCmdFactory))
 	gitalypb.RegisterCommitServiceServer(srv.GrpcServer(), commit.NewServer(config.Config, locator, gitCmdFactory))
 	gitalypb.RegisterSSHServiceServer(srv.GrpcServer(), ssh.NewServer(config.Config, locator, gitCmdFactory))
@@ -115,7 +112,7 @@ func newOperationClient(t *testing.T, serverSocketPath string) (gitalypb.Operati
 }
 
 func setupAndStartGitlabServer(t testing.TB, glID, glRepository string, gitPushOptions ...string) func() {
-	url, cleanup := testhelper.SetupAndStartGitlabServer(t, &testhelper.GitlabTestServerOptions{
+	url, cleanup := testhelper.SetupAndStartGitlabServer(t, config.Config.GitlabShell.Dir, &testhelper.GitlabTestServerOptions{
 		SecretToken:                 "secretToken",
 		GLID:                        glID,
 		GLRepository:                glRepository,
diff --git a/internal/gitaly/service/remote/testhelper_test.go b/internal/gitaly/service/remote/testhelper_test.go
index e6480b920..7e343677a 100644
--- a/internal/gitaly/service/remote/testhelper_test.go
+++ b/internal/gitaly/service/remote/testhelper_test.go
@@ -14,7 +14,7 @@ import (
 	"google.golang.org/grpc/reflection"
 )
 
-var RubyServer = &rubyserver.Server{}
+var RubyServer *rubyserver.Server
 
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
@@ -27,6 +27,7 @@ func testMain(m *testing.M) int {
 	defer cleanup()
 	testhelper.ConfigureGitalySSH()
 
+	RubyServer = rubyserver.New(config.Config)
 	if err := RubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
diff --git a/internal/gitaly/service/repository/archive_test.go b/internal/gitaly/service/repository/archive_test.go
index 79a8a5942..2966be443 100644
--- a/internal/gitaly/service/repository/archive_test.go
+++ b/internal/gitaly/service/repository/archive_test.go
@@ -190,7 +190,7 @@ func TestGetArchiveWithLfsSuccess(t *testing.T) {
 	config.Config.GitlabShell.Dir = gitlabShellDir
 	config.Config.Gitlab.SecretFile = filepath.Join(config.Config.GitlabShell.Dir, ".gitlab_shell_secret")
 
-	url, cleanup := testhelper.SetupAndStartGitlabServer(t, &defaultOptions)
+	url, cleanup := testhelper.SetupAndStartGitlabServer(t, config.Config.GitlabShell.Dir, &defaultOptions)
 	defer cleanup()
 
 	config.Config.Gitlab.URL = url
diff --git a/internal/gitaly/service/repository/testhelper_test.go b/internal/gitaly/service/repository/testhelper_test.go
index f38a6f0b3..3bcd4f2c5 100644
--- a/internal/gitaly/service/repository/testhelper_test.go
+++ b/internal/gitaly/service/repository/testhelper_test.go
@@ -34,7 +34,7 @@ const testTimeString = "200601021504.05"
 
 var (
 	testTime   = time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)
-	RubyServer = &rubyserver.Server{}
+	RubyServer *rubyserver.Server
 )
 
 func TestMain(m *testing.M) {
@@ -59,6 +59,7 @@ func testMain(m *testing.M) int {
 	testhelper.ConfigureGitalyHooksBinary()
 	testhelper.ConfigureGitalySSH()
 
+	RubyServer = rubyserver.New(config.Config)
 	if err := RubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
diff --git a/internal/gitaly/service/wiki/testhelper_test.go b/internal/gitaly/service/wiki/testhelper_test.go
index 22728debb..b134e10d1 100644
--- a/internal/gitaly/service/wiki/testhelper_test.go
+++ b/internal/gitaly/service/wiki/testhelper_test.go
@@ -31,7 +31,7 @@ type createWikiPageOpts struct {
 
 var (
 	mockPageContent = bytes.Repeat([]byte("Mock wiki page content"), 10000)
-	rubyServer      = &rubyserver.Server{}
+	rubyServer      *rubyserver.Server
 )
 
 func TestMain(m *testing.M) {
@@ -54,6 +54,7 @@ func testMain(m *testing.M) int {
 	hooks.Override = tempDir + "/hooks"
 	config.Config.InternalSocketDir = tempDir + "/sock"
 
+	rubyServer = rubyserver.New(config.Config)
 	if err := rubyServer.Start(); err != nil {
 		log.Error(err)
 		return 1
diff --git a/internal/praefect/replicator_test.go b/internal/praefect/replicator_test.go
index 84591c68c..303bce251 100644
--- a/internal/praefect/replicator_test.go
+++ b/internal/praefect/replicator_test.go
@@ -48,7 +48,7 @@ import (
 	"google.golang.org/grpc/reflection"
 )
 
-var RubyServer = &rubyserver.Server{}
+var RubyServer *rubyserver.Server
 
 func TestMain(m *testing.M) {
 	os.Exit(testMain(m))
@@ -74,6 +74,7 @@ func testMain(m *testing.M) int {
 	testhelper.ConfigureGitalySSH()
 	testhelper.ConfigureGitalyHooksBinary()
 
+	RubyServer = rubyserver.New(gitaly_config.Config)
 	if err := RubyServer.Start(); err != nil {
 		logrus.Error(err)
 		return 1
diff --git a/internal/testhelper/testserver.go b/internal/testhelper/testserver.go
index 119e387f4..57bd3e15c 100644
--- a/internal/testhelper/testserver.go
+++ b/internal/testhelper/testserver.go
@@ -825,6 +825,8 @@ type GitlabTestServerOptions struct {
 
 // NewGitlabTestServer returns a mock gitlab server that responds to the hook api endpoints
 func NewGitlabTestServer(t testing.TB, options GitlabTestServerOptions) (url string, cleanup func()) {
+	t.Helper()
+
 	mux := http.NewServeMux()
 	prefix := strings.TrimRight(options.RelativeURLRoot, "/") + "/api/v4/internal"
 	mux.Handle(prefix+"/allowed", http.HandlerFunc(handleAllowed(t, options)))
@@ -911,6 +913,8 @@ func WriteTemporaryGitalyConfigFile(t testing.TB, tempDir, gitlabURL, user, pass
 
 // WriteShellSecretFile writes a .gitlab_shell_secret file in the specified directory
 func WriteShellSecretFile(t testing.TB, dir, secretToken string) {
+	t.Helper()
+
 	require.NoError(t, os.MkdirAll(dir, os.ModeDir))
 	require.NoError(t, ioutil.WriteFile(filepath.Join(dir, ".gitlab_shell_secret"), []byte(secretToken), 0644))
 }
@@ -945,10 +949,10 @@ func NewHealthServerWithListener(t testing.TB, listener net.Listener) (*grpc.Ser
 
 // SetupAndStartGitlabServer creates a new GitlabTestServer, starts it and sets
 // up the gitlab-shell secret.
-func SetupAndStartGitlabServer(t testing.TB, c *GitlabTestServerOptions) (string, func()) {
+func SetupAndStartGitlabServer(t testing.TB, shellDir string, c *GitlabTestServerOptions) (string, func()) {
 	url, cleanup := NewGitlabTestServer(t, *c)
 
-	WriteShellSecretFile(t, config.Config.GitlabShell.Dir, c.SecretToken)
+	WriteShellSecretFile(t, shellDir, c.SecretToken)
 
 	return url, cleanup
 }