diff options
author | Vladimir Shushlin <vshushlin@gitlab.com> | 2020-12-02 11:37:17 +0300 |
---|---|---|
committer | Vladimir Shushlin <vshushlin@gitlab.com> | 2020-12-02 11:37:17 +0300 |
commit | a452ff89165e29d018180e043f3eb0250d85e6e4 (patch) | |
tree | 16241f3a1178e3a348e4025088c15f9cdf8113eb | |
parent | f520a339a6b7880170a65dfcddcef5afff4faaef (diff) | |
parent | d13acbb585182980c0fcdba72df26001fafbce38 (diff) |
Merge branch '464-make-cache-configurable' into 'master'
Add zip serving configuration flags
See merge request gitlab-org/gitlab-pages!392
-rw-r--r-- | app.go | 23 | ||||
-rw-r--r-- | app_config.go | 5 | ||||
-rw-r--r-- | internal/config/config.go | 34 | ||||
-rw-r--r-- | internal/middleware/headers.go | 31 | ||||
-rw-r--r-- | internal/middleware/headers_test.go (renamed from internal/config/config_test.go) | 2 | ||||
-rw-r--r-- | internal/serving/disk/serving.go | 6 | ||||
-rw-r--r-- | internal/serving/disk/zip/serving.go | 3 | ||||
-rw-r--r-- | internal/serving/disk/zip/serving_test.go | 13 | ||||
-rw-r--r-- | internal/serving/serverless/serverless.go | 6 | ||||
-rw-r--r-- | internal/serving/serving.go | 3 | ||||
-rw-r--r-- | internal/vfs/local/vfs.go | 6 | ||||
-rw-r--r-- | internal/vfs/vfs.go | 6 | ||||
-rw-r--r-- | internal/vfs/zip/archive_test.go | 19 | ||||
-rw-r--r-- | internal/vfs/zip/lru_cache.go | 6 | ||||
-rw-r--r-- | internal/vfs/zip/vfs.go | 80 | ||||
-rw-r--r-- | internal/vfs/zip/vfs_test.go | 15 | ||||
-rw-r--r-- | main.go | 19 | ||||
-rw-r--r-- | test/acceptance/zip_test.go | 25 |
18 files changed, 208 insertions, 94 deletions
@@ -21,13 +21,15 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/acme" "gitlab.com/gitlab-org/gitlab-pages/internal/artifact" "gitlab.com/gitlab-org/gitlab-pages/internal/auth" - headerConfig "gitlab.com/gitlab-org/gitlab-pages/internal/config" + cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/handlers" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" + "gitlab.com/gitlab-org/gitlab-pages/internal/middleware" "gitlab.com/gitlab-org/gitlab-pages/internal/netutil" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/zip" "gitlab.com/gitlab-org/gitlab-pages/internal/source" "gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig" "gitlab.com/gitlab-org/gitlab-pages/metrics" @@ -186,7 +188,7 @@ func (a *theApp) healthCheckMiddleware(handler http.Handler) (http.Handler, erro // customHeadersMiddleware will inject custom headers into the response func (a *theApp) customHeadersMiddleware(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - headerConfig.AddCustomHeaders(w, a.CustomHeaders) + middleware.AddCustomHeaders(w, a.CustomHeaders) handler.ServeHTTP(w, r) }) @@ -493,7 +495,7 @@ func runApp(config appConfig) { } if len(config.CustomHeaders) != 0 { - customHeaders, err := headerConfig.ParseHeaderString(config.CustomHeaders) + customHeaders, err := middleware.ParseHeaderString(config.CustomHeaders) if err != nil { log.WithError(err).Fatal("Unable to parse header string") } @@ -504,6 +506,21 @@ func runApp(config appConfig) { log.WithError(err).Warn("Loading extended MIME database failed") } + c := &cfg.Config{ + Zip: &cfg.ZipServing{ + ExpirationInterval: config.ZipCacheExpiry, + CleanupInterval: config.ZipCacheCleanup, + RefreshInterval: config.ZipCacheRefresh, + OpenTimeout: config.ZipeOpenTimeout, + }, + } + + // TODO: reconfigure all VFS' + // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/512 + if err := zip.Instance().Reconfigure(c); err != nil { + fatal(err, "failed to reconfigure zip VFS") + } + a.Run() } diff --git a/app_config.go b/app_config.go index bb4aa917..0dd192d5 100644 --- a/app_config.go +++ b/app_config.go @@ -41,6 +41,11 @@ type appConfig struct { SentryDSN string SentryEnvironment string CustomHeaders []string + + ZipCacheExpiry time.Duration + ZipCacheRefresh time.Duration + ZipCacheCleanup time.Duration + ZipeOpenTimeout time.Duration } // InternalGitLabServerURL returns URL to a GitLab instance. diff --git a/internal/config/config.go b/internal/config/config.go index d415b21d..c52beef8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,32 +1,18 @@ package config import ( - "errors" - "net/http" - "strings" + "time" ) -var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter") - -// AddCustomHeaders adds a map of Headers to a Response -func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error { - for k, v := range headers { - for _, value := range v { - w.Header().Add(k, value) - } - } - return nil +type Config struct { + Zip *ZipServing } -// ParseHeaderString parses a string of key values into a map -func ParseHeaderString(customHeaders []string) (http.Header, error) { - headers := http.Header{} - for _, keyValueString := range customHeaders { - keyValue := strings.SplitN(keyValueString, ":", 2) - if len(keyValue) != 2 { - return nil, errInvalidHeaderParameter - } - headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1])) - } - return headers, nil +// ZipServing stores all configuration values to be used by the zip VFS opening and +// caching +type ZipServing struct { + ExpirationInterval time.Duration + CleanupInterval time.Duration + RefreshInterval time.Duration + OpenTimeout time.Duration } diff --git a/internal/middleware/headers.go b/internal/middleware/headers.go new file mode 100644 index 00000000..77b008f3 --- /dev/null +++ b/internal/middleware/headers.go @@ -0,0 +1,31 @@ +package middleware + +import ( + "errors" + "net/http" + "strings" +) + +var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter") + +// AddCustomHeaders adds a map of Headers to a Response +func AddCustomHeaders(w http.ResponseWriter, headers http.Header) { + for k, v := range headers { + for _, value := range v { + w.Header().Add(k, value) + } + } +} + +// ParseHeaderString parses a string of key values into a map +func ParseHeaderString(customHeaders []string) (http.Header, error) { + headers := http.Header{} + for _, keyValueString := range customHeaders { + keyValue := strings.SplitN(keyValueString, ":", 2) + if len(keyValue) != 2 { + return nil, errInvalidHeaderParameter + } + headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1])) + } + return headers, nil +} diff --git a/internal/config/config_test.go b/internal/middleware/headers_test.go index 44afd470..17d31b50 100644 --- a/internal/config/config_test.go +++ b/internal/middleware/headers_test.go @@ -1,4 +1,4 @@ -package config +package middleware import ( "net/http/httptest" diff --git a/internal/serving/disk/serving.go b/internal/serving/disk/serving.go index 30c821ea..fbcdf9f2 100644 --- a/internal/serving/disk/serving.go +++ b/internal/serving/disk/serving.go @@ -3,6 +3,7 @@ package disk import ( "os" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" @@ -40,6 +41,11 @@ func (s *Disk) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } +// Reconfigure VFS +func (s *Disk) Reconfigure(cfg *config.Config) error { + return s.reader.vfs.Reconfigure(cfg) +} + // New returns a serving instance that is capable of reading files // from the VFS func New(vfs vfs.VFS) serving.Serving { diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 95894fc9..6db0be10 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -1,13 +1,14 @@ package zip import ( + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var instance = disk.New(vfs.Instrumented(zip.New())) +var instance = disk.New(vfs.Instrumented(zip.New(&config.ZipServing{}))) // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go index bca2ae51..e64a761a 100644 --- a/internal/serving/disk/zip/serving_test.go +++ b/internal/serving/disk/zip/serving_test.go @@ -5,9 +5,11 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) @@ -53,7 +55,18 @@ func TestZip_ServeFileHTTP(t *testing.T) { }, } + cfg := &config.Config{ + Zip: &config.ZipServing{ + ExpirationInterval: 10 * time.Second, + CleanupInterval: 5 * time.Second, + RefreshInterval: 5 * time.Second, + OpenTimeout: 5 * time.Second, + }, + } + s := Instance() + err := s.Reconfigure(cfg) + require.NoError(t, err) for name, test := range tests { t.Run(name, func(t *testing.T) { diff --git a/internal/serving/serverless/serverless.go b/internal/serving/serverless/serverless.go index e1881362..f8bd4e87 100644 --- a/internal/serving/serverless/serverless.go +++ b/internal/serving/serverless/serverless.go @@ -4,6 +4,7 @@ import ( "errors" "net/http/httputil" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api" @@ -65,3 +66,8 @@ func (s *Serverless) ServeFileHTTP(h serving.Handler) bool { func (s *Serverless) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } + +// Reconfigure noop +func (s *Serverless) Reconfigure(*config.Config) error { + return nil +} diff --git a/internal/serving/serving.go b/internal/serving/serving.go index 6fde8216..786ee569 100644 --- a/internal/serving/serving.go +++ b/internal/serving/serving.go @@ -1,7 +1,10 @@ package serving +import "gitlab.com/gitlab-org/gitlab-pages/internal/config" + // Serving is an interface used to define a serving driver type Serving interface { ServeFileHTTP(Handler) bool ServeNotFoundHTTP(Handler) + Reconfigure(config *config.Config) error } diff --git a/internal/vfs/local/vfs.go b/internal/vfs/local/vfs.go index bdcd5160..ea54e8e8 100644 --- a/internal/vfs/local/vfs.go +++ b/internal/vfs/local/vfs.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" ) @@ -43,3 +44,8 @@ func (fs VFS) Root(ctx context.Context, path string) (vfs.Root, error) { func (fs *VFS) Name() string { return "local" } + +func (fs *VFS) Reconfigure(*config.Config) error { + // noop + return nil +} diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 7bd51db2..2304f903 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -6,6 +6,7 @@ import ( log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) @@ -13,6 +14,7 @@ import ( type VFS interface { Root(ctx context.Context, path string) (Root, error) Name() string + Reconfigure(config *config.Config) error } func Instrumented(fs VFS) VFS { @@ -50,3 +52,7 @@ func (i *instrumentedVFS) Root(ctx context.Context, path string) (Root, error) { func (i *instrumentedVFS) Name() string { return i.fs.Name() } + +func (i *instrumentedVFS) Reconfigure(cfg *config.Config) error { + return i.fs.Reconfigure(cfg) +} diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index e1b0d116..da778e62 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -12,11 +12,20 @@ import ( "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) -var chdirSet = false +var ( + chdirSet = false + zipCfg = &config.ZipServing{ + ExpirationInterval: 10 * time.Second, + CleanupInterval: 5 * time.Second, + RefreshInterval: 5 * time.Second, + OpenTimeout: 5 * time.Second, + } +) func TestOpen(t *testing.T) { zip, cleanup := openZipArchive(t, nil) @@ -75,7 +84,7 @@ func TestOpenCached(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", &requests) defer cleanup() - fs := New() + fs := New(zipCfg) // We use array instead of map to ensure // predictable ordering of test execution @@ -322,7 +331,7 @@ func TestArchiveCanBeReadAfterOpenCtxCanceled(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil) defer cleanup() - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -345,7 +354,7 @@ func TestReadArchiveFails(t *testing.T) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil) defer cleanup() - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) err := zip.openArchive(context.Background(), testServerURL+"/unkown.html") @@ -365,7 +374,7 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) { testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", requests) - fs := New().(*zipVFS) + fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) err := zip.openArchive(context.Background(), testServerURL+"/public.zip") diff --git a/internal/vfs/zip/lru_cache.go b/internal/vfs/zip/lru_cache.go index 36817e66..9810e245 100644 --- a/internal/vfs/zip/lru_cache.go +++ b/internal/vfs/zip/lru_cache.go @@ -24,10 +24,10 @@ type lruCache struct { cache *ccache.Cache } -func newLruCache(op string, maxEntries uint32, duration time.Duration) *lruCache { +func newLruCache(op string, maxEntries int64, duration time.Duration) *lruCache { configuration := ccache.Configure() - configuration.MaxSize(int64(maxEntries)) - configuration.ItemsToPrune(maxEntries / lruCacheItemsToPruneDiv) + configuration.MaxSize(maxEntries) + configuration.ItemsToPrune(uint32(maxEntries) / lruCacheItemsToPruneDiv) configuration.GetsPerPromote(lruCacheGetsPerPromote) // if item gets requested frequently promote it configuration.OnDelete(func(*ccache.Item) { metrics.ZipCachedEntries.WithLabelValues(op).Dec() diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index 692a4a69..b27424c6 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -9,18 +9,13 @@ import ( "github.com/patrickmn/go-cache" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httprange" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) const ( - // TODO: make these configurable https://gitlab.com/gitlab-org/gitlab-pages/-/issues/464 - defaultCacheExpirationInterval = time.Minute - defaultCacheCleanupInterval = time.Minute / 2 - defaultCacheRefreshInterval = time.Minute / 2 - defaultOpenTimeout = time.Minute / 2 - // we assume that each item costs around 100 bytes // this gives around 5MB of raw memory needed without acceleration structures defaultDataOffsetItems = 50000 @@ -52,62 +47,47 @@ type zipVFS struct { archiveCount int64 } -// Option function allows to override default values -type Option func(*zipVFS) - -// WithCacheRefreshInterval when used it can override defaultCacheRefreshInterval -func WithCacheRefreshInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheRefreshInterval = interval +// New creates a zipVFS instance that can be used by a serving request +func New(cfg *config.ZipServing) vfs.VFS { + zipVFS := &zipVFS{ + cacheExpirationInterval: cfg.ExpirationInterval, + cacheRefreshInterval: cfg.RefreshInterval, + cacheCleanupInterval: cfg.CleanupInterval, + openTimeout: cfg.OpenTimeout, } -} -// WithCacheExpirationInterval when used it can override defaultCacheExpirationInterval -func WithCacheExpirationInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheExpirationInterval = interval - } -} + zipVFS.resetCache() -// WithCacheCleanupInterval when used it can override defaultCacheCleanupInterval -func WithCacheCleanupInterval(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.cacheCleanupInterval = interval - } -} + // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 + zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) + zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval) -// WithOpenTimeout when used it can override openTimeout -func WithOpenTimeout(interval time.Duration) Option { - return func(vfs *zipVFS) { - vfs.openTimeout = interval - } + return zipVFS } -// New creates a zipVFS instance that can be used by a serving request -func New(options ...Option) vfs.VFS { - zipVFS := &zipVFS{ - cacheExpirationInterval: defaultCacheExpirationInterval, - cacheRefreshInterval: defaultCacheRefreshInterval, - cacheCleanupInterval: defaultCacheCleanupInterval, - openTimeout: defaultOpenTimeout, - } +// Reconfigure will update the zipVFS configuration values and will reset the +// cache +func (fs *zipVFS) Reconfigure(cfg *config.Config) error { + fs.cacheLock.Lock() + defer fs.cacheLock.Unlock() - for _, option := range options { - option(zipVFS) - } + fs.openTimeout = cfg.Zip.OpenTimeout + fs.cacheExpirationInterval = cfg.Zip.ExpirationInterval + fs.cacheRefreshInterval = cfg.Zip.RefreshInterval + fs.cacheCleanupInterval = cfg.Zip.CleanupInterval + + fs.resetCache() + + return nil +} - zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval) - zipVFS.cache.OnEvicted(func(s string, i interface{}) { +func (fs *zipVFS) resetCache() { + fs.cache = cache.New(fs.cacheExpirationInterval, fs.cacheCleanupInterval) + fs.cache.OnEvicted(func(s string, i interface{}) { metrics.ZipCachedEntries.WithLabelValues("archive").Dec() i.(*zipArchive).onEvicted() }) - - // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 - zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) - zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval) - - return zipVFS } func (fs *zipVFS) keyFromPath(path string) (string, error) { diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go index dff2ff43..ffda1fb6 100644 --- a/internal/vfs/zip/vfs_test.go +++ b/internal/vfs/zip/vfs_test.go @@ -35,7 +35,7 @@ func TestVFSRoot(t *testing.T) { }, } - vfs := New() + vfs := New(zipCfg) for name, tt := range tests { t.Run(name, func(t *testing.T) { @@ -73,7 +73,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) { path := testServerURL + "/public.zip" - vfs := New().(*zipVFS) + vfs := New(zipCfg).(*zipVFS) root, err := vfs.Root(context.Background(), path) require.NoError(t, err) @@ -98,7 +98,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) { require.Eventually(t, func() bool { _, err := vfs.findOrOpenArchive(context.Background(), path, path) return err == errAlreadyCached - }, time.Second, time.Nanosecond) + }, 3*time.Second, time.Nanosecond) } func TestVFSFindOrOpenArchiveRefresh(t *testing.T) { @@ -158,10 +158,11 @@ func TestVFSFindOrOpenArchiveRefresh(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { withExpectedArchiveCount(t, 1, func(t *testing.T) { - vfs := New( - WithCacheExpirationInterval(test.expirationInterval), - WithCacheRefreshInterval(test.refreshInterval), - ).(*zipVFS) + cfg := *zipCfg + cfg.ExpirationInterval = test.expirationInterval + cfg.RefreshInterval = test.refreshInterval + + vfs := New(&cfg).(*zipVFS) path := testServerURL + test.path @@ -13,6 +13,7 @@ import ( "github.com/namsral/flag" log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/labkit/errortracking" "gitlab.com/gitlab-org/gitlab-pages/internal/logging" @@ -29,6 +30,7 @@ var VERSION = "dev" var REVISION = "HEAD" func init() { + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests") flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests") flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests") @@ -37,6 +39,7 @@ func init() { } var ( + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages") pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages") redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS") @@ -75,6 +78,11 @@ var ( insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4") tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min")) tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max")) + // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507 + zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval") + zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval") + zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval") + zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout") disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests") @@ -210,6 +218,11 @@ func configFromFlags() appConfig { config.SentryDSN = *sentryDSN config.SentryEnvironment = *sentryEnvironment + config.ZipCacheExpiry = *zipCacheExpiration + config.ZipCacheCleanup = *zipCacheCleanup + config.ZipCacheRefresh = *zipCacheRefresh + config.ZipeOpenTimeout = *zipOpenTimeout + checkAuthenticationConfig(config) return config @@ -294,6 +307,10 @@ func loadConfig() appConfig { "api-secret-key": *gitLabAPISecretKey, "domain-config-source": config.DomainConfigurationSource, "auth-redirect-uri": config.RedirectURI, + "zip-cache-expiration": config.ZipCacheExpiry, + "zip-cache-cleanup": config.ZipCacheCleanup, + "zip-cache-refresh": config.ZipCacheRefresh, + "zip-open-timeout": config.ZipeOpenTimeout, }).Debug("Start daemon with configuration") return config @@ -304,7 +321,9 @@ func appMain() { // read from -config=/path/to/gitlab-pages-config flag.String(flag.DefaultConfigFlagname, "", "path to config file") + flag.Parse() + if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil { fatal(err, "invalid TLS version") } diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go index ea703ebe..3bfa21bf 100644 --- a/test/acceptance/zip_test.go +++ b/test/acceptance/zip_test.go @@ -104,6 +104,31 @@ func TestZipServing(t *testing.T) { } } +func TestZipServingConfigShortTimeout(t *testing.T) { + skipUnlessEnabled(t) + + var apiCalled bool + source := NewGitlabDomainsSourceStub(t, &apiCalled) + defer source.Close() + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", + "-zip-open-timeout=1ns"} // <- test purpose + + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) + defer teardown() + + _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + + response, err := GetPageFromListener(t, httpListener, "zip.gitlab.io", "/") + require.NoError(t, err) + defer response.Body.Close() + + require.Equal(t, http.StatusInternalServerError, response.StatusCode, "should fail to serve") +} + func newZipFileServerURL(t *testing.T, zipFilePath string) (string, func()) { t.Helper() |