Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-pages.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladimir Shushlin <vshushlin@gitlab.com>2020-12-02 11:37:17 +0300
committerVladimir Shushlin <vshushlin@gitlab.com>2020-12-02 11:37:17 +0300
commita452ff89165e29d018180e043f3eb0250d85e6e4 (patch)
tree16241f3a1178e3a348e4025088c15f9cdf8113eb
parentf520a339a6b7880170a65dfcddcef5afff4faaef (diff)
parentd13acbb585182980c0fcdba72df26001fafbce38 (diff)
Merge branch '464-make-cache-configurable' into 'master'
Add zip serving configuration flags See merge request gitlab-org/gitlab-pages!392
-rw-r--r--app.go23
-rw-r--r--app_config.go5
-rw-r--r--internal/config/config.go34
-rw-r--r--internal/middleware/headers.go31
-rw-r--r--internal/middleware/headers_test.go (renamed from internal/config/config_test.go)2
-rw-r--r--internal/serving/disk/serving.go6
-rw-r--r--internal/serving/disk/zip/serving.go3
-rw-r--r--internal/serving/disk/zip/serving_test.go13
-rw-r--r--internal/serving/serverless/serverless.go6
-rw-r--r--internal/serving/serving.go3
-rw-r--r--internal/vfs/local/vfs.go6
-rw-r--r--internal/vfs/vfs.go6
-rw-r--r--internal/vfs/zip/archive_test.go19
-rw-r--r--internal/vfs/zip/lru_cache.go6
-rw-r--r--internal/vfs/zip/vfs.go80
-rw-r--r--internal/vfs/zip/vfs_test.go15
-rw-r--r--main.go19
-rw-r--r--test/acceptance/zip_test.go25
18 files changed, 208 insertions, 94 deletions
diff --git a/app.go b/app.go
index ca495073..c411470b 100644
--- a/app.go
+++ b/app.go
@@ -21,13 +21,15 @@ import (
"gitlab.com/gitlab-org/gitlab-pages/internal/acme"
"gitlab.com/gitlab-org/gitlab-pages/internal/artifact"
"gitlab.com/gitlab-org/gitlab-pages/internal/auth"
- headerConfig "gitlab.com/gitlab-org/gitlab-pages/internal/config"
+ cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/domain"
"gitlab.com/gitlab-org/gitlab-pages/internal/handlers"
"gitlab.com/gitlab-org/gitlab-pages/internal/httperrors"
"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/middleware"
"gitlab.com/gitlab-org/gitlab-pages/internal/netutil"
"gitlab.com/gitlab-org/gitlab-pages/internal/request"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/zip"
"gitlab.com/gitlab-org/gitlab-pages/internal/source"
"gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
@@ -186,7 +188,7 @@ func (a *theApp) healthCheckMiddleware(handler http.Handler) (http.Handler, erro
// customHeadersMiddleware will inject custom headers into the response
func (a *theApp) customHeadersMiddleware(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- headerConfig.AddCustomHeaders(w, a.CustomHeaders)
+ middleware.AddCustomHeaders(w, a.CustomHeaders)
handler.ServeHTTP(w, r)
})
@@ -493,7 +495,7 @@ func runApp(config appConfig) {
}
if len(config.CustomHeaders) != 0 {
- customHeaders, err := headerConfig.ParseHeaderString(config.CustomHeaders)
+ customHeaders, err := middleware.ParseHeaderString(config.CustomHeaders)
if err != nil {
log.WithError(err).Fatal("Unable to parse header string")
}
@@ -504,6 +506,21 @@ func runApp(config appConfig) {
log.WithError(err).Warn("Loading extended MIME database failed")
}
+ c := &cfg.Config{
+ Zip: &cfg.ZipServing{
+ ExpirationInterval: config.ZipCacheExpiry,
+ CleanupInterval: config.ZipCacheCleanup,
+ RefreshInterval: config.ZipCacheRefresh,
+ OpenTimeout: config.ZipeOpenTimeout,
+ },
+ }
+
+ // TODO: reconfigure all VFS'
+ // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/512
+ if err := zip.Instance().Reconfigure(c); err != nil {
+ fatal(err, "failed to reconfigure zip VFS")
+ }
+
a.Run()
}
diff --git a/app_config.go b/app_config.go
index bb4aa917..0dd192d5 100644
--- a/app_config.go
+++ b/app_config.go
@@ -41,6 +41,11 @@ type appConfig struct {
SentryDSN string
SentryEnvironment string
CustomHeaders []string
+
+ ZipCacheExpiry time.Duration
+ ZipCacheRefresh time.Duration
+ ZipCacheCleanup time.Duration
+ ZipeOpenTimeout time.Duration
}
// InternalGitLabServerURL returns URL to a GitLab instance.
diff --git a/internal/config/config.go b/internal/config/config.go
index d415b21d..c52beef8 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,32 +1,18 @@
package config
import (
- "errors"
- "net/http"
- "strings"
+ "time"
)
-var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter")
-
-// AddCustomHeaders adds a map of Headers to a Response
-func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error {
- for k, v := range headers {
- for _, value := range v {
- w.Header().Add(k, value)
- }
- }
- return nil
+type Config struct {
+ Zip *ZipServing
}
-// ParseHeaderString parses a string of key values into a map
-func ParseHeaderString(customHeaders []string) (http.Header, error) {
- headers := http.Header{}
- for _, keyValueString := range customHeaders {
- keyValue := strings.SplitN(keyValueString, ":", 2)
- if len(keyValue) != 2 {
- return nil, errInvalidHeaderParameter
- }
- headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1]))
- }
- return headers, nil
+// ZipServing stores all configuration values to be used by the zip VFS opening and
+// caching
+type ZipServing struct {
+ ExpirationInterval time.Duration
+ CleanupInterval time.Duration
+ RefreshInterval time.Duration
+ OpenTimeout time.Duration
}
diff --git a/internal/middleware/headers.go b/internal/middleware/headers.go
new file mode 100644
index 00000000..77b008f3
--- /dev/null
+++ b/internal/middleware/headers.go
@@ -0,0 +1,31 @@
+package middleware
+
+import (
+ "errors"
+ "net/http"
+ "strings"
+)
+
+var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter")
+
+// AddCustomHeaders adds a map of Headers to a Response
+func AddCustomHeaders(w http.ResponseWriter, headers http.Header) {
+ for k, v := range headers {
+ for _, value := range v {
+ w.Header().Add(k, value)
+ }
+ }
+}
+
+// ParseHeaderString parses a string of key values into a map
+func ParseHeaderString(customHeaders []string) (http.Header, error) {
+ headers := http.Header{}
+ for _, keyValueString := range customHeaders {
+ keyValue := strings.SplitN(keyValueString, ":", 2)
+ if len(keyValue) != 2 {
+ return nil, errInvalidHeaderParameter
+ }
+ headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1]))
+ }
+ return headers, nil
+}
diff --git a/internal/config/config_test.go b/internal/middleware/headers_test.go
index 44afd470..17d31b50 100644
--- a/internal/config/config_test.go
+++ b/internal/middleware/headers_test.go
@@ -1,4 +1,4 @@
-package config
+package middleware
import (
"net/http/httptest"
diff --git a/internal/serving/disk/serving.go b/internal/serving/disk/serving.go
index 30c821ea..fbcdf9f2 100644
--- a/internal/serving/disk/serving.go
+++ b/internal/serving/disk/serving.go
@@ -3,6 +3,7 @@ package disk
import (
"os"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httperrors"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
@@ -40,6 +41,11 @@ func (s *Disk) ServeNotFoundHTTP(h serving.Handler) {
httperrors.Serve404(h.Writer)
}
+// Reconfigure VFS
+func (s *Disk) Reconfigure(cfg *config.Config) error {
+ return s.reader.vfs.Reconfigure(cfg)
+}
+
// New returns a serving instance that is capable of reading files
// from the VFS
func New(vfs vfs.VFS) serving.Serving {
diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go
index 95894fc9..6db0be10 100644
--- a/internal/serving/disk/zip/serving.go
+++ b/internal/serving/disk/zip/serving.go
@@ -1,13 +1,14 @@
package zip
import (
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip"
)
-var instance = disk.New(vfs.Instrumented(zip.New()))
+var instance = disk.New(vfs.Instrumented(zip.New(&config.ZipServing{})))
// Instance returns a serving instance that is capable of reading files
// from a zip archives opened from a URL, most likely stored in object storage
diff --git a/internal/serving/disk/zip/serving_test.go b/internal/serving/disk/zip/serving_test.go
index bca2ae51..e64a761a 100644
--- a/internal/serving/disk/zip/serving_test.go
+++ b/internal/serving/disk/zip/serving_test.go
@@ -5,9 +5,11 @@ import (
"net/http"
"net/http/httptest"
"testing"
+ "time"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
)
@@ -53,7 +55,18 @@ func TestZip_ServeFileHTTP(t *testing.T) {
},
}
+ cfg := &config.Config{
+ Zip: &config.ZipServing{
+ ExpirationInterval: 10 * time.Second,
+ CleanupInterval: 5 * time.Second,
+ RefreshInterval: 5 * time.Second,
+ OpenTimeout: 5 * time.Second,
+ },
+ }
+
s := Instance()
+ err := s.Reconfigure(cfg)
+ require.NoError(t, err)
for name, test := range tests {
t.Run(name, func(t *testing.T) {
diff --git a/internal/serving/serverless/serverless.go b/internal/serving/serverless/serverless.go
index e1881362..f8bd4e87 100644
--- a/internal/serving/serverless/serverless.go
+++ b/internal/serving/serverless/serverless.go
@@ -4,6 +4,7 @@ import (
"errors"
"net/http/httputil"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httperrors"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api"
@@ -65,3 +66,8 @@ func (s *Serverless) ServeFileHTTP(h serving.Handler) bool {
func (s *Serverless) ServeNotFoundHTTP(h serving.Handler) {
httperrors.Serve404(h.Writer)
}
+
+// Reconfigure noop
+func (s *Serverless) Reconfigure(*config.Config) error {
+ return nil
+}
diff --git a/internal/serving/serving.go b/internal/serving/serving.go
index 6fde8216..786ee569 100644
--- a/internal/serving/serving.go
+++ b/internal/serving/serving.go
@@ -1,7 +1,10 @@
package serving
+import "gitlab.com/gitlab-org/gitlab-pages/internal/config"
+
// Serving is an interface used to define a serving driver
type Serving interface {
ServeFileHTTP(Handler) bool
ServeNotFoundHTTP(Handler)
+ Reconfigure(config *config.Config) error
}
diff --git a/internal/vfs/local/vfs.go b/internal/vfs/local/vfs.go
index bdcd5160..ea54e8e8 100644
--- a/internal/vfs/local/vfs.go
+++ b/internal/vfs/local/vfs.go
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
)
@@ -43,3 +44,8 @@ func (fs VFS) Root(ctx context.Context, path string) (vfs.Root, error) {
func (fs *VFS) Name() string {
return "local"
}
+
+func (fs *VFS) Reconfigure(*config.Config) error {
+ // noop
+ return nil
+}
diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go
index 7bd51db2..2304f903 100644
--- a/internal/vfs/vfs.go
+++ b/internal/vfs/vfs.go
@@ -6,6 +6,7 @@ import (
log "github.com/sirupsen/logrus"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
)
@@ -13,6 +14,7 @@ import (
type VFS interface {
Root(ctx context.Context, path string) (Root, error)
Name() string
+ Reconfigure(config *config.Config) error
}
func Instrumented(fs VFS) VFS {
@@ -50,3 +52,7 @@ func (i *instrumentedVFS) Root(ctx context.Context, path string) (Root, error) {
func (i *instrumentedVFS) Name() string {
return i.fs.Name()
}
+
+func (i *instrumentedVFS) Reconfigure(cfg *config.Config) error {
+ return i.fs.Reconfigure(cfg)
+}
diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go
index e1b0d116..da778e62 100644
--- a/internal/vfs/zip/archive_test.go
+++ b/internal/vfs/zip/archive_test.go
@@ -12,11 +12,20 @@ import (
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httprange"
"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
)
-var chdirSet = false
+var (
+ chdirSet = false
+ zipCfg = &config.ZipServing{
+ ExpirationInterval: 10 * time.Second,
+ CleanupInterval: 5 * time.Second,
+ RefreshInterval: 5 * time.Second,
+ OpenTimeout: 5 * time.Second,
+ }
+)
func TestOpen(t *testing.T) {
zip, cleanup := openZipArchive(t, nil)
@@ -75,7 +84,7 @@ func TestOpenCached(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", &requests)
defer cleanup()
- fs := New()
+ fs := New(zipCfg)
// We use array instead of map to ensure
// predictable ordering of test execution
@@ -322,7 +331,7 @@ func TestArchiveCanBeReadAfterOpenCtxCanceled(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil)
defer cleanup()
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -345,7 +354,7 @@ func TestReadArchiveFails(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil)
defer cleanup()
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
err := zip.openArchive(context.Background(), testServerURL+"/unkown.html")
@@ -365,7 +374,7 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", requests)
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
err := zip.openArchive(context.Background(), testServerURL+"/public.zip")
diff --git a/internal/vfs/zip/lru_cache.go b/internal/vfs/zip/lru_cache.go
index 36817e66..9810e245 100644
--- a/internal/vfs/zip/lru_cache.go
+++ b/internal/vfs/zip/lru_cache.go
@@ -24,10 +24,10 @@ type lruCache struct {
cache *ccache.Cache
}
-func newLruCache(op string, maxEntries uint32, duration time.Duration) *lruCache {
+func newLruCache(op string, maxEntries int64, duration time.Duration) *lruCache {
configuration := ccache.Configure()
- configuration.MaxSize(int64(maxEntries))
- configuration.ItemsToPrune(maxEntries / lruCacheItemsToPruneDiv)
+ configuration.MaxSize(maxEntries)
+ configuration.ItemsToPrune(uint32(maxEntries) / lruCacheItemsToPruneDiv)
configuration.GetsPerPromote(lruCacheGetsPerPromote) // if item gets requested frequently promote it
configuration.OnDelete(func(*ccache.Item) {
metrics.ZipCachedEntries.WithLabelValues(op).Dec()
diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go
index 692a4a69..b27424c6 100644
--- a/internal/vfs/zip/vfs.go
+++ b/internal/vfs/zip/vfs.go
@@ -9,18 +9,13 @@ import (
"github.com/patrickmn/go-cache"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httprange"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
)
const (
- // TODO: make these configurable https://gitlab.com/gitlab-org/gitlab-pages/-/issues/464
- defaultCacheExpirationInterval = time.Minute
- defaultCacheCleanupInterval = time.Minute / 2
- defaultCacheRefreshInterval = time.Minute / 2
- defaultOpenTimeout = time.Minute / 2
-
// we assume that each item costs around 100 bytes
// this gives around 5MB of raw memory needed without acceleration structures
defaultDataOffsetItems = 50000
@@ -52,62 +47,47 @@ type zipVFS struct {
archiveCount int64
}
-// Option function allows to override default values
-type Option func(*zipVFS)
-
-// WithCacheRefreshInterval when used it can override defaultCacheRefreshInterval
-func WithCacheRefreshInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheRefreshInterval = interval
+// New creates a zipVFS instance that can be used by a serving request
+func New(cfg *config.ZipServing) vfs.VFS {
+ zipVFS := &zipVFS{
+ cacheExpirationInterval: cfg.ExpirationInterval,
+ cacheRefreshInterval: cfg.RefreshInterval,
+ cacheCleanupInterval: cfg.CleanupInterval,
+ openTimeout: cfg.OpenTimeout,
}
-}
-// WithCacheExpirationInterval when used it can override defaultCacheExpirationInterval
-func WithCacheExpirationInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheExpirationInterval = interval
- }
-}
+ zipVFS.resetCache()
-// WithCacheCleanupInterval when used it can override defaultCacheCleanupInterval
-func WithCacheCleanupInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheCleanupInterval = interval
- }
-}
+ // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480
+ zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval)
+ zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval)
-// WithOpenTimeout when used it can override openTimeout
-func WithOpenTimeout(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.openTimeout = interval
- }
+ return zipVFS
}
-// New creates a zipVFS instance that can be used by a serving request
-func New(options ...Option) vfs.VFS {
- zipVFS := &zipVFS{
- cacheExpirationInterval: defaultCacheExpirationInterval,
- cacheRefreshInterval: defaultCacheRefreshInterval,
- cacheCleanupInterval: defaultCacheCleanupInterval,
- openTimeout: defaultOpenTimeout,
- }
+// Reconfigure will update the zipVFS configuration values and will reset the
+// cache
+func (fs *zipVFS) Reconfigure(cfg *config.Config) error {
+ fs.cacheLock.Lock()
+ defer fs.cacheLock.Unlock()
- for _, option := range options {
- option(zipVFS)
- }
+ fs.openTimeout = cfg.Zip.OpenTimeout
+ fs.cacheExpirationInterval = cfg.Zip.ExpirationInterval
+ fs.cacheRefreshInterval = cfg.Zip.RefreshInterval
+ fs.cacheCleanupInterval = cfg.Zip.CleanupInterval
+
+ fs.resetCache()
+
+ return nil
+}
- zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval)
- zipVFS.cache.OnEvicted(func(s string, i interface{}) {
+func (fs *zipVFS) resetCache() {
+ fs.cache = cache.New(fs.cacheExpirationInterval, fs.cacheCleanupInterval)
+ fs.cache.OnEvicted(func(s string, i interface{}) {
metrics.ZipCachedEntries.WithLabelValues("archive").Dec()
i.(*zipArchive).onEvicted()
})
-
- // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480
- zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval)
- zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval)
-
- return zipVFS
}
func (fs *zipVFS) keyFromPath(path string) (string, error) {
diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go
index dff2ff43..ffda1fb6 100644
--- a/internal/vfs/zip/vfs_test.go
+++ b/internal/vfs/zip/vfs_test.go
@@ -35,7 +35,7 @@ func TestVFSRoot(t *testing.T) {
},
}
- vfs := New()
+ vfs := New(zipCfg)
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
@@ -73,7 +73,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) {
path := testServerURL + "/public.zip"
- vfs := New().(*zipVFS)
+ vfs := New(zipCfg).(*zipVFS)
root, err := vfs.Root(context.Background(), path)
require.NoError(t, err)
@@ -98,7 +98,7 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) {
require.Eventually(t, func() bool {
_, err := vfs.findOrOpenArchive(context.Background(), path, path)
return err == errAlreadyCached
- }, time.Second, time.Nanosecond)
+ }, 3*time.Second, time.Nanosecond)
}
func TestVFSFindOrOpenArchiveRefresh(t *testing.T) {
@@ -158,10 +158,11 @@ func TestVFSFindOrOpenArchiveRefresh(t *testing.T) {
for name, test := range tests {
t.Run(name, func(t *testing.T) {
withExpectedArchiveCount(t, 1, func(t *testing.T) {
- vfs := New(
- WithCacheExpirationInterval(test.expirationInterval),
- WithCacheRefreshInterval(test.refreshInterval),
- ).(*zipVFS)
+ cfg := *zipCfg
+ cfg.ExpirationInterval = test.expirationInterval
+ cfg.RefreshInterval = test.refreshInterval
+
+ vfs := New(&cfg).(*zipVFS)
path := testServerURL + test.path
diff --git a/main.go b/main.go
index 95805b25..8527295e 100644
--- a/main.go
+++ b/main.go
@@ -13,6 +13,7 @@ import (
"github.com/namsral/flag"
log "github.com/sirupsen/logrus"
+
"gitlab.com/gitlab-org/labkit/errortracking"
"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
@@ -29,6 +30,7 @@ var VERSION = "dev"
var REVISION = "HEAD"
func init() {
+ // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
flag.Var(&listenHTTP, "listen-http", "The address(es) to listen on for HTTP requests")
flag.Var(&listenHTTPS, "listen-https", "The address(es) to listen on for HTTPS requests")
flag.Var(&listenProxy, "listen-proxy", "The address(es) to listen on for proxy requests")
@@ -37,6 +39,7 @@ func init() {
}
var (
+ // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
pagesRootCert = flag.String("root-cert", "", "The default path to file certificate to serve static pages")
pagesRootKey = flag.String("root-key", "", "The default path to file certificate to serve static pages")
redirectHTTP = flag.Bool("redirect-http", false, "Redirect pages from HTTP to HTTPS")
@@ -75,6 +78,11 @@ var (
insecureCiphers = flag.Bool("insecure-ciphers", false, "Use default list of cipher suites, may contain insecure ones like 3DES and RC4")
tlsMinVersion = flag.String("tls-min-version", "tls1.2", tlsconfig.FlagUsage("min"))
tlsMaxVersion = flag.String("tls-max-version", "", tlsconfig.FlagUsage("max"))
+ // TODO: move all flags to config pkg https://gitlab.com/gitlab-org/gitlab-pages/-/issues/507
+ zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval")
+ zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval")
+ zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval")
+ zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout")
disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests")
@@ -210,6 +218,11 @@ func configFromFlags() appConfig {
config.SentryDSN = *sentryDSN
config.SentryEnvironment = *sentryEnvironment
+ config.ZipCacheExpiry = *zipCacheExpiration
+ config.ZipCacheCleanup = *zipCacheCleanup
+ config.ZipCacheRefresh = *zipCacheRefresh
+ config.ZipeOpenTimeout = *zipOpenTimeout
+
checkAuthenticationConfig(config)
return config
@@ -294,6 +307,10 @@ func loadConfig() appConfig {
"api-secret-key": *gitLabAPISecretKey,
"domain-config-source": config.DomainConfigurationSource,
"auth-redirect-uri": config.RedirectURI,
+ "zip-cache-expiration": config.ZipCacheExpiry,
+ "zip-cache-cleanup": config.ZipCacheCleanup,
+ "zip-cache-refresh": config.ZipCacheRefresh,
+ "zip-open-timeout": config.ZipeOpenTimeout,
}).Debug("Start daemon with configuration")
return config
@@ -304,7 +321,9 @@ func appMain() {
// read from -config=/path/to/gitlab-pages-config
flag.String(flag.DefaultConfigFlagname, "", "path to config file")
+
flag.Parse()
+
if err := tlsconfig.ValidateTLSVersions(*tlsMinVersion, *tlsMaxVersion); err != nil {
fatal(err, "invalid TLS version")
}
diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go
index ea703ebe..3bfa21bf 100644
--- a/test/acceptance/zip_test.go
+++ b/test/acceptance/zip_test.go
@@ -104,6 +104,31 @@ func TestZipServing(t *testing.T) {
}
}
+func TestZipServingConfigShortTimeout(t *testing.T) {
+ skipUnlessEnabled(t)
+
+ var apiCalled bool
+ source := NewGitlabDomainsSourceStub(t, &apiCalled)
+ defer source.Close()
+
+ gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
+
+ pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab",
+ "-zip-open-timeout=1ns"} // <- test purpose
+
+ teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...)
+ defer teardown()
+
+ _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
+ defer cleanup()
+
+ response, err := GetPageFromListener(t, httpListener, "zip.gitlab.io", "/")
+ require.NoError(t, err)
+ defer response.Body.Close()
+
+ require.Equal(t, http.StatusInternalServerError, response.StatusCode, "should fail to serve")
+}
+
func newZipFileServerURL(t *testing.T, zipFilePath string) (string, func()) {
t.Helper()