Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-pages.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--internal/config/config.go42
-rw-r--r--internal/config/headers.go32
-rw-r--r--internal/config/headers_test.go (renamed from internal/config/config_test.go)0
-rw-r--r--internal/serving/disk/zip/serving.go7
-rw-r--r--internal/vfs/zip/archive_test.go20
-rw-r--r--internal/vfs/zip/lru_cache.go6
-rw-r--r--internal/vfs/zip/vfs.go68
-rw-r--r--internal/vfs/zip/vfs_test.go18
-rw-r--r--main.go30
9 files changed, 123 insertions, 100 deletions
diff --git a/internal/config/config.go b/internal/config/config.go
index d415b21d..ef2af686 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,32 +1,22 @@
package config
-import (
- "errors"
- "net/http"
- "strings"
-)
+import "time"
-var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter")
+// TODO: refactor config flags in main.go and find a better way to handle all settings
-// AddCustomHeaders adds a map of Headers to a Response
-func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error {
- for k, v := range headers {
- for _, value := range v {
- w.Header().Add(k, value)
- }
- }
- return nil
-}
+// ZipVFSConfig struct that can be accessed by different packages to share
+// configuration parameters
+var ZipVFSConfig *ZipServing
-// ParseHeaderString parses a string of key values into a map
-func ParseHeaderString(customHeaders []string) (http.Header, error) {
- headers := http.Header{}
- for _, keyValueString := range customHeaders {
- keyValue := strings.SplitN(keyValueString, ":", 2)
- if len(keyValue) != 2 {
- return nil, errInvalidHeaderParameter
- }
- headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1]))
- }
- return headers, nil
+// ZipServing stores all configuration values to be used by the zip VFS opening and
+// caching
+type ZipServing struct {
+ ExpirationInterval time.Duration
+ CleanupInterval time.Duration
+ RefreshInterval time.Duration
+ OpenTimeout time.Duration
+ DataOffsetItems int64
+ DataOffsetExpirationInterval time.Duration
+ ReadlinkItems int64
+ ReadlinkExpirationInterval time.Duration
}
diff --git a/internal/config/headers.go b/internal/config/headers.go
new file mode 100644
index 00000000..d415b21d
--- /dev/null
+++ b/internal/config/headers.go
@@ -0,0 +1,32 @@
+package config
+
+import (
+ "errors"
+ "net/http"
+ "strings"
+)
+
+var errInvalidHeaderParameter = errors.New("invalid syntax specified as header parameter")
+
+// AddCustomHeaders adds a map of Headers to a Response
+func AddCustomHeaders(w http.ResponseWriter, headers http.Header) error {
+ for k, v := range headers {
+ for _, value := range v {
+ w.Header().Add(k, value)
+ }
+ }
+ return nil
+}
+
+// ParseHeaderString parses a string of key values into a map
+func ParseHeaderString(customHeaders []string) (http.Header, error) {
+ headers := http.Header{}
+ for _, keyValueString := range customHeaders {
+ keyValue := strings.SplitN(keyValueString, ":", 2)
+ if len(keyValue) != 2 {
+ return nil, errInvalidHeaderParameter
+ }
+ headers[strings.TrimSpace(keyValue[0])] = append(headers[strings.TrimSpace(keyValue[0])], strings.TrimSpace(keyValue[1]))
+ }
+ return headers, nil
+}
diff --git a/internal/config/config_test.go b/internal/config/headers_test.go
index 44afd470..44afd470 100644
--- a/internal/config/config_test.go
+++ b/internal/config/headers_test.go
diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go
index 95894fc9..9b3a7aa7 100644
--- a/internal/serving/disk/zip/serving.go
+++ b/internal/serving/disk/zip/serving.go
@@ -1,16 +1,21 @@
package zip
import (
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving"
"gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip"
)
-var instance = disk.New(vfs.Instrumented(zip.New()))
+var instance serving.Serving
// Instance returns a serving instance that is capable of reading files
// from a zip archives opened from a URL, most likely stored in object storage
func Instance() serving.Serving {
+ if instance == nil {
+ instance = disk.New(vfs.Instrumented(zip.New(config.ZipVFSConfig)))
+ }
+
return instance
}
diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go
index e1b0d116..f4fd6b5a 100644
--- a/internal/vfs/zip/archive_test.go
+++ b/internal/vfs/zip/archive_test.go
@@ -12,12 +12,24 @@ import (
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httprange"
"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
)
var chdirSet = false
+var zipCfg = &config.ZipServing{
+ ExpirationInterval: time.Minute,
+ CleanupInterval: time.Minute / 2,
+ RefreshInterval: time.Minute / 2,
+ OpenTimeout: time.Minute / 2,
+ DataOffsetItems: 1000,
+ DataOffsetExpirationInterval: time.Hour,
+ ReadlinkItems: 500,
+ ReadlinkExpirationInterval: time.Hour,
+}
+
func TestOpen(t *testing.T) {
zip, cleanup := openZipArchive(t, nil)
defer cleanup()
@@ -75,7 +87,7 @@ func TestOpenCached(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", &requests)
defer cleanup()
- fs := New()
+ fs := New(zipCfg)
// We use array instead of map to ensure
// predictable ordering of test execution
@@ -322,7 +334,7 @@ func TestArchiveCanBeReadAfterOpenCtxCanceled(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil)
defer cleanup()
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -345,7 +357,7 @@ func TestReadArchiveFails(t *testing.T) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public.zip", nil)
defer cleanup()
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
err := zip.openArchive(context.Background(), testServerURL+"/unkown.html")
@@ -365,7 +377,7 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) {
testServerURL, cleanup := newZipFileServerURL(t, "group/zip.gitlab.io/public-without-dirs.zip", requests)
- fs := New().(*zipVFS)
+ fs := New(zipCfg).(*zipVFS)
zip := newArchive(fs, time.Second)
err := zip.openArchive(context.Background(), testServerURL+"/public.zip")
diff --git a/internal/vfs/zip/lru_cache.go b/internal/vfs/zip/lru_cache.go
index 36817e66..9810e245 100644
--- a/internal/vfs/zip/lru_cache.go
+++ b/internal/vfs/zip/lru_cache.go
@@ -24,10 +24,10 @@ type lruCache struct {
cache *ccache.Cache
}
-func newLruCache(op string, maxEntries uint32, duration time.Duration) *lruCache {
+func newLruCache(op string, maxEntries int64, duration time.Duration) *lruCache {
configuration := ccache.Configure()
- configuration.MaxSize(int64(maxEntries))
- configuration.ItemsToPrune(maxEntries / lruCacheItemsToPruneDiv)
+ configuration.MaxSize(maxEntries)
+ configuration.ItemsToPrune(uint32(maxEntries) / lruCacheItemsToPruneDiv)
configuration.GetsPerPromote(lruCacheGetsPerPromote) // if item gets requested frequently promote it
configuration.OnDelete(func(*ccache.Item) {
metrics.ZipCachedEntries.WithLabelValues(op).Dec()
diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go
index 692a4a69..feeab974 100644
--- a/internal/vfs/zip/vfs.go
+++ b/internal/vfs/zip/vfs.go
@@ -9,29 +9,12 @@ import (
"github.com/patrickmn/go-cache"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httprange"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
)
-const (
- // TODO: make these configurable https://gitlab.com/gitlab-org/gitlab-pages/-/issues/464
- defaultCacheExpirationInterval = time.Minute
- defaultCacheCleanupInterval = time.Minute / 2
- defaultCacheRefreshInterval = time.Minute / 2
- defaultOpenTimeout = time.Minute / 2
-
- // we assume that each item costs around 100 bytes
- // this gives around 5MB of raw memory needed without acceleration structures
- defaultDataOffsetItems = 50000
- defaultDataOffsetExpirationInterval = time.Hour
-
- // we assume that each item costs around 200 bytes
- // this gives around 2MB of raw memory needed without acceleration structures
- defaultReadlinkItems = 10000
- defaultReadlinkExpirationInterval = time.Hour
-)
-
var (
errAlreadyCached = errors.New("archive already cached")
)
@@ -52,48 +35,13 @@ type zipVFS struct {
archiveCount int64
}
-// Option function allows to override default values
-type Option func(*zipVFS)
-
-// WithCacheRefreshInterval when used it can override defaultCacheRefreshInterval
-func WithCacheRefreshInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheRefreshInterval = interval
- }
-}
-
-// WithCacheExpirationInterval when used it can override defaultCacheExpirationInterval
-func WithCacheExpirationInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheExpirationInterval = interval
- }
-}
-
-// WithCacheCleanupInterval when used it can override defaultCacheCleanupInterval
-func WithCacheCleanupInterval(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.cacheCleanupInterval = interval
- }
-}
-
-// WithOpenTimeout when used it can override openTimeout
-func WithOpenTimeout(interval time.Duration) Option {
- return func(vfs *zipVFS) {
- vfs.openTimeout = interval
- }
-}
-
// New creates a zipVFS instance that can be used by a serving request
-func New(options ...Option) vfs.VFS {
+func New(cfg *config.ZipServing) vfs.VFS {
zipVFS := &zipVFS{
- cacheExpirationInterval: defaultCacheExpirationInterval,
- cacheRefreshInterval: defaultCacheRefreshInterval,
- cacheCleanupInterval: defaultCacheCleanupInterval,
- openTimeout: defaultOpenTimeout,
- }
-
- for _, option := range options {
- option(zipVFS)
+ cacheExpirationInterval: cfg.ExpirationInterval,
+ cacheRefreshInterval: cfg.RefreshInterval,
+ cacheCleanupInterval: cfg.CleanupInterval,
+ openTimeout: cfg.OpenTimeout,
}
zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval)
@@ -104,8 +52,8 @@ func New(options ...Option) vfs.VFS {
})
// TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480
- zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval)
- zipVFS.readlinkCache = newLruCache("readlink", defaultReadlinkItems, defaultReadlinkExpirationInterval)
+ zipVFS.dataOffsetCache = newLruCache("data-offset", cfg.DataOffsetItems, cfg.DataOffsetExpirationInterval)
+ zipVFS.readlinkCache = newLruCache("readlink", cfg.ReadlinkItems, cfg.ReadlinkExpirationInterval)
return zipVFS
}
diff --git a/internal/vfs/zip/vfs_test.go b/internal/vfs/zip/vfs_test.go
index dff2ff43..aa1ed109 100644
--- a/internal/vfs/zip/vfs_test.go
+++ b/internal/vfs/zip/vfs_test.go
@@ -9,6 +9,7 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/require"
+ "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/httprange"
"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
"gitlab.com/gitlab-org/gitlab-pages/metrics"
@@ -35,7 +36,7 @@ func TestVFSRoot(t *testing.T) {
},
}
- vfs := New()
+ vfs := New(zipCfg)
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
@@ -73,7 +74,11 @@ func TestVFSFindOrOpenArchiveConcurrentAccess(t *testing.T) {
path := testServerURL + "/public.zip"
- vfs := New().(*zipVFS)
+ vfs := New(config.ZipServing{
+ OpenTimeout: time.Second,
+ DataOffsetItems: 20,
+ ReadlinkItems: 20,
+ }).(*zipVFS)
root, err := vfs.Root(context.Background(), path)
require.NoError(t, err)
@@ -158,10 +163,11 @@ func TestVFSFindOrOpenArchiveRefresh(t *testing.T) {
for name, test := range tests {
t.Run(name, func(t *testing.T) {
withExpectedArchiveCount(t, 1, func(t *testing.T) {
- vfs := New(
- WithCacheExpirationInterval(test.expirationInterval),
- WithCacheRefreshInterval(test.refreshInterval),
- ).(*zipVFS)
+ cfg := *zipCfg
+ cfg.ExpirationInterval = test.expirationInterval
+ cfg.RefreshInterval = test.refreshInterval
+
+ vfs := New(&cfg).(*zipVFS)
path := testServerURL + test.path
diff --git a/main.go b/main.go
index 7defd281..b5056fa9 100644
--- a/main.go
+++ b/main.go
@@ -13,8 +13,10 @@ import (
"github.com/namsral/flag"
log "github.com/sirupsen/logrus"
+
"gitlab.com/gitlab-org/labkit/errortracking"
+ cfg "gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/logging"
"gitlab.com/gitlab-org/gitlab-pages/internal/request"
"gitlab.com/gitlab-org/gitlab-pages/internal/tlsconfig"
@@ -78,6 +80,22 @@ var (
disableCrossOriginRequests = flag.Bool("disable-cross-origin-requests", false, "Disable cross-origin requests")
+ // zip serving settings
+ zipCacheExpiration = flag.Duration("zip-cache-expiration", 60*time.Second, "Zip serving archive cache expiration interval (default: 60s)")
+ zipCacheCleanup = flag.Duration("zip-cache-cleanup", 30*time.Second, "Zip serving archive cache cleanup interval (default: 30s)")
+ zipCacheRefresh = flag.Duration("zip-cache-refresh", 30*time.Second, "Zip serving archive cache refresh interval (default: 30s)")
+
+ // we assume that each item costs around 100 bytes
+ // this gives around 5MB of raw memory needed without acceleration structures
+ zipCacheDataOffsetItems = flag.Int64("zip-cache-dataoffset-items", 50000, "Zip serving number of files to cache per archive (default: 50,000)")
+ zipCachetDataOffsetExpiration = flag.Duration("zip-cache-dataoffset-expiration", time.Hour, "Zip serving cached files expiration interval (default: 1h)")
+ // we assume that each item costs around 200 bytes
+ // this gives around 2MB of raw memory needed without acceleration structures
+ zipCacheReadlinkItems = flag.Int64("zip-cache-readlink-items", 10000, "Zip serving number of symbolic links to cache per archive (default: 10,000)")
+ zipCacheReadlinkExpirationInterval = flag.Duration("zip-cache-readlink-expiration", time.Hour, "Zip serving cached symbolic links expiration interval (default: 1h)")
+
+ zipOpenTimeout = flag.Duration("zip-open-timeout", 30*time.Second, "Zip archive open timeout (default: 30s)")
+
// See init()
listenHTTP MultiStringFlag
listenHTTPS MultiStringFlag
@@ -212,6 +230,18 @@ func configFromFlags() appConfig {
checkAuthenticationConfig(config)
+ cfg.ZipVFSConfig = &cfg.ZipServing{
+ ExpirationInterval: *zipCacheExpiration,
+ CleanupInterval: *zipCacheCleanup,
+ RefreshInterval: *zipCacheRefresh,
+ OpenTimeout: *zipOpenTimeout,
+ DataOffsetItems: *zipCacheDataOffsetItems,
+ DataOffsetExpirationInterval: *zipCachetDataOffsetExpiration,
+ ReadlinkItems: *zipCacheReadlinkItems,
+ ReadlinkExpirationInterval: *zipCacheReadlinkExpirationInterval,
+ }
+
+ fmt.Printf("THE CONFIG FIRST:\n%+v\n", cfg.ZipVFSConfig)
return config
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-23 13:02:58 +0300