From 9c51d46d39c1a2614aafca02bf1f3376a88e0280 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Mon, 30 Nov 2020 11:09:30 +1100 Subject: Add Reconfigure to serving and VFS interfaces --- internal/serving/disk/serving.go | 6 ++++++ internal/serving/disk/zip/serving.go | 12 +++++++++--- internal/serving/serverless/serverless.go | 6 ++++++ internal/serving/serving.go | 3 +++ internal/vfs/local/vfs.go | 6 ++++++ internal/vfs/vfs.go | 6 ++++++ internal/vfs/zip/vfs.go | 32 +++++++++++++++++++++++++------ 7 files changed, 62 insertions(+), 9 deletions(-) diff --git a/internal/serving/disk/serving.go b/internal/serving/disk/serving.go index 30c821ea..fbcdf9f2 100644 --- a/internal/serving/disk/serving.go +++ b/internal/serving/disk/serving.go @@ -3,6 +3,7 @@ package disk import ( "os" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" @@ -40,6 +41,11 @@ func (s *Disk) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } +// Reconfigure VFS +func (s *Disk) Reconfigure(cfg *config.Config) error { + return s.reader.vfs.Reconfigure(cfg) +} + // New returns a serving instance that is capable of reading files // from the VFS func New(vfs vfs.VFS) serving.Serving { diff --git a/internal/serving/disk/zip/serving.go b/internal/serving/disk/zip/serving.go index 61d186da..b31d9090 100644 --- a/internal/serving/disk/zip/serving.go +++ b/internal/serving/disk/zip/serving.go @@ -3,6 +3,8 @@ package zip import ( "sync" + "gitlab.com/gitlab-org/labkit/log" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk" @@ -10,14 +12,18 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/vfs/zip" ) -var instance serving.Serving -var once sync.Once +var ( + once sync.Once + instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) +) // Instance returns a serving instance that is capable of reading files // from a zip archives opened from a URL, most likely stored in object storage func Instance() serving.Serving { once.Do(func() { - instance = disk.New(vfs.Instrumented(zip.New(config.Default.Zip))) + if err := instance.Reconfigure(config.Default); err != nil { + log.WithError(err).Fatal("failed to reconfigure zip serving") + } }) return instance diff --git a/internal/serving/serverless/serverless.go b/internal/serving/serverless/serverless.go index e1881362..f8bd4e87 100644 --- a/internal/serving/serverless/serverless.go +++ b/internal/serving/serverless/serverless.go @@ -4,6 +4,7 @@ import ( "errors" "net/http/httputil" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/httperrors" "gitlab.com/gitlab-org/gitlab-pages/internal/serving" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api" @@ -65,3 +66,8 @@ func (s *Serverless) ServeFileHTTP(h serving.Handler) bool { func (s *Serverless) ServeNotFoundHTTP(h serving.Handler) { httperrors.Serve404(h.Writer) } + +// Reconfigure noop +func (s *Serverless) Reconfigure(*config.Config) error { + return nil +} diff --git a/internal/serving/serving.go b/internal/serving/serving.go index 6fde8216..786ee569 100644 --- a/internal/serving/serving.go +++ b/internal/serving/serving.go @@ -1,7 +1,10 @@ package serving +import "gitlab.com/gitlab-org/gitlab-pages/internal/config" + // Serving is an interface used to define a serving driver type Serving interface { ServeFileHTTP(Handler) bool ServeNotFoundHTTP(Handler) + Reconfigure(config *config.Config) error } diff --git a/internal/vfs/local/vfs.go b/internal/vfs/local/vfs.go index bdcd5160..ea54e8e8 100644 --- a/internal/vfs/local/vfs.go +++ b/internal/vfs/local/vfs.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/vfs" ) @@ -43,3 +44,8 @@ func (fs VFS) Root(ctx context.Context, path string) (vfs.Root, error) { func (fs *VFS) Name() string { return "local" } + +func (fs *VFS) Reconfigure(*config.Config) error { + // noop + return nil +} diff --git a/internal/vfs/vfs.go b/internal/vfs/vfs.go index 7bd51db2..2304f903 100644 --- a/internal/vfs/vfs.go +++ b/internal/vfs/vfs.go @@ -6,6 +6,7 @@ import ( log "github.com/sirupsen/logrus" + "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/metrics" ) @@ -13,6 +14,7 @@ import ( type VFS interface { Root(ctx context.Context, path string) (Root, error) Name() string + Reconfigure(config *config.Config) error } func Instrumented(fs VFS) VFS { @@ -50,3 +52,7 @@ func (i *instrumentedVFS) Root(ctx context.Context, path string) (Root, error) { func (i *instrumentedVFS) Name() string { return i.fs.Name() } + +func (i *instrumentedVFS) Reconfigure(cfg *config.Config) error { + return i.fs.Reconfigure(cfg) +} diff --git a/internal/vfs/zip/vfs.go b/internal/vfs/zip/vfs.go index d90d4796..b27424c6 100644 --- a/internal/vfs/zip/vfs.go +++ b/internal/vfs/zip/vfs.go @@ -56,12 +56,7 @@ func New(cfg *config.ZipServing) vfs.VFS { openTimeout: cfg.OpenTimeout, } - zipVFS.cache = cache.New(zipVFS.cacheExpirationInterval, zipVFS.cacheCleanupInterval) - zipVFS.cache.OnEvicted(func(s string, i interface{}) { - metrics.ZipCachedEntries.WithLabelValues("archive").Dec() - - i.(*zipArchive).onEvicted() - }) + zipVFS.resetCache() // TODO: To be removed with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/480 zipVFS.dataOffsetCache = newLruCache("data-offset", defaultDataOffsetItems, defaultDataOffsetExpirationInterval) @@ -70,6 +65,31 @@ func New(cfg *config.ZipServing) vfs.VFS { return zipVFS } +// Reconfigure will update the zipVFS configuration values and will reset the +// cache +func (fs *zipVFS) Reconfigure(cfg *config.Config) error { + fs.cacheLock.Lock() + defer fs.cacheLock.Unlock() + + fs.openTimeout = cfg.Zip.OpenTimeout + fs.cacheExpirationInterval = cfg.Zip.ExpirationInterval + fs.cacheRefreshInterval = cfg.Zip.RefreshInterval + fs.cacheCleanupInterval = cfg.Zip.CleanupInterval + + fs.resetCache() + + return nil +} + +func (fs *zipVFS) resetCache() { + fs.cache = cache.New(fs.cacheExpirationInterval, fs.cacheCleanupInterval) + fs.cache.OnEvicted(func(s string, i interface{}) { + metrics.ZipCachedEntries.WithLabelValues("archive").Dec() + + i.(*zipArchive).onEvicted() + }) +} + func (fs *zipVFS) keyFromPath(path string) (string, error) { // We assume that our URL is https://.../artifacts.zip?content-sign=aaa // our caching key is `https://.../artifacts.zip` -- cgit v1.2.3