From 17d7ecde2b261d2ab29049d12361b66504e3f995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 14 Nov 2018 10:44:04 +0100 Subject: cache/filecache: Split implementation and config into separate files --- cache/filecache/filecache.go | 145 -------------------------- cache/filecache/filecache_config.go | 168 +++++++++++++++++++++++++++++++ cache/filecache/filecache_config_test.go | 131 ++++++++++++++++++++++++ cache/filecache/filecache_test.go | 106 ------------------- 4 files changed, 299 insertions(+), 251 deletions(-) create mode 100644 cache/filecache/filecache_config.go create mode 100644 cache/filecache/filecache_config_test.go (limited to 'cache') diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 4232c791a..c6ea02681 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -17,7 +17,6 @@ import ( "bytes" "io" "io/ioutil" - "path" "path/filepath" "strings" "time" @@ -28,56 +27,10 @@ import ( "github.com/gohugoio/hugo/hugolib/paths" - "github.com/pkg/errors" - "github.com/BurntSushi/locker" - "github.com/bep/mapstructure" "github.com/spf13/afero" ) -const ( - cachesConfigKey = "caches" - - resourcesGenDir = ":resourceDir/_gen" -) - -var defaultCacheConfig = cacheConfig{ - MaxAge: -1, // Never expire - Dir: ":cacheDir", -} - -const ( - cacheKeyGetJSON = "getjson" - cacheKeyGetCSV = "getcsv" - cacheKeyImages = "images" - cacheKeyAssets = "assets" -) - -var defaultCacheConfigs = map[string]cacheConfig{ - cacheKeyGetJSON: defaultCacheConfig, - cacheKeyGetCSV: defaultCacheConfig, - cacheKeyImages: cacheConfig{ - MaxAge: -1, - Dir: resourcesGenDir, - }, - cacheKeyAssets: cacheConfig{ - MaxAge: -1, - Dir: resourcesGenDir, - }, -} - -type cachesConfig map[string]cacheConfig - -type cacheConfig struct { - // Max age of cache entries in this cache. Any items older than this will - // be removed and not returned from the cache. - // -1 means forever, 0 means cache is disabled. - MaxAge int - - // The directory where files are stored. - Dir string -} - // Cache caches a set of files in a directory. This is usually a file on // disk, but since this is backed by an Afero file system, it can be anything. type Cache struct { @@ -316,26 +269,6 @@ func (f Caches) Get(name string) *Cache { return f[strings.ToLower(name)] } -// GetJSONCache gets the file cache for getJSON. -func (f Caches) GetJSONCache() *Cache { - return f[cacheKeyGetJSON] -} - -// GetCSVCache gets the file cache for getCSV. -func (f Caches) GetCSVCache() *Cache { - return f[cacheKeyGetCSV] -} - -// ImageCache gets the file cache for processed images. -func (f Caches) ImageCache() *Cache { - return f[cacheKeyImages] -} - -// AssetsCache gets the file cache for assets (processed resources, SCSS etc.). -func (f Caches) AssetsCache() *Cache { - return f[cacheKeyAssets] -} - // NewCachesFromPaths creates a new set of file caches from the given // configuration. func NewCachesFromPaths(p *paths.Paths) (Caches, error) { @@ -359,84 +292,6 @@ func NewCachesFromPaths(p *paths.Paths) (Caches, error) { return m, nil } -func decodeConfig(p *paths.Paths) (cachesConfig, error) { - c := make(cachesConfig) - valid := make(map[string]bool) - // Add defaults - for k, v := range defaultCacheConfigs { - c[k] = v - valid[k] = true - } - - cfg := p.Cfg - - m := cfg.GetStringMap(cachesConfigKey) - - _, isOsFs := p.Fs.Source.(*afero.OsFs) - - for k, v := range m { - cc := defaultCacheConfig - - if err := mapstructure.WeakDecode(v, &cc); err != nil { - return nil, err - } - - if cc.Dir == "" { - return c, errors.New("must provide cache Dir") - } - - name := strings.ToLower(k) - if !valid[name] { - return nil, errors.Errorf("%q is not a valid cache name", name) - } - - c[name] = cc - } - - // This is a very old flag in Hugo, but we need to respect it. - disabled := cfg.GetBool("ignoreCache") - - for k, v := range c { - v.Dir = filepath.Clean(v.Dir) - dir := filepath.ToSlash(v.Dir) - parts := strings.Split(dir, "/") - first := parts[0] - - if strings.HasPrefix(first, ":") { - resolved, err := resolveDirPlaceholder(p, first) - if err != nil { - return c, err - } - resolved = filepath.ToSlash(resolved) - - v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...)) - - } else if isOsFs && !path.IsAbs(dir) { - return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir) - } - - if disabled { - v.MaxAge = 0 - } - - c[k] = v - } - - return c, nil -} - -// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... -func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) { - switch strings.ToLower(placeholder) { - case ":resourcedir": - return p.AbsResourcesDir, nil - case ":cachedir": - return helpers.GetCacheDir(p.Fs.Source, p.Cfg) - } - - return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) -} - func cleanID(name string) string { return filepath.Clean(name) } diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go new file mode 100644 index 000000000..f83e19e0e --- /dev/null +++ b/cache/filecache/filecache_config.go @@ -0,0 +1,168 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "path" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/paths" + + "github.com/bep/mapstructure" + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +const ( + cachesConfigKey = "caches" + + resourcesGenDir = ":resourceDir/_gen" +) + +var defaultCacheConfig = cacheConfig{ + MaxAge: -1, // Never expire + Dir: ":cacheDir", +} + +const ( + cacheKeyGetJSON = "getjson" + cacheKeyGetCSV = "getcsv" + cacheKeyImages = "images" + cacheKeyAssets = "assets" +) + +var defaultCacheConfigs = map[string]cacheConfig{ + cacheKeyGetJSON: defaultCacheConfig, + cacheKeyGetCSV: defaultCacheConfig, + cacheKeyImages: cacheConfig{ + MaxAge: -1, + Dir: resourcesGenDir, + }, + cacheKeyAssets: cacheConfig{ + MaxAge: -1, + Dir: resourcesGenDir, + }, +} + +type cachesConfig map[string]cacheConfig + +type cacheConfig struct { + // Max age of cache entries in this cache. Any items older than this will + // be removed and not returned from the cache. + // -1 means forever, 0 means cache is disabled. + MaxAge int + + // The directory where files are stored. + Dir string +} + +// GetJSONCache gets the file cache for getJSON. +func (f Caches) GetJSONCache() *Cache { + return f[cacheKeyGetJSON] +} + +// GetCSVCache gets the file cache for getCSV. +func (f Caches) GetCSVCache() *Cache { + return f[cacheKeyGetCSV] +} + +// ImageCache gets the file cache for processed images. +func (f Caches) ImageCache() *Cache { + return f[cacheKeyImages] +} + +// AssetsCache gets the file cache for assets (processed resources, SCSS etc.). +func (f Caches) AssetsCache() *Cache { + return f[cacheKeyAssets] +} + +func decodeConfig(p *paths.Paths) (cachesConfig, error) { + c := make(cachesConfig) + valid := make(map[string]bool) + // Add defaults + for k, v := range defaultCacheConfigs { + c[k] = v + valid[k] = true + } + + cfg := p.Cfg + + m := cfg.GetStringMap(cachesConfigKey) + + _, isOsFs := p.Fs.Source.(*afero.OsFs) + + for k, v := range m { + cc := defaultCacheConfig + + if err := mapstructure.WeakDecode(v, &cc); err != nil { + return nil, err + } + + if cc.Dir == "" { + return c, errors.New("must provide cache Dir") + } + + name := strings.ToLower(k) + if !valid[name] { + return nil, errors.Errorf("%q is not a valid cache name", name) + } + + c[name] = cc + } + + // This is a very old flag in Hugo, but we need to respect it. + disabled := cfg.GetBool("ignoreCache") + + for k, v := range c { + v.Dir = filepath.Clean(v.Dir) + dir := filepath.ToSlash(v.Dir) + parts := strings.Split(dir, "/") + first := parts[0] + + if strings.HasPrefix(first, ":") { + resolved, err := resolveDirPlaceholder(p, first) + if err != nil { + return c, err + } + resolved = filepath.ToSlash(resolved) + + v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...)) + + } else if isOsFs && !path.IsAbs(dir) { + return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir) + } + + if disabled { + v.MaxAge = 0 + } + + c[k] = v + } + + return c, nil +} + +// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... +func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) { + switch strings.ToLower(placeholder) { + case ":resourcedir": + return p.AbsResourcesDir, nil + case ":cachedir": + return helpers.GetCacheDir(p.Fs.Source, p.Cfg) + } + + return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) +} diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go new file mode 100644 index 000000000..209be823d --- /dev/null +++ b/cache/filecache/filecache_config_test.go @@ -0,0 +1,131 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filecache + +import ( + "path/filepath" + "runtime" + "testing" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/paths" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestDecodeConfig(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + configStr := ` +[caches] +[caches.getJSON] +maxAge = 1234 +dir = "/path/to/c1" +[caches.getCSV] +maxAge = 3456 +dir = "/path/to/c2" +[caches.images] +dir = "/path/to/c3" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + fs := hugofs.NewMem(cfg) + p, err := paths.New(fs, cfg) + assert.NoError(err) + + decoded, err := decodeConfig(p) + assert.NoError(err) + + assert.Equal(4, len(decoded)) + + c2 := decoded["getcsv"] + assert.Equal(3456, c2.MaxAge) + assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir) + + c3 := decoded["images"] + assert.Equal(-1, c3.MaxAge) + assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir) + +} + +func TestDecodeConfigIgnoreCache(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + configStr := ` +ignoreCache = true +[caches] +[caches.getJSON] +maxAge = 1234 +dir = "/path/to/c1" +[caches.getCSV] +maxAge = 3456 +dir = "/path/to/c2" +[caches.images] +dir = "/path/to/c3" + +` + + cfg, err := config.FromConfigString(configStr, "toml") + assert.NoError(err) + fs := hugofs.NewMem(cfg) + p, err := paths.New(fs, cfg) + assert.NoError(err) + + decoded, err := decodeConfig(p) + assert.NoError(err) + + assert.Equal(4, len(decoded)) + + for _, v := range decoded { + assert.Equal(0, v.MaxAge) + } + +} + +func TestDecodeConfigDefault(t *testing.T) { + assert := require.New(t) + cfg := viper.New() + if runtime.GOOS == "windows" { + cfg.Set("resourceDir", "c:\\cache\\resources") + cfg.Set("cacheDir", "c:\\cache\\thecache") + + } else { + cfg.Set("resourceDir", "/cache/resources") + cfg.Set("cacheDir", "/cache/thecache") + } + + fs := hugofs.NewMem(cfg) + p, err := paths.New(fs, cfg) + assert.NoError(err) + + decoded, err := decodeConfig(p) + + assert.NoError(err) + + assert.Equal(4, len(decoded)) + + if runtime.GOOS == "windows" { + assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir) + } else { + assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir) + } +} diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index d483fc1a7..4f5336be5 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "path/filepath" "regexp" - "runtime" "strings" "sync" "testing" @@ -31,7 +30,6 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/paths" "github.com/spf13/afero" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -200,107 +198,3 @@ dir = "/cache/c" } wg.Wait() } - -func TestDecodeConfig(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - configStr := ` -[caches] -[caches.getJSON] -maxAge = 1234 -dir = "/path/to/c1" -[caches.getCSV] -maxAge = 3456 -dir = "/path/to/c2" -[caches.images] -dir = "/path/to/c3" - -` - - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := paths.New(fs, cfg) - assert.NoError(err) - - decoded, err := decodeConfig(p) - assert.NoError(err) - - assert.Equal(4, len(decoded)) - - c2 := decoded["getcsv"] - assert.Equal(3456, c2.MaxAge) - assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir) - - c3 := decoded["images"] - assert.Equal(-1, c3.MaxAge) - assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir) - -} - -func TestDecodeConfigIgnoreCache(t *testing.T) { - t.Parallel() - - assert := require.New(t) - - configStr := ` -ignoreCache = true -[caches] -[caches.getJSON] -maxAge = 1234 -dir = "/path/to/c1" -[caches.getCSV] -maxAge = 3456 -dir = "/path/to/c2" -[caches.images] -dir = "/path/to/c3" - -` - - cfg, err := config.FromConfigString(configStr, "toml") - assert.NoError(err) - fs := hugofs.NewMem(cfg) - p, err := paths.New(fs, cfg) - assert.NoError(err) - - decoded, err := decodeConfig(p) - assert.NoError(err) - - assert.Equal(4, len(decoded)) - - for _, v := range decoded { - assert.Equal(0, v.MaxAge) - } - -} - -func TestDecodeConfigDefault(t *testing.T) { - assert := require.New(t) - cfg := viper.New() - if runtime.GOOS == "windows" { - cfg.Set("resourceDir", "c:\\cache\\resources") - cfg.Set("cacheDir", "c:\\cache\\thecache") - - } else { - cfg.Set("resourceDir", "/cache/resources") - cfg.Set("cacheDir", "/cache/thecache") - } - - fs := hugofs.NewMem(cfg) - p, err := paths.New(fs, cfg) - assert.NoError(err) - - decoded, err := decodeConfig(p) - - assert.NoError(err) - - assert.Equal(4, len(decoded)) - - if runtime.GOOS == "windows" { - assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir) - } else { - assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir) - } -} -- cgit v1.2.3