Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/gohugoio/hugo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/cache
diff options
context:
space:
mode:
Diffstat (limited to 'cache')
-rw-r--r--cache/filecache/filecache.go49
-rw-r--r--cache/filecache/filecache_config.go56
-rw-r--r--cache/filecache/filecache_config_test.go45
-rw-r--r--cache/filecache/filecache_pruner.go117
-rw-r--r--cache/filecache/filecache_pruner_test.go11
-rw-r--r--cache/filecache/filecache_test.go64
6 files changed, 223 insertions, 119 deletions
diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go
index 6ad417117..bf004c8f7 100644
--- a/cache/filecache/filecache.go
+++ b/cache/filecache/filecache.go
@@ -44,6 +44,9 @@ type Cache struct {
// 0 is effectively turning this cache off.
maxAge time.Duration
+ // When set, we just remove this entire root directory on expiration.
+ pruneAllRootDir string
+
nlocker *lockTracker
}
@@ -77,11 +80,12 @@ type ItemInfo struct {
}
// NewCache creates a new file cache with the given filesystem and max age.
-func NewCache(fs afero.Fs, maxAge time.Duration) *Cache {
+func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
return &Cache{
- Fs: fs,
- nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
- maxAge: maxAge,
+ Fs: fs,
+ nlocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+ maxAge: maxAge,
+ pruneAllRootDir: pruneAllRootDir,
}
}
@@ -307,9 +311,15 @@ func (f Caches) Get(name string) *Cache {
// NewCaches creates a new set of file caches from the given
// configuration.
func NewCaches(p *helpers.PathSpec) (Caches, error) {
- dcfg, err := decodeConfig(p)
- if err != nil {
- return nil, err
+ var dcfg Configs
+ if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok {
+ dcfg = c
+ } else {
+ var err error
+ dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg)
+ if err != nil {
+ return nil, err
+ }
}
fs := p.Fs.Source
@@ -319,30 +329,25 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
var cfs afero.Fs
if v.isResourceDir {
- cfs = p.BaseFs.Resources.Fs
+ cfs = p.BaseFs.ResourcesCache
} else {
cfs = fs
}
- var baseDir string
- if !strings.HasPrefix(v.Dir, "_gen") {
- // We do cache eviction (file removes) and since the user can set
- // his/hers own cache directory, we really want to make sure
- // we do not delete any files that do not belong to this cache.
- // We do add the cache name as the root, but this is an extra safe
- // guard. We skip the files inside /resources/_gen/ because
- // that would be breaking.
- baseDir = filepath.Join(v.Dir, filecacheRootDirname, k)
- } else {
- baseDir = filepath.Join(v.Dir, k)
- }
- if err = cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
+ baseDir := v.Dir
+
+ if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) {
return nil, err
}
bfs := afero.NewBasePathFs(cfs, baseDir)
- m[k] = NewCache(bfs, v.MaxAge)
+ var pruneAllRootDir string
+ if k == cacheKeyModules {
+ pruneAllRootDir = "pkg"
+ }
+
+ m[k] = NewCache(bfs, v.MaxAge, pruneAllRootDir)
}
return m, nil
diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go
index a6a0252b2..0c6b569c1 100644
--- a/cache/filecache/filecache_config.go
+++ b/cache/filecache/filecache_config.go
@@ -19,6 +19,8 @@ import (
"strings"
"time"
+ "github.com/gohugoio/hugo/config"
+
"github.com/gohugoio/hugo/helpers"
"github.com/mitchellh/mapstructure"
@@ -32,7 +34,7 @@ const (
resourcesGenDir = ":resourceDir/_gen"
)
-var defaultCacheConfig = cacheConfig{
+var defaultCacheConfig = Config{
MaxAge: -1, // Never expire
Dir: ":cacheDir/:project",
}
@@ -42,9 +44,20 @@ const (
cacheKeyGetCSV = "getcsv"
cacheKeyImages = "images"
cacheKeyAssets = "assets"
+ cacheKeyModules = "modules"
)
-var defaultCacheConfigs = map[string]cacheConfig{
+type Configs map[string]Config
+
+func (c Configs) CacheDirModules() string {
+ return c[cacheKeyModules].Dir
+}
+
+var defaultCacheConfigs = Configs{
+ cacheKeyModules: {
+ MaxAge: -1,
+ Dir: ":cacheDir/modules",
+ },
cacheKeyGetJSON: defaultCacheConfig,
cacheKeyGetCSV: defaultCacheConfig,
cacheKeyImages: {
@@ -57,9 +70,7 @@ var defaultCacheConfigs = map[string]cacheConfig{
},
}
-type cachesConfig map[string]cacheConfig
-
-type cacheConfig struct {
+type Config struct {
// Max age of cache entries in this cache. Any items older than this will
// be removed and not returned from the cache.
// a negative value means forever, 0 means cache is disabled.
@@ -88,13 +99,18 @@ func (f Caches) ImageCache() *Cache {
return f[cacheKeyImages]
}
+// ModulesCache gets the file cache for Hugo Modules.
+func (f Caches) ModulesCache() *Cache {
+ return f[cacheKeyModules]
+}
+
// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
return f[cacheKeyAssets]
}
-func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
- c := make(cachesConfig)
+func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) {
+ c := make(Configs)
valid := make(map[string]bool)
// Add defaults
for k, v := range defaultCacheConfigs {
@@ -102,11 +118,9 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
valid[k] = true
}
- cfg := p.Cfg
-
m := cfg.GetStringMap(cachesConfigKey)
- _, isOsFs := p.Fs.Source.(*afero.OsFs)
+ _, isOsFs := fs.(*afero.OsFs)
for k, v := range m {
cc := defaultCacheConfig
@@ -148,7 +162,7 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
for i, part := range parts {
if strings.HasPrefix(part, ":") {
- resolved, isResource, err := resolveDirPlaceholder(p, part)
+ resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part)
if err != nil {
return c, err
}
@@ -176,6 +190,18 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
}
}
+ if !strings.HasPrefix(v.Dir, "_gen") {
+ // We do cache eviction (file removes) and since the user can set
+ // his/hers own cache directory, we really want to make sure
+ // we do not delete any files that do not belong to this cache.
+ // We do add the cache name as the root, but this is an extra safe
+ // guard. We skip the files inside /resources/_gen/ because
+ // that would be breaking.
+ v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k)
+ } else {
+ v.Dir = filepath.Join(v.Dir, k)
+ }
+
if disabled {
v.MaxAge = 0
}
@@ -187,15 +213,17 @@ func decodeConfig(p *helpers.PathSpec) (cachesConfig, error) {
}
// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *helpers.PathSpec, placeholder string) (cacheDir string, isResource bool, err error) {
+func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) {
+ workingDir := cfg.GetString("workingDir")
+
switch strings.ToLower(placeholder) {
case ":resourcedir":
return "", true, nil
case ":cachedir":
- d, err := helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+ d, err := helpers.GetCacheDir(fs, cfg)
return d, false, err
case ":project":
- return filepath.Base(p.WorkingDir), false, nil
+ return filepath.Base(workingDir), false, nil
}
return "", false, errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go
index b0f5d2dc0..f2f75344b 100644
--- a/cache/filecache/filecache_config_test.go
+++ b/cache/filecache/filecache_config_test.go
@@ -20,10 +20,9 @@ import (
"testing"
"time"
- "github.com/gohugoio/hugo/helpers"
+ "github.com/spf13/afero"
"github.com/gohugoio/hugo/config"
- "github.com/gohugoio/hugo/hugofs"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
@@ -57,22 +56,19 @@ dir = "/path/to/c3"
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
+ fs := afero.NewMemMapFs()
+ decoded, err := DecodeConfig(fs, cfg)
assert.NoError(err)
- decoded, err := decodeConfig(p)
- assert.NoError(err)
-
- assert.Equal(4, len(decoded))
+ assert.Equal(5, len(decoded))
c2 := decoded["getcsv"]
assert.Equal("11h0m0s", c2.MaxAge.String())
- assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
+ assert.Equal(filepath.FromSlash("/path/to/c2/filecache/getcsv"), c2.Dir)
c3 := decoded["images"]
assert.Equal(time.Duration(-1), c3.MaxAge)
- assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
+ assert.Equal(filepath.FromSlash("/path/to/c3/filecache/images"), c3.Dir)
}
@@ -105,14 +101,11 @@ dir = "/path/to/c3"
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
-
- decoded, err := decodeConfig(p)
+ fs := afero.NewMemMapFs()
+ decoded, err := DecodeConfig(fs, cfg)
assert.NoError(err)
- assert.Equal(4, len(decoded))
+ assert.Equal(5, len(decoded))
for _, v := range decoded {
assert.Equal(time.Duration(0), v.MaxAge)
@@ -133,24 +126,22 @@ func TestDecodeConfigDefault(t *testing.T) {
cfg.Set("cacheDir", "/cache/thecache")
}
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
+ fs := afero.NewMemMapFs()
- decoded, err := decodeConfig(p)
+ decoded, err := DecodeConfig(fs, cfg)
assert.NoError(err)
- assert.Equal(4, len(decoded))
+ assert.Equal(5, len(decoded))
imgConfig := decoded[cacheKeyImages]
jsonConfig := decoded[cacheKeyGetJSON]
if runtime.GOOS == "windows" {
- assert.Equal("_gen", imgConfig.Dir)
+ assert.Equal(filepath.FromSlash("_gen/images"), imgConfig.Dir)
} else {
- assert.Equal("_gen", imgConfig.Dir)
- assert.Equal("/cache/thecache/hugoproject", jsonConfig.Dir)
+ assert.Equal("_gen/images", imgConfig.Dir)
+ assert.Equal("/cache/thecache/hugoproject/filecache/getjson", jsonConfig.Dir)
}
assert.True(imgConfig.isResourceDir)
@@ -183,11 +174,9 @@ dir = "/"
cfg, err := config.FromConfigString(configStr, "toml")
assert.NoError(err)
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
+ fs := afero.NewMemMapFs()
- _, err = decodeConfig(p)
+ _, err = DecodeConfig(fs, cfg)
assert.Error(err)
}
diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go
index 322eabf92..c6fd4497e 100644
--- a/cache/filecache/filecache_pruner.go
+++ b/cache/filecache/filecache_pruner.go
@@ -28,53 +28,100 @@ import (
func (c Caches) Prune() (int, error) {
counter := 0
for k, cache := range c {
- err := afero.Walk(cache.Fs, "", func(name string, info os.FileInfo, err error) error {
- if info == nil {
- return nil
- }
- name = cleanID(name)
-
- if info.IsDir() {
- f, err := cache.Fs.Open(name)
- if err != nil {
- // This cache dir may not exist.
- return nil
- }
- defer f.Close()
- _, err = f.Readdirnames(1)
- if err == io.EOF {
- // Empty dir.
- return cache.Fs.Remove(name)
- }
+ count, err := cache.Prune(false)
+
+ if err != nil {
+ return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+ }
+
+ counter += count
+
+ }
+
+ return counter, nil
+}
+
+// Prune removes expired and unused items from this cache.
+// If force is set, everything will be removed not considering expiry time.
+func (c *Cache) Prune(force bool) (int, error) {
+ if c.pruneAllRootDir != "" {
+ return c.pruneRootDir(force)
+ }
+
+ counter := 0
+
+ err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error {
+ if info == nil {
+ return nil
+ }
+
+ name = cleanID(name)
+ if info.IsDir() {
+ f, err := c.Fs.Open(name)
+ if err != nil {
+ // This cache dir may not exist.
return nil
}
+ defer f.Close()
+ _, err = f.Readdirnames(1)
+ if err == io.EOF {
+ // Empty dir.
+ return c.Fs.Remove(name)
+ }
+
+ return nil
+ }
- shouldRemove := cache.isExpired(info.ModTime())
+ shouldRemove := force || c.isExpired(info.ModTime())
- if !shouldRemove && len(cache.nlocker.seen) > 0 {
- // Remove it if it's not been touched/used in the last build.
- _, seen := cache.nlocker.seen[name]
- shouldRemove = !seen
- }
+ if !shouldRemove && len(c.nlocker.seen) > 0 {
+ // Remove it if it's not been touched/used in the last build.
+ _, seen := c.nlocker.seen[name]
+ shouldRemove = !seen
+ }
- if shouldRemove {
- err := cache.Fs.Remove(name)
- if err == nil {
- counter++
- }
- return err
+ if shouldRemove {
+ err := c.Fs.Remove(name)
+ if err == nil {
+ counter++
}
+ return err
+ }
- return nil
- })
+ return nil
+ })
- if err != nil {
- return counter, errors.Wrapf(err, "failed to prune cache %q", k)
+ return counter, err
+}
+
+func (c *Cache) pruneRootDir(force bool) (int, error) {
+
+ info, err := c.Fs.Stat(c.pruneAllRootDir)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return 0, nil
}
+ return 0, err
+ }
+ if !force && !c.isExpired(info.ModTime()) {
+ return 0, nil
}
- return counter, nil
+ counter := 0
+ // Module cache has 0555 directories; make them writable in order to remove content.
+ afero.Walk(c.Fs, c.pruneAllRootDir, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return nil
+ }
+ if info.IsDir() {
+ counter++
+ c.Fs.Chmod(path, 0777)
+ }
+ return nil
+ })
+ return 1, c.Fs.RemoveAll(c.pruneAllRootDir)
+
}
diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go
index e62a6315a..72c6781ac 100644
--- a/cache/filecache/filecache_pruner_test.go
+++ b/cache/filecache/filecache_pruner_test.go
@@ -18,9 +18,7 @@ import (
"testing"
"time"
- "github.com/gohugoio/hugo/config"
- "github.com/gohugoio/hugo/helpers"
- "github.com/gohugoio/hugo/hugofs"
+ "github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
@@ -54,14 +52,9 @@ maxAge = "200ms"
dir = ":resourceDir/_gen"
`
- cfg, err := config.FromConfigString(configStr, "toml")
- assert.NoError(err)
-
for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} {
msg := fmt.Sprintf("cache: %s", name)
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
+ p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
caches, err := NewCaches(p)
assert.NoError(err)
cache := caches[name]
diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go
index 5ac2e9beb..a03c3116a 100644
--- a/cache/filecache/filecache_test.go
+++ b/cache/filecache/filecache_test.go
@@ -25,6 +25,9 @@ import (
"testing"
"time"
+ "github.com/gohugoio/hugo/langs"
+ "github.com/gohugoio/hugo/modules"
+
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
@@ -83,12 +86,7 @@ dir = ":cacheDir/c"
configStr = replacer.Replace(configStr)
configStr = strings.Replace(configStr, "\\", winPathSep, -1)
- cfg, err := config.FromConfigString(configStr, "toml")
- assert.NoError(err)
-
- fs := hugofs.NewFrom(osfs, cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
+ p := newPathsSpec(t, osfs, configStr)
caches, err := NewCaches(p)
assert.NoError(err)
@@ -207,11 +205,7 @@ dir = "/cache/c"
`
- cfg, err := config.FromConfigString(configStr, "toml")
- assert.NoError(err)
- fs := hugofs.NewMem(cfg)
- p, err := helpers.NewPathSpec(fs, cfg)
- assert.NoError(err)
+ p := newPathsSpec(t, afero.NewMemMapFs(), configStr)
caches, err := NewCaches(p)
assert.NoError(err)
@@ -255,3 +249,51 @@ func TestCleanID(t *testing.T) {
assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("/a/b//c.txt")))
assert.Equal(filepath.FromSlash("a/b/c.txt"), cleanID(filepath.FromSlash("a/b//c.txt")))
}
+
+func initConfig(fs afero.Fs, cfg config.Provider) error {
+ if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil {
+ return err
+ }
+
+ modConfig, err := modules.DecodeConfig(cfg)
+ if err != nil {
+ return err
+ }
+
+ workingDir := cfg.GetString("workingDir")
+ themesDir := cfg.GetString("themesDir")
+ if !filepath.IsAbs(themesDir) {
+ themesDir = filepath.Join(workingDir, themesDir)
+ }
+ modulesClient := modules.NewClient(modules.ClientConfig{
+ Fs: fs,
+ WorkingDir: workingDir,
+ ThemesDir: themesDir,
+ ModuleConfig: modConfig,
+ IgnoreVendor: true,
+ })
+
+ moduleConfig, err := modulesClient.Collect()
+ if err != nil {
+ return err
+ }
+
+ if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil {
+ return err
+ }
+
+ cfg.Set("allModules", moduleConfig.ActiveModules)
+
+ return nil
+}
+
+func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec {
+ assert := require.New(t)
+ cfg, err := config.FromConfigString(configStr, "toml")
+ assert.NoError(err)
+ initConfig(fs, cfg)
+ p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg)
+ assert.NoError(err)
+ return p
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-25 22:29:44 +0300