Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/gohugoio/hugo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--hugofs/glob.go85
-rw-r--r--hugofs/glob/glob.go (renamed from resources/internal/glob.go)35
-rw-r--r--hugofs/glob/glob_test.go63
-rw-r--r--hugofs/glob_test.go61
-rw-r--r--hugolib/resource_chain_test.go30
-rw-r--r--resources/resource/resources.go6
-rw-r--r--resources/resource_cache.go28
-rw-r--r--resources/resource_factories/create/create.go68
-rw-r--r--resources/resource_metadata.go4
-rw-r--r--tpl/resources/resources.go46
10 files changed, 407 insertions, 19 deletions
diff --git a/hugofs/glob.go b/hugofs/glob.go
new file mode 100644
index 000000000..e4115ea7c
--- /dev/null
+++ b/hugofs/glob.go
@@ -0,0 +1,85 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "errors"
+ "path/filepath"
+ "strings"
+
+ "github.com/gohugoio/hugo/hugofs/glob"
+
+ "github.com/spf13/afero"
+)
+
+// Glob walks the fs and passes all matches to the handle func.
+// The handle func can return true to signal a stop.
+func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error {
+ pattern = glob.NormalizePath(pattern)
+ if pattern == "" {
+ return nil
+ }
+
+ g, err := glob.GetGlob(pattern)
+ if err != nil {
+ return nil
+ }
+
+ hasSuperAsterisk := strings.Contains(pattern, "**")
+ levels := strings.Count(pattern, "/")
+ root := glob.ResolveRootDir(pattern)
+
+ // Signals that we're done.
+ done := errors.New("done")
+
+ wfn := func(p string, info FileMetaInfo, err error) error {
+ p = glob.NormalizePath(p)
+ if info.IsDir() {
+ if !hasSuperAsterisk {
+ // Avoid walking to the bottom if we can avoid it.
+ if p != "" && strings.Count(p, "/") >= levels {
+ return filepath.SkipDir
+ }
+ }
+ return nil
+ }
+
+ if g.Match(p) {
+ d, err := handle(info)
+ if err != nil {
+ return err
+ }
+ if d {
+ return done
+ }
+ }
+
+ return nil
+ }
+
+ w := NewWalkway(WalkwayConfig{
+ Root: root,
+ Fs: fs,
+ WalkFn: wfn,
+ })
+
+ err = w.Walk()
+
+ if err != done {
+ return err
+ }
+
+ return nil
+
+}
diff --git a/resources/internal/glob.go b/hugofs/glob/glob.go
index a87a23f13..18d8d44cb 100644
--- a/resources/internal/glob.go
+++ b/hugofs/glob/glob.go
@@ -11,13 +11,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package internal
+package glob
import (
+ "path"
+ "path/filepath"
"strings"
"sync"
"github.com/gobwas/glob"
+ "github.com/gobwas/glob/syntax"
)
var (
@@ -46,3 +49,33 @@ func GetGlob(pattern string) (glob.Glob, error) {
return g, nil
}
+
+func NormalizePath(p string) string {
+ return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.")
+}
+
+// ResolveRootDir takes a normalized path on the form "assets/**.json" and
+// determines any root dir, i.e. any start path without any wildcards.
+func ResolveRootDir(p string) string {
+ parts := strings.Split(path.Dir(p), "/")
+ var roots []string
+ for _, part := range parts {
+ isSpecial := false
+ for i := 0; i < len(part); i++ {
+ if syntax.Special(part[i]) {
+ isSpecial = true
+ break
+ }
+ }
+ if isSpecial {
+ break
+ }
+ roots = append(roots, part)
+ }
+
+ if len(roots) == 0 {
+ return ""
+ }
+
+ return strings.Join(roots, "/")
+}
diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go
new file mode 100644
index 000000000..2b1c74107
--- /dev/null
+++ b/hugofs/glob/glob_test.go
@@ -0,0 +1,63 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+ "path/filepath"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestResolveRootDir(t *testing.T) {
+ c := qt.New(t)
+
+ for _, test := range []struct {
+ in string
+ expect string
+ }{
+ {"data/foo.json", "data"},
+ {"a/b/**/foo.json", "a/b"},
+ {"dat?a/foo.json", ""},
+ {"a/b[a-c]/foo.json", "a"},
+ } {
+
+ c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect)
+ }
+}
+
+func TestNormalizePath(t *testing.T) {
+ c := qt.New(t)
+
+ for _, test := range []struct {
+ in string
+ expect string
+ }{
+ {filepath.FromSlash("data/FOO.json"), "data/foo.json"},
+ {filepath.FromSlash("/data/FOO.json"), "data/foo.json"},
+ {filepath.FromSlash("./FOO.json"), "foo.json"},
+ {"//", ""},
+ } {
+
+ c.Assert(NormalizePath(test.in), qt.Equals, test.expect)
+ }
+}
+
+func TestGetGlob(t *testing.T) {
+ c := qt.New(t)
+ g, err := GetGlob("**.JSON")
+ c.Assert(err, qt.IsNil)
+ c.Assert(g.Match("data/my.json"), qt.Equals, true)
+
+}
diff --git a/hugofs/glob_test.go b/hugofs/glob_test.go
new file mode 100644
index 000000000..3c7780685
--- /dev/null
+++ b/hugofs/glob_test.go
@@ -0,0 +1,61 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/spf13/afero"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestGlob(t *testing.T) {
+ c := qt.New(t)
+
+ fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+ create := func(filename string) {
+ err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777)
+ c.Assert(err, qt.IsNil)
+ }
+
+ collect := func(pattern string) []string {
+ var paths []string
+ h := func(fi FileMetaInfo) (bool, error) {
+ paths = append(paths, fi.Meta().Path())
+ return false, nil
+ }
+ err := Glob(fs, pattern, h)
+ c.Assert(err, qt.IsNil)
+ return paths
+ }
+
+ create("root.json")
+ create("jsonfiles/d1.json")
+ create("jsonfiles/d2.json")
+ create("jsonfiles/sub/d3.json")
+ create("jsonfiles/d1.xml")
+ create("a/b/c/e/f.json")
+
+ c.Assert(collect("**.json"), qt.HasLen, 5)
+ c.Assert(collect("**"), qt.HasLen, 6)
+ c.Assert(collect(""), qt.HasLen, 0)
+ c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2)
+ c.Assert(collect("*.json"), qt.HasLen, 1)
+ c.Assert(collect("**.xml"), qt.HasLen, 1)
+ c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2)
+
+}
diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go
index 75f5595e1..84c871e4d 100644
--- a/hugolib/resource_chain_test.go
+++ b/hugolib/resource_chain_test.go
@@ -502,3 +502,33 @@ func TestMultiSiteResource(t *testing.T) {
b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes")
}
+
+func TestResourcesMatch(t *testing.T) {
+ t.Parallel()
+
+ b := newTestSitesBuilder(t)
+
+ b.WithContent("page.md", "")
+
+ b.WithSourceFile(
+ "assets/jsons/data1.json", "json1 content",
+ "assets/jsons/data2.json", "json2 content",
+ "assets/jsons/data3.xml", "xml content",
+ )
+
+ b.WithTemplates("index.html", `
+{{ $jsons := (resources.Match "jsons/*.json") }}
+{{ $json := (resources.GetMatch "jsons/*.json") }}
+{{ printf "JSONS: %d" (len $jsons) }}
+JSON: {{ $json.RelPermalink }}: {{ $json.Content }}
+{{ range $jsons }}
+{{- .RelPermalink }}: {{ .Content }}
+{{ end }}
+`)
+
+ b.Build(BuildCfg{})
+
+ b.AssertFileContent("public/index.html",
+ "JSON: /jsons/data1.json: json1 content",
+ "JSONS: 2", "/jsons/data1.json: json1 content")
+}
diff --git a/resources/resource/resources.go b/resources/resource/resources.go
index 5c661c24e..ac5dd0b2b 100644
--- a/resources/resource/resources.go
+++ b/resources/resource/resources.go
@@ -17,7 +17,7 @@ import (
"fmt"
"strings"
- "github.com/gohugoio/hugo/resources/internal"
+ "github.com/gohugoio/hugo/hugofs/glob"
)
// Resources represents a slice of resources, which can be a mix of different types.
@@ -44,7 +44,7 @@ func (r Resources) ByType(tp string) Resources {
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
// See Match for a more complete explanation about the rules used.
func (r Resources) GetMatch(pattern string) Resource {
- g, err := internal.GetGlob(pattern)
+ g, err := glob.GetGlob(pattern)
if err != nil {
return nil
}
@@ -68,7 +68,7 @@ func (r Resources) GetMatch(pattern string) Resource {
// path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
// See https://github.com/gobwas/glob for the full rules set.
func (r Resources) Match(pattern string) Resources {
- g, err := internal.GetGlob(pattern)
+ g, err := glob.GetGlob(pattern)
if err != nil {
return nil
}
diff --git a/resources/resource_cache.go b/resources/resource_cache.go
index 8ff63beb0..656d4f826 100644
--- a/resources/resource_cache.go
+++ b/resources/resource_cache.go
@@ -37,7 +37,9 @@ type ResourceCache struct {
rs *Spec
sync.RWMutex
- cache map[string]resource.Resource
+
+ // Either resource.Resource or resource.Resources.
+ cache map[string]interface{}
fileCache *filecache.Cache
@@ -61,7 +63,7 @@ func newResourceCache(rs *Spec) *ResourceCache {
return &ResourceCache{
rs: rs,
fileCache: rs.FileCaches.AssetsCache(),
- cache: make(map[string]resource.Resource),
+ cache: make(map[string]interface{}),
nlocker: locker.NewLocker(),
}
}
@@ -70,7 +72,7 @@ func (c *ResourceCache) clear() {
c.Lock()
defer c.Unlock()
- c.cache = make(map[string]resource.Resource)
+ c.cache = make(map[string]interface{})
c.nlocker = locker.NewLocker()
}
@@ -84,7 +86,7 @@ func (c *ResourceCache) cleanKey(key string) string {
return strings.TrimPrefix(path.Clean(key), "/")
}
-func (c *ResourceCache) get(key string) (resource.Resource, bool) {
+func (c *ResourceCache) get(key string) (interface{}, bool) {
c.RLock()
defer c.RUnlock()
r, found := c.cache[key]
@@ -92,6 +94,22 @@ func (c *ResourceCache) get(key string) (resource.Resource, bool) {
}
func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) {
+ r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
+ if r == nil || err != nil {
+ return nil, err
+ }
+ return r.(resource.Resource), nil
+}
+
+func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) {
+ r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
+ if r == nil || err != nil {
+ return nil, err
+ }
+ return r.(resource.Resources), nil
+}
+
+func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) {
key = c.cleanKey(path.Join(partition, key))
// First check in-memory cache.
r, found := c.get(key)
@@ -174,7 +192,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata)
}
-func (c *ResourceCache) set(key string, r resource.Resource) {
+func (c *ResourceCache) set(key string, r interface{}) {
c.Lock()
defer c.Unlock()
c.cache[key] = r
diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
index 36a29e733..e42843c75 100644
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -16,9 +16,12 @@
package create
import (
+ "path"
"path/filepath"
- "github.com/spf13/afero"
+ "github.com/gohugoio/hugo/hugofs/glob"
+
+ "github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/resources"
@@ -36,18 +39,75 @@ func New(rs *resources.Spec) *Client {
return &Client{rs: rs}
}
-// Get creates a new Resource by opening the given filename in the given filesystem.
-func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
+// Get creates a new Resource by opening the given filename in the assets filesystem.
+func (c *Client) Get(filename string) (resource.Resource, error) {
filename = filepath.Clean(filename)
return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
return c.rs.New(resources.ResourceSourceDescriptor{
- Fs: fs,
+ Fs: c.rs.BaseFs.Assets.Fs,
LazyPublish: true,
SourceFilename: filename})
})
}
+// Match gets the resources matching the given pattern from the assets filesystem.
+func (c *Client) Match(pattern string) (resource.Resources, error) {
+ return c.match(pattern, false)
+}
+
+// GetMatch gets first resource matching the given pattern from the assets filesystem.
+func (c *Client) GetMatch(pattern string) (resource.Resource, error) {
+ res, err := c.match(pattern, true)
+ if err != nil || len(res) == 0 {
+ return nil, err
+ }
+ return res[0], err
+}
+
+func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) {
+ var partition string
+ if firstOnly {
+ partition = "__get-match"
+ } else {
+ partition = "__match"
+ }
+
+ // TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199
+ partition = path.Join(resources.CACHE_OTHER, partition)
+ key := glob.NormalizePath(pattern)
+
+ return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) {
+ var res resource.Resources
+
+ handle := func(info hugofs.FileMetaInfo) (bool, error) {
+ meta := info.Meta()
+ r, err := c.rs.New(resources.ResourceSourceDescriptor{
+ LazyPublish: true,
+ OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+ return meta.Open()
+ },
+ RelTargetFilename: meta.Path()})
+
+ if err != nil {
+ return true, err
+ }
+
+ res = append(res, r)
+
+ return firstOnly, nil
+
+ }
+
+ if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil {
+ return nil, err
+ }
+
+ return res, nil
+
+ })
+}
+
// FromString creates a new Resource from a string with the given relative target path.
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go
index e019133d7..adb9d6867 100644
--- a/resources/resource_metadata.go
+++ b/resources/resource_metadata.go
@@ -17,7 +17,7 @@ import (
"fmt"
"strconv"
- "github.com/gohugoio/hugo/resources/internal"
+ "github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/resources/resource"
"github.com/pkg/errors"
@@ -70,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
srcKey := strings.ToLower(cast.ToString(src))
- glob, err := internal.GetGlob(srcKey)
+ glob, err := glob.GetGlob(srcKey)
if err != nil {
return errors.Wrap(err, "failed to match resource with metadata")
}
diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go
index d32e12a05..3d688e21c 100644
--- a/tpl/resources/resources.go
+++ b/tpl/resources/resources.go
@@ -68,7 +68,7 @@ type Namespace struct {
templatesClient *templates.Client
}
-// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order)
+// Get locates the filename given in Hugo's assets filesystem
// and creates a Resource object that can be used for further transformations.
func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
filenamestr, err := cast.ToStringE(filename)
@@ -78,12 +78,50 @@ func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
filenamestr = filepath.Clean(filenamestr)
- // Resource Get'ing is currently limited to /assets to make it simpler
- // to control the behaviour of publishing and partial rebuilding.
- return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr)
+ return ns.createClient.Get(filenamestr)
}
+// GetMatch finds the first Resource matching the given pattern, or nil if none found.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) {
+ patternStr, err := cast.ToStringE(pattern)
+ if err != nil {
+ return nil, err
+ }
+
+ return ns.createClient.GetMatch(patternStr)
+
+}
+
+// Match gets all resources matching the given base path prefix, e.g
+// "*.png" will match all png files. The "*" does not match path delimiters (/),
+// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.:
+// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and
+// to match all PNG images below the images folder, use "images/**.jpg".
+//
+// The matching is case insensitive.
+//
+// Match matches by using the files name with path relative to the file system root
+// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
+//
+// See https://github.com/gobwas/glob for the full rules set.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) {
+ patternStr, err := cast.ToStringE(pattern)
+ if err != nil {
+ return nil, err
+ }
+
+ return ns.createClient.Match(patternStr)
+}
+
// Concat concatenates a slice of Resource objects. These resources must
// (currently) be of the same Media Type.
func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.Resource, error) {
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-25 14:38:46 +0300