Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/gohugoio/hugo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2018-02-20 12:02:14 +0300
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2018-07-06 12:46:12 +0300
commitdea71670c059ab4d5a42bd22503f18c087dd22d4 (patch)
tree52889fd27a2d316fad5a04c0f2fe2198491c6cd1 /resource
parenta5d0a57e6bdab583134a68c035aac9b3007f006a (diff)
Add Hugo Piper with SCSS support and much more
Before this commit, you would have to use page bundles to do image processing etc. in Hugo. This commit adds * A new `/assets` top-level project or theme dir (configurable via `assetDir`) * A new template func, `resources.Get` which can be used to "get a resource" that can be further processed. This means that you can now do this in your templates (or shortcodes): ```bash {{ $sunset := (resources.Get "images/sunset.jpg").Fill "300x200" }} ``` This also adds a new `extended` build tag that enables powerful SCSS/SASS support with source maps. To compile this from source, you will also need a C compiler installed: ``` HUGO_BUILD_TAGS=extended mage install ``` Note that you can use output of the SCSS processing later in a non-SCSSS-enabled Hugo. The `SCSS` processor is a _Resource transformation step_ and it can be chained with the many others in a pipeline: ```bash {{ $css := resources.Get "styles.scss" | resources.ToCSS | resources.PostCSS | resources.Minify | resources.Fingerprint }} <link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen"> ``` The transformation funcs above have aliases, so it can be shortened to: ```bash {{ $css := resources.Get "styles.scss" | toCSS | postCSS | minify | fingerprint }} <link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen"> ``` A quick tip would be to avoid the fingerprinting part, and possibly also the not-superfast `postCSS` when you're doing development, as it allows Hugo to be smarter about the rebuilding. Documentation will follow, but have a look at the demo repo in https://github.com/bep/hugo-sass-test New functions to create `Resource` objects: * `resources.Get` (see above) * `resources.FromString`: Create a Resource from a string. New `Resource` transformation funcs: * `resources.ToCSS`: Compile `SCSS` or `SASS` into `CSS`. * `resources.PostCSS`: Process your CSS with PostCSS. Config file support (project or theme or passed as an option). * `resources.Minify`: Currently supports `css`, `js`, `json`, `html`, `svg`, `xml`. * `resources.Fingerprint`: Creates a fingerprinted version of the given Resource with Subresource Integrity.. * `resources.Concat`: Concatenates a list of Resource objects. Think of this as a poor man's bundler. * `resources.ExecuteAsTemplate`: Parses and executes the given Resource and data context (e.g. .Site) as a Go template. Fixes #4381 Fixes #4903 Fixes #4858
Diffstat (limited to 'resource')
-rw-r--r--resource/bundler/bundler.go121
-rw-r--r--resource/create/create.go77
-rw-r--r--resource/image.go70
-rw-r--r--resource/image_cache.go16
-rw-r--r--resource/image_test.go18
-rw-r--r--resource/integrity/integrity.go106
-rw-r--r--resource/integrity/integrity_test.go54
-rw-r--r--resource/minifiers/minify.go115
-rw-r--r--resource/postcss/postcss.go175
-rw-r--r--resource/resource.go691
-rw-r--r--resource/resource_cache.go241
-rw-r--r--resource/resource_metadata.go129
-rw-r--r--resource/resource_metadata_test.go230
-rw-r--r--resource/resource_test.go312
-rw-r--r--resource/templates/execute_as_template.go76
-rw-r--r--resource/testhelpers_test.go17
-rw-r--r--resource/tocss/scss/client.go101
-rw-r--r--resource/tocss/scss/tocss.go111
-rw-r--r--resource/tocss/scss/tocss_notavailable.go30
-rw-r--r--resource/transform.go487
-rw-r--r--resource/transform_test.go36
21 files changed, 2618 insertions, 595 deletions
diff --git a/resource/bundler/bundler.go b/resource/bundler/bundler.go
new file mode 100644
index 000000000..2f3981485
--- /dev/null
+++ b/resource/bundler/bundler.go
@@ -0,0 +1,121 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package bundler contains functions for concatenation etc. of Resource objects.
+package bundler
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "path/filepath"
+
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resource"
+)
+
+// Client contains methods perform concatenation and other bundling related
+// tasks to Resource objects.
+type Client struct {
+ rs *resource.Spec
+}
+
+// New creates a new Client with the given specification.
+func New(rs *resource.Spec) *Client {
+ return &Client{rs: rs}
+}
+
+type multiReadSeekCloser struct {
+ mr io.Reader
+ sources []resource.ReadSeekCloser
+}
+
+func (r *multiReadSeekCloser) Read(p []byte) (n int, err error) {
+ return r.mr.Read(p)
+}
+
+func (r *multiReadSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) {
+ for _, s := range r.sources {
+ newOffset, err = s.Seek(offset, whence)
+ if err != nil {
+ return
+ }
+ }
+ return
+}
+
+func (r *multiReadSeekCloser) Close() error {
+ for _, s := range r.sources {
+ s.Close()
+ }
+ return nil
+}
+
+// Concat concatenates the list of Resource objects.
+func (c *Client) Concat(targetPath string, resources []resource.Resource) (resource.Resource, error) {
+ // The CACHE_OTHER will make sure this will be re-created and published on rebuilds.
+ return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
+ var resolvedm media.Type
+
+ // The given set of resources must be of the same Media Type.
+ // We may improve on that in the future, but then we need to know more.
+ for i, r := range resources {
+ if i > 0 && r.MediaType() != resolvedm {
+ return nil, errors.New("resources in Concat must be of the same Media Type")
+ }
+ resolvedm = r.MediaType()
+ }
+
+ concatr := func() (resource.ReadSeekCloser, error) {
+ var rcsources []resource.ReadSeekCloser
+ for _, s := range resources {
+ rcr, ok := s.(resource.ReadSeekCloserResource)
+ if !ok {
+ return nil, fmt.Errorf("resource %T does not implement resource.ReadSeekerCloserResource", s)
+ }
+ rc, err := rcr.ReadSeekCloser()
+ if err != nil {
+ // Close the already opened.
+ for _, rcs := range rcsources {
+ rcs.Close()
+ }
+ return nil, err
+ }
+ rcsources = append(rcsources, rc)
+ }
+
+ readers := make([]io.Reader, len(rcsources))
+ for i := 0; i < len(rcsources); i++ {
+ readers[i] = rcsources[i]
+ }
+
+ mr := io.MultiReader(readers...)
+
+ return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil
+ }
+
+ composite, err := c.rs.NewForFs(
+ c.rs.BaseFs.Resources.Fs,
+ resource.ResourceSourceDescriptor{
+ LazyPublish: true,
+ OpenReadSeekCloser: concatr,
+ RelTargetFilename: filepath.Clean(targetPath)})
+
+ if err != nil {
+ return nil, err
+ }
+
+ return composite, nil
+ })
+
+}
diff --git a/resource/create/create.go b/resource/create/create.go
new file mode 100644
index 000000000..1c7894232
--- /dev/null
+++ b/resource/create/create.go
@@ -0,0 +1,77 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package create contains functions for to create Resource objects. This will
+// typically non-files.
+package create
+
+import (
+ "io"
+ "path/filepath"
+
+ "github.com/spf13/afero"
+
+ "github.com/dsnet/golib/memfile"
+ "github.com/gohugoio/hugo/resource"
+)
+
+// Client contains methods to create Resource objects.
+// tasks to Resource objects.
+type Client struct {
+ rs *resource.Spec
+}
+
+// New creates a new Client with the given specification.
+func New(rs *resource.Spec) *Client {
+ return &Client{rs: rs}
+}
+
+type memFileCloser struct {
+ *memfile.File
+ io.Closer
+}
+
+func (m *memFileCloser) Close() error {
+ return nil
+}
+
+// Get creates a new Resource by opening the given filename in the given filesystem.
+func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
+ filename = filepath.Clean(filename)
+ return c.rs.ResourceCache.GetOrCreate(resource.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
+ return c.rs.NewForFs(fs,
+ resource.ResourceSourceDescriptor{
+ LazyPublish: true,
+ SourceFilename: filename})
+
+ })
+
+}
+
+// FromString creates a new Resource from a string with the given relative target path.
+func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
+ return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
+ return c.rs.NewForFs(
+ c.rs.BaseFs.Resources.Fs,
+ resource.ResourceSourceDescriptor{
+ LazyPublish: true,
+ OpenReadSeekCloser: func() (resource.ReadSeekCloser, error) {
+ return &memFileCloser{
+ File: memfile.New([]byte(content)),
+ }, nil
+ },
+ RelTargetFilename: filepath.Clean(targetPath)})
+
+ })
+
+}
diff --git a/resource/image.go b/resource/image.go
index 19b68a296..6aa382331 100644
--- a/resource/image.go
+++ b/resource/image.go
@@ -19,14 +19,12 @@ import (
"image/color"
"io"
"os"
- "path/filepath"
"strconv"
"strings"
"github.com/mitchellh/mapstructure"
"github.com/gohugoio/hugo/helpers"
- "github.com/spf13/afero"
// Importing image codecs for image.DecodeConfig
"image"
@@ -132,8 +130,6 @@ type Image struct {
format imaging.Format
- hash string
-
*genericResource
}
@@ -151,7 +147,6 @@ func (i *Image) Height() int {
func (i *Image) WithNewBase(base string) Resource {
return &Image{
imaging: i.imaging,
- hash: i.hash,
format: i.format,
genericResource: i.genericResource.WithNewBase(base).(*genericResource)}
}
@@ -209,7 +204,7 @@ type imageConfig struct {
}
func (i *Image) isJPEG() bool {
- name := strings.ToLower(i.relTargetPath.file)
+ name := strings.ToLower(i.relTargetDirFile.file)
return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg")
}
@@ -241,7 +236,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c
ci := i.clone()
errOp := action
- errPath := i.AbsSourceFilename()
+ errPath := i.sourceFilename
ci.setBasePath(conf)
@@ -273,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c
ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y}
ci.configLoaded = true
- return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.target())
+ return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilename())
})
}
@@ -415,11 +410,11 @@ func (i *Image) initConfig() error {
}
var (
- f afero.File
+ f ReadSeekCloser
config image.Config
)
- f, err = i.sourceFs().Open(i.AbsSourceFilename())
+ f, err = i.ReadSeekCloser()
if err != nil {
return
}
@@ -440,19 +435,19 @@ func (i *Image) initConfig() error {
}
func (i *Image) decodeSource() (image.Image, error) {
- file, err := i.sourceFs().Open(i.AbsSourceFilename())
+ f, err := i.ReadSeekCloser()
if err != nil {
return nil, fmt.Errorf("failed to open image for decode: %s", err)
}
- defer file.Close()
- img, _, err := image.Decode(file)
+ defer f.Close()
+ img, _, err := image.Decode(f)
return img, err
}
func (i *Image) copyToDestination(src string) error {
var res error
i.copyToDestinationInit.Do(func() {
- target := i.target()
+ target := i.targetFilename()
// Fast path:
// This is a processed version of the original.
@@ -469,20 +464,9 @@ func (i *Image) copyToDestination(src string) error {
}
defer in.Close()
- out, err := i.spec.BaseFs.PublishFs.Create(target)
- if err != nil && os.IsNotExist(err) {
- // When called from shortcodes, the target directory may not exist yet.
- // See https://github.com/gohugoio/hugo/issues/4202
- if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
- res = err
- return
- }
- out, err = i.spec.BaseFs.PublishFs.Create(target)
- if err != nil {
- res = err
- return
- }
- } else if err != nil {
+ out, err := openFileForWriting(i.spec.BaseFs.PublishFs, target)
+
+ if err != nil {
res = err
return
}
@@ -501,21 +485,10 @@ func (i *Image) copyToDestination(src string) error {
return nil
}
-func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error {
- target := filepath.Clean(filename)
+func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, targetFilename string) error {
- file1, err := i.spec.BaseFs.PublishFs.Create(target)
- if err != nil && os.IsNotExist(err) {
- // When called from shortcodes, the target directory may not exist yet.
- // See https://github.com/gohugoio/hugo/issues/4202
- if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
- return err
- }
- file1, err = i.spec.BaseFs.PublishFs.Create(target)
- if err != nil {
- return err
- }
- } else if err != nil {
+ file1, err := openFileForWriting(i.spec.BaseFs.PublishFs, targetFilename)
+ if err != nil {
return err
}
@@ -525,11 +498,7 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource
if resourceCacheFilename != "" {
// Also save it to the image resource cache for later reuse.
- if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil {
- return err
- }
-
- file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename)
+ file2, err := openFileForWriting(i.spec.BaseFs.Resources.Fs, resourceCacheFilename)
if err != nil {
return err
}
@@ -572,17 +541,16 @@ func (i *Image) clone() *Image {
return &Image{
imaging: i.imaging,
- hash: i.hash,
format: i.format,
genericResource: &g}
}
func (i *Image) setBasePath(conf imageConfig) {
- i.relTargetPath = i.relTargetPathFromConfig(conf)
+ i.relTargetDirFile = i.relTargetPathFromConfig(conf)
}
func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile {
- p1, p2 := helpers.FileAndExt(i.relTargetPath.file)
+ p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file)
idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size())
@@ -611,7 +579,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile {
}
return dirFile{
- dir: i.relTargetPath.dir,
+ dir: i.relTargetDirFile.dir,
file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2),
}
diff --git a/resource/image_cache.go b/resource/image_cache.go
index 5985797d6..4fb45c17f 100644
--- a/resource/image_cache.go
+++ b/resource/image_cache.go
@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -60,12 +60,6 @@ func (c *imageCache) getOrCreate(
relTarget := parent.relTargetPathFromConfig(conf)
key := parent.relTargetPathForRel(relTarget.path(), false)
- if c.pathSpec.Language != nil {
- // Avoid do and store more work than needed. The language versions will in
- // most cases be duplicates of the same image files.
- key = strings.TrimPrefix(key, "/"+c.pathSpec.Language.Lang)
- }
-
// First check the in-memory store, then the disk.
c.mu.RLock()
img, found := c.store[key]
@@ -88,17 +82,17 @@ func (c *imageCache) getOrCreate(
// but the count of processed image variations for this site.
c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages)
- exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs)
+ exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.Resources.Fs)
if err != nil {
return nil, err
}
if exists {
img = parent.clone()
- img.relTargetPath.file = relTarget.file
+ img.relTargetDirFile.file = relTarget.file
img.sourceFilename = cacheFilename
- // We have to look resources file system for this.
- img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs
+ // We have to look in the resources file system for this.
+ img.overriddenSourceFs = img.spec.BaseFs.Resources.Fs
} else {
img, err = create(cacheFilename)
if err != nil {
diff --git a/resource/image_test.go b/resource/image_test.go
index 11807d695..f4d91bd99 100644
--- a/resource/image_test.go
+++ b/resource/image_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -78,19 +78,19 @@ func TestImageTransformBasic(t *testing.T) {
assert.NoError(err)
assert.Equal(320, resized0x.Width())
assert.Equal(200, resized0x.Height())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized0x.RelPermalink(), 320, 200)
resizedx0, err := image.Resize("200x")
assert.NoError(err)
assert.Equal(200, resizedx0.Width())
assert.Equal(125, resizedx0.Height())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedx0.RelPermalink(), 200, 125)
resizedAndRotated, err := image.Resize("x200 r90")
assert.NoError(err)
assert.Equal(125, resizedAndRotated.Width())
assert.Equal(200, resizedAndRotated.Height())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAndRotated.RelPermalink(), 125, 200)
assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink())
assert.Equal(300, resized.Width())
@@ -115,20 +115,20 @@ func TestImageTransformBasic(t *testing.T) {
assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink())
assert.Equal(200, filled.Width())
assert.Equal(100, filled.Height())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filled.RelPermalink(), 200, 100)
smart, err := image.Fill("200x100 smart")
assert.NoError(err)
assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink())
assert.Equal(200, smart.Width())
assert.Equal(100, smart.Height())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, smart.RelPermalink(), 200, 100)
// Check cache
filledAgain, err := image.Fill("200x100 bottomLeft")
assert.NoError(err)
assert.True(filled == filledAgain)
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filledAgain.RelPermalink(), 200, 100)
}
@@ -298,7 +298,7 @@ func TestImageResizeInSubPath(t *testing.T) {
assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink())
assert.Equal(101, resized.Width())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized.RelPermalink(), 101, 101)
publishedImageFilename := filepath.Clean(resized.RelPermalink())
assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename))
@@ -310,7 +310,7 @@ func TestImageResizeInSubPath(t *testing.T) {
assert.NoError(err)
assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink())
assert.Equal(101, resizedAgain.Width())
- assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101)
+ assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAgain.RelPermalink(), 101, 101)
assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
}
diff --git a/resource/integrity/integrity.go b/resource/integrity/integrity.go
new file mode 100644
index 000000000..8b4a5a263
--- /dev/null
+++ b/resource/integrity/integrity.go
@@ -0,0 +1,106 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integrity
+
+import (
+ "crypto/md5"
+ "crypto/sha256"
+ "crypto/sha512"
+ "encoding/base64"
+ "encoding/hex"
+ "fmt"
+ "hash"
+ "io"
+
+ "github.com/gohugoio/hugo/resource"
+)
+
+const defaultHashAlgo = "sha256"
+
+// Client contains methods to fingerprint (cachebusting) and other integrity-related
+// methods.
+type Client struct {
+ rs *resource.Spec
+}
+
+// New creates a new Client with the given specification.
+func New(rs *resource.Spec) *Client {
+ return &Client{rs: rs}
+}
+
+type fingerprintTransformation struct {
+ algo string
+}
+
+func (t *fingerprintTransformation) Key() resource.ResourceTransformationKey {
+ return resource.NewResourceTransformationKey("fingerprint", t.algo)
+}
+
+// Transform creates a MD5 hash of the Resource content and inserts that hash before
+// the extension in the filename.
+func (t *fingerprintTransformation) Transform(ctx *resource.ResourceTransformationCtx) error {
+ algo := t.algo
+
+ var h hash.Hash
+
+ switch algo {
+ case "md5":
+ h = md5.New()
+ case "sha256":
+ h = sha256.New()
+ case "sha512":
+ h = sha512.New()
+ default:
+ return fmt.Errorf("unsupported crypto algo: %q, use either md5, sha256 or sha512", algo)
+ }
+
+ io.Copy(io.MultiWriter(h, ctx.To), ctx.From)
+ d, err := digest(h)
+ if err != nil {
+ return err
+ }
+
+ ctx.Data["Integrity"] = integrity(algo, d)
+ ctx.AddOutPathIdentifier("." + hex.EncodeToString(d[:]))
+ return nil
+}
+
+// Fingerprint applies fingerprinting of the given resource and hash algorithm.
+// It defaults to sha256 if none given, and the options are md5, sha256 or sha512.
+// The same algo is used for both the fingerprinting part (aka cache busting) and
+// the base64-encoded Subresource Integrity hash, so you will have to stay away from
+// md5 if you plan to use both.
+// See https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity
+func (c *Client) Fingerprint(res resource.Resource, algo string) (resource.Resource, error) {
+ if algo == "" {
+ algo = defaultHashAlgo
+ }
+
+ return c.rs.Transform(
+ res,
+ &fingerprintTransformation{algo: algo},
+ )
+}
+
+func integrity(algo string, sum []byte) string {
+ encoded := base64.StdEncoding.EncodeToString(sum)
+ return fmt.Sprintf("%s-%s", algo, encoded)
+
+}
+
+func digest(h hash.Hash) ([]byte, error) {
+ sum := h.Sum(nil)
+ //enc := hex.EncodeToString(sum[:])
+ return sum, nil
+}
diff --git a/resource/integrity/integrity_test.go b/resource/integrity/integrity_test.go
new file mode 100644
index 000000000..602db4e38
--- /dev/null
+++ b/resource/integrity/integrity_test.go
@@ -0,0 +1,54 @@
+// Copyright 2018-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integrity
+
+import (
+ "github.com/gohugoio/hugo/media"
+)
+
+type testResource struct {
+ content string
+}
+
+func (r testResource) Permalink() string {
+ panic("not implemented")
+}
+
+func (r testResource) RelPermalink() string {
+ panic("not implemented")
+}
+
+func (r testResource) ResourceType() string {
+ panic("not implemented")
+}
+
+func (r testResource) Name() string {
+ panic("not implemented")
+}
+
+func (r testResource) MediaType() media.Type {
+ panic("not implemented")
+}
+
+func (r testResource) Title() string {
+ panic("not implemented")
+}
+
+func (r testResource) Params() map[string]interface{} {
+ panic("not implemented")
+}
+
+func (r testResource) Bytes() ([]byte, error) {
+ return []byte(r.content), nil
+}
diff --git a/resource/minifiers/minify.go b/resource/minifiers/minify.go
new file mode 100644
index 000000000..609b9a694
--- /dev/null
+++ b/resource/minifiers/minify.go
@@ -0,0 +1,115 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package minifiers
+
+import (
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/media"
+
+ "github.com/gohugoio/hugo/resource"
+ "github.com/tdewolff/minify"
+ "github.com/tdewolff/minify/css"
+ "github.com/tdewolff/minify/html"
+ "github.com/tdewolff/minify/js"
+ "github.com/tdewolff/minify/json"
+ "github.com/tdewolff/minify/svg"
+ "github.com/tdewolff/minify/xml"
+)
+
+// Client for minification of Resource objects. Supported minfiers are:
+// css, html, js, json, svg and xml.
+type Client struct {
+ rs *resource.Spec
+ m *minify.M
+}
+
+// New creates a new Client given a specification. Note that it is the media types
+// configured for the site that is used to match files to the correct minifier.
+func New(rs *resource.Spec) *Client {
+ m := minify.New()
+ mt := rs.MediaTypes
+
+ // We use the Type definition of the media types defined in the site if found.
+ addMinifierFunc(m, mt, "text/css", "css", css.Minify)
+ addMinifierFunc(m, mt, "text/html", "html", html.Minify)
+ addMinifierFunc(m, mt, "application/javascript", "js", js.Minify)
+ addMinifierFunc(m, mt, "application/json", "json", json.Minify)
+ addMinifierFunc(m, mt, "image/svg", "xml", svg.Minify)
+ addMinifierFunc(m, mt, "application/xml", "xml", xml.Minify)
+ addMinifierFunc(m, mt, "application/rss", "xml", xml.Minify)
+
+ return &Client{rs: rs, m: m}
+}
+
+func addMinifierFunc(m *minify.M, mt media.Types, typeString, suffix string, fn minify.MinifierFunc) {
+ resolvedTypeStr := resolveMediaTypeString(mt, typeString, suffix)
+ m.AddFunc(resolvedTypeStr, fn)
+ if resolvedTypeStr != typeString {
+ m.AddFunc(typeString, fn)
+ }
+}
+
+type minifyTransformation struct {
+ rs *resource.Spec
+ m *minify.M
+}
+
+func (t *minifyTransformation) Key() resource.ResourceTransformationKey {
+ return resource.NewResourceTransformationKey("minify")
+}
+
+func (t *minifyTransformation) Transform(ctx *resource.ResourceTransformationCtx) error {
+ mtype := resolveMediaTypeString(
+ t.rs.MediaTypes,
+ ctx.InMediaType.Type(),
+ helpers.ExtNoDelimiter(ctx.InPath),
+ )
+ if err := t.m.Minify(mtype, ctx.To, ctx.From); err != nil {
+ return err
+ }
+ ctx.AddOutPathIdentifier(".min")
+ return nil
+}
+
+func (c *Client) Minify(res resource.Resource) (resource.Resource, error) {
+ return c.rs.Transform(
+ res,
+ &minifyTransformation{
+ rs: c.rs,
+ m: c.m},
+ )
+}
+
+func resolveMediaTypeString(types media.Types, typeStr, suffix string) string {
+ if m, found := resolveMediaType(types, typeStr, suffix); found {
+ return m.Type()
+ }
+ // Fall back to the default.
+ return typeStr
+}
+
+// Make sure we match the matching pattern with what the user have actually defined
+// in his or hers media types configuration.
+func resolveMediaType(types media.Types, typeStr, suffix string) (media.Type, bool) {
+ if m, found := types.GetByType(typeStr); found {
+ return m, true
+ }
+
+ if m, found := types.GetFirstBySuffix(suffix); found {
+ return m, true
+ }
+
+ return media.Type{}, false
+
+}
diff --git a/resource/postcss/postcss.go b/resource/postcss/postcss.go
new file mode 100644
index 000000000..7dd27b2f9
--- /dev/null
+++ b/resource/postcss/postcss.go
@@ -0,0 +1,175 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package postcss
+
+import (
+ "fmt"
+ "io"
+ "path/filepath"
+
+ "github.com/gohugoio/hugo/hugofs"
+
+ "github.com/mitchellh/mapstructure"
+ // "io/ioutil"
+ "os"
+ "os/exec"
+
+ "github.com/gohugoio/hugo/common/errors"
+
+ "github.com/gohugoio/hugo/resource"
+)
+
+// Some of the options from https://github.com/postcss/postcss-cli
+type Options struct {
+
+ // Set a custom path to look for a config file.
+ Config string
+
+ NoMap bool `mapstructure:"no-map"` // Disable the default inline sourcemaps
+
+ // Options for when not using a config file
+ Use string // List of postcss plugins to use
+ Parser string // Custom postcss parser
+ Stringifier string // Custom postcss stringifier
+ Syntax string // Custom postcss syntax
+}
+
+func DecodeOptions(m map[string]interface{}) (opts Options, err error) {
+ if m == nil {
+ return
+ }
+ err = mapstructure.WeakDecode(m, &opts)
+ return
+}
+
+func (opts Options) toArgs() []string {
+ var args []string
+ if opts.NoMap {
+ args = append(args, "--no-map")
+ }
+ if opts.Use != "" {
+ args = append(args, "--use", opts.Use)
+ }
+ if opts.Parser != "" {
+ args = append(args, "--parser", opts.Parser)
+ }
+ if opts.Stringifier != "" {
+ args = append(args, "--stringifier", opts.Stringifier)
+ }
+ if opts.Syntax != "" {
+ args = append(args, "--syntax", opts.Syntax)
+ }
+ return args
+}
+
+// Client is the client used to do PostCSS transformations.
+type Client struct {
+ rs *resource.Spec
+}
+
+// New creates a new Client with the given specification.
+func New(rs *resource.Spec) *Client {
+ return &Client{rs: rs}
+}
+
+type postcssTransformation struct {
+ options Options
+ rs *resource.Spec
+}
+
+func (t *postcssTransformation) Key() resource.ResourceTransformationKey {
+ return resource.NewResourceTransformationKey("postcss", t.options)
+}
+
+// Transform shells out to postcss-cli to do the heavy lifting.
+// For this to work, you need some additional tools. To install them globally:
+// npm install -g postcss-cli
+// npm install -g autoprefixer
+func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCtx) error {
+
+ const binary = "postcss"
+
+ if _, err := exec.LookPath(binary); err != nil {
+ // This may be on a CI server etc. Will fall back to pre-built assets.
+ return errors.FeatureNotAvailableErr
+ }
+
+ var configFile string
+ logger := t.rs.Logger
+
+ if t.options.Config != "" {
+ configFile = t.options.Config
+ } else {
+ configFile = "postcss.config.js"
+ }
+
+ configFile = filepath.Clean(configFile)
+
+ // We need an abolute filename to the config file.
+ if !filepath.IsAbs(configFile) {
+ // We resolve this against the virtual Work filesystem, to allow
+ // this config file to live in one of the themes if needed.
+ fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile)
+ if err != nil {
+ if t.options.Config != "" {
+ // Only fail if the user specificed config file is not found.
+ return fmt.Errorf("postcss config %q not found: %s", configFile, err)
+ }
+ configFile = ""
+ } else {
+ configFile = fi.(hugofs.RealFilenameInfo).RealFilename()
+ }
+ }
+
+ var cmdArgs []string
+
+ if configFile != "" {
+ logger.INFO.Println("postcss: use config file", configFile)
+ cmdArgs = []string{"--config", configFile}
+ }
+
+ if optArgs := t.options.toArgs(); len(optArgs) > 0 {
+ cmdArgs = append(cmdArgs, optArgs...)
+ }
+
+ cmd := exec.Command(binary, cmdArgs...)
+
+ cmd.Stdout = ctx.To
+ cmd.Stderr = os.Stderr
+
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ return err
+ }
+
+ go func() {
+ defer stdin.Close()
+ io.Copy(stdin, ctx.From)
+ }()
+
+ err = cmd.Run()
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Process transforms the given Resource with the PostCSS processor.
+func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) {
+ return c.rs.Transform(
+ res,
+ &postcssTransformation{rs: c.rs, options: options},
+ )
+}
diff --git a/resource/resource.go b/resource/resource.go
index 9a3725f8a..f0989e51e 100644
--- a/resource/resource.go
+++ b/resource/resource.go
@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,20 +14,24 @@
package resource
import (
+ "errors"
"fmt"
+ "io"
+ "io/ioutil"
"mime"
"os"
"path"
"path/filepath"
- "strconv"
"strings"
"sync"
- "github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/tpl"
- "github.com/spf13/afero"
+ "github.com/gohugoio/hugo/common/loggers"
+
+ jww "github.com/spf13/jwalterweatherman"
- "github.com/spf13/cast"
+ "github.com/spf13/afero"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/helpers"
@@ -36,34 +40,39 @@ import (
)
var (
+ _ ContentResource = (*genericResource)(nil)
+ _ ReadSeekCloserResource = (*genericResource)(nil)
_ Resource = (*genericResource)(nil)
- _ metaAssigner = (*genericResource)(nil)
_ Source = (*genericResource)(nil)
_ Cloner = (*genericResource)(nil)
_ ResourcesLanguageMerger = (*Resources)(nil)
+ _ permalinker = (*genericResource)(nil)
)
const DefaultResourceType = "unknown"
+var noData = make(map[string]interface{})
+
// Source is an internal template and not meant for use in the templates. It
// may change without notice.
type Source interface {
- AbsSourceFilename() string
Publish() error
}
+type permalinker interface {
+ relPermalinkFor(target string) string
+ permalinkFor(target string) string
+ relTargetPathFor(target string) string
+ relTargetPath() string
+ targetPath() string
+}
+
// Cloner is an internal template and not meant for use in the templates. It
// may change without notice.
type Cloner interface {
WithNewBase(base string) Resource
}
-type metaAssigner interface {
- setTitle(title string)
- setName(name string)
- updateParams(params map[string]interface{})
-}
-
// Resource represents a linkable resource, i.e. a content page, image etc.
type Resource interface {
// Permalink represents the absolute link to this resource.
@@ -77,6 +86,9 @@ type Resource interface {
// For content pages, this value is "page".
ResourceType() string
+ // MediaType is this resource's MIME type.
+ MediaType() media.Type
+
// Name is the logical name of this resource. This can be set in the front matter
// metadata for this resource. If not set, Hugo will assign a value.
// This will in most cases be the base filename.
@@ -88,8 +100,30 @@ type Resource interface {
// Title returns the title if set in front matter. For content pages, this will be the expected value.
Title() string
+ // Resource specific data set by Hugo.
+ // One example would be.Data.Digest for fingerprinted resources.
+ Data() interface{}
+
// Params set in front matter for this resource.
Params() map[string]interface{}
+}
+
+type ResourcesLanguageMerger interface {
+ MergeByLanguage(other Resources) Resources
+ // Needed for integration with the tpl package.
+ MergeByLanguageInterface(other interface{}) (interface{}, error)
+}
+
+type translatedResource interface {
+ TranslationKey() string
+}
+
+// ContentResource represents a Resource that provides a way to get to its content.
+// Most Resource types in Hugo implements this interface, including Page.
+// This should be used with care, as it will read the file content into memory, but it
+// should be cached as effectively as possible by the implementation.
+type ContentResource interface {
+ Resource
// Content returns this resource's content. It will be equivalent to reading the content
// that RelPermalink points to in the published folder.
@@ -100,14 +134,22 @@ type Resource interface {
Content() (interface{}, error)
}
-type ResourcesLanguageMerger interface {
- MergeByLanguage(other Resources) Resources
- // Needed for integration with the tpl package.
- MergeByLanguageInterface(other interface{}) (interface{}, error)
+// ReadSeekCloser is implemented by afero.File. We use this as the common type for
+// content in Resource objects, even for strings.
+type ReadSeekCloser interface {
+ io.Reader
+ io.Seeker
+ io.Closer
}
-type translatedResource interface {
- TranslationKey() string
+// OpenReadSeekeCloser allows setting some other way (than reading from a filesystem)
+// to open or create a ReadSeekCloser.
+type OpenReadSeekCloser func() (ReadSeekCloser, error)
+
+// ReadSeekCloserResource is a Resource that supports loading its content.
+type ReadSeekCloserResource interface {
+ Resource
+ ReadSeekCloser() (ReadSeekCloser, error)
}
// Resources represents a slice of resources, which can be a mix of different types.
@@ -125,44 +167,6 @@ func (r Resources) ByType(tp string) Resources {
return filtered
}
-const prefixDeprecatedMsg = `We have added the more flexible Resources.GetMatch (find one) and Resources.Match (many) to replace the "prefix" methods.
-
-These matches by a given globbing pattern, e.g. "*.jpg".
-
-Some examples:
-
-* To find all resources by its prefix in the root dir of the bundle: .Match image*
-* To find one resource by its prefix in the root dir of the bundle: .GetMatch image*
-* To find all JPEG images anywhere in the bundle: .Match **.jpg`
-
-// GetByPrefix gets the first resource matching the given filename prefix, e.g
-// "logo" will match logo.png. It returns nil of none found.
-// In potential ambiguous situations, combine it with ByType.
-func (r Resources) GetByPrefix(prefix string) Resource {
- helpers.Deprecated("Resources", "GetByPrefix", prefixDeprecatedMsg, true)
- prefix = strings.ToLower(prefix)
- for _, resource := range r {
- if matchesPrefix(resource, prefix) {
- return resource
- }
- }
- return nil
-}
-
-// ByPrefix gets all resources matching the given base filename prefix, e.g
-// "logo" will match logo.png.
-func (r Resources) ByPrefix(prefix string) Resources {
- helpers.Deprecated("Resources", "ByPrefix", prefixDeprecatedMsg, true)
- var matches Resources
- prefix = strings.ToLower(prefix)
- for _, resource := range r {
- if matchesPrefix(resource, prefix) {
- matches = append(matches, resource)
- }
- }
- return matches
-}
-
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
// See Match for a more complete explanation about the rules used.
func (r Resources) GetMatch(pattern string) Resource {
@@ -204,10 +208,6 @@ func (r Resources) Match(pattern string) Resources {
return matches
}
-func matchesPrefix(r Resource, prefix string) bool {
- return strings.HasPrefix(strings.ToLower(r.Name()), prefix)
-}
-
var (
globCache = make(map[string]glob.Glob)
globMu sync.RWMutex
@@ -268,81 +268,180 @@ func (r1 Resources) MergeByLanguageInterface(in interface{}) (interface{}, error
type Spec struct {
*helpers.PathSpec
- mimeTypes media.Types
+ MediaTypes media.Types
+
+ Logger *jww.Notepad
+
+ TextTemplates tpl.TemplateParseFinder
// Holds default filter settings etc.
imaging *Imaging
- imageCache *imageCache
+ imageCache *imageCache
+ ResourceCache *ResourceCache
- GenImagePath string
+ GenImagePath string
+ GenAssetsPath string
}
-func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) {
+func NewSpec(s *helpers.PathSpec, logger *jww.Notepad, mimeTypes media.Types) (*Spec, error) {
imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging"))
if err != nil {
return nil, err
}
- genImagePath := filepath.FromSlash("_gen/images")
+ if logger == nil {
+ logger = loggers.NewErrorLogger()
+ }
- return &Spec{PathSpec: s,
- GenImagePath: genImagePath,
- imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache(
+ genImagePath := filepath.FromSlash("_gen/images")
+ // The transformed assets (CSS etc.)
+ genAssetsPath := filepath.FromSlash("_gen/assets")
+
+ rs := &Spec{PathSpec: s,
+ Logger: logger,
+ GenImagePath: genImagePath,
+ GenAssetsPath: genAssetsPath,
+ imaging: &imaging,
+ MediaTypes: mimeTypes,
+ imageCache: newImageCache(
s,
// We're going to write a cache pruning routine later, so make it extremely
// unlikely that the user shoots him or herself in the foot
// and this is set to a value that represents data he/she
// cares about. This should be set in stone once released.
genImagePath,
- )}, nil
-}
+ )}
-func (r *Spec) NewResourceFromFile(
- targetPathBuilder func(base string) string,
- file source.File, relTargetFilename string) (Resource, error) {
+ rs.ResourceCache = newResourceCache(rs)
+
+ return rs, nil
- return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename)
}
-func (r *Spec) NewResourceFromFilename(
- targetPathBuilder func(base string) string,
- absSourceFilename, relTargetFilename string) (Resource, error) {
+type ResourceSourceDescriptor struct {
+ // TargetPathBuilder is a callback to create target paths's relative to its owner.
+ TargetPathBuilder func(base string) string
- fi, err := r.sourceFs().Stat(absSourceFilename)
- if err != nil {
- return nil, err
+ // Need one of these to load the resource content.
+ SourceFile source.File
+ OpenReadSeekCloser OpenReadSeekCloser
+
+ // If OpenReadSeekerCloser is not set, we use this to open the file.
+ SourceFilename string
+
+ // The relative target filename without any language code.
+ RelTargetFilename string
+
+ // Any base path prepeneded to the permalink.
+ // Typically the language code if this resource should be published to its sub-folder.
+ URLBase string
+
+ // Any base path prepended to the target path. This will also typically be the
+ // language code, but setting it here means that it should not have any effect on
+ // the permalink.
+ TargetPathBase string
+
+ // Delay publishing until either Permalink or RelPermalink is called. Maybe never.
+ LazyPublish bool
+}
+
+func (r ResourceSourceDescriptor) Filename() string {
+ if r.SourceFile != nil {
+ return r.SourceFile.Filename()
}
- return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename)
+ return r.SourceFilename
}
func (r *Spec) sourceFs() afero.Fs {
- return r.PathSpec.BaseFs.ContentFs
+ return r.PathSpec.BaseFs.Content.Fs
}
-func (r *Spec) newResource(
- targetPathBuilder func(base string) string,
- absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) {
+func (r *Spec) New(fd ResourceSourceDescriptor) (Resource, error) {
+ return r.newResourceForFs(r.sourceFs(), fd)
+}
- var mimeType string
- ext := filepath.Ext(relTargetFilename)
- m, found := r.mimeTypes.GetBySuffix(strings.TrimPrefix(ext, "."))
- if found {
- mimeType = m.SubType
- } else {
- mimeType = mime.TypeByExtension(ext)
- if mimeType == "" {
- mimeType = DefaultResourceType
- } else {
- mimeType = mimeType[:strings.Index(mimeType, "/")]
+func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) {
+ return r.newResourceForFs(sourceFs, fd)
+}
+
+func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) {
+ if fd.OpenReadSeekCloser == nil {
+ if fd.SourceFile != nil && fd.SourceFilename != "" {
+ return nil, errors.New("both SourceFile and AbsSourceFilename provided")
+ } else if fd.SourceFile == nil && fd.SourceFilename == "" {
+ return nil, errors.New("either SourceFile or AbsSourceFilename must be provided")
}
}
- gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType)
+ if fd.URLBase == "" {
+ fd.URLBase = r.GetURLLanguageBasePath()
+ }
+
+ if fd.TargetPathBase == "" {
+ fd.TargetPathBase = r.GetTargetLanguageBasePath()
+ }
+
+ if fd.RelTargetFilename == "" {
+ fd.RelTargetFilename = fd.Filename()
+ }
+
+ return r.newResource(sourceFs, fd)
+}
- if mimeType == "image" {
- ext := strings.ToLower(helpers.Ext(absSourceFilename))
+func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) {
+ var fi os.FileInfo
+ var sourceFilename string
+
+ if fd.OpenReadSeekCloser != nil {
+
+ } else if fd.SourceFilename != "" {
+ var err error
+ fi, err = sourceFs.Stat(fd.SourceFilename)
+ if err != nil {
+ return nil, err
+ }
+ sourceFilename = fd.SourceFilename
+ } else {
+ fi = fd.SourceFile.FileInfo()
+ sourceFilename = fd.SourceFile.Filename()
+ }
+
+ if fd.RelTargetFilename == "" {
+ fd.RelTargetFilename = sourceFilename
+ }
+
+ ext := filepath.Ext(fd.RelTargetFilename)
+ mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, "."))
+ // TODO(bep) we need to handle these ambigous types better, but in this context
+ // we most likely want the application/xml type.
+ if mimeType.Suffix == "xml" && mimeType.SubType == "rss" {
+ mimeType, found = r.MediaTypes.GetByType("application/xml")
+ }
+
+ if !found {
+ mimeStr := mime.TypeByExtension(ext)
+ if mimeStr != "" {
+ mimeType, _ = media.FromString(mimeStr)
+ }
+
+ }
+
+ gr := r.newGenericResourceWithBase(
+ sourceFs,
+ fd.LazyPublish,
+ fd.OpenReadSeekCloser,
+ fd.URLBase,
+ fd.TargetPathBase,
+ fd.TargetPathBuilder,
+ fi,
+ sourceFilename,
+ fd.RelTargetFilename,
+ mimeType)
+
+ if mimeType.MainType == "image" {
+ ext := strings.ToLower(helpers.Ext(sourceFilename))
imgFormat, ok := imageFormats[ext]
if !ok {
@@ -351,27 +450,21 @@ func (r *Spec) newResource(
return gr, nil
}
- f, err := gr.sourceFs().Open(absSourceFilename)
- if err != nil {
- return nil, fmt.Errorf("failed to open image source file: %s", err)
- }
- defer f.Close()
-
- hash, err := helpers.MD5FromFileFast(f)
- if err != nil {
+ if err := gr.initHash(); err != nil {
return nil, err
}
return &Image{
- hash: hash,
format: imgFormat,
imaging: r.imaging,
genericResource: gr}, nil
}
return gr, nil
+
}
-func (r *Spec) IsInCache(key string) bool {
+// TODO(bep) unify
+func (r *Spec) IsInImageCache(key string) bool {
// This is used for cache pruning. We currently only have images, but we could
// imagine expanding on this.
return r.imageCache.isInCache(key)
@@ -381,6 +474,11 @@ func (r *Spec) DeleteCacheByPrefix(prefix string) {
r.imageCache.deleteByPrefix(prefix)
}
+func (r *Spec) ClearCaches() {
+ r.imageCache.clear()
+ r.ResourceCache.clear()
+}
+
func (r *Spec) CacheStats() string {
r.imageCache.mu.RLock()
defer r.imageCache.mu.RUnlock()
@@ -410,18 +508,54 @@ func (d dirFile) path() string {
return path.Join(d.dir, d.file)
}
+type resourcePathDescriptor struct {
+ // The relative target directory and filename.
+ relTargetDirFile dirFile
+
+ // Callback used to construct a target path relative to its owner.
+ targetPathBuilder func(rel string) string
+
+ // baseURLDir is the fixed sub-folder for a resource in permalinks. This will typically
+ // be the language code if we publish to the language's sub-folder.
+ baseURLDir string
+
+ // This will normally be the same as above, but this will only apply to publishing
+ // of resources.
+ baseTargetPathDir string
+
+ // baseOffset is set when the output format's path has a offset, e.g. for AMP.
+ baseOffset string
+}
+
type resourceContent struct {
content string
contentInit sync.Once
}
+type resourceHash struct {
+ hash string
+ hashInit sync.Once
+}
+
+type publishOnce struct {
+ publisherInit sync.Once
+ publisherErr error
+ logger *jww.Notepad
+}
+
+func (l *publishOnce) publish(s Source) error {
+ l.publisherInit.Do(func() {
+ l.publisherErr = s.Publish()
+ if l.publisherErr != nil {
+ l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr)
+ }
+ })
+ return l.publisherErr
+}
+
// genericResource represents a generic linkable resource.
type genericResource struct {
- // The relative path to this resource.
- relTargetPath dirFile
-
- // Base is set when the output format's path has a offset, e.g. for AMP.
- base string
+ resourcePathDescriptor
title string
name string
@@ -433,6 +567,12 @@ type genericResource struct {
// the path to the file on the real filesystem.
sourceFilename string
+ // Will be set if this resource is backed by something other than a file.
+ openReadSeekerCloser OpenReadSeekCloser
+
+ // A hash of the source content. Is only calculated in caching situations.
+ *resourceHash
+
// This may be set to tell us to look in another filesystem for this resource.
// We, by default, use the sourceFs filesystem in the spec below.
overriddenSourceFs afero.Fs
@@ -440,20 +580,87 @@ type genericResource struct {
spec *Spec
resourceType string
- osFileInfo os.FileInfo
+ mediaType media.Type
- targetPathBuilder func(rel string) string
+ osFileInfo os.FileInfo
// We create copies of this struct, so this needs to be a pointer.
*resourceContent
+
+ // May be set to signal lazy/delayed publishing.
+ *publishOnce
+}
+
+func (l *genericResource) Data() interface{} {
+ return noData
}
func (l *genericResource) Content() (interface{}, error) {
+ if err := l.initContent(); err != nil {
+ return nil, err
+ }
+
+ return l.content, nil
+}
+
+func (l *genericResource) ReadSeekCloser() (ReadSeekCloser, error) {
+ if l.openReadSeekerCloser != nil {
+ return l.openReadSeekerCloser()
+ }
+ f, err := l.sourceFs().Open(l.sourceFilename)
+ if err != nil {
+ return nil, err
+ }
+ return f, nil
+
+}
+
+func (l *genericResource) MediaType() media.Type {
+ return l.mediaType
+}
+
+// Implement the Cloner interface.
+func (l genericResource) WithNewBase(base string) Resource {
+ l.baseOffset = base
+ l.resourceContent = &resourceContent{}
+ return &l
+}
+
+func (l *genericResource) initHash() error {
+ var err error
+ l.hashInit.Do(func() {
+ var hash string
+ var f ReadSeekCloser
+ f, err = l.ReadSeekCloser()
+ if err != nil {
+ err = fmt.Errorf("failed to open source file: %s", err)
+ return
+ }
+ defer f.Close()
+
+ hash, err = helpers.MD5FromFileFast(f)
+ if err != nil {
+ return
+ }
+ l.hash = hash
+
+ })
+
+ return err
+}
+
+func (l *genericResource) initContent() error {
var err error
l.contentInit.Do(func() {
- var b []byte
+ var r ReadSeekCloser
+ r, err = l.ReadSeekCloser()
+ if err != nil {
+ return
+ }
+ defer r.Close()
- b, err := afero.ReadFile(l.sourceFs(), l.AbsSourceFilename())
+ var b []byte
+ b, err = ioutil.ReadAll(r)
if err != nil {
return
}
@@ -462,7 +669,7 @@ func (l *genericResource) Content() (interface{}, error) {
})
- return l.content, err
+ return err
}
func (l *genericResource) sourceFs() afero.Fs {
@@ -472,12 +679,36 @@ func (l *genericResource) sourceFs() afero.Fs {
return l.spec.sourceFs()
}
+func (l *genericResource) publishIfNeeded() {
+ if l.publishOnce != nil {
+ l.publishOnce.publish(l)
+ }
+}
+
func (l *genericResource) Permalink() string {
- return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String())
+ l.publishIfNeeded()
+ return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path()), l.spec.BaseURL.HostURL())
}
func (l *genericResource) RelPermalink() string {
- return l.relPermalinkForRel(l.relTargetPath.path(), true)
+ l.publishIfNeeded()
+ return l.relPermalinkFor(l.relTargetDirFile.path())
+}
+
+func (l *genericResource) relPermalinkFor(target string) string {
+ return l.relPermalinkForRel(target)
+
+}
+func (l *genericResource) permalinkFor(target string) string {
+ return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target), l.spec.BaseURL.HostURL())
+
+}
+func (l *genericResource) relTargetPathFor(target string) string {
+ return l.relTargetPathForRel(target, false)
+}
+
+func (l *genericResource) relTargetPath() string {
+ return l.relTargetPathForRel(l.targetPath(), false)
}
func (l *genericResource) Name() string {
@@ -514,31 +745,33 @@ func (l *genericResource) updateParams(params map[string]interface{}) {
}
}
-// Implement the Cloner interface.
-func (l genericResource) WithNewBase(base string) Resource {
- l.base = base
- l.resourceContent = &resourceContent{}
- return &l
+func (l *genericResource) relPermalinkForRel(rel string) string {
+ return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, true))
}
-func (l *genericResource) relPermalinkForRel(rel string, addBasePath bool) string {
- return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, addBasePath))
-}
+func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string {
-func (l *genericResource) relTargetPathForRel(rel string, addBasePath bool) string {
if l.targetPathBuilder != nil {
rel = l.targetPathBuilder(rel)
}
- if l.base != "" {
- rel = path.Join(l.base, rel)
+ if isURL && l.baseURLDir != "" {
+ rel = path.Join(l.baseURLDir, rel)
}
- if addBasePath && l.spec.PathSpec.BasePath != "" {
+ if !isURL && l.baseTargetPathDir != "" {
+ rel = path.Join(l.baseTargetPathDir, rel)
+ }
+
+ if l.baseOffset != "" {
+ rel = path.Join(l.baseOffset, rel)
+ }
+
+ if isURL && l.spec.PathSpec.BasePath != "" {
rel = path.Join(l.spec.PathSpec.BasePath, rel)
}
- if rel[0] != '/' {
+ if len(rel) == 0 || rel[0] != '/' {
rel = "/" + rel
}
@@ -549,146 +782,100 @@ func (l *genericResource) ResourceType() string {
return l.resourceType
}
-func (l *genericResource) AbsSourceFilename() string {
- return l.sourceFilename
-}
-
func (l *genericResource) String() string {
return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name)
}
func (l *genericResource) Publish() error {
- f, err := l.sourceFs().Open(l.AbsSourceFilename())
+ f, err := l.ReadSeekCloser()
if err != nil {
return err
}
defer f.Close()
- return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs)
-}
-
-const counterPlaceHolder = ":counter"
-
-// AssignMetadata assigns the given metadata to those resources that supports updates
-// and matching by wildcard given in `src` using `filepath.Match` with lower cased values.
-// This assignment is additive, but the most specific match needs to be first.
-// The `name` and `title` metadata field support shell-matched collection it got a match in.
-// See https://golang.org/pkg/path/#Match
-func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error {
-
- counters := make(map[string]int)
-
- for _, r := range resources {
- if _, ok := r.(metaAssigner); !ok {
- continue
- }
-
- var (
- nameSet, titleSet bool
- nameCounter, titleCounter = 0, 0
- nameCounterFound, titleCounterFound bool
- resourceSrcKey = strings.ToLower(r.Name())
- )
-
- ma := r.(metaAssigner)
- for _, meta := range metadata {
- src, found := meta["src"]
- if !found {
- return fmt.Errorf("missing 'src' in metadata for resource")
- }
-
- srcKey := strings.ToLower(cast.ToString(src))
-
- glob, err := getGlob(srcKey)
- if err != nil {
- return fmt.Errorf("failed to match resource with metadata: %s", err)
- }
-
- match := glob.Match(resourceSrcKey)
-
- if match {
- if !nameSet {
- name, found := meta["name"]
- if found {
- name := cast.ToString(name)
- if !nameCounterFound {
- nameCounterFound = strings.Contains(name, counterPlaceHolder)
- }
- if nameCounterFound && nameCounter == 0 {
- counterKey := "name_" + srcKey
- nameCounter = counters[counterKey] + 1
- counters[counterKey] = nameCounter
- }
-
- ma.setName(replaceResourcePlaceholders(name, nameCounter))
- nameSet = true
- }
- }
-
- if !titleSet {
- title, found := meta["title"]
- if found {
- title := cast.ToString(title)
- if !titleCounterFound {
- titleCounterFound = strings.Contains(title, counterPlaceHolder)
- }
- if titleCounterFound && titleCounter == 0 {
- counterKey := "title_" + srcKey
- titleCounter = counters[counterKey] + 1
- counters[counterKey] = titleCounter
- }
- ma.setTitle((replaceResourcePlaceholders(title, titleCounter)))
- titleSet = true
- }
- }
-
- params, found := meta["params"]
- if found {
- m := cast.ToStringMap(params)
- // Needed for case insensitive fetching of params values
- maps.ToLower(m)
- ma.updateParams(m)
- }
- }
- }
- }
-
- return nil
+ return helpers.WriteToDisk(l.targetFilename(), f, l.spec.BaseFs.PublishFs)
}
-func replaceResourcePlaceholders(in string, counter int) string {
- return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1)
+// Path is stored with Unix style slashes.
+func (l *genericResource) targetPath() string {
+ return l.relTargetDirFile.path()
}
-func (l *genericResource) target() string {
- target := l.relTargetPathForRel(l.relTargetPath.path(), false)
- if l.spec.PathSpec.Languages.IsMultihost() {
- target = path.Join(l.spec.PathSpec.Language.Lang, target)
- }
- return filepath.Clean(target)
+func (l *genericResource) targetFilename() string {
+ return filepath.Clean(l.relTargetPath())
}
-func (r *Spec) newGenericResource(
+// TODO(bep) clean up below
+func (r *Spec) newGenericResource(sourceFs afero.Fs,
+ targetPathBuilder func(base string) string,
+ osFileInfo os.FileInfo,
+ sourceFilename,
+ baseFilename string,
+ mediaType media.Type) *genericResource {
+ return r.newGenericResourceWithBase(
+ sourceFs,
+ false,
+ nil,
+ "",
+ "",
+ targetPathBuilder,
+ osFileInfo,
+ sourceFilename,
+ baseFilename,
+ mediaType,
+ )
+
+}
+
+func (r *Spec) newGenericResourceWithBase(
+ sourceFs afero.Fs,
+ lazyPublish bool,
+ openReadSeekerCloser OpenReadSeekCloser,
+ urlBaseDir string,
+ targetPathBaseDir string,
targetPathBuilder func(base string) string,
osFileInfo os.FileInfo,
sourceFilename,
- baseFilename,
- resourceType string) *genericResource {
+ baseFilename string,
+ mediaType media.Type) *genericResource {
// This value is used both to construct URLs and file paths, but start
// with a Unix-styled path.
- baseFilename = filepath.ToSlash(baseFilename)
+ baseFilename = helpers.ToSlashTrimLeading(baseFilename)
fpath, fname := path.Split(baseFilename)
- return &genericResource{
+ var resourceType string
+ if mediaType.MainType == "image" {
+ resourceType = mediaType.MainType
+ } else {
+ resourceType = mediaType.SubType
+ }
+
+ pathDescriptor := resourcePathDescriptor{
+ baseURLDir: urlBaseDir,
+ baseTargetPathDir: targetPathBaseDir,
targetPathBuilder: targetPathBuilder,
- osFileInfo: osFileInfo,
- sourceFilename: sourceFilename,
- relTargetPath: dirFile{dir: fpath, file: fname},
- resourceType: resourceType,
- spec: r,
- params: make(map[string]interface{}),
- name: baseFilename,
- title: baseFilename,
- resourceContent: &resourceContent{},
+ relTargetDirFile: dirFile{dir: fpath, file: fname},
+ }
+
+ var po *publishOnce
+ if lazyPublish {
+ po = &publishOnce{logger: r.Logger}
+ }
+
+ return &genericResource{
+ openReadSeekerCloser: openReadSeekerCloser,
+ publishOnce: po,
+ resourcePathDescriptor: pathDescriptor,
+ overriddenSourceFs: sourceFs,
+ osFileInfo: osFileInfo,
+ sourceFilename: sourceFilename,
+ mediaType: mediaType,
+ resourceType: resourceType,
+ spec: r,
+ params: make(map[string]interface{}),
+ name: baseFilename,
+ title: baseFilename,
+ resourceContent: &resourceContent{},
+ resourceHash: &resourceHash{},
}
}
diff --git a/resource/resource_cache.go b/resource/resource_cache.go
new file mode 100644
index 000000000..28c3c23a2
--- /dev/null
+++ b/resource/resource_cache.go
@@ -0,0 +1,241 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resource
+
+import (
+ "encoding/json"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "github.com/spf13/afero"
+
+ "github.com/BurntSushi/locker"
+)
+
+const (
+ CACHE_CLEAR_ALL = "clear_all"
+ CACHE_OTHER = "other"
+)
+
+type ResourceCache struct {
+ rs *Spec
+
+ cache map[string]Resource
+ sync.RWMutex
+
+ // Provides named resource locks.
+ nlocker *locker.Locker
+}
+
+// ResourceKeyPartition returns a partition name
+// to allow for more fine grained cache flushes.
+// It will return the file extension without the leading ".". If no
+// extension, it will return "other".
+func ResourceKeyPartition(filename string) string {
+ ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".")
+ if ext == "" {
+ ext = CACHE_OTHER
+ }
+ return ext
+}
+
+func newResourceCache(rs *Spec) *ResourceCache {
+ return &ResourceCache{
+ rs: rs,
+ cache: make(map[string]Resource),
+ nlocker: locker.NewLocker(),
+ }
+}
+
+func (c *ResourceCache) clear() {
+ c.Lock()
+ defer c.Unlock()
+
+ c.cache = make(map[string]Resource)
+ c.nlocker = locker.NewLocker()
+}
+
+func (c *ResourceCache) Contains(key string) bool {
+ key = c.cleanKey(filepath.ToSlash(key))
+ _, found := c.get(key)
+ return found
+}
+
+func (c *ResourceCache) cleanKey(key string) string {
+ return strings.TrimPrefix(path.Clean(key), "/")
+}
+
+func (c *ResourceCache) get(key string) (Resource, bool) {
+ c.RLock()
+ defer c.RUnlock()
+ r, found := c.cache[key]
+ return r, found
+}
+
+func (c *ResourceCache) GetOrCreate(partition, key string, f func() (Resource, error)) (Resource, error) {
+ key = c.cleanKey(path.Join(partition, key))
+ // First check in-memory cache.
+ r, found := c.get(key)
+ if found {
+ return r, nil
+ }
+ // This is a potentially long running operation, so get a named lock.
+ c.nlocker.Lock(key)
+
+ // Double check in-memory cache.
+ r, found = c.get(key)
+ if found {
+ c.nlocker.Unlock(key)
+ return r, nil
+ }
+
+ defer c.nlocker.Unlock(key)
+
+ r, err := f()
+ if err != nil {
+ return nil, err
+ }
+
+ c.set(key, r)
+
+ return r, nil
+
+}
+
+func (c *ResourceCache) getFilenames(key string) (string, string) {
+ filenameBase := filepath.Join(c.rs.GenAssetsPath, key)
+ filenameMeta := filenameBase + ".json"
+ filenameContent := filenameBase + ".content"
+
+ return filenameMeta, filenameContent
+}
+
+func (c *ResourceCache) getFromFile(key string) (afero.File, transformedResourceMetadata, bool) {
+ c.RLock()
+ defer c.RUnlock()
+
+ var meta transformedResourceMetadata
+ filenameMeta, filenameContent := c.getFilenames(key)
+ fMeta, err := c.rs.Resources.Fs.Open(filenameMeta)
+ if err != nil {
+ return nil, meta, false
+ }
+ defer fMeta.Close()
+
+ jsonContent, err := ioutil.ReadAll(fMeta)
+ if err != nil {
+ return nil, meta, false
+ }
+
+ if err := json.Unmarshal(jsonContent, &meta); err != nil {
+ return nil, meta, false
+ }
+
+ fContent, err := c.rs.Resources.Fs.Open(filenameContent)
+ if err != nil {
+ return nil, meta, false
+ }
+
+ return fContent, meta, true
+}
+
+// writeMeta writes the metadata to file and returns a writer for the content part.
+func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (afero.File, error) {
+ filenameMeta, filenameContent := c.getFilenames(key)
+ raw, err := json.Marshal(meta)
+ if err != nil {
+ return nil, err
+ }
+
+ fm, err := c.openResourceFileForWriting(filenameMeta)
+ if err != nil {
+ return nil, err
+ }
+
+ if _, err := fm.Write(raw); err != nil {
+ return nil, err
+ }
+
+ return c.openResourceFileForWriting(filenameContent)
+
+}
+
+func (c *ResourceCache) openResourceFileForWriting(filename string) (afero.File, error) {
+ return openFileForWriting(c.rs.Resources.Fs, filename)
+}
+
+// openFileForWriting opens or creates the given file. If the target directory
+// does not exist, it gets created.
+func openFileForWriting(fs afero.Fs, filename string) (afero.File, error) {
+ filename = filepath.Clean(filename)
+ // Create will truncate if file already exists.
+ f, err := fs.Create(filename)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return nil, err
+ }
+ if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil {
+ return nil, err
+ }
+ f, err = fs.Create(filename)
+ }
+
+ return f, err
+}
+
+func (c *ResourceCache) set(key string, r Resource) {
+ c.Lock()
+ defer c.Unlock()
+ c.cache[key] = r
+}
+
+func (c *ResourceCache) DeletePartitions(partitions ...string) {
+ partitionsSet := map[string]bool{
+ // Always clear out the resources not matching the partition.
+ "other": true,
+ }
+ for _, p := range partitions {
+ partitionsSet[p] = true
+ }
+
+ if partitionsSet[CACHE_CLEAR_ALL] {
+ c.clear()
+ return
+ }
+
+ c.Lock()
+ defer c.Unlock()
+
+ for k := range c.cache {
+ clear := false
+ partIdx := strings.Index(k, "/")
+ if partIdx == -1 {
+ clear = true
+ } else {
+ partition := k[:partIdx]
+ if partitionsSet[partition] {
+ clear = true
+ }
+ }
+
+ if clear {
+ delete(c.cache, k)
+ }
+ }
+
+}
diff --git a/resource/resource_metadata.go b/resource/resource_metadata.go
new file mode 100644
index 000000000..2c82aeaf6
--- /dev/null
+++ b/resource/resource_metadata.go
@@ -0,0 +1,129 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resource
+
+import (
+ "fmt"
+ "strconv"
+
+ "github.com/spf13/cast"
+
+ "strings"
+
+ "github.com/gohugoio/hugo/common/maps"
+)
+
+var (
+ _ metaAssigner = (*genericResource)(nil)
+)
+
+// metaAssigner allows updating metadata in resources that supports it.
+type metaAssigner interface {
+ setTitle(title string)
+ setName(name string)
+ updateParams(params map[string]interface{})
+}
+
+const counterPlaceHolder = ":counter"
+
+// AssignMetadata assigns the given metadata to those resources that supports updates
+// and matching by wildcard given in `src` using `filepath.Match` with lower cased values.
+// This assignment is additive, but the most specific match needs to be first.
+// The `name` and `title` metadata field support shell-matched collection it got a match in.
+// See https://golang.org/pkg/path/#Match
+func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error {
+
+ counters := make(map[string]int)
+
+ for _, r := range resources {
+ if _, ok := r.(metaAssigner); !ok {
+ continue
+ }
+
+ var (
+ nameSet, titleSet bool
+ nameCounter, titleCounter = 0, 0
+ nameCounterFound, titleCounterFound bool
+ resourceSrcKey = strings.ToLower(r.Name())
+ )
+
+ ma := r.(metaAssigner)
+ for _, meta := range metadata {
+ src, found := meta["src"]
+ if !found {
+ return fmt.Errorf("missing 'src' in metadata for resource")
+ }
+
+ srcKey := strings.ToLower(cast.ToString(src))
+
+ glob, err := getGlob(srcKey)
+ if err != nil {
+ return fmt.Errorf("failed to match resource with metadata: %s", err)
+ }
+
+ match := glob.Match(resourceSrcKey)
+
+ if match {
+ if !nameSet {
+ name, found := meta["name"]
+ if found {
+ name := cast.ToString(name)
+ if !nameCounterFound {
+ nameCounterFound = strings.Contains(name, counterPlaceHolder)
+ }
+ if nameCounterFound && nameCounter == 0 {
+ counterKey := "name_" + srcKey
+ nameCounter = counters[counterKey] + 1
+ counters[counterKey] = nameCounter
+ }
+
+ ma.setName(replaceResourcePlaceholders(name, nameCounter))
+ nameSet = true
+ }
+ }
+
+ if !titleSet {
+ title, found := meta["title"]
+ if found {
+ title := cast.ToString(title)
+ if !titleCounterFound {
+ titleCounterFound = strings.Contains(title, counterPlaceHolder)
+ }
+ if titleCounterFound && titleCounter == 0 {
+ counterKey := "title_" + srcKey
+ titleCounter = counters[counterKey] + 1
+ counters[counterKey] = titleCounter
+ }
+ ma.setTitle((replaceResourcePlaceholders(title, titleCounter)))
+ titleSet = true
+ }
+ }
+
+ params, found := meta["params"]
+ if found {
+ m := cast.ToStringMap(params)
+ // Needed for case insensitive fetching of params values
+ maps.ToLower(m)
+ ma.updateParams(m)
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
+func replaceResourcePlaceholders(in string, counter int) string {
+ return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1)
+}
diff --git a/resource/resource_metadata_test.go b/resource/resource_metadata_test.go
new file mode 100644
index 000000000..85fb25b57
--- /dev/null
+++ b/resource/resource_metadata_test.go
@@ -0,0 +1,230 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resource
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/media"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestAssignMetadata(t *testing.T) {
+ assert := require.New(t)
+ spec := newTestResourceSpec(assert)
+
+ var foo1, foo2, foo3, logo1, logo2, logo3 Resource
+ var resources Resources
+
+ for _, this := range []struct {
+ metaData []map[string]interface{}
+ assertFunc func(err error)
+ }{
+ {[]map[string]interface{}{
+ {
+ "title": "My Resource",
+ "name": "My Name",
+ "src": "*",
+ },
+ }, func(err error) {
+ assert.Equal("My Resource", logo1.Title())
+ assert.Equal("My Name", logo1.Name())
+ assert.Equal("My Name", foo2.Name())
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "My Logo",
+ "src": "*loGo*",
+ },
+ {
+ "title": "My Resource",
+ "name": "My Name",
+ "src": "*",
+ },
+ }, func(err error) {
+ assert.Equal("My Logo", logo1.Title())
+ assert.Equal("My Logo", logo2.Title())
+ assert.Equal("My Name", logo1.Name())
+ assert.Equal("My Name", foo2.Name())
+ assert.Equal("My Name", foo3.Name())
+ assert.Equal("My Resource", foo3.Title())
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "My Logo",
+ "src": "*loGo*",
+ "params": map[string]interface{}{
+ "Param1": true,
+ "icon": "logo",
+ },
+ },
+ {
+ "title": "My Resource",
+ "src": "*",
+ "params": map[string]interface{}{
+ "Param2": true,
+ "icon": "resource",
+ },
+ },
+ }, func(err error) {
+ assert.NoError(err)
+ assert.Equal("My Logo", logo1.Title())
+ assert.Equal("My Resource", foo3.Title())
+ _, p1 := logo2.Params()["param1"]
+ _, p2 := foo2.Params()["param2"]
+ _, p1_2 := foo2.Params()["param1"]
+ _, p2_2 := logo2.Params()["param2"]
+
+ icon1, _ := logo2.Params()["icon"]
+ icon2, _ := foo2.Params()["icon"]
+
+ assert.True(p1)
+ assert.True(p2)
+
+ // Check merge
+ assert.True(p2_2)
+ assert.False(p1_2)
+
+ assert.Equal("logo", icon1)
+ assert.Equal("resource", icon2)
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "name": "Logo Name #:counter",
+ "src": "*logo*",
+ },
+ {
+ "title": "Resource #:counter",
+ "name": "Name #:counter",
+ "src": "*",
+ },
+ }, func(err error) {
+ assert.NoError(err)
+ assert.Equal("Resource #2", logo2.Title())
+ assert.Equal("Logo Name #1", logo2.Name())
+ assert.Equal("Resource #4", logo1.Title())
+ assert.Equal("Logo Name #2", logo1.Name())
+ assert.Equal("Resource #1", foo2.Title())
+ assert.Equal("Resource #3", foo1.Title())
+ assert.Equal("Name #2", foo1.Name())
+ assert.Equal("Resource #5", foo3.Title())
+
+ assert.Equal(logo2, resources.GetMatch("logo name #1*"))
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "Third Logo #:counter",
+ "src": "logo3.png",
+ },
+ {
+ "title": "Other Logo #:counter",
+ "name": "Name #:counter",
+ "src": "logo*",
+ },
+ }, func(err error) {
+ assert.NoError(err)
+ assert.Equal("Third Logo #1", logo3.Title())
+ assert.Equal("Name #3", logo3.Name())
+ assert.Equal("Other Logo #1", logo2.Title())
+ assert.Equal("Name #1", logo2.Name())
+ assert.Equal("Other Logo #2", logo1.Title())
+ assert.Equal("Name #2", logo1.Name())
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "Third Logo",
+ "src": "logo3.png",
+ },
+ {
+ "title": "Other Logo #:counter",
+ "name": "Name #:counter",
+ "src": "logo*",
+ },
+ }, func(err error) {
+ assert.NoError(err)
+ assert.Equal("Third Logo", logo3.Title())
+ assert.Equal("Name #3", logo3.Name())
+ assert.Equal("Other Logo #1", logo2.Title())
+ assert.Equal("Name #1", logo2.Name())
+ assert.Equal("Other Logo #2", logo1.Title())
+ assert.Equal("Name #2", logo1.Name())
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "name": "third-logo",
+ "src": "logo3.png",
+ },
+ {
+ "title": "Logo #:counter",
+ "name": "Name #:counter",
+ "src": "logo*",
+ },
+ }, func(err error) {
+ assert.NoError(err)
+ assert.Equal("Logo #3", logo3.Title())
+ assert.Equal("third-logo", logo3.Name())
+ assert.Equal("Logo #1", logo2.Title())
+ assert.Equal("Name #1", logo2.Name())
+ assert.Equal("Logo #2", logo1.Title())
+ assert.Equal("Name #2", logo1.Name())
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "Third Logo #:counter",
+ },
+ }, func(err error) {
+ // Missing src
+ assert.Error(err)
+
+ }},
+ {[]map[string]interface{}{
+ {
+ "title": "Title",
+ "src": "[]",
+ },
+ }, func(err error) {
+ // Invalid pattern
+ assert.Error(err)
+
+ }},
+ } {
+
+ foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType)
+ logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType)
+ foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType)
+ logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType)
+ foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)
+ logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType)
+
+ resources = Resources{
+ foo2,
+ logo2,
+ foo1,
+ logo1,
+ foo3,
+ logo3,
+ }
+
+ this.assertFunc(AssignMetadata(this.metaData, resources...))
+ }
+
+}
diff --git a/resource/resource_test.go b/resource/resource_test.go
index 40061e5c4..659994c36 100644
--- a/resource/resource_test.go
+++ b/resource/resource_test.go
@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -22,6 +22,8 @@ import (
"testing"
"time"
+ "github.com/gohugoio/hugo/media"
+
"github.com/stretchr/testify/require"
)
@@ -29,7 +31,7 @@ func TestGenericResource(t *testing.T) {
assert := require.New(t)
spec := newTestResourceSpec(assert)
- r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css")
+ r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType)
assert.Equal("https://example.com/foo.css", r.Permalink())
assert.Equal("/foo.css", r.RelPermalink())
@@ -44,7 +46,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) {
factory := func(s string) string {
return path.Join("/foo", s)
}
- r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css")
+ r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType)
assert.Equal("https://example.com/foo/foo.css", r.Permalink())
assert.Equal("/foo/foo.css", r.RelPermalink())
@@ -58,8 +60,7 @@ func TestNewResourceFromFilename(t *testing.T) {
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
writeSource(t, spec.Fs, "content/a/b/data.json", "json")
- r, err := spec.NewResourceFromFilename(nil,
- filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+ r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"})
assert.NoError(err)
assert.NotNil(r)
@@ -67,7 +68,7 @@ func TestNewResourceFromFilename(t *testing.T) {
assert.Equal("/a/b/logo.png", r.RelPermalink())
assert.Equal("https://example.com/a/b/logo.png", r.Permalink())
- r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json")
+ r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"})
assert.NoError(err)
assert.NotNil(r)
@@ -84,8 +85,7 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
- r, err := spec.NewResourceFromFilename(nil,
- filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+ r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")})
assert.NoError(err)
assert.NotNil(r)
@@ -93,18 +93,20 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
assert.Equal("/docs/a/b/logo.png", r.RelPermalink())
assert.Equal("https://example.com/docs/a/b/logo.png", r.Permalink())
img := r.(*Image)
- assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.target())
+ assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilename())
}
+var pngType, _ = media.FromString("image/png")
+
func TestResourcesByType(t *testing.T) {
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"),
- spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")}
+ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType),
+ spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)}
assert.Len(resources.ByType("css"), 3)
assert.Len(resources.ByType("image"), 1)
@@ -115,25 +117,25 @@ func TestResourcesGetByPrefix(t *testing.T) {
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
- spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
- spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")}
-
- assert.Nil(resources.GetByPrefix("asdf"))
- assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink())
- assert.Equal("/logo1.png", resources.GetByPrefix("loGo").RelPermalink())
- assert.Equal("/Logo2.png", resources.GetByPrefix("logo2").RelPermalink())
- assert.Equal("/foo2.css", resources.GetByPrefix("foo2").RelPermalink())
- assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink())
- assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink())
- assert.Nil(resources.GetByPrefix("asdfasdf"))
-
- assert.Equal(2, len(resources.ByPrefix("logo")))
- assert.Equal(1, len(resources.ByPrefix("logo2")))
-
- logo := resources.GetByPrefix("logo")
+ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType),
+ spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType),
+ spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)}
+
+ assert.Nil(resources.GetMatch("asdf*"))
+ assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink())
+ assert.Equal("/logo1.png", resources.GetMatch("loGo*").RelPermalink())
+ assert.Equal("/Logo2.png", resources.GetMatch("logo2*").RelPermalink())
+ assert.Equal("/foo2.css", resources.GetMatch("foo2*").RelPermalink())
+ assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink())
+ assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink())
+ assert.Nil(resources.GetMatch("asdfasdf*"))
+
+ assert.Equal(2, len(resources.Match("logo*")))
+ assert.Equal(1, len(resources.Match("logo2*")))
+
+ logo := resources.GetMatch("logo*")
assert.NotNil(logo.Params())
assert.Equal("logo1.png", logo.Name())
assert.Equal("logo1.png", logo.Title())
@@ -144,14 +146,14 @@ func TestResourcesGetMatch(t *testing.T) {
assert := require.New(t)
spec := newTestResourceSpec(assert)
resources := Resources{
- spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
- spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
- spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
- spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
- spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"),
- spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"),
- spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"),
- spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"),
+ spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType),
+ spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType),
+ spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType),
+ spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType),
}
assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink())
@@ -186,226 +188,6 @@ func TestResourcesGetMatch(t *testing.T) {
}
-func TestAssignMetadata(t *testing.T) {
- assert := require.New(t)
- spec := newTestResourceSpec(assert)
-
- var foo1, foo2, foo3, logo1, logo2, logo3 Resource
- var resources Resources
-
- for _, this := range []struct {
- metaData []map[string]interface{}
- assertFunc func(err error)
- }{
- {[]map[string]interface{}{
- {
- "title": "My Resource",
- "name": "My Name",
- "src": "*",
- },
- }, func(err error) {
- assert.Equal("My Resource", logo1.Title())
- assert.Equal("My Name", logo1.Name())
- assert.Equal("My Name", foo2.Name())
-
- }},
- {[]map[string]interface{}{
- {
- "title": "My Logo",
- "src": "*loGo*",
- },
- {
- "title": "My Resource",
- "name": "My Name",
- "src": "*",
- },
- }, func(err error) {
- assert.Equal("My Logo", logo1.Title())
- assert.Equal("My Logo", logo2.Title())
- assert.Equal("My Name", logo1.Name())
- assert.Equal("My Name", foo2.Name())
- assert.Equal("My Name", foo3.Name())
- assert.Equal("My Resource", foo3.Title())
-
- }},
- {[]map[string]interface{}{
- {
- "title": "My Logo",
- "src": "*loGo*",
- "params": map[string]interface{}{
- "Param1": true,
- "icon": "logo",
- },
- },
- {
- "title": "My Resource",
- "src": "*",
- "params": map[string]interface{}{
- "Param2": true,
- "icon": "resource",
- },
- },
- }, func(err error) {
- assert.NoError(err)
- assert.Equal("My Logo", logo1.Title())
- assert.Equal("My Resource", foo3.Title())
- _, p1 := logo2.Params()["param1"]
- _, p2 := foo2.Params()["param2"]
- _, p1_2 := foo2.Params()["param1"]
- _, p2_2 := logo2.Params()["param2"]
-
- icon1, _ := logo2.Params()["icon"]
- icon2, _ := foo2.Params()["icon"]
-
- assert.True(p1)
- assert.True(p2)
-
- // Check merge
- assert.True(p2_2)
- assert.False(p1_2)
-
- assert.Equal("logo", icon1)
- assert.Equal("resource", icon2)
-
- }},
- {[]map[string]interface{}{
- {
- "name": "Logo Name #:counter",
- "src": "*logo*",
- },
- {
- "title": "Resource #:counter",
- "name": "Name #:counter",
- "src": "*",
- },
- }, func(err error) {
- assert.NoError(err)
- assert.Equal("Resource #2", logo2.Title())
- assert.Equal("Logo Name #1", logo2.Name())
- assert.Equal("Resource #4", logo1.Title())
- assert.Equal("Logo Name #2", logo1.Name())
- assert.Equal("Resource #1", foo2.Title())
- assert.Equal("Resource #3", foo1.Title())
- assert.Equal("Name #2", foo1.Name())
- assert.Equal("Resource #5", foo3.Title())
-
- assert.Equal(logo2, resources.GetByPrefix("logo name #1"))
-
- }},
- {[]map[string]interface{}{
- {
- "title": "Third Logo #:counter",
- "src": "logo3.png",
- },
- {
- "title": "Other Logo #:counter",
- "name": "Name #:counter",
- "src": "logo*",
- },
- }, func(err error) {
- assert.NoError(err)
- assert.Equal("Third Logo #1", logo3.Title())
- assert.Equal("Name #3", logo3.Name())
- assert.Equal("Other Logo #1", logo2.Title())
- assert.Equal("Name #1", logo2.Name())
- assert.Equal("Other Logo #2", logo1.Title())
- assert.Equal("Name #2", logo1.Name())
-
- }},
- {[]map[string]interface{}{
- {
- "title": "Third Logo",
- "src": "logo3.png",
- },
- {
- "title": "Other Logo #:counter",
- "name": "Name #:counter",
- "src": "logo*",
- },
- }, func(err error) {
- assert.NoError(err)
- assert.Equal("Third Logo", logo3.Title())
- assert.Equal("Name #3", logo3.Name())
- assert.Equal("Other Logo #1", logo2.Title())
- assert.Equal("Name #1", logo2.Name())
- assert.Equal("Other Logo #2", logo1.Title())
- assert.Equal("Name #2", logo1.Name())
-
- }},
- {[]map[string]interface{}{
- {
- "name": "third-logo",
- "src": "logo3.png",
- },
- {
- "title": "Logo #:counter",
- "name": "Name #:counter",
- "src": "logo*",
- },
- }, func(err error) {
- assert.NoError(err)
- assert.Equal("Logo #3", logo3.Title())
- assert.Equal("third-logo", logo3.Name())
- assert.Equal("Logo #1", logo2.Title())
- assert.Equal("Name #1", logo2.Name())
- assert.Equal("Logo #2", logo1.Title())
- assert.Equal("Name #2", logo1.Name())
-
- }},
- {[]map[string]interface{}{
- {
- "title": "Third Logo #:counter",
- },
- }, func(err error) {
- // Missing src
- assert.Error(err)
-
- }},
- {[]map[string]interface{}{
- {
- "title": "Title",
- "src": "[]",
- },
- }, func(err error) {
- // Invalid pattern
- assert.Error(err)
-
- }},
- } {
-
- foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css")
- logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image")
- foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css")
- logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image")
- foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")
- logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image")
-
- resources = Resources{
- foo2,
- logo2,
- foo1,
- logo1,
- foo3,
- logo3,
- }
-
- this.assertFunc(AssignMetadata(this.metaData, resources...))
- }
-
-}
-
-func BenchmarkResourcesByPrefix(b *testing.B) {
- resources := benchResources(b)
- prefixes := []string{"abc", "jkl", "nomatch", "sub/"}
- rnd := rand.New(rand.NewSource(time.Now().Unix()))
-
- b.RunParallel(func(pb *testing.PB) {
- for pb.Next() {
- resources.ByPrefix(prefixes[rnd.Intn(len(prefixes))])
- }
- })
-}
-
func BenchmarkResourcesMatch(b *testing.B) {
resources := benchResources(b)
prefixes := []string{"abc*", "jkl*", "nomatch*", "sub/*"}
@@ -428,7 +210,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) {
a100 := strings.Repeat("a", 100)
pattern := "a*a*a*a*a*a*a*a*b"
- resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")}
+ resources := Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)}
b.ResetTimer()
for i := 0; i < b.N; i++ {
@@ -444,17 +226,17 @@ func benchResources(b *testing.B) Resources {
for i := 0; i < 30; i++ {
name := fmt.Sprintf("abcde%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType))
}
for i := 0; i < 30; i++ {
name := fmt.Sprintf("efghi%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType))
}
for i := 0; i < 30; i++ {
name := fmt.Sprintf("jklmn%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType))
}
return resources
@@ -482,7 +264,7 @@ func BenchmarkAssignMetadata(b *testing.B) {
}
for i := 0; i < 20; i++ {
name := fmt.Sprintf("foo%d_%d.css", i%5, i)
- resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
+ resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType))
}
b.StartTimer()
diff --git a/resource/templates/execute_as_template.go b/resource/templates/execute_as_template.go
new file mode 100644
index 000000000..dee9d0d9a
--- /dev/null
+++ b/resource/templates/execute_as_template.go
@@ -0,0 +1,76 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package templates contains functions for template processing of Resource objects.
+package templates
+
+import (
+ "fmt"
+
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/resource"
+ "github.com/gohugoio/hugo/tpl"
+)
+
+// Client contains methods to perform template processing of Resource objects.
+type Client struct {
+ rs *resource.Spec
+
+ textTemplate tpl.TemplateParseFinder
+}
+
+// New creates a new Client with the given specification.
+func New(rs *resource.Spec, textTemplate tpl.TemplateParseFinder) *Client {
+ if rs == nil {
+ panic("must provice a resource Spec")
+ }
+ if textTemplate == nil {
+ panic("must provide a textTemplate")
+ }
+ return &Client{rs: rs, textTemplate: textTemplate}
+}
+
+type executeAsTemplateTransform struct {
+ rs *resource.Spec
+ textTemplate tpl.TemplateParseFinder
+ targetPath string
+ data interface{}
+}
+
+func (t *executeAsTemplateTransform) Key() resource.ResourceTransformationKey {
+ return resource.NewResourceTransformationKey("execute-as-template", t.targetPath)
+}
+
+func (t *executeAsTemplateTransform) Transform(ctx *resource.ResourceTransformationCtx) error {
+ tplStr := helpers.ReaderToString(ctx.From)
+ templ, err := t.textTemplate.Parse(ctx.InPath, tplStr)
+ if err != nil {
+ return fmt.Errorf("failed to parse Resource %q as Template: %s", ctx.InPath, err)
+ }
+
+ ctx.OutPath = t.targetPath
+
+ return templ.Execute(ctx.To, t.data)
+}
+
+func (c *Client) ExecuteAsTemplate(res resource.Resource, targetPath string, data interface{}) (resource.Resource, error) {
+ return c.rs.Transform(
+ res,
+ &executeAsTemplateTransform{
+ rs: c.rs,
+ targetPath: helpers.ToSlashTrimLeading(targetPath),
+ textTemplate: c.textTemplate,
+ data: data,
+ },
+ )
+}
diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go
index 360adc038..e78a536a2 100644
--- a/resource/testhelpers_test.go
+++ b/resource/testhelpers_test.go
@@ -33,7 +33,9 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *
cfg.Set("dataDir", "data")
cfg.Set("i18nDir", "i18n")
cfg.Set("layoutDir", "layouts")
+ cfg.Set("assetDir", "assets")
cfg.Set("archetypeDir", "archetypes")
+ cfg.Set("publishDir", "public")
imagingCfg := map[string]interface{}{
"resampleFilter": "linear",
@@ -49,7 +51,7 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *
assert.NoError(err)
- spec, err := NewSpec(s, media.DefaultTypes)
+ spec, err := NewSpec(s, nil, media.DefaultTypes)
assert.NoError(err)
return spec
}
@@ -72,7 +74,9 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
cfg.Set("dataDir", "data")
cfg.Set("i18nDir", "i18n")
cfg.Set("layoutDir", "layouts")
+ cfg.Set("assetDir", "assets")
cfg.Set("archetypeDir", "archetypes")
+ cfg.Set("publishDir", "public")
fs := hugofs.NewFrom(hugofs.Os, cfg)
fs.Destination = &afero.MemMapFs{}
@@ -81,7 +85,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
assert.NoError(err)
- spec, err := NewSpec(s, media.DefaultTypes)
+ spec, err := NewSpec(s, nil, media.DefaultTypes)
assert.NoError(err)
return spec
@@ -102,12 +106,11 @@ func fetchImageForSpec(spec *Spec, assert *require.Assertions, name string) *Ima
return r.(*Image)
}
-func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) Resource {
+func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) ContentResource {
src, err := os.Open(filepath.FromSlash("testdata/" + name))
assert.NoError(err)
- assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755))
- out, err := spec.BaseFs.ContentFs.Create(name)
+ out, err := openFileForWriting(spec.BaseFs.Content.Fs, name)
assert.NoError(err)
_, err = io.Copy(out, src)
out.Close()
@@ -118,10 +121,10 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R
return path.Join("/a", s)
}
- r, err := spec.NewResourceFromFilename(factory, name, name)
+ r, err := spec.New(ResourceSourceDescriptor{TargetPathBuilder: factory, SourceFilename: name})
assert.NoError(err)
- return r
+ return r.(ContentResource)
}
func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
diff --git a/resource/tocss/scss/client.go b/resource/tocss/scss/client.go
new file mode 100644
index 000000000..610ea3845
--- /dev/null
+++ b/resource/tocss/scss/client.go
@@ -0,0 +1,101 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scss
+
+import (
+ "github.com/bep/go-tocss/scss"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/hugolib/filesystems"
+ "github.com/gohugoio/hugo/resource"
+ "github.com/mitchellh/mapstructure"
+)
+
+type Client struct {
+ rs *resource.Spec
+ sfs *filesystems.SourceFilesystem
+}
+
+func New(fs *filesystems.SourceFilesystem, rs *resource.Spec) (*Client, error) {
+ return &Client{sfs: fs, rs: rs}, nil
+}
+
+type Options struct {
+
+ // Hugo, will by default, just replace the extension of the source
+ // to .css, e.g. "scss/main.scss" becomes "scss/main.css". You can
+ // control this by setting this, e.g. "styles/main.css" will create
+ // a Resource with that as a base for RelPermalink etc.
+ TargetPath string
+
+ // Default is nested.
+ // One of nested, expanded, compact, compressed.
+ OutputStyle string
+
+ // Precision of floating point math.
+ Precision int
+
+ // When enabled, Hugo will generate a source map.
+ EnableSourceMap bool
+}
+
+type options struct {
+ // The options we receive from the end user.
+ from Options
+
+ // The options we send to the SCSS library.
+ to scss.Options
+}
+
+func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) {
+ internalOptions := options{
+ from: opts,
+ }
+
+ // Transfer values from client.
+ internalOptions.to.Precision = opts.Precision
+ internalOptions.to.OutputStyle = scss.OutputStyleFromString(opts.OutputStyle)
+
+ if internalOptions.to.Precision == 0 {
+ // bootstrap-sass requires 8 digits precision. The libsass default is 5.
+ // https://github.com/twbs/bootstrap-sass/blob/master/README.md#sass-number-precision
+ internalOptions.to.Precision = 8
+ }
+
+ return c.rs.Transform(
+ res,
+ &toCSSTransformation{c: c, options: internalOptions},
+ )
+}
+
+type toCSSTransformation struct {
+ c *Client
+ options options
+}
+
+func (t *toCSSTransformation) Key() resource.ResourceTransformationKey {
+ return resource.NewResourceTransformationKey("tocss", t.options.from)
+}
+
+func DecodeOptions(m map[string]interface{}) (opts Options, err error) {
+ if m == nil {
+ return
+ }
+ err = mapstructure.WeakDecode(m, &opts)
+
+ if opts.TargetPath != "" {
+ opts.TargetPath = helpers.ToSlashTrimLeading(opts.TargetPath)
+ }
+
+ return
+}
diff --git a/resource/tocss/scss/tocss.go b/resource/tocss/scss/tocss.go
new file mode 100644
index 000000000..d606e9832
--- /dev/null
+++ b/resource/tocss/scss/tocss.go
@@ -0,0 +1,111 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build extended
+
+package scss
+
+import (
+ "fmt"
+ "io"
+ "path"
+ "strings"
+
+ "github.com/bep/go-tocss/scss"
+ "github.com/bep/go-tocss/scss/libsass"
+ "github.com/bep/go-tocss/tocss"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/gohugoio/hugo/media"
+ "github.com/gohugoio/hugo/resource"
+)
+
+// Used in tests. This feature requires Hugo to be built with the extended tag.
+func Supports() bool {
+ return true
+}
+
+func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error {
+ ctx.OutMediaType = media.CSSType
+
+ var outName string
+ if t.options.from.TargetPath != "" {
+ ctx.OutPath = t.options.from.TargetPath
+ } else {
+ ctx.ReplaceOutPathExtension(".css")
+ }
+
+ outName = path.Base(ctx.OutPath)
+
+ options := t.options
+
+ // We may allow the end user to add IncludePaths later, if we find a use
+ // case for that.
+ options.to.IncludePaths = t.c.sfs.RealDirs(path.Dir(ctx.SourcePath))
+
+ if ctx.InMediaType.SubType == media.SASSType.SubType {
+ options.to.SassSyntax = true
+ }
+
+ if options.from.EnableSourceMap {
+
+ options.to.SourceMapFilename = outName + ".map"
+ options.to.SourceMapRoot = t.c.rs.WorkingDir
+
+ // Setting this to the relative input filename will get the source map
+ // more correct for the main entry path (main.scss typically), but
+ // it will mess up the import mappings. As a workaround, we do a replacement
+ // in the source map itself (see below).
+ //options.InputPath = inputPath
+ options.to.OutputPath = outName
+ options.to.SourceMapContents = true
+ options.to.OmitSourceMapURL = false
+ options.to.EnableEmbeddedSourceMap = false
+ }
+
+ res, err := t.c.toCSS(options.to, ctx.To, ctx.From)
+ if err != nil {
+ return err
+ }
+
+ if options.from.EnableSourceMap && res.SourceMapContent != "" {
+ sourcePath := t.c.sfs.RealFilename(ctx.SourcePath)
+
+ if strings.HasPrefix(sourcePath, t.c.rs.WorkingDir) {
+ sourcePath = strings.TrimPrefix(sourcePath, t.c.rs.WorkingDir+helpers.FilePathSeparator)
+ }
+
+ // This is a workaround for what looks like a bug in Libsass. But
+ // getting this resolution correct in tools like Chrome Workspaces
+ // is important enough to go this extra mile.
+ mapContent := strings.Replace(res.SourceMapContent, `stdin",`, fmt.Sprintf("%s\",", sourcePath), 1)
+
+ return ctx.PublishSourceMap(mapContent)
+ }
+ return nil
+}
+
+func (c *Client) toCSS(options scss.Options, dst io.Writer, src io.Reader) (tocss.Result, error) {
+ var res tocss.Result
+
+ transpiler, err := libsass.New(options)
+ if err != nil {
+ return res, err
+ }
+
+ res, err = transpiler.Execute(dst, src)
+ if err != nil {
+ return res, fmt.Errorf("SCSS processing failed: %s", err)
+ }
+
+ return res, nil
+}
diff --git a/resource/tocss/scss/tocss_notavailable.go b/resource/tocss/scss/tocss_notavailable.go
new file mode 100644
index 000000000..69b4fc655
--- /dev/null
+++ b/resource/tocss/scss/tocss_notavailable.go
@@ -0,0 +1,30 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !extended
+
+package scss
+
+import (
+ "github.com/gohugoio/hugo/common/errors"
+ "github.com/gohugoio/hugo/resource"
+)
+
+// Used in tests.
+func Supports() bool {
+ return false
+}
+
+func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error {
+ return errors.FeatureNotAvailableErr
+}
diff --git a/resource/transform.go b/resource/transform.go
new file mode 100644
index 000000000..6a100ddc4
--- /dev/null
+++ b/resource/transform.go
@@ -0,0 +1,487 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resource
+
+import (
+ "bytes"
+ "path"
+ "strconv"
+
+ "github.com/gohugoio/hugo/common/errors"
+ "github.com/gohugoio/hugo/helpers"
+ "github.com/mitchellh/hashstructure"
+ "github.com/spf13/afero"
+
+ "fmt"
+ "io"
+ "sync"
+
+ "github.com/gohugoio/hugo/media"
+
+ bp "github.com/gohugoio/hugo/bufferpool"
+)
+
+var (
+ _ ContentResource = (*transformedResource)(nil)
+ _ ReadSeekCloserResource = (*transformedResource)(nil)
+)
+
+func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) {
+ return &transformedResource{
+ Resource: r,
+ transformation: t,
+ transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})},
+ cache: s.ResourceCache}, nil
+}
+
+type ResourceTransformationCtx struct {
+ // The content to transform.
+ From io.Reader
+
+ // The target of content transformation.
+ // The current implementation requires that r is written to w
+ // even if no transformation is performed.
+ To io.Writer
+
+ // This is the relative path to the original source. Unix styled slashes.
+ SourcePath string
+
+ // This is the relative target path to the resource. Unix styled slashes.
+ InPath string
+
+ // The relative target path to the transformed resource. Unix styled slashes.
+ OutPath string
+
+ // The input media type
+ InMediaType media.Type
+
+ // The media type of the transformed resource.
+ OutMediaType media.Type
+
+ // Data data can be set on the transformed Resource. Not that this need
+ // to be simple types, as it needs to be serialized to JSON and back.
+ Data map[string]interface{}
+
+ // This is used to publis additional artifacts, e.g. source maps.
+ // We may improve this.
+ OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error)
+}
+
+// AddOutPathIdentifier transforming InPath to OutPath adding an identifier,
+// eg '.min' before any extension.
+func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
+ ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
+}
+
+func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
+ dir, file := path.Split(inPath)
+ base, ext := helpers.PathAndExt(file)
+ return path.Join(dir, (base + identifier + ext))
+}
+
+// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
+// extension, e.g. ".scss"
+func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
+ dir, file := path.Split(ctx.InPath)
+ base, _ := helpers.PathAndExt(file)
+ ctx.OutPath = path.Join(dir, (base + newExt))
+}
+
+// PublishSourceMap writes the content to the target folder of the main resource
+// with the ".map" extension added.
+func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
+ target := ctx.OutPath + ".map"
+ f, err := ctx.OpenResourcePublisher(target)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ _, err = f.Write([]byte(content))
+ return err
+}
+
+// ResourceTransformationKey are provided by the different transformation implementations.
+// It identifies the transformation (name) and its configuration (elements).
+// We combine this in a chain with the rest of the transformations
+// with the target filename and a content hash of the origin to use as cache key.
+type ResourceTransformationKey struct {
+ name string
+ elements []interface{}
+}
+
+// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation
+// name and elements. We will create a 64 bit FNV hash from the elements, which when combined
+// with the other key elements should be unique for all practical applications.
+func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey {
+ return ResourceTransformationKey{name: name, elements: elements}
+}
+
+// Do not change this without good reasons.
+func (k ResourceTransformationKey) key() string {
+ if len(k.elements) == 0 {
+ return k.name
+ }
+
+ sb := bp.GetBuffer()
+ defer bp.PutBuffer(sb)
+
+ sb.WriteString(k.name)
+ for _, element := range k.elements {
+ hash, err := hashstructure.Hash(element, nil)
+ if err != nil {
+ panic(err)
+ }
+ sb.WriteString("_")
+ sb.WriteString(strconv.FormatUint(hash, 10))
+ }
+
+ return sb.String()
+}
+
+// ResourceTransformation is the interface that a resource transformation step
+// needs to implement.
+type ResourceTransformation interface {
+ Key() ResourceTransformationKey
+ Transform(ctx *ResourceTransformationCtx) error
+}
+
+// We will persist this information to disk.
+type transformedResourceMetadata struct {
+ Target string `json:"Target"`
+ MediaTypeV string `json:"MediaType"`
+ MetaData map[string]interface{} `json:"Data"`
+}
+
+type transformedResource struct {
+ cache *ResourceCache
+
+ // This is the filename inside resources/_gen/assets
+ sourceFilename string
+
+ linker permalinker
+
+ // The transformation to apply.
+ transformation ResourceTransformation
+
+ // We apply the tranformations lazily.
+ transformInit sync.Once
+ transformErr error
+
+ // The transformed values
+ content string
+ contentInit sync.Once
+ transformedResourceMetadata
+
+ // The source
+ Resource
+}
+
+func (r *transformedResource) ReadSeekCloser() (ReadSeekCloser, error) {
+ rc, ok := r.Resource.(ReadSeekCloserResource)
+ if !ok {
+ return nil, fmt.Errorf("resource %T is not a ReadSeekerCloserResource", rc)
+ }
+ return rc.ReadSeekCloser()
+}
+
+func (r *transformedResource) transferTransformedValues(another *transformedResource) {
+ if another.content != "" {
+ r.contentInit.Do(func() {
+ r.content = another.content
+ })
+ }
+ r.transformedResourceMetadata = another.transformedResourceMetadata
+}
+
+func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser {
+ f, meta, found := r.cache.getFromFile(key)
+ if !found {
+ return nil
+ }
+ r.transformedResourceMetadata = meta
+ r.sourceFilename = f.Name()
+
+ return f
+}
+
+func (r *transformedResource) Content() (interface{}, error) {
+ if err := r.initTransform(true); err != nil {
+ return nil, err
+ }
+ if err := r.initContent(); err != nil {
+ return "", err
+ }
+ return r.content, nil
+}
+
+func (r *transformedResource) Data() interface{} {
+ return r.MetaData
+}
+
+func (r *transformedResource) MediaType() media.Type {
+ if err := r.initTransform(false); err != nil {
+ return media.Type{}
+ }
+ m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV)
+ return m
+}
+
+func (r *transformedResource) Permalink() string {
+ if err := r.initTransform(false); err != nil {
+ return ""
+ }
+ return r.linker.permalinkFor(r.Target)
+}
+
+func (r *transformedResource) RelPermalink() string {
+ if err := r.initTransform(false); err != nil {
+ return ""
+ }
+ return r.linker.relPermalinkFor(r.Target)
+}
+
+func (r *transformedResource) initContent() error {
+ var err error
+ r.contentInit.Do(func() {
+ var b []byte
+ b, err := afero.ReadFile(r.cache.rs.Resources.Fs, r.sourceFilename)
+ if err != nil {
+ return
+ }
+ r.content = string(b)
+ })
+ return err
+}
+
+func (r *transformedResource) transform(setContent bool) (err error) {
+
+ openPublishFileForWriting := func(relTargetPath string) (io.WriteCloser, error) {
+ return openFileForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathFor(relTargetPath))
+ }
+
+ // This can be the last resource in a chain.
+ // Rewind and create a processing chain.
+ var chain []Resource
+ current := r
+ for {
+ rr := current.Resource
+ chain = append(chain[:0], append([]Resource{rr}, chain[0:]...)...)
+ if tr, ok := rr.(*transformedResource); ok {
+ current = tr
+ } else {
+ break
+ }
+ }
+
+ // Append the current transformer at the end
+ chain = append(chain, r)
+
+ first := chain[0]
+
+ contentrc, err := contentReadSeekerCloser(first)
+ if err != nil {
+ return err
+ }
+ defer contentrc.Close()
+
+ // Files with a suffix will be stored in cache (both on disk and in memory)
+ // partitioned by their suffix. There will be other files below /other.
+ // This partition is also how we determine what to delete on server reloads.
+ var key, base string
+ for _, element := range chain {
+ switch v := element.(type) {
+ case *transformedResource:
+ key = key + "_" + v.transformation.Key().key()
+ case permalinker:
+ r.linker = v
+ p := v.relTargetPath()
+ if p == "" {
+ panic("target path needed for key creation")
+ }
+ partition := ResourceKeyPartition(p)
+ base = partition + "/" + p
+ default:
+ return fmt.Errorf("transformation not supported for type %T", element)
+ }
+ }
+
+ key = r.cache.cleanKey(base + "_" + helpers.MD5String(key))
+
+ cached, found := r.cache.get(key)
+ if found {
+ r.transferTransformedValues(cached.(*transformedResource))
+ return
+ }
+
+ // Acquire a write lock for the named transformation.
+ r.cache.nlocker.Lock(key)
+ // Check the cache again.
+ cached, found = r.cache.get(key)
+ if found {
+ r.transferTransformedValues(cached.(*transformedResource))
+ r.cache.nlocker.Unlock(key)
+ return
+ }
+ defer r.cache.nlocker.Unlock(key)
+ defer r.cache.set(key, r)
+
+ b1 := bp.GetBuffer()
+ b2 := bp.GetBuffer()
+ defer bp.PutBuffer(b1)
+ defer bp.PutBuffer(b2)
+
+ tctx := &ResourceTransformationCtx{
+ Data: r.transformedResourceMetadata.MetaData,
+ OpenResourcePublisher: openPublishFileForWriting,
+ }
+
+ tctx.InMediaType = first.MediaType()
+ tctx.OutMediaType = first.MediaType()
+ tctx.From = contentrc
+ tctx.To = b1
+
+ if r.linker != nil {
+ tctx.InPath = r.linker.targetPath()
+ tctx.SourcePath = tctx.InPath
+ }
+
+ counter := 0
+
+ var transformedContentr io.Reader
+
+ for _, element := range chain {
+ tr, ok := element.(*transformedResource)
+ if !ok {
+ continue
+ }
+ counter++
+ if counter != 1 {
+ tctx.InMediaType = tctx.OutMediaType
+ }
+ if counter%2 == 0 {
+ tctx.From = b1
+ b2.Reset()
+ tctx.To = b2
+ } else {
+ if counter != 1 {
+ // The first reader is the file.
+ tctx.From = b2
+ }
+ b1.Reset()
+ tctx.To = b1
+ }
+
+ if err := tr.transformation.Transform(tctx); err != nil {
+ if err == errors.FeatureNotAvailableErr {
+ // This transformation is not available in this
+ // Hugo installation (scss not compiled in, PostCSS not available etc.)
+ // If a prepared bundle for this transformation chain is available, use that.
+ f := r.tryTransformedFileCache(key)
+ if f == nil {
+ return fmt.Errorf("failed to transform %q (%s): %s", tctx.InPath, tctx.InMediaType.Type(), err)
+ }
+ transformedContentr = f
+ defer f.Close()
+
+ // The reader above is all we need.
+ break
+ }
+
+ // Abort.
+ return err
+ }
+
+ if tctx.OutPath != "" {
+ tctx.InPath = tctx.OutPath
+ tctx.OutPath = ""
+ }
+ }
+
+ if transformedContentr == nil {
+ r.Target = tctx.InPath
+ r.MediaTypeV = tctx.OutMediaType.Type()
+ }
+
+ publicw, err := openPublishFileForWriting(r.Target)
+ if err != nil {
+ r.transformErr = err
+ return
+ }
+ defer publicw.Close()
+
+ publishwriters := []io.Writer{publicw}
+
+ if transformedContentr == nil {
+ // Also write it to the cache
+ metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata)
+ if err != nil {
+ return err
+ }
+ r.sourceFilename = metaw.Name()
+ defer metaw.Close()
+
+ publishwriters = append(publishwriters, metaw)
+
+ if counter > 0 {
+ transformedContentr = tctx.To.(*bytes.Buffer)
+ } else {
+ transformedContentr = contentrc
+ }
+ }
+
+ // Also write it to memory
+ var contentmemw *bytes.Buffer
+
+ if setContent {
+ contentmemw = bp.GetBuffer()
+ defer bp.PutBuffer(contentmemw)
+ publishwriters = append(publishwriters, contentmemw)
+ }
+
+ publishw := io.MultiWriter(publishwriters...)
+ _, r.transformErr = io.Copy(publishw, transformedContentr)
+
+ if setContent {
+ r.contentInit.Do(func() {
+ r.content = contentmemw.String()
+ })
+ }
+
+ return nil
+
+}
+func (r *transformedResource) initTransform(setContent bool) error {
+ r.transformInit.Do(func() {
+ if err := r.transform(setContent); err != nil {
+ r.transformErr = err
+ r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err)
+ }
+ })
+ return r.transformErr
+}
+
+// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
+func contentReadSeekerCloser(r Resource) (ReadSeekCloser, error) {
+ switch rr := r.(type) {
+ case ReadSeekCloserResource:
+ rc, err := rr.ReadSeekCloser()
+ if err != nil {
+ return nil, err
+ }
+ return rc, nil
+ default:
+ return nil, fmt.Errorf("cannot tranform content of Resource of type %T", r)
+
+ }
+}
diff --git a/resource/transform_test.go b/resource/transform_test.go
new file mode 100644
index 000000000..df68e780d
--- /dev/null
+++ b/resource/transform_test.go
@@ -0,0 +1,36 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resource
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+type testStruct struct {
+ Name string
+ V1 int64
+ V2 int32
+ V3 int
+ V4 uint64
+}
+
+func TestResourceTransformationKey(t *testing.T) {
+ // We really need this key to be portable across OSes.
+ key := NewResourceTransformationKey("testing",
+ testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)})
+ assert := require.New(t)
+ assert.Equal(key.key(), "testing_518996646957295636")
+}