diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-02-20 12:02:14 +0300 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-07-06 12:46:12 +0300 |
commit | dea71670c059ab4d5a42bd22503f18c087dd22d4 (patch) | |
tree | 52889fd27a2d316fad5a04c0f2fe2198491c6cd1 /resource | |
parent | a5d0a57e6bdab583134a68c035aac9b3007f006a (diff) |
Add Hugo Piper with SCSS support and much more
Before this commit, you would have to use page bundles to do image processing etc. in Hugo.
This commit adds
* A new `/assets` top-level project or theme dir (configurable via `assetDir`)
* A new template func, `resources.Get` which can be used to "get a resource" that can be further processed.
This means that you can now do this in your templates (or shortcodes):
```bash
{{ $sunset := (resources.Get "images/sunset.jpg").Fill "300x200" }}
```
This also adds a new `extended` build tag that enables powerful SCSS/SASS support with source maps. To compile this from source, you will also need a C compiler installed:
```
HUGO_BUILD_TAGS=extended mage install
```
Note that you can use output of the SCSS processing later in a non-SCSSS-enabled Hugo.
The `SCSS` processor is a _Resource transformation step_ and it can be chained with the many others in a pipeline:
```bash
{{ $css := resources.Get "styles.scss" | resources.ToCSS | resources.PostCSS | resources.Minify | resources.Fingerprint }}
<link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen">
```
The transformation funcs above have aliases, so it can be shortened to:
```bash
{{ $css := resources.Get "styles.scss" | toCSS | postCSS | minify | fingerprint }}
<link rel="stylesheet" href="{{ $styles.RelPermalink }}" integrity="{{ $styles.Data.Digest }}" media="screen">
```
A quick tip would be to avoid the fingerprinting part, and possibly also the not-superfast `postCSS` when you're doing development, as it allows Hugo to be smarter about the rebuilding.
Documentation will follow, but have a look at the demo repo in https://github.com/bep/hugo-sass-test
New functions to create `Resource` objects:
* `resources.Get` (see above)
* `resources.FromString`: Create a Resource from a string.
New `Resource` transformation funcs:
* `resources.ToCSS`: Compile `SCSS` or `SASS` into `CSS`.
* `resources.PostCSS`: Process your CSS with PostCSS. Config file support (project or theme or passed as an option).
* `resources.Minify`: Currently supports `css`, `js`, `json`, `html`, `svg`, `xml`.
* `resources.Fingerprint`: Creates a fingerprinted version of the given Resource with Subresource Integrity..
* `resources.Concat`: Concatenates a list of Resource objects. Think of this as a poor man's bundler.
* `resources.ExecuteAsTemplate`: Parses and executes the given Resource and data context (e.g. .Site) as a Go template.
Fixes #4381
Fixes #4903
Fixes #4858
Diffstat (limited to 'resource')
-rw-r--r-- | resource/bundler/bundler.go | 121 | ||||
-rw-r--r-- | resource/create/create.go | 77 | ||||
-rw-r--r-- | resource/image.go | 70 | ||||
-rw-r--r-- | resource/image_cache.go | 16 | ||||
-rw-r--r-- | resource/image_test.go | 18 | ||||
-rw-r--r-- | resource/integrity/integrity.go | 106 | ||||
-rw-r--r-- | resource/integrity/integrity_test.go | 54 | ||||
-rw-r--r-- | resource/minifiers/minify.go | 115 | ||||
-rw-r--r-- | resource/postcss/postcss.go | 175 | ||||
-rw-r--r-- | resource/resource.go | 691 | ||||
-rw-r--r-- | resource/resource_cache.go | 241 | ||||
-rw-r--r-- | resource/resource_metadata.go | 129 | ||||
-rw-r--r-- | resource/resource_metadata_test.go | 230 | ||||
-rw-r--r-- | resource/resource_test.go | 312 | ||||
-rw-r--r-- | resource/templates/execute_as_template.go | 76 | ||||
-rw-r--r-- | resource/testhelpers_test.go | 17 | ||||
-rw-r--r-- | resource/tocss/scss/client.go | 101 | ||||
-rw-r--r-- | resource/tocss/scss/tocss.go | 111 | ||||
-rw-r--r-- | resource/tocss/scss/tocss_notavailable.go | 30 | ||||
-rw-r--r-- | resource/transform.go | 487 | ||||
-rw-r--r-- | resource/transform_test.go | 36 |
21 files changed, 2618 insertions, 595 deletions
diff --git a/resource/bundler/bundler.go b/resource/bundler/bundler.go new file mode 100644 index 000000000..2f3981485 --- /dev/null +++ b/resource/bundler/bundler.go @@ -0,0 +1,121 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bundler contains functions for concatenation etc. of Resource objects. +package bundler + +import ( + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods perform concatenation and other bundling related +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type multiReadSeekCloser struct { + mr io.Reader + sources []resource.ReadSeekCloser +} + +func (r *multiReadSeekCloser) Read(p []byte) (n int, err error) { + return r.mr.Read(p) +} + +func (r *multiReadSeekCloser) Seek(offset int64, whence int) (newOffset int64, err error) { + for _, s := range r.sources { + newOffset, err = s.Seek(offset, whence) + if err != nil { + return + } + } + return +} + +func (r *multiReadSeekCloser) Close() error { + for _, s := range r.sources { + s.Close() + } + return nil +} + +// Concat concatenates the list of Resource objects. +func (c *Client) Concat(targetPath string, resources []resource.Resource) (resource.Resource, error) { + // The CACHE_OTHER will make sure this will be re-created and published on rebuilds. + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + var resolvedm media.Type + + // The given set of resources must be of the same Media Type. + // We may improve on that in the future, but then we need to know more. + for i, r := range resources { + if i > 0 && r.MediaType() != resolvedm { + return nil, errors.New("resources in Concat must be of the same Media Type") + } + resolvedm = r.MediaType() + } + + concatr := func() (resource.ReadSeekCloser, error) { + var rcsources []resource.ReadSeekCloser + for _, s := range resources { + rcr, ok := s.(resource.ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T does not implement resource.ReadSeekerCloserResource", s) + } + rc, err := rcr.ReadSeekCloser() + if err != nil { + // Close the already opened. + for _, rcs := range rcsources { + rcs.Close() + } + return nil, err + } + rcsources = append(rcsources, rc) + } + + readers := make([]io.Reader, len(rcsources)) + for i := 0; i < len(rcsources); i++ { + readers[i] = rcsources[i] + } + + mr := io.MultiReader(readers...) + + return &multiReadSeekCloser{mr: mr, sources: rcsources}, nil + } + + composite, err := c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: concatr, + RelTargetFilename: filepath.Clean(targetPath)}) + + if err != nil { + return nil, err + } + + return composite, nil + }) + +} diff --git a/resource/create/create.go b/resource/create/create.go new file mode 100644 index 000000000..1c7894232 --- /dev/null +++ b/resource/create/create.go @@ -0,0 +1,77 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package create contains functions for to create Resource objects. This will +// typically non-files. +package create + +import ( + "io" + "path/filepath" + + "github.com/spf13/afero" + + "github.com/dsnet/golib/memfile" + "github.com/gohugoio/hugo/resource" +) + +// Client contains methods to create Resource objects. +// tasks to Resource objects. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type memFileCloser struct { + *memfile.File + io.Closer +} + +func (m *memFileCloser) Close() error { + return nil +} + +// Get creates a new Resource by opening the given filename in the given filesystem. +func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { + filename = filepath.Clean(filename) + return c.rs.ResourceCache.GetOrCreate(resource.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { + return c.rs.NewForFs(fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + SourceFilename: filename}) + + }) + +} + +// FromString creates a new Resource from a string with the given relative target path. +func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { + return c.rs.ResourceCache.GetOrCreate(resource.CACHE_OTHER, targetPath, func() (resource.Resource, error) { + return c.rs.NewForFs( + c.rs.BaseFs.Resources.Fs, + resource.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (resource.ReadSeekCloser, error) { + return &memFileCloser{ + File: memfile.New([]byte(content)), + }, nil + }, + RelTargetFilename: filepath.Clean(targetPath)}) + + }) + +} diff --git a/resource/image.go b/resource/image.go index 19b68a296..6aa382331 100644 --- a/resource/image.go +++ b/resource/image.go @@ -19,14 +19,12 @@ import ( "image/color" "io" "os" - "path/filepath" "strconv" "strings" "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/afero" // Importing image codecs for image.DecodeConfig "image" @@ -132,8 +130,6 @@ type Image struct { format imaging.Format - hash string - *genericResource } @@ -151,7 +147,6 @@ func (i *Image) Height() int { func (i *Image) WithNewBase(base string) Resource { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: i.genericResource.WithNewBase(base).(*genericResource)} } @@ -209,7 +204,7 @@ type imageConfig struct { } func (i *Image) isJPEG() bool { - name := strings.ToLower(i.relTargetPath.file) + name := strings.ToLower(i.relTargetDirFile.file) return strings.HasSuffix(name, ".jpg") || strings.HasSuffix(name, ".jpeg") } @@ -241,7 +236,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci := i.clone() errOp := action - errPath := i.AbsSourceFilename() + errPath := i.sourceFilename ci.setBasePath(conf) @@ -273,7 +268,7 @@ func (i *Image) doWithImageConfig(action, spec string, f func(src image.Image, c ci.config = image.Config{Width: b.Max.X, Height: b.Max.Y} ci.configLoaded = true - return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.target()) + return ci, i.encodeToDestinations(converted, conf, resourceCacheFilename, ci.targetFilename()) }) } @@ -415,11 +410,11 @@ func (i *Image) initConfig() error { } var ( - f afero.File + f ReadSeekCloser config image.Config ) - f, err = i.sourceFs().Open(i.AbsSourceFilename()) + f, err = i.ReadSeekCloser() if err != nil { return } @@ -440,19 +435,19 @@ func (i *Image) initConfig() error { } func (i *Image) decodeSource() (image.Image, error) { - file, err := i.sourceFs().Open(i.AbsSourceFilename()) + f, err := i.ReadSeekCloser() if err != nil { return nil, fmt.Errorf("failed to open image for decode: %s", err) } - defer file.Close() - img, _, err := image.Decode(file) + defer f.Close() + img, _, err := image.Decode(f) return img, err } func (i *Image) copyToDestination(src string) error { var res error i.copyToDestinationInit.Do(func() { - target := i.target() + target := i.targetFilename() // Fast path: // This is a processed version of the original. @@ -469,20 +464,9 @@ func (i *Image) copyToDestination(src string) error { } defer in.Close() - out, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - res = err - return - } - out, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - res = err - return - } - } else if err != nil { + out, err := openFileForWriting(i.spec.BaseFs.PublishFs, target) + + if err != nil { res = err return } @@ -501,21 +485,10 @@ func (i *Image) copyToDestination(src string) error { return nil } -func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error { - target := filepath.Clean(filename) +func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, targetFilename string) error { - file1, err := i.spec.BaseFs.PublishFs.Create(target) - if err != nil && os.IsNotExist(err) { - // When called from shortcodes, the target directory may not exist yet. - // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { - return err - } - file1, err = i.spec.BaseFs.PublishFs.Create(target) - if err != nil { - return err - } - } else if err != nil { + file1, err := openFileForWriting(i.spec.BaseFs.PublishFs, targetFilename) + if err != nil { return err } @@ -525,11 +498,7 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource if resourceCacheFilename != "" { // Also save it to the image resource cache for later reuse. - if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil { - return err - } - - file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename) + file2, err := openFileForWriting(i.spec.BaseFs.Resources.Fs, resourceCacheFilename) if err != nil { return err } @@ -572,17 +541,16 @@ func (i *Image) clone() *Image { return &Image{ imaging: i.imaging, - hash: i.hash, format: i.format, genericResource: &g} } func (i *Image) setBasePath(conf imageConfig) { - i.relTargetPath = i.relTargetPathFromConfig(conf) + i.relTargetDirFile = i.relTargetPathFromConfig(conf) } func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { - p1, p2 := helpers.FileAndExt(i.relTargetPath.file) + p1, p2 := helpers.FileAndExt(i.relTargetDirFile.file) idStr := fmt.Sprintf("_hu%s_%d", i.hash, i.osFileInfo.Size()) @@ -611,7 +579,7 @@ func (i *Image) relTargetPathFromConfig(conf imageConfig) dirFile { } return dirFile{ - dir: i.relTargetPath.dir, + dir: i.relTargetDirFile.dir, file: fmt.Sprintf("%s%s_%s%s", p1, idStr, key, p2), } diff --git a/resource/image_cache.go b/resource/image_cache.go index 5985797d6..4fb45c17f 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -60,12 +60,6 @@ func (c *imageCache) getOrCreate( relTarget := parent.relTargetPathFromConfig(conf) key := parent.relTargetPathForRel(relTarget.path(), false) - if c.pathSpec.Language != nil { - // Avoid do and store more work than needed. The language versions will in - // most cases be duplicates of the same image files. - key = strings.TrimPrefix(key, "/"+c.pathSpec.Language.Lang) - } - // First check the in-memory store, then the disk. c.mu.RLock() img, found := c.store[key] @@ -88,17 +82,17 @@ func (c *imageCache) getOrCreate( // but the count of processed image variations for this site. c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs) + exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.Resources.Fs) if err != nil { return nil, err } if exists { img = parent.clone() - img.relTargetPath.file = relTarget.file + img.relTargetDirFile.file = relTarget.file img.sourceFilename = cacheFilename - // We have to look resources file system for this. - img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs + // We have to look in the resources file system for this. + img.overriddenSourceFs = img.spec.BaseFs.Resources.Fs } else { img, err = create(cacheFilename) if err != nil { diff --git a/resource/image_test.go b/resource/image_test.go index 11807d695..f4d91bd99 100644 --- a/resource/image_test.go +++ b/resource/image_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -78,19 +78,19 @@ func TestImageTransformBasic(t *testing.T) { assert.NoError(err) assert.Equal(320, resized0x.Width()) assert.Equal(200, resized0x.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") assert.NoError(err) assert.Equal(200, resizedx0.Width()) assert.Equal(125, resizedx0.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") assert.NoError(err) assert.Equal(125, resizedAndRotated.Width()) assert.Equal(200, resizedAndRotated.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAndRotated.RelPermalink(), 125, 200) assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink()) assert.Equal(300, resized.Width()) @@ -115,20 +115,20 @@ func TestImageTransformBasic(t *testing.T) { assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink()) assert.Equal(200, filled.Width()) assert.Equal(100, filled.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") assert.NoError(err) assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink()) assert.Equal(200, smart.Width()) assert.Equal(100, smart.Height()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") assert.NoError(err) assert.True(filled == filledAgain) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, filledAgain.RelPermalink(), 200, 100) } @@ -298,7 +298,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink()) assert.Equal(101, resized.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resized.RelPermalink(), 101, 101) publishedImageFilename := filepath.Clean(resized.RelPermalink()) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename)) @@ -310,7 +310,7 @@ func TestImageResizeInSubPath(t *testing.T) { assert.NoError(err) assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink()) assert.Equal(101, resizedAgain.Width()) - assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101) + assertFileCache(assert, image.spec.BaseFs.Resources.Fs, resizedAgain.RelPermalink(), 101, 101) assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) } diff --git a/resource/integrity/integrity.go b/resource/integrity/integrity.go new file mode 100644 index 000000000..8b4a5a263 --- /dev/null +++ b/resource/integrity/integrity.go @@ -0,0 +1,106 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "crypto/md5" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "fmt" + "hash" + "io" + + "github.com/gohugoio/hugo/resource" +) + +const defaultHashAlgo = "sha256" + +// Client contains methods to fingerprint (cachebusting) and other integrity-related +// methods. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type fingerprintTransformation struct { + algo string +} + +func (t *fingerprintTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("fingerprint", t.algo) +} + +// Transform creates a MD5 hash of the Resource content and inserts that hash before +// the extension in the filename. +func (t *fingerprintTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + algo := t.algo + + var h hash.Hash + + switch algo { + case "md5": + h = md5.New() + case "sha256": + h = sha256.New() + case "sha512": + h = sha512.New() + default: + return fmt.Errorf("unsupported crypto algo: %q, use either md5, sha256 or sha512", algo) + } + + io.Copy(io.MultiWriter(h, ctx.To), ctx.From) + d, err := digest(h) + if err != nil { + return err + } + + ctx.Data["Integrity"] = integrity(algo, d) + ctx.AddOutPathIdentifier("." + hex.EncodeToString(d[:])) + return nil +} + +// Fingerprint applies fingerprinting of the given resource and hash algorithm. +// It defaults to sha256 if none given, and the options are md5, sha256 or sha512. +// The same algo is used for both the fingerprinting part (aka cache busting) and +// the base64-encoded Subresource Integrity hash, so you will have to stay away from +// md5 if you plan to use both. +// See https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity +func (c *Client) Fingerprint(res resource.Resource, algo string) (resource.Resource, error) { + if algo == "" { + algo = defaultHashAlgo + } + + return c.rs.Transform( + res, + &fingerprintTransformation{algo: algo}, + ) +} + +func integrity(algo string, sum []byte) string { + encoded := base64.StdEncoding.EncodeToString(sum) + return fmt.Sprintf("%s-%s", algo, encoded) + +} + +func digest(h hash.Hash) ([]byte, error) { + sum := h.Sum(nil) + //enc := hex.EncodeToString(sum[:]) + return sum, nil +} diff --git a/resource/integrity/integrity_test.go b/resource/integrity/integrity_test.go new file mode 100644 index 000000000..602db4e38 --- /dev/null +++ b/resource/integrity/integrity_test.go @@ -0,0 +1,54 @@ +// Copyright 2018-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integrity + +import ( + "github.com/gohugoio/hugo/media" +) + +type testResource struct { + content string +} + +func (r testResource) Permalink() string { + panic("not implemented") +} + +func (r testResource) RelPermalink() string { + panic("not implemented") +} + +func (r testResource) ResourceType() string { + panic("not implemented") +} + +func (r testResource) Name() string { + panic("not implemented") +} + +func (r testResource) MediaType() media.Type { + panic("not implemented") +} + +func (r testResource) Title() string { + panic("not implemented") +} + +func (r testResource) Params() map[string]interface{} { + panic("not implemented") +} + +func (r testResource) Bytes() ([]byte, error) { + return []byte(r.content), nil +} diff --git a/resource/minifiers/minify.go b/resource/minifiers/minify.go new file mode 100644 index 000000000..609b9a694 --- /dev/null +++ b/resource/minifiers/minify.go @@ -0,0 +1,115 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifiers + +import ( + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/resource" + "github.com/tdewolff/minify" + "github.com/tdewolff/minify/css" + "github.com/tdewolff/minify/html" + "github.com/tdewolff/minify/js" + "github.com/tdewolff/minify/json" + "github.com/tdewolff/minify/svg" + "github.com/tdewolff/minify/xml" +) + +// Client for minification of Resource objects. Supported minfiers are: +// css, html, js, json, svg and xml. +type Client struct { + rs *resource.Spec + m *minify.M +} + +// New creates a new Client given a specification. Note that it is the media types +// configured for the site that is used to match files to the correct minifier. +func New(rs *resource.Spec) *Client { + m := minify.New() + mt := rs.MediaTypes + + // We use the Type definition of the media types defined in the site if found. + addMinifierFunc(m, mt, "text/css", "css", css.Minify) + addMinifierFunc(m, mt, "text/html", "html", html.Minify) + addMinifierFunc(m, mt, "application/javascript", "js", js.Minify) + addMinifierFunc(m, mt, "application/json", "json", json.Minify) + addMinifierFunc(m, mt, "image/svg", "xml", svg.Minify) + addMinifierFunc(m, mt, "application/xml", "xml", xml.Minify) + addMinifierFunc(m, mt, "application/rss", "xml", xml.Minify) + + return &Client{rs: rs, m: m} +} + +func addMinifierFunc(m *minify.M, mt media.Types, typeString, suffix string, fn minify.MinifierFunc) { + resolvedTypeStr := resolveMediaTypeString(mt, typeString, suffix) + m.AddFunc(resolvedTypeStr, fn) + if resolvedTypeStr != typeString { + m.AddFunc(typeString, fn) + } +} + +type minifyTransformation struct { + rs *resource.Spec + m *minify.M +} + +func (t *minifyTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("minify") +} + +func (t *minifyTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + mtype := resolveMediaTypeString( + t.rs.MediaTypes, + ctx.InMediaType.Type(), + helpers.ExtNoDelimiter(ctx.InPath), + ) + if err := t.m.Minify(mtype, ctx.To, ctx.From); err != nil { + return err + } + ctx.AddOutPathIdentifier(".min") + return nil +} + +func (c *Client) Minify(res resource.Resource) (resource.Resource, error) { + return c.rs.Transform( + res, + &minifyTransformation{ + rs: c.rs, + m: c.m}, + ) +} + +func resolveMediaTypeString(types media.Types, typeStr, suffix string) string { + if m, found := resolveMediaType(types, typeStr, suffix); found { + return m.Type() + } + // Fall back to the default. + return typeStr +} + +// Make sure we match the matching pattern with what the user have actually defined +// in his or hers media types configuration. +func resolveMediaType(types media.Types, typeStr, suffix string) (media.Type, bool) { + if m, found := types.GetByType(typeStr); found { + return m, true + } + + if m, found := types.GetFirstBySuffix(suffix); found { + return m, true + } + + return media.Type{}, false + +} diff --git a/resource/postcss/postcss.go b/resource/postcss/postcss.go new file mode 100644 index 000000000..7dd27b2f9 --- /dev/null +++ b/resource/postcss/postcss.go @@ -0,0 +1,175 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postcss + +import ( + "fmt" + "io" + "path/filepath" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/mitchellh/mapstructure" + // "io/ioutil" + "os" + "os/exec" + + "github.com/gohugoio/hugo/common/errors" + + "github.com/gohugoio/hugo/resource" +) + +// Some of the options from https://github.com/postcss/postcss-cli +type Options struct { + + // Set a custom path to look for a config file. + Config string + + NoMap bool `mapstructure:"no-map"` // Disable the default inline sourcemaps + + // Options for when not using a config file + Use string // List of postcss plugins to use + Parser string // Custom postcss parser + Stringifier string // Custom postcss stringifier + Syntax string // Custom postcss syntax +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + return +} + +func (opts Options) toArgs() []string { + var args []string + if opts.NoMap { + args = append(args, "--no-map") + } + if opts.Use != "" { + args = append(args, "--use", opts.Use) + } + if opts.Parser != "" { + args = append(args, "--parser", opts.Parser) + } + if opts.Stringifier != "" { + args = append(args, "--stringifier", opts.Stringifier) + } + if opts.Syntax != "" { + args = append(args, "--syntax", opts.Syntax) + } + return args +} + +// Client is the client used to do PostCSS transformations. +type Client struct { + rs *resource.Spec +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec) *Client { + return &Client{rs: rs} +} + +type postcssTransformation struct { + options Options + rs *resource.Spec +} + +func (t *postcssTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("postcss", t.options) +} + +// Transform shells out to postcss-cli to do the heavy lifting. +// For this to work, you need some additional tools. To install them globally: +// npm install -g postcss-cli +// npm install -g autoprefixer +func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + + const binary = "postcss" + + if _, err := exec.LookPath(binary); err != nil { + // This may be on a CI server etc. Will fall back to pre-built assets. + return errors.FeatureNotAvailableErr + } + + var configFile string + logger := t.rs.Logger + + if t.options.Config != "" { + configFile = t.options.Config + } else { + configFile = "postcss.config.js" + } + + configFile = filepath.Clean(configFile) + + // We need an abolute filename to the config file. + if !filepath.IsAbs(configFile) { + // We resolve this against the virtual Work filesystem, to allow + // this config file to live in one of the themes if needed. + fi, err := t.rs.BaseFs.Work.Fs.Stat(configFile) + if err != nil { + if t.options.Config != "" { + // Only fail if the user specificed config file is not found. + return fmt.Errorf("postcss config %q not found: %s", configFile, err) + } + configFile = "" + } else { + configFile = fi.(hugofs.RealFilenameInfo).RealFilename() + } + } + + var cmdArgs []string + + if configFile != "" { + logger.INFO.Println("postcss: use config file", configFile) + cmdArgs = []string{"--config", configFile} + } + + if optArgs := t.options.toArgs(); len(optArgs) > 0 { + cmdArgs = append(cmdArgs, optArgs...) + } + + cmd := exec.Command(binary, cmdArgs...) + + cmd.Stdout = ctx.To + cmd.Stderr = os.Stderr + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, ctx.From) + }() + + err = cmd.Run() + if err != nil { + return err + } + + return nil +} + +// Process transforms the given Resource with the PostCSS processor. +func (c *Client) Process(res resource.Resource, options Options) (resource.Resource, error) { + return c.rs.Transform( + res, + &postcssTransformation{rs: c.rs, options: options}, + ) +} diff --git a/resource/resource.go b/resource/resource.go index 9a3725f8a..f0989e51e 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,20 +14,24 @@ package resource import ( + "errors" "fmt" + "io" + "io/ioutil" "mime" "os" "path" "path/filepath" - "strconv" "strings" "sync" - "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/tpl" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/common/loggers" + + jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/cast" + "github.com/spf13/afero" "github.com/gobwas/glob" "github.com/gohugoio/hugo/helpers" @@ -36,34 +40,39 @@ import ( ) var ( + _ ContentResource = (*genericResource)(nil) + _ ReadSeekCloserResource = (*genericResource)(nil) _ Resource = (*genericResource)(nil) - _ metaAssigner = (*genericResource)(nil) _ Source = (*genericResource)(nil) _ Cloner = (*genericResource)(nil) _ ResourcesLanguageMerger = (*Resources)(nil) + _ permalinker = (*genericResource)(nil) ) const DefaultResourceType = "unknown" +var noData = make(map[string]interface{}) + // Source is an internal template and not meant for use in the templates. It // may change without notice. type Source interface { - AbsSourceFilename() string Publish() error } +type permalinker interface { + relPermalinkFor(target string) string + permalinkFor(target string) string + relTargetPathFor(target string) string + relTargetPath() string + targetPath() string +} + // Cloner is an internal template and not meant for use in the templates. It // may change without notice. type Cloner interface { WithNewBase(base string) Resource } -type metaAssigner interface { - setTitle(title string) - setName(name string) - updateParams(params map[string]interface{}) -} - // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { // Permalink represents the absolute link to this resource. @@ -77,6 +86,9 @@ type Resource interface { // For content pages, this value is "page". ResourceType() string + // MediaType is this resource's MIME type. + MediaType() media.Type + // Name is the logical name of this resource. This can be set in the front matter // metadata for this resource. If not set, Hugo will assign a value. // This will in most cases be the base filename. @@ -88,8 +100,30 @@ type Resource interface { // Title returns the title if set in front matter. For content pages, this will be the expected value. Title() string + // Resource specific data set by Hugo. + // One example would be.Data.Digest for fingerprinted resources. + Data() interface{} + // Params set in front matter for this resource. Params() map[string]interface{} +} + +type ResourcesLanguageMerger interface { + MergeByLanguage(other Resources) Resources + // Needed for integration with the tpl package. + MergeByLanguageInterface(other interface{}) (interface{}, error) +} + +type translatedResource interface { + TranslationKey() string +} + +// ContentResource represents a Resource that provides a way to get to its content. +// Most Resource types in Hugo implements this interface, including Page. +// This should be used with care, as it will read the file content into memory, but it +// should be cached as effectively as possible by the implementation. +type ContentResource interface { + Resource // Content returns this resource's content. It will be equivalent to reading the content // that RelPermalink points to in the published folder. @@ -100,14 +134,22 @@ type Resource interface { Content() (interface{}, error) } -type ResourcesLanguageMerger interface { - MergeByLanguage(other Resources) Resources - // Needed for integration with the tpl package. - MergeByLanguageInterface(other interface{}) (interface{}, error) +// ReadSeekCloser is implemented by afero.File. We use this as the common type for +// content in Resource objects, even for strings. +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer } -type translatedResource interface { - TranslationKey() string +// OpenReadSeekeCloser allows setting some other way (than reading from a filesystem) +// to open or create a ReadSeekCloser. +type OpenReadSeekCloser func() (ReadSeekCloser, error) + +// ReadSeekCloserResource is a Resource that supports loading its content. +type ReadSeekCloserResource interface { + Resource + ReadSeekCloser() (ReadSeekCloser, error) } // Resources represents a slice of resources, which can be a mix of different types. @@ -125,44 +167,6 @@ func (r Resources) ByType(tp string) Resources { return filtered } -const prefixDeprecatedMsg = `We have added the more flexible Resources.GetMatch (find one) and Resources.Match (many) to replace the "prefix" methods. - -These matches by a given globbing pattern, e.g. "*.jpg". - -Some examples: - -* To find all resources by its prefix in the root dir of the bundle: .Match image* -* To find one resource by its prefix in the root dir of the bundle: .GetMatch image* -* To find all JPEG images anywhere in the bundle: .Match **.jpg` - -// GetByPrefix gets the first resource matching the given filename prefix, e.g -// "logo" will match logo.png. It returns nil of none found. -// In potential ambiguous situations, combine it with ByType. -func (r Resources) GetByPrefix(prefix string) Resource { - helpers.Deprecated("Resources", "GetByPrefix", prefixDeprecatedMsg, true) - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - return resource - } - } - return nil -} - -// ByPrefix gets all resources matching the given base filename prefix, e.g -// "logo" will match logo.png. -func (r Resources) ByPrefix(prefix string) Resources { - helpers.Deprecated("Resources", "ByPrefix", prefixDeprecatedMsg, true) - var matches Resources - prefix = strings.ToLower(prefix) - for _, resource := range r { - if matchesPrefix(resource, prefix) { - matches = append(matches, resource) - } - } - return matches -} - // GetMatch finds the first Resource matching the given pattern, or nil if none found. // See Match for a more complete explanation about the rules used. func (r Resources) GetMatch(pattern string) Resource { @@ -204,10 +208,6 @@ func (r Resources) Match(pattern string) Resources { return matches } -func matchesPrefix(r Resource, prefix string) bool { - return strings.HasPrefix(strings.ToLower(r.Name()), prefix) -} - var ( globCache = make(map[string]glob.Glob) globMu sync.RWMutex @@ -268,81 +268,180 @@ func (r1 Resources) MergeByLanguageInterface(in interface{}) (interface{}, error type Spec struct { *helpers.PathSpec - mimeTypes media.Types + MediaTypes media.Types + + Logger *jww.Notepad + + TextTemplates tpl.TemplateParseFinder // Holds default filter settings etc. imaging *Imaging - imageCache *imageCache + imageCache *imageCache + ResourceCache *ResourceCache - GenImagePath string + GenImagePath string + GenAssetsPath string } -func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { +func NewSpec(s *helpers.PathSpec, logger *jww.Notepad, mimeTypes media.Types) (*Spec, error) { imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) if err != nil { return nil, err } - genImagePath := filepath.FromSlash("_gen/images") + if logger == nil { + logger = loggers.NewErrorLogger() + } - return &Spec{PathSpec: s, - GenImagePath: genImagePath, - imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( + genImagePath := filepath.FromSlash("_gen/images") + // The transformed assets (CSS etc.) + genAssetsPath := filepath.FromSlash("_gen/assets") + + rs := &Spec{PathSpec: s, + Logger: logger, + GenImagePath: genImagePath, + GenAssetsPath: genAssetsPath, + imaging: &imaging, + MediaTypes: mimeTypes, + imageCache: newImageCache( s, // We're going to write a cache pruning routine later, so make it extremely // unlikely that the user shoots him or herself in the foot // and this is set to a value that represents data he/she // cares about. This should be set in stone once released. genImagePath, - )}, nil -} + )} -func (r *Spec) NewResourceFromFile( - targetPathBuilder func(base string) string, - file source.File, relTargetFilename string) (Resource, error) { + rs.ResourceCache = newResourceCache(rs) + + return rs, nil - return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename) } -func (r *Spec) NewResourceFromFilename( - targetPathBuilder func(base string) string, - absSourceFilename, relTargetFilename string) (Resource, error) { +type ResourceSourceDescriptor struct { + // TargetPathBuilder is a callback to create target paths's relative to its owner. + TargetPathBuilder func(base string) string - fi, err := r.sourceFs().Stat(absSourceFilename) - if err != nil { - return nil, err + // Need one of these to load the resource content. + SourceFile source.File + OpenReadSeekCloser OpenReadSeekCloser + + // If OpenReadSeekerCloser is not set, we use this to open the file. + SourceFilename string + + // The relative target filename without any language code. + RelTargetFilename string + + // Any base path prepeneded to the permalink. + // Typically the language code if this resource should be published to its sub-folder. + URLBase string + + // Any base path prepended to the target path. This will also typically be the + // language code, but setting it here means that it should not have any effect on + // the permalink. + TargetPathBase string + + // Delay publishing until either Permalink or RelPermalink is called. Maybe never. + LazyPublish bool +} + +func (r ResourceSourceDescriptor) Filename() string { + if r.SourceFile != nil { + return r.SourceFile.Filename() } - return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename) + return r.SourceFilename } func (r *Spec) sourceFs() afero.Fs { - return r.PathSpec.BaseFs.ContentFs + return r.PathSpec.BaseFs.Content.Fs } -func (r *Spec) newResource( - targetPathBuilder func(base string) string, - absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) { +func (r *Spec) New(fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(r.sourceFs(), fd) +} - var mimeType string - ext := filepath.Ext(relTargetFilename) - m, found := r.mimeTypes.GetBySuffix(strings.TrimPrefix(ext, ".")) - if found { - mimeType = m.SubType - } else { - mimeType = mime.TypeByExtension(ext) - if mimeType == "" { - mimeType = DefaultResourceType - } else { - mimeType = mimeType[:strings.Index(mimeType, "/")] +func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + return r.newResourceForFs(sourceFs, fd) +} + +func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + if fd.OpenReadSeekCloser == nil { + if fd.SourceFile != nil && fd.SourceFilename != "" { + return nil, errors.New("both SourceFile and AbsSourceFilename provided") + } else if fd.SourceFile == nil && fd.SourceFilename == "" { + return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") } } - gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType) + if fd.URLBase == "" { + fd.URLBase = r.GetURLLanguageBasePath() + } + + if fd.TargetPathBase == "" { + fd.TargetPathBase = r.GetTargetLanguageBasePath() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = fd.Filename() + } + + return r.newResource(sourceFs, fd) +} - if mimeType == "image" { - ext := strings.ToLower(helpers.Ext(absSourceFilename)) +func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (Resource, error) { + var fi os.FileInfo + var sourceFilename string + + if fd.OpenReadSeekCloser != nil { + + } else if fd.SourceFilename != "" { + var err error + fi, err = sourceFs.Stat(fd.SourceFilename) + if err != nil { + return nil, err + } + sourceFilename = fd.SourceFilename + } else { + fi = fd.SourceFile.FileInfo() + sourceFilename = fd.SourceFile.Filename() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = sourceFilename + } + + ext := filepath.Ext(fd.RelTargetFilename) + mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) + // TODO(bep) we need to handle these ambigous types better, but in this context + // we most likely want the application/xml type. + if mimeType.Suffix == "xml" && mimeType.SubType == "rss" { + mimeType, found = r.MediaTypes.GetByType("application/xml") + } + + if !found { + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromString(mimeStr) + } + + } + + gr := r.newGenericResourceWithBase( + sourceFs, + fd.LazyPublish, + fd.OpenReadSeekCloser, + fd.URLBase, + fd.TargetPathBase, + fd.TargetPathBuilder, + fi, + sourceFilename, + fd.RelTargetFilename, + mimeType) + + if mimeType.MainType == "image" { + ext := strings.ToLower(helpers.Ext(sourceFilename)) imgFormat, ok := imageFormats[ext] if !ok { @@ -351,27 +450,21 @@ func (r *Spec) newResource( return gr, nil } - f, err := gr.sourceFs().Open(absSourceFilename) - if err != nil { - return nil, fmt.Errorf("failed to open image source file: %s", err) - } - defer f.Close() - - hash, err := helpers.MD5FromFileFast(f) - if err != nil { + if err := gr.initHash(); err != nil { return nil, err } return &Image{ - hash: hash, format: imgFormat, imaging: r.imaging, genericResource: gr}, nil } return gr, nil + } -func (r *Spec) IsInCache(key string) bool { +// TODO(bep) unify +func (r *Spec) IsInImageCache(key string) bool { // This is used for cache pruning. We currently only have images, but we could // imagine expanding on this. return r.imageCache.isInCache(key) @@ -381,6 +474,11 @@ func (r *Spec) DeleteCacheByPrefix(prefix string) { r.imageCache.deleteByPrefix(prefix) } +func (r *Spec) ClearCaches() { + r.imageCache.clear() + r.ResourceCache.clear() +} + func (r *Spec) CacheStats() string { r.imageCache.mu.RLock() defer r.imageCache.mu.RUnlock() @@ -410,18 +508,54 @@ func (d dirFile) path() string { return path.Join(d.dir, d.file) } +type resourcePathDescriptor struct { + // The relative target directory and filename. + relTargetDirFile dirFile + + // Callback used to construct a target path relative to its owner. + targetPathBuilder func(rel string) string + + // baseURLDir is the fixed sub-folder for a resource in permalinks. This will typically + // be the language code if we publish to the language's sub-folder. + baseURLDir string + + // This will normally be the same as above, but this will only apply to publishing + // of resources. + baseTargetPathDir string + + // baseOffset is set when the output format's path has a offset, e.g. for AMP. + baseOffset string +} + type resourceContent struct { content string contentInit sync.Once } +type resourceHash struct { + hash string + hashInit sync.Once +} + +type publishOnce struct { + publisherInit sync.Once + publisherErr error + logger *jww.Notepad +} + +func (l *publishOnce) publish(s Source) error { + l.publisherInit.Do(func() { + l.publisherErr = s.Publish() + if l.publisherErr != nil { + l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr) + } + }) + return l.publisherErr +} + // genericResource represents a generic linkable resource. type genericResource struct { - // The relative path to this resource. - relTargetPath dirFile - - // Base is set when the output format's path has a offset, e.g. for AMP. - base string + resourcePathDescriptor title string name string @@ -433,6 +567,12 @@ type genericResource struct { // the path to the file on the real filesystem. sourceFilename string + // Will be set if this resource is backed by something other than a file. + openReadSeekerCloser OpenReadSeekCloser + + // A hash of the source content. Is only calculated in caching situations. + *resourceHash + // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. overriddenSourceFs afero.Fs @@ -440,20 +580,87 @@ type genericResource struct { spec *Spec resourceType string - osFileInfo os.FileInfo + mediaType media.Type - targetPathBuilder func(rel string) string + osFileInfo os.FileInfo // We create copies of this struct, so this needs to be a pointer. *resourceContent + + // May be set to signal lazy/delayed publishing. + *publishOnce +} + +func (l *genericResource) Data() interface{} { + return noData } func (l *genericResource) Content() (interface{}, error) { + if err := l.initContent(); err != nil { + return nil, err + } + + return l.content, nil +} + +func (l *genericResource) ReadSeekCloser() (ReadSeekCloser, error) { + if l.openReadSeekerCloser != nil { + return l.openReadSeekerCloser() + } + f, err := l.sourceFs().Open(l.sourceFilename) + if err != nil { + return nil, err + } + return f, nil + +} + +func (l *genericResource) MediaType() media.Type { + return l.mediaType +} + +// Implement the Cloner interface. +func (l genericResource) WithNewBase(base string) Resource { + l.baseOffset = base + l.resourceContent = &resourceContent{} + return &l +} + +func (l *genericResource) initHash() error { + var err error + l.hashInit.Do(func() { + var hash string + var f ReadSeekCloser + f, err = l.ReadSeekCloser() + if err != nil { + err = fmt.Errorf("failed to open source file: %s", err) + return + } + defer f.Close() + + hash, err = helpers.MD5FromFileFast(f) + if err != nil { + return + } + l.hash = hash + + }) + + return err +} + +func (l *genericResource) initContent() error { var err error l.contentInit.Do(func() { - var b []byte + var r ReadSeekCloser + r, err = l.ReadSeekCloser() + if err != nil { + return + } + defer r.Close() - b, err := afero.ReadFile(l.sourceFs(), l.AbsSourceFilename()) + var b []byte + b, err = ioutil.ReadAll(r) if err != nil { return } @@ -462,7 +669,7 @@ func (l *genericResource) Content() (interface{}, error) { }) - return l.content, err + return err } func (l *genericResource) sourceFs() afero.Fs { @@ -472,12 +679,36 @@ func (l *genericResource) sourceFs() afero.Fs { return l.spec.sourceFs() } +func (l *genericResource) publishIfNeeded() { + if l.publishOnce != nil { + l.publishOnce.publish(l) + } +} + func (l *genericResource) Permalink() string { - return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String()) + l.publishIfNeeded() + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path()), l.spec.BaseURL.HostURL()) } func (l *genericResource) RelPermalink() string { - return l.relPermalinkForRel(l.relTargetPath.path(), true) + l.publishIfNeeded() + return l.relPermalinkFor(l.relTargetDirFile.path()) +} + +func (l *genericResource) relPermalinkFor(target string) string { + return l.relPermalinkForRel(target) + +} +func (l *genericResource) permalinkFor(target string) string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target), l.spec.BaseURL.HostURL()) + +} +func (l *genericResource) relTargetPathFor(target string) string { + return l.relTargetPathForRel(target, false) +} + +func (l *genericResource) relTargetPath() string { + return l.relTargetPathForRel(l.targetPath(), false) } func (l *genericResource) Name() string { @@ -514,31 +745,33 @@ func (l *genericResource) updateParams(params map[string]interface{}) { } } -// Implement the Cloner interface. -func (l genericResource) WithNewBase(base string) Resource { - l.base = base - l.resourceContent = &resourceContent{} - return &l +func (l *genericResource) relPermalinkForRel(rel string) string { + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, true)) } -func (l *genericResource) relPermalinkForRel(rel string, addBasePath bool) string { - return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, addBasePath)) -} +func (l *genericResource) relTargetPathForRel(rel string, isURL bool) string { -func (l *genericResource) relTargetPathForRel(rel string, addBasePath bool) string { if l.targetPathBuilder != nil { rel = l.targetPathBuilder(rel) } - if l.base != "" { - rel = path.Join(l.base, rel) + if isURL && l.baseURLDir != "" { + rel = path.Join(l.baseURLDir, rel) } - if addBasePath && l.spec.PathSpec.BasePath != "" { + if !isURL && l.baseTargetPathDir != "" { + rel = path.Join(l.baseTargetPathDir, rel) + } + + if l.baseOffset != "" { + rel = path.Join(l.baseOffset, rel) + } + + if isURL && l.spec.PathSpec.BasePath != "" { rel = path.Join(l.spec.PathSpec.BasePath, rel) } - if rel[0] != '/' { + if len(rel) == 0 || rel[0] != '/' { rel = "/" + rel } @@ -549,146 +782,100 @@ func (l *genericResource) ResourceType() string { return l.resourceType } -func (l *genericResource) AbsSourceFilename() string { - return l.sourceFilename -} - func (l *genericResource) String() string { return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) } func (l *genericResource) Publish() error { - f, err := l.sourceFs().Open(l.AbsSourceFilename()) + f, err := l.ReadSeekCloser() if err != nil { return err } defer f.Close() - return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs) -} - -const counterPlaceHolder = ":counter" - -// AssignMetadata assigns the given metadata to those resources that supports updates -// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. -// This assignment is additive, but the most specific match needs to be first. -// The `name` and `title` metadata field support shell-matched collection it got a match in. -// See https://golang.org/pkg/path/#Match -func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { - - counters := make(map[string]int) - - for _, r := range resources { - if _, ok := r.(metaAssigner); !ok { - continue - } - - var ( - nameSet, titleSet bool - nameCounter, titleCounter = 0, 0 - nameCounterFound, titleCounterFound bool - resourceSrcKey = strings.ToLower(r.Name()) - ) - - ma := r.(metaAssigner) - for _, meta := range metadata { - src, found := meta["src"] - if !found { - return fmt.Errorf("missing 'src' in metadata for resource") - } - - srcKey := strings.ToLower(cast.ToString(src)) - - glob, err := getGlob(srcKey) - if err != nil { - return fmt.Errorf("failed to match resource with metadata: %s", err) - } - - match := glob.Match(resourceSrcKey) - - if match { - if !nameSet { - name, found := meta["name"] - if found { - name := cast.ToString(name) - if !nameCounterFound { - nameCounterFound = strings.Contains(name, counterPlaceHolder) - } - if nameCounterFound && nameCounter == 0 { - counterKey := "name_" + srcKey - nameCounter = counters[counterKey] + 1 - counters[counterKey] = nameCounter - } - - ma.setName(replaceResourcePlaceholders(name, nameCounter)) - nameSet = true - } - } - - if !titleSet { - title, found := meta["title"] - if found { - title := cast.ToString(title) - if !titleCounterFound { - titleCounterFound = strings.Contains(title, counterPlaceHolder) - } - if titleCounterFound && titleCounter == 0 { - counterKey := "title_" + srcKey - titleCounter = counters[counterKey] + 1 - counters[counterKey] = titleCounter - } - ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) - titleSet = true - } - } - - params, found := meta["params"] - if found { - m := cast.ToStringMap(params) - // Needed for case insensitive fetching of params values - maps.ToLower(m) - ma.updateParams(m) - } - } - } - } - - return nil + return helpers.WriteToDisk(l.targetFilename(), f, l.spec.BaseFs.PublishFs) } -func replaceResourcePlaceholders(in string, counter int) string { - return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +// Path is stored with Unix style slashes. +func (l *genericResource) targetPath() string { + return l.relTargetDirFile.path() } -func (l *genericResource) target() string { - target := l.relTargetPathForRel(l.relTargetPath.path(), false) - if l.spec.PathSpec.Languages.IsMultihost() { - target = path.Join(l.spec.PathSpec.Language.Lang, target) - } - return filepath.Clean(target) +func (l *genericResource) targetFilename() string { + return filepath.Clean(l.relTargetPath()) } -func (r *Spec) newGenericResource( +// TODO(bep) clean up below +func (r *Spec) newGenericResource(sourceFs afero.Fs, + targetPathBuilder func(base string) string, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + false, + nil, + "", + "", + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} + +func (r *Spec) newGenericResourceWithBase( + sourceFs afero.Fs, + lazyPublish bool, + openReadSeekerCloser OpenReadSeekCloser, + urlBaseDir string, + targetPathBaseDir string, targetPathBuilder func(base string) string, osFileInfo os.FileInfo, sourceFilename, - baseFilename, - resourceType string) *genericResource { + baseFilename string, + mediaType media.Type) *genericResource { // This value is used both to construct URLs and file paths, but start // with a Unix-styled path. - baseFilename = filepath.ToSlash(baseFilename) + baseFilename = helpers.ToSlashTrimLeading(baseFilename) fpath, fname := path.Split(baseFilename) - return &genericResource{ + var resourceType string + if mediaType.MainType == "image" { + resourceType = mediaType.MainType + } else { + resourceType = mediaType.SubType + } + + pathDescriptor := resourcePathDescriptor{ + baseURLDir: urlBaseDir, + baseTargetPathDir: targetPathBaseDir, targetPathBuilder: targetPathBuilder, - osFileInfo: osFileInfo, - sourceFilename: sourceFilename, - relTargetPath: dirFile{dir: fpath, file: fname}, - resourceType: resourceType, - spec: r, - params: make(map[string]interface{}), - name: baseFilename, - title: baseFilename, - resourceContent: &resourceContent{}, + relTargetDirFile: dirFile{dir: fpath, file: fname}, + } + + var po *publishOnce + if lazyPublish { + po = &publishOnce{logger: r.Logger} + } + + return &genericResource{ + openReadSeekerCloser: openReadSeekerCloser, + publishOnce: po, + resourcePathDescriptor: pathDescriptor, + overriddenSourceFs: sourceFs, + osFileInfo: osFileInfo, + sourceFilename: sourceFilename, + mediaType: mediaType, + resourceType: resourceType, + spec: r, + params: make(map[string]interface{}), + name: baseFilename, + title: baseFilename, + resourceContent: &resourceContent{}, + resourceHash: &resourceHash{}, } } diff --git a/resource/resource_cache.go b/resource/resource_cache.go new file mode 100644 index 000000000..28c3c23a2 --- /dev/null +++ b/resource/resource_cache.go @@ -0,0 +1,241 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "encoding/json" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/spf13/afero" + + "github.com/BurntSushi/locker" +) + +const ( + CACHE_CLEAR_ALL = "clear_all" + CACHE_OTHER = "other" +) + +type ResourceCache struct { + rs *Spec + + cache map[string]Resource + sync.RWMutex + + // Provides named resource locks. + nlocker *locker.Locker +} + +// ResourceKeyPartition returns a partition name +// to allow for more fine grained cache flushes. +// It will return the file extension without the leading ".". If no +// extension, it will return "other". +func ResourceKeyPartition(filename string) string { + ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(filename)), ".") + if ext == "" { + ext = CACHE_OTHER + } + return ext +} + +func newResourceCache(rs *Spec) *ResourceCache { + return &ResourceCache{ + rs: rs, + cache: make(map[string]Resource), + nlocker: locker.NewLocker(), + } +} + +func (c *ResourceCache) clear() { + c.Lock() + defer c.Unlock() + + c.cache = make(map[string]Resource) + c.nlocker = locker.NewLocker() +} + +func (c *ResourceCache) Contains(key string) bool { + key = c.cleanKey(filepath.ToSlash(key)) + _, found := c.get(key) + return found +} + +func (c *ResourceCache) cleanKey(key string) string { + return strings.TrimPrefix(path.Clean(key), "/") +} + +func (c *ResourceCache) get(key string) (Resource, bool) { + c.RLock() + defer c.RUnlock() + r, found := c.cache[key] + return r, found +} + +func (c *ResourceCache) GetOrCreate(partition, key string, f func() (Resource, error)) (Resource, error) { + key = c.cleanKey(path.Join(partition, key)) + // First check in-memory cache. + r, found := c.get(key) + if found { + return r, nil + } + // This is a potentially long running operation, so get a named lock. + c.nlocker.Lock(key) + + // Double check in-memory cache. + r, found = c.get(key) + if found { + c.nlocker.Unlock(key) + return r, nil + } + + defer c.nlocker.Unlock(key) + + r, err := f() + if err != nil { + return nil, err + } + + c.set(key, r) + + return r, nil + +} + +func (c *ResourceCache) getFilenames(key string) (string, string) { + filenameBase := filepath.Join(c.rs.GenAssetsPath, key) + filenameMeta := filenameBase + ".json" + filenameContent := filenameBase + ".content" + + return filenameMeta, filenameContent +} + +func (c *ResourceCache) getFromFile(key string) (afero.File, transformedResourceMetadata, bool) { + c.RLock() + defer c.RUnlock() + + var meta transformedResourceMetadata + filenameMeta, filenameContent := c.getFilenames(key) + fMeta, err := c.rs.Resources.Fs.Open(filenameMeta) + if err != nil { + return nil, meta, false + } + defer fMeta.Close() + + jsonContent, err := ioutil.ReadAll(fMeta) + if err != nil { + return nil, meta, false + } + + if err := json.Unmarshal(jsonContent, &meta); err != nil { + return nil, meta, false + } + + fContent, err := c.rs.Resources.Fs.Open(filenameContent) + if err != nil { + return nil, meta, false + } + + return fContent, meta, true +} + +// writeMeta writes the metadata to file and returns a writer for the content part. +func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) (afero.File, error) { + filenameMeta, filenameContent := c.getFilenames(key) + raw, err := json.Marshal(meta) + if err != nil { + return nil, err + } + + fm, err := c.openResourceFileForWriting(filenameMeta) + if err != nil { + return nil, err + } + + if _, err := fm.Write(raw); err != nil { + return nil, err + } + + return c.openResourceFileForWriting(filenameContent) + +} + +func (c *ResourceCache) openResourceFileForWriting(filename string) (afero.File, error) { + return openFileForWriting(c.rs.Resources.Fs, filename) +} + +// openFileForWriting opens or creates the given file. If the target directory +// does not exist, it gets created. +func openFileForWriting(fs afero.Fs, filename string) (afero.File, error) { + filename = filepath.Clean(filename) + // Create will truncate if file already exists. + f, err := fs.Create(filename) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + if err = fs.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return nil, err + } + f, err = fs.Create(filename) + } + + return f, err +} + +func (c *ResourceCache) set(key string, r Resource) { + c.Lock() + defer c.Unlock() + c.cache[key] = r +} + +func (c *ResourceCache) DeletePartitions(partitions ...string) { + partitionsSet := map[string]bool{ + // Always clear out the resources not matching the partition. + "other": true, + } + for _, p := range partitions { + partitionsSet[p] = true + } + + if partitionsSet[CACHE_CLEAR_ALL] { + c.clear() + return + } + + c.Lock() + defer c.Unlock() + + for k := range c.cache { + clear := false + partIdx := strings.Index(k, "/") + if partIdx == -1 { + clear = true + } else { + partition := k[:partIdx] + if partitionsSet[partition] { + clear = true + } + } + + if clear { + delete(c.cache, k) + } + } + +} diff --git a/resource/resource_metadata.go b/resource/resource_metadata.go new file mode 100644 index 000000000..2c82aeaf6 --- /dev/null +++ b/resource/resource_metadata.go @@ -0,0 +1,129 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "fmt" + "strconv" + + "github.com/spf13/cast" + + "strings" + + "github.com/gohugoio/hugo/common/maps" +) + +var ( + _ metaAssigner = (*genericResource)(nil) +) + +// metaAssigner allows updating metadata in resources that supports it. +type metaAssigner interface { + setTitle(title string) + setName(name string) + updateParams(params map[string]interface{}) +} + +const counterPlaceHolder = ":counter" + +// AssignMetadata assigns the given metadata to those resources that supports updates +// and matching by wildcard given in `src` using `filepath.Match` with lower cased values. +// This assignment is additive, but the most specific match needs to be first. +// The `name` and `title` metadata field support shell-matched collection it got a match in. +// See https://golang.org/pkg/path/#Match +func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) error { + + counters := make(map[string]int) + + for _, r := range resources { + if _, ok := r.(metaAssigner); !ok { + continue + } + + var ( + nameSet, titleSet bool + nameCounter, titleCounter = 0, 0 + nameCounterFound, titleCounterFound bool + resourceSrcKey = strings.ToLower(r.Name()) + ) + + ma := r.(metaAssigner) + for _, meta := range metadata { + src, found := meta["src"] + if !found { + return fmt.Errorf("missing 'src' in metadata for resource") + } + + srcKey := strings.ToLower(cast.ToString(src)) + + glob, err := getGlob(srcKey) + if err != nil { + return fmt.Errorf("failed to match resource with metadata: %s", err) + } + + match := glob.Match(resourceSrcKey) + + if match { + if !nameSet { + name, found := meta["name"] + if found { + name := cast.ToString(name) + if !nameCounterFound { + nameCounterFound = strings.Contains(name, counterPlaceHolder) + } + if nameCounterFound && nameCounter == 0 { + counterKey := "name_" + srcKey + nameCounter = counters[counterKey] + 1 + counters[counterKey] = nameCounter + } + + ma.setName(replaceResourcePlaceholders(name, nameCounter)) + nameSet = true + } + } + + if !titleSet { + title, found := meta["title"] + if found { + title := cast.ToString(title) + if !titleCounterFound { + titleCounterFound = strings.Contains(title, counterPlaceHolder) + } + if titleCounterFound && titleCounter == 0 { + counterKey := "title_" + srcKey + titleCounter = counters[counterKey] + 1 + counters[counterKey] = titleCounter + } + ma.setTitle((replaceResourcePlaceholders(title, titleCounter))) + titleSet = true + } + } + + params, found := meta["params"] + if found { + m := cast.ToStringMap(params) + // Needed for case insensitive fetching of params values + maps.ToLower(m) + ma.updateParams(m) + } + } + } + } + + return nil +} + +func replaceResourcePlaceholders(in string, counter int) string { + return strings.Replace(in, counterPlaceHolder, strconv.Itoa(counter), -1) +} diff --git a/resource/resource_metadata_test.go b/resource/resource_metadata_test.go new file mode 100644 index 000000000..85fb25b57 --- /dev/null +++ b/resource/resource_metadata_test.go @@ -0,0 +1,230 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/gohugoio/hugo/media" + + "github.com/stretchr/testify/require" +) + +func TestAssignMetadata(t *testing.T) { + assert := require.New(t) + spec := newTestResourceSpec(assert) + + var foo1, foo2, foo3, logo1, logo2, logo3 Resource + var resources Resources + + for _, this := range []struct { + metaData []map[string]interface{} + assertFunc func(err error) + }{ + {[]map[string]interface{}{ + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Resource", logo1.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + }, + { + "title": "My Resource", + "name": "My Name", + "src": "*", + }, + }, func(err error) { + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Logo", logo2.Title()) + assert.Equal("My Name", logo1.Name()) + assert.Equal("My Name", foo2.Name()) + assert.Equal("My Name", foo3.Name()) + assert.Equal("My Resource", foo3.Title()) + + }}, + {[]map[string]interface{}{ + { + "title": "My Logo", + "src": "*loGo*", + "params": map[string]interface{}{ + "Param1": true, + "icon": "logo", + }, + }, + { + "title": "My Resource", + "src": "*", + "params": map[string]interface{}{ + "Param2": true, + "icon": "resource", + }, + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("My Logo", logo1.Title()) + assert.Equal("My Resource", foo3.Title()) + _, p1 := logo2.Params()["param1"] + _, p2 := foo2.Params()["param2"] + _, p1_2 := foo2.Params()["param1"] + _, p2_2 := logo2.Params()["param2"] + + icon1, _ := logo2.Params()["icon"] + icon2, _ := foo2.Params()["icon"] + + assert.True(p1) + assert.True(p2) + + // Check merge + assert.True(p2_2) + assert.False(p1_2) + + assert.Equal("logo", icon1) + assert.Equal("resource", icon2) + + }}, + {[]map[string]interface{}{ + { + "name": "Logo Name #:counter", + "src": "*logo*", + }, + { + "title": "Resource #:counter", + "name": "Name #:counter", + "src": "*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Resource #2", logo2.Title()) + assert.Equal("Logo Name #1", logo2.Name()) + assert.Equal("Resource #4", logo1.Title()) + assert.Equal("Logo Name #2", logo1.Name()) + assert.Equal("Resource #1", foo2.Title()) + assert.Equal("Resource #3", foo1.Title()) + assert.Equal("Name #2", foo1.Name()) + assert.Equal("Resource #5", foo3.Title()) + + assert.Equal(logo2, resources.GetMatch("logo name #1*")) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo #1", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo", + "src": "logo3.png", + }, + { + "title": "Other Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Third Logo", logo3.Title()) + assert.Equal("Name #3", logo3.Name()) + assert.Equal("Other Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Other Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "name": "third-logo", + "src": "logo3.png", + }, + { + "title": "Logo #:counter", + "name": "Name #:counter", + "src": "logo*", + }, + }, func(err error) { + assert.NoError(err) + assert.Equal("Logo #3", logo3.Title()) + assert.Equal("third-logo", logo3.Name()) + assert.Equal("Logo #1", logo2.Title()) + assert.Equal("Name #1", logo2.Name()) + assert.Equal("Logo #2", logo1.Title()) + assert.Equal("Name #2", logo1.Name()) + + }}, + {[]map[string]interface{}{ + { + "title": "Third Logo #:counter", + }, + }, func(err error) { + // Missing src + assert.Error(err) + + }}, + {[]map[string]interface{}{ + { + "title": "Title", + "src": "[]", + }, + }, func(err error) { + // Invalid pattern + assert.Error(err) + + }}, + } { + + foo2 = spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType) + logo2 = spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType) + foo1 = spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType) + logo1 = spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType) + foo3 = spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType) + logo3 = spec.newGenericResource(nil, nil, nil, "/b/logo3.png", "logo3.png", pngType) + + resources = Resources{ + foo2, + logo2, + foo1, + logo1, + foo3, + logo3, + } + + this.assertFunc(AssignMetadata(this.metaData, resources...)) + } + +} diff --git a/resource/resource_test.go b/resource/resource_test.go index 40061e5c4..659994c36 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/media" + "github.com/stretchr/testify/require" ) @@ -29,7 +31,7 @@ func TestGenericResource(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) - r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, nil, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo.css", r.Permalink()) assert.Equal("/foo.css", r.RelPermalink()) @@ -44,7 +46,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := func(s string) string { return path.Join("/foo", s) } - r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, factory, nil, "/a/foo.css", "foo.css", media.CSSType) assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink()) @@ -58,8 +60,7 @@ func TestNewResourceFromFilename(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/logo.png"}) assert.NoError(err) assert.NotNil(r) @@ -67,7 +68,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json") + r, err = spec.New(ResourceSourceDescriptor{SourceFilename: "a/b/data.json"}) assert.NoError(err) assert.NotNil(r) @@ -84,8 +85,7 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { writeSource(t, spec.Fs, "content/a/b/logo.png", "image") - r, err := spec.NewResourceFromFilename(nil, - filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.New(ResourceSourceDescriptor{SourceFilename: filepath.FromSlash("a/b/logo.png")}) assert.NoError(err) assert.NotNil(r) @@ -93,18 +93,20 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { assert.Equal("/docs/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/docs/a/b/logo.png", r.Permalink()) img := r.(*Image) - assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.target()) + assert.Equal(filepath.FromSlash("/a/b/logo.png"), img.targetFilename()) } +var pngType, _ = media.FromString("image/png") + func TestResourcesByType(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"), - spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")} + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo.png", "logo.css", pngType), + spec.newGenericResource(nil, nil, nil, "/a/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/foo3.css", "foo3.css", media.CSSType)} assert.Len(resources.ByType("css"), 3) assert.Len(resources.ByType("image"), 1) @@ -115,25 +117,25 @@ func TestResourcesGetByPrefix(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")} - - assert.Nil(resources.GetByPrefix("asdf")) - assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink()) - assert.Equal("/logo1.png", resources.GetByPrefix("loGo").RelPermalink()) - assert.Equal("/Logo2.png", resources.GetByPrefix("logo2").RelPermalink()) - assert.Equal("/foo2.css", resources.GetByPrefix("foo2").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Equal("/foo1.css", resources.GetByPrefix("foo1").RelPermalink()) - assert.Nil(resources.GetByPrefix("asdfasdf")) - - assert.Equal(2, len(resources.ByPrefix("logo"))) - assert.Equal(1, len(resources.ByPrefix("logo2"))) - - logo := resources.GetByPrefix("logo") + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType)} + + assert.Nil(resources.GetMatch("asdf*")) + assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) + assert.Equal("/logo1.png", resources.GetMatch("loGo*").RelPermalink()) + assert.Equal("/Logo2.png", resources.GetMatch("logo2*").RelPermalink()) + assert.Equal("/foo2.css", resources.GetMatch("foo2*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Equal("/foo1.css", resources.GetMatch("foo1*").RelPermalink()) + assert.Nil(resources.GetMatch("asdfasdf*")) + + assert.Equal(2, len(resources.Match("logo*"))) + assert.Equal(1, len(resources.Match("logo2*"))) + + logo := resources.GetMatch("logo*") assert.NotNil(logo.Params()) assert.Equal("logo1.png", logo.Name()) assert.Equal("logo1.png", logo.Title()) @@ -144,14 +146,14 @@ func TestResourcesGetMatch(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"), - spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"), + spec.newGenericResource(nil, nil, nil, "/a/foo1.css", "foo1.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/a/logo1.png", "logo1.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/Logo2.png", "Logo2.png", pngType), + spec.newGenericResource(nil, nil, nil, "/b/foo2.css", "foo2.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/foo3.css", "foo3.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo4.css", "c/foo4.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/foo5.css", "c/foo5.css", media.CSSType), + spec.newGenericResource(nil, nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", media.CSSType), } assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) @@ -186,226 +188,6 @@ func TestResourcesGetMatch(t *testing.T) { } -func TestAssignMetadata(t *testing.T) { - assert := require.New(t) - spec := newTestResourceSpec(assert) - - var foo1, foo2, foo3, logo1, logo2, logo3 Resource - var resources Resources - - for _, this := range []struct { - metaData []map[string]interface{} - assertFunc func(err error) - }{ - {[]map[string]interface{}{ - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Resource", logo1.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - }, - { - "title": "My Resource", - "name": "My Name", - "src": "*", - }, - }, func(err error) { - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Logo", logo2.Title()) - assert.Equal("My Name", logo1.Name()) - assert.Equal("My Name", foo2.Name()) - assert.Equal("My Name", foo3.Name()) - assert.Equal("My Resource", foo3.Title()) - - }}, - {[]map[string]interface{}{ - { - "title": "My Logo", - "src": "*loGo*", - "params": map[string]interface{}{ - "Param1": true, - "icon": "logo", - }, - }, - { - "title": "My Resource", - "src": "*", - "params": map[string]interface{}{ - "Param2": true, - "icon": "resource", - }, - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("My Logo", logo1.Title()) - assert.Equal("My Resource", foo3.Title()) - _, p1 := logo2.Params()["param1"] - _, p2 := foo2.Params()["param2"] - _, p1_2 := foo2.Params()["param1"] - _, p2_2 := logo2.Params()["param2"] - - icon1, _ := logo2.Params()["icon"] - icon2, _ := foo2.Params()["icon"] - - assert.True(p1) - assert.True(p2) - - // Check merge - assert.True(p2_2) - assert.False(p1_2) - - assert.Equal("logo", icon1) - assert.Equal("resource", icon2) - - }}, - {[]map[string]interface{}{ - { - "name": "Logo Name #:counter", - "src": "*logo*", - }, - { - "title": "Resource #:counter", - "name": "Name #:counter", - "src": "*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Resource #2", logo2.Title()) - assert.Equal("Logo Name #1", logo2.Name()) - assert.Equal("Resource #4", logo1.Title()) - assert.Equal("Logo Name #2", logo1.Name()) - assert.Equal("Resource #1", foo2.Title()) - assert.Equal("Resource #3", foo1.Title()) - assert.Equal("Name #2", foo1.Name()) - assert.Equal("Resource #5", foo3.Title()) - - assert.Equal(logo2, resources.GetByPrefix("logo name #1")) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo #1", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo", - "src": "logo3.png", - }, - { - "title": "Other Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Third Logo", logo3.Title()) - assert.Equal("Name #3", logo3.Name()) - assert.Equal("Other Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Other Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "name": "third-logo", - "src": "logo3.png", - }, - { - "title": "Logo #:counter", - "name": "Name #:counter", - "src": "logo*", - }, - }, func(err error) { - assert.NoError(err) - assert.Equal("Logo #3", logo3.Title()) - assert.Equal("third-logo", logo3.Name()) - assert.Equal("Logo #1", logo2.Title()) - assert.Equal("Name #1", logo2.Name()) - assert.Equal("Logo #2", logo1.Title()) - assert.Equal("Name #2", logo1.Name()) - - }}, - {[]map[string]interface{}{ - { - "title": "Third Logo #:counter", - }, - }, func(err error) { - // Missing src - assert.Error(err) - - }}, - {[]map[string]interface{}{ - { - "title": "Title", - "src": "[]", - }, - }, func(err error) { - // Invalid pattern - assert.Error(err) - - }}, - } { - - foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css") - logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image") - foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css") - logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image") - foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css") - logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image") - - resources = Resources{ - foo2, - logo2, - foo1, - logo1, - foo3, - logo3, - } - - this.assertFunc(AssignMetadata(this.metaData, resources...)) - } - -} - -func BenchmarkResourcesByPrefix(b *testing.B) { - resources := benchResources(b) - prefixes := []string{"abc", "jkl", "nomatch", "sub/"} - rnd := rand.New(rand.NewSource(time.Now().Unix())) - - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resources.ByPrefix(prefixes[rnd.Intn(len(prefixes))]) - } - }) -} - func BenchmarkResourcesMatch(b *testing.B) { resources := benchResources(b) prefixes := []string{"abc*", "jkl*", "nomatch*", "sub/*"} @@ -428,7 +210,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")} + resources := Resources{spec.newGenericResource(nil, nil, nil, "/a/"+a100, a100, media.CSSType)} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -444,17 +226,17 @@ func benchResources(b *testing.B) Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/b/sub/"+name, "sub/"+name, media.CSSType)) } return resources @@ -482,7 +264,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, nil, "/a/"+name, name, media.CSSType)) } b.StartTimer() diff --git a/resource/templates/execute_as_template.go b/resource/templates/execute_as_template.go new file mode 100644 index 000000000..dee9d0d9a --- /dev/null +++ b/resource/templates/execute_as_template.go @@ -0,0 +1,76 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package templates contains functions for template processing of Resource objects. +package templates + +import ( + "fmt" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/resource" + "github.com/gohugoio/hugo/tpl" +) + +// Client contains methods to perform template processing of Resource objects. +type Client struct { + rs *resource.Spec + + textTemplate tpl.TemplateParseFinder +} + +// New creates a new Client with the given specification. +func New(rs *resource.Spec, textTemplate tpl.TemplateParseFinder) *Client { + if rs == nil { + panic("must provice a resource Spec") + } + if textTemplate == nil { + panic("must provide a textTemplate") + } + return &Client{rs: rs, textTemplate: textTemplate} +} + +type executeAsTemplateTransform struct { + rs *resource.Spec + textTemplate tpl.TemplateParseFinder + targetPath string + data interface{} +} + +func (t *executeAsTemplateTransform) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("execute-as-template", t.targetPath) +} + +func (t *executeAsTemplateTransform) Transform(ctx *resource.ResourceTransformationCtx) error { + tplStr := helpers.ReaderToString(ctx.From) + templ, err := t.textTemplate.Parse(ctx.InPath, tplStr) + if err != nil { + return fmt.Errorf("failed to parse Resource %q as Template: %s", ctx.InPath, err) + } + + ctx.OutPath = t.targetPath + + return templ.Execute(ctx.To, t.data) +} + +func (c *Client) ExecuteAsTemplate(res resource.Resource, targetPath string, data interface{}) (resource.Resource, error) { + return c.rs.Transform( + res, + &executeAsTemplateTransform{ + rs: c.rs, + targetPath: helpers.ToSlashTrimLeading(targetPath), + textTemplate: c.textTemplate, + data: data, + }, + ) +} diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 360adc038..e78a536a2 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -33,7 +33,9 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -49,7 +51,7 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec } @@ -72,7 +74,9 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { cfg.Set("dataDir", "data") cfg.Set("i18nDir", "i18n") cfg.Set("layoutDir", "layouts") + cfg.Set("assetDir", "assets") cfg.Set("archetypeDir", "archetypes") + cfg.Set("publishDir", "public") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} @@ -81,7 +85,7 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { assert.NoError(err) - spec, err := NewSpec(s, media.DefaultTypes) + spec, err := NewSpec(s, nil, media.DefaultTypes) assert.NoError(err) return spec @@ -102,12 +106,11 @@ func fetchImageForSpec(spec *Spec, assert *require.Assertions, name string) *Ima return r.(*Image) } -func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) Resource { +func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) ContentResource { src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755)) - out, err := spec.BaseFs.ContentFs.Create(name) + out, err := openFileForWriting(spec.BaseFs.Content.Fs, name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -118,10 +121,10 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R return path.Join("/a", s) } - r, err := spec.NewResourceFromFilename(factory, name, name) + r, err := spec.New(ResourceSourceDescriptor{TargetPathBuilder: factory, SourceFilename: name}) assert.NoError(err) - return r + return r.(ContentResource) } func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { diff --git a/resource/tocss/scss/client.go b/resource/tocss/scss/client.go new file mode 100644 index 000000000..610ea3845 --- /dev/null +++ b/resource/tocss/scss/client.go @@ -0,0 +1,101 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scss + +import ( + "github.com/bep/go-tocss/scss" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/resource" + "github.com/mitchellh/mapstructure" +) + +type Client struct { + rs *resource.Spec + sfs *filesystems.SourceFilesystem +} + +func New(fs *filesystems.SourceFilesystem, rs *resource.Spec) (*Client, error) { + return &Client{sfs: fs, rs: rs}, nil +} + +type Options struct { + + // Hugo, will by default, just replace the extension of the source + // to .css, e.g. "scss/main.scss" becomes "scss/main.css". You can + // control this by setting this, e.g. "styles/main.css" will create + // a Resource with that as a base for RelPermalink etc. + TargetPath string + + // Default is nested. + // One of nested, expanded, compact, compressed. + OutputStyle string + + // Precision of floating point math. + Precision int + + // When enabled, Hugo will generate a source map. + EnableSourceMap bool +} + +type options struct { + // The options we receive from the end user. + from Options + + // The options we send to the SCSS library. + to scss.Options +} + +func (c *Client) ToCSS(res resource.Resource, opts Options) (resource.Resource, error) { + internalOptions := options{ + from: opts, + } + + // Transfer values from client. + internalOptions.to.Precision = opts.Precision + internalOptions.to.OutputStyle = scss.OutputStyleFromString(opts.OutputStyle) + + if internalOptions.to.Precision == 0 { + // bootstrap-sass requires 8 digits precision. The libsass default is 5. + // https://github.com/twbs/bootstrap-sass/blob/master/README.md#sass-number-precision + internalOptions.to.Precision = 8 + } + + return c.rs.Transform( + res, + &toCSSTransformation{c: c, options: internalOptions}, + ) +} + +type toCSSTransformation struct { + c *Client + options options +} + +func (t *toCSSTransformation) Key() resource.ResourceTransformationKey { + return resource.NewResourceTransformationKey("tocss", t.options.from) +} + +func DecodeOptions(m map[string]interface{}) (opts Options, err error) { + if m == nil { + return + } + err = mapstructure.WeakDecode(m, &opts) + + if opts.TargetPath != "" { + opts.TargetPath = helpers.ToSlashTrimLeading(opts.TargetPath) + } + + return +} diff --git a/resource/tocss/scss/tocss.go b/resource/tocss/scss/tocss.go new file mode 100644 index 000000000..d606e9832 --- /dev/null +++ b/resource/tocss/scss/tocss.go @@ -0,0 +1,111 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build extended + +package scss + +import ( + "fmt" + "io" + "path" + "strings" + + "github.com/bep/go-tocss/scss" + "github.com/bep/go-tocss/scss/libsass" + "github.com/bep/go-tocss/tocss" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. This feature requires Hugo to be built with the extended tag. +func Supports() bool { + return true +} + +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + ctx.OutMediaType = media.CSSType + + var outName string + if t.options.from.TargetPath != "" { + ctx.OutPath = t.options.from.TargetPath + } else { + ctx.ReplaceOutPathExtension(".css") + } + + outName = path.Base(ctx.OutPath) + + options := t.options + + // We may allow the end user to add IncludePaths later, if we find a use + // case for that. + options.to.IncludePaths = t.c.sfs.RealDirs(path.Dir(ctx.SourcePath)) + + if ctx.InMediaType.SubType == media.SASSType.SubType { + options.to.SassSyntax = true + } + + if options.from.EnableSourceMap { + + options.to.SourceMapFilename = outName + ".map" + options.to.SourceMapRoot = t.c.rs.WorkingDir + + // Setting this to the relative input filename will get the source map + // more correct for the main entry path (main.scss typically), but + // it will mess up the import mappings. As a workaround, we do a replacement + // in the source map itself (see below). + //options.InputPath = inputPath + options.to.OutputPath = outName + options.to.SourceMapContents = true + options.to.OmitSourceMapURL = false + options.to.EnableEmbeddedSourceMap = false + } + + res, err := t.c.toCSS(options.to, ctx.To, ctx.From) + if err != nil { + return err + } + + if options.from.EnableSourceMap && res.SourceMapContent != "" { + sourcePath := t.c.sfs.RealFilename(ctx.SourcePath) + + if strings.HasPrefix(sourcePath, t.c.rs.WorkingDir) { + sourcePath = strings.TrimPrefix(sourcePath, t.c.rs.WorkingDir+helpers.FilePathSeparator) + } + + // This is a workaround for what looks like a bug in Libsass. But + // getting this resolution correct in tools like Chrome Workspaces + // is important enough to go this extra mile. + mapContent := strings.Replace(res.SourceMapContent, `stdin",`, fmt.Sprintf("%s\",", sourcePath), 1) + + return ctx.PublishSourceMap(mapContent) + } + return nil +} + +func (c *Client) toCSS(options scss.Options, dst io.Writer, src io.Reader) (tocss.Result, error) { + var res tocss.Result + + transpiler, err := libsass.New(options) + if err != nil { + return res, err + } + + res, err = transpiler.Execute(dst, src) + if err != nil { + return res, fmt.Errorf("SCSS processing failed: %s", err) + } + + return res, nil +} diff --git a/resource/tocss/scss/tocss_notavailable.go b/resource/tocss/scss/tocss_notavailable.go new file mode 100644 index 000000000..69b4fc655 --- /dev/null +++ b/resource/tocss/scss/tocss_notavailable.go @@ -0,0 +1,30 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !extended + +package scss + +import ( + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/resource" +) + +// Used in tests. +func Supports() bool { + return false +} + +func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { + return errors.FeatureNotAvailableErr +} diff --git a/resource/transform.go b/resource/transform.go new file mode 100644 index 000000000..6a100ddc4 --- /dev/null +++ b/resource/transform.go @@ -0,0 +1,487 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "bytes" + "path" + "strconv" + + "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/helpers" + "github.com/mitchellh/hashstructure" + "github.com/spf13/afero" + + "fmt" + "io" + "sync" + + "github.com/gohugoio/hugo/media" + + bp "github.com/gohugoio/hugo/bufferpool" +) + +var ( + _ ContentResource = (*transformedResource)(nil) + _ ReadSeekCloserResource = (*transformedResource)(nil) +) + +func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) { + return &transformedResource{ + Resource: r, + transformation: t, + transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})}, + cache: s.ResourceCache}, nil +} + +type ResourceTransformationCtx struct { + // The content to transform. + From io.Reader + + // The target of content transformation. + // The current implementation requires that r is written to w + // even if no transformation is performed. + To io.Writer + + // This is the relative path to the original source. Unix styled slashes. + SourcePath string + + // This is the relative target path to the resource. Unix styled slashes. + InPath string + + // The relative target path to the transformed resource. Unix styled slashes. + OutPath string + + // The input media type + InMediaType media.Type + + // The media type of the transformed resource. + OutMediaType media.Type + + // Data data can be set on the transformed Resource. Not that this need + // to be simple types, as it needs to be serialized to JSON and back. + Data map[string]interface{} + + // This is used to publis additional artifacts, e.g. source maps. + // We may improve this. + OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error) +} + +// AddOutPathIdentifier transforming InPath to OutPath adding an identifier, +// eg '.min' before any extension. +func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) { + ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier) +} + +func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { + dir, file := path.Split(inPath) + base, ext := helpers.PathAndExt(file) + return path.Join(dir, (base + identifier + ext)) +} + +// ReplaceOutPathExtension transforming InPath to OutPath replacing the file +// extension, e.g. ".scss" +func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { + dir, file := path.Split(ctx.InPath) + base, _ := helpers.PathAndExt(file) + ctx.OutPath = path.Join(dir, (base + newExt)) +} + +// PublishSourceMap writes the content to the target folder of the main resource +// with the ".map" extension added. +func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { + target := ctx.OutPath + ".map" + f, err := ctx.OpenResourcePublisher(target) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write([]byte(content)) + return err +} + +// ResourceTransformationKey are provided by the different transformation implementations. +// It identifies the transformation (name) and its configuration (elements). +// We combine this in a chain with the rest of the transformations +// with the target filename and a content hash of the origin to use as cache key. +type ResourceTransformationKey struct { + name string + elements []interface{} +} + +// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation +// name and elements. We will create a 64 bit FNV hash from the elements, which when combined +// with the other key elements should be unique for all practical applications. +func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { + return ResourceTransformationKey{name: name, elements: elements} +} + +// Do not change this without good reasons. +func (k ResourceTransformationKey) key() string { + if len(k.elements) == 0 { + return k.name + } + + sb := bp.GetBuffer() + defer bp.PutBuffer(sb) + + sb.WriteString(k.name) + for _, element := range k.elements { + hash, err := hashstructure.Hash(element, nil) + if err != nil { + panic(err) + } + sb.WriteString("_") + sb.WriteString(strconv.FormatUint(hash, 10)) + } + + return sb.String() +} + +// ResourceTransformation is the interface that a resource transformation step +// needs to implement. +type ResourceTransformation interface { + Key() ResourceTransformationKey + Transform(ctx *ResourceTransformationCtx) error +} + +// We will persist this information to disk. +type transformedResourceMetadata struct { + Target string `json:"Target"` + MediaTypeV string `json:"MediaType"` + MetaData map[string]interface{} `json:"Data"` +} + +type transformedResource struct { + cache *ResourceCache + + // This is the filename inside resources/_gen/assets + sourceFilename string + + linker permalinker + + // The transformation to apply. + transformation ResourceTransformation + + // We apply the tranformations lazily. + transformInit sync.Once + transformErr error + + // The transformed values + content string + contentInit sync.Once + transformedResourceMetadata + + // The source + Resource +} + +func (r *transformedResource) ReadSeekCloser() (ReadSeekCloser, error) { + rc, ok := r.Resource.(ReadSeekCloserResource) + if !ok { + return nil, fmt.Errorf("resource %T is not a ReadSeekerCloserResource", rc) + } + return rc.ReadSeekCloser() +} + +func (r *transformedResource) transferTransformedValues(another *transformedResource) { + if another.content != "" { + r.contentInit.Do(func() { + r.content = another.content + }) + } + r.transformedResourceMetadata = another.transformedResourceMetadata +} + +func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { + f, meta, found := r.cache.getFromFile(key) + if !found { + return nil + } + r.transformedResourceMetadata = meta + r.sourceFilename = f.Name() + + return f +} + +func (r *transformedResource) Content() (interface{}, error) { + if err := r.initTransform(true); err != nil { + return nil, err + } + if err := r.initContent(); err != nil { + return "", err + } + return r.content, nil +} + +func (r *transformedResource) Data() interface{} { + return r.MetaData +} + +func (r *transformedResource) MediaType() media.Type { + if err := r.initTransform(false); err != nil { + return media.Type{} + } + m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV) + return m +} + +func (r *transformedResource) Permalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.permalinkFor(r.Target) +} + +func (r *transformedResource) RelPermalink() string { + if err := r.initTransform(false); err != nil { + return "" + } + return r.linker.relPermalinkFor(r.Target) +} + +func (r *transformedResource) initContent() error { + var err error + r.contentInit.Do(func() { + var b []byte + b, err := afero.ReadFile(r.cache.rs.Resources.Fs, r.sourceFilename) + if err != nil { + return + } + r.content = string(b) + }) + return err +} + +func (r *transformedResource) transform(setContent bool) (err error) { + + openPublishFileForWriting := func(relTargetPath string) (io.WriteCloser, error) { + return openFileForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathFor(relTargetPath)) + } + + // This can be the last resource in a chain. + // Rewind and create a processing chain. + var chain []Resource + current := r + for { + rr := current.Resource + chain = append(chain[:0], append([]Resource{rr}, chain[0:]...)...) + if tr, ok := rr.(*transformedResource); ok { + current = tr + } else { + break + } + } + + // Append the current transformer at the end + chain = append(chain, r) + + first := chain[0] + + contentrc, err := contentReadSeekerCloser(first) + if err != nil { + return err + } + defer contentrc.Close() + + // Files with a suffix will be stored in cache (both on disk and in memory) + // partitioned by their suffix. There will be other files below /other. + // This partition is also how we determine what to delete on server reloads. + var key, base string + for _, element := range chain { + switch v := element.(type) { + case *transformedResource: + key = key + "_" + v.transformation.Key().key() + case permalinker: + r.linker = v + p := v.relTargetPath() + if p == "" { + panic("target path needed for key creation") + } + partition := ResourceKeyPartition(p) + base = partition + "/" + p + default: + return fmt.Errorf("transformation not supported for type %T", element) + } + } + + key = r.cache.cleanKey(base + "_" + helpers.MD5String(key)) + + cached, found := r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + return + } + + // Acquire a write lock for the named transformation. + r.cache.nlocker.Lock(key) + // Check the cache again. + cached, found = r.cache.get(key) + if found { + r.transferTransformedValues(cached.(*transformedResource)) + r.cache.nlocker.Unlock(key) + return + } + defer r.cache.nlocker.Unlock(key) + defer r.cache.set(key, r) + + b1 := bp.GetBuffer() + b2 := bp.GetBuffer() + defer bp.PutBuffer(b1) + defer bp.PutBuffer(b2) + + tctx := &ResourceTransformationCtx{ + Data: r.transformedResourceMetadata.MetaData, + OpenResourcePublisher: openPublishFileForWriting, + } + + tctx.InMediaType = first.MediaType() + tctx.OutMediaType = first.MediaType() + tctx.From = contentrc + tctx.To = b1 + + if r.linker != nil { + tctx.InPath = r.linker.targetPath() + tctx.SourcePath = tctx.InPath + } + + counter := 0 + + var transformedContentr io.Reader + + for _, element := range chain { + tr, ok := element.(*transformedResource) + if !ok { + continue + } + counter++ + if counter != 1 { + tctx.InMediaType = tctx.OutMediaType + } + if counter%2 == 0 { + tctx.From = b1 + b2.Reset() + tctx.To = b2 + } else { + if counter != 1 { + // The first reader is the file. + tctx.From = b2 + } + b1.Reset() + tctx.To = b1 + } + + if err := tr.transformation.Transform(tctx); err != nil { + if err == errors.FeatureNotAvailableErr { + // This transformation is not available in this + // Hugo installation (scss not compiled in, PostCSS not available etc.) + // If a prepared bundle for this transformation chain is available, use that. + f := r.tryTransformedFileCache(key) + if f == nil { + return fmt.Errorf("failed to transform %q (%s): %s", tctx.InPath, tctx.InMediaType.Type(), err) + } + transformedContentr = f + defer f.Close() + + // The reader above is all we need. + break + } + + // Abort. + return err + } + + if tctx.OutPath != "" { + tctx.InPath = tctx.OutPath + tctx.OutPath = "" + } + } + + if transformedContentr == nil { + r.Target = tctx.InPath + r.MediaTypeV = tctx.OutMediaType.Type() + } + + publicw, err := openPublishFileForWriting(r.Target) + if err != nil { + r.transformErr = err + return + } + defer publicw.Close() + + publishwriters := []io.Writer{publicw} + + if transformedContentr == nil { + // Also write it to the cache + metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) + if err != nil { + return err + } + r.sourceFilename = metaw.Name() + defer metaw.Close() + + publishwriters = append(publishwriters, metaw) + + if counter > 0 { + transformedContentr = tctx.To.(*bytes.Buffer) + } else { + transformedContentr = contentrc + } + } + + // Also write it to memory + var contentmemw *bytes.Buffer + + if setContent { + contentmemw = bp.GetBuffer() + defer bp.PutBuffer(contentmemw) + publishwriters = append(publishwriters, contentmemw) + } + + publishw := io.MultiWriter(publishwriters...) + _, r.transformErr = io.Copy(publishw, transformedContentr) + + if setContent { + r.contentInit.Do(func() { + r.content = contentmemw.String() + }) + } + + return nil + +} +func (r *transformedResource) initTransform(setContent bool) error { + r.transformInit.Do(func() { + if err := r.transform(setContent); err != nil { + r.transformErr = err + r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err) + } + }) + return r.transformErr +} + +// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource. +func contentReadSeekerCloser(r Resource) (ReadSeekCloser, error) { + switch rr := r.(type) { + case ReadSeekCloserResource: + rc, err := rr.ReadSeekCloser() + if err != nil { + return nil, err + } + return rc, nil + default: + return nil, fmt.Errorf("cannot tranform content of Resource of type %T", r) + + } +} diff --git a/resource/transform_test.go b/resource/transform_test.go new file mode 100644 index 000000000..df68e780d --- /dev/null +++ b/resource/transform_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resource + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +type testStruct struct { + Name string + V1 int64 + V2 int32 + V3 int + V4 uint64 +} + +func TestResourceTransformationKey(t *testing.T) { + // We really need this key to be portable across OSes. + key := NewResourceTransformationKey("testing", + testStruct{Name: "test", V1: int64(10), V2: int32(20), V3: 30, V4: uint64(40)}) + assert := require.New(t) + assert.Equal(key.key(), "testing_518996646957295636") +} |