diff options
author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-01-02 13:58:32 +0300 |
---|---|---|
committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2019-01-02 16:25:37 +0300 |
commit | ce8a09a4c0661dece931ab1173e4f09e8e04aa38 (patch) | |
tree | 870372211e342312d0ab3034ee03285c6f698f9d /resources/resource.go | |
parent | 669ada436787311cc5d02dae5b88e60a09adda58 (diff) |
resources: Move resource interfaces into its own package
Diffstat (limited to 'resources/resource.go')
-rw-r--r-- | resources/resource.go | 741 |
1 files changed, 741 insertions, 0 deletions
diff --git a/resources/resource.go b/resources/resource.go new file mode 100644 index 000000000..742903e80 --- /dev/null +++ b/resources/resource.go @@ -0,0 +1,741 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "fmt" + "io" + "io/ioutil" + "mime" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/tpl" + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/collections" + "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/resources/resource" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/source" +) + +var ( + _ resource.ContentResource = (*genericResource)(nil) + _ resource.ReadSeekCloserResource = (*genericResource)(nil) + _ resource.Resource = (*genericResource)(nil) + _ resource.Source = (*genericResource)(nil) + _ resource.Cloner = (*genericResource)(nil) + _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) + _ permalinker = (*genericResource)(nil) + _ collections.Slicer = (*genericResource)(nil) + _ resource.Identifier = (*genericResource)(nil) +) + +var noData = make(map[string]interface{}) + +type permalinker interface { + relPermalinkFor(target string) string + permalinkFor(target string) string + relTargetPathsFor(target string) []string + relTargetPaths() []string + targetPath() string +} + +type Spec struct { + *helpers.PathSpec + + MediaTypes media.Types + OutputFormats output.Formats + + Logger *loggers.Logger + + TextTemplates tpl.TemplateParseFinder + + // Holds default filter settings etc. + imaging *Imaging + + imageCache *imageCache + ResourceCache *ResourceCache + FileCaches filecache.Caches +} + +func NewSpec( + s *helpers.PathSpec, + fileCaches filecache.Caches, + logger *loggers.Logger, + outputFormats output.Formats, + mimeTypes media.Types) (*Spec, error) { + + imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) + if err != nil { + return nil, err + } + + if logger == nil { + logger = loggers.NewErrorLogger() + } + + rs := &Spec{PathSpec: s, + Logger: logger, + imaging: &imaging, + MediaTypes: mimeTypes, + OutputFormats: outputFormats, + FileCaches: fileCaches, + imageCache: newImageCache( + fileCaches.ImageCache(), + + s, + )} + + rs.ResourceCache = newResourceCache(rs) + + return rs, nil + +} + +type ResourceSourceDescriptor struct { + // TargetPathBuilder is a callback to create target paths's relative to its owner. + TargetPathBuilder func(base string) string + + // Need one of these to load the resource content. + SourceFile source.File + OpenReadSeekCloser resource.OpenReadSeekCloser + + // If OpenReadSeekerCloser is not set, we use this to open the file. + SourceFilename string + + // The relative target filename without any language code. + RelTargetFilename string + + // Any base path prepeneded to the permalink. + // Typically the language code if this resource should be published to its sub-folder. + URLBase string + + // Any base paths prepended to the target path. This will also typically be the + // language code, but setting it here means that it should not have any effect on + // the permalink. + // This may be several values. In multihost mode we may publish the same resources to + // multiple targets. + TargetBasePaths []string + + // Delay publishing until either Permalink or RelPermalink is called. Maybe never. + LazyPublish bool +} + +func (r ResourceSourceDescriptor) Filename() string { + if r.SourceFile != nil { + return r.SourceFile.Filename() + } + return r.SourceFilename +} + +func (r *Spec) sourceFs() afero.Fs { + return r.PathSpec.BaseFs.Content.Fs +} + +func (r *Spec) New(fd ResourceSourceDescriptor) (resource.Resource, error) { + return r.newResourceForFs(r.sourceFs(), fd) +} + +func (r *Spec) NewForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { + return r.newResourceForFs(sourceFs, fd) +} + +func (r *Spec) newResourceForFs(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { + if fd.OpenReadSeekCloser == nil { + if fd.SourceFile != nil && fd.SourceFilename != "" { + return nil, errors.New("both SourceFile and AbsSourceFilename provided") + } else if fd.SourceFile == nil && fd.SourceFilename == "" { + return nil, errors.New("either SourceFile or AbsSourceFilename must be provided") + } + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = fd.Filename() + } + + if len(fd.TargetBasePaths) == 0 { + // If not set, we publish the same resource to all hosts. + fd.TargetBasePaths = r.MultihostTargetBasePaths + } + + return r.newResource(sourceFs, fd) +} + +func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (resource.Resource, error) { + var fi os.FileInfo + var sourceFilename string + + if fd.OpenReadSeekCloser != nil { + + } else if fd.SourceFilename != "" { + var err error + fi, err = sourceFs.Stat(fd.SourceFilename) + if err != nil { + return nil, err + } + sourceFilename = fd.SourceFilename + } else { + fi = fd.SourceFile.FileInfo() + sourceFilename = fd.SourceFile.Filename() + } + + if fd.RelTargetFilename == "" { + fd.RelTargetFilename = sourceFilename + } + + ext := filepath.Ext(fd.RelTargetFilename) + mimeType, found := r.MediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")) + // TODO(bep) we need to handle these ambigous types better, but in this context + // we most likely want the application/xml type. + if mimeType.Suffix() == "xml" && mimeType.SubType == "rss" { + mimeType, found = r.MediaTypes.GetByType("application/xml") + } + + if !found { + mimeStr := mime.TypeByExtension(ext) + if mimeStr != "" { + mimeType, _ = media.FromStringAndExt(mimeStr, ext) + } + } + + gr := r.newGenericResourceWithBase( + sourceFs, + fd.LazyPublish, + fd.OpenReadSeekCloser, + fd.URLBase, + fd.TargetBasePaths, + fd.TargetPathBuilder, + fi, + sourceFilename, + fd.RelTargetFilename, + mimeType) + + if mimeType.MainType == "image" { + ext := strings.ToLower(helpers.Ext(sourceFilename)) + + imgFormat, ok := imageFormats[ext] + if !ok { + // This allows SVG etc. to be used as resources. They will not have the methods of the Image, but + // that would not (currently) have worked. + return gr, nil + } + + if err := gr.initHash(); err != nil { + return nil, err + } + + return &Image{ + format: imgFormat, + imaging: r.imaging, + genericResource: gr}, nil + } + return gr, nil + +} + +// TODO(bep) unify +func (r *Spec) IsInImageCache(key string) bool { + // This is used for cache pruning. We currently only have images, but we could + // imagine expanding on this. + return r.imageCache.isInCache(key) +} + +func (r *Spec) DeleteCacheByPrefix(prefix string) { + r.imageCache.deleteByPrefix(prefix) +} + +func (r *Spec) ClearCaches() { + r.imageCache.clear() + r.ResourceCache.clear() +} + +func (r *Spec) CacheStats() string { + r.imageCache.mu.RLock() + defer r.imageCache.mu.RUnlock() + + s := fmt.Sprintf("Cache entries: %d", len(r.imageCache.store)) + + count := 0 + for k := range r.imageCache.store { + if count > 5 { + break + } + s += "\n" + k + count++ + } + + return s +} + +type dirFile struct { + // This is the directory component with Unix-style slashes. + dir string + // This is the file component. + file string +} + +func (d dirFile) path() string { + return path.Join(d.dir, d.file) +} + +type resourcePathDescriptor struct { + // The relative target directory and filename. + relTargetDirFile dirFile + + // Callback used to construct a target path relative to its owner. + targetPathBuilder func(rel string) string + + // baseURLDir is the fixed sub-folder for a resource in permalinks. This will typically + // be the language code if we publish to the language's sub-folder. + baseURLDir string + + // This will normally be the same as above, but this will only apply to publishing + // of resources. It may be mulltiple values when in multihost mode. + baseTargetPathDirs []string + + // baseOffset is set when the output format's path has a offset, e.g. for AMP. + baseOffset string +} + +type resourceContent struct { + content string + contentInit sync.Once +} + +type resourceHash struct { + hash string + hashInit sync.Once +} + +type publishOnce struct { + publisherInit sync.Once + publisherErr error + logger *loggers.Logger +} + +func (l *publishOnce) publish(s resource.Source) error { + l.publisherInit.Do(func() { + l.publisherErr = s.Publish() + if l.publisherErr != nil { + l.logger.ERROR.Printf("failed to publish Resource: %s", l.publisherErr) + } + }) + return l.publisherErr +} + +// genericResource represents a generic linkable resource. +type genericResource struct { + commonResource + resourcePathDescriptor + + title string + name string + params map[string]interface{} + + // Absolute filename to the source, including any content folder path. + // Note that this is absolute in relation to the filesystem it is stored in. + // It can be a base path filesystem, and then this filename will not match + // the path to the file on the real filesystem. + sourceFilename string + + // Will be set if this resource is backed by something other than a file. + openReadSeekerCloser resource.OpenReadSeekCloser + + // A hash of the source content. Is only calculated in caching situations. + *resourceHash + + // This may be set to tell us to look in another filesystem for this resource. + // We, by default, use the sourceFs filesystem in the spec below. + overriddenSourceFs afero.Fs + + spec *Spec + + resourceType string + mediaType media.Type + + osFileInfo os.FileInfo + + // We create copies of this struct, so this needs to be a pointer. + *resourceContent + + // May be set to signal lazy/delayed publishing. + *publishOnce +} + +type commonResource struct { +} + +func (l *genericResource) Data() interface{} { + return noData +} + +func (l *genericResource) Content() (interface{}, error) { + if err := l.initContent(); err != nil { + return nil, err + } + + return l.content, nil +} + +func (l *genericResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { + if l.openReadSeekerCloser != nil { + return l.openReadSeekerCloser() + } + f, err := l.sourceFs().Open(l.sourceFilename) + if err != nil { + return nil, err + } + return f, nil + +} + +func (l *genericResource) MediaType() media.Type { + return l.mediaType +} + +// Implement the Cloner interface. +func (l genericResource) WithNewBase(base string) resource.Resource { + l.baseOffset = base + l.resourceContent = &resourceContent{} + return &l +} + +// Slice is not meant to be used externally. It's a bridge function +// for the template functions. See collections.Slice. +func (commonResource) Slice(in interface{}) (interface{}, error) { + switch items := in.(type) { + case resource.Resources: + return items, nil + case []interface{}: + groups := make(resource.Resources, len(items)) + for i, v := range items { + g, ok := v.(resource.Resource) + if !ok { + return nil, fmt.Errorf("type %T is not a Resource", v) + } + groups[i] = g + } + return groups, nil + default: + return nil, fmt.Errorf("invalid slice type %T", items) + } +} + +func (l *genericResource) initHash() error { + var err error + l.hashInit.Do(func() { + var hash string + var f hugio.ReadSeekCloser + f, err = l.ReadSeekCloser() + if err != nil { + err = errors.Wrap(err, "failed to open source file") + return + } + defer f.Close() + + hash, err = helpers.MD5FromFileFast(f) + if err != nil { + return + } + l.hash = hash + + }) + + return err +} + +func (l *genericResource) initContent() error { + var err error + l.contentInit.Do(func() { + var r hugio.ReadSeekCloser + r, err = l.ReadSeekCloser() + if err != nil { + return + } + defer r.Close() + + var b []byte + b, err = ioutil.ReadAll(r) + if err != nil { + return + } + + l.content = string(b) + + }) + + return err +} + +func (l *genericResource) sourceFs() afero.Fs { + if l.overriddenSourceFs != nil { + return l.overriddenSourceFs + } + return l.spec.sourceFs() +} + +func (l *genericResource) publishIfNeeded() { + if l.publishOnce != nil { + l.publishOnce.publish(l) + } +} + +func (l *genericResource) Permalink() string { + l.publishIfNeeded() + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL()) +} + +func (l *genericResource) RelPermalink() string { + l.publishIfNeeded() + return l.relPermalinkFor(l.relTargetDirFile.path()) +} + +func (l *genericResource) Key() string { + return l.relTargetDirFile.path() +} + +func (l *genericResource) relPermalinkFor(target string) string { + return l.relPermalinkForRel(target, false) + +} +func (l *genericResource) permalinkFor(target string) string { + return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL()) + +} +func (l *genericResource) relTargetPathsFor(target string) []string { + return l.relTargetPathsForRel(target) +} + +func (l *genericResource) relTargetPaths() []string { + return l.relTargetPathsForRel(l.targetPath()) +} + +func (l *genericResource) Name() string { + return l.name +} + +func (l *genericResource) Title() string { + return l.title +} + +func (l *genericResource) Params() map[string]interface{} { + return l.params +} + +func (l *genericResource) setTitle(title string) { + l.title = title +} + +func (l *genericResource) setName(name string) { + l.name = name +} + +func (l *genericResource) updateParams(params map[string]interface{}) { + if l.params == nil { + l.params = params + return + } + + // Sets the params not already set + for k, v := range params { + if _, found := l.params[k]; !found { + l.params[k] = v + } + } +} + +func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string { + return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true)) +} + +func (l *genericResource) relTargetPathsForRel(rel string) []string { + if len(l.baseTargetPathDirs) == 0 { + return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)} + } + + var targetPaths = make([]string, len(l.baseTargetPathDirs)) + for i, dir := range l.baseTargetPathDirs { + targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false) + } + return targetPaths +} + +func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string { + if addBaseTargetPath && len(l.baseTargetPathDirs) > 1 { + panic("multiple baseTargetPathDirs") + } + var basePath string + if addBaseTargetPath && len(l.baseTargetPathDirs) > 0 { + basePath = l.baseTargetPathDirs[0] + } + + return l.relTargetPathForRelAndBasePath(rel, basePath, isAbs, isURL) +} + +func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, isAbs, isURL bool) string { + if l.targetPathBuilder != nil { + rel = l.targetPathBuilder(rel) + } + + if isURL && l.baseURLDir != "" { + rel = path.Join(l.baseURLDir, rel) + } + + if basePath != "" { + rel = path.Join(basePath, rel) + } + + if l.baseOffset != "" { + rel = path.Join(l.baseOffset, rel) + } + + if isURL { + bp := l.spec.PathSpec.GetBasePath(!isAbs) + if bp != "" { + rel = path.Join(bp, rel) + } + } + + if len(rel) == 0 || rel[0] != '/' { + rel = "/" + rel + } + + return rel +} + +func (l *genericResource) ResourceType() string { + return l.resourceType +} + +func (l *genericResource) String() string { + return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) +} + +func (l *genericResource) Publish() error { + fr, err := l.ReadSeekCloser() + if err != nil { + return err + } + defer fr.Close() + fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.targetFilenames()...) + if err != nil { + return err + } + defer fw.Close() + + _, err = io.Copy(fw, fr) + return err +} + +// Path is stored with Unix style slashes. +func (l *genericResource) targetPath() string { + return l.relTargetDirFile.path() +} + +func (l *genericResource) targetFilenames() []string { + paths := l.relTargetPaths() + for i, p := range paths { + paths[i] = filepath.Clean(p) + } + return paths +} + +// TODO(bep) clean up below +func (r *Spec) newGenericResource(sourceFs afero.Fs, + targetPathBuilder func(base string) string, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + return r.newGenericResourceWithBase( + sourceFs, + false, + nil, + "", + nil, + targetPathBuilder, + osFileInfo, + sourceFilename, + baseFilename, + mediaType, + ) + +} + +func (r *Spec) newGenericResourceWithBase( + sourceFs afero.Fs, + lazyPublish bool, + openReadSeekerCloser resource.OpenReadSeekCloser, + urlBaseDir string, + targetPathBaseDirs []string, + targetPathBuilder func(base string) string, + osFileInfo os.FileInfo, + sourceFilename, + baseFilename string, + mediaType media.Type) *genericResource { + + // This value is used both to construct URLs and file paths, but start + // with a Unix-styled path. + baseFilename = helpers.ToSlashTrimLeading(baseFilename) + fpath, fname := path.Split(baseFilename) + + var resourceType string + if mediaType.MainType == "image" { + resourceType = mediaType.MainType + } else { + resourceType = mediaType.SubType + } + + pathDescriptor := resourcePathDescriptor{ + baseURLDir: urlBaseDir, + baseTargetPathDirs: targetPathBaseDirs, + targetPathBuilder: targetPathBuilder, + relTargetDirFile: dirFile{dir: fpath, file: fname}, + } + + var po *publishOnce + if lazyPublish { + po = &publishOnce{logger: r.Logger} + } + + return &genericResource{ + openReadSeekerCloser: openReadSeekerCloser, + publishOnce: po, + resourcePathDescriptor: pathDescriptor, + overriddenSourceFs: sourceFs, + osFileInfo: osFileInfo, + sourceFilename: sourceFilename, + mediaType: mediaType, + resourceType: resourceType, + spec: r, + params: make(map[string]interface{}), + name: baseFilename, + title: baseFilename, + resourceContent: &resourceContent{}, + resourceHash: &resourceHash{}, + } +} |