From 3cdf19e9b7e46c57a9bb43ff02199177feb55768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 24 Jul 2017 09:00:23 +0200 Subject: :sparkles: Implement Page bundling and image handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit is not the smallest in Hugo's history. Some hightlights include: * Page bundles (for complete articles, keeping images and content together etc.). * Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`. * Processed images are cached inside `resources/_gen/images` (default) in your project. * Symbolic links (both files and dirs) are now allowed anywhere inside /content * A new table based build summary * The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below). A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory: ```bash ▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render" benchmark old ns/op new ns/op delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86% benchmark old allocs new allocs delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30% benchmark old bytes new bytes delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64% ``` Fixes #3651 Closes #3158 Fixes #1014 Closes #2021 Fixes #1240 Updates #3757 --- source/content_directory_test.go | 11 +- source/dirs.go | 11 +- source/file.go | 172 ---------------------------- source/fileInfo.go | 213 +++++++++++++++++++++++++++++++++++ source/fileInfo_test.go | 22 ++++ source/file_test.go | 62 ---------- source/filesystem.go | 108 +++++------------- source/filesystem_test.go | 66 +++-------- source/inmemory.go | 25 ----- source/lazy_file_reader.go | 170 ---------------------------- source/lazy_file_reader_test.go | 236 --------------------------------------- source/sourceSpec.go | 117 +++++++++++++++++++ 12 files changed, 406 insertions(+), 807 deletions(-) delete mode 100644 source/file.go create mode 100644 source/fileInfo.go create mode 100644 source/fileInfo_test.go delete mode 100644 source/file_test.go delete mode 100644 source/inmemory.go delete mode 100644 source/lazy_file_reader.go delete mode 100644 source/lazy_file_reader_test.go create mode 100644 source/sourceSpec.go (limited to 'source') diff --git a/source/content_directory_test.go b/source/content_directory_test.go index 4ff12af8d..9874acec2 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -14,6 +14,7 @@ package source import ( + "path/filepath" "testing" "github.com/gohugoio/hugo/hugofs" @@ -41,21 +42,21 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { {"foobar/bar~foo.md", false, nil}, {"foobar/foo.md", true, []string{"\\.md$", "\\.boo$"}}, {"foobar/foo.html", false, []string{"\\.md$", "\\.boo$"}}, - {"foobar/foo.md", true, []string{"^foo"}}, - {"foobar/foo.md", false, []string{"*", "\\.md$", "\\.boo$"}}, + {"foobar/foo.md", true, []string{"foo.md$"}}, + {"foobar/foo.md", true, []string{"*", "\\.md$", "\\.boo$"}}, {"foobar/.#content.md", true, []string{"/\\.#"}}, {".#foobar.md", true, []string{"^\\.#"}}, } - for _, test := range tests { + for i, test := range tests { v := viper.New() v.Set("ignoreFiles", test.ignoreFilesRegexpes) s := NewSourceSpec(v, hugofs.NewMem(v)) - if ignored := s.isNonProcessablePath(test.path); test.ignore != ignored { - t.Errorf("File not ignored. Expected: %t, got: %t", test.ignore, ignored) + if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored { + t.Errorf("[%d] File not ignored", i) } } } diff --git a/source/dirs.go b/source/dirs.go index 1e6850da7..49a849453 100644 --- a/source/dirs.go +++ b/source/dirs.go @@ -38,7 +38,7 @@ type Dirs struct { staticDirs []string AbsStaticDirs []string - publishDir string + Language *helpers.Language } // NewDirs creates a new dirs with the given configuration and filesystem. @@ -48,7 +48,12 @@ func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, er return nil, err } - d := &Dirs{pathSpec: ps, logger: logger} + var l *helpers.Language + if language, ok := cfg.(*helpers.Language); ok { + l = language + } + + d := &Dirs{Language: l, pathSpec: ps, logger: logger} return d, d.init(cfg) @@ -96,8 +101,6 @@ func (d *Dirs) init(cfg config.Provider) error { d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator } - d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator - return nil } diff --git a/source/file.go b/source/file.go deleted file mode 100644 index a630431c6..000000000 --- a/source/file.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "io" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" -) - -// SourceSpec abstracts language-specific file creation. -type SourceSpec struct { - Cfg config.Provider - Fs *hugofs.Fs - - languages map[string]interface{} - defaultContentLanguage string -} - -// NewSourceSpec initializes SourceSpec using languages from a given configuration. -func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) SourceSpec { - defaultLang := cfg.GetString("defaultContentLanguage") - languages := cfg.GetStringMap("languages") - return SourceSpec{Cfg: cfg, Fs: fs, languages: languages, defaultContentLanguage: defaultLang} -} - -// File represents a source content file. -// All paths are relative from the source directory base -type File struct { - relpath string // Original relative path, e.g. section/foo.txt - logicalName string // foo.txt - baseName string // `post` for `post.md`, also `post.en` for `post.en.md` - Contents io.Reader - section string // The first directory - dir string // The relative directory Path (minus file name) - ext string // Just the ext (eg txt) - uniqueID string // MD5 of the file's path - - translationBaseName string // `post` for `post.es.md` (if `Multilingual` is enabled.) - lang string // The language code if `Multilingual` is enabled -} - -// UniqueID is the MD5 hash of the file's path and is for most practical applications, -// Hugo content files being one of them, considered to be unique. -func (f *File) UniqueID() string { - return f.uniqueID -} - -// String returns the file's content as a string. -func (f *File) String() string { - return helpers.ReaderToString(f.Contents) -} - -// Bytes returns the file's content as a byte slice. -func (f *File) Bytes() []byte { - return helpers.ReaderToBytes(f.Contents) -} - -// BaseFileName is a filename without extension. -func (f *File) BaseFileName() string { - return f.baseName -} - -// TranslationBaseName is a filename with no extension, -// not even the optional language extension part. -func (f *File) TranslationBaseName() string { - return f.translationBaseName -} - -// Lang for this page, if `Multilingual` is enabled on your site. -func (f *File) Lang() string { - return f.lang -} - -// Section is first directory below the content root. -func (f *File) Section() string { - return f.section -} - -// LogicalName is filename and extension of the file. -func (f *File) LogicalName() string { - return f.logicalName -} - -// SetDir sets the relative directory where this file lives. -// TODO(bep) Get rid of this. -func (f *File) SetDir(dir string) { - f.dir = dir -} - -// Dir gets the name of the directory that contains this file. -// The directory is relative to the content root. -func (f *File) Dir() string { - return f.dir -} - -// Extension gets the file extension, i.e "myblogpost.md" will return "md". -func (f *File) Extension() string { - return f.ext -} - -// Ext is an alias for Extension. -func (f *File) Ext() string { - return f.Extension() -} - -// Path gets the relative path including file name and extension. -// The directory is relative to the content root. -func (f *File) Path() string { - return f.relpath -} - -// NewFileWithContents creates a new File pointer with the given relative path and -// content. The language defaults to "en". -func (sp SourceSpec) NewFileWithContents(relpath string, content io.Reader) *File { - file := sp.NewFile(relpath) - file.Contents = content - file.lang = "en" - return file -} - -// NewFile creates a new File pointer with the given relative path. -func (sp SourceSpec) NewFile(relpath string) *File { - f := &File{ - relpath: relpath, - } - - f.dir, f.logicalName = filepath.Split(f.relpath) - f.ext = strings.TrimPrefix(filepath.Ext(f.LogicalName()), ".") - f.baseName = helpers.Filename(f.LogicalName()) - - lang := strings.TrimPrefix(filepath.Ext(f.baseName), ".") - if _, ok := sp.languages[lang]; lang == "" || !ok { - f.lang = sp.defaultContentLanguage - f.translationBaseName = f.baseName - } else { - f.lang = lang - f.translationBaseName = helpers.Filename(f.baseName) - } - - f.section = helpers.GuessSection(f.Dir()) - f.uniqueID = helpers.Md5String(filepath.ToSlash(f.relpath)) - - return f -} - -// NewFileFromAbs creates a new File pointer with the given full file path path and -// content. -func (sp SourceSpec) NewFileFromAbs(base, fullpath string, content io.Reader) (f *File, err error) { - var name string - if name, err = helpers.GetRelativePath(fullpath, base); err != nil { - return nil, err - } - - return sp.NewFileWithContents(name, content), nil -} diff --git a/source/fileInfo.go b/source/fileInfo.go new file mode 100644 index 000000000..e4b4a80fb --- /dev/null +++ b/source/fileInfo.go @@ -0,0 +1,213 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/gohugoio/hugo/helpers" +) + +// fileInfo implements the File interface. +var ( + _ File = (*FileInfo)(nil) + _ ReadableFile = (*FileInfo)(nil) +) + +type File interface { + + // Filename gets the full path and filename to the file. + Filename() string + + // Path gets the relative path including file name and extension. + // The directory is relative to the content root. + Path() string + + // Dir gets the name of the directory that contains this file. + // The directory is relative to the content root. + Dir() string + + // Extension gets the file extension, i.e "myblogpost.md" will return "md". + Extension() string + // Ext is an alias for Extension. + Ext() string // Hmm... Deprecate Extension + + // Lang for this page, if `Multilingual` is enabled on your site. + Lang() string + + // LogicalName is filename and extension of the file. + LogicalName() string + + // Section is first directory below the content root. + Section() string + + // BaseFileName is a filename without extension. + BaseFileName() string + + // TranslationBaseName is a filename with no extension, + // not even the optional language extension part. + TranslationBaseName() string + + // UniqueID is the MD5 hash of the file's path and is for most practical applications, + // Hugo content files being one of them, considered to be unique. + UniqueID() string + + FileInfo() os.FileInfo + + String() string + + // Deprecated + Bytes() []byte +} + +// A ReadableFile is a File that is readable. +type ReadableFile interface { + File + Open() (io.ReadCloser, error) +} + +type FileInfo struct { + + // Absolute filename to the file on disk. + filename string + fi os.FileInfo + + // Derived from filename + ext string // Extension without any "." + lang string + + name string + + dir string + relDir string + relPath string + baseName string + translationBaseName string + section string + + uniqueID string + + sp *SourceSpec + + lazyInit sync.Once +} + +func (fi *FileInfo) Filename() string { return fi.filename } +func (fi *FileInfo) Path() string { return fi.relPath } +func (fi *FileInfo) Dir() string { return fi.relDir } +func (fi *FileInfo) Extension() string { return fi.Ext() } +func (fi *FileInfo) Ext() string { return fi.ext } +func (fi *FileInfo) Lang() string { return fi.lang } +func (fi *FileInfo) LogicalName() string { return fi.name } +func (fi *FileInfo) BaseFileName() string { return fi.baseName } +func (fi *FileInfo) TranslationBaseName() string { return fi.translationBaseName } + +func (fi *FileInfo) Section() string { + fi.init() + return fi.section +} + +func (fi *FileInfo) UniqueID() string { + fi.init() + return fi.uniqueID +} +func (fi *FileInfo) FileInfo() os.FileInfo { + return fi.fi +} + +func (fi *FileInfo) Bytes() []byte { + // Remove in Hugo 0.34 + helpers.Deprecated("File", "Bytes", "", false) + return []byte("") +} + +func (fi *FileInfo) String() string { return fi.BaseFileName() } + +// We create a lot of these FileInfo objects, but there are parts of it used only +// in some cases that is slightly expensive to construct. +func (fi *FileInfo) init() { + fi.lazyInit.Do(func() { + parts := strings.Split(fi.relDir, helpers.FilePathSeparator) + var section string + if len(parts) == 1 { + section = parts[0] + } else if len(parts) > 1 { + if parts[0] == "" { + section = parts[1] + } else { + section = parts[0] + } + } + + fi.section = section + + fi.uniqueID = helpers.MD5String(filepath.ToSlash(fi.relPath)) + + }) +} + +func (sp *SourceSpec) NewFileInfo(baseDir, filename string, fi os.FileInfo) *FileInfo { + dir, name := filepath.Split(filename) + + dir = strings.TrimSuffix(dir, helpers.FilePathSeparator) + baseDir = strings.TrimSuffix(baseDir, helpers.FilePathSeparator) + + relDir := "" + if dir != baseDir { + relDir = strings.TrimPrefix(dir, baseDir) + } + + relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator) + + relPath := filepath.Join(relDir, name) + + ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) + baseName := helpers.Filename(name) + + lang := strings.TrimPrefix(filepath.Ext(baseName), ".") + var translationBaseName string + + if _, ok := sp.Languages[lang]; lang == "" || !ok { + lang = sp.DefaultContentLanguage + translationBaseName = baseName + } else { + translationBaseName = helpers.Filename(baseName) + } + + f := &FileInfo{ + sp: sp, + filename: filename, + fi: fi, + lang: lang, + ext: ext, + dir: dir, + relDir: relDir, + relPath: relPath, + name: name, + baseName: baseName, + translationBaseName: translationBaseName, + } + + return f + +} + +// Open implements ReadableFile. +func (fi *FileInfo) Open() (io.ReadCloser, error) { + return fi.sp.Fs.Source.Open(fi.Filename()) +} diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go new file mode 100644 index 000000000..3f99497ad --- /dev/null +++ b/source/fileInfo_test.go @@ -0,0 +1,22 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" +) + +func TestFileInfo(t *testing.T) { + +} diff --git a/source/file_test.go b/source/file_test.go deleted file mode 100644 index 64ad6fb46..000000000 --- a/source/file_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "path/filepath" - "strings" - "testing" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" - - "github.com/stretchr/testify/assert" -) - -func TestFileUniqueID(t *testing.T) { - ss := newTestSourceSpec() - - f1 := File{uniqueID: "123"} - f2 := ss.NewFile("a") - - assert.Equal(t, "123", f1.UniqueID()) - assert.Equal(t, "0cc175b9c0f1b6a831c399e269772661", f2.UniqueID()) - - f3 := ss.NewFile(filepath.FromSlash("test1/index.md")) - f4 := ss.NewFile(filepath.FromSlash("test2/index.md")) - - assert.NotEqual(t, f3.UniqueID(), f4.UniqueID()) - - f5l := ss.NewFile("test3/index.md") - f5w := ss.NewFile(filepath.FromSlash("test3/index.md")) - - assert.Equal(t, f5l.UniqueID(), f5w.UniqueID()) -} - -func TestFileString(t *testing.T) { - ss := newTestSourceSpec() - assert.Equal(t, "abc", ss.NewFileWithContents("a", strings.NewReader("abc")).String()) - assert.Equal(t, "", ss.NewFile("a").String()) -} - -func TestFileBytes(t *testing.T) { - ss := newTestSourceSpec() - assert.Equal(t, []byte("abc"), ss.NewFileWithContents("a", strings.NewReader("abc")).Bytes()) - assert.Equal(t, []byte(""), ss.NewFile("a").Bytes()) -} - -func newTestSourceSpec() SourceSpec { - v := viper.New() - return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v} -} diff --git a/source/filesystem.go b/source/filesystem.go index e6e354e99..a5f2988e9 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -14,73 +14,52 @@ package source import ( - "io" "os" "path/filepath" - "regexp" "runtime" - "strings" + "sync" "github.com/gohugoio/hugo/helpers" - "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "golang.org/x/text/unicode/norm" ) -type Input interface { - Files() []*File -} - type Filesystem struct { - files []*File - Base string - AvoidPaths []string + files []ReadableFile + filesInit sync.Once + + Base string SourceSpec } -func (sp SourceSpec) NewFilesystem(base string, avoidPaths ...string) *Filesystem { - return &Filesystem{SourceSpec: sp, Base: base, AvoidPaths: avoidPaths} +type Input interface { + Files() []ReadableFile } -func (f *Filesystem) FilesByExts(exts ...string) []*File { - var newFiles []*File - - if len(exts) == 0 { - return f.Files() - } - - for _, x := range f.Files() { - for _, e := range exts { - if x.Ext() == strings.TrimPrefix(e, ".") { - newFiles = append(newFiles, x) - } - } - } - return newFiles +func (sp SourceSpec) NewFilesystem(base string) *Filesystem { + return &Filesystem{SourceSpec: sp, Base: base} } -func (f *Filesystem) Files() []*File { - if len(f.files) < 1 { +func (f *Filesystem) Files() []ReadableFile { + f.filesInit.Do(func() { f.captureFiles() - } + }) return f.files } // add populates a file in the Filesystem.files -func (f *Filesystem) add(name string, reader io.Reader) (err error) { - var file *File +func (f *Filesystem) add(name string, fi os.FileInfo) (err error) { + var file ReadableFile if runtime.GOOS == "darwin" { // When a file system is HFS+, its filepath is in NFD form. name = norm.NFC.String(name) } - file, err = f.SourceSpec.NewFileFromAbs(f.Base, name, reader) + file = f.SourceSpec.NewFileInfo(f.Base, name, fi) + f.files = append(f.files, file) - if err == nil { - f.files = append(f.files, file) - } return err } @@ -90,16 +69,12 @@ func (f *Filesystem) captureFiles() { return nil } - b, err := f.ShouldRead(filePath, fi) + b, err := f.shouldRead(filePath, fi) if err != nil { return err } if b { - rd, err := NewLazyFileReader(f.Fs.Source, filePath) - if err != nil { - return err - } - f.add(filePath, rd) + f.add(filePath, fi) } return err } @@ -118,11 +93,11 @@ func (f *Filesystem) captureFiles() { } -func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) { +func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(filePath) + link, err := filepath.EvalSymlinks(filename) if err != nil { - jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err) + jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } linkfi, err := f.Fs.Source.Stat(link) @@ -130,52 +105,25 @@ func (f *Filesystem) ShouldRead(filePath string, fi os.FileInfo) (bool, error) { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) return false, nil } + if !linkfi.Mode().IsRegular() { - jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath) + jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filename) } return false, nil } + ignore := f.SourceSpec.IgnoreFile(filename) + if fi.IsDir() { - if f.avoid(filePath) || f.isNonProcessablePath(filePath) { + if ignore { return false, filepath.SkipDir } return false, nil } - if f.isNonProcessablePath(filePath) { + if ignore { return false, nil } - return true, nil -} - -func (f *Filesystem) avoid(filePath string) bool { - for _, avoid := range f.AvoidPaths { - if avoid == filePath { - return true - } - } - return false -} -func (sp SourceSpec) isNonProcessablePath(filePath string) bool { - base := filepath.Base(filePath) - if strings.HasPrefix(base, ".") || - strings.HasPrefix(base, "#") || - strings.HasSuffix(base, "~") { - return true - } - ignoreFiles := cast.ToStringSlice(sp.Cfg.Get("ignoreFiles")) - if len(ignoreFiles) > 0 { - for _, ignorePattern := range ignoreFiles { - match, err := regexp.MatchString(ignorePattern, filePath) - if err != nil { - helpers.DistinctErrorLog.Printf("Invalid regexp '%s' in ignoreFiles: %s", ignorePattern, err) - return false - } else if match { - return true - } - } - } - return false + return true, nil } diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 90512ce3f..25ce0268f 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -14,11 +14,13 @@ package source import ( - "bytes" - "path/filepath" + "os" "runtime" - "strings" "testing" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/viper" ) func TestEmptySourceFilesystem(t *testing.T) { @@ -37,54 +39,6 @@ type TestPath struct { dir string } -func TestAddFile(t *testing.T) { - ss := newTestSourceSpec() - tests := platformPaths - for _, test := range tests { - base := platformBase - srcDefault := ss.NewFilesystem("") - srcWithBase := ss.NewFilesystem(base) - - for _, src := range []*Filesystem{srcDefault, srcWithBase} { - - p := test.filename - if !filepath.IsAbs(test.filename) { - p = filepath.Join(src.Base, test.filename) - } - - if err := src.add(p, bytes.NewReader([]byte(test.content))); err != nil { - if err.Error() == "source: missing base directory" { - continue - } - t.Fatalf("%s add returned an error: %s", p, err) - } - - if len(src.Files()) != 1 { - t.Fatalf("%s Files() should return 1 file", p) - } - - f := src.Files()[0] - if f.LogicalName() != test.logical { - t.Errorf("Filename (Base: %q) expected: %q, got: %q", src.Base, test.logical, f.LogicalName()) - } - - b := new(bytes.Buffer) - b.ReadFrom(f.Contents) - if b.String() != test.content { - t.Errorf("File (Base: %q) contents should be %q, got: %q", src.Base, test.content, b.String()) - } - - if f.Section() != test.section { - t.Errorf("File section (Base: %q) expected: %q, got: %q", src.Base, test.section, f.Section()) - } - - if f.Dir() != test.dir { - t.Errorf("Dir path (Base: %q) expected: %q, got: %q", src.Base, test.dir, f.Dir()) - } - } - } -} - func TestUnicodeNorm(t *testing.T) { if runtime.GOOS != "darwin" { // Normalization code is only for Mac OS, since it is not necessary for other OSes. @@ -100,10 +54,11 @@ func TestUnicodeNorm(t *testing.T) { } ss := newTestSourceSpec() + var fi os.FileInfo for _, path := range paths { - src := ss.NewFilesystem("") - _ = src.add(path.NFD, strings.NewReader("")) + src := ss.NewFilesystem("base") + _ = src.add(path.NFD, fi) f := src.Files()[0] if f.BaseFileName() != path.NFC { t.Fatalf("file name in NFD form should be normalized (%s)", path.NFC) @@ -111,3 +66,8 @@ func TestUnicodeNorm(t *testing.T) { } } + +func newTestSourceSpec() SourceSpec { + v := viper.New() + return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v} +} diff --git a/source/inmemory.go b/source/inmemory.go deleted file mode 100644 index 387bde3b8..000000000 --- a/source/inmemory.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -// ByteSource represents a source's name and content. -// It's currently only used for testing purposes. -type ByteSource struct { - Name string - Content []byte -} - -func (b *ByteSource) String() string { - return b.Name + " " + string(b.Content) -} diff --git a/source/lazy_file_reader.go b/source/lazy_file_reader.go deleted file mode 100644 index 7cc484f0b..000000000 --- a/source/lazy_file_reader.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// Portions Copyright 2009 The Go Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "bytes" - "errors" - "fmt" - "io" - - "github.com/spf13/afero" -) - -// LazyFileReader is an io.Reader implementation to postpone reading the file -// contents until it is really needed. It keeps filename and file contents once -// it is read. -type LazyFileReader struct { - fs afero.Fs - filename string - contents *bytes.Reader - pos int64 -} - -// NewLazyFileReader creates and initializes a new LazyFileReader of filename. -// It checks whether the file can be opened. If it fails, it returns nil and an -// error. -func NewLazyFileReader(fs afero.Fs, filename string) (*LazyFileReader, error) { - f, err := fs.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - return &LazyFileReader{fs: fs, filename: filename, contents: nil, pos: 0}, nil -} - -// Filename returns a file name which LazyFileReader keeps -func (l *LazyFileReader) Filename() string { - return l.filename -} - -// Read reads up to len(p) bytes from the LazyFileReader's file and copies them -// into p. It returns the number of bytes read and any error encountered. If -// the file is once read, it returns its contents from cache, doesn't re-read -// the file. -func (l *LazyFileReader) Read(p []byte) (n int, err error) { - if l.contents == nil { - b, err := afero.ReadFile(l.fs, l.filename) - if err != nil { - return 0, fmt.Errorf("failed to read content from %s: %s", l.filename, err.Error()) - } - l.contents = bytes.NewReader(b) - } - if _, err = l.contents.Seek(l.pos, 0); err != nil { - return 0, errors.New("failed to set read position: " + err.Error()) - } - n, err = l.contents.Read(p) - l.pos += int64(n) - return n, err -} - -// Seek implements the io.Seeker interface. Once reader contents is consumed by -// Read, WriteTo etc, to read it again, it must be rewinded by this function -func (l *LazyFileReader) Seek(offset int64, whence int) (pos int64, err error) { - if l.contents == nil { - switch whence { - case 0: - pos = offset - case 1: - pos = l.pos + offset - case 2: - fi, err := l.fs.Stat(l.filename) - if err != nil { - return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error()) - } - pos = fi.Size() + offset - default: - return 0, errors.New("invalid whence") - } - if pos < 0 { - return 0, errors.New("negative position") - } - } else { - pos, err = l.contents.Seek(offset, whence) - if err != nil { - return 0, err - } - } - l.pos = pos - return pos, nil -} - -// WriteTo writes data to w until all the LazyFileReader's file contents is -// drained or an error occurs. If the file is once read, it just writes its -// read cache to w, doesn't re-read the file but this method itself doesn't try -// to keep the contents in cache. -func (l *LazyFileReader) WriteTo(w io.Writer) (n int64, err error) { - if l.contents != nil { - l.contents.Seek(l.pos, 0) - if err != nil { - return 0, errors.New("failed to set read position: " + err.Error()) - } - n, err = l.contents.WriteTo(w) - l.pos += n - return n, err - } - f, err := l.fs.Open(l.filename) - if err != nil { - return 0, fmt.Errorf("failed to open %s to read content: %s", l.filename, err.Error()) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - return 0, fmt.Errorf("failed to get %q info: %s", l.filename, err.Error()) - } - - if l.pos >= fi.Size() { - return 0, nil - } - - return l.copyBuffer(w, f, nil) -} - -// copyBuffer is the actual implementation of Copy and CopyBuffer. -// If buf is nil, one is allocated. -// -// Most of this function is copied from the Go stdlib 'io/io.go'. -func (l *LazyFileReader) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) { - if buf == nil { - buf = make([]byte, 32*1024) - } - for { - nr, er := src.Read(buf) - if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - l.pos += int64(nw) - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} diff --git a/source/lazy_file_reader_test.go b/source/lazy_file_reader_test.go deleted file mode 100644 index 778a9513b..000000000 --- a/source/lazy_file_reader_test.go +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "bytes" - "io" - "os" - "testing" - - "github.com/spf13/afero" -) - -func TestNewLazyFileReader(t *testing.T) { - fs := afero.NewOsFs() - filename := "itdoesnotexistfile" - _, err := NewLazyFileReader(fs, filename) - if err == nil { - t.Errorf("NewLazyFileReader %s: error expected but no error is returned", filename) - } - - filename = "lazy_file_reader_test.go" - _, err = NewLazyFileReader(fs, filename) - if err != nil { - t.Errorf("NewLazyFileReader %s: %v", filename, err) - } -} - -func TestFilename(t *testing.T) { - fs := afero.NewOsFs() - filename := "lazy_file_reader_test.go" - rd, err := NewLazyFileReader(fs, filename) - if err != nil { - t.Fatalf("NewLazyFileReader %s: %v", filename, err) - } - if rd.Filename() != filename { - t.Errorf("Filename: expected filename %q, got %q", filename, rd.Filename()) - } -} - -func TestRead(t *testing.T) { - fs := afero.NewOsFs() - filename := "lazy_file_reader_test.go" - fi, err := fs.Stat(filename) - if err != nil { - t.Fatalf("os.Stat: %v", err) - } - - b, err := afero.ReadFile(fs, filename) - if err != nil { - t.Fatalf("afero.ReadFile: %v", err) - } - - rd, err := NewLazyFileReader(fs, filename) - if err != nil { - t.Fatalf("NewLazyFileReader %s: %v", filename, err) - } - - tst := func(testcase string) { - p := make([]byte, fi.Size()) - n, err := rd.Read(p) - if err != nil { - t.Fatalf("Read %s case: %v", testcase, err) - } - if int64(n) != fi.Size() { - t.Errorf("Read %s case: read bytes length expected %d, got %d", testcase, fi.Size(), n) - } - if !bytes.Equal(b, p) { - t.Errorf("Read %s case: read bytes are different from expected", testcase) - } - } - tst("No cache") - _, err = rd.Seek(0, 0) - if err != nil { - t.Fatalf("Seek: %v", err) - } - tst("Cache") -} - -func TestSeek(t *testing.T) { - type testcase struct { - seek int - offset int64 - length int - moveto int64 - expected []byte - } - fs := afero.NewOsFs() - filename := "lazy_file_reader_test.go" - b, err := afero.ReadFile(fs, filename) - if err != nil { - t.Fatalf("afero.ReadFile: %v", err) - } - - // no cache case - for i, this := range []testcase{ - {seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]}, - {seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, - {seek: os.SEEK_CUR, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, // current pos = 0 - {seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]}, - {seek: 3, expected: nil}, - {seek: os.SEEK_SET, offset: -1, expected: nil}, - } { - rd, err := NewLazyFileReader(fs, filename) - if err != nil { - t.Errorf("[%d] NewLazyFileReader %s: %v", i, filename, err) - continue - } - - pos, err := rd.Seek(this.offset, this.seek) - if this.expected == nil { - if err == nil { - t.Errorf("[%d] Seek didn't return an expected error", i) - } - } else { - if err != nil { - t.Errorf("[%d] Seek failed unexpectedly: %v", i, err) - continue - } - if pos != this.moveto { - t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto) - } - - buf := make([]byte, this.length) - n, err := rd.Read(buf) - if err != nil { - t.Errorf("[%d] Read failed unexpectedly: %v", i, err) - } - if !bytes.Equal(this.expected, buf[:n]) { - t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected) - } - } - } - - // cache case - rd, err := NewLazyFileReader(fs, filename) - if err != nil { - t.Fatalf("NewLazyFileReader %s: %v", filename, err) - } - dummy := make([]byte, len(b)) - _, err = rd.Read(dummy) - if err != nil { - t.Fatalf("Read failed unexpectedly: %v", err) - } - - for i, this := range []testcase{ - {seek: os.SEEK_SET, offset: 0, length: 10, moveto: 0, expected: b[:10]}, - {seek: os.SEEK_SET, offset: 5, length: 10, moveto: 5, expected: b[5:15]}, - {seek: os.SEEK_CUR, offset: 1, length: 10, moveto: 16, expected: b[16:26]}, // current pos = 15 - {seek: os.SEEK_END, offset: -1, length: 1, moveto: int64(len(b) - 1), expected: b[len(b)-1:]}, - {seek: 3, expected: nil}, - {seek: os.SEEK_SET, offset: -1, expected: nil}, - } { - pos, err := rd.Seek(this.offset, this.seek) - if this.expected == nil { - if err == nil { - t.Errorf("[%d] Seek didn't return an expected error", i) - } - } else { - if err != nil { - t.Errorf("[%d] Seek failed unexpectedly: %v", i, err) - continue - } - if pos != this.moveto { - t.Errorf("[%d] Seek failed to move the pointer: got %d, expected: %d", i, pos, this.moveto) - } - - buf := make([]byte, this.length) - n, err := rd.Read(buf) - if err != nil { - t.Errorf("[%d] Read failed unexpectedly: %v", i, err) - } - if !bytes.Equal(this.expected, buf[:n]) { - t.Errorf("[%d] Seek and Read got %q but expected %q", i, buf[:n], this.expected) - } - } - } -} - -func TestWriteTo(t *testing.T) { - fs := afero.NewOsFs() - filename := "lazy_file_reader_test.go" - fi, err := fs.Stat(filename) - if err != nil { - t.Fatalf("os.Stat: %v", err) - } - - b, err := afero.ReadFile(fs, filename) - if err != nil { - t.Fatalf("afero.ReadFile: %v", err) - } - - rd, err := NewLazyFileReader(fs, filename) - if err != nil { - t.Fatalf("NewLazyFileReader %s: %v", filename, err) - } - - tst := func(testcase string, expectedSize int64, checkEqual bool) { - buf := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - n, err := rd.WriteTo(buf) - if err != nil { - t.Fatalf("WriteTo %s case: %v", testcase, err) - } - if n != expectedSize { - t.Errorf("WriteTo %s case: written bytes length expected %d, got %d", testcase, expectedSize, n) - } - if checkEqual && !bytes.Equal(b, buf.Bytes()) { - t.Errorf("WriteTo %s case: written bytes are different from expected", testcase) - } - } - tst("No cache", fi.Size(), true) - tst("No cache 2nd", 0, false) - - p := make([]byte, fi.Size()) - _, err = rd.Read(p) - if err != nil && err != io.EOF { - t.Fatalf("Read: %v", err) - } - _, err = rd.Seek(0, 0) - if err != nil { - t.Fatalf("Seek: %v", err) - } - - tst("Cache", fi.Size(), true) -} diff --git a/source/sourceSpec.go b/source/sourceSpec.go new file mode 100644 index 000000000..74a754a26 --- /dev/null +++ b/source/sourceSpec.go @@ -0,0 +1,117 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "os" + "path/filepath" + "regexp" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/cast" +) + +// SourceSpec abstracts language-specific file creation. +// TODO(bep) rename to Spec +type SourceSpec struct { + Cfg config.Provider + Fs *hugofs.Fs + + // This is set if the ignoreFiles config is set. + ignoreFilesRe []*regexp.Regexp + + Languages map[string]interface{} + DefaultContentLanguage string +} + +// NewSourceSpec initializes SourceSpec using languages from a given configuration. +func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec { + defaultLang := cfg.GetString("defaultContentLanguage") + languages := cfg.GetStringMap("languages") + + if len(languages) == 0 { + l := helpers.NewDefaultLanguage(cfg) + languages[l.Lang] = l + defaultLang = l.Lang + } + + ignoreFiles := cast.ToStringSlice(cfg.Get("ignoreFiles")) + var regexps []*regexp.Regexp + if len(ignoreFiles) > 0 { + for _, ignorePattern := range ignoreFiles { + re, err := regexp.Compile(ignorePattern) + if err != nil { + helpers.DistinctErrorLog.Printf("Invalid regexp %q in ignoreFiles: %s", ignorePattern, err) + } else { + regexps = append(regexps, re) + } + + } + } + + return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang} +} + +func (s *SourceSpec) IgnoreFile(filename string) bool { + base := filepath.Base(filename) + + if len(base) > 0 { + first := base[0] + last := base[len(base)-1] + if first == '.' || + first == '#' || + last == '~' { + return true + } + } + + if len(s.ignoreFilesRe) == 0 { + return false + } + + for _, re := range s.ignoreFilesRe { + if re.MatchString(filename) { + return true + } + } + + return false +} + +func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { + fi, err := helpers.LstatIfOs(s.Fs.Source, filename) + if err != nil { + return false, err + } + + if fi.IsDir() { + return false, nil + } + + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := filepath.EvalSymlinks(filename) + fi, err = helpers.LstatIfOs(s.Fs.Source, link) + if err != nil { + return false, err + } + + if fi.IsDir() { + return false, nil + } + } + + return true, nil +} -- cgit v1.2.3