From 022c4795510306e08a4aba31504ca382d41c7fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 13 Jul 2021 11:41:02 +0200 Subject: hugofs: Make FileMeta a struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit started out investigating a `concurrent map read write` issue, ending by replacing the map with a struct. This is easier to reason about, and it's more effective: ``` name old time/op new time/op delta SiteNew/Regular_Deep_content_tree-16 71.5ms ± 3% 69.4ms ± 5% ~ (p=0.200 n=4+4) name old alloc/op new alloc/op delta SiteNew/Regular_Deep_content_tree-16 29.7MB ± 0% 27.9MB ± 0% -5.82% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Regular_Deep_content_tree-16 313k ± 0% 303k ± 0% -3.35% (p=0.029 n=4+4) ``` See #8749 --- hugofs/decorators.go | 12 +- hugofs/fileinfo.go | 277 +++++++++++++--------------------------- hugofs/fileinfo_test.go | 51 ++++++++ hugofs/filter_fs.go | 32 ++--- hugofs/glob_test.go | 2 +- hugofs/language_composite_fs.go | 2 +- hugofs/nosymlink_fs.go | 2 +- hugofs/rootmapping_fs.go | 34 ++--- hugofs/rootmapping_fs_test.go | 36 +++--- hugofs/slice_fs.go | 8 +- hugofs/walk.go | 28 ++-- hugofs/walk_test.go | 98 +++++++++----- 12 files changed, 287 insertions(+), 295 deletions(-) create mode 100644 hugofs/fileinfo_test.go (limited to 'hugofs') diff --git a/hugofs/decorators.go b/hugofs/decorators.go index 62d234944..364a3e23e 100644 --- a/hugofs/decorators.go +++ b/hugofs/decorators.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/afero" ) -func decorateDirs(fs afero.Fs, meta FileMeta) afero.Fs { +func decorateDirs(fs afero.Fs, meta *FileMeta) afero.Fs { ffs := &baseFileDecoratorFs{Fs: fs} decorator := func(fi os.FileInfo, name string) (os.FileInfo, error) { @@ -82,9 +82,11 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) { // Store away the original in case it's a symlink. - meta := FileMeta{metaKeyName: fi.Name()} + meta := NewFileMeta() + meta.Name = fi.Name() + if fi.IsDir() { - meta[metaKeyJoinStat] = func(name string) (FileMetaInfo, error) { + meta.JoinStatFunc = func(name string) (FileMetaInfo, error) { joinedFilename := filepath.Join(filename, name) fi, _, err := lstatIfPossible(fs, joinedFilename) if err != nil { @@ -102,7 +104,7 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero isSymlink := isSymlink(fi) if isSymlink { - meta[metaKeyOriginalFilename] = filename + meta.OriginalFilename = filename var link string var err error link, fi, err = evalSymlinks(fs, filename) @@ -110,7 +112,7 @@ func NewBaseFileDecorator(fs afero.Fs, callbacks ...func(fi FileMetaInfo)) afero return nil, err } filename = link - meta[metaKeyIsSymlink] = true + meta.IsSymlink = true } opener := func() (afero.File, error) { diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index 4f318205a..fcf35d956 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -17,6 +17,7 @@ package hugofs import ( "os" "path/filepath" + "reflect" "runtime" "sort" "strings" @@ -27,242 +28,128 @@ import ( "github.com/pkg/errors" - "github.com/spf13/cast" - "github.com/gohugoio/hugo/common/hreflect" "github.com/spf13/afero" ) -const ( - metaKeyFilename = "filename" - - metaKeySourceRoot = "sourceRoot" - metaKeyBaseDir = "baseDir" // Abs base directory of source file. - metaKeyMountRoot = "mountRoot" - metaKeyModule = "module" - metaKeyOriginalFilename = "originalFilename" - metaKeyName = "name" - metaKeyPath = "path" - metaKeyPathWalk = "pathWalk" - metaKeyLang = "lang" - metaKeyWeight = "weight" - metaKeyOrdinal = "ordinal" - metaKeyFs = "fs" - metaKeyOpener = "opener" - metaKeyIsOrdered = "isOrdered" - metaKeyIsSymlink = "isSymlink" - metaKeyJoinStat = "joinStat" - metaKeySkipDir = "skipDir" - metaKeyClassifier = "classifier" - metaKeyTranslationBaseName = "translationBaseName" - metaKeyTranslationBaseNameWithExt = "translationBaseNameWithExt" - metaKeyTranslations = "translations" - metaKeyDecoraterPath = "decoratorPath" -) - -type FileMeta map[string]interface{} - -func (f FileMeta) GetInt(key string) int { - return cast.ToInt(f[key]) -} - -func (f FileMeta) GetString(key string) string { - return cast.ToString(f[key]) -} - -func (f FileMeta) GetBool(key string) bool { - return cast.ToBool(f[key]) -} - -func (f FileMeta) Filename() string { - return f.stringV(metaKeyFilename) -} - -func (f FileMeta) OriginalFilename() string { - return f.stringV(metaKeyOriginalFilename) -} - -func (f FileMeta) SkipDir() bool { - return f.GetBool(metaKeySkipDir) -} - -func (f FileMeta) TranslationBaseName() string { - return f.stringV(metaKeyTranslationBaseName) -} - -func (f FileMeta) TranslationBaseNameWithExt() string { - return f.stringV(metaKeyTranslationBaseNameWithExt) -} - -func (f FileMeta) Translations() []string { - return cast.ToStringSlice(f[metaKeyTranslations]) -} - -func (f FileMeta) Name() string { - return f.stringV(metaKeyName) -} - -func (f FileMeta) Classifier() files.ContentClass { - c, found := f[metaKeyClassifier] - if found { - return c.(files.ContentClass) - } - - return files.ContentClassFile // For sorting -} - -func (f FileMeta) Lang() string { - return f.stringV(metaKeyLang) -} - -// Path returns the relative file path to where this file is mounted. -func (f FileMeta) Path() string { - return f.stringV(metaKeyPath) +func NewFileMeta() *FileMeta { + return &FileMeta{} } // PathFile returns the relative file path for the file source. -func (f FileMeta) PathFile() string { - base := f.stringV(metaKeyBaseDir) - if base == "" { +func (f *FileMeta) PathFile() string { + if f.BaseDir == "" { return "" } - return strings.TrimPrefix(strings.TrimPrefix(f.Filename(), base), filepathSeparator) + return strings.TrimPrefix(strings.TrimPrefix(f.Filename, f.BaseDir), filepathSeparator) } -func (f FileMeta) SourceRoot() string { - return f.stringV(metaKeySourceRoot) -} +type FileMeta struct { + Name string + Filename string + Path string + PathWalk string + OriginalFilename string + BaseDir string -func (f FileMeta) MountRoot() string { - return f.stringV(metaKeyMountRoot) -} + SourceRoot string + MountRoot string + Module string -func (f FileMeta) Module() string { - return f.stringV(metaKeyModule) -} + Weight int + Ordinal int + IsOrdered bool + IsSymlink bool + IsRootFile bool + Watch bool -func (f FileMeta) Weight() int { - return f.GetInt(metaKeyWeight) -} + Classifier files.ContentClass -func (f FileMeta) Ordinal() int { - return f.GetInt(metaKeyOrdinal) -} + SkipDir bool -func (f FileMeta) IsOrdered() bool { - return f.GetBool(metaKeyIsOrdered) -} + Lang string + TranslationBaseName string + TranslationBaseNameWithExt string + Translations []string -// IsSymlink returns whether this comes from a symlinked file or directory. -func (f FileMeta) IsSymlink() bool { - return f.GetBool(metaKeyIsSymlink) + Fs afero.Fs + OpenFunc func() (afero.File, error) + JoinStatFunc func(name string) (FileMetaInfo, error) } -func (f FileMeta) Watch() bool { - if v, found := f["watch"]; found { - return v.(bool) +func (m *FileMeta) Copy() *FileMeta { + if m == nil { + return NewFileMeta() } - return false + c := *m + return &c } -func (f FileMeta) Fs() afero.Fs { - if v, found := f[metaKeyFs]; found { - return v.(afero.Fs) +func (m *FileMeta) Merge(from *FileMeta) { + if m == nil || from == nil { + return } - return nil -} + dstv := reflect.Indirect(reflect.ValueOf(m)) + srcv := reflect.Indirect(reflect.ValueOf(from)) -func (f FileMeta) GetOpener() func() (afero.File, error) { - o, found := f[metaKeyOpener] - if !found { - return nil + for i := 0; i < dstv.NumField(); i++ { + v := dstv.Field(i) + if !hreflect.IsTruthfulValue(v) { + v.Set(srcv.Field(i)) + } } - return o.(func() (afero.File, error)) } -func (f FileMeta) Open() (afero.File, error) { - v, found := f[metaKeyOpener] - if !found { - return nil, errors.New("file opener not found") +func (f *FileMeta) Open() (afero.File, error) { + if f.OpenFunc == nil { + return nil, errors.New("OpenFunc not set") } - return v.(func() (afero.File, error))() + return f.OpenFunc() } -func (f FileMeta) JoinStat(name string) (FileMetaInfo, error) { - v, found := f[metaKeyJoinStat] - if !found { +func (f *FileMeta) JoinStat(name string) (FileMetaInfo, error) { + if f.JoinStatFunc == nil { return nil, os.ErrNotExist } - return v.(func(name string) (FileMetaInfo, error))(name) -} - -func (f FileMeta) stringV(key string) string { - if v, found := f[key]; found { - return v.(string) - } - return "" -} - -func (f FileMeta) setIfNotZero(key string, val interface{}) { - if !hreflect.IsTruthful(val) { - return - } - f[key] = val + return f.JoinStatFunc(name) } type FileMetaInfo interface { os.FileInfo - Meta() FileMeta + Meta() *FileMeta } type fileInfoMeta struct { os.FileInfo - m FileMeta + m *FileMeta } // Name returns the file's name. Note that we follow symlinks, // if supported by the file system, and the Name given here will be the // name of the symlink, which is what Hugo needs in all situations. func (fi *fileInfoMeta) Name() string { - if name := fi.m.Name(); name != "" { + if name := fi.m.Name; name != "" { return name } return fi.FileInfo.Name() } -func (fi *fileInfoMeta) Meta() FileMeta { +func (fi *fileInfoMeta) Meta() *FileMeta { return fi.m } -func NewFileMetaInfo(fi os.FileInfo, m FileMeta) FileMetaInfo { +func NewFileMetaInfo(fi os.FileInfo, m *FileMeta) FileMetaInfo { + if m == nil { + panic("FileMeta must be set") + } if fim, ok := fi.(FileMetaInfo); ok { - mergeFileMeta(fim.Meta(), m) + m.Merge(fim.Meta()) } return &fileInfoMeta{FileInfo: fi, m: m} } -func copyFileMeta(m FileMeta) FileMeta { - c := make(FileMeta) - for k, v := range m { - c[k] = v - } - return c -} - -// Merge metadata, last entry wins. -func mergeFileMeta(from, to FileMeta) { - if from == nil { - return - } - for k, v := range from { - if _, found := to[k]; !found { - to[k] = v - } - } -} - type dirNameOnlyFileInfo struct { name string modTime time.Time @@ -292,16 +179,16 @@ func (fi *dirNameOnlyFileInfo) Sys() interface{} { return nil } -func newDirNameOnlyFileInfo(name string, meta FileMeta, fileOpener func() (afero.File, error)) FileMetaInfo { +func newDirNameOnlyFileInfo(name string, meta *FileMeta, fileOpener func() (afero.File, error)) FileMetaInfo { name = normalizeFilename(name) _, base := filepath.Split(name) - m := copyFileMeta(meta) - if _, found := m[metaKeyFilename]; !found { - m.setIfNotZero(metaKeyFilename, name) + m := meta.Copy() + if m.Filename == "" { + m.Filename = name } - m[metaKeyOpener] = fileOpener - m[metaKeyIsOrdered] = false + m.OpenFunc = fileOpener + m.IsOrdered = false return NewFileMetaInfo( &dirNameOnlyFileInfo{name: base, modTime: time.Now()}, @@ -312,8 +199,8 @@ func newDirNameOnlyFileInfo(name string, meta FileMeta, fileOpener func() (afero func decorateFileInfo( fi os.FileInfo, fs afero.Fs, opener func() (afero.File, error), - filename, filepath string, inMeta FileMeta) FileMetaInfo { - var meta FileMeta + filename, filepath string, inMeta *FileMeta) FileMetaInfo { + var meta *FileMeta var fim FileMetaInfo filepath = strings.TrimPrefix(filepath, filepathSeparator) @@ -322,16 +209,26 @@ func decorateFileInfo( if fim, ok = fi.(FileMetaInfo); ok { meta = fim.Meta() } else { - meta = make(FileMeta) + meta = NewFileMeta() fim = NewFileMetaInfo(fi, meta) } - meta.setIfNotZero(metaKeyOpener, opener) - meta.setIfNotZero(metaKeyFs, fs) - meta.setIfNotZero(metaKeyPath, normalizeFilename(filepath)) - meta.setIfNotZero(metaKeyFilename, normalizeFilename(filename)) + if opener != nil { + meta.OpenFunc = opener + } + if fs != nil { + meta.Fs = fs + } + nfilepath := normalizeFilename(filepath) + nfilename := normalizeFilename(filename) + if nfilepath != "" { + meta.Path = nfilepath + } + if nfilename != "" { + meta.Filename = nfilename + } - mergeFileMeta(inMeta, meta) + meta.Merge(inMeta) return fim } @@ -377,6 +274,6 @@ func fromSlash(filenames []string) []string { func sortFileInfos(fis []os.FileInfo) { sort.Slice(fis, func(i, j int) bool { fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo) - return fimi.Meta().Filename() < fimj.Meta().Filename() + return fimi.Meta().Filename < fimj.Meta().Filename }) } diff --git a/hugofs/fileinfo_test.go b/hugofs/fileinfo_test.go new file mode 100644 index 000000000..8d6a2ff7a --- /dev/null +++ b/hugofs/fileinfo_test.go @@ -0,0 +1,51 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestFileMeta(t *testing.T) { + c := qt.New(t) + + c.Run("Merge", func(c *qt.C) { + src := &FileMeta{ + Filename: "fs1", + Path: "ps1", + } + dst := &FileMeta{ + Filename: "fd1", + } + + dst.Merge(src) + + c.Assert(dst.Path, qt.Equals, "ps1") + c.Assert(dst.Filename, qt.Equals, "fd1") + }) + + c.Run("Copy", func(c *qt.C) { + src := &FileMeta{ + Filename: "fs1", + Path: "ps1", + } + dst := src.Copy() + + c.Assert(dst, qt.Not(qt.Equals), src) + c.Assert(dst, qt.DeepEquals, src) + }) + +} diff --git a/hugofs/filter_fs.go b/hugofs/filter_fs.go index bee4d05d2..9da63bbb7 100644 --- a/hugofs/filter_fs.go +++ b/hugofs/filter_fs.go @@ -44,7 +44,7 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { } meta := fi.(FileMetaInfo).Meta() - lang := meta.Lang() + lang := meta.Lang fileLang, translationBaseName, translationBaseNameWithExt := langInfoFrom(langs, fi.Name()) weight := 0 @@ -58,14 +58,16 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { lang = fileLang } - fim := NewFileMetaInfo(fi, FileMeta{ - metaKeyLang: lang, - metaKeyWeight: weight, - metaKeyOrdinal: langs[lang], - metaKeyTranslationBaseName: translationBaseName, - metaKeyTranslationBaseNameWithExt: translationBaseNameWithExt, - metaKeyClassifier: files.ClassifyContentFile(fi.Name(), meta.GetOpener()), - }) + fim := NewFileMetaInfo( + fi, + &FileMeta{ + Lang: lang, + Weight: weight, + Ordinal: langs[lang], + TranslationBaseName: translationBaseName, + TranslationBaseNameWithExt: translationBaseNameWithExt, + Classifier: files.ClassifyContentFile(fi.Name(), meta.OpenFunc), + }) fis[i] = fim } @@ -74,9 +76,9 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { all := func(fis []os.FileInfo) { // Maps translation base name to a list of language codes. translations := make(map[string][]string) - trackTranslation := func(meta FileMeta) { - name := meta.TranslationBaseNameWithExt() - translations[name] = append(translations[name], meta.Lang()) + trackTranslation := func(meta *FileMeta) { + name := meta.TranslationBaseNameWithExt + translations[name] = append(translations[name], meta.Lang) } for _, fi := range fis { if fi.IsDir() { @@ -90,9 +92,9 @@ func NewLanguageFs(langs map[string]int, fs afero.Fs) (afero.Fs, error) { for _, fi := range fis { fim := fi.(FileMetaInfo) - langs := translations[fim.Meta().TranslationBaseNameWithExt()] + langs := translations[fim.Meta().TranslationBaseNameWithExt] if len(langs) > 0 { - fim.Meta()["translations"] = sortAndremoveStringDuplicates(langs) + fim.Meta().Translations = sortAndremoveStringDuplicates(langs) } } } @@ -108,7 +110,7 @@ func NewFilterFs(fs afero.Fs) (afero.Fs, error) { applyMeta := func(fs *FilterFs, name string, fis []os.FileInfo) { for i, fi := range fis { if fi.IsDir() { - fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + fis[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename), "", "", nil) } } } diff --git a/hugofs/glob_test.go b/hugofs/glob_test.go index b18864ef2..29cd1e0ca 100644 --- a/hugofs/glob_test.go +++ b/hugofs/glob_test.go @@ -35,7 +35,7 @@ func TestGlob(t *testing.T) { collect := func(pattern string) []string { var paths []string h := func(fi FileMetaInfo) (bool, error) { - paths = append(paths, fi.Meta().Path()) + paths = append(paths, fi.Meta().Path) return false, nil } err := Glob(fs, pattern, h) diff --git a/hugofs/language_composite_fs.go b/hugofs/language_composite_fs.go index 5dbd252c0..09c4540a9 100644 --- a/hugofs/language_composite_fs.go +++ b/hugofs/language_composite_fs.go @@ -59,7 +59,7 @@ var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { m := make(map[string]FileMetaInfo) getKey := func(fim FileMetaInfo) string { - return path.Join(fim.Meta().Lang(), fim.Name()) + return path.Join(fim.Meta().Lang, fim.Name()) } for _, fi := range lofi { diff --git a/hugofs/nosymlink_fs.go b/hugofs/nosymlink_fs.go index e55d369e9..ff9503257 100644 --- a/hugofs/nosymlink_fs.go +++ b/hugofs/nosymlink_fs.go @@ -103,7 +103,7 @@ func (fs *noSymlinkFs) checkSymlinkStatus(name string, fi os.FileInfo) (os.FileI if fim, ok := fi.(FileMetaInfo); ok { meta := fim.Meta() - metaIsSymlink = meta.IsSymlink() + metaIsSymlink = meta.IsSymlink } if metaIsSymlink { diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 4ffdb9c66..6441693ad 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -55,19 +55,19 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { // Extract "blog" from "content/blog" rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) if rm.Meta == nil { - rm.Meta = make(FileMeta) + rm.Meta = NewFileMeta() } - rm.Meta[metaKeySourceRoot] = rm.To - rm.Meta[metaKeyBaseDir] = rm.ToBasedir - rm.Meta[metaKeyMountRoot] = rm.path - rm.Meta[metaKeyModule] = rm.Module + rm.Meta.SourceRoot = rm.To + rm.Meta.BaseDir = rm.ToBasedir + rm.Meta.MountRoot = rm.path + rm.Meta.Module = rm.Module - meta := copyFileMeta(rm.Meta) + meta := rm.Meta.Copy() if !fi.IsDir() { _, name := filepath.Split(rm.From) - meta[metaKeyName] = name + meta.Name = name } rm.fi = NewFileMetaInfo(fi, meta) @@ -114,11 +114,11 @@ func newRootMappingFsFromFromTo( // RootMapping describes a virtual file or directory mount. type RootMapping struct { - From string // The virtual mount. - To string // The source directory or file. - ToBasedir string // The base of To. May be empty if an absolute path was provided. - Module string // The module path/ID. - Meta FileMeta // File metadata (lang etc.) + From string // The virtual mount. + To string // The source directory or file. + ToBasedir string // The base of To. May be empty if an absolute path was provided. + Module string // The module path/ID. + Meta *FileMeta // File metadata (lang etc.) fi FileMetaInfo path string // The virtual mount point, e.g. "blog". @@ -177,7 +177,7 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { } if !fi.IsDir() { - mergeFileMeta(r.Meta, fi.(FileMetaInfo).Meta()) + fi.(FileMetaInfo).Meta().Merge(r.Meta) } fss[i] = fi.(FileMetaInfo) @@ -304,7 +304,7 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { return f, nil } - rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} + rf := &rootMappingFile{File: f, fs: fs, name: meta.Name, meta: meta} if len(fis) == 1 { return rf, err } @@ -367,7 +367,7 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) for _, fi := range direntries { meta := fi.(FileMetaInfo).Meta() - mergeFileMeta(rm.Meta, meta) + meta.Merge(rm.Meta) if fi.IsDir() { name := fi.Name() if seen[name] { @@ -556,7 +556,7 @@ func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil } } -func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) { +func (fs *RootMappingFs) realDirOpener(name string, meta *FileMeta) func() (afero.File, error) { return func() (afero.File, error) { f, err := fs.Fs.Open(name) if err != nil { @@ -570,7 +570,7 @@ type rootMappingFile struct { afero.File fs *RootMappingFs name string - meta FileMeta + meta *FileMeta } func (f *rootMappingFile) Close() error { diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index db9ed25cd..e83a46a87 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -49,27 +49,27 @@ func TestLanguageRootMapping(t *testing.T) { RootMapping{ From: "content/blog", // Virtual path, first element is one of content, static, layouts etc. To: "themes/a/mysvblogcontent", // Real path - Meta: FileMeta{"lang": "sv"}, + Meta: &FileMeta{Lang: "sv"}, }, RootMapping{ From: "content/blog", To: "themes/a/myenblogcontent", - Meta: FileMeta{"lang": "en"}, + Meta: &FileMeta{Lang: "en"}, }, RootMapping{ From: "content/blog", To: "content/sv", - Meta: FileMeta{"lang": "sv"}, + Meta: &FileMeta{Lang: "sv"}, }, RootMapping{ From: "content/blog", To: "themes/a/myotherenblogcontent", - Meta: FileMeta{"lang": "en"}, + Meta: &FileMeta{Lang: "en"}, }, RootMapping{ From: "content/docs", To: "themes/a/mysvdocs", - Meta: FileMeta{"lang": "sv"}, + Meta: &FileMeta{Lang: "sv"}, }, ) @@ -122,13 +122,13 @@ func TestLanguageRootMapping(t *testing.T) { } rfsEn := rfs.Filter(func(rm RootMapping) bool { - return rm.Meta.Lang() == "en" + return rm.Meta.Lang == "en" }) c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"}) rfsSv := rfs.Filter(func(rm RootMapping) bool { - return rm.Meta.Lang() == "sv" + return rm.Meta.Lang == "sv" }) c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"}) @@ -157,7 +157,7 @@ func TestRootMappingFsDirnames(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(fif.Name(), qt.Equals, "myfile.txt") fifm := fif.(FileMetaInfo).Meta() - c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt")) + c.Assert(fifm.Filename, qt.Equals, filepath.FromSlash("f2t/myfile.txt")) root, err := rfs.Open("static") c.Assert(err, qt.IsNil) @@ -185,7 +185,7 @@ func TestRootMappingFsFilename(t *testing.T) { fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt")) c.Assert(err, qt.IsNil) fim := fi.(FileMetaInfo) - c.Assert(fim.Meta().Filename(), qt.Equals, testfilename) + c.Assert(fim.Meta().Filename, qt.Equals, testfilename) _, err = rfs.Stat(filepath.FromSlash("static/f1")) c.Assert(err, qt.IsNil) } @@ -209,30 +209,30 @@ func TestRootMappingFsMount(t *testing.T) { { From: "content/blog", To: "mynoblogcontent", - Meta: FileMeta{"lang": "no"}, + Meta: &FileMeta{Lang: "no"}, }, { From: "content/blog", To: "myenblogcontent", - Meta: FileMeta{"lang": "en"}, + Meta: &FileMeta{Lang: "en"}, }, { From: "content/blog", To: "mysvblogcontent", - Meta: FileMeta{"lang": "sv"}, + Meta: &FileMeta{Lang: "sv"}, }, // Files { From: "content/singles/p1.md", To: "singlefiles/no.txt", ToBasedir: "singlefiles", - Meta: FileMeta{"lang": "no"}, + Meta: &FileMeta{Lang: "no"}, }, { From: "content/singles/p1.md", To: "singlefiles/sv.txt", ToBasedir: "singlefiles", - Meta: FileMeta{"lang": "sv"}, + Meta: &FileMeta{Lang: "sv"}, }, } @@ -243,7 +243,7 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(blog.IsDir(), qt.Equals, true) blogm := blog.(FileMetaInfo).Meta() - c.Assert(blogm.Lang(), qt.Equals, "no") // First match + c.Assert(blogm.Lang, qt.Equals, "no") // First match f, err := blogm.Open() c.Assert(err, qt.IsNil) @@ -261,7 +261,7 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(testfilefi.Name(), qt.Equals, testfile) testfilem := testfilefi.(FileMetaInfo).Meta() - c.Assert(testfilem.Filename(), qt.Equals, filepath.FromSlash("themes/a/mynoblogcontent/test.txt")) + c.Assert(testfilem.Filename, qt.Equals, filepath.FromSlash("themes/a/mynoblogcontent/test.txt")) tf, err := testfilem.Open() c.Assert(err, qt.IsNil) @@ -283,7 +283,7 @@ func TestRootMappingFsMount(t *testing.T) { for i, lang := range []string{"no", "sv"} { fi := singles[i].(FileMetaInfo) c.Assert(fi.Meta().PathFile(), qt.Equals, filepath.FromSlash("themes/a/singlefiles/"+lang+".txt")) - c.Assert(fi.Meta().Lang(), qt.Equals, lang) + c.Assert(fi.Meta().Lang, qt.Equals, lang) c.Assert(fi.Name(), qt.Equals, "p1.md") } } @@ -431,7 +431,7 @@ func TestRootMappingFsOs(t *testing.T) { } i++ meta := fi.(FileMetaInfo).Meta() - c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i))) + c.Assert(meta.Filename, qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i))) c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i))) } diff --git a/hugofs/slice_fs.go b/hugofs/slice_fs.go index fd442ee8f..0f0d3850a 100644 --- a/hugofs/slice_fs.go +++ b/hugofs/slice_fs.go @@ -144,7 +144,7 @@ func (fs *SliceFs) getOpener(name string) func() (afero.File, error) { func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) { for i, mfs := range fs.dirs { meta := mfs.Meta() - fs := meta.Fs() + fs := meta.Fs fi, _, err := lstatIfPossible(fs, name) if err == nil { // Gotta match! @@ -162,8 +162,8 @@ func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) { } func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, error) { - collect := func(lfs FileMeta) ([]os.FileInfo, error) { - d, err := lfs.Fs().Open(name) + collect := func(lfs *FileMeta) ([]os.FileInfo, error) { + d, err := lfs.Fs.Open(name) if err != nil { if !os.IsNotExist(err) { return nil, err @@ -204,7 +204,7 @@ func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, er duplicates = append(duplicates, i) } else { // Make sure it's opened by this filesystem. - dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename()), "", "", nil) + dirs[i] = decorateFileInfo(fi, fs, fs.getOpener(fi.(FileMetaInfo).Meta().Filename), "", "", nil) seen[fi.Name()] = true } } diff --git a/hugofs/walk.go b/hugofs/walk.go index a46b36e50..44d58f060 100644 --- a/hugofs/walk.go +++ b/hugofs/walk.go @@ -73,7 +73,7 @@ type WalkwayConfig struct { func NewWalkway(cfg WalkwayConfig) *Walkway { var fs afero.Fs if cfg.Info != nil { - fs = cfg.Info.Meta().Fs() + fs = cfg.Info.Meta().Fs } else { fs = cfg.Fs } @@ -184,7 +184,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo } meta := info.Meta() - filename := meta.Filename() + filename := meta.Filename if dirEntries == nil { f, err := w.fs.Open(path) @@ -206,7 +206,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo dirEntries = fileInfosToFileMetaInfos(fis) - if !meta.IsOrdered() { + if !meta.IsOrdered { sort.Slice(dirEntries, func(i, j int) bool { fii := dirEntries[i] fij := dirEntries[j] @@ -214,7 +214,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo fim, fjm := fii.Meta(), fij.Meta() // Pull bundle headers to the top. - ficlass, fjclass := fim.Classifier(), fjm.Classifier() + ficlass, fjclass := fim.Classifier, fjm.Classifier if ficlass != fjclass { return ficlass < fjclass } @@ -222,20 +222,20 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo // With multiple content dirs with different languages, // there can be duplicate files, and a weight will be added // to the closest one. - fiw, fjw := fim.Weight(), fjm.Weight() + fiw, fjw := fim.Weight, fjm.Weight if fiw != fjw { return fiw > fjw } // Explicit order set. - fio, fjo := fim.Ordinal(), fjm.Ordinal() + fio, fjo := fim.Ordinal, fjm.Ordinal if fio != fjo { return fio < fjo } // When we walk into a symlink, we keep the reference to // the original name. - fin, fjn := fim.Name(), fjm.Name() + fin, fjn := fim.Name, fjm.Name if fin != "" && fjn != "" { return fin < fjn } @@ -252,7 +252,7 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo meta := fim.Meta() // Note that we use the original Name even if it's a symlink. - name := meta.Name() + name := meta.Name if name == "" { name = fim.Name() } @@ -267,13 +267,13 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo pathMeta = strings.TrimPrefix(pathn, w.basePath) } - meta[metaKeyPath] = normalizeFilename(pathMeta) - meta[metaKeyPathWalk] = pathn + meta.Path = normalizeFilename(pathMeta) + meta.PathWalk = pathn - if fim.IsDir() && w.isSeen(meta.Filename()) { + if fim.IsDir() && w.isSeen(meta.Filename) { // Prevent infinite recursion // Possible cyclic reference - meta[metaKeySkipDir] = true + meta.SkipDir = true } } @@ -291,11 +291,11 @@ func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo fim := fi.(FileMetaInfo) meta := fim.Meta() - if meta.SkipDir() { + if meta.SkipDir { continue } - err := w.walk(meta.GetString(metaKeyPathWalk), fim, nil, walkFn) + err := w.walk(meta.PathWalk, fim, nil, walkFn) if err != nil { if !fi.IsDir() || err != filepath.SkipDir { return err diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go index 49e011d74..4825ba36e 100644 --- a/hugofs/walk_test.go +++ b/hugofs/walk_test.go @@ -14,6 +14,7 @@ package hugofs import ( + "context" "fmt" "os" "path/filepath" @@ -23,6 +24,7 @@ import ( "github.com/pkg/errors" + "github.com/gohugoio/hugo/common/para" "github.com/gohugoio/hugo/htesting" "github.com/spf13/afero" @@ -47,38 +49,76 @@ func TestWalk(t *testing.T) { func TestWalkRootMappingFs(t *testing.T) { c := qt.New(t) - fs := NewBaseFileDecorator(afero.NewMemMapFs()) - testfile := "test.txt" - - c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil) - c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil) - c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil) - - rm := []RootMapping{ - { - From: "static/b", - To: "e/f", - }, - { - From: "static/a", - To: "c/d", - }, - - { - From: "static/c", - To: "a/b", - }, + prepare := func(c *qt.C) afero.Fs { + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + testfile := "test.txt" + + c.Assert(afero.WriteFile(fs, filepath.Join("a/b", testfile), []byte("some content"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("c/d", testfile), []byte("some content"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("e/f", testfile), []byte("some content"), 0755), qt.IsNil) + + rm := []RootMapping{ + { + From: "static/b", + To: "e/f", + }, + { + From: "static/a", + To: "c/d", + }, + + { + From: "static/c", + To: "a/b", + }, + } + + rfs, err := NewRootMappingFs(fs, rm...) + c.Assert(err, qt.IsNil) + return afero.NewBasePathFs(rfs, "static") } - rfs, err := NewRootMappingFs(fs, rm...) - c.Assert(err, qt.IsNil) - bfs := afero.NewBasePathFs(rfs, "static") + c.Run("Basic", func(c *qt.C) { - names, err := collectFilenames(bfs, "", "") + bfs := prepare(c) - c.Assert(err, qt.IsNil) - c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"}) + names, err := collectFilenames(bfs, "", "") + + c.Assert(err, qt.IsNil) + c.Assert(names, qt.DeepEquals, []string{"a/test.txt", "b/test.txt", "c/test.txt"}) + + }) + + c.Run("Para", func(c *qt.C) { + bfs := prepare(c) + + p := para.New(4) + r, _ := p.Start(context.Background()) + + for i := 0; i < 8; i++ { + r.Run(func() error { + _, err := collectFilenames(bfs, "", "") + if err != nil { + return err + } + fi, err := bfs.Stat("b/test.txt") + if err != nil { + return err + } + meta := fi.(FileMetaInfo).Meta() + if meta.Filename == "" { + return errors.New("fail") + } + return nil + + }) + } + + c.Assert(r.Wait(), qt.IsNil) + + }) } func skipSymlink() bool { @@ -157,7 +197,7 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) { return nil } - filename := info.Meta().Path() + filename := info.Meta().Path filename = filepath.ToSlash(filename) names = append(names, filename) @@ -221,7 +261,7 @@ func BenchmarkWalk(b *testing.B) { return nil } - filename := info.Meta().Filename() + filename := info.Meta().Filename if !strings.HasPrefix(filename, "root") { return errors.New(filename) } -- cgit v1.2.3