diff options
86 files changed, 2820 insertions, 1914 deletions
diff --git a/Gopkg.lock b/Gopkg.lock index 86c9e7c8f..51fb96c52 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -298,8 +298,8 @@ ".", "mem" ] - revision = "63644898a8da0bc22138abf860edaf5277b6102e" - version = "v1.1.0" + revision = "787d034dfe70e44075ccc060d346146ef53270ad" + version = "v1.1.1" [[projects]] name = "github.com/spf13/cast" diff --git a/commands/benchmark.go b/commands/benchmark.go index 3938acf1b..b0a12db7f 100644 --- a/commands/benchmark.go +++ b/commands/benchmark.go @@ -56,7 +56,7 @@ func (c *benchmarkCmd) benchmark(cmd *cobra.Command, args []string) error { return nil } - comm, err := initializeConfig(false, &c.hugoBuilderCommon, c, cfgInit) + comm, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, cfgInit) if err != nil { return err } diff --git a/commands/commandeer.go b/commands/commandeer.go index d43b7c9f1..d5d2740bf 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -34,7 +34,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/langs" ) type commandeer struct { @@ -45,11 +45,8 @@ type commandeer struct { h *hugoBuilderCommon ftch flagsToConfigHandler - pathSpec *helpers.PathSpec visitedURLs *types.EvictingStringQueue - staticDirsConfig []*src.Dirs - // We watch these for changes. configFiles []string @@ -63,7 +60,7 @@ type commandeer struct { serverPorts []int languagesConfigured bool - languages helpers.Languages + languages langs.Languages configured bool } @@ -75,31 +72,13 @@ func (c *commandeer) Set(key string, value interface{}) { c.Cfg.Set(key, value) } -// PathSpec lazily creates a new PathSpec, as all the paths must -// be configured before it is created. -func (c *commandeer) PathSpec() *helpers.PathSpec { - c.configured = true - return c.pathSpec -} - func (c *commandeer) initFs(fs *hugofs.Fs) error { c.DepsCfg.Fs = fs - ps, err := helpers.NewPathSpec(fs, c.Cfg) - if err != nil { - return err - } - c.pathSpec = ps - - dirsConfig, err := c.createStaticDirsConfig() - if err != nil { - return err - } - c.staticDirsConfig = dirsConfig return nil } -func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { +func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { var rebuildDebouncer func(f func()) if running { @@ -117,10 +96,10 @@ func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, d debounce: rebuildDebouncer, } - return c, c.loadConfig(running) + return c, c.loadConfig(mustHaveConfigFile, running) } -func (c *commandeer) loadConfig(running bool) error { +func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { if c.DepsCfg == nil { c.DepsCfg = &deps.DepsCfg{} @@ -168,12 +147,18 @@ func (c *commandeer) loadConfig(running bool) error { doWithConfig) if err != nil { - return err + if mustHaveConfigFile { + return err + } + if err != hugolib.ErrNoConfigFile { + return err + } + } c.configFiles = configFiles - if l, ok := c.Cfg.Get("languagesSorted").(helpers.Languages); ok { + if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok { c.languagesConfigured = true c.languages = l } @@ -209,6 +194,15 @@ func (c *commandeer) loadConfig(running bool) error { } err = c.initFs(fs) + if err != nil { + return + } + + var h *hugolib.HugoSites + + h, err = hugolib.NewHugoSites(*c.DepsCfg) + c.hugo = h + }) if err != nil { @@ -232,7 +226,7 @@ func (c *commandeer) loadConfig(running bool) error { cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed()) - themeDir := c.PathSpec().GetThemeDir() + themeDir := c.hugo.PathSpec.GetFirstThemeDir() if themeDir != "" { if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) { return newSystemError("Unable to find theme Directory:", themeDir) diff --git a/commands/commands.go b/commands/commands.go index 8ba28e10d..74bc709cc 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -148,7 +148,7 @@ Complete documentation is available at http://gohugo.io/.`, return nil } - c, err := initializeConfig(cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } diff --git a/commands/commands_test.go b/commands/commands_test.go index 907f003c0..d576b4428 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -237,6 +237,11 @@ List: {{ .Title }} `) + writeFile(t, filepath.Join(d, "static", "my.txt"), ` +MyMy + +`) + return d, nil } diff --git a/commands/config.go b/commands/config.go index 951b57540..33a61733d 100644 --- a/commands/config.go +++ b/commands/config.go @@ -44,7 +44,7 @@ func newConfigCmd() *configCmd { } func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error { - cfg, err := initializeConfig(false, &c.hugoBuilderCommon, c, nil) + cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil) if err != nil { return err diff --git a/commands/convert.go b/commands/convert.go index fb70a148d..8de155e9b 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -96,7 +96,7 @@ func (cc *convertCmd) convertContents(mark rune) error { return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, nil) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, nil) if err != nil { return err } diff --git a/commands/hugo.go b/commands/hugo.go index 8f7860f76..c4fee122d 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -23,6 +23,8 @@ import ( "sync/atomic" "syscall" + "github.com/gohugoio/hugo/hugolib/filesystems" + "golang.org/x/sync/errgroup" "log" @@ -32,8 +34,6 @@ import ( "strings" "time" - src "github.com/gohugoio/hugo/source" - "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/parser" @@ -103,12 +103,12 @@ func Execute(args []string) Response { } // InitializeConfig initializes a config file with sensible default configuration flags. -func initializeConfig(running bool, +func initializeConfig(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error) (*commandeer, error) { - c, err := newCommandeer(running, h, f, doWithCommandeer) + c, err := newCommandeer(mustHaveConfigFile, running, h, f, doWithCommandeer) if err != nil { return nil, err } @@ -280,6 +280,7 @@ func (c *commandeer) fullBuild() error { return fmt.Errorf("Error copying static files: %s", err) } langCount = cnt + langCount = cnt return nil } buildSitesFunc := func() error { @@ -344,7 +345,7 @@ func (c *commandeer) build() error { if err != nil { return err } - c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir"))) + c.Logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir"))) c.Logger.FEEDBACK.Println("Press Ctrl+C to stop") watcher, err := c.newWatcher(watchDirs...) utils.CheckErr(c.Logger, err) @@ -380,49 +381,30 @@ func (c *commandeer) copyStatic() (map[string]uint64, error) { return c.doWithPublishDirs(c.copyStaticTo) } -func (c *commandeer) createStaticDirsConfig() ([]*src.Dirs, error) { - var dirsConfig []*src.Dirs - - if !c.languages.IsMultihost() { - dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - dirsConfig = append(dirsConfig, dirs) - } else { - for _, l := range c.languages { - dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - dirsConfig = append(dirsConfig, dirs) - } - } - - return dirsConfig, nil - -} - -func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) (uint64, error)) (map[string]uint64, error) { +func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { langCount := make(map[string]uint64) - for _, dirs := range c.staticDirsConfig { + staticFilesystems := c.hugo.BaseFs.SourceFilesystems.Static - cnt, err := f(dirs, c.pathSpec.PublishDir) + if len(staticFilesystems) == 0 { + c.Logger.WARN.Println("No static directories found to sync") + return langCount, nil + } + + for lang, fs := range staticFilesystems { + cnt, err := f(fs) if err != nil { return langCount, err } - - if dirs.Language == nil { + if lang == "" { // Not multihost for _, l := range c.languages { langCount[l.Lang] = cnt } } else { - langCount[dirs.Language.Lang] = cnt + langCount[lang] = cnt } - } return langCount, nil @@ -443,29 +425,18 @@ func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { return f, err } -func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, error) { - +func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := c.hugo.PathSpec.PublishDir // If root, remove the second '/' if publishDir == "//" { publishDir = helpers.FilePathSeparator } - if dirs.Language != nil { - // Multihost setup. - publishDir = filepath.Join(publishDir, dirs.Language.Lang) + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) } - staticSourceFs, err := dirs.CreateStaticFs() - if err != nil { - return 0, err - } - - if staticSourceFs == nil { - c.Logger.WARN.Println("No static directories found to sync") - return 0, nil - } - - fs := &countingStatFs{Fs: staticSourceFs} + fs := &countingStatFs{Fs: sourceFs.Fs} syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") @@ -485,6 +456,8 @@ func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, er } c.Logger.INFO.Println("syncing static files to", publishDir) + var err error + // because we are using a baseFs (to get the union right). // set sync src to root err = syncer.Sync(publishDir, helpers.FilePathSeparator) @@ -514,41 +487,10 @@ func (c *commandeer) getDirList() ([]string, error) { var seen = make(map[string]bool) var nested []string - dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir")) - i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir")) - staticSyncer, err := newStaticSyncer(c) - if err != nil { - return nil, err - } - - layoutDir := c.PathSpec().GetLayoutDirPath() - staticDirs := staticSyncer.d.AbsStaticDirs - newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error { return func(path string, fi os.FileInfo, err error) error { if err != nil { - if path == dataDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip dataDir:", err) - return nil - } - - if path == i18nDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip i18nDir:", err) - return nil - } - - if path == layoutDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip layoutDir:", err) - return nil - } - if os.IsNotExist(err) { - for _, staticDir := range staticDirs { - if path == staticDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip staticDir:", err) - } - } - // Ignore. return nil } @@ -605,23 +547,28 @@ func (c *commandeer) getDirList() ([]string, error) { regularWalker := newWalker(false) // SymbolicWalk will log anny ERRORs - _ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker) - - for _, contentDir := range c.PathSpec().ContentDirs() { + // Also note that the Dirnames fetched below will contain any relevant theme + // directories. + for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs { _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker) } - for _, staticDir := range staticDirs { + for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames { _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) } - if c.PathSpec().ThemeSet() { - themesDir := c.PathSpec().GetThemeDir() - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "data"), regularWalker) + for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } + + for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } + + for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static { + for _, staticDir := range staticFilesystem.Dirnames { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) + } } if len(nested) > 0 { @@ -648,9 +595,6 @@ func (c *commandeer) getDirList() ([]string, error) { func (c *commandeer) recreateAndBuildSites(watching bool) (err error) { defer c.timeTrack(time.Now(), "Total") - if err := c.initSites(); err != nil { - return err - } if !c.h.quiet { c.Logger.FEEDBACK.Println("Started building sites ...") } @@ -658,56 +602,30 @@ func (c *commandeer) recreateAndBuildSites(watching bool) (err error) { } func (c *commandeer) resetAndBuildSites() (err error) { - if err = c.initSites(); err != nil { - return - } if !c.h.quiet { c.Logger.FEEDBACK.Println("Started building sites ...") } return c.hugo.Build(hugolib.BuildCfg{ResetState: true}) } -func (c *commandeer) initSites() error { - if c.hugo != nil { - c.hugo.Cfg = c.Cfg - return nil - } - - h, err := hugolib.NewHugoSites(*c.DepsCfg) - - if err != nil { - return err - } - - c.hugo = h - - return nil -} - func (c *commandeer) buildSites() (err error) { - if err := c.initSites(); err != nil { - return err - } return c.hugo.Build(hugolib.BuildCfg{}) } func (c *commandeer) rebuildSites(events []fsnotify.Event) error { defer c.timeTrack(time.Now(), "Total") - if err := c.initSites(); err != nil { - return err - } visited := c.visitedURLs.PeekAllSet() doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") if doLiveReload && !c.Cfg.GetBool("disableFastRender") { // Make sure we always render the home pages for _, l := range c.languages { - langPath := c.PathSpec().GetLangSubDir(l.Lang) + langPath := c.hugo.PathSpec.GetLangSubDir(l.Lang) if langPath != "" { langPath = langPath + "/" } - home := c.pathSpec.PrependBasePath("/" + langPath) + home := c.hugo.PathSpec.PrependBasePath("/" + langPath) visited[home] = true } @@ -716,7 +634,7 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error { } func (c *commandeer) fullRebuild() { - if err := c.loadConfig(true); err != nil { + if err := c.loadConfig(true, true); err != nil { jww.ERROR.Println("Failed to reload config:", err) } else if err := c.recreateAndBuildSites(true); err != nil { jww.ERROR.Println(err) @@ -906,7 +824,8 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { // force refresh when more than one file if len(staticEvents) > 0 { for _, ev := range staticEvents { - path := staticSyncer.d.MakeStaticPathRelative(ev.Name) + + path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) livereload.RefreshPath(path) } @@ -975,32 +894,36 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { } // isThemeVsHugoVersionMismatch returns whether the current Hugo version is -// less than the theme's min_version. +// less than any of the themes' min_version. func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (mismatch bool, requiredMinVersion string) { - if !c.PathSpec().ThemeSet() { + if !c.hugo.PathSpec.ThemeSet() { return } - themeDir := c.PathSpec().GetThemeDir() + for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs { - path := filepath.Join(themeDir, "theme.toml") + path := filepath.Join(absThemeDir, "theme.toml") - exists, err := helpers.Exists(path, fs) + exists, err := helpers.Exists(path, fs) - if err != nil || !exists { - return - } + if err != nil || !exists { + continue + } - b, err := afero.ReadFile(fs, path) + b, err := afero.ReadFile(fs, path) - tomlMeta, err := parser.HandleTOMLMetaData(b) + tomlMeta, err := parser.HandleTOMLMetaData(b) - if err != nil { - return - } + if err != nil { + continue + } + + if minVersion, ok := tomlMeta["min_version"]; ok { + if helpers.CompareVersion(minVersion) > 0 { + return true, fmt.Sprint(minVersion) + } + } - if minVersion, ok := tomlMeta["min_version"]; ok { - return helpers.CompareVersion(minVersion) > 0, fmt.Sprint(minVersion) } return diff --git a/commands/list.go b/commands/list.go index 57a92082c..9922e957d 100644 --- a/commands/list.go +++ b/commands/list.go @@ -50,7 +50,7 @@ List requires a subcommand, e.g. ` + "`hugo list drafts`.", c.Set("buildDrafts", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } @@ -86,7 +86,7 @@ posted in the future.`, c.Set("buildFuture", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } @@ -122,7 +122,7 @@ expired.`, c.Set("buildExpired", true) return nil } - c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit) + c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit) if err != nil { return err } diff --git a/commands/new.go b/commands/new.go index c088dca9b..27d079b0d 100644 --- a/commands/new.go +++ b/commands/new.go @@ -71,7 +71,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return nil } - c, err := initializeConfig(false, &n.hugoBuilderCommon, n, cfgInit) + c, err := initializeConfig(true, false, &n.hugoBuilderCommon, n, cfgInit) if err != nil { return err @@ -104,9 +104,6 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return hugolib.NewSite(*cfg) } var s *hugolib.Site - if err := c.initSites(); err != nil { - return nil, err - } if err := c.hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { return nil, err diff --git a/commands/new_theme.go b/commands/new_theme.go index 3b00cb1df..9464e1968 100644 --- a/commands/new_theme.go +++ b/commands/new_theme.go @@ -54,7 +54,7 @@ as you see fit.`, } func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error { - c, err := initializeConfig(false, &n.hugoBuilderCommon, n, nil) + c, err := initializeConfig(false, false, &n.hugoBuilderCommon, n, nil) if err != nil { return err @@ -64,7 +64,7 @@ func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error { return newUserError("theme name needs to be provided") } - createpath := c.PathSpec().AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0])) + createpath := c.hugo.PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0])) jww.INFO.Println("creating theme at", createpath) cfg := c.DepsCfg @@ -140,7 +140,7 @@ description = "" homepage = "http://example.com/" tags = [] features = [] -min_version = "0.38" +min_version = "0.41" [author] name = "" diff --git a/commands/server.go b/commands/server.go index c05180de9..8089b0ade 100644 --- a/commands/server.go +++ b/commands/server.go @@ -226,7 +226,7 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error { jww.ERROR.Println("memstats error:", err) } - c, err := initializeConfig(true, &s.hugoBuilderCommon, s, cfgInit) + c, err := initializeConfig(true, true, &s.hugoBuilderCommon, s, cfgInit) if err != nil { return err } @@ -288,7 +288,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro publishDir = filepath.Join(publishDir, root) } - absPublishDir := f.c.PathSpec().AbsPathify(publishDir) + absPublishDir := f.c.hugo.PathSpec.AbsPathify(publishDir) if i == 0 { if f.s.renderToDisk { diff --git a/commands/static_syncer.go b/commands/static_syncer.go index a04904f95..1e73e7fc2 100644 --- a/commands/static_syncer.go +++ b/commands/static_syncer.go @@ -17,53 +17,43 @@ import ( "os" "path/filepath" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" - src "github.com/gohugoio/hugo/source" "github.com/spf13/fsync" ) type staticSyncer struct { c *commandeer - d *src.Dirs } func newStaticSyncer(c *commandeer) (*staticSyncer, error) { - dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) - if err != nil { - return nil, err - } - - return &staticSyncer{c: c, d: dirs}, nil + return &staticSyncer{c: c}, nil } -func (s *staticSyncer) isStatic(path string) bool { - return s.d.IsStatic(path) +func (s *staticSyncer) isStatic(filename string) bool { + return s.c.hugo.BaseFs.SourceFilesystems.IsStatic(filename) } func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { c := s.c - syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) { - staticSourceFs, err := dirs.CreateStaticFs() - if err != nil { - return 0, err - } - - if dirs.Language != nil { - // Multihost setup - publishDir = filepath.Join(publishDir, dirs.Language.Lang) + syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := c.hugo.PathSpec.PublishDir + // If root, remove the second '/' + if publishDir == "//" { + publishDir = helpers.FilePathSeparator } - if staticSourceFs == nil { - c.Logger.WARN.Println("No static directories found to sync") - return 0, nil + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) } syncer := fsync.NewSyncer() syncer.NoTimes = c.Cfg.GetBool("noTimes") syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.SrcFs = staticSourceFs + syncer.SrcFs = sourceFs.Fs syncer.DestFs = c.Fs.Destination // prevent spamming the log on changes @@ -88,8 +78,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { fromPath := ev.Name - // If we are here we already know the event took place in a static dir - relPath := dirs.MakeStaticPathRelative(fromPath) + relPath := sourceFs.MakePathRelative(fromPath) if relPath == "" { // Not member of this virtual host. continue @@ -105,7 +94,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { // the source of that static file. In this case Hugo will incorrectly remove that file // from the published directory. if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { - if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) { + if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) { // If file doesn't exist in any static dir, remove it toRemove := filepath.Join(publishDir, relPath) diff --git a/common/loggers/loggers.go b/common/loggers/loggers.go new file mode 100644 index 000000000..2f7f36b34 --- /dev/null +++ b/common/loggers/loggers.go @@ -0,0 +1,37 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loggers + +import ( + "io/ioutil" + "log" + "os" + + jww "github.com/spf13/jwalterweatherman" +) + +// NewDebugLogger is a convenience function to create a debug logger. +func NewDebugLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} + +// NewWarningLogger is a convenience function to create a warning logger. +func NewWarningLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} + +// NewErrorLogger is a convenience function to create an error logger. +func NewErrorLogger() *jww.Notepad { + return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +} diff --git a/common/maps/maps.go b/common/maps/maps.go new file mode 100644 index 000000000..a114b557c --- /dev/null +++ b/common/maps/maps.go @@ -0,0 +1,44 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "strings" + + "github.com/spf13/cast" +) + +// ToLower makes all the keys in the given map lower cased and will do so +// recursively. +// Notes: +// * This will modify the map given. +// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}. +func ToLower(m map[string]interface{}) { + for k, v := range m { + switch v.(type) { + case map[interface{}]interface{}: + v = cast.ToStringMap(v) + ToLower(v.(map[string]interface{})) + case map[string]interface{}: + ToLower(v.(map[string]interface{})) + } + + lKey := strings.ToLower(k) + if k != lKey { + delete(m, k) + m[lKey] = v + } + + } +} diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go new file mode 100644 index 000000000..37add5dc5 --- /dev/null +++ b/common/maps/maps_test.go @@ -0,0 +1,72 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maps + +import ( + "reflect" + "testing" +) + +func TestToLower(t *testing.T) { + + tests := []struct { + input map[string]interface{} + expected map[string]interface{} + }{ + { + map[string]interface{}{ + "abC": 32, + }, + map[string]interface{}{ + "abc": 32, + }, + }, + { + map[string]interface{}{ + "abC": 32, + "deF": map[interface{}]interface{}{ + 23: "A value", + 24: map[string]interface{}{ + "AbCDe": "A value", + "eFgHi": "Another value", + }, + }, + "gHi": map[string]interface{}{ + "J": 25, + }, + }, + map[string]interface{}{ + "abc": 32, + "def": map[string]interface{}{ + "23": "A value", + "24": map[string]interface{}{ + "abcde": "A value", + "efghi": "Another value", + }, + }, + "ghi": map[string]interface{}{ + "j": 25, + }, + }, + }, + } + + for i, test := range tests { + // ToLower modifies input. + ToLower(test.input) + if !reflect.DeepEqual(test.expected, test.input) { + t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) + } + } +} diff --git a/config/configProvider.go b/config/configProvider.go index 335294d73..a7dc18960 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -16,6 +16,8 @@ package config import ( "strings" + "github.com/spf13/cast" + "github.com/spf13/viper" ) @@ -40,5 +42,16 @@ func FromConfigString(config, configType string) (Provider, error) { return nil, err } return v, nil +} +// GetStringSlicePreserveString returns a string slice from the given config and key. +// It differs from the GetStringSlice method in that if the config value is a string, +// we do not attempt to split it into fields. +func GetStringSlicePreserveString(cfg Provider, key string) []string { + sd := cfg.Get(key) + if sds, ok := sd.(string); ok { + return []string{sds} + } else { + return cast.ToStringSlice(sd) + } } diff --git a/config/configProvider_test.go b/config/configProvider_test.go new file mode 100644 index 000000000..7e9c2223b --- /dev/null +++ b/config/configProvider_test.go @@ -0,0 +1,36 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestGetStringSlicePreserveString(t *testing.T) { + assert := require.New(t) + cfg := viper.New() + + s := "This is a string" + sSlice := []string{"This", "is", "a", "slice"} + + cfg.Set("s1", s) + cfg.Set("s2", sSlice) + + assert.Equal([]string{s}, GetStringSlicePreserveString(cfg, "s1")) + assert.Equal(sSlice, GetStringSlicePreserveString(cfg, "s2")) + assert.Nil(GetStringSlicePreserveString(cfg, "s3")) +} diff --git a/create/content.go b/create/content.go index 29fe47394..6d022282e 100644 --- a/create/content.go +++ b/create/content.go @@ -16,6 +16,7 @@ package create import ( "bytes" + "fmt" "os" "os/exec" "path/filepath" @@ -31,6 +32,7 @@ func NewContent( ps *helpers.PathSpec, siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error { ext := helpers.Ext(targetPath) + fs := ps.BaseFs.SourceFilesystems.Archetypes.Fs jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext) @@ -40,9 +42,9 @@ func NewContent( siteUsed := false if archetypeFilename != "" { - f, err := ps.Fs.Source.Open(archetypeFilename) + f, err := fs.Open(archetypeFilename) if err != nil { - return err + return fmt.Errorf("failed to open archetype file: %s", err) } defer f.Close() @@ -71,7 +73,7 @@ func NewContent( targetDir := filepath.Dir(targetPath) if targetDir != "" && targetDir != "." { - exists, _ = helpers.Exists(targetDir, ps.Fs.Source) + exists, _ = helpers.Exists(targetDir, fs) } if exists { @@ -101,42 +103,27 @@ func NewContent( return nil } -// FindArchetype takes a given kind/archetype of content and returns an output -// path for that archetype. If no archetype is found, an empty string is -// returned. +// FindArchetype takes a given kind/archetype of content and returns the path +// to the archetype in the archetype filesystem, blank if none found. func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) { - search := []string{ps.AbsPathify(ps.Cfg.GetString("archetypeDir"))} + fs := ps.BaseFs.Archetypes.Fs - if ps.Cfg.GetString("theme") != "" { - themeDir := filepath.Join(ps.AbsPathify(ps.Cfg.GetString("themesDir")+"/"+ps.Cfg.GetString("theme")), "/archetypes/") - if _, err := ps.Fs.Source.Stat(themeDir); os.IsNotExist(err) { - jww.ERROR.Printf("Unable to find archetypes directory for theme %q at %q", ps.Cfg.GetString("theme"), themeDir) + // If the new content isn't in a subdirectory, kind == "". + // Therefore it should be excluded otherwise `is a directory` + // error will occur. github.com/gohugoio/hugo/issues/411 + var pathsToCheck = []string{"default"} + + if ext != "" { + if kind != "" { + pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...) } else { - search = append(search, themeDir) + pathsToCheck = append([]string{"default" + ext}, pathsToCheck...) } } - for _, x := range search { - // If the new content isn't in a subdirectory, kind == "". - // Therefore it should be excluded otherwise `is a directory` - // error will occur. github.com/gohugoio/hugo/issues/411 - var pathsToCheck = []string{"default"} - - if ext != "" { - if kind != "" { - pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...) - } else { - pathsToCheck = append([]string{"default" + ext}, pathsToCheck...) - } - } - - for _, p := range pathsToCheck { - curpath := filepath.Join(x, p) - jww.DEBUG.Println("checking", curpath, "for archetypes") - if exists, _ := helpers.Exists(curpath, ps.Fs.Source); exists { - jww.INFO.Println("curpath: " + curpath) - return curpath - } + for _, p := range pathsToCheck { + if exists, _ := helpers.Exists(p, fs); exists { + return p } } diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 17e52cae0..37eed52cf 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -89,10 +89,11 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile ) ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg) - sp := source.NewSourceSpec(ps, ps.Fs.Source) if err != nil { return nil, err } + sp := source.NewSourceSpec(ps, ps.Fs.Source) + f := sp.NewFileInfo("", targetPath, false, nil) name := f.TranslationBaseName() @@ -115,9 +116,9 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile // TODO(bep) archetype revive the issue about wrong tpl funcs arg order archetypeTemplate = []byte(ArchetypeTemplateTemplate) } else { - archetypeTemplate, err = afero.ReadFile(s.Fs.Source, archetypeFilename) + archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename) if err != nil { - return nil, fmt.Errorf("Failed to read archetype file %q: %s", archetypeFilename, err) + return nil, fmt.Errorf("failed to read archetype file %s", err) } } diff --git a/create/content_test.go b/create/content_test.go index 62d5ed1da..e9d46becf 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -58,17 +58,15 @@ func TestNewContent(t *testing.T) { for _, c := range cases { cfg, fs := newTestCfg() - ps, err := helpers.NewPathSpec(fs, cfg) - require.NoError(t, err) + require.NoError(t, initFs(fs)) h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) require.NoError(t, err) - require.NoError(t, initFs(fs)) siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) { return h.Sites[0], nil } - require.NoError(t, create.NewContent(ps, siteFactory, c.kind, c.path)) + require.NoError(t, create.NewContent(h.PathSpec, siteFactory, c.kind, c.path)) fname := filepath.Join("content", filepath.FromSlash(c.path)) content := readFileFromFs(t, fs.Source, fname) @@ -89,6 +87,7 @@ func initViper(v *viper.Viper) { v.Set("layoutDir", "layouts") v.Set("i18nDir", "i18n") v.Set("theme", "sample") + v.Set("archetypeDir", "archetypes") } func initFs(fs *hugofs.Fs) error { @@ -187,6 +186,12 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { func newTestCfg() (*viper.Viper, *hugofs.Fs) { v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + fs := hugofs.NewMem(v) v.SetFs(fs.Source) diff --git a/deps/deps.go b/deps/deps.go index 733de03b3..d233025d3 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -9,6 +9,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/metrics" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/source" @@ -47,7 +48,7 @@ type Deps struct { // The translation func to use Translate func(translationID string, args ...interface{}) string `json:"-"` - Language *helpers.Language + Language *langs.Language // All the output formats available for the current site. OutputFormatsConfig output.Formats @@ -166,10 +167,10 @@ func New(cfg DepsCfg) (*Deps, error) { // ForLanguage creates a copy of the Deps with the language dependent // parts switched out. -func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) { +func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) { var err error - d.PathSpec, err = helpers.NewPathSpec(d.Fs, l) + d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs) if err != nil { return nil, err } @@ -206,7 +207,7 @@ type DepsCfg struct { Fs *hugofs.Fs // The language to use. - Language *helpers.Language + Language *langs.Language // The configuration to use. Cfg config.Provider diff --git a/helpers/content.go b/helpers/content.go index 1c0a7b7e9..55d8ce202 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -25,6 +25,8 @@ import ( "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/common/maps" + "github.com/chaseadamsio/goorgeous" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/config" @@ -134,7 +136,7 @@ func newBlackfriday(config map[string]interface{}) *BlackFriday { "taskLists": true, } - ToLowerMap(defaultParam) + maps.ToLower(defaultParam) siteConfig := make(map[string]interface{}) diff --git a/helpers/general.go b/helpers/general.go index 5b46520e5..b442b1eb4 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -20,18 +20,20 @@ import ( "fmt" "io" "net" + "os" "path/filepath" "strings" "sync" "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" "github.com/jdkato/prose/transform" bp "github.com/gohugoio/hugo/bufferpool" - "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) @@ -129,30 +131,6 @@ func ReaderToBytes(lines io.Reader) []byte { return bc } -// ToLowerMap makes all the keys in the given map lower cased and will do so -// recursively. -// Notes: -// * This will modify the map given. -// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}. -func ToLowerMap(m map[string]interface{}) { - for k, v := range m { - switch v.(type) { - case map[interface{}]interface{}: - v = cast.ToStringMap(v) - ToLowerMap(v.(map[string]interface{})) - case map[string]interface{}: - ToLowerMap(v.(map[string]interface{})) - } - - lKey := strings.ToLower(k) - if k != lKey { - delete(m, k) - m[lKey] = v - } - - } -} - // ReaderToString is the same as ReaderToBytes, but returns a string. func ReaderToString(lines io.Reader) string { if lines == nil { @@ -255,11 +233,6 @@ func compareStringSlices(a, b []string) bool { return true } -// ThemeSet checks whether a theme is in use or not. -func (p *PathSpec) ThemeSet() bool { - return p.theme != "" -} - // LogPrinter is the common interface of the JWWs loggers. type LogPrinter interface { // Println is the only common method that works in all of JWWs loggers. @@ -477,3 +450,24 @@ func DiffStringSlices(slice1 []string, slice2 []string) []string { func DiffStrings(s1, s2 string) []string { return DiffStringSlices(strings.Fields(s1), strings.Fields(s2)) } + +// PrintFs prints the given filesystem to the given writer starting from the given path. +// This is useful for debugging. +func PrintFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\tLANG: " + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} diff --git a/helpers/general_test.go b/helpers/general_test.go index 16df69d24..08fe4890e 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -220,59 +220,6 @@ func TestFindAvailablePort(t *testing.T) { assert.True(t, addr.Port > 0) } -func TestToLowerMap(t *testing.T) { - - tests := []struct { - input map[string]interface{} - expected map[string]interface{} - }{ - { - map[string]interface{}{ - "abC": 32, - }, - map[string]interface{}{ - "abc": 32, - }, - }, - { - map[string]interface{}{ - "abC": 32, - "deF": map[interface{}]interface{}{ - 23: "A value", - 24: map[string]interface{}{ - "AbCDe": "A value", - "eFgHi": "Another value", - }, - }, - "gHi": map[string]interface{}{ - "J": 25, - }, - }, - map[string]interface{}{ - "abc": 32, - "def": map[string]interface{}{ - "23": "A value", - "24": map[string]interface{}{ - "abcde": "A value", - "efghi": "Another value", - }, - }, - "ghi": map[string]interface{}{ - "j": 25, - }, - }, - }, - } - - for i, test := range tests { - // ToLowerMap modifies input. - ToLowerMap(test.input) - if !reflect.DeepEqual(test.expected, test.input) { - t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) - } - } -} - func TestFastMD5FromFile(t *testing.T) { fs := afero.NewMemMapFs() diff --git a/helpers/path.go b/helpers/path.go index 7ac9208bf..76f13d653 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -20,6 +20,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "unicode" @@ -31,9 +32,6 @@ import ( var ( // ErrThemeUndefined is returned when a theme has not be defined by the user. ErrThemeUndefined = errors.New("no theme set") - - // ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters. - ErrPathTooShort = errors.New("file path is too short") ) // filepathPathBridge is a bridge for common functionality in filepath vs path @@ -86,7 +84,7 @@ func (p *PathSpec) MakePath(s string) string { // MakePathSanitized creates a Unicode-sanitized string, with the spaces replaced func (p *PathSpec) MakePathSanitized(s string) string { - if p.disablePathToLower { + if p.DisablePathToLower { return p.MakePath(s) } return strings.ToLower(p.MakePath(s)) @@ -129,7 +127,7 @@ func (p *PathSpec) UnicodeSanitize(s string) string { var result string - if p.removePathAccents { + if p.RemovePathAccents { // remove accents - see https://blog.golang.org/normalization t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) result, _, _ = transform.String(t, string(target)) @@ -151,32 +149,19 @@ func ReplaceExtension(path string, newExt string) string { return f + "." + newExt } -// AbsPathify creates an absolute path if given a relative path. If already -// absolute, the path is just cleaned. -func (p *PathSpec) AbsPathify(inPath string) string { - return AbsPathify(p.workingDir, inPath) -} - -// AbsPathify creates an absolute path if given a working dir and arelative path. -// If already absolute, the path is just cleaned. -func AbsPathify(workingDir, inPath string) string { - if filepath.IsAbs(inPath) { - return filepath.Clean(inPath) +// GetFirstThemeDir gets the root directory of the first theme, if there is one. +// If there is no theme, returns the empty string. +func (p *PathSpec) GetFirstThemeDir() string { + if p.ThemeSet() { + return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0])) } - return filepath.Join(workingDir, inPath) -} - -// GetLayoutDirPath returns the absolute path to the layout file dir -// for the current Hugo project. -func (p *PathSpec) GetLayoutDirPath() string { - return p.AbsPathify(p.layoutDir) + return "" } -// GetThemeDir gets the root directory of the current theme, if there is one. -// If there is no theme, returns the empty string. -func (p *PathSpec) GetThemeDir() string { +// GetThemesDir gets the absolute root theme dir path. +func (p *PathSpec) GetThemesDir() string { if p.ThemeSet() { - return p.AbsPathify(filepath.Join(p.themesDir, p.theme)) + return p.AbsPathify(p.ThemesDir) } return "" } @@ -185,50 +170,11 @@ func (p *PathSpec) GetThemeDir() string { // If there is no theme, returns the empty string. func (p *PathSpec) GetRelativeThemeDir() string { if p.ThemeSet() { - return strings.TrimPrefix(filepath.Join(p.themesDir, p.theme), FilePathSeparator) + return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator) } return "" } -// GetThemeStaticDirPath returns the theme's static dir path if theme is set. -// If theme is set and the static dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeStaticDirPath() (string, error) { - return p.getThemeDirPath("static") -} - -// GetThemeDataDirPath returns the theme's data dir path if theme is set. -// If theme is set and the data dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeDataDirPath() (string, error) { - return p.getThemeDirPath("data") -} - -// GetThemeI18nDirPath returns the theme's i18n dir path if theme is set. -// If theme is set and the i18n dir doesn't exist, an error is returned. -func (p *PathSpec) GetThemeI18nDirPath() (string, error) { - return p.getThemeDirPath("i18n") -} - -func (p *PathSpec) getThemeDirPath(path string) (string, error) { - if !p.ThemeSet() { - return "", ErrThemeUndefined - } - - themeDir := filepath.Join(p.GetThemeDir(), path) - if _, err := p.Fs.Source.Stat(themeDir); os.IsNotExist(err) { - return "", fmt.Errorf("Unable to find %s directory for theme %s in %s", path, p.theme, themeDir) - } - - return themeDir, nil -} - -// GetThemesDirPath gets the static files directory of the current theme, if there is one. -// Ignores underlying errors. -// TODO(bep) Candidate for deprecation? -func (p *PathSpec) GetThemesDirPath() string { - dir, _ := p.getThemeDirPath("static") - return dir -} - func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { for _, currentPath := range possibleDirectories { @@ -445,8 +391,8 @@ func FindCWD() (string, error) { func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { // Sanity check - if len(root) < 4 { - return ErrPathTooShort + if root != "" && len(root) < 4 { + return errors.New("Path is too short") } // Handle the root first @@ -464,7 +410,10 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { return err } - rootContent, err := afero.ReadDir(fs, root) + // Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders. + // Make sure that order is preserved. afero.Walk will sort the directories down in the file tree, + // but we don't care about that. + rootContent, err := readDir(fs, root, false) if err != nil { return walker(root, nil, err) @@ -480,6 +429,22 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { } +func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) { + f, err := fs.Open(dirname) + if err != nil { + return nil, err + } + list, err := f.Readdir(-1) + f.Close() + if err != nil { + return nil, err + } + if doSort { + sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) + } + return list, nil +} + func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { fileInfo, err := LstatIfPossible(fs, path) realPath := path diff --git a/helpers/path_test.go b/helpers/path_test.go index c2ac19675..2c6cb9f37 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -25,6 +25,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/langs" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" @@ -56,11 +58,10 @@ func TestMakePath(t *testing.T) { } for _, test := range tests { - v := viper.New() - v.Set("contentDir", "content") + v := newTestCfg() v.Set("removePathAccents", test.removeAccents) - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) @@ -74,8 +75,12 @@ func TestMakePath(t *testing.T) { func TestMakePathSanitized(t *testing.T) { v := viper.New() v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { @@ -99,12 +104,11 @@ func TestMakePathSanitized(t *testing.T) { } func TestMakePathSanitizedDisablePathToLower(t *testing.T) { - v := viper.New() + v := newTestCfg() v.Set("disablePathToLower", true) - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { diff --git a/helpers/pathspec.go b/helpers/pathspec.go index b18408590..847029f44 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -14,354 +14,71 @@ package helpers import ( - "fmt" "strings" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/cast" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/hugolib/paths" ) // PathSpec holds methods that decides how paths in URLs and files in Hugo should look like. type PathSpec struct { - BaseURL - - // If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath. - // This will not be set if canonifyURLs is enabled. - BasePath string - - disablePathToLower bool - removePathAccents bool - uglyURLs bool - canonifyURLs bool - - Language *Language - Languages Languages - - // pagination path handling - paginatePath string - - theme string - - // Directories - contentDir string - themesDir string - layoutDir string - workingDir string - staticDirs []string - absContentDirs []types.KeyValueStr - - PublishDir string - - // The PathSpec looks up its config settings in both the current language - // and then in the global Viper config. - // Some settings, the settings listed below, does not make sense to be set - // on per-language-basis. We have no good way of protecting against this - // other than a "white-list". See language.go. - defaultContentLanguageInSubdir bool - defaultContentLanguage string - multilingual bool + *paths.Paths + *filesystems.BaseFs ProcessingStats *ProcessingStats // The file systems to use Fs *hugofs.Fs - // The fine grained filesystems in play (resources, content etc.). - BaseFs *hugofs.BaseFs - // The config provider to use Cfg config.Provider } -func (p PathSpec) String() string { - return fmt.Sprintf("PathSpec, language %q, prefix %q, multilingual: %T", p.Language.Lang, p.getLanguagePrefix(), p.multilingual) -} - -// NewPathSpec creats a new PathSpec from the given filesystems and Language. +// NewPathSpec creats a new PathSpec from the given filesystems and language. func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { + return NewPathSpecWithBaseBaseFsProvided(fs, cfg, nil) +} - baseURLstr := cfg.GetString("baseURL") - baseURL, err := newBaseURLFromString(baseURLstr) - - if err != nil { - return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) - } - - var staticDirs []string - - for i := -1; i <= 10; i++ { - staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) - } - - var ( - lang string - language *Language - languages Languages - ) - - if l, ok := cfg.(*Language); ok { - language = l - lang = l.Lang - - } - - if l, ok := cfg.Get("languagesSorted").(Languages); ok { - languages = l - } - - defaultContentLanguage := cfg.GetString("defaultContentLanguage") - - // We will eventually pull out this badly placed path logic. - contentDir := cfg.GetString("contentDir") - workingDir := cfg.GetString("workingDir") - resourceDir := cfg.GetString("resourceDir") - publishDir := cfg.GetString("publishDir") - - if len(languages) == 0 { - // We have some old tests that does not test the entire chain, hence - // they have no languages. So create one so we get the proper filesystem. - languages = Languages{&Language{Lang: "en", ContentDir: contentDir}} - } +// NewPathSpecWithBaseBaseFsProvided creats a new PathSpec from the given filesystems and language. +// If an existing BaseFs is provided, parts of that is reused. +func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) { - absPuslishDir := AbsPathify(workingDir, publishDir) - if !strings.HasSuffix(absPuslishDir, FilePathSeparator) { - absPuslishDir += FilePathSeparator - } - // If root, remove the second '/' - if absPuslishDir == "//" { - absPuslishDir = FilePathSeparator - } - absResourcesDir := AbsPathify(workingDir, resourceDir) - if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { - absResourcesDir += FilePathSeparator - } - if absResourcesDir == "//" { - absResourcesDir = FilePathSeparator - } - - contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages) + p, err := paths.New(fs, cfg) if err != nil { return nil, err } - // Make sure we don't have any overlapping content dirs. That will never work. - for i, d1 := range absContentDirs { - for j, d2 := range absContentDirs { - if i == j { - continue - } - if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { - return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) - } + var options []func(*filesystems.BaseFs) error + if baseBaseFs != nil { + options = []func(*filesystems.BaseFs) error{ + filesystems.WithBaseFs(baseBaseFs), } } - - resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir) - publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir) - - baseFs := &hugofs.BaseFs{ - ContentFs: contentFs, - ResourcesFs: resourcesFs, - PublishFs: publishFs, + bfs, err := filesystems.NewBase(p, options...) + if err != nil { + return nil, err } ps := &PathSpec{ - Fs: fs, - BaseFs: baseFs, - Cfg: cfg, - disablePathToLower: cfg.GetBool("disablePathToLower"), - removePathAccents: cfg.GetBool("removePathAccents"), - uglyURLs: cfg.GetBool("uglyURLs"), - canonifyURLs: cfg.GetBool("canonifyURLs"), - multilingual: cfg.GetBool("multilingual"), - Language: language, - Languages: languages, - defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), - defaultContentLanguage: defaultContentLanguage, - paginatePath: cfg.GetString("paginatePath"), - BaseURL: baseURL, - contentDir: contentDir, - themesDir: cfg.GetString("themesDir"), - layoutDir: cfg.GetString("layoutDir"), - workingDir: workingDir, - staticDirs: staticDirs, - absContentDirs: absContentDirs, - theme: cfg.GetString("theme"), - ProcessingStats: NewProcessingStats(lang), + Paths: p, + BaseFs: bfs, + Fs: fs, + Cfg: cfg, + ProcessingStats: NewProcessingStats(p.Lang()), } - if !ps.canonifyURLs { - basePath := ps.BaseURL.url.Path + if !ps.CanonifyURLs { + basePath := ps.BaseURL.Path() if basePath != "" && basePath != "/" { ps.BasePath = basePath } } - // TODO(bep) remove this, eventually - ps.PublishDir = absPuslishDir - return ps, nil } -func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { - - if id >= 0 { - key = fmt.Sprintf("%s%d", key, id) - } - - var out []string - - sd := cfg.Get(key) - - if sds, ok := sd.(string); ok { - out = []string{sds} - } else if sd != nil { - out = cast.ToStringSlice(sd) - } - - return out -} - -func createContentFs(fs afero.Fs, - workingDir, - defaultContentLanguage string, - languages Languages) (afero.Fs, []types.KeyValueStr, error) { - - var contentLanguages Languages - var contentDirSeen = make(map[string]bool) - languageSet := make(map[string]bool) - - // The default content language needs to be first. - for _, language := range languages { - if language.Lang == defaultContentLanguage { - contentLanguages = append(contentLanguages, language) - contentDirSeen[language.ContentDir] = true - } - languageSet[language.Lang] = true - } - - for _, language := range languages { - if contentDirSeen[language.ContentDir] { - continue - } - if language.ContentDir == "" { - language.ContentDir = defaultContentLanguage - } - contentDirSeen[language.ContentDir] = true - contentLanguages = append(contentLanguages, language) - - } - - var absContentDirs []types.KeyValueStr - - fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) - return fs, absContentDirs, err - -} - -func createContentOverlayFs(source afero.Fs, - workingDir string, - languages Languages, - languageSet map[string]bool, - absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { - if len(languages) == 0 { - return source, nil - } - - language := languages[0] - - contentDir := language.ContentDir - if contentDir == "" { - panic("missing contentDir") - } - - absContentDir := AbsPathify(workingDir, language.ContentDir) - if !strings.HasSuffix(absContentDir, FilePathSeparator) { - absContentDir += FilePathSeparator - } - - // If root, remove the second '/' - if absContentDir == "//" { - absContentDir = FilePathSeparator - } - - if len(absContentDir) < 6 { - return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort) - } - - *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) - - overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) - if len(languages) == 1 { - return overlay, nil - } - - base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) - if err != nil { - return nil, err - } - - return hugofs.NewLanguageCompositeFs(base, overlay), nil - -} - -// RelContentDir tries to create a path relative to the content root from -// the given filename. The return value is the path and language code. -func (p *PathSpec) RelContentDir(filename string) (string, string) { - for _, dir := range p.absContentDirs { - if strings.HasPrefix(filename, dir.Value) { - rel := strings.TrimPrefix(filename, dir.Value) - return strings.TrimPrefix(rel, FilePathSeparator), dir.Key - } - } - // Either not a content dir or already relative. - return filename, "" -} - -// ContentDirs returns all the content dirs (absolute paths). -func (p *PathSpec) ContentDirs() []types.KeyValueStr { - return p.absContentDirs -} - -// PaginatePath returns the configured root path used for paginator pages. -func (p *PathSpec) PaginatePath() string { - return p.paginatePath -} - -// ContentDir returns the configured workingDir. -func (p *PathSpec) ContentDir() string { - return p.contentDir -} - -// WorkingDir returns the configured workingDir. -func (p *PathSpec) WorkingDir() string { - return p.workingDir -} - -// StaticDirs returns the relative static dirs for the current configuration. -func (p *PathSpec) StaticDirs() []string { - return p.staticDirs -} - -// LayoutDir returns the relative layout dir in the current configuration. -func (p *PathSpec) LayoutDir() string { - return p.layoutDir -} - -// Theme returns the theme name if set. -func (p *PathSpec) Theme() string { - return p.theme -} - -// Theme returns the theme relative theme dir. -func (p *PathSpec) ThemesDir() string { - return p.themesDir -} - // PermalinkForBaseURL creates a permalink from the given link and baseURL. func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string { link = strings.TrimPrefix(link, "/") diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index dc2079e06..00dd9cd7b 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,20 +18,16 @@ import ( "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/langs" "github.com/stretchr/testify/require" ) func TestNewPathSpecFromConfig(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - l := NewLanguage("no", v) + v := newTestCfg() + l := langs.NewLanguage("no", v) v.Set("disablePathToLower", true) v.Set("removePathAccents", true) v.Set("uglyURLs", true) - v.Set("multilingual", true) - v.Set("defaultContentLanguageInSubdir", true) - v.Set("defaultContentLanguage", "no") v.Set("canonifyURLs", true) v.Set("paginatePath", "side") v.Set("baseURL", "http://base.com") @@ -44,19 +40,15 @@ func TestNewPathSpecFromConfig(t *testing.T) { p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) - require.True(t, p.canonifyURLs) - require.True(t, p.defaultContentLanguageInSubdir) - require.True(t, p.disablePathToLower) - require.True(t, p.multilingual) - require.True(t, p.removePathAccents) - require.True(t, p.uglyURLs) - require.Equal(t, "no", p.defaultContentLanguage) + require.True(t, p.CanonifyURLs) + require.True(t, p.DisablePathToLower) + require.True(t, p.RemovePathAccents) + require.True(t, p.UglyURLs) require.Equal(t, "no", p.Language.Lang) - require.Equal(t, "side", p.paginatePath) + require.Equal(t, "side", p.PaginatePath) require.Equal(t, "http://base.com", p.BaseURL.String()) - require.Equal(t, "thethemes", p.themesDir) - require.Equal(t, "thelayouts", p.layoutDir) - require.Equal(t, "thework", p.workingDir) - require.Equal(t, "thetheme", p.theme) + require.Equal(t, "thethemes", p.ThemesDir) + require.Equal(t, "thework", p.WorkingDir) + require.Equal(t, []string{"thetheme"}, p.Themes()) } diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 215ae9188..fda1c9ea2 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -4,10 +4,11 @@ import ( "github.com/spf13/viper" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" ) func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) ps, _ := NewPathSpec(fs, l) return ps } @@ -15,7 +16,7 @@ func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec { func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { v := viper.New() fs := hugofs.NewMem(v) - cfg := newTestCfg(fs) + cfg := newTestCfgFor(fs) for i := 0; i < len(configKeyValues); i += 2 { cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) @@ -23,16 +24,24 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { return newTestPathSpec(fs, cfg) } -func newTestCfg(fs *hugofs.Fs) *viper.Viper { - v := viper.New() - v.Set("contentDir", "content") - +func newTestCfgFor(fs *hugofs.Fs) *viper.Viper { + v := newTestCfg() v.SetFs(fs.Source) return v } +func newTestCfg() *viper.Viper { + v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + func newTestContentSpec() *ContentSpec { v := viper.New() spec, err := NewContentSpec(v) diff --git a/helpers/url.go b/helpers/url.go index ef08a7530..f167fd3d2 100644 --- a/helpers/url.go +++ b/helpers/url.go @@ -177,7 +177,7 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string { } if addLanguage { - prefix := p.getLanguagePrefix() + prefix := p.GetLanguagePrefix() if prefix != "" { hasPrefix := false // avoid adding language prefix if already present @@ -200,38 +200,6 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string { return MakePermalink(baseURL, in).String() } -func (p *PathSpec) getLanguagePrefix() string { - if !p.multilingual { - return "" - } - - defaultLang := p.defaultContentLanguage - defaultInSubDir := p.defaultContentLanguageInSubdir - - currentLang := p.Language.Lang - if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) { - return "" - } - return currentLang -} - -// GetLangSubDir returns the given language's subdir if needed. -func (p *PathSpec) GetLangSubDir(lang string) string { - if !p.multilingual { - return "" - } - - if p.Languages.IsMultihost() { - return "" - } - - if lang == "" || (lang == p.defaultContentLanguage && !p.defaultContentLanguageInSubdir) { - return "" - } - - return lang -} - // IsAbsURL determines whether the given path points to an absolute URL. func IsAbsURL(path string) bool { url, err := url.Parse(path) @@ -246,7 +214,7 @@ func IsAbsURL(path string) bool { // Note: The result URL will not include the context root if canonifyURLs is enabled. func (p *PathSpec) RelURL(in string, addLanguage bool) string { baseURL := p.BaseURL.String() - canonifyURLs := p.canonifyURLs + canonifyURLs := p.CanonifyURLs if (!strings.HasPrefix(in, baseURL) && strings.HasPrefix(in, "http")) || strings.HasPrefix(in, "//") { return in } @@ -258,7 +226,7 @@ func (p *PathSpec) RelURL(in string, addLanguage bool) string { } if addLanguage { - prefix := p.getLanguagePrefix() + prefix := p.GetLanguagePrefix() if prefix != "" { hasPrefix := false // avoid adding language prefix if already present @@ -339,7 +307,7 @@ func (p *PathSpec) URLizeAndPrep(in string) string { // URLPrep applies misc sanitation to the given URL. func (p *PathSpec) URLPrep(in string) string { - if p.uglyURLs { + if p.UglyURLs { return Uglify(SanitizeURL(in)) } pretty := PrettifyURL(SanitizeURL(in)) diff --git a/helpers/url_test.go b/helpers/url_test.go index 0ca3c8df2..a2c945dfe 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -19,16 +19,15 @@ import ( "testing" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" + "github.com/gohugoio/hugo/langs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestURLize(t *testing.T) { - v := viper.New() - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + v := newTestCfg() + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) tests := []struct { @@ -64,7 +63,7 @@ func TestAbsURL(t *testing.T) { } func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) { - v := viper.New() + v := newTestCfg() v.Set("multilingual", multilingual) v.Set("defaultContentLanguage", "en") v.Set("defaultContentLanguageInSubdir", defaultInSubDir) @@ -90,7 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for _, test := range tests { v.Set("baseURL", test.baseURL) v.Set("contentDir", "content") - l := NewLanguage(lang, v) + l := langs.NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.AbsURL(test.input, addLanguage) @@ -140,7 +139,7 @@ func TestRelURL(t *testing.T) { } func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) { - v := viper.New() + v := newTestCfg() v.Set("multilingual", multilingual) v.Set("defaultContentLanguage", "en") v.Set("defaultContentLanguageInSubdir", defaultInSubDir) @@ -168,8 +167,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for i, test := range tests { v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) - v.Set("contentDir", "content") - l := NewLanguage(lang, v) + l := langs.NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.RelURL(test.input, addLanguage) @@ -255,10 +253,9 @@ func TestURLPrep(t *testing.T) { } for i, d := range data { - v := viper.New() + v := newTestCfg() v.Set("uglyURLs", d.ugly) - v.Set("contentDir", "content") - l := NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) output := p.URLPrep(d.input) diff --git a/hugofs/base_fs.go b/hugofs/base_fs.go deleted file mode 100644 index 77af66dfe..000000000 --- a/hugofs/base_fs.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugofs - -import ( - "github.com/spf13/afero" -) - -// BaseFs contains the core base filesystems used by Hugo. The name "base" is used -// to underline that even if they can be composites, they all have a base path set to a specific -// resource folder, e.g "/my-project/content". So, no absolute filenames needed. -type BaseFs struct { - // The filesystem used to capture content. This can be a composite and - // language aware file system. - ContentFs afero.Fs - - // The filesystem used to store resources (processed images etc.). - // This usually maps to /my-project/resources. - ResourcesFs afero.Fs - - // The filesystem used to publish the rendered site. - // This usually maps to /my-project/public. - PublishFs afero.Fs -} diff --git a/hugofs/noop_fs.go b/hugofs/noop_fs.go new file mode 100644 index 000000000..2d06622e4 --- /dev/null +++ b/hugofs/noop_fs.go @@ -0,0 +1,79 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "os" + "time" + + "github.com/spf13/afero" +) + +var ( + noOpErr = errors.New("this is a filesystem that does nothing and this operation is not supported") + _ afero.Fs = (*noOpFs)(nil) + NoOpFs = &noOpFs{} +) + +type noOpFs struct { +} + +func (fs noOpFs) Create(name string) (afero.File, error) { + return nil, noOpErr +} + +func (fs noOpFs) Mkdir(name string, perm os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) MkdirAll(path string, perm os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) Open(name string) (afero.File, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) Remove(name string) error { + return noOpErr +} + +func (fs noOpFs) RemoveAll(path string) error { + return noOpErr +} + +func (fs noOpFs) Rename(oldname string, newname string) error { + return noOpErr +} + +func (fs noOpFs) Stat(name string) (os.FileInfo, error) { + return nil, os.ErrNotExist +} + +func (fs noOpFs) Name() string { + return "noOpFs" +} + +func (fs noOpFs) Chmod(name string, mode os.FileMode) error { + return noOpErr +} + +func (fs noOpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return noOpErr +} diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go new file mode 100644 index 000000000..59f49f3a9 --- /dev/null +++ b/hugofs/rootmapping_fs.go @@ -0,0 +1,180 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "path/filepath" + "strings" + "time" + + radix "github.com/hashicorp/go-immutable-radix" + "github.com/spf13/afero" +) + +var filepathSeparator = string(filepath.Separator) + +// A RootMappingFs maps several roots into one. Note that the root of this filesystem +// is directories only, and they will be returned in Readdir and Readdirnames +// in the order given. +type RootMappingFs struct { + afero.Fs + rootMapToReal *radix.Node + virtualRoots []string +} + +type rootMappingFile struct { + afero.File + fs *RootMappingFs + name string +} + +type rootMappingFileInfo struct { + name string +} + +func (fi *rootMappingFileInfo) Name() string { + return fi.name +} + +func (fi *rootMappingFileInfo) Size() int64 { + panic("not implemented") +} + +func (fi *rootMappingFileInfo) Mode() os.FileMode { + return os.ModeDir +} + +func (fi *rootMappingFileInfo) ModTime() time.Time { + panic("not implemented") +} + +func (fi *rootMappingFileInfo) IsDir() bool { + return true +} + +func (fi *rootMappingFileInfo) Sys() interface{} { + return nil +} + +func newRootMappingDirFileInfo(name string) *rootMappingFileInfo { + return &rootMappingFileInfo{name: name} +} + +// NewRootMappingFs creates a new RootMappingFs on top of the provided with +// a list of from, to string pairs of root mappings. +// Note that 'from' represents a virtual root that maps to the actual filename in 'to'. +func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { + rootMapToReal := radix.New().Txn() + var virtualRoots []string + + for i := 0; i < len(fromTo); i += 2 { + vr := filepath.Clean(fromTo[i]) + rr := filepath.Clean(fromTo[i+1]) + + // We need to preserve the original order for Readdir + virtualRoots = append(virtualRoots, vr) + + rootMapToReal.Insert([]byte(vr), rr) + } + + return &RootMappingFs{Fs: fs, + virtualRoots: virtualRoots, + rootMapToReal: rootMapToReal.Commit().Root()}, nil +} + +func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { + if fs.isRoot(name) { + return newRootMappingDirFileInfo(name), nil + } + realName := fs.realName(name) + return fs.Fs.Stat(realName) +} + +func (fs *RootMappingFs) isRoot(name string) bool { + return name == "" || name == filepathSeparator + +} + +func (fs *RootMappingFs) Open(name string) (afero.File, error) { + if fs.isRoot(name) { + return &rootMappingFile{name: name, fs: fs}, nil + } + realName := fs.realName(name) + f, err := fs.Fs.Open(realName) + if err != nil { + return nil, err + } + return &rootMappingFile{File: f, name: name, fs: fs}, nil +} + +func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + if fs.isRoot(name) { + return newRootMappingDirFileInfo(name), false, nil + } + name = fs.realName(name) + if ls, ok := fs.Fs.(afero.Lstater); ok { + return ls.LstatIfPossible(name) + } + fi, err := fs.Stat(name) + return fi, false, err +} + +func (fs *RootMappingFs) realName(name string) string { + key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name))) + if !found { + return name + } + keystr := string(key) + + return filepath.Join(val.(string), strings.TrimPrefix(name, keystr)) +} + +func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { + if f.File == nil { + dirsn := make([]os.FileInfo, 0) + for i := 0; i < len(f.fs.virtualRoots); i++ { + if count != -1 && i >= count { + break + } + dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i])) + } + return dirsn, nil + } + return f.File.Readdir(count) + +} + +func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { + dirs, err := f.Readdir(count) + if err != nil { + return nil, err + } + dirss := make([]string, len(dirs)) + for i, d := range dirs { + dirss[i] = d.Name() + } + return dirss, nil +} + +func (f *rootMappingFile) Name() string { + return f.name +} + +func (f *rootMappingFile) Close() error { + if f.File == nil { + return nil + } + return f.File.Close() +} diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go new file mode 100644 index 000000000..a84f41151 --- /dev/null +++ b/hugofs/rootmapping_fs_test.go @@ -0,0 +1,93 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestRootMappingFsRealName(t *testing.T) { + assert := require.New(t) + fs := afero.NewMemMapFs() + + rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t") + assert.NoError(err) + + assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt"))) + +} + +func TestRootMappingFsDirnames(t *testing.T) { + assert := require.New(t) + fs := afero.NewMemMapFs() + + testfile := "myfile.txt" + assert.NoError(fs.Mkdir("f1t", 0755)) + assert.NoError(fs.Mkdir("f2t", 0755)) + assert.NoError(fs.Mkdir("f3t", 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755)) + + rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t") + assert.NoError(err) + + fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + assert.NoError(err) + assert.Equal("myfile.txt", fif.Name()) + + root, err := rfs.Open(filepathSeparator) + assert.NoError(err) + + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames) + +} + +func TestRootMappingFsOs(t *testing.T) { + assert := require.New(t) + fs := afero.NewOsFs() + + d, err := ioutil.TempDir("", "hugo-root-mapping") + assert.NoError(err) + defer func() { + os.RemoveAll(d) + }() + + testfile := "myfile.txt" + assert.NoError(fs.Mkdir(filepath.Join(d, "f1t"), 0755)) + assert.NoError(fs.Mkdir(filepath.Join(d, "f2t"), 0755)) + assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755)) + assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755)) + + rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t")) + assert.NoError(err) + + fif, err := rfs.Stat(filepath.Join("cf2", testfile)) + assert.NoError(err) + assert.Equal("myfile.txt", fif.Name()) + + root, err := rfs.Open(filepathSeparator) + assert.NoError(err) + + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames) + +} diff --git a/hugolib/alias_test.go b/hugolib/alias_test.go index d20409512..04c5b4358 100644 --- a/hugolib/alias_test.go +++ b/hugolib/alias_test.go @@ -18,6 +18,8 @@ import ( "runtime" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/stretchr/testify/require" ) @@ -97,7 +99,7 @@ func TestAliasTemplate(t *testing.T) { } func TestTargetPathHTMLRedirectAlias(t *testing.T) { - h := newAliasHandler(nil, newErrorLogger(), false) + h := newAliasHandler(nil, loggers.NewErrorLogger(), false) errIsNilForThisOS := runtime.GOOS != "windows" diff --git a/hugolib/case_insensitive_test.go b/hugolib/case_insensitive_test.go index 52ef198a5..f3ba5f933 100644 --- a/hugolib/case_insensitive_test.go +++ b/hugolib/case_insensitive_test.go @@ -19,8 +19,9 @@ import ( "strings" "testing" - "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" + + "github.com/gohugoio/hugo/deps" "github.com/spf13/afero" "github.com/stretchr/testify/require" ) diff --git a/hugolib/config.go b/hugolib/config.go index 73ba84686..dec5b870d 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -16,11 +16,14 @@ package hugolib import ( "errors" "fmt" - "path/filepath" + + "github.com/gohugoio/hugo/hugolib/paths" "io" "strings" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" "github.com/gohugoio/hugo/config/services" @@ -81,6 +84,8 @@ func LoadConfigDefault(fs afero.Fs) (*viper.Viper, error) { return v, err } +var ErrNoConfigFile = errors.New("Unable to locate Config file. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") + // LoadConfig loads Hugo configuration into a new Viper and then adds // a set of defaults. func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) { @@ -100,41 +105,50 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid v.SetConfigFile(configFilenames[0]) v.AddConfigPath(d.Path) + var configFileErr error + err := v.ReadInConfig() if err != nil { if _, ok := err.(viper.ConfigParseError); ok { return nil, configFiles, err } - return nil, configFiles, fmt.Errorf("Unable to locate Config file. Perhaps you need to create a new site.\n Run `hugo help new` for details. (%s)\n", err) + configFileErr = ErrNoConfigFile } - if cf := v.ConfigFileUsed(); cf != "" { - configFiles = append(configFiles, cf) - } + if configFileErr == nil { - for _, configFile := range configFilenames[1:] { - var r io.Reader - var err error - if r, err = fs.Open(configFile); err != nil { - return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err) + if cf := v.ConfigFileUsed(); cf != "" { + configFiles = append(configFiles, cf) } - if err = v.MergeConfig(r); err != nil { - return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err) + + for _, configFile := range configFilenames[1:] { + var r io.Reader + var err error + if r, err = fs.Open(configFile); err != nil { + return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err) + } + if err = v.MergeConfig(r); err != nil { + return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err) + } + configFiles = append(configFiles, configFile) } - configFiles = append(configFiles, configFile) + } if err := loadDefaultSettingsFor(v); err != nil { return v, configFiles, err } - themeConfigFile, err := loadThemeConfig(d, v) - if err != nil { - return v, configFiles, err - } + if configFileErr == nil { - if themeConfigFile != "" { - configFiles = append(configFiles, themeConfigFile) + themeConfigFiles, err := loadThemeConfig(d, v) + if err != nil { + return v, configFiles, err + } + + if len(themeConfigFiles) > 0 { + configFiles = append(configFiles, themeConfigFiles...) + } } // We create languages based on the settings, so we need to make sure that @@ -149,11 +163,11 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid return v, configFiles, err } - return v, configFiles, nil + return v, configFiles, configFileErr } -func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error { +func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { defaultLang := cfg.GetString("defaultContentLanguage") @@ -182,14 +196,14 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error } var ( - langs helpers.Languages - err error + languages2 langs.Languages + err error ) if len(languages) == 0 { - langs = append(langs, helpers.NewDefaultLanguage(cfg)) + languages2 = append(languages2, langs.NewDefaultLanguage(cfg)) } else { - langs, err = toSortedLanguages(cfg, languages) + languages2, err = toSortedLanguages(cfg, languages) if err != nil { return fmt.Errorf("Failed to parse multilingual config: %s", err) } @@ -201,10 +215,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The validation below isn't complete, but should cover the most // important cases. var invalid bool - if langs.IsMultihost() != oldLangs.IsMultihost() { + if languages2.IsMultihost() != oldLangs.IsMultihost() { invalid = true } else { - if langs.IsMultihost() && len(langs) != len(oldLangs) { + if languages2.IsMultihost() && len(languages2) != len(oldLangs) { invalid = true } } @@ -213,10 +227,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return errors.New("language change needing a server restart detected") } - if langs.IsMultihost() { + if languages2.IsMultihost() { // We need to transfer any server baseURL to the new language for i, ol := range oldLangs { - nl := langs[i] + nl := languages2[i] nl.Set("baseURL", ol.GetString("baseURL")) } } @@ -225,7 +239,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The defaultContentLanguage is something the user has to decide, but it needs // to match a language in the language definition list. langExists := false - for _, lang := range langs { + for _, lang := range languages2 { if lang.Lang == defaultLang { langExists = true break @@ -236,10 +250,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang) } - cfg.Set("languagesSorted", langs) - cfg.Set("multilingual", len(langs) > 1) + cfg.Set("languagesSorted", languages2) + cfg.Set("multilingual", len(languages2) > 1) - multihost := langs.IsMultihost() + multihost := languages2.IsMultihost() if multihost { cfg.Set("defaultContentLanguageInSubdir", true) @@ -250,7 +264,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error // The baseURL may be provided at the language level. If that is true, // then every language must have a baseURL. In this case we always render // to a language sub folder, which is then stripped from all the Permalink URLs etc. - for _, l := range langs { + for _, l := range languages2 { burl := l.GetLocal("baseURL") if burl == nil { return errors.New("baseURL must be set on all or none of the languages") @@ -262,49 +276,32 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error return nil } -func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) { +func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) ([]string, error) { + themesDir := paths.AbsPathify(d.WorkingDir, v1.GetString("themesDir")) + themes := config.GetStringSlicePreserveString(v1, "theme") - theme := v1.GetString("theme") - if theme == "" { - return "", nil + // CollectThemes(fs afero.Fs, themesDir string, themes []strin + themeConfigs, err := paths.CollectThemes(d.Fs, themesDir, themes) + if err != nil { + return nil, err } - - themesDir := helpers.AbsPathify(d.WorkingDir, v1.GetString("themesDir")) - configDir := filepath.Join(themesDir, theme) - - var ( - configPath string - exists bool - err error - ) - - // Viper supports more, but this is the sub-set supported by Hugo. - for _, configFormats := range []string{"toml", "yaml", "yml", "json"} { - configPath = filepath.Join(configDir, "config."+configFormats) - exists, err = helpers.Exists(configPath, d.Fs) - if err != nil { - return "", err - } - if exists { - break + v1.Set("allThemes", themeConfigs) + + var configFilenames []string + for _, tc := range themeConfigs { + if tc.ConfigFilename != "" { + configFilenames = append(configFilenames, tc.ConfigFilename) + if err := applyThemeConfig(v1, tc); err != nil { + return nil, err + } } } - if !exists { - // No theme config set. - return "", nil - } + return configFilenames, nil - v2 := viper.New() - v2.SetFs(d.Fs) - v2.AutomaticEnv() - v2.SetEnvPrefix("hugo") - v2.SetConfigFile(configPath) +} - err = v2.ReadInConfig() - if err != nil { - return "", err - } +func applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error { const ( paramsKey = "params" @@ -312,11 +309,13 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) menuKey = "menu" ) + v2 := theme.Cfg + for _, key := range []string{paramsKey, "outputformats", "mediatypes"} { mergeStringMapKeepLeft("", key, v1, v2) } - themeLower := strings.ToLower(theme) + themeLower := strings.ToLower(theme.Name) themeParamsNamespace := paramsKey + "." + themeLower // Set namespaced params @@ -371,11 +370,11 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) } } - return v2.ConfigFileUsed(), nil + return nil } -func mergeStringMapKeepLeft(rootKey, key string, v1, v2 *viper.Viper) { +func mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Provider) { if !v2.IsSet(key) { return } diff --git a/hugolib/datafiles_test.go b/hugolib/datafiles_test.go index cd1ad8411..8b2dc8c0f 100644 --- a/hugolib/datafiles_test.go +++ b/hugolib/datafiles_test.go @@ -19,6 +19,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/deps" "fmt" @@ -322,7 +324,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey } var ( - logger = newErrorLogger() + logger = loggers.NewErrorLogger() depsCfg = deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: logger} ) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go new file mode 100644 index 000000000..deecd69a5 --- /dev/null +++ b/hugolib/filesystems/basefs.go @@ -0,0 +1,644 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package filesystems provides the fine grained file systems used by Hugo. These +// are typically virtual filesystems that are composites of project and theme content. +package filesystems + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + + "github.com/gohugoio/hugo/hugofs" + + "fmt" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/gohugoio/hugo/langs" + "github.com/spf13/afero" +) + +// When we create a virtual filesystem with data and i18n bundles for the project and the themes, +// this is the name of the project's virtual root. It got it's funky name to make sure +// (or very unlikely) that it collides with a theme name. +const projectVirtualFolder = "__h__project" + +var filePathSeparator = string(filepath.Separator) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + // TODO(bep) make this go away + AbsContentDirs []types.KeyValueStr + + // The filesystem used to capture content. This can be a composite and + // language aware file system. + ContentFs afero.Fs + + // SourceFilesystems contains the different source file systems. + *SourceFilesystems + + // The filesystem used to store resources (processed images etc.). + // This usually maps to /my-project/resources. + ResourcesFs afero.Fs + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs + + themeFs afero.Fs + + // TODO(bep) improve the "theme interaction" + AbsThemeDirs []string +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (b *BaseFs) RelContentDir(filename string) (string, string) { + for _, dir := range b.AbsContentDirs { + if strings.HasPrefix(filename, dir.Value) { + rel := strings.TrimPrefix(filename, dir.Value) + return strings.TrimPrefix(rel, filePathSeparator), dir.Key + } + } + // Either not a content dir or already relative. + return filename, "" +} + +// IsContent returns whether the given filename is in the content filesystem. +func (b *BaseFs) IsContent(filename string) bool { + for _, dir := range b.AbsContentDirs { + if strings.HasPrefix(filename, dir.Value) { + return true + } + } + return false +} + +// SourceFilesystems contains the different source file systems. These can be +// composite file systems (theme and project etc.), and they have all root +// set to the source type the provides: data, i18n, static, layouts. +type SourceFilesystems struct { + Data *SourceFilesystem + I18n *SourceFilesystem + Layouts *SourceFilesystem + Archetypes *SourceFilesystem + + // When in multihost we have one static filesystem per language. The sync + // static files is currently done outside of the Hugo build (where there is + // a concept of a site per language). + // When in non-multihost mode there will be one entry in this map with a blank key. + Static map[string]*SourceFilesystem +} + +// A SourceFilesystem holds the filesystem for a given source type in Hugo (data, +// i18n, layouts, static) and additional metadata to be able to use that filesystem +// in server mode. +type SourceFilesystem struct { + Fs afero.Fs + + Dirnames []string + + // When syncing a source folder to the target (e.g. /public), this may + // be set to publish into a subfolder. This is used for static syncing + // in multihost mode. + PublishFolder string +} + +// IsStatic returns true if the given filename is a member of one of the static +// filesystems. +func (s SourceFilesystems) IsStatic(filename string) bool { + for _, staticFs := range s.Static { + if staticFs.Contains(filename) { + return true + } + } + return false +} + +// IsLayout returns true if the given filename is a member of the layouts filesystem. +func (s SourceFilesystems) IsLayout(filename string) bool { + return s.Layouts.Contains(filename) +} + +// IsData returns true if the given filename is a member of the data filesystem. +func (s SourceFilesystems) IsData(filename string) bool { + return s.Data.Contains(filename) +} + +// IsI18n returns true if the given filename is a member of the i18n filesystem. +func (s SourceFilesystems) IsI18n(filename string) bool { + return s.I18n.Contains(filename) +} + +// MakeStaticPathRelative makes an absolute static filename into a relative one. +// It will return an empty string if the filename is not a member of a static filesystem. +func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { + for _, staticFs := range s.Static { + rel := staticFs.MakePathRelative(filename) + if rel != "" { + return rel + } + } + return "" +} + +// MakePathRelative creates a relative path from the given filename. +// It will return an empty string if the filename is not a member of this filesystem. +func (d *SourceFilesystem) MakePathRelative(filename string) string { + for _, currentPath := range d.Dirnames { + if strings.HasPrefix(filename, currentPath) { + return strings.TrimPrefix(filename, currentPath) + } + } + return "" +} + +// Contains returns whether the given filename is a member of the current filesystem. +func (d *SourceFilesystem) Contains(filename string) bool { + for _, dir := range d.Dirnames { + if strings.HasPrefix(filename, dir) { + return true + } + } + return false +} + +// WithBaseFs allows reuse of some potentially expensive to create parts that remain +// the same across sites/languages. +func WithBaseFs(b *BaseFs) func(*BaseFs) error { + return func(bb *BaseFs) error { + bb.themeFs = b.themeFs + bb.AbsThemeDirs = b.AbsThemeDirs + return nil + } +} + +// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase +func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { + fs := p.Fs + + resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir) + publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) + + contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) + if err != nil { + return nil, err + } + + // Make sure we don't have any overlapping content dirs. That will never work. + for i, d1 := range absContentDirs { + for j, d2 := range absContentDirs { + if i == j { + continue + } + if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) + } + } + } + + b := &BaseFs{ + AbsContentDirs: absContentDirs, + ContentFs: contentFs, + ResourcesFs: resourcesFs, + PublishFs: publishFs, + } + + for _, opt := range options { + if err := opt(b); err != nil { + return nil, err + } + } + + builder := newSourceFilesystemsBuilder(p, b) + sourceFilesystems, err := builder.Build() + if err != nil { + return nil, err + } + + b.SourceFilesystems = sourceFilesystems + b.themeFs = builder.themeFs + b.AbsThemeDirs = builder.absThemeDirs + + return b, nil +} + +type sourceFilesystemsBuilder struct { + p *paths.Paths + result *SourceFilesystems + themeFs afero.Fs + hasTheme bool + absThemeDirs []string +} + +func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { + return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}} +} + +func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { + if b.themeFs == nil && b.p.ThemeSet() { + themeFs, absThemeDirs, err := createThemesOverlayFs(b.p) + if err != nil { + return nil, err + } + if themeFs == nil { + panic("createThemesFs returned nil") + } + b.themeFs = themeFs + b.absThemeDirs = absThemeDirs + + } + + b.hasTheme = len(b.absThemeDirs) > 0 + + sfs, err := b.createRootMappingFs("dataDir", "data") + if err != nil { + return nil, err + } + b.result.Data = sfs + + sfs, err = b.createRootMappingFs("i18nDir", "i18n") + if err != nil { + return nil, err + } + b.result.I18n = sfs + + sfs, err = b.createFs("layoutDir", "layouts") + if err != nil { + return nil, err + } + b.result.Layouts = sfs + + sfs, err = b.createFs("archetypeDir", "archetypes") + if err != nil { + return nil, err + } + b.result.Archetypes = sfs + + err = b.createStaticFs() + if err != nil { + return nil, err + } + + return b.result, nil +} + +func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{} + dir := b.p.Cfg.GetString(dirKey) + if dir == "" { + return s, fmt.Errorf("config %q not set", dirKey) + } + + var fs afero.Fs + + absDir := b.p.AbsPathify(dir) + if b.existsInSource(absDir) { + fs = afero.NewBasePathFs(b.p.Fs.Source, absDir) + s.Dirnames = []string{absDir} + } + + if b.hasTheme { + themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder) + if fs == nil { + fs = themeFolderFs + } else { + fs = afero.NewCopyOnWriteFs(themeFolderFs, fs) + } + + for _, absThemeDir := range b.absThemeDirs { + absThemeFolderDir := filepath.Join(absThemeDir, themeFolder) + if b.existsInSource(absThemeFolderDir) { + s.Dirnames = append(s.Dirnames, absThemeFolderDir) + } + } + } + + if fs == nil { + s.Fs = hugofs.NoOpFs + } else { + s.Fs = afero.NewReadOnlyFs(fs) + } + + return s, nil +} + +// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need +// to keep a strict order. +func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { + s := &SourceFilesystem{} + + projectDir := b.p.Cfg.GetString(dirKey) + if projectDir == "" { + return nil, fmt.Errorf("config %q not set", dirKey) + } + + var fromTo []string + to := b.p.AbsPathify(projectDir) + + if b.existsInSource(to) { + s.Dirnames = []string{to} + fromTo = []string{projectVirtualFolder, to} + } + + for _, theme := range b.p.AllThemes { + to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder)) + if b.existsInSource(to) { + s.Dirnames = append(s.Dirnames, to) + from := theme + fromTo = append(fromTo, from.Name, to) + } + } + + if len(fromTo) == 0 { + s.Fs = hugofs.NoOpFs + return s, nil + } + + fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...) + if err != nil { + return nil, err + } + + s.Fs = afero.NewReadOnlyFs(fs) + + return s, nil + +} + +func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool { + exists, _ := afero.Exists(b.p.Fs.Source, abspath) + return exists +} + +func (b *sourceFilesystemsBuilder) createStaticFs() error { + isMultihost := b.p.Cfg.GetBool("multihost") + ms := make(map[string]*SourceFilesystem) + b.result.Static = ms + + if isMultihost { + for _, l := range b.p.Languages { + s := &SourceFilesystem{PublishFolder: l.Lang} + staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) + if len(staticDirs) == 0 { + continue + } + + for _, dir := range staticDirs { + absDir := b.p.AbsPathify(dir) + if !b.existsInSource(absDir) { + continue + } + + s.Dirnames = append(s.Dirnames, absDir) + } + + fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) + if err != nil { + return err + } + + s.Fs = fs + ms[l.Lang] = s + + } + + return nil + } + + s := &SourceFilesystem{} + var staticDirs []string + + for _, l := range b.p.Languages { + staticDirs = append(staticDirs, getStaticDirs(l)...) + } + + staticDirs = removeDuplicatesKeepRight(staticDirs) + if len(staticDirs) == 0 { + return nil + } + + for _, dir := range staticDirs { + absDir := b.p.AbsPathify(dir) + if !b.existsInSource(absDir) { + continue + } + s.Dirnames = append(s.Dirnames, absDir) + } + + fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) + if err != nil { + return err + } + + if b.hasTheme { + themeFolder := "static" + fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs) + for _, absThemeDir := range b.absThemeDirs { + s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) + } + } + + s.Fs = fs + ms[""] = s + + return nil +} + +func getStaticDirs(cfg config.Provider) []string { + var staticDirs []string + for i := -1; i <= 10; i++ { + staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + } + return staticDirs +} + +func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { + + if id >= 0 { + key = fmt.Sprintf("%s%d", key, id) + } + + return config.GetStringSlicePreserveString(cfg, key) + +} + +func createContentFs(fs afero.Fs, + workingDir, + defaultContentLanguage string, + languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) { + + var contentLanguages langs.Languages + var contentDirSeen = make(map[string]bool) + languageSet := make(map[string]bool) + + // The default content language needs to be first. + for _, language := range languages { + if language.Lang == defaultContentLanguage { + contentLanguages = append(contentLanguages, language) + contentDirSeen[language.ContentDir] = true + } + languageSet[language.Lang] = true + } + + for _, language := range languages { + if contentDirSeen[language.ContentDir] { + continue + } + if language.ContentDir == "" { + language.ContentDir = defaultContentLanguage + } + contentDirSeen[language.ContentDir] = true + contentLanguages = append(contentLanguages, language) + + } + + var absContentDirs []types.KeyValueStr + + fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) + return fs, absContentDirs, err + +} + +func createContentOverlayFs(source afero.Fs, + workingDir string, + languages langs.Languages, + languageSet map[string]bool, + absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + if len(languages) == 0 { + return source, nil + } + + language := languages[0] + + contentDir := language.ContentDir + if contentDir == "" { + panic("missing contentDir") + } + + absContentDir := paths.AbsPathify(workingDir, language.ContentDir) + if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) { + absContentDir += paths.FilePathSeparator + } + + // If root, remove the second '/' + if absContentDir == "//" { + absContentDir = paths.FilePathSeparator + } + + if len(absContentDir) < 6 { + return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) + } + + *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + + overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) + if len(languages) == 1 { + return overlay, nil + } + + base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) + if err != nil { + return nil, err + } + + return hugofs.NewLanguageCompositeFs(base, overlay), nil + +} + +func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) { + + themes := p.AllThemes + + if len(themes) == 0 { + panic("AllThemes not set") + } + + themesDir := p.AbsPathify(p.ThemesDir) + if themesDir == "" { + return nil, nil, errors.New("no themes dir set") + } + + absPaths := make([]string, len(themes)) + + // The themes are ordered from left to right. We need to revert it to get the + // overlay logic below working as expected. + for i := 0; i < len(themes); i++ { + absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name) + } + + fs, err := createOverlayFs(p.Fs.Source, absPaths) + + return fs, absPaths, err + +} + +func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { + if len(absPaths) == 0 { + return hugofs.NoOpFs, nil + } + + if len(absPaths) == 1 { + return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil + } + + base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + overlay, err := createOverlayFs(source, absPaths[1:]) + if err != nil { + return nil, err + } + + return afero.NewCopyOnWriteFs(base, overlay), nil +} + +func removeDuplicatesKeepRight(in []string) []string { + seen := make(map[string]bool) + var out []string + for i := len(in) - 1; i >= 0; i-- { + v := in[i] + if seen[v] { + continue + } + out = append([]string{v}, out...) + seen[v] = true + } + + return out +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\tLANG: " + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go new file mode 100644 index 000000000..ea09cd8fd --- /dev/null +++ b/hugolib/filesystems/basefs_test.go @@ -0,0 +1,170 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filesystems + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib/paths" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestNewBaseFs(t *testing.T) { + assert := require.New(t) + v := viper.New() + + fs := hugofs.NewMem(v) + + themes := []string{"btheme", "atheme"} + + workingDir := filepath.FromSlash("/my/work") + v.Set("workingDir", workingDir) + v.Set("themesDir", "themes") + v.Set("theme", themes[:1]) + + // Write some data to the themes + for _, theme := range themes { + for _, dir := range []string{"i18n", "data"} { + base := filepath.Join(workingDir, "themes", theme, dir) + fs.Source.Mkdir(base, 0755) + afero.WriteFile(fs.Source, filepath.Join(base, fmt.Sprintf("theme-file-%s-%s.txt", theme, dir)), []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755) + } + } + + afero.WriteFile(fs.Source, filepath.Join(workingDir, "themes", "btheme", "config.toml"), []byte(` +theme = ["atheme"] +`), 0755) + + setConfigAndWriteSomeFilesTo(fs.Source, v, "contentDir", "mycontent", 3) + setConfigAndWriteSomeFilesTo(fs.Source, v, "i18nDir", "myi18n", 4) + setConfigAndWriteSomeFilesTo(fs.Source, v, "layoutDir", "mylayouts", 5) + setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6) + setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7) + setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8) + + p, err := paths.New(fs, v) + assert.NoError(err) + + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + + root, err := bfs.I18n.Fs.Open("") + assert.NoError(err) + dirnames, err := root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) + ff, err := bfs.I18n.Fs.Open("myi18n") + assert.NoError(err) + _, err = ff.Readdirnames(-1) + assert.NoError(err) + + root, err = bfs.Data.Fs.Open("") + assert.NoError(err) + dirnames, err = root.Readdirnames(-1) + assert.NoError(err) + assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames) + ff, err = bfs.I18n.Fs.Open("mydata") + assert.NoError(err) + _, err = ff.Readdirnames(-1) + assert.NoError(err) + + checkFileCount(bfs.ContentFs, "", assert, 3) + checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes + checkFileCount(bfs.Layouts.Fs, "", assert, 5) + checkFileCount(bfs.Static[""].Fs, "", assert, 6) + checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes + checkFileCount(bfs.Archetypes.Fs, "", assert, 8) + + assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames) + + assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt"))) + assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt"))) + assert.True(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt"))) + assert.True(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt"))) + contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt") + assert.True(bfs.IsContent(contentFilename)) + rel, _ := bfs.RelContentDir(contentFilename) + assert.Equal("file1.txt", rel) + +} + +func TestNewBaseFsEmpty(t *testing.T) { + assert := require.New(t) + v := viper.New() + v.Set("contentDir", "mycontent") + v.Set("i18nDir", "myi18n") + v.Set("staticDir", "mystatic") + v.Set("dataDir", "mydata") + v.Set("layoutDir", "mylayouts") + v.Set("archetypeDir", "myarchetypes") + + fs := hugofs.NewMem(v) + p, err := paths.New(fs, v) + bfs, err := NewBase(p) + assert.NoError(err) + assert.NotNil(bfs) + assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs) + assert.Equal(hugofs.NoOpFs, bfs.Data.Fs) + assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs) + assert.NotNil(hugofs.NoOpFs, bfs.ContentFs) + assert.NotNil(hugofs.NoOpFs, bfs.Static) +} + +func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) { + count, _, err := countFileaAndGetDirs(fs, dirname) + assert.NoError(err) + assert.Equal(expected, count) +} + +func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) { + if fs == nil { + return 0, nil, errors.New("no fs") + } + + counter := 0 + var dirs []string + + afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error { + if info != nil { + if !info.IsDir() { + counter++ + } else if info.Name() != "." { + dirs = append(dirs, filepath.Join(path, info.Name())) + } + } + + return nil + }) + + return counter, dirs, nil +} + +func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) { + workingDir := v.GetString("workingDir") + v.Set(key, val) + fs.Mkdir(val, 0755) + for i := 0; i < num; i++ { + afero.WriteFile(fs, filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1)), []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755) + } +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index ad233f1c2..a0ac72d67 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -25,6 +25,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/i18n" "github.com/gohugoio/hugo/tpl" @@ -228,10 +229,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { - templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "") - if s.PathSpec.ThemeSet() { - templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme") - } + templ.LoadTemplates("") for _, wt := range withTemplates { if wt == nil { @@ -289,7 +287,7 @@ func (h *HugoSites) resetLogs() { } func (h *HugoSites) createSitesFromConfig() error { - oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages) + oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { return err diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 4192580bd..221987b37 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -3,7 +3,6 @@ package hugolib import ( "bytes" "fmt" - "io" "strings" "testing" @@ -12,6 +11,8 @@ import ( "path/filepath" "time" + "github.com/gohugoio/hugo/langs" + "github.com/fortytw2/leaktest" "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" @@ -660,7 +661,7 @@ title = "Svenska" sites := b.H // Watching does not work with in-memory fs, so we trigger a reload manually - assert.NoError(sites.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig()) + assert.NoError(sites.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig()) err := b.H.Build(BuildCfg{CreateSitesFromConfig: true}) if err != nil { @@ -723,7 +724,7 @@ func TestChangeDefaultLanguage(t *testing.T) { // Watching does not work with in-memory fs, so we trigger a reload manually // This does not look pretty, so we should think of something else. - assert.NoError(b.H.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig()) + assert.NoError(b.H.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig()) err := b.H.Build(BuildCfg{CreateSitesFromConfig: true}) if err != nil { t.Fatalf("Failed to rebuild sites: %s", err) @@ -1177,31 +1178,12 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { if err != nil { // Print some debug info root := strings.Split(filename, helpers.FilePathSeparator)[0] - printFs(fs, root, os.Stdout) + helpers.PrintFs(fs, root, os.Stdout) Fatalf(t, "Failed to read file: %s", err) } return string(b) } -func printFs(fs afero.Fs, path string, w io.Writer) { - if fs == nil { - return - } - afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - s := path - if lang, ok := info.(hugofs.LanguageAnnouncer); ok { - s = s + "\tLANG: " + lang.Lang() - } - if fp, ok := info.(hugofs.FilePather); ok { - s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() - } - fmt.Fprintln(w, " ", s) - } - return nil - }) -} - const testPageTemplate = `--- title: "%s" publishdate: "%s" diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 7dc2d8e1c..2ccbb6ca1 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -55,8 +55,6 @@ languageName = "Nynorsk" s1 := b.H.Sites[0] - assert.Equal([]string{"s1", "s2", "ens1", "ens2"}, s1.StaticDirs()) - s1h := s1.getPage(KindHome) assert.True(s1h.IsTranslated()) assert.Len(s1h.Translations(), 2) @@ -79,7 +77,6 @@ languageName = "Nynorsk" b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`) s2 := b.H.Sites[1] - assert.Equal([]string{"s1", "s2", "frs1", "frs2"}, s2.StaticDirs()) s2h := s2.getPage(KindHome) assert.Equal("https://example.fr/", s2h.Permalink()) diff --git a/hugolib/hugo_themes_test.go b/hugolib/hugo_themes_test.go new file mode 100644 index 000000000..05bfaa692 --- /dev/null +++ b/hugolib/hugo_themes_test.go @@ -0,0 +1,268 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/common/loggers" +) + +func TestThemesGraph(t *testing.T) { + t.Parallel() + + const ( + themeStandalone = ` +title = "Theme Standalone" +[params] +v1 = "v1s" +v2 = "v2s" +` + themeCyclic = ` +title = "Theme Cyclic" +theme = "theme3" +[params] +v1 = "v1c" +v2 = "v2c" +` + theme1 = ` +title = "Theme #1" +theme = "themeStandalone" +[params] +v2 = "v21" +` + + theme2 = ` +title = "Theme #2" +theme = "theme1" +[params] +v1 = "v12" +` + + theme3 = ` +title = "Theme #3" +theme = ["theme2", "themeStandalone", "themeCyclic"] +[params] +v1 = "v13" +v2 = "v24" +` + + theme4 = ` +title = "Theme #4" +theme = "theme3" +[params] +v1 = "v14" +v2 = "v24" +` + + site1 = ` + theme = "theme4" + + [params] + v1 = "site" +` + site2 = ` + theme = ["theme2", "themeStandalone"] +` + ) + + var ( + testConfigs = []struct { + siteConfig string + + // The name of theme somewhere in the middle to write custom key/files. + offset string + + check func(b *sitesBuilder) + }{ + {site1, "theme3", func(b *sitesBuilder) { + + // site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic + + // Check data + // theme3 should win the offset competition + b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3") + b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1") + b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3") + b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|") + b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|") + + // Check layouts + b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3") + + // Check i18n + b.AssertFileContent("public/index.html", "i18n: project theme4") + + // Check static files + // TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO") + + // Check site params + b.AssertFileContent("public/index.html", "v1::site", "v2::v24") + }}, + {site2, "", func(b *sitesBuilder) { + + // site2: theme2 theme1 themeStandalone + b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") + b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|") + b.AssertFileContent("public/index.html", "i18n: project theme2") + b.AssertFileContent("public/index.html", "partial ntheme: theme2") + + // Params only set in themes + b.AssertFileContent("public/index.html", "v1::v12", "v2::v21") + + }}, + } + + themeConfigs = []struct { + name string + config string + }{ + {"themeStandalone", themeStandalone}, + {"themeCyclic", themeCyclic}, + {"theme1", theme1}, + {"theme2", theme2}, + {"theme3", theme3}, + {"theme4", theme4}, + } + ) + + for i, testConfig := range testConfigs { + t.Log(fmt.Sprintf("Test %d", i)) + b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger()) + b.WithConfigFile("toml", testConfig.siteConfig) + + for _, tc := range themeConfigs { + var variationsNameBase = []string{"nproject", "ntheme", tc.name} + + themeRoot := filepath.Join("themes", tc.name) + b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config) + + b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }} +{{ end }} +{{ define "printv" }} +{{- $tp := printf "%T" . -}} +{{- if (strings.HasSuffix $tp "map[string]interface {}") -}} +{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}} +{{- else -}} +{{- . }}| +{{- end -}} +{{ end }} +`) + + for _, nameVariaton := range variationsNameBase { + roots := []string{"", themeRoot} + + for _, root := range roots { + name := tc.name + if root == "" { + name = "project" + } + + if nameVariaton == "ntheme" && name == "project" { + continue + } + + // static + b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name) + + // layouts + if i == 1 { + b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set") + } + b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name) + if root != "" && testConfig.offset == tc.name { + for _, tc2 := range themeConfigs { + b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name) + } + } + + // i18n + data + + var dataContent string + if root == "" { + dataContent = fmt.Sprintf(` +[%s] +other = %q + +[inner] +other = %q + +`, name, name, name) + } else { + dataContent = fmt.Sprintf(` +[%s] +other = %q + +[inner] +other = %q + +[theme] +other = %q + +`, name, name, name, name) + } + + b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent) + b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent) + + // If an offset is set, duplicate a data key with a winner in the middle. + if root != "" && testConfig.offset == tc.name { + for _, tc2 := range themeConfigs { + dataContent := fmt.Sprintf(` +[offset] +v = %q +`, tc.name) + b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent) + } + } + } + + } + + } + + for _, themeConfig := range themeConfigs { + b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config) + } + + b.WithContent(filepath.Join("content", "page.md"), `--- +title: "Page" +--- + +`) + + homeTpl := ` +data: {{ partial "m" .Site.Data }} +i18n: {{ i18n "inner" }} {{ i18n "theme" }} +partial ntheme: {{ partial "ntheme" . }} +partial theme2o: {{ partial "theme2o" . }} +params: {{ partial "m" .Site.Params }} + +` + + b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl) + + b.Build(BuildCfg{}) + + var _ = os.Stdout + + // printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout) + testConfig.check(b) + + } + +} diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index a3f3828ef..c09e3667e 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -16,30 +16,33 @@ package hugolib import ( "sync" + "github.com/gohugoio/hugo/common/maps" + "sort" "errors" "fmt" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" ) // Multilingual manages the all languages used in a multilingual site. type Multilingual struct { - Languages helpers.Languages + Languages langs.Languages - DefaultLang *helpers.Language + DefaultLang *langs.Language - langMap map[string]*helpers.Language + langMap map[string]*langs.Language langMapInit sync.Once } // Language returns the Language associated with the given string. -func (ml *Multilingual) Language(lang string) *helpers.Language { +func (ml *Multilingual) Language(lang string) *langs.Language { ml.langMapInit.Do(func() { - ml.langMap = make(map[string]*helpers.Language) + ml.langMap = make(map[string]*langs.Language) for _, l := range ml.Languages { ml.langMap[l.Lang] = l } @@ -47,16 +50,16 @@ func (ml *Multilingual) Language(lang string) *helpers.Language { return ml.langMap[lang] } -func getLanguages(cfg config.Provider) helpers.Languages { +func getLanguages(cfg config.Provider) langs.Languages { if cfg.IsSet("languagesSorted") { - return cfg.Get("languagesSorted").(helpers.Languages) + return cfg.Get("languagesSorted").(langs.Languages) } - return helpers.Languages{helpers.NewDefaultLanguage(cfg)} + return langs.Languages{langs.NewDefaultLanguage(cfg)} } func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) { - languages := make(helpers.Languages, len(sites)) + languages := make(langs.Languages, len(sites)) for i, s := range sites { if s.Language == nil { @@ -71,12 +74,12 @@ func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingua defaultLang = "en" } - return &Multilingual{Languages: languages, DefaultLang: helpers.NewLanguage(defaultLang, cfg)}, nil + return &Multilingual{Languages: languages, DefaultLang: langs.NewLanguage(defaultLang, cfg)}, nil } -func newMultiLingualForLanguage(language *helpers.Language) *Multilingual { - languages := helpers.Languages{language} +func newMultiLingualForLanguage(language *langs.Language) *Multilingual { + languages := langs.Languages{language} return &Multilingual{Languages: languages, DefaultLang: language} } func (ml *Multilingual) enabled() bool { @@ -90,8 +93,8 @@ func (s *Site) multilingualEnabled() bool { return s.owner.multilingual != nil && s.owner.multilingual.enabled() } -func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.Languages, error) { - langs := make(helpers.Languages, len(l)) +func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) { + languages := make(langs.Languages, len(l)) i := 0 for lang, langConf := range l { @@ -101,7 +104,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L return nil, fmt.Errorf("Language config is not a map: %T", langConf) } - language := helpers.NewLanguage(lang, cfg) + language := langs.NewLanguage(lang, cfg) for loki, v := range langsMap { switch loki { @@ -118,7 +121,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L case "params": m := cast.ToStringMap(v) // Needed for case insensitive fetching of params values - helpers.ToLowerMap(m) + maps.ToLower(m) for k, vv := range m { language.SetParam(k, vv) } @@ -131,11 +134,11 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L language.Set(loki, v) } - langs[i] = language + languages[i] = language i++ } - sort.Sort(langs) + sort.Sort(languages) - return langs, nil + return languages, nil } diff --git a/hugolib/page.go b/hugolib/page.go index 89d68084e..322660647 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -21,6 +21,10 @@ import ( "reflect" "unicode" + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/related" "github.com/bep/gitmap" @@ -254,7 +258,7 @@ type Page struct { // It would be tempting to use the language set on the Site, but in they way we do // multi-site processing, these values may differ during the initial page processing. - language *helpers.Language + language *langs.Language lang string @@ -1281,7 +1285,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error { return errors.New("missing frontmatter data") } // Needed for case insensitive fetching of params values - helpers.ToLowerMap(frontmatter) + maps.ToLower(frontmatter) var mtime time.Time if p.Source.FileInfo() != nil { @@ -2028,7 +2032,7 @@ func (p *Page) Scratch() *Scratch { return p.scratch } -func (p *Page) Language() *helpers.Language { +func (p *Page) Language() *langs.Language { p.initLanguage() return p.language } diff --git a/hugolib/page_bundler_capture.go b/hugolib/page_bundler_capture.go index 255a8efda..92b3efe49 100644 --- a/hugolib/page_bundler_capture.go +++ b/hugolib/page_bundler_capture.go @@ -75,7 +75,7 @@ func newCapturer( sem: make(chan bool, numWorkers), handler: handler, sourceSpec: sourceSpec, - fs: sourceSpec.Fs, + fs: sourceSpec.SourceFs, logger: logger, contentChanges: contentChanges, seen: make(map[string]bool), diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index c07383797..14d8a4368 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -20,6 +20,8 @@ import ( "path/filepath" "sort" + "github.com/gohugoio/hugo/common/loggers" + jww "github.com/spf13/jwalterweatherman" "runtime" @@ -92,7 +94,7 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) { sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - logger := newErrorLogger() + logger := loggers.NewErrorLogger() c := newCapturer(logger, sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -139,12 +141,10 @@ func TestPageBundlerCaptureBasic(t *testing.T) { fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) - printFs(fs.Source, "", os.Stdout) - expected := ` F: /work/base/_1.md @@ -185,7 +185,7 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) { sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) + c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -265,7 +265,7 @@ func BenchmarkPageBundlerCapture(b *testing.B) { writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content") } - capturers[i] = newCapturer(newErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) + capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base) } b.ResetTimer() diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index 2d83e3af1..f66b8c0db 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -15,6 +15,9 @@ package hugolib import ( "io/ioutil" + + "github.com/gohugoio/hugo/common/loggers" + "os" "runtime" "strings" @@ -75,7 +78,7 @@ func TestPageBundlerSiteRegular(t *testing.T) { cfg.Set("uglyURLs", ugly) - s := buildSingleSite(t, deps.DepsCfg{Logger: newWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{}) th := testHelper{s.Cfg, s.Fs, t} @@ -158,7 +161,6 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename()) assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) - printFs(th.Fs.Destination, "", os.Stdout) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") @@ -329,7 +331,7 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { cfg := ps.Cfg fs := ps.Fs - s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{}) + s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{}) th := testHelper{s.Cfg, s.Fs, t} diff --git a/hugolib/pagination.go b/hugolib/pagination.go index 86113271b..84ad74b07 100644 --- a/hugolib/pagination.go +++ b/hugolib/pagination.go @@ -532,7 +532,7 @@ func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory { pathDescriptor := d var rel string if page > 1 { - rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath(), page) + rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath, page) pathDescriptor.Addends = rel } diff --git a/helpers/baseURL.go b/hugolib/paths/baseURL.go index eb39ced5b..9cb5627ba 100644 --- a/helpers/baseURL.go +++ b/hugolib/paths/baseURL.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package paths import ( "fmt" @@ -30,6 +30,10 @@ func (b BaseURL) String() string { return b.urlStr } +func (b BaseURL) Path() string { + return b.url.Path +} + // WithProtocol returns the BaseURL prefixed with the given protocol. // The Protocol is normally of the form "scheme://", i.e. "webcal://". func (b BaseURL) WithProtocol(protocol string) (string, error) { diff --git a/helpers/baseURL_test.go b/hugolib/paths/baseURL_test.go index 437152f34..af1d2e38d 100644 --- a/helpers/baseURL_test.go +++ b/hugolib/paths/baseURL_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package paths import ( "testing" diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go new file mode 100644 index 000000000..cf8792e5a --- /dev/null +++ b/hugolib/paths/paths.go @@ -0,0 +1,231 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/langs" + + "github.com/gohugoio/hugo/hugofs" +) + +var FilePathSeparator = string(filepath.Separator) + +type Paths struct { + Fs *hugofs.Fs + Cfg config.Provider + + BaseURL + + // If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath. + // This will not be set if canonifyURLs is enabled. + BasePath string + + // Directories + // TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make + // these into an interface. + ContentDir string + ThemesDir string + WorkingDir string + AbsResourcesDir string + AbsPublishDir string + + // pagination path handling + PaginatePath string + + PublishDir string + + DisablePathToLower bool + RemovePathAccents bool + UglyURLs bool + CanonifyURLs bool + + Language *langs.Language + Languages langs.Languages + + // The PathSpec looks up its config settings in both the current language + // and then in the global Viper config. + // Some settings, the settings listed below, does not make sense to be set + // on per-language-basis. We have no good way of protecting against this + // other than a "white-list". See language.go. + defaultContentLanguageInSubdir bool + DefaultContentLanguage string + multilingual bool + + themes []string + AllThemes []ThemeConfig +} + +func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { + baseURLstr := cfg.GetString("baseURL") + baseURL, err := newBaseURLFromString(baseURLstr) + + if err != nil { + return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) + } + + // TODO(bep) + contentDir := cfg.GetString("contentDir") + workingDir := cfg.GetString("workingDir") + resourceDir := cfg.GetString("resourceDir") + publishDir := cfg.GetString("publishDir") + + defaultContentLanguage := cfg.GetString("defaultContentLanguage") + + var ( + language *langs.Language + languages langs.Languages + ) + + if l, ok := cfg.(*langs.Language); ok { + language = l + + } + + if l, ok := cfg.Get("languagesSorted").(langs.Languages); ok { + languages = l + } + + if len(languages) == 0 { + // We have some old tests that does not test the entire chain, hence + // they have no languages. So create one so we get the proper filesystem. + languages = langs.Languages{&langs.Language{Lang: "en", Cfg: cfg, ContentDir: contentDir}} + } + + absPublishDir := AbsPathify(workingDir, publishDir) + if !strings.HasSuffix(absPublishDir, FilePathSeparator) { + absPublishDir += FilePathSeparator + } + // If root, remove the second '/' + if absPublishDir == "//" { + absPublishDir = FilePathSeparator + } + absResourcesDir := AbsPathify(workingDir, resourceDir) + if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { + absResourcesDir += FilePathSeparator + } + if absResourcesDir == "//" { + absResourcesDir = FilePathSeparator + } + + p := &Paths{ + Fs: fs, + Cfg: cfg, + BaseURL: baseURL, + + DisablePathToLower: cfg.GetBool("disablePathToLower"), + RemovePathAccents: cfg.GetBool("removePathAccents"), + UglyURLs: cfg.GetBool("uglyURLs"), + CanonifyURLs: cfg.GetBool("canonifyURLs"), + + ContentDir: contentDir, + ThemesDir: cfg.GetString("themesDir"), + WorkingDir: workingDir, + + AbsResourcesDir: absResourcesDir, + AbsPublishDir: absPublishDir, + + themes: config.GetStringSlicePreserveString(cfg, "theme"), + + multilingual: cfg.GetBool("multilingual"), + defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), + DefaultContentLanguage: defaultContentLanguage, + + Language: language, + Languages: languages, + + PaginatePath: cfg.GetString("paginatePath"), + } + + if cfg.IsSet("allThemes") { + p.AllThemes = cfg.Get("allThemes").([]ThemeConfig) + } else { + p.AllThemes, err = collectThemeNames(p) + if err != nil { + return nil, err + } + } + + // TODO(bep) remove this, eventually + p.PublishDir = absPublishDir + + return p, nil +} + +func (p *Paths) Lang() string { + if p == nil || p.Language == nil { + return "" + } + return p.Language.Lang +} + +// ThemeSet checks whether a theme is in use or not. +func (p *Paths) ThemeSet() bool { + return len(p.themes) > 0 +} + +func (p *Paths) Themes() []string { + return p.themes +} + +func (p *Paths) GetLanguagePrefix() string { + if !p.multilingual { + return "" + } + + defaultLang := p.DefaultContentLanguage + defaultInSubDir := p.defaultContentLanguageInSubdir + + currentLang := p.Language.Lang + if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) { + return "" + } + return currentLang +} + +// GetLangSubDir returns the given language's subdir if needed. +func (p *Paths) GetLangSubDir(lang string) string { + if !p.multilingual { + return "" + } + + if p.Languages.IsMultihost() { + return "" + } + + if lang == "" || (lang == p.DefaultContentLanguage && !p.defaultContentLanguageInSubdir) { + return "" + } + + return lang +} + +// AbsPathify creates an absolute path if given a relative path. If already +// absolute, the path is just cleaned. +func (p *Paths) AbsPathify(inPath string) string { + return AbsPathify(p.WorkingDir, inPath) +} + +// AbsPathify creates an absolute path if given a working dir and arelative path. +// If already absolute, the path is just cleaned. +func AbsPathify(workingDir, inPath string) string { + if filepath.IsAbs(inPath) { + return filepath.Clean(inPath) + } + return filepath.Join(workingDir, inPath) +} diff --git a/hugolib/paths/paths_test.go b/hugolib/paths/paths_test.go new file mode 100644 index 000000000..6cadc747f --- /dev/null +++ b/hugolib/paths/paths_test.go @@ -0,0 +1,40 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "testing" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestNewPaths(t *testing.T) { + assert := require.New(t) + + v := viper.New() + fs := hugofs.NewMem(v) + + v.Set("defaultContentLanguageInSubdir", true) + v.Set("defaultContentLanguage", "no") + v.Set("multilingual", true) + + p, err := New(fs, v) + assert.NoError(err) + + assert.Equal(true, p.defaultContentLanguageInSubdir) + assert.Equal("no", p.DefaultContentLanguage) + assert.Equal(true, p.multilingual) +} diff --git a/hugolib/paths/themes.go b/hugolib/paths/themes.go new file mode 100644 index 000000000..abe6121ae --- /dev/null +++ b/hugolib/paths/themes.go @@ -0,0 +1,162 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package paths + +import ( + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +type ThemeConfig struct { + // The theme name as provided by the folder name below /themes. + Name string + + // Optional configuration filename (e.g. "/themes/mytheme/config.json"). + ConfigFilename string + + // Optional config read from the ConfigFile above. + Cfg config.Provider +} + +// Create file system, an ordered theme list from left to right, no duplicates. +type themesCollector struct { + themesDir string + fs afero.Fs + seen map[string]bool + themes []ThemeConfig +} + +func (c *themesCollector) isSeen(theme string) bool { + loki := strings.ToLower(theme) + if c.seen[loki] { + return true + } + c.seen[loki] = true + return false +} + +func (c *themesCollector) addAndRecurse(themes ...string) error { + for i := 0; i < len(themes); i++ { + theme := themes[i] + configFilename := c.getConfigFileIfProvided(theme) + if !c.isSeen(theme) { + tc, err := c.add(theme, configFilename) + if err != nil { + return err + } + if err := c.addTemeNamesFromTheme(tc); err != nil { + return err + } + } + } + return nil +} + +func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) { + var cfg config.Provider + var tc ThemeConfig + + if configFilename != "" { + v := viper.New() + v.SetFs(c.fs) + v.AutomaticEnv() + v.SetEnvPrefix("hugo") + v.SetConfigFile(configFilename) + + err := v.ReadInConfig() + if err != nil { + return tc, err + } + cfg = v + + } + + tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg} + c.themes = append(c.themes, tc) + return tc, nil + +} + +func collectThemeNames(p *Paths) ([]ThemeConfig, error) { + return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes()) + +} + +func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) { + if len(themes) == 0 { + return nil, nil + } + + c := &themesCollector{ + fs: fs, + themesDir: themesDir, + seen: make(map[string]bool)} + + for i := 0; i < len(themes); i++ { + theme := themes[i] + if err := c.addAndRecurse(theme); err != nil { + return nil, err + } + } + + return c.themes, nil + +} + +func (c *themesCollector) getConfigFileIfProvided(theme string) string { + configDir := filepath.Join(c.themesDir, theme) + + var ( + configFilename string + exists bool + ) + + // Viper supports more, but this is the sub-set supported by Hugo. + for _, configFormats := range []string{"toml", "yaml", "yml", "json"} { + configFilename = filepath.Join(configDir, "config."+configFormats) + exists, _ = afero.Exists(c.fs, configFilename) + if exists { + break + } + } + + if !exists { + // No theme config set. + return "" + } + + return configFilename + +} + +func (c *themesCollector) addTemeNamesFromTheme(theme ThemeConfig) error { + if theme.Cfg != nil && theme.Cfg.IsSet("theme") { + v := theme.Cfg.Get("theme") + switch vv := v.(type) { + case []string: + return c.addAndRecurse(vv...) + case []interface{}: + return c.addAndRecurse(cast.ToStringSlice(vv)...) + default: + return c.addAndRecurse(cast.ToString(vv)) + } + } + + return nil +} diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index a4c6ca20d..1437ae0cf 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -34,7 +34,9 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl" + "github.com/stretchr/testify/require" ) @@ -46,7 +48,7 @@ func pageFromString(in, filename string, withTemplate ...func(templ tpl.Template var err error cfg, fs := newTestCfg() - d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]} + d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]} s, err = NewSiteForCfg(d) if err != nil { diff --git a/hugolib/site.go b/hugolib/site.go index 04a18bb48..8ff724d0a 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -27,6 +27,10 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/langs" + + src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/resource" "golang.org/x/sync/errgroup" @@ -107,7 +111,7 @@ type Site struct { expiredCount int Data map[string]interface{} - Language *helpers.Language + Language *langs.Language disabledKinds map[string]bool @@ -175,7 +179,7 @@ func (s *Site) isEnabled(kind string) bool { // reset returns a new Site prepared for rebuild. func (s *Site) reset() *Site { return &Site{Deps: s.Deps, - layoutHandler: output.NewLayoutHandler(s.PathSpec.ThemeSet()), + layoutHandler: output.NewLayoutHandler(), disabledKinds: s.disabledKinds, titleFunc: s.titleFunc, relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), @@ -195,7 +199,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { c := newPageCollections() if cfg.Language == nil { - cfg.Language = helpers.NewDefaultLanguage(cfg.Cfg) + cfg.Language = langs.NewDefaultLanguage(cfg.Cfg) } disabledKinds := make(map[string]bool) @@ -261,7 +265,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) { s := &Site{ PageCollections: c, - layoutHandler: output.NewLayoutHandler(cfg.Cfg.GetString("themesDir") != ""), + layoutHandler: output.NewLayoutHandler(), Language: cfg.Language, disabledKinds: disabledKinds, titleFunc: titleFunc, @@ -304,7 +308,7 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) ( if err := loadDefaultSettingsFor(v); err != nil { return nil, err } - return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...) + return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...) } // NewEnglishSite creates a new site in English language. @@ -316,11 +320,11 @@ func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Sit if err := loadDefaultSettingsFor(v); err != nil { return nil, err } - return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...) + return newSiteForLang(langs.NewLanguage("en", v), withTemplate...) } // newSiteForLang creates a new site in the given language. -func newSiteForLang(lang *helpers.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { +func newSiteForLang(lang *langs.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { withTemplates := func(templ tpl.TemplateHandler) error { for _, wt := range withTemplate { if err := wt(templ); err != nil { @@ -389,9 +393,9 @@ type SiteInfo struct { owner *HugoSites s *Site multilingual *Multilingual - Language *helpers.Language + Language *langs.Language LanguagePrefix string - Languages helpers.Languages + Languages langs.Languages defaultContentLanguageInSubdir bool sectionPagesMenu string } @@ -431,7 +435,7 @@ func (s *SiteInfo) DisqusShortname() string { // Used in tests. type siteBuilderCfg struct { - language *helpers.Language + language *langs.Language s *Site pageCollections *PageCollections } @@ -805,15 +809,13 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } -func (s *Site) loadData(sourceDirs []string) (err error) { - s.Log.DEBUG.Printf("Load Data from %d source(s)", len(sourceDirs)) +func (s *Site) loadData(fs afero.Fs) (err error) { + spec := src.NewSourceSpec(s.PathSpec, fs) + fileSystem := spec.NewFilesystem("") s.Data = make(map[string]interface{}) - for _, sourceDir := range sourceDirs { - fs := s.SourceSpec.NewFilesystem(sourceDir) - for _, r := range fs.Files() { - if err := s.handleDataFile(r); err != nil { - return err - } + for _, r := range fileSystem.Files() { + if err := s.handleDataFile(r); err != nil { + return err } } @@ -831,12 +833,17 @@ func (s *Site) handleDataFile(r source.ReadableFile) error { // Crawl in data tree to insert data current = s.Data - for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) + keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) + // The first path element is the virtual folder (typically theme name), which is + // not part of the key. + if len(keyParts) > 1 { + for _, key := range keyParts[1:] { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) + } + current = current[key].(map[string]interface{}) } - current = current[key].(map[string]interface{}) } } @@ -919,18 +926,7 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) { } func (s *Site) readDataFromSourceFS() error { - var dataSourceDirs []string - - // have to be last - duplicate keys in earlier entries will win - themeDataDir, err := s.PathSpec.GetThemeDataDirPath() - if err == nil { - dataSourceDirs = []string{s.absDataDir(), themeDataDir} - } else { - dataSourceDirs = []string{s.absDataDir()} - - } - - err = s.loadData(dataSourceDirs) + err := s.loadData(s.PathSpec.BaseFs.Data.Fs) s.timerStep("load data") return err } @@ -1041,10 +1037,6 @@ func (s *Site) Initialise() (err error) { func (s *Site) initialize() (err error) { s.Menus = Menus{} - if err = s.checkDirectories(); err != nil { - return err - } - return s.initializeSiteInfo() } @@ -1071,7 +1063,7 @@ func (s *SiteInfo) SitemapAbsURL() string { func (s *Site) initializeSiteInfo() error { var ( lang = s.Language - languages helpers.Languages + languages langs.Languages ) if s.owner != nil && s.owner.multilingual != nil { @@ -1166,126 +1158,24 @@ func (s *Site) initializeSiteInfo() error { return nil } -func (s *Site) dataDir() string { - return s.Cfg.GetString("dataDir") -} - -func (s *Site) absDataDir() string { - return s.PathSpec.AbsPathify(s.dataDir()) -} - -func (s *Site) i18nDir() string { - return s.Cfg.GetString("i18nDir") -} - -func (s *Site) absI18nDir() string { - return s.PathSpec.AbsPathify(s.i18nDir()) -} - func (s *Site) isI18nEvent(e fsnotify.Event) bool { - if s.getI18nDir(e.Name) != "" { - return true - } - return s.getThemeI18nDir(e.Name) != "" -} - -func (s *Site) getI18nDir(path string) string { - return s.getRealDir(s.absI18nDir(), path) -} - -func (s *Site) getThemeI18nDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.i18nDir()), path) + return s.BaseFs.SourceFilesystems.IsI18n(e.Name) } func (s *Site) isDataDirEvent(e fsnotify.Event) bool { - if s.getDataDir(e.Name) != "" { - return true - } - return s.getThemeDataDir(e.Name) != "" -} - -func (s *Site) getDataDir(path string) string { - return s.getRealDir(s.absDataDir(), path) -} - -func (s *Site) getThemeDataDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.dataDir()), path) -} - -func (s *Site) layoutDir() string { - return s.Cfg.GetString("layoutDir") + return s.BaseFs.SourceFilesystems.IsData(e.Name) } func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { - if s.getLayoutDir(e.Name) != "" { - return true - } - return s.getThemeLayoutDir(e.Name) != "" -} - -func (s *Site) getLayoutDir(path string) string { - return s.getRealDir(s.PathSpec.GetLayoutDirPath(), path) -} - -func (s *Site) getThemeLayoutDir(path string) string { - if !s.PathSpec.ThemeSet() { - return "" - } - return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.layoutDir()), path) + return s.BaseFs.SourceFilesystems.IsLayout(e.Name) } func (s *Site) absContentDir() string { - return s.PathSpec.AbsPathify(s.PathSpec.ContentDir()) + return s.PathSpec.AbsPathify(s.PathSpec.ContentDir) } func (s *Site) isContentDirEvent(e fsnotify.Event) bool { - relDir, _ := s.PathSpec.RelContentDir(e.Name) - return relDir != e.Name -} - -func (s *Site) getContentDir(path string) string { - return s.getRealDir(s.absContentDir(), path) -} - -// getRealDir gets the base path of the given path, also handling the case where -// base is a symlinked folder. -func (s *Site) getRealDir(base, path string) string { - - if strings.HasPrefix(path, base) { - return base - } - - realDir, err := helpers.GetRealPath(s.Fs.Source, base) - - if err != nil { - if !os.IsNotExist(err) { - s.Log.ERROR.Printf("Failed to get real path for %s: %s", path, err) - } - return "" - } - - if strings.HasPrefix(path, realDir) { - return realDir - } - - return "" -} - -func (s *Site) absPublishDir() string { - return s.PathSpec.AbsPathify(s.Cfg.GetString("publishDir")) -} - -func (s *Site) checkDirectories() (err error) { - if b, _ := helpers.DirExists(s.absContentDir(), s.Fs.Source); !b { - return errors.New("No source directory found, expecting to find it at " + s.absContentDir()) - } - return + return s.BaseFs.IsContent(e.Name) } type contentCaptureResultHandler struct { @@ -1871,9 +1761,7 @@ func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) { s.PathSpec.ProcessingStats.Incr(statCounter) - path = filepath.Join(s.absPublishDir(), path) - - return helpers.WriteToDisk(path, r, s.Fs.Destination) + return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs) } func getGoMaxProcs() int { diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 231200a7b..93ea5032e 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -10,8 +10,8 @@ import ( "strings" "text/template" + "github.com/gohugoio/hugo/langs" "github.com/sanity-io/litter" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/config" @@ -22,11 +22,8 @@ import ( "github.com/gohugoio/hugo/tpl" "github.com/spf13/viper" - "io/ioutil" "os" - "log" - "github.com/gohugoio/hugo/hugofs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -135,6 +132,11 @@ func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder { return s } +func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder { + writeSource(s.T, s.Fs, filepath.FromSlash(filename), content) + return s +} + const commonConfigSections = ` [services] @@ -304,15 +306,17 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { s.writeFilePairs("i18n", s.i18nFilePairsAdded) if s.Cfg == nil { - cfg, configFiles, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) + cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) if err != nil { s.Fatalf("Failed to load config: %s", err) } - expectedConfigs := 1 - if s.theme != "" { - expectedConfigs = 2 - } - require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles)) + // TODO(bep) + /* expectedConfigs := 1 + if s.theme != "" { + expectedConfigs = 2 + } + require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles)) + */ s.Cfg = cfg } @@ -337,6 +341,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { if s.H == nil { s.CreateSites() } + err := s.H.Build(cfg) if err == nil { logErrorCount := s.H.NumLogErrors() @@ -436,7 +441,7 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) { content := readDestination(s.T, s.Fs, filename) for _, match := range matches { if !strings.Contains(content, match) { - s.Fatalf("No match for %q in content for %s\n%q", match, filename, content) + s.Fatalf("No match for %q in content for %s\n%s", match, filename, content) } } } @@ -509,7 +514,7 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string { } func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec { - l := helpers.NewDefaultLanguage(v) + l := langs.NewDefaultLanguage(v) ps, _ := helpers.NewPathSpec(fs, l) return ps } @@ -519,6 +524,10 @@ func newTestDefaultPathSpec() *helpers.PathSpec { // Easier to reason about in tests. v.Set("disablePathToLower", true) v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") fs := hugofs.NewDefault(v) ps, _ := helpers.NewPathSpec(fs, v) return ps @@ -551,7 +560,7 @@ func newTestSite(t testing.TB, configKeyValues ...interface{}) *Site { cfg.Set(configKeyValues[i].(string), configKeyValues[i+1]) } - d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Fs: fs, Cfg: cfg} + d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Fs: fs, Cfg: cfg} s, err := NewSiteForCfg(d) @@ -593,18 +602,6 @@ func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string) ) } -func newDebugLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - -func newErrorLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - -func newWarningLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) -} - func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 4f5b3fbac..c5c962c16 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -19,6 +19,7 @@ import ( "github.com/gohugoio/hugo/tpl/tplimpl" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/deps" @@ -26,8 +27,6 @@ import ( "io/ioutil" "os" - "github.com/gohugoio/hugo/helpers" - "log" "github.com/gohugoio/hugo/config" @@ -168,15 +167,16 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin assert := require.New(t) fs := hugofs.NewMem(cfg) tp := NewTranslationProvider() - depsCfg := newDepsConfig(tp, cfg, fs) - d, err := deps.New(depsCfg) - assert.NoError(err) for file, content := range test.data { err := afero.WriteFile(fs.Source, filepath.Join("i18n", file), []byte(content), 0755) assert.NoError(err) } + depsCfg := newDepsConfig(tp, cfg, fs) + d, err := deps.New(depsCfg) + assert.NoError(err) + assert.NoError(d.LoadResources()) f := tp.t.Func(test.lang) return f(test.id, test.args) @@ -184,7 +184,7 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin } func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) deps.DepsCfg { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") return deps.DepsCfg{ Language: l, @@ -201,6 +201,10 @@ func TestI18nTranslate(t *testing.T) { v := viper.New() v.SetDefault("defaultContentLanguage", "en") v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") // Test without and with placeholders for _, enablePlaceholders := range []bool{false, true} { diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index fa5664210..8749360b3 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -38,17 +38,8 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) - sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source) - sources := []source.Input{sp.NewFilesystem(dir)} - - themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() - - if err == nil { - sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]} - } - - d.Log.DEBUG.Printf("Load I18n from %q", sources) + sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs) + src := sp.NewFilesystem("") i18nBundle := bundle.New() @@ -58,14 +49,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { } var newLangs []string - for _, currentSource := range sources { - for _, r := range currentSource.Files() { - currentSpec := language.GetPluralSpec(r.BaseFileName()) - if currentSpec == nil { - // This may is a language code not supported by go-i18n, it may be - // Klingon or ... not even a fake language. Make sure it works. - newLangs = append(newLangs, r.BaseFileName()) - } + for _, r := range src.Files() { + currentSpec := language.GetPluralSpec(r.BaseFileName()) + if currentSpec == nil { + // This may is a language code not supported by go-i18n, it may be + // Klingon or ... not even a fake language. Make sure it works. + newLangs = append(newLangs, r.BaseFileName()) } } @@ -73,11 +62,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { language.RegisterPluralSpec(newLangs, en) } - for _, currentSource := range sources { - for _, r := range currentSource.Files() { - if err := addTranslationFile(i18nBundle, r); err != nil { - return err - } + // The source files are ordered so the most important comes first. Since this is a + // last key win situation, we have to reverse the iteration order. + files := src.Files() + for i := len(files) - 1; i >= 0; i-- { + if err := addTranslationFile(i18nBundle, files[i]); err != nil { + return err } } diff --git a/helpers/language.go b/langs/language.go index 731e9b088..6f3e1de64 100644 --- a/helpers/language.go +++ b/langs/language.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,12 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package langs import ( "sort" "strings" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/spf13/cast" ) @@ -73,7 +74,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language { for k, v := range cfg.GetStringMap("params") { params[k] = v } - ToLowerMap(params) + maps.ToLower(params) defaultContentDir := cfg.GetString("contentDir") if defaultContentDir == "" { diff --git a/helpers/language_test.go b/langs/language_test.go index 4c4670321..8783172fb 100644 --- a/helpers/language_test.go +++ b/langs/language_test.go @@ -1,4 +1,4 @@ -// Copyright 2016-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package langs import ( "testing" diff --git a/output/docshelper.go b/output/docshelper.go index 6b7826002..fcf9ae61c 100644 --- a/output/docshelper.go +++ b/output/docshelper.go @@ -68,7 +68,7 @@ func createLayoutExamples() interface{} { {"Taxonomy term in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, HTMLFormat}, } { - l := NewLayoutHandler(example.hasTheme) + l := NewLayoutHandler() layouts, _ := l.For(example.d, example.f) basicExamples = append(basicExamples, Example{ diff --git a/output/layout.go b/output/layout.go index 206293842..f83490d81 100644 --- a/output/layout.go +++ b/output/layout.go @@ -41,8 +41,6 @@ type LayoutDescriptor struct { // LayoutHandler calculates the layout template to use to render a given output type. type LayoutHandler struct { - hasTheme bool - mu sync.RWMutex cache map[layoutCacheKey][]string } @@ -53,8 +51,8 @@ type layoutCacheKey struct { } // NewLayoutHandler creates a new LayoutHandler. -func NewLayoutHandler(hasTheme bool) *LayoutHandler { - return &LayoutHandler{hasTheme: hasTheme, cache: make(map[layoutCacheKey][]string)} +func NewLayoutHandler() *LayoutHandler { + return &LayoutHandler{cache: make(map[layoutCacheKey][]string)} } // For returns a layout for the given LayoutDescriptor and options. @@ -72,30 +70,6 @@ func (l *LayoutHandler) For(d LayoutDescriptor, f Format) ([]string, error) { layouts := resolvePageTemplate(d, f) - if l.hasTheme { - // From Hugo 0.33 we interleave the project/theme templates. This was kind of a fundamental change, but the - // previous behaviour was surprising. - // As an example, an `index.html` in theme for the home page will now win over a `_default/list.html` in the project. - layoutsWithThemeLayouts := []string{} - - // First place all non internal templates - for _, t := range layouts { - if !strings.HasPrefix(t, "_internal/") { - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t) - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, "theme/"+t) - } - } - - // Lastly place internal templates - for _, t := range layouts { - if strings.HasPrefix(t, "_internal/") { - layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t) - } - } - - layouts = layoutsWithThemeLayouts - } - layouts = prependTextPrefixIfNeeded(f, layouts...) layouts = helpers.UniqueStrings(layouts) diff --git a/output/layout_base.go b/output/layout_base.go index 49ae1d64e..31e1194f4 100644 --- a/output/layout_base.go +++ b/output/layout_base.go @@ -40,26 +40,16 @@ type TemplateNames struct { } type TemplateLookupDescriptor struct { - // TemplateDir is the project or theme root of the current template. - // This will be the same as WorkingDir for non-theme templates. - TemplateDir string - // The full path to the site root. WorkingDir string - // Main project layout dir, defaults to "layouts" - LayoutDir string - // The path to the template relative the the base. // I.e. shortcodes/youtube.html RelPath string - // The template name prefix to look for, i.e. "theme". + // The template name prefix to look for. Prefix string - // The theme dir if theme active. - ThemeDir string - // All the output formats in play. This is used to decide if text/template or // html/template. OutputFormats Formats @@ -71,6 +61,7 @@ type TemplateLookupDescriptor struct { func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { name := filepath.ToSlash(d.RelPath) + name = strings.TrimPrefix(name, "/") if d.Prefix != "" { name = strings.Trim(d.Prefix, "/") + "/" + name @@ -78,22 +69,8 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { var ( id TemplateNames - - // This is the path to the actual template in process. This may - // be in the theme's or the project's /layouts. - baseLayoutDir = filepath.Join(d.TemplateDir, d.LayoutDir) - fullPath = filepath.Join(baseLayoutDir, d.RelPath) - - // This is always the project's layout dir. - baseWorkLayoutDir = filepath.Join(d.WorkingDir, d.LayoutDir) - - baseThemeLayoutDir string ) - if d.ThemeDir != "" { - baseThemeLayoutDir = filepath.Join(d.ThemeDir, "layouts") - } - // The filename will have a suffix with an optional type indicator. // Examples: // index.html @@ -119,7 +96,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { filenameNoSuffix := parts[0] - id.OverlayFilename = fullPath + id.OverlayFilename = d.RelPath id.Name = name if isPlainText { @@ -127,7 +104,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { } // Ace and Go templates may have both a base and inner template. - pathDir := filepath.Dir(fullPath) + pathDir := filepath.Dir(d.RelPath) if ext == "amber" || strings.HasSuffix(pathDir, "partials") || strings.HasSuffix(pathDir, "shortcodes") { // No base template support @@ -150,7 +127,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { // This may be a view that shouldn't have base template // Have to look inside it to make sure - needsBase, err := d.ContainsAny(fullPath, innerMarkers) + needsBase, err := d.ContainsAny(d.RelPath, innerMarkers) if err != nil { return id, err } @@ -158,21 +135,14 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { if needsBase { currBaseFilename := fmt.Sprintf("%s-%s", filenameNoSuffix, baseFilename) - templateDir := filepath.Dir(fullPath) - - // Find the base, e.g. "_default". - baseTemplatedDir := strings.TrimPrefix(templateDir, baseLayoutDir) - baseTemplatedDir = strings.TrimPrefix(baseTemplatedDir, helpers.FilePathSeparator) - // Look for base template in the follwing order: // 1. <current-path>/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>. // 2. <current-path>/baseof.<outputFormat>(optional).<suffix> // 3. _default/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>. // 4. _default/baseof.<outputFormat>(optional).<suffix> - // For each of the steps above, it will first look in the project, then, if theme is set, - // in the theme's layouts folder. - // Also note that the <current-path> may be both the project's layout folder and the theme's. - pairsToCheck := createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename) + // + // The filesystem it looks in a a composite of the project and potential theme(s). + pathsToCheck := createPathsToCheck(pathDir, baseFilename, currBaseFilename) // We may have language code and/or "terms" in the template name. We want the most specific, // but need to fall back to the baseof.html or baseof.ace if needed. @@ -183,20 +153,15 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { if len(p1) > 0 && len(p1) == len(p2) { for i := len(p1); i > 0; i-- { v1, v2 := strings.Join(p1[:i], ".")+"."+ext, strings.Join(p2[:i], ".")+"."+ext - pairsToCheck = append(pairsToCheck, createPairsToCheck(baseTemplatedDir, v1, v2)...) + pathsToCheck = append(pathsToCheck, createPathsToCheck(pathDir, v1, v2)...) } } - Loop: - for _, pair := range pairsToCheck { - pathsToCheck := basePathsToCheck(pair, baseLayoutDir, baseWorkLayoutDir, baseThemeLayoutDir) - - for _, pathToCheck := range pathsToCheck { - if ok, err := d.FileExists(pathToCheck); err == nil && ok { - id.MasterFilename = pathToCheck - break Loop - } + for _, p := range pathsToCheck { + if ok, err := d.FileExists(p); err == nil && ok { + id.MasterFilename = p + break } } } @@ -205,29 +170,11 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) { } -func createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) [][]string { - return [][]string{ - {baseTemplatedDir, currBaseFilename}, - {baseTemplatedDir, baseFilename}, - {"_default", currBaseFilename}, - {"_default", baseFilename}, - } -} - -func basePathsToCheck(path []string, layoutDir, workLayoutDir, themeLayoutDir string) []string { - // workLayoutDir will always be the most specific, so start there. - pathsToCheck := []string{filepath.Join((append([]string{workLayoutDir}, path...))...)} - - if layoutDir != "" && layoutDir != workLayoutDir { - pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{layoutDir}, path...))...)) +func createPathsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) []string { + return []string{ + filepath.Join(baseTemplatedDir, currBaseFilename), + filepath.Join(baseTemplatedDir, baseFilename), + filepath.Join("_default", currBaseFilename), + filepath.Join("_default", baseFilename), } - - // May have a theme - if themeLayoutDir != "" && themeLayoutDir != layoutDir { - pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{themeLayoutDir}, path...))...)) - - } - - return pathsToCheck - } diff --git a/output/layout_base_test.go b/output/layout_base_test.go index d7c7fbb90..719407524 100644 --- a/output/layout_base_test.go +++ b/output/layout_base_test.go @@ -25,8 +25,6 @@ func TestLayoutBase(t *testing.T) { var ( workingDir = "/sites/mysite/" - themeDir = "/themes/mytheme/" - layoutBase1 = "layouts" layoutPath1 = "_default/single.html" layoutPathAmp = "_default/single.amp.html" layoutPathJSON = "_default/single.json" @@ -39,108 +37,72 @@ func TestLayoutBase(t *testing.T) { basePathMatchStrings string expect TemplateNames }{ - {"No base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, false, "", + {"No base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, false, "", TemplateNames{ Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", + OverlayFilename: "_default/single.html", }}, - {"Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, true, "", + {"Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, true, "", TemplateNames{ Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.html", + OverlayFilename: "_default/single.html", + MasterFilename: "_default/single-baseof.html", }}, // Issue #3893 - {"Base Lang, Default Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html", + {"Base Lang, Default Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html", TemplateNames{ Name: "_default/list.en.html", - OverlayFilename: "/sites/mysite/layouts/_default/list.en.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", + OverlayFilename: "_default/list.en.html", + MasterFilename: "_default/baseof.html", }}, - {"Base Lang, Lang Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html", + {"Base Lang, Lang Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html", TemplateNames{ Name: "_default/list.en.html", - OverlayFilename: "/sites/mysite/layouts/_default/list.en.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.en.html", + OverlayFilename: "_default/list.en.html", + MasterFilename: "_default/baseof.en.html", }}, // Issue #3856 - {"Base Taxonomy Term", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html", + {"Base Taxonomy Term", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html", TemplateNames{ Name: "taxonomy/tag.terms.html", - OverlayFilename: "/sites/mysite/layouts/taxonomy/tag.terms.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", + OverlayFilename: "taxonomy/tag.terms.html", + MasterFilename: "_default/baseof.html", }}, - {"Base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Template in theme, base in theme", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/themes/mytheme/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Template in theme, base in site", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "/sites/mysite/layouts/_default/baseof.html", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/themes/mytheme/layouts/_default/single.html", - MasterFilename: "/sites/mysite/layouts/_default/baseof.html", - }}, - {"Template in site, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true, - "/themes/mytheme", - TemplateNames{ - Name: "_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/single-baseof.html", - }}, - {"With prefix, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, - ThemeDir: themeDir, Prefix: "someprefix"}, true, - "mytheme/layouts/_default/baseof.html", - TemplateNames{ - Name: "someprefix/_default/single.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.html", - MasterFilename: "/themes/mytheme/layouts/_default/baseof.html", - }}, - {"Partial", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "partials/menu.html"}, true, + {"Partial", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "partials/menu.html"}, true, "mytheme/layouts/_default/baseof.html", TemplateNames{ Name: "partials/menu.html", - OverlayFilename: "/sites/mysite/layouts/partials/menu.html", + OverlayFilename: "partials/menu.html", }}, - {"AMP, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, false, "", + {"AMP, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, false, "", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", + OverlayFilename: "_default/single.amp.html", }}, - {"JSON, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, false, "", + {"JSON, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, false, "", TemplateNames{ Name: "_default/single.json", - OverlayFilename: "/sites/mysite/layouts/_default/single.json", + OverlayFilename: "_default/single.json", }}, - {"AMP with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html", + {"AMP with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.amp.html", + OverlayFilename: "_default/single.amp.html", + MasterFilename: "_default/single-baseof.amp.html", }}, - {"AMP with no AMP base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html", + {"AMP with no AMP base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html", TemplateNames{ Name: "_default/single.amp.html", - OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.html", + OverlayFilename: "_default/single.amp.html", + MasterFilename: "_default/single-baseof.html", }}, - {"JSON with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, true, "single-baseof.json", + {"JSON with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, true, "single-baseof.json", TemplateNames{ Name: "_default/single.json", - OverlayFilename: "/sites/mysite/layouts/_default/single.json", - MasterFilename: "/sites/mysite/layouts/_default/single-baseof.json", + OverlayFilename: "_default/single.json", + MasterFilename: "_default/single-baseof.json", }}, } { t.Run(this.name, func(t *testing.T) { @@ -164,7 +126,6 @@ func TestLayoutBase(t *testing.T) { this.d.OutputFormats = Formats{AMPFormat, HTMLFormat, RSSFormat, JSONFormat} this.d.WorkingDir = filepath.FromSlash(this.d.WorkingDir) - this.d.LayoutDir = filepath.FromSlash(this.d.LayoutDir) this.d.RelPath = filepath.FromSlash(this.d.RelPath) this.d.ContainsAny = needsBase this.d.FileExists = fileExists diff --git a/output/layout_test.go b/output/layout_test.go index 3c7fde41a..4b958e9ff 100644 --- a/output/layout_test.go +++ b/output/layout_test.go @@ -57,62 +57,61 @@ func TestLayout(t *testing.T) { for _, this := range []struct { name string d LayoutDescriptor - hasTheme bool layoutOverride string tp Format expect []string expectCount int }{ - {"Home", LayoutDescriptor{Kind: "home"}, true, "", ampType, - []string{"index.amp.html", "theme/index.amp.html", "home.amp.html", "theme/home.amp.html", "list.amp.html", "theme/list.amp.html", "index.html", "theme/index.html", "home.html", "theme/home.html", "list.html", "theme/list.html", "_default/index.amp.html"}, 24}, - {"Home, HTML", LayoutDescriptor{Kind: "home"}, true, "", htmlFormat, + {"Home", LayoutDescriptor{Kind: "home"}, "", ampType, + []string{"index.amp.html", "home.amp.html", "list.amp.html", "index.html", "home.html", "list.html", "_default/index.amp.html"}, 12}, + {"Home, HTML", LayoutDescriptor{Kind: "home"}, "", htmlFormat, // We will eventually get to index.html. This looks stuttery, but makes the lookup logic easy to understand. - []string{"index.html.html", "theme/index.html.html", "home.html.html"}, 24}, - {"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, true, "", ampType, - []string{"index.fr.amp.html", "theme/index.fr.amp.html"}, - 48}, - {"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, true, "", noExtDelimFormat, - []string{"index.nem", "theme/index.nem", "home.nem", "theme/home.nem", "list.nem"}, 12}, - {"Home, no ext", LayoutDescriptor{Kind: "home"}, true, "", noExt, - []string{"index.nex", "theme/index.nex", "home.nex", "theme/home.nex", "list.nex"}, 12}, - {"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, true, "", noExtDelimFormat, - []string{"_default/single.nem", "theme/_default/single.nem"}, 2}, - {"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", ampType, + []string{"index.html.html", "home.html.html"}, 12}, + {"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, "", ampType, + []string{"index.fr.amp.html"}, + 24}, + {"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, "", noExtDelimFormat, + []string{"index.nem", "home.nem", "list.nem"}, 6}, + {"Home, no ext", LayoutDescriptor{Kind: "home"}, "", noExt, + []string{"index.nex", "home.nex", "list.nex"}, 6}, + {"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, "", noExtDelimFormat, + []string{"_default/single.nem"}, 1}, + {"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", ampType, []string{"sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/sect1.html", "sect1/section.html", "sect1/list.html", "section/sect1.amp.html", "section/section.amp.html"}, 18}, - {"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, false, "", ampType, + {"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, "", ampType, []string{"sect1/mylayout.amp.html", "sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/mylayout.html", "sect1/sect1.html"}, 24}, - {"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", ampType, + {"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", ampType, []string{"taxonomy/tag.amp.html", "taxonomy/taxonomy.amp.html", "taxonomy/list.amp.html", "taxonomy/tag.html", "taxonomy/taxonomy.html"}, 18}, - {"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, false, "", ampType, + {"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, "", ampType, []string{"taxonomy/categories.terms.amp.html", "taxonomy/terms.amp.html", "taxonomy/list.amp.html", "taxonomy/categories.terms.html", "taxonomy/terms.html"}, 18}, - {"Page", LayoutDescriptor{Kind: "page"}, true, "", ampType, - []string{"_default/single.amp.html", "theme/_default/single.amp.html", "_default/single.html", "theme/_default/single.html"}, 4}, - {"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, false, "", ampType, + {"Page", LayoutDescriptor{Kind: "page"}, "", ampType, + []string{"_default/single.amp.html", "_default/single.html"}, 2}, + {"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, "", ampType, []string{"_default/mylayout.amp.html", "_default/single.amp.html", "_default/mylayout.html", "_default/single.html"}, 4}, - {"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, false, "", ampType, + {"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, "", ampType, []string{"myttype/mylayout.amp.html", "myttype/single.amp.html", "myttype/mylayout.html"}, 8}, - {"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, false, "", ampType, + {"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, "", ampType, []string{"myttype/mysubtype/mylayout.amp.html", "myttype/mysubtype/single.amp.html", "myttype/mysubtype/mylayout.html"}, 8}, // RSS - {"RSS Home with theme", LayoutDescriptor{Kind: "home"}, true, "", RSSFormat, - []string{"index.rss.xml", "theme/index.rss.xml", "home.rss.xml", "theme/home.rss.xml", "rss.xml"}, 29}, - {"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", RSSFormat, + {"RSS Home", LayoutDescriptor{Kind: "home"}, "", RSSFormat, + []string{"index.rss.xml", "home.rss.xml", "rss.xml"}, 15}, + {"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", RSSFormat, []string{"sect1/sect1.rss.xml", "sect1/section.rss.xml", "sect1/rss.xml", "sect1/list.rss.xml", "sect1/sect1.xml", "sect1/section.xml"}, 22}, - {"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", RSSFormat, + {"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", RSSFormat, []string{"taxonomy/tag.rss.xml", "taxonomy/taxonomy.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.xml", "taxonomy/taxonomy.xml"}, 22}, - {"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, false, "", RSSFormat, + {"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, "", RSSFormat, []string{"taxonomy/tag.terms.rss.xml", "taxonomy/terms.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.terms.xml"}, 22}, - {"Home plain text", LayoutDescriptor{Kind: "home"}, true, "", JSONFormat, - []string{"_text/index.json.json", "_text/theme/index.json.json", "_text/home.json.json", "_text/theme/home.json.json"}, 24}, - {"Page plain text", LayoutDescriptor{Kind: "page"}, true, "", JSONFormat, - []string{"_text/_default/single.json.json", "_text/theme/_default/single.json.json", "_text/_default/single.json", "_text/theme/_default/single.json"}, 4}, - {"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, true, "", ampType, - []string{"section/shortcodes.amp.html", "theme/section/shortcodes.amp.html"}, 24}, - {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, true, "", ampType, - []string{"section/partials.amp.html", "theme/section/partials.amp.html"}, 24}, + {"Home plain text", LayoutDescriptor{Kind: "home"}, "", JSONFormat, + []string{"_text/index.json.json", "_text/home.json.json"}, 12}, + {"Page plain text", LayoutDescriptor{Kind: "page"}, "", JSONFormat, + []string{"_text/_default/single.json.json", "_text/_default/single.json"}, 2}, + {"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, "", ampType, + []string{"section/shortcodes.amp.html"}, 12}, + {"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, "", ampType, + []string{"section/partials.amp.html"}, 12}, } { t.Run(this.name, func(t *testing.T) { - l := NewLayoutHandler(this.hasTheme) + l := NewLayoutHandler() layouts, err := l.For(this.d, this.tp) @@ -130,11 +129,6 @@ func TestLayout(t *testing.T) { } - if !this.hasTheme { - for _, layout := range layouts { - require.NotContains(t, layout, "theme") - } - } }) } @@ -142,7 +136,7 @@ func TestLayout(t *testing.T) { func BenchmarkLayout(b *testing.B) { descriptor := LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"} - l := NewLayoutHandler(false) + l := NewLayoutHandler() for i := 0; i < b.N; i++ { layouts, err := l.For(descriptor, HTMLFormat) diff --git a/resource/resource.go b/resource/resource.go index 0714805e8..9a3725f8a 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/common/maps" + "github.com/spf13/afero" "github.com/spf13/cast" @@ -282,7 +284,6 @@ func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { if err != nil { return nil, err } - s.GetLayoutDirPath() genImagePath := filepath.FromSlash("_gen/images") @@ -644,7 +645,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) er if found { m := cast.ToStringMap(params) // Needed for case insensitive fetching of params values - helpers.ToLowerMap(m) + maps.ToLower(m) ma.updateParams(m) } } diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index 9b50633bd..360adc038 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -30,6 +30,10 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) * cfg.Set("baseURL", baseURL) cfg.Set("resourceDir", "resources") cfg.Set("contentDir", "content") + cfg.Set("dataDir", "data") + cfg.Set("i18nDir", "i18n") + cfg.Set("layoutDir", "layouts") + cfg.Set("archetypeDir", "archetypes") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -63,8 +67,12 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { } cfg.Set("workingDir", workDir) - cfg.Set("contentDir", filepath.Join(workDir, "content")) cfg.Set("resourceDir", filepath.Join(workDir, "res")) + cfg.Set("contentDir", "content") + cfg.Set("dataDir", "data") + cfg.Set("i18nDir", "i18n") + cfg.Set("layoutDir", "layouts") + cfg.Set("archetypeDir", "archetypes") fs := hugofs.NewFrom(hugofs.Os, cfg) fs.Destination = &afero.MemMapFs{} diff --git a/source/content_directory_test.go b/source/content_directory_test.go index ed00af625..7f050e0da 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -20,7 +20,6 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -52,9 +51,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { } for i, test := range tests { - - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() v.Set("ignoreFiles", test.ignoreFilesRegexpes) fs := hugofs.NewMem(v) ps, err := helpers.NewPathSpec(fs, v) diff --git a/source/dirs.go b/source/dirs.go deleted file mode 100644 index 49a849453..000000000 --- a/source/dirs.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "errors" - "os" - "path/filepath" - "strings" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - jww "github.com/spf13/jwalterweatherman" -) - -// Dirs holds the source directories for a given build. -// In case where there are more than one of a kind, the order matters: -// It will be used to construct a union filesystem, so the right-most directory -// will "win" on duplicates. Typically, the theme version will be the first. -type Dirs struct { - logger *jww.Notepad - pathSpec *helpers.PathSpec - - staticDirs []string - AbsStaticDirs []string - - Language *helpers.Language -} - -// NewDirs creates a new dirs with the given configuration and filesystem. -func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) { - ps, err := helpers.NewPathSpec(fs, cfg) - if err != nil { - return nil, err - } - - var l *helpers.Language - if language, ok := cfg.(*helpers.Language); ok { - l = language - } - - d := &Dirs{Language: l, pathSpec: ps, logger: logger} - - return d, d.init(cfg) - -} - -func (d *Dirs) init(cfg config.Provider) error { - - var ( - statics []string - ) - - if d.pathSpec.Theme() != "" { - statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static")) - } - - _, isLanguage := cfg.(*helpers.Language) - languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages) - - if !isLanguage && !hasLanguages { - return errors.New("missing languagesSorted in config") - } - - if !isLanguage { - // Merge all the static dirs. - for _, l := range languages { - addend, err := d.staticDirsFor(l) - if err != nil { - return err - } - - statics = append(statics, addend...) - } - } else { - addend, err := d.staticDirsFor(cfg) - if err != nil { - return err - } - - statics = append(statics, addend...) - } - - d.staticDirs = removeDuplicatesKeepRight(statics) - d.AbsStaticDirs = make([]string, len(d.staticDirs)) - for i, di := range d.staticDirs { - d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator - } - - return nil -} - -func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) { - var statics []string - ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg) - if err != nil { - return statics, err - } - - statics = append(statics, ps.StaticDirs()...) - - return statics, nil -} - -// CreateStaticFs will create a union filesystem with the static paths configured. -// Any missing directories will be logged as warnings. -func (d *Dirs) CreateStaticFs() (afero.Fs, error) { - var ( - source = d.pathSpec.Fs.Source - absPaths []string - ) - - for _, staticDir := range d.AbsStaticDirs { - if _, err := source.Stat(staticDir); os.IsNotExist(err) { - d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir) - } else { - absPaths = append(absPaths, staticDir) - } - - } - - if len(absPaths) == 0 { - return nil, nil - } - - return d.createOverlayFs(absPaths), nil - -} - -// IsStatic returns whether the given filename is located in one of the static -// source dirs. -func (d *Dirs) IsStatic(filename string) bool { - for _, absPath := range d.AbsStaticDirs { - if strings.HasPrefix(filename, absPath) { - return true - } - } - return false -} - -// MakeStaticPathRelative creates a relative path from the given filename. -// It will return an empty string if the filename is not a member of dirs. -func (d *Dirs) MakeStaticPathRelative(filename string) string { - for _, currentPath := range d.AbsStaticDirs { - if strings.HasPrefix(filename, currentPath) { - return strings.TrimPrefix(filename, currentPath) - } - } - - return "" - -} - -func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs { - source := d.pathSpec.Fs.Source - - if len(absPaths) == 1 { - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) - } - - base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) - overlay := d.createOverlayFs(absPaths[1:]) - - return afero.NewCopyOnWriteFs(base, overlay) -} - -func removeDuplicatesKeepRight(in []string) []string { - seen := make(map[string]bool) - var out []string - for i := len(in) - 1; i >= 0; i-- { - v := in[i] - if seen[v] { - continue - } - out = append([]string{v}, out...) - seen[v] = true - } - - return out -} diff --git a/source/dirs_test.go b/source/dirs_test.go deleted file mode 100644 index 46236120e..000000000 --- a/source/dirs_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package source - -import ( - "testing" - - "github.com/gohugoio/hugo/helpers" - - "fmt" - - "io/ioutil" - "log" - "os" - "path/filepath" - - "github.com/gohugoio/hugo/config" - "github.com/spf13/afero" - - jww "github.com/spf13/jwalterweatherman" - - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" -) - -var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) - -func TestStaticDirs(t *testing.T) { - assert := require.New(t) - - tests := []struct { - setup func(cfg config.Provider, fs *hugofs.Fs) config.Provider - expected []string - }{ - - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - return cfg - }, []string{"s1"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", []string{"s2", "s1", "s2"}) - return cfg - }, []string{"s1", "s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("theme", "mytheme") - cfg.Set("themesDir", "themes") - cfg.Set("staticDir", []string{"s1", "s2"}) - return cfg - }, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"s1", "l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", []string{"s1", "s2"}) - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - return l1 - - }, []string{"s1", "s2", "l1s1", "l1s2"}}, - {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { - cfg.Set("staticDir", "s1") - - l1 := helpers.NewLanguage("en", cfg) - l1.Set("staticDir2", []string{"l1s1", "l1s2"}) - l2 := helpers.NewLanguage("nn", cfg) - l2.Set("staticDir3", []string{"l2s1", "l2s2"}) - l2.Set("staticDir", []string{"l2"}) - - cfg.Set("languagesSorted", helpers.Languages{l1, l2}) - return cfg - - }, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}}, - } - - for i, test := range tests { - msg := fmt.Sprintf("Test %d", i) - v := viper.New() - v.Set("contentDir", "content") - - fs := hugofs.NewMem(v) - cfg := test.setup(v, fs) - cfg.Set("workingDir", filepath.FromSlash("/work")) - _, isLanguage := cfg.(*helpers.Language) - if !isLanguage && !cfg.IsSet("languagesSorted") { - cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)}) - } - dirs, err := NewDirs(fs, cfg, logger) - assert.NoError(err) - assert.Equal(test.expected, dirs.staticDirs, msg) - assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs)) - - for i, d := range dirs.staticDirs { - abs := dirs.AbsStaticDirs[i] - assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs) - assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png"))) - rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png")) - assert.Equal("logo.png", rel) - } - - assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png"))) - - } - -} - -func TestStaticDirsFs(t *testing.T) { - assert := require.New(t) - v := viper.New() - fs := hugofs.NewMem(v) - v.Set("workingDir", filepath.FromSlash("/work")) - v.Set("theme", "mytheme") - v.Set("themesDir", "themes") - v.Set("contentDir", "content") - v.Set("staticDir", []string{"s1", "s2"}) - v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)}) - - writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1") - writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2") - writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2") - writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1") - writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3") - - dirs, err := NewDirs(fs, v, logger) - assert.NoError(err) - - sfs, err := dirs.CreateStaticFs() - assert.NoError(err) - - assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt")) - assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt")) - assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt")) - -} - -func TestRemoveDuplicatesKeepRight(t *testing.T) { - in := []string{"a", "b", "c", "a"} - out := removeDuplicatesKeepRight(in) - - require.Equal(t, []string{"b", "c", "a"}, out) -} - -func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { - if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { - t.Fatalf("Failed to write file: %s", err) - } -} - -func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { - filename = filepath.FromSlash(filename) - b, err := afero.ReadFile(fs, filename) - if err != nil { - afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { - fmt.Println(" ", path, " ", info) - return nil - }) - t.Fatalf("Failed to read file: %s", err) - } - return string(b) -} diff --git a/source/fileInfo.go b/source/fileInfo.go index 9adb96df4..31885bfd4 100644 --- a/source/fileInfo.go +++ b/source/fileInfo.go @@ -220,7 +220,7 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f // Open implements ReadableFile. func (fi *FileInfo) Open() (io.ReadCloser, error) { - f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename()) + f, err := fi.sp.SourceFs.Open(fi.Filename()) return f, err } diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go index ec2a17c65..9d3566240 100644 --- a/source/fileInfo_test.go +++ b/source/fileInfo_test.go @@ -19,8 +19,6 @@ import ( "github.com/gohugoio/hugo/helpers" - "github.com/spf13/viper" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/afero" "github.com/stretchr/testify/require" @@ -72,14 +70,13 @@ func TestFileInfoLanguage(t *testing.T) { m := afero.NewMemMapFs() lfs := hugofs.NewLanguageFs("sv", langs, m) - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() fs := hugofs.NewFrom(m, v) ps, err := helpers.NewPathSpec(fs, v) assert.NoError(err) - s := SourceSpec{Fs: lfs, PathSpec: ps} + s := SourceSpec{SourceFs: lfs, PathSpec: ps} s.Languages = map[string]interface{}{ "en": true, } diff --git a/source/filesystem.go b/source/filesystem.go index 50075e3c4..3f4bf0ff1 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -79,16 +79,13 @@ func (f *Filesystem) captureFiles() { return err } - if f.Fs == nil { + if f.SourceFs == nil { panic("Must have a fs") } - err := helpers.SymbolicWalk(f.Fs, f.Base, walker) + err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker) if err != nil { jww.ERROR.Println(err) - if err == helpers.ErrPathTooShort { - panic("The root path is too short. If this is a test, make sure to init the content paths.") - } } } @@ -100,7 +97,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } - linkfi, err := f.Fs.Stat(link) + linkfi, err := f.SourceFs.Stat(link) if err != nil { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) return false, nil diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 82f02d404..ee86c1487 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" @@ -69,9 +68,19 @@ func TestUnicodeNorm(t *testing.T) { } -func newTestSourceSpec() SourceSpec { +func newTestConfig() *viper.Viper { v := viper.New() v.Set("contentDir", "content") - ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v) - return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps} + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + +func newTestSourceSpec() *SourceSpec { + v := newTestConfig() + fs := hugofs.NewMem(v) + ps, _ := helpers.NewPathSpec(fs, v) + return NewSourceSpec(ps, fs.Source) } diff --git a/source/sourceSpec.go b/source/sourceSpec.go index 634306e5f..144d86ca3 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -18,6 +18,7 @@ import ( "path/filepath" "regexp" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" @@ -29,7 +30,7 @@ import ( type SourceSpec struct { *helpers.PathSpec - Fs afero.Fs + SourceFs afero.Fs // This is set if the ignoreFiles config is set. ignoreFilesRe []*regexp.Regexp @@ -52,7 +53,7 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } if len(languages) == 0 { - l := helpers.NewDefaultLanguage(cfg) + l := langs.NewDefaultLanguage(cfg) languages[l.Lang] = l defaultLang = l.Lang } @@ -71,12 +72,13 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } } - return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + } func (s *SourceSpec) IgnoreFile(filename string) bool { if filename == "" { - if _, ok := s.Fs.(*afero.OsFs); ok { + if _, ok := s.SourceFs.(*afero.OsFs); ok { return true } return false @@ -108,7 +110,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { - fi, err := helpers.LstatIfPossible(s.Fs, filename) + fi, err := helpers.LstatIfPossible(s.SourceFs, filename) if err != nil { return false, err } @@ -119,7 +121,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(filename) - fi, err = helpers.LstatIfPossible(s.Fs, link) + fi, err = helpers.LstatIfPossible(s.SourceFs, link) if err != nil { return false, err } diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index 68e7c59d6..ac2c2fe63 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/assert" @@ -774,7 +775,7 @@ type TstX struct { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 79e9b3907..f6baae18b 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -27,6 +27,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/spf13/viper" "github.com/stretchr/testify/assert" @@ -164,7 +165,7 @@ func TestScpGetRemoteParallel(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { diff --git a/tpl/template.go b/tpl/template.go index bdb917ba9..e04d2cc6c 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -35,7 +35,7 @@ type TemplateHandler interface { TemplateFinder AddTemplate(name, tpl string) error AddLateTemplate(name, tpl string) error - LoadTemplates(absPath, prefix string) + LoadTemplates(prefix string) PrintErrors() MarkReady() diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index 8f91113a8..e838ebc57 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -86,6 +86,10 @@ type templateHandler struct { errors []*templateErr + // This is the filesystem to load the templates from. All the templates are + // stored in the root of this filesystem. + layoutsFs afero.Fs + *deps.Deps } @@ -129,10 +133,11 @@ func (t *templateHandler) Lookup(name string) *tpl.TemplateAdapter { func (t *templateHandler) clone(d *deps.Deps) *templateHandler { c := &templateHandler{ - Deps: d, - html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, - text: &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)}, - errors: make([]*templateErr, 0), + Deps: d, + layoutsFs: d.BaseFs.Layouts.Fs, + html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, + text: &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)}, + errors: make([]*templateErr, 0), } d.Tmpl = c @@ -170,10 +175,11 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { overlays: make(map[string]*texttemplate.Template), } return &templateHandler{ - Deps: deps, - html: htmlT, - text: textT, - errors: make([]*templateErr, 0), + Deps: deps, + layoutsFs: deps.BaseFs.Layouts.Fs, + html: htmlT, + text: textT, + errors: make([]*templateErr, 0), } } @@ -208,15 +214,18 @@ func (t *htmlTemplates) Lookup(name string) *tpl.TemplateAdapter { } func (t *htmlTemplates) lookup(name string) *template.Template { - if templ := t.t.Lookup(name); templ != nil { - return templ - } + + // Need to check in the overlay registry first as it will also be found below. if t.overlays != nil { if templ, ok := t.overlays[name]; ok { return templ } } + if templ := t.t.Lookup(name); templ != nil { + return templ + } + if t.clone != nil { return t.clone.Lookup(name) } @@ -248,15 +257,18 @@ func (t *textTemplates) Lookup(name string) *tpl.TemplateAdapter { } func (t *textTemplates) lookup(name string) *texttemplate.Template { - if templ := t.t.Lookup(name); templ != nil { - return templ - } + + // Need to check in the overlay registry first as it will also be found below. if t.overlays != nil { if templ, ok := t.overlays[name]; ok { return templ } } + if templ := t.t.Lookup(name); templ != nil { + return templ + } + if t.clone != nil { return t.clone.Lookup(name) } @@ -287,11 +299,11 @@ func (t *textTemplates) setFuncs(funcMap map[string]interface{}) { t.t.Funcs(funcMap) } -// LoadTemplates loads the templates, starting from the given absolute path. +// LoadTemplates loads the templates from the layouts filesystem. // A prefix can be given to indicate a template namespace to load the templates // into, i.e. "_internal" etc. -func (t *templateHandler) LoadTemplates(absPath, prefix string) { - t.loadTemplates(absPath, prefix) +func (t *templateHandler) LoadTemplates(prefix string) { + t.loadTemplates(prefix) } @@ -406,85 +418,49 @@ func (t *templateHandler) RebuildClone() { t.text.clone = texttemplate.Must(t.text.cloneClone.Clone()) } -func (t *templateHandler) loadTemplates(absPath string, prefix string) { - t.Log.DEBUG.Printf("Load templates from path %q prefix %q", absPath, prefix) +func (t *templateHandler) loadTemplates(prefix string) { walker := func(path string, fi os.FileInfo, err error) error { - if err != nil { + if err != nil || fi.IsDir() { return nil } - t.Log.DEBUG.Println("Template path", path) - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(absPath) - if err != nil { - t.Log.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", absPath, err) - return nil - } - - linkfi, err := t.Fs.Source.Stat(link) - if err != nil { - t.Log.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) - return nil - } - - if !linkfi.Mode().IsRegular() { - t.Log.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", absPath) - } + if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) { return nil } - if !fi.IsDir() { - if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) { - return nil - } - - var ( - workingDir = t.PathSpec.WorkingDir() - themeDir = t.PathSpec.GetThemeDir() - layoutDir = t.PathSpec.LayoutDir() - ) - - if themeDir != "" && strings.HasPrefix(absPath, themeDir) { - layoutDir = "layouts" - } - - li := strings.LastIndex(path, layoutDir) + len(layoutDir) + 1 - relPath := path[li:] - templateDir := path[:li-len(layoutDir)-1] - - descriptor := output.TemplateLookupDescriptor{ - TemplateDir: templateDir, - WorkingDir: workingDir, - LayoutDir: layoutDir, - RelPath: relPath, - Prefix: prefix, - ThemeDir: themeDir, - OutputFormats: t.OutputFormatsConfig, - FileExists: func(filename string) (bool, error) { - return helpers.Exists(filename, t.Fs.Source) - }, - ContainsAny: func(filename string, subslices [][]byte) (bool, error) { - return helpers.FileContainsAny(filename, subslices, t.Fs.Source) - }, - } - - tplID, err := output.CreateTemplateNames(descriptor) - if err != nil { - t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err) + workingDir := t.PathSpec.WorkingDir + + descriptor := output.TemplateLookupDescriptor{ + WorkingDir: workingDir, + RelPath: path, + Prefix: prefix, + OutputFormats: t.OutputFormatsConfig, + FileExists: func(filename string) (bool, error) { + return helpers.Exists(filename, t.Layouts.Fs) + }, + ContainsAny: func(filename string, subslices [][]byte) (bool, error) { + return helpers.FileContainsAny(filename, subslices, t.Layouts.Fs) + }, + } - return nil - } + tplID, err := output.CreateTemplateNames(descriptor) + if err != nil { + t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err) - if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil { - t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err) - } + return nil + } + if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil { + t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err) } + return nil } - if err := helpers.SymbolicWalk(t.Fs.Source, absPath, walker); err != nil { + + if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil { t.Log.ERROR.Printf("Failed to load templates: %s", err) } + } func (t *templateHandler) initFuncs() { @@ -534,6 +510,7 @@ func (t *templateHandler) handleMaster(name, overlayFilename, masterFilename str } func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { + masterTpl := t.lookup(masterFilename) if masterTpl == nil { @@ -565,6 +542,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin if err := applyTemplateTransformersToHMLTTemplate(overlayTpl); err != nil { return err } + t.overlays[name] = overlayTpl return err @@ -572,6 +550,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin } func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { + name = strings.TrimPrefix(name, textTmplNamePrefix) masterTpl := t.lookup(masterFilename) @@ -610,12 +589,16 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) error { t.checkState() + t.Log.DEBUG.Printf("Add template file: name %q, baseTemplatePath %q, path %q", name, baseTemplatePath, path) + getTemplate := func(filename string) (string, error) { - b, err := afero.ReadFile(t.Fs.Source, filename) + b, err := afero.ReadFile(t.Layouts.Fs, filename) if err != nil { return "", err } - return string(b), nil + s := string(b) + + return s, nil } // get the suffix and switch on that @@ -625,7 +608,7 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e // Only HTML support for Amber withoutExt := strings.TrimSuffix(name, filepath.Ext(name)) templateName := withoutExt + ".html" - b, err := afero.ReadFile(t.Fs.Source, path) + b, err := afero.ReadFile(t.Layouts.Fs, path) if err != nil { return err @@ -654,14 +637,14 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e case ".ace": // Only HTML support for Ace var innerContent, baseContent []byte - innerContent, err := afero.ReadFile(t.Fs.Source, path) + innerContent, err := afero.ReadFile(t.Layouts.Fs, path) if err != nil { return err } if baseTemplatePath != "" { - baseContent, err = afero.ReadFile(t.Fs.Source, baseTemplatePath) + baseContent, err = afero.ReadFile(t.Layouts.Fs, baseTemplatePath) if err != nil { return err } @@ -680,8 +663,6 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e return err } - t.Log.DEBUG.Printf("Add template file from path %s", path) - return t.AddTemplate(name, templ) } } diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 72842d308..a1745282d 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -30,6 +30,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/i18n" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/internal" "github.com/gohugoio/hugo/tpl/partials" @@ -43,9 +44,18 @@ var ( logger = jww.NewNotepad(jww.LevelFatal, jww.LevelFatal, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) ) +func newTestConfig() config.Provider { + v := viper.New() + v.Set("contentDir", "content") + v.Set("dataDir", "data") + v.Set("i18nDir", "i18n") + v.Set("layoutDir", "layouts") + v.Set("archetypeDir", "archetypes") + return v +} + func newDepsConfig(cfg config.Provider) deps.DepsCfg { - l := helpers.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") + l := langs.NewLanguage("en", cfg) return deps.DepsCfg{ Language: l, Cfg: cfg, @@ -61,13 +71,13 @@ func TestTemplateFuncsExamples(t *testing.T) { workingDir := "/home/hugo" - v := viper.New() + v := newTestConfig() v.Set("workingDir", workingDir) v.Set("multilingual", true) v.Set("contentDir", "content") v.Set("baseURL", "http://mysite.com/hugo/") - v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v)) + v.Set("CurrentContentLanguage", langs.NewLanguage("en", v)) fs := hugofs.NewMem(v) @@ -126,8 +136,7 @@ func TestPartialCached(t *testing.T) { var data struct { } - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() config := newDepsConfig(v) diff --git a/tpl/tplimpl/template_test.go b/tpl/tplimpl/template_test.go index 78682e9ab..3ce2a88a2 100644 --- a/tpl/tplimpl/template_test.go +++ b/tpl/tplimpl/template_test.go @@ -18,7 +18,6 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -34,8 +33,7 @@ func TestHTMLEscape(t *testing.T) { "html": "<h1>Hi!</h1>", "other": "<h1>Hi!</h1>", } - v := viper.New() - v.Set("contentDir", "content") + v := newTestConfig() fs := hugofs.NewMem(v) //afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755) diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index ab3beb804..34de4a6fd 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -22,6 +22,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/langs" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -240,7 +241,7 @@ func TestPlainify(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { - l := helpers.NewLanguage("en", cfg) + l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") cs, err := helpers.NewContentSpec(l) if err != nil { |