diff options
| author | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-03 15:58:09 +0300 |
|---|---|---|
| committer | Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> | 2018-10-16 23:10:56 +0300 |
| commit | 35fbfb19a173b01bc881f2bbc5d104136633a7ec (patch) | |
| tree | 636d0d51fa262dc808eb3c5cc9cf92ad977a0c6a | |
| parent | 3a3089121b852332b5744d1f566959c8cf93cef4 (diff) | |
commands: Show server error info in browser
The main item in this commit is showing of errors with a file context when running `hugo server`.
This can be turned off: `hugo server --disableBrowserError` (can also be set in `config.toml`).
But to get there, the error handling in Hugo needed a revision. There are some items left TODO for commits soon to follow, most notable errors in content and config files.
Fixes #5284
Fixes #5290
See #5325
See #5324
73 files changed, 1886 insertions, 640 deletions
diff --git a/commands/commandeer.go b/commands/commandeer.go index c55806980..2b76462fe 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -14,6 +14,15 @@ package commands import ( + "bytes" + "errors" + + "github.com/gohugoio/hugo/common/herrors" + + "io/ioutil" + + jww "github.com/spf13/jwalterweatherman" + "os" "path/filepath" "regexp" @@ -21,13 +30,13 @@ import ( "sync" "time" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/spf13/cobra" - "github.com/spf13/afero" - "github.com/gohugoio/hugo/hugolib" + "github.com/spf13/afero" "github.com/bep/debounce" "github.com/gohugoio/hugo/common/types" @@ -46,6 +55,8 @@ type commandeerHugoState struct { type commandeer struct { *commandeerHugoState + logger *loggers.Logger + // Currently only set when in "fast render mode". But it seems to // be fast enough that we could maybe just add it for all server modes. changeDetector *fileChangeDetector @@ -69,9 +80,45 @@ type commandeer struct { serverPorts []int languagesConfigured bool languages langs.Languages + doLiveReload bool + fastRenderMode bool + showErrorInBrowser bool configured bool paused bool + + // Any error from the last build. + buildErr error +} + +func (c *commandeer) errCount() int { + return int(c.logger.ErrorCounter.Count()) +} + +func (c *commandeer) getErrorWithContext() interface{} { + errCount := c.errCount() + + if errCount == 0 { + return nil + } + + m := make(map[string]interface{}) + + m["Error"] = errors.New(removeErrorPrefixFromLog(c.logger.Errors.String())) + m["Version"] = hugoVersionString() + + fe := herrors.UnwrapErrorWithFileContext(c.buildErr) + if fe != nil { + m["File"] = fe + } + + if c.h.verbose { + var b bytes.Buffer + herrors.FprintStackTrace(&b, c.buildErr) + m["StackTrace"] = b.String() + } + + return m } func (c *commandeer) Set(key string, value interface{}) { @@ -105,6 +152,8 @@ func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f fla doWithCommandeer: doWithCommandeer, visitedURLs: types.NewEvictingStringQueue(10), debounce: rebuildDebouncer, + // This will be replaced later, but we need something to log to before the configuration is read. + logger: loggers.NewLogger(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, running), } return c, c.loadConfig(mustHaveConfigFile, running) @@ -236,6 +285,11 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { c.languages = l } + // Set some commonly used flags + c.doLiveReload = !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") + c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender") + c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError") + // This is potentially double work, but we need to do this one more time now // that all the languages have been configured. if c.doWithCommandeer != nil { @@ -244,12 +298,13 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error { } } - logger, err := c.createLogger(config) + logger, err := c.createLogger(config, running) if err != nil { return err } cfg.Logger = logger + c.logger = logger createMemFs := config.GetBool("renderToMemory") diff --git a/commands/commands.go b/commands/commands.go index 54eb03b5b..8670d4983 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -14,12 +14,10 @@ package commands import ( - "os" - + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/nitro" ) @@ -242,7 +240,7 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { _ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"}) } -func checkErr(logger *jww.Notepad, err error, s ...string) { +func checkErr(logger *loggers.Logger, err error, s ...string) { if err == nil { return } @@ -255,25 +253,3 @@ func checkErr(logger *jww.Notepad, err error, s ...string) { } logger.ERROR.Println(err) } - -func stopOnErr(logger *jww.Notepad, err error, s ...string) { - if err == nil { - return - } - - defer os.Exit(-1) - - if len(s) == 0 { - newMessage := err.Error() - // Printing an empty string results in a error with - // no message, no bueno. - if newMessage != "" { - logger.CRITICAL.Println(newMessage) - } - } - for _, message := range s { - if message != "" { - logger.CRITICAL.Println(message) - } - } -} diff --git a/commands/convert.go b/commands/convert.go index 8de155e9b..dc6b8fe15 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -14,10 +14,10 @@ package commands import ( - "fmt" "time" src "github.com/gohugoio/hugo/source" + "github.com/pkg/errors" "github.com/gohugoio/hugo/hugolib" @@ -187,7 +187,7 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma } if err = newPage.SaveSourceAs(newFilename); err != nil { - return fmt.Errorf("Failed to save file %q: %s", newFilename, err) + return errors.Wrapf(err, "Failed to save file %q:", newFilename) } return nil diff --git a/commands/hugo.go b/commands/hugo.go index 2e7353d51..6cb2ec012 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -18,16 +18,22 @@ package commands import ( "fmt" "io/ioutil" + "os/signal" "sort" "sync/atomic" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/loggers" + "syscall" "github.com/gohugoio/hugo/hugolib/filesystems" "golang.org/x/sync/errgroup" - "log" "os" "path/filepath" "runtime" @@ -85,7 +91,7 @@ func Execute(args []string) Response { } if err == nil { - errCount := int(jww.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError)) + errCount := int(loggers.GlobalErrorCounter.Count()) if errCount > 0 { err = fmt.Errorf("logged %d errors", errCount) } else if resp.Result != nil { @@ -118,7 +124,7 @@ func initializeConfig(mustHaveConfigFile, running bool, } -func (c *commandeer) createLogger(cfg config.Provider) (*jww.Notepad, error) { +func (c *commandeer) createLogger(cfg config.Provider, running bool) (*loggers.Logger, error) { var ( logHandle = ioutil.Discard logThreshold = jww.LevelWarn @@ -161,7 +167,7 @@ func (c *commandeer) createLogger(cfg config.Provider) (*jww.Notepad, error) { jww.SetStdoutThreshold(stdoutThreshold) helpers.InitLoggers() - return jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime), nil + return loggers.NewLogger(stdoutThreshold, logThreshold, outHandle, logHandle, running), nil } func initializeFlags(cmd *cobra.Command, cfg config.Provider) { @@ -275,9 +281,9 @@ func (c *commandeer) fullBuild() error { cnt, err := c.copyStatic() if err != nil { if !os.IsNotExist(err) { - return fmt.Errorf("Error copying static files: %s", err) + return errors.Wrap(err, "Error copying static files") } - c.Logger.WARN.Println("No Static directory found") + c.logger.WARN.Println("No Static directory found") } langCount = cnt langCount = cnt @@ -285,7 +291,7 @@ func (c *commandeer) fullBuild() error { } buildSitesFunc := func() error { if err := c.buildSites(); err != nil { - return fmt.Errorf("Error building site: %s", err) + return errors.Wrap(err, "Error building site") } return nil } @@ -345,8 +351,8 @@ func (c *commandeer) build() error { if err != nil { return err } - c.Logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir"))) - c.Logger.FEEDBACK.Println("Press Ctrl+C to stop") + c.logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir"))) + c.logger.FEEDBACK.Println("Press Ctrl+C to stop") watcher, err := c.newWatcher(watchDirs...) checkErr(c.Logger, err) defer watcher.Close() @@ -388,7 +394,7 @@ func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesy staticFilesystems := c.hugo.BaseFs.SourceFilesystems.Static if len(staticFilesystems) == 0 { - c.Logger.WARN.Println("No static directories found to sync") + c.logger.WARN.Println("No static directories found to sync") return langCount, nil } @@ -448,13 +454,13 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6 syncer.Delete = c.Cfg.GetBool("cleanDestinationDir") if syncer.Delete { - c.Logger.INFO.Println("removing all files from destination that don't exist in static dirs") + c.logger.INFO.Println("removing all files from destination that don't exist in static dirs") syncer.DeleteFilter = func(f os.FileInfo) bool { return f.IsDir() && strings.HasPrefix(f.Name(), ".") } } - c.Logger.INFO.Println("syncing static files to", publishDir) + c.logger.INFO.Println("syncing static files to", publishDir) var err error @@ -480,7 +486,7 @@ func (c *commandeer) timeTrack(start time.Time, name string) { return } elapsed := time.Since(start) - c.Logger.FEEDBACK.Printf("%s in %v ms", name, int(1000*elapsed.Seconds())) + c.logger.FEEDBACK.Printf("%s in %v ms", name, int(1000*elapsed.Seconds())) } // getDirList provides NewWatcher() with a list of directories to watch for changes. @@ -498,7 +504,7 @@ func (c *commandeer) getDirList() ([]string, error) { return nil } - c.Logger.ERROR.Println("Walker: ", err) + c.logger.ERROR.Println("Walker: ", err) return nil } @@ -511,16 +517,16 @@ func (c *commandeer) getDirList() ([]string, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(path) if err != nil { - c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) + c.logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) return nil } linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link) if err != nil { - c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err) + c.logger.ERROR.Printf("Cannot stat %q: %s", link, err) return nil } if !allowSymbolicDirs && !linkfi.Mode().IsRegular() { - c.Logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path) + c.logger.ERROR.Printf("Symbolic links for directories not supported, skipping %q", path) return nil } @@ -603,7 +609,7 @@ func (c *commandeer) getDirList() ([]string, error) { func (c *commandeer) resetAndBuildSites() (err error) { if !c.h.quiet { - c.Logger.FEEDBACK.Println("Started building sites ...") + c.logger.FEEDBACK.Println("Started building sites ...") } return c.hugo.Build(hugolib.BuildCfg{ResetState: true}) } @@ -615,6 +621,7 @@ func (c *commandeer) buildSites() (err error) { func (c *commandeer) rebuildSites(events []fsnotify.Event) error { defer c.timeTrack(time.Now(), "Total") + c.buildErr = nil visited := c.visitedURLs.PeekAllSet() doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") if doLiveReload && !c.Cfg.GetBool("disableFastRender") { @@ -637,7 +644,7 @@ func (c *commandeer) fullRebuild() { c.commandeerHugoState = &commandeerHugoState{} err := c.loadConfig(true, true) if err != nil { - jww.ERROR.Println("Failed to reload config:", err) + c.logger.ERROR.Println("Failed to reload config:", err) // Set the processing on pause until the state is recovered. c.paused = true } else { @@ -645,8 +652,9 @@ func (c *commandeer) fullRebuild() { } if !c.paused { - if err := c.buildSites(); err != nil { - jww.ERROR.Println(err) + err := c.buildSites() + if err != nil { + c.logger.ERROR.Println(err) } else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { livereload.ForceRefresh() } @@ -680,7 +688,7 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { configSet := make(map[string]bool) for _, configFile := range c.configFiles { - c.Logger.FEEDBACK.Println("Watching for config changes in", configFile) + c.logger.FEEDBACK.Println("Watching for config changes in", configFile) watcher.Add(configFile) configSet[configFile] = true } @@ -689,241 +697,259 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) { for { select { case evs := <-watcher.Events: - for _, ev := range evs { - if configSet[ev.Name] { - if ev.Op&fsnotify.Chmod == fsnotify.Chmod { - continue - } - if ev.Op&fsnotify.Remove == fsnotify.Remove { - for _, configFile := range c.configFiles { - counter := 0 - for watcher.Add(configFile) != nil { - counter++ - if counter >= 100 { - break - } - time.Sleep(100 * time.Millisecond) - } - } - } - // Config file changed. Need full rebuild. - c.fullRebuild() - break - } + c.handleEvents(watcher, staticSyncer, evs, configSet) + if c.showErrorInBrowser && c.errCount() > 0 { + // Need to reload browser to show the error + livereload.ForceRefresh() } - - if c.paused { - // Wait for the server to get into a consistent state before - // we continue with processing. - continue - } - - if len(evs) > 50 { - // This is probably a mass edit of the content dir. - // Schedule a full rebuild for when it slows down. - c.debounce(c.fullRebuild) - continue + case err := <-watcher.Errors: + if err != nil { + c.logger.ERROR.Println("Error while watching:", err) } + } + } + }() - c.Logger.INFO.Println("Received System Events:", evs) + return watcher, nil +} - staticEvents := []fsnotify.Event{} - dynamicEvents := []fsnotify.Event{} +func (c *commandeer) handleEvents(watcher *watcher.Batcher, + staticSyncer *staticSyncer, + evs []fsnotify.Event, + configSet map[string]bool) { - // Special handling for symbolic links inside /content. - filtered := []fsnotify.Event{} - for _, ev := range evs { - // Check the most specific first, i.e. files. - contentMapped := c.hugo.ContentChanges.GetSymbolicLinkMappings(ev.Name) - if len(contentMapped) > 0 { - for _, mapped := range contentMapped { - filtered = append(filtered, fsnotify.Event{Name: mapped, Op: ev.Op}) + for _, ev := range evs { + if configSet[ev.Name] { + if ev.Op&fsnotify.Chmod == fsnotify.Chmod { + continue + } + if ev.Op&fsnotify.Remove == fsnotify.Remove { + for _, configFile := range c.configFiles { + counter := 0 + for watcher.Add(configFile) != nil { + counter++ + if counter >= 100 { + break } - continue + time.Sleep(100 * time.Millisecond) } + } + } + // Config file changed. Need full rebuild. + c.fullRebuild() + break + } + } - // Check for any symbolic directory mapping. + if c.paused { + // Wait for the server to get into a consistent state before + // we continue with processing. + return + } - dir, name := filepath.Split(ev.Name) + if len(evs) > 50 { + // This is probably a mass edit of the content dir. + // Schedule a full rebuild for when it slows down. + c.debounce(c.fullRebuild) + return + } - contentMapped = c.hugo.ContentChanges.GetSymbolicLinkMappings(dir) + c.logger.INFO.Println("Received System Events:", evs) - if len(contentMapped) == 0 { - filtered = append(filtered, ev) - continue - } + staticEvents := []fsnotify.Event{} + dynamicEvents := []fsnotify.Event{} - for _, mapped := range contentMapped { - mappedFilename := filepath.Join(mapped, name) - filtered = append(filtered, fsnotify.Event{Name: mappedFilename, Op: ev.Op}) - } - } + // Special handling for symbolic links inside /content. + filtered := []fsnotify.Event{} + for _, ev := range evs { + // Check the most specific first, i.e. files. + contentMapped := c.hugo.ContentChanges.GetSymbolicLinkMappings(ev.Name) + if len(contentMapped) > 0 { + for _, mapped := range contentMapped { + filtered = append(filtered, fsnotify.Event{Name: mapped, Op: ev.Op}) + } + continue + } - evs = filtered - - for _, ev := range evs { - ext := filepath.Ext(ev.Name) - baseName := filepath.Base(ev.Name) - istemp := strings.HasSuffix(ext, "~") || - (ext == ".swp") || // vim - (ext == ".swx") || // vim - (ext == ".tmp") || // generic temp file - (ext == ".DS_Store") || // OSX Thumbnail - baseName == "4913" || // vim - strings.HasPrefix(ext, ".goutputstream") || // gnome - strings.HasSuffix(ext, "jb_old___") || // intelliJ - strings.HasSuffix(ext, "jb_tmp___") || // intelliJ - strings.HasSuffix(ext, "jb_bak___") || // intelliJ - strings.HasPrefix(ext, ".sb-") || // byword - strings.HasPrefix(baseName, ".#") || // emacs - strings.HasPrefix(baseName, "#") // emacs - if istemp { - continue - } - // Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these - if ev.Name == "" { - continue - } + // Check for any symbolic directory mapping. - // Write and rename operations are often followed by CHMOD. - // There may be valid use cases for rebuilding the site on CHMOD, - // but that will require more complex logic than this simple conditional. - // On OS X this seems to be related to Spotlight, see: - // https://github.com/go-fsnotify/fsnotify/issues/15 - // A workaround is to put your site(s) on the Spotlight exception list, - // but that may be a little mysterious for most end users. - // So, for now, we skip reload on CHMOD. - // We do have to check for WRITE though. On slower laptops a Chmod - // could be aggregated with other important events, and we still want - // to rebuild on those - if ev.Op&(fsnotify.Chmod|fsnotify.Write|fsnotify.Create) == fsnotify.Chmod { - continue - } + dir, name := filepath.Split(ev.Name) - walkAdder := func(path string, f os.FileInfo, err error) error { - if f.IsDir() { - c.Logger.FEEDBACK.Println("adding created directory to watchlist", path) - if err := watcher.Add(path); err != nil { - return err - } - } else if !staticSyncer.isStatic(path) { - // Hugo's rebuilding logic is entirely file based. When you drop a new folder into - // /content on OSX, the above logic will handle future watching of those files, - // but the initial CREATE is lost. - dynamicEvents = append(dynamicEvents, fsnotify.Event{Name: path, Op: fsnotify.Create}) - } - return nil - } + contentMapped = c.hugo.ContentChanges.GetSymbolicLinkMappings(dir) - // recursively add new directories to watch list - // When mkdir -p is used, only the top directory triggers an event (at least on OSX) - if ev.Op&fsnotify.Create == fsnotify.Create { - if s, err := c.Fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() { - _ = helpers.SymbolicWalk(c.Fs.Source, ev.Name, walkAdder) - } - } + if len(contentMapped) == 0 { + filtered = append(filtered, ev) + continue + } - if staticSyncer.isStatic(ev.Name) { - staticEvents = append(staticEvents, ev) - } else { - dynamicEvents = append(dynamicEvents, ev) - } - } + for _, mapped := range contentMapped { + mappedFilename := filepath.Join(mapped, name) + filtered = append(filtered, fsnotify.Event{Name: mappedFilename, Op: ev.Op}) + } + } - if len(staticEvents) > 0 { - c.Logger.FEEDBACK.Println("\nStatic file changes detected") - const layout = "2006-01-02 15:04:05.000 -0700" - c.Logger.FEEDBACK.Println(time.Now().Format(layout)) + evs = filtered + + for _, ev := range evs { + ext := filepath.Ext(ev.Name) + baseName := filepath.Base(ev.Name) + istemp := strings.HasSuffix(ext, "~") || + (ext == ".swp") || // vim + (ext == ".swx") || // vim + (ext == ".tmp") || // generic temp file + (ext == ".DS_Store") || // OSX Thumbnail + baseName == "4913" || // vim + strings.HasPrefix(ext, ".goutputstream") || // gnome + strings.HasSuffix(ext, "jb_old___") || // intelliJ + strings.HasSuffix(ext, "jb_tmp___") || // intelliJ + strings.HasSuffix(ext, "jb_bak___") || // intelliJ + strings.HasPrefix(ext, ".sb-") || // byword + strings.HasPrefix(baseName, ".#") || // emacs + strings.HasPrefix(baseName, "#") // emacs + if istemp { + continue + } + // Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these + if ev.Name == "" { + continue + } - if c.Cfg.GetBool("forceSyncStatic") { - c.Logger.FEEDBACK.Printf("Syncing all static files\n") - _, err := c.copyStatic() - if err != nil { - stopOnErr(c.Logger, err, "Error copying static files to publish dir") - } - } else { - if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil { - c.Logger.ERROR.Println(err) - continue - } - } + // Write and rename operations are often followed by CHMOD. + // There may be valid use cases for rebuilding the site on CHMOD, + // but that will require more complex logic than this simple conditional. + // On OS X this seems to be related to Spotlight, see: + // https://github.com/go-fsnotify/fsnotify/issues/15 + // A workaround is to put your site(s) on the Spotlight exception list, + // but that may be a little mysterious for most end users. + // So, for now, we skip reload on CHMOD. + // We do have to check for WRITE though. On slower laptops a Chmod + // could be aggregated with other important events, and we still want + // to rebuild on those + if ev.Op&(fsnotify.Chmod|fsnotify.Write|fsnotify.Create) == fsnotify.Chmod { + continue + } - if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { - // Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized - - // force refresh when more than one file - if len(staticEvents) == 1 { - ev := staticEvents[0] - path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) - path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false) - livereload.RefreshPath(path) - } else { - livereload.ForceRefresh() - } - } + walkAdder := func(path string, f os.FileInfo, err error) error { + if f.IsDir() { + c.logger.FEEDBACK.Println("adding created directory to watchlist", path) + if err := watcher.Add(path); err != nil { + return err } + } else if !staticSyncer.isStatic(path) { + // Hugo's rebuilding logic is entirely file based. When you drop a new folder into + // /content on OSX, the above logic will handle future watching of those files, + // but the initial CREATE is lost. + dynamicEvents = append(dynamicEvents, fsnotify.Event{Name: path, Op: fsnotify.Create}) + } + return nil + } - if len(dynamicEvents) > 0 { - partitionedEvents := partitionDynamicEvents( - c.firstPathSpec().BaseFs.SourceFilesystems, - dynamicEvents) + // recursively add new directories to watch list + // When mkdir -p is used, only the top directory triggers an event (at least on OSX) + if ev.Op&fsnotify.Create == fsnotify.Create { + if s, err := c.Fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() { + _ = helpers.SymbolicWalk(c.Fs.Source, ev.Name, walkAdder) + } + } - doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") - onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents) + if staticSyncer.isStatic(ev.Name) { + staticEvents = append(staticEvents, ev) + } else { + dynamicEvents = append(dynamicEvents, ev) + } + } - c.Logger.FEEDBACK.Println("\nChange detected, rebuilding site") - const layout = "2006-01-02 15:04:05.000 -0700" - c.Logger.FEEDBACK.Println(time.Now().Format(layout)) + if len(staticEvents) > 0 { + c.logger.FEEDBACK.Println("\nStatic file changes detected") + const layout = "2006-01-02 15:04:05.000 -0700" + c.logger.FEEDBACK.Println(time.Now().Format(layout)) - c.changeDetector.PrepareNew() - if err := c.rebuildSites(dynamicEvents); err != nil { - c.Logger.ERROR.Println("Failed to rebuild site:", err) - } + if c.Cfg.GetBool("forceSyncStatic") { + c.logger.FEEDBACK.Printf("Syncing all static files\n") + _, err := c.copyStatic() + if err != nil { + c.logger.ERROR.Println("Error copying static files to publish dir:", err) + return + } + } else { + if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil { + c.logger.ERROR.Println("Error syncing static files to publish dir:", err) + return + } + } - if doLiveReload { - if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 { - changed := c.changeDetector.changed() - if c.changeDetector != nil && len(changed) == 0 { - // Nothing has changed. - continue - } else if len(changed) == 1 { - pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false) - livereload.RefreshPath(pathToRefresh) - } else { - livereload.ForceRefresh() - } - } + if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { + // Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized + + // force refresh when more than one file + if len(staticEvents) == 1 { + ev := staticEvents[0] + path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) + path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false) + livereload.RefreshPath(path) + } else { + livereload.ForceRefresh() + } + } + } - if len(partitionedEvents.ContentEvents) > 0 { + if len(dynamicEvents) > 0 { + partitionedEvents := partitionDynamicEvents( + c.firstPathSpec().BaseFs.SourceFilesystems, + dynamicEvents) - navigate := c.Cfg.GetBool("navigateToChanged") - // We have fetched the same page above, but it may have - // changed. - var p *hugolib.Page + doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") + onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents) - if navigate { - if onePageName != "" { - p = c.hugo.GetContentPage(onePageName) - } - } + c.logger.FEEDBACK.Println("\nChange detected, rebuilding site") + const layout = "2006-01-02 15:04:05.000 -0700" + c.logger.FEEDBACK.Println(time.Now().Format(layout)) - if p != nil { - livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort()) - } else { - livereload.ForceRefresh() - } - } + c.changeDetector.PrepareNew() + if err := c.rebuildSites(dynamicEvents); err != nil { + c.buildErr = err + c.logger.ERROR.Printf("Rebuild failed: %s", err) + if !c.h.quiet && c.h.verbose { + herrors.PrintStackTrace(err) + } + } + + if doLiveReload { + + if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 { + changed := c.changeDetector.changed() + if c.changeDetector != nil && len(changed) == 0 { + // Nothing has changed. + return + } else if len(changed) == 1 { + pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false) + livereload.RefreshPath(pathToRefresh) + } else { + livereload.ForceRefresh() + } + } + + if len(partitionedEvents.ContentEvents) > 0 { + + navigate := c.Cfg.GetBool("navigateToChanged") + // We have fetched the same page above, but it may have + // changed. + var p *hugolib.Page + + if navigate { + if onePageName != "" { + p = c.hugo.GetContentPage(onePageName) } } - case err := <-watcher.Errors: - if err != nil { - c.Logger.ERROR.Println(err) + + if p != nil { + livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort()) + } else { + livereload.ForceRefresh() } } } - }() - - return watcher, nil + } } // dynamicEvents contains events that is considered dynamic, as in "not static". diff --git a/commands/new_site.go b/commands/new_site.go index 3e4cf4561..2233157ed 100644 --- a/commands/new_site.go +++ b/commands/new_site.go @@ -16,10 +16,11 @@ package commands import ( "bytes" "errors" - "fmt" "path/filepath" "strings" + _errors "github.com/pkg/errors" + "github.com/gohugoio/hugo/create" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -92,7 +93,7 @@ func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error for _, dir := range dirs { if err := fs.Source.MkdirAll(dir, 0777); err != nil { - return fmt.Errorf("Failed to create dir: %s", err) + return _errors.Wrap(err, "Failed to create dir") } } diff --git a/commands/server.go b/commands/server.go index 27999fa6c..ffdbc95c9 100644 --- a/commands/server.go +++ b/commands/server.go @@ -14,6 +14,7 @@ package commands import ( + "bytes" "fmt" "net" "net/http" @@ -21,6 +22,7 @@ import ( "os" "os/signal" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -28,7 +30,10 @@ import ( "syscall" "time" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/livereload" + "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/config" @@ -52,7 +57,8 @@ type serverCmd struct { serverWatch bool noHTTPCache bool - disableFastRender bool + disableFastRender bool + disableBrowserError bool *baseBuilderCmd } @@ -93,6 +99,7 @@ of a second, you will be able to save and see your changes nearly instantly.`, cc.cmd.Flags().BoolVar(&cc.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload") cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "render to Destination path (default is render to memory & serve from there)") cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes") + cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser") cc.cmd.Flags().String("memstats", "", "log memory usage to this file") cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".") @@ -142,6 +149,9 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("disableFastRender") { c.Set("disableFastRender", sc.disableFastRender) } + if cmd.Flags().Changed("disableBrowserError") { + c.Set("disableBrowserError", sc.disableBrowserError) + } if sc.serverWatch { c.Set("watch", true) } @@ -176,7 +186,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { // port set explicitly by user -- he/she probably meant it! err = newSystemErrorF("Server startup failed: %s", err) } - jww.ERROR.Println("port", sc.serverPort, "already in use, attempting to use an available port") + c.logger.FEEDBACK.Println("port", sc.serverPort, "already in use, attempting to use an available port") sp, err := helpers.FindAvailablePort() if err != nil { err = newSystemError("Unable to find alternative port to use:", err) @@ -223,7 +233,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { } if err := memStats(); err != nil { - jww.ERROR.Println("memstats error:", err) + jww.WARN.Println("memstats error:", err) } c, err := initializeConfig(true, true, &sc.hugoBuilderCommon, sc, cfgInit) @@ -271,10 +281,11 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { } type fileServer struct { - baseURLs []string - roots []string - c *commandeer - s *serverCmd + baseURLs []string + roots []string + errorTemplate tpl.Template + c *commandeer + s *serverCmd } func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, error) { @@ -301,27 +312,40 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro httpFs := afero.NewHttpFs(f.c.destinationFs) fs := filesOnlyFs{httpFs.Dir(absPublishDir)} - doLiveReload := !f.s.buildWatch && !f.c.Cfg.GetBool("disableLiveReload") - fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender") - - if i == 0 && fastRenderMode { + if i == 0 && f.c.fastRenderMode { jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender") } // We're only interested in the path u, err := url.Parse(baseURL) if err != nil { - return nil, "", "", fmt.Errorf("Invalid baseURL: %s", err) + return nil, "", "", errors.Wrap(err, "Invalid baseURL") } decorate := func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if f.c.showErrorInBrowser { + // First check the error state + err := f.c.getErrorWithContext() + if err != nil { + w.WriteHeader(500) + var b bytes.Buffer + err := f.errorTemplate.Execute(&b, err) + if err != nil { + f.c.logger.ERROR.Println(err) + } + fmt.Fprint(w, injectLiveReloadScript(&b, f.c.Cfg.GetInt("liveReloadPort"))) + + return + } + } + if f.s.noHTTPCache { w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0") w.Header().Set("Pragma", "no-cache") } - if fastRenderMode { + if f.c.fastRenderMode { p := r.RequestURI if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") { f.c.visitedURLs.Add(p) @@ -345,6 +369,11 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro return mu, u.String(), endpoint, nil } +var logErrorRe = regexp.MustCompile("(?s)ERROR \\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} ") + +func removeErrorPrefixFromLog(content string) string { + return logErrorRe.ReplaceAllLiteralString(content, "") +} func (c *commandeer) serve(s *serverCmd) error { isMultiHost := c.hugo.IsMultihost() @@ -365,11 +394,17 @@ func (c *commandeer) serve(s *serverCmd) error { roots = []string{""} } + templ, err := c.hugo.TextTmpl.Parse("__default_server_error", buildErrorTemplate) + if err != nil { + return err + } + srv := &fileServer{ - baseURLs: baseURLs, - roots: roots, - c: c, - s: s, + baseURLs: baseURLs, + roots: roots, + c: c, + s: s, + errorTemplate: templ, } doLiveReload := !c.Cfg.GetBool("disableLiveReload") @@ -392,7 +427,7 @@ func (c *commandeer) serve(s *serverCmd) error { go func() { err = http.ListenAndServe(endpoint, mu) if err != nil { - jww.ERROR.Printf("Error: %s\n", err.Error()) + c.logger.ERROR.Printf("Error: %s\n", err.Error()) os.Exit(1) } }() @@ -453,7 +488,7 @@ func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, er if strings.Contains(u.Host, ":") { u.Host, _, err = net.SplitHostPort(u.Host) if err != nil { - return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err) + return "", errors.Wrap(err, "Failed to split baseURL hostpost") } } u.Host += fmt.Sprintf(":%d", port) diff --git a/commands/server_errors.go b/commands/server_errors.go new file mode 100644 index 000000000..1a469dac8 --- /dev/null +++ b/commands/server_errors.go @@ -0,0 +1,95 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "bytes" + "io" + + "github.com/gohugoio/hugo/transform" + "github.com/gohugoio/hugo/transform/livereloadinject" +) + +var buildErrorTemplate = `<!doctype html> +<html class="no-js" lang=""> + <head> + <meta charset="utf-8"> + <title>Hugo Server: Error</title> + <style type="text/css"> + body { + font-family: "Muli",avenir, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; + font-size: 16px; + background-color: black; + color: rgba(255, 255, 255, 0.9); + } + main { + margin: auto; + width: 95%; + padding: 1rem; + } + .version { + color: #ccc; + padding: 1rem 0; + } + .stack { + margin-top: 6rem; + } + pre { + white-space: pre-wrap; + white-space: -moz-pre-wrap; + white-space: -pre-wrap; + white-space: -o-pre-wrap; + word-wrap: break-word; + } + .highlight { + overflow-x: scroll; + padding: 0.75rem; + margin-bottom: 1rem; + background-color: #272822; + border: 1px solid black; + } + a { + color: #0594cb; + text-decoration: none; + } + a:hover { + color: #ccc; + } + </style> + </head> + <body> + <main> + {{ highlight .Error "apl" "noclasses=true,style=monokai" }} + {{ with .File }} + {{ $params := printf "noclasses=true,style=monokai,linenos=table,hl_lines=%d,linenostart=%d" (add .Pos 1) .LineNumber }} + {{ $lexer := .ChromaLexer | default "go-html-template" }} + {{ highlight (delimit .Lines "\n") $lexer $params }} + {{ end }} + {{ with .StackTrace }} + {{ highlight . "apl" "noclasses=true,style=monokai" }} + {{ end }} + <p class="version">{{ .Version }}</p> + <a href="">Reload Page</a> + </main> +</body> +</html> +` + +func injectLiveReloadScript(src io.Reader, port int) string { + var b bytes.Buffer + chain := transform.Chain{livereloadinject.New(port)} + chain.Apply(&b, src) + + return b.String() +} diff --git a/commands/server_test.go b/commands/server_test.go index 72d81d70d..438837a90 100644 --- a/commands/server_test.go +++ b/commands/server_test.go @@ -18,6 +18,7 @@ import ( "net/http" "os" "runtime" + "strings" "testing" "time" @@ -113,6 +114,18 @@ func TestFixURL(t *testing.T) { } } +func TestRemoveErrorPrefixFromLog(t *testing.T) { + assert := require.New(t) + content := `ERROR 2018/10/07 13:11:12 Error while rendering "home": template: _default/baseof.html:4:3: executing "main" at <partial "logo" .>: error calling partial: template: partials/logo.html:5:84: executing "partials/logo.html" at <$resized.AHeight>: can't evaluate field AHeight in type *resource.Image +ERROR 2018/10/07 13:11:12 Rebuild failed: logged 1 error(s) +` + + withoutError := removeErrorPrefixFromLog(content) + + assert.False(strings.Contains(withoutError, "ERROR"), withoutError) + +} + func isWindowsCI() bool { return runtime.GOOS == "windows" && os.Getenv("CI") != "" } diff --git a/commands/static_syncer.go b/commands/static_syncer.go index 1e73e7fc2..237453868 100644 --- a/commands/static_syncer.go +++ b/commands/static_syncer.go @@ -105,10 +105,10 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { logger.Println("Syncing", relPath, "to", publishDir) if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { - c.Logger.ERROR.Println(err) + c.logger.ERROR.Println(err) } } else { - c.Logger.ERROR.Println(err) + c.logger.ERROR.Println(err) } continue @@ -117,7 +117,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { // For all other event operations Hugo will sync static. logger.Println("Syncing", relPath, "to", publishDir) if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { - c.Logger.ERROR.Println(err) + c.logger.ERROR.Println(err) } } diff --git a/commands/version.go b/commands/version.go index ea4e4c926..b85f53725 100644 --- a/commands/version.go +++ b/commands/version.go @@ -14,14 +14,16 @@ package commands import ( + "fmt" "runtime" "strings" + jww "github.com/spf13/jwalterweatherman" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugolib" "github.com/gohugoio/hugo/resource/tocss/scss" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" ) var _ cmder = (*versionCmd)(nil) @@ -45,6 +47,10 @@ func newVersionCmd() *versionCmd { } func printHugoVersion() { + jww.FEEDBACK.Println(hugoVersionString()) +} + +func hugoVersionString() string { program := "Hugo Static Site Generator" version := "v" + helpers.CurrentHugoVersion.String() @@ -64,5 +70,6 @@ func printHugoVersion() { buildDate = "unknown" } - jww.FEEDBACK.Println(program, version, osArch, "BuildDate:", buildDate) + return fmt.Sprintf("%s %s %s BuildDate: %s", program, version, osArch, buildDate) + } diff --git a/common/herrors/error_locator.go b/common/herrors/error_locator.go new file mode 100644 index 000000000..cc41e8868 --- /dev/null +++ b/common/herrors/error_locator.go @@ -0,0 +1,194 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors contains common Hugo errors and error related utilities. +package herrors + +import ( + "bufio" + "io" + "strings" + + "github.com/spf13/afero" +) + +// LineMatcher is used to match a line with an error. +type LineMatcher func(le FileError, lineNumber int, line string) bool + +// SimpleLineMatcher matches if the current line number matches the line number +// in the error. +var SimpleLineMatcher = func(le FileError, lineNumber int, line string) bool { + return le.LineNumber() == lineNumber +} + +// ErrorContext contains contextual information about an error. This will +// typically be the lines surrounding some problem in a file. +type ErrorContext struct { + + // If a match will contain the matched line and up to 2 lines before and after. + // Will be empty if no match. + Lines []string + + // The position of the error in the Lines above. 0 based. + Pos int + + // The linenumber in the source file from where the Lines start. Starting at 1. + LineNumber int + + // The lexer to use for syntax highlighting. + // https://gohugo.io/content-management/syntax-highlighting/#list-of-chroma-highlighting-languages + ChromaLexer string +} + +var _ causer = (*ErrorWithFileContext)(nil) + +// ErrorWithFileContext is an error with some additional file context related +// to that error. +type ErrorWithFileContext struct { + cause error + ErrorContext +} + +func (e *ErrorWithFileContext) Error() string { + return e.cause.Error() +} + +func (e *ErrorWithFileContext) Cause() error { + return e.cause +} + +// WithFileContextForFile will try to add a file context with lines matching the given matcher. +// If no match could be found, the original error is returned with false as the second return value. +func WithFileContextForFile(e error, filename string, fs afero.Fs, chromaLexer string, matcher LineMatcher) (error, bool) { + f, err := fs.Open(filename) + if err != nil { + return e, false + } + defer f.Close() + return WithFileContext(e, f, chromaLexer, matcher) +} + +// WithFileContextForFile will try to add a file context with lines matching the given matcher. +// If no match could be found, the original error is returned with false as the second return value. +func WithFileContext(e error, r io.Reader, chromaLexer string, matcher LineMatcher) (error, bool) { + if e == nil { + panic("error missing") + } + le := UnwrapFileError(e) + if le == nil { + var ok bool + if le, ok = ToFileError("bash", e).(FileError); !ok { + return e, false + } + } + + errCtx := locateError(r, le, matcher) + + if errCtx.LineNumber == -1 { + return e, false + } + + if chromaLexer != "" { + errCtx.ChromaLexer = chromaLexer + } else { + errCtx.ChromaLexer = chromaLexerFromType(le.Type()) + } + + return &ErrorWithFileContext{cause: e, ErrorContext: errCtx}, true +} + +// UnwrapErrorWithFileContext tries to unwrap an ErrorWithFileContext from err. +// It returns nil if this is not possible. +func UnwrapErrorWithFileContext(err error) *ErrorWithFileContext { + for err != nil { + switch v := err.(type) { + case *ErrorWithFileContext: + return v + case causer: + err = v.Cause() + default: + return nil + } + } + return nil +} + +func chromaLexerFromType(fileType string) string { + return fileType +} + +func locateErrorInString(le FileError, src string, matcher LineMatcher) ErrorContext { + return locateError(strings.NewReader(src), nil, matcher) +} + +func locateError(r io.Reader, le FileError, matches LineMatcher) ErrorContext { + var errCtx ErrorContext + s := bufio.NewScanner(r) + + lineNo := 0 + + var buff [6]string + i := 0 + errCtx.Pos = -1 + + for s.Scan() { + lineNo++ + txt := s.Text() + buff[i] = txt + + if errCtx.Pos != -1 && i >= 5 { + break + } + + if errCtx.Pos == -1 && matches(le, lineNo, txt) { + errCtx.Pos = i + errCtx.LineNumber = lineNo - i + } + + if errCtx.Pos == -1 && i == 2 { + // Shift left + buff[0], buff[1] = buff[i-1], buff[i] + } else { + i++ + } + } + + // Go's template parser will typically report "unexpected EOF" errors on the + // empty last line that is supressed by the scanner. + // Do an explicit check for that. + if errCtx.Pos == -1 { + lineNo++ + if matches(le, lineNo, "") { + buff[i] = "" + errCtx.Pos = i + errCtx.LineNumber = lineNo - 1 + + i++ + } + } + + if errCtx.Pos != -1 { + low := errCtx.Pos - 2 + if low < 0 { + low = 0 + } + high := i + errCtx.Lines = buff[low:high] + + } else { + errCtx.Pos = -1 + errCtx.LineNumber = -1 + } + + return errCtx +} diff --git a/common/herrors/error_locator_test.go b/common/herrors/error_locator_test.go new file mode 100644 index 000000000..6c879727e --- /dev/null +++ b/common/herrors/error_locator_test.go @@ -0,0 +1,112 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors contains common Hugo errors and error related utilities. +package herrors + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestErrorLocator(t *testing.T) { + assert := require.New(t) + + lineMatcher := func(le FileError, lineno int, line string) bool { + return strings.Contains(line, "THEONE") + } + + lines := `LINE 1 +LINE 2 +LINE 3 +LINE 4 +This is THEONE +LINE 6 +LINE 7 +LINE 8 +` + + location := locateErrorInString(nil, lines, lineMatcher) + assert.Equal([]string{"LINE 3", "LINE 4", "This is THEONE", "LINE 6", "LINE 7"}, location.Lines) + + assert.Equal(3, location.LineNumber) + assert.Equal(2, location.Pos) + + assert.Equal([]string{"This is THEONE"}, locateErrorInString(nil, `This is THEONE`, lineMatcher).Lines) + + location = locateErrorInString(nil, `L1 +This is THEONE +L2 +`, lineMatcher) + assert.Equal(1, location.Pos) + assert.Equal([]string{"L1", "This is THEONE", "L2"}, location.Lines) + + location = locateErrorInString(nil, `This is THEONE +L2 +`, lineMatcher) + assert.Equal(0, location.Pos) + assert.Equal([]string{"This is THEONE", "L2"}, location.Lines) + + location = locateErrorInString(nil, `L1 +This THEONE +`, lineMatcher) + assert.Equal([]string{"L1", "This THEONE"}, location.Lines) + assert.Equal(1, location.Pos) + + location = locateErrorInString(nil, `L1 +L2 +This THEONE +`, lineMatcher) + assert.Equal([]string{"L1", "L2", "This THEONE"}, location.Lines) + assert.Equal(2, location.Pos) + + location = locateErrorInString(nil, "NO MATCH", lineMatcher) + assert.Equal(-1, location.LineNumber) + assert.Equal(-1, location.Pos) + assert.Equal(0, len(location.Lines)) + + lineMatcher = func(le FileError, lineno int, line string) bool { + return lineno == 6 + } + location = locateErrorInString(nil, `A +B +C +D +E +F +G +H +I +J`, lineMatcher) + + assert.Equal([]string{"D", "E", "F", "G", "H"}, location.Lines) + assert.Equal(4, location.LineNumber) + assert.Equal(2, location.Pos) + + // Test match EOF + lineMatcher = func(le FileError, lineno int, line string) bool { + return lineno == 4 + } + + location = locateErrorInString(nil, `A +B +C +`, lineMatcher) + + assert.Equal([]string{"B", "C", ""}, location.Lines) + assert.Equal(3, location.LineNumber) + assert.Equal(2, location.Pos) + +} diff --git a/common/errors/errors.go b/common/herrors/errors.go index 673cd23ff..fe92c5467 100644 --- a/common/errors/errors.go +++ b/common/herrors/errors.go @@ -11,13 +11,41 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package errors contains common Hugo errors and error related utilities. -package errors +// Package herrors contains common Hugo errors and error related utilities. +package herrors import ( "errors" + "fmt" + "io" + "os" + + _errors "github.com/pkg/errors" ) +// As defined in https://godoc.org/github.com/pkg/errors +type causer interface { + Cause() error +} + +type stackTracer interface { + StackTrace() _errors.StackTrace +} + +// PrintStackTrace prints the error's stack trace to stdoud. +func PrintStackTrace(err error) { + FprintStackTrace(os.Stdout, err) +} + +// FprintStackTrace prints the error's stack trace to w. +func FprintStackTrace(w io.Writer, err error) { + if err, ok := err.(stackTracer); ok { + for _, f := range err.StackTrace() { + fmt.Fprintf(w, "%+s:%d\n", f, f) + } + } +} + // ErrFeatureNotAvailable denotes that a feature is unavailable. // // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, diff --git a/common/herrors/file_error.go b/common/herrors/file_error.go new file mode 100644 index 000000000..f29f91fcc --- /dev/null +++ b/common/herrors/file_error.go @@ -0,0 +1,111 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitatio ns under the License. + +package herrors + +import ( + "fmt" +) + +var _ causer = (*fileError)(nil) + +// FileError represents an error when handling a file: Parsing a config file, +// execute a template etc. +type FileError interface { + error + + // LineNumber gets the error location, starting at line 1. + LineNumber() int + + // A string identifying the type of file, e.g. JSON, TOML, markdown etc. + Type() string +} + +var _ FileError = (*fileError)(nil) + +type fileError struct { + lineNumber int + fileType string + msg string + + cause error +} + +func (e *fileError) LineNumber() int { + return e.lineNumber +} + +func (e *fileError) Type() string { + return e.fileType +} + +func (e *fileError) Error() string { + return e.msg +} + +func (f *fileError) Cause() error { + return f.cause +} + +func (e *fileError) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + fallthrough + case 's': + fmt.Fprintf(s, "%s:%d: %s:%s", e.fileType, e.lineNumber, e.msg, e.cause) + case 'q': + fmt.Fprintf(s, "%q:%d: %q:%q", e.fileType, e.lineNumber, e.msg, e.cause) + } +} + +// NewFileError creates a new FileError. +func NewFileError(fileType string, lineNumber int, msg string, err error) FileError { + return &fileError{cause: err, fileType: fileType, lineNumber: lineNumber, msg: msg} +} + +// UnwrapFileError tries to unwrap a FileError from err. +// It returns nil if this is not possible. +func UnwrapFileError(err error) FileError { + for err != nil { + switch v := err.(type) { + case FileError: + return v + case causer: + err = v.Cause() + default: + return nil + } + } + return nil +} + +// ToFileError will try to convert the given error to an error supporting +// the FileError interface. +// If will fall back to returning the original error if a line number cannot be extracted. +func ToFileError(fileType string, err error) error { + return ToFileErrorWithOffset(fileType, err, 0) +} + +// ToFileErrorWithOffset will try to convert the given error to an error supporting +// the FileError interface. It will take any line number offset given into account. +// If will fall back to returning the original error if a line number cannot be extracted. +func ToFileErrorWithOffset(fileType string, err error, offset int) error { + for _, handle := range lineNumberExtractors { + lno, msg := handle(err, offset) + if lno > 0 { + return NewFileError(fileType, lno, msg, err) + } + } + // Fall back to the original. + return err +} diff --git a/common/herrors/file_error_test.go b/common/herrors/file_error_test.go new file mode 100644 index 000000000..e266ff1dc --- /dev/null +++ b/common/herrors/file_error_test.go @@ -0,0 +1,56 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package herrors + +import ( + "errors" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestToLineNumberError(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + for i, test := range []struct { + in error + offset int + lineNumber int + }{ + {errors.New("no line number for you"), 0, -1}, + {errors.New(`template: _default/single.html:2:15: executing "_default/single.html" at <.Titles>: can't evaluate field`), 0, 2}, + {errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11}, + {errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2}, + {errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32}, + {errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 2, 34}, + } { + + got := ToFileErrorWithOffset("template", test.in, test.offset) + + errMsg := fmt.Sprintf("[%d][%T]", i, got) + le, ok := got.(FileError) + + if test.lineNumber > 0 { + assert.True(ok) + assert.Equal(test.lineNumber, le.LineNumber(), errMsg) + assert.Contains(got.Error(), strconv.Itoa(le.LineNumber())) + } else { + assert.False(ok) + } + } +} diff --git a/common/herrors/line_number_extractors.go b/common/herrors/line_number_extractors.go new file mode 100644 index 000000000..01a7450f9 --- /dev/null +++ b/common/herrors/line_number_extractors.go @@ -0,0 +1,59 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitatio ns under the License. + +package herrors + +import ( + "fmt" + "regexp" + "strconv" +) + +var lineNumberExtractors = []lineNumberExtractor{ + // Template/shortcode parse errors + newLineNumberErrHandlerFromRegexp("(.*?:)(\\d+)(:.*)"), + + // TOML parse errors + newLineNumberErrHandlerFromRegexp("(.*Near line )(\\d+)(\\s.*)"), + + // YAML parse errors + newLineNumberErrHandlerFromRegexp("(line )(\\d+)(:)"), +} + +type lineNumberExtractor func(e error, offset int) (int, string) + +func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor { + re := regexp.MustCompile(expression) + return extractLineNo(re) +} + +func extractLineNo(re *regexp.Regexp) lineNumberExtractor { + return func(e error, offset int) (int, string) { + if e == nil { + panic("no error") + } + s := e.Error() + m := re.FindStringSubmatch(s) + if len(m) == 4 { + i, _ := strconv.Atoi(m[2]) + msg := e.Error() + if offset != 0 { + i = i + offset + msg = re.ReplaceAllString(s, fmt.Sprintf("${1}%d${3}", i)) + } + return i, msg + } + + return -1, "" + } +} diff --git a/common/loggers/loggers.go b/common/loggers/loggers.go index 2f7f36b34..a26cbd8ca 100644 --- a/common/loggers/loggers.go +++ b/common/loggers/loggers.go @@ -14,6 +14,8 @@ package loggers import ( + "bytes" + "io" "io/ioutil" "log" "os" @@ -21,17 +23,78 @@ import ( jww "github.com/spf13/jwalterweatherman" ) +var ( + // Counts ERROR logs to the global jww logger. + GlobalErrorCounter *jww.Counter +) + +func init() { + GlobalErrorCounter = &jww.Counter{} + jww.SetLogListeners(jww.LogCounter(GlobalErrorCounter, jww.LevelError)) +} + +// Logger wraps a *loggers.Logger and some other related logging state. +type Logger struct { + *jww.Notepad + ErrorCounter *jww.Counter + + // This is only set in server mode. + Errors *bytes.Buffer +} + +// Reset resets the logger's internal state. +func (l *Logger) Reset() { + l.ErrorCounter.Reset() + if l.Errors != nil { + l.Errors.Reset() + } +} + +// NewLogger creates a new Logger for the given thresholds +func NewLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger { + return newLogger(stdoutThreshold, logThreshold, outHandle, logHandle, saveErrors) +} + // NewDebugLogger is a convenience function to create a debug logger. -func NewDebugLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +func NewDebugLogger() *Logger { + return newBasicLogger(jww.LevelDebug) } // NewWarningLogger is a convenience function to create a warning logger. -func NewWarningLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +func NewWarningLogger() *Logger { + return newBasicLogger(jww.LevelWarn) } // NewErrorLogger is a convenience function to create an error logger. -func NewErrorLogger() *jww.Notepad { - return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +func NewErrorLogger() *Logger { + return newBasicLogger(jww.LevelError) +} + +func newLogger(stdoutThreshold, logThreshold jww.Threshold, outHandle, logHandle io.Writer, saveErrors bool) *Logger { + errorCounter := &jww.Counter{} + listeners := []jww.LogListener{jww.LogCounter(errorCounter, jww.LevelError)} + var errorBuff *bytes.Buffer + if saveErrors { + errorBuff = new(bytes.Buffer) + errorCapture := func(t jww.Threshold) io.Writer { + if t != jww.LevelError { + // Only interested in ERROR + return nil + } + + return errorBuff + } + + listeners = append(listeners, errorCapture) + } + + return &Logger{ + Notepad: jww.NewNotepad(stdoutThreshold, logThreshold, outHandle, logHandle, "", log.Ldate|log.Ltime, listeners...), + ErrorCounter: errorCounter, + Errors: errorBuff, + } +} + +func newBasicLogger(t jww.Threshold) *Logger { + return newLogger(t, jww.LevelError, os.Stdout, ioutil.Discard, false) } diff --git a/create/content.go b/create/content.go index 00924941f..388f2b4a0 100644 --- a/create/content.go +++ b/create/content.go @@ -16,7 +16,9 @@ package create import ( "bytes" - "fmt" + + "github.com/pkg/errors" + "io" "os" "os/exec" @@ -135,7 +137,7 @@ func newContentFromDir( targetDir := filepath.Dir(targetFilename) if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create target directory for %s: %s", targetDir, err) + return errors.Wrapf(err, "failed to create target directory for %s:", targetDir) } out, err := targetFs.Create(targetFilename) @@ -223,7 +225,7 @@ func mapArcheTypeDir( func usesSiteVar(fs afero.Fs, filename string) (bool, error) { f, err := fs.Open(filename) if err != nil { - return false, fmt.Errorf("failed to open archetype file: %s", err) + return false, errors.Wrap(err, "failed to open archetype file") } defer f.Close() return helpers.ReaderContains(f, []byte(".Site")), nil diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 458b7285c..5a8b4f63c 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -20,6 +20,8 @@ import ( "strings" "time" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/source" @@ -127,14 +129,14 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety templateHandler := s.Deps.Tmpl.(tpl.TemplateHandler) templateName := "_text/" + helpers.Filename(archetypeFilename) if err := templateHandler.AddTemplate(templateName, string(archetypeTemplate)); err != nil { - return nil, fmt.Errorf("Failed to parse archetype file %q: %s", archetypeFilename, err) + return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename) } templ, _ := templateHandler.Lookup(templateName) var buff bytes.Buffer if err := templ.Execute(&buff, data); err != nil { - return nil, fmt.Errorf("Failed to process archetype file %q: %s", archetypeFilename, err) + return nil, errors.Wrapf(err, "Failed to process archetype file %q:", archetypeFilename) } archetypeContent = []byte(archetypeShortcodeReplacementsPost.Replace(buff.String())) diff --git a/deps/deps.go b/deps/deps.go index 2b66a153f..1e2686421 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -16,7 +16,6 @@ import ( "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" - jww "github.com/spf13/jwalterweatherman" ) // Deps holds dependencies used by many. @@ -25,7 +24,7 @@ import ( type Deps struct { // The logger to use. - Log *jww.Notepad `json:"-"` + Log *loggers.Logger `json:"-"` // Used to log errors that may repeat itself many times. DistinctErrorLog *helpers.DistinctLogger @@ -122,10 +121,6 @@ func (d *Deps) LoadResources() error { return err } - if th, ok := d.Tmpl.(tpl.TemplateHandler); ok { - th.PrintErrors() - } - return nil } @@ -256,7 +251,7 @@ func (d Deps) ForLanguage(cfg DepsCfg) (*Deps, error) { type DepsCfg struct { // The Logger to use. - Logger *jww.Notepad + Logger *loggers.Logger // The file systems to use Fs *hugofs.Fs @@ -38,7 +38,7 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nicksnyder/go-i18n v1.10.0 github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pkg/errors v0.8.0 github.com/russross/blackfriday v0.0.0-20180804101149-46c73eb196ba github.com/sanity-io/litter v1.1.0 github.com/sergi/go-diff v1.0.0 // indirect @@ -47,7 +47,7 @@ require ( github.com/spf13/cast v1.2.0 github.com/spf13/cobra v0.0.3 github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05 - github.com/spf13/jwalterweatherman v1.0.0 + github.com/spf13/jwalterweatherman v1.0.1-0.20181005085228-103a6da826d0 github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d github.com/spf13/pflag v1.0.2 github.com/spf13/viper v1.2.0 @@ -60,6 +60,7 @@ require ( golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f + golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect golang.org/x/text v0.3.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v2 v2.2.1 @@ -87,6 +87,8 @@ github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84 h1:fiKJgB4J github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v0.0.0-20180804101149-46c73eb196ba h1:8Vzt8HxRjy7hp1eqPKVoAEPK9npQFW2510qlobGzvi0= @@ -107,6 +109,8 @@ github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05 h1:pQHm7pxjSgC54M1rtLS github.com/spf13/fsync v0.0.0-20170320142552-12a01e648f05/go.mod h1:jdsEoy1w+v0NpuwXZEaRAH6ADTDmzfRnE2eVwshwFrM= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.1-0.20181005085228-103a6da826d0 h1:kPJPXmEs6V1YyXfHFbp1NCpdqhvFVssh2FGx7+OoJLM= +github.com/spf13/jwalterweatherman v1.0.1-0.20181005085228-103a6da826d0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d h1:ihvj2nmx8eqWjlgNgdW6h0DyGJuq5GiwHadJkG0wXtQ= github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d/go.mod h1:jU8A+8xL+6n1OX4XaZtCj4B3mIa64tULUsD6YegdpFo= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= @@ -133,6 +137,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6Zh golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992 h1:BH3eQWeGbwRU2+wxxuuPOdFBmaiBH81O8BugSjHeTFg= golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/helpers/path.go b/helpers/path.go index 4463b8925..f96156dcf 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -25,6 +25,7 @@ import ( "unicode" "github.com/gohugoio/hugo/common/hugio" + _errors "github.com/pkg/errors" "github.com/spf13/afero" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" @@ -493,11 +494,11 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(path) if err != nil { - return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err) + return nil, "", _errors.Wrapf(err, "Cannot read symbolic link %q", path) } fileInfo, err = LstatIfPossible(fs, link) if err != nil { - return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err) + return nil, "", _errors.Wrapf(err, "Cannot stat %q", link) } realPath = link } diff --git a/hugolib/alias.go b/hugolib/alias.go index 73d8acafc..bcf8f1963 100644 --- a/hugolib/alias.go +++ b/hugolib/alias.go @@ -22,12 +22,12 @@ import ( "runtime" "strings" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/tpl" - jww "github.com/spf13/jwalterweatherman" - "github.com/gohugoio/hugo/helpers" ) @@ -47,11 +47,11 @@ func init() { type aliasHandler struct { t tpl.TemplateFinder - log *jww.Notepad + log *loggers.Logger allowRoot bool } -func newAliasHandler(t tpl.TemplateFinder, l *jww.Notepad, allowRoot bool) aliasHandler { +func newAliasHandler(t tpl.TemplateFinder, l *loggers.Logger, allowRoot bool) aliasHandler { return aliasHandler{t, l, allowRoot} } diff --git a/hugolib/config.go b/hugolib/config.go index 3a0e665c1..b21981304 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -17,11 +17,12 @@ import ( "errors" "fmt" - "github.com/gohugoio/hugo/hugolib/paths" - "io" "strings" + "github.com/gohugoio/hugo/hugolib/paths" + _errors "github.com/pkg/errors" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/config" @@ -205,7 +206,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error { } else { languages2, err = toSortedLanguages(cfg, languages) if err != nil { - return fmt.Errorf("Failed to parse multilingual config: %s", err) + return _errors.Wrap(err, "Failed to parse multilingual config") } } diff --git a/hugolib/datafiles_test.go b/hugolib/datafiles_test.go index 8b2dc8c0f..6685de4cc 100644 --- a/hugolib/datafiles_test.go +++ b/hugolib/datafiles_test.go @@ -347,7 +347,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey } }() - s := buildSingleSiteExpected(t, expectBuildError, depsCfg, BuildCfg{SkipRender: true}) + s := buildSingleSiteExpected(t, false, expectBuildError, depsCfg, BuildCfg{SkipRender: true}) if !expectBuildError && !reflect.DeepEqual(expected, s.Data) { // This disabled code detects the situation described in the WARNING message below. diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go index e4af42fd3..86429352d 100644 --- a/hugolib/fileInfo.go +++ b/hugolib/fileInfo.go @@ -54,6 +54,9 @@ func (fi *fileInfo) Lang() string { } func (fi *fileInfo) Filename() string { + if fi == nil || fi.basePather == nil { + return "" + } return fi.basePather.Filename() } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 3ff31ece3..d9eb9f57d 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -21,6 +21,7 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/langs" @@ -29,7 +30,6 @@ import ( "github.com/gohugoio/hugo/i18n" "github.com/gohugoio/hugo/tpl" "github.com/gohugoio/hugo/tpl/tplimpl" - jww "github.com/spf13/jwalterweatherman" ) // HugoSites represents the sites to build. Each site represents a language. @@ -69,7 +69,7 @@ func (h *HugoSites) NumLogErrors() int { if h == nil { return 0 } - return int(h.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError)) + return int(h.Log.ErrorCounter.Count()) } func (h *HugoSites) PrintProcessingStats(w io.Writer) { @@ -250,7 +250,9 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error { return func(templ tpl.TemplateHandler) error { - templ.LoadTemplates("") + if err := templ.LoadTemplates(""); err != nil { + return err + } for _, wt := range withTemplates { if wt == nil { @@ -301,7 +303,8 @@ func (h *HugoSites) reset() { // resetLogs resets the log counters etc. Used to do a new build on the same sites. func (h *HugoSites) resetLogs() { - h.Log.ResetLogCounters() + h.Log.Reset() + loggers.GlobalErrorCounter.Reset() for _, s := range h.Sites { s.Deps.DistinctErrorLog = helpers.NewDistinctLogger(h.Log.ERROR) } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 8ca2128a1..5bb328aa2 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -19,8 +19,6 @@ import ( "errors" - jww "github.com/spf13/jwalterweatherman" - "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" ) @@ -79,7 +77,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { h.Log.FEEDBACK.Println() } - errorCount := h.Log.LogCountForLevel(jww.LevelError) + errorCount := h.Log.ErrorCounter.Count() if errorCount > 0 { return fmt.Errorf("logged %d error(s)", errorCount) } diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go new file mode 100644 index 000000000..074220758 --- /dev/null +++ b/hugolib/hugo_sites_build_errors_test.go @@ -0,0 +1,182 @@ +package hugolib + +import ( + "fmt" + "strings" + "testing" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/stretchr/testify/require" +) + +type testSiteBuildErrorAsserter struct { + name string + assert *require.Assertions +} + +func (t testSiteBuildErrorAsserter) getFileError(err error) *herrors.ErrorWithFileContext { + t.assert.NotNil(err, t.name) + ferr := herrors.UnwrapErrorWithFileContext(err) + t.assert.NotNil(ferr, fmt.Sprintf("[%s] got %T: %+v", t.name, err, err)) + return ferr +} + +func (t testSiteBuildErrorAsserter) assertLineNumber(lineNumber int, err error) { + fe := t.getFileError(err) + t.assert.Equal(lineNumber, fe.LineNumber, fmt.Sprintf("[%s] got => %s", t.name, fe)) +} + +func TestSiteBuildErrors(t *testing.T) { + t.Parallel() + assert := require.New(t) + + const ( + yamlcontent = "yamlcontent" + shortcode = "shortcode" + base = "base" + single = "single" + ) + + // TODO(bep) add content tests after https://github.com/gohugoio/hugo/issues/5324 + // is implemented. + + tests := []struct { + name string + fileType string + fileFixer func(content string) string + assertCreateError func(a testSiteBuildErrorAsserter, err error) + assertBuildError func(a testSiteBuildErrorAsserter, err error) + }{ + + { + name: "Base template parse failed", + fileType: base, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title }}", ".Title }", 1) + }, + assertCreateError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(2, err) + }, + }, + { + name: "Base template execute failed", + fileType: base, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title", ".Titles", 1) + }, + assertBuildError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(2, err) + }, + }, + { + name: "Single template parse failed", + fileType: single, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title }}", ".Title }", 1) + }, + assertCreateError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(3, err) + }, + }, + { + name: "Single template execute failed", + fileType: single, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title", ".Titles", 1) + }, + assertBuildError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(3, err) + }, + }, + { + name: "Shortcode parse failed", + fileType: shortcode, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title }}", ".Title }", 1) + }, + assertCreateError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(2, err) + }, + }, + // TODO(bep) 2errors + /* { + name: "Shortode execute failed", + fileType: shortcode, + fileFixer: func(content string) string { + return strings.Replace(content, ".Title", ".Titles", 1) + }, + assertBuildError: func(a testSiteBuildErrorAsserter, err error) { + a.assertLineNumber(2, err) + }, + },*/ + } + + for _, test := range tests { + + errorAsserter := testSiteBuildErrorAsserter{ + assert: assert, + name: test.name, + } + + b := newTestSitesBuilder(t).WithSimpleConfigFile() + + f := func(fileType, content string) string { + if fileType != test.fileType { + return content + } + return test.fileFixer(content) + + } + + b.WithTemplatesAdded("layouts/shortcodes/sc.html", f(shortcode, `SHORTCODE L1 +SHORTCODE L2 +SHORTCODE L3: +SHORTCODE L4: {{ .Page.Title }} +`)) + b.WithTemplatesAdded("layouts/_default/baseof.html", f(base, `BASEOF L1 +BASEOF L2 +BASEOF L3 +BASEOF L4{{ if .Title }}{{ end }} +{{block "main" .}}This is the main content.{{end}} +BASEOF L6 +`)) + + b.WithTemplatesAdded("layouts/_default/single.html", f(single, `{{ define "main" }} +SINGLE L2: +SINGLE L3: +SINGLE L4: +SINGLE L5: {{ .Title }} {{ .Content }} +{{ end }} +`)) + + b.WithContent("myyaml.md", f(yamlcontent, `--- +title: "The YAML" +--- + +Some content. + +{{< sc >}} + +Some more text. + +The end. + +`)) + + createErr := b.CreateSitesE() + if test.assertCreateError != nil { + test.assertCreateError(errorAsserter, createErr) + } else { + assert.NoError(createErr) + } + + if createErr == nil { + buildErr := b.BuildE(BuildCfg{}) + if test.assertBuildError != nil { + test.assertBuildError(errorAsserter, buildErr) + } else { + assert.NoError(buildErr) + } + } + } +} diff --git a/hugolib/hugo_sites_build_failures_test.go b/hugolib/hugo_sites_build_failures_test.go deleted file mode 100644 index b347490cd..000000000 --- a/hugolib/hugo_sites_build_failures_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package hugolib - -import ( - "fmt" - "testing" -) - -// https://github.com/gohugoio/hugo/issues/4526 -func TestSiteBuildFailureInvalidPageMetadata(t *testing.T) { - t.Parallel() - - validContentFile := ` ---- -title = "This is good" ---- - -Some content. -` - - invalidContentFile := ` ---- -title = "PDF EPUB: Anne Bradstreet: Poems "The Prologue Summary And Analysis EBook Full Text " ---- - -Some content. -` - - var contentFiles []string - for i := 0; i <= 30; i++ { - name := fmt.Sprintf("valid%d.md", i) - contentFiles = append(contentFiles, name, validContentFile) - if i%5 == 0 { - name = fmt.Sprintf("invalid%d.md", i) - contentFiles = append(contentFiles, name, invalidContentFile) - } - } - - b := newTestSitesBuilder(t) - b.WithSimpleConfigFile().WithContent(contentFiles...) - b.CreateSites().BuildFail(BuildCfg{}) - -} diff --git a/hugolib/page.go b/hugolib/page.go index 1fefd945a..8a2864e7f 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -22,6 +22,7 @@ import ( "unicode" "github.com/gohugoio/hugo/media" + _errors "github.com/pkg/errors" "github.com/gohugoio/hugo/common/maps" @@ -307,13 +308,13 @@ func (p *Page) initContent() { err = p.prepareForRender() if err != nil { - p.s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.Path(), err) + c <- err return } if len(p.summary) == 0 { if err = p.setAutoSummary(); err != nil { - err = fmt.Errorf("Failed to set user auto summary for page %q: %s", p.pathOrTitle(), err) + err = _errors.Wrapf(err, "Failed to set user auto summary for page %q:", p.pathOrTitle()) } } c <- err @@ -324,11 +325,11 @@ func (p *Page) initContent() { p.s.Log.WARN.Printf("WARNING: Timed out creating content for page %q (.Content will be empty). This is most likely a circular shortcode content loop that should be fixed. If this is just a shortcode calling a slow remote service, try to set \"timeout=20000\" (or higher, value is in milliseconds) in config.toml.\n", p.pathOrTitle()) case err := <-c: if err != nil { + // TODO(bep) 2errors needs to be transported to the caller. p.s.Log.ERROR.Println(err) } } }) - } // This is sent to the shortcodes for this page. Not doing that will create an infinite regress. So, @@ -989,11 +990,20 @@ func (s *Site) NewPage(name string) (*Page, error) { return p, nil } +func (p *Page) errorf(err error, format string, a ...interface{}) error { + args := append([]interface{}{p.Lang(), p.pathOrTitle()}, a...) + format = "[%s] Page %q: " + format + if err == nil { + return fmt.Errorf(format, args...) + } + return _errors.Wrapf(err, format, args...) +} + func (p *Page) ReadFrom(buf io.Reader) (int64, error) { // Parse for metadata & body if err := p.parse(buf); err != nil { - p.s.Log.ERROR.Printf("%s for %s", err, p.File.Path()) - return 0, err + return 0, p.errorf(err, "parse failed") + } return int64(len(p.rawContent)), nil @@ -1205,7 +1215,7 @@ func (p *Page) initMainOutputFormat() error { pageOutput, err := newPageOutput(p, false, false, outFormat) if err != nil { - return fmt.Errorf("Failed to create output page for type %q for page %q: %s", outFormat.Name, p.pathOrTitle(), err) + return _errors.Wrapf(err, "Failed to create output page for type %q for page %q:", outFormat.Name, p.pathOrTitle()) } p.mainPageOutput = pageOutput @@ -1271,7 +1281,7 @@ func (p *Page) prepareForRender() error { // Note: The shortcodes in a page cannot access the page content it lives in, // hence the withoutContent(). if workContentCopy, err = handleShortcodes(p.withoutContent(), workContentCopy); err != nil { - s.Log.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err) + return err } if p.Markup != "html" { @@ -1294,8 +1304,6 @@ func (p *Page) prepareForRender() error { return nil } -var ErrHasDraftAndPublished = errors.New("both draft and published parameters were found in page's frontmatter") - func (p *Page) update(frontmatter map[string]interface{}) error { if frontmatter == nil { return errors.New("missing frontmatter data") @@ -1512,8 +1520,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error { if draft != nil && published != nil { p.Draft = *draft - p.s.Log.ERROR.Printf("page %s has both draft and published settings in its frontmatter. Using draft.", p.File.Path()) - return ErrHasDraftAndPublished + p.s.Log.WARN.Printf("page %q has both draft and published settings in its frontmatter. Using draft.", p.File.Path()) } else if draft != nil { p.Draft = *draft } else if published != nil { @@ -1751,6 +1758,7 @@ func (p *Page) shouldRenderTo(f output.Format) bool { func (p *Page) parse(reader io.Reader) error { psr, err := parser.ReadFrom(reader) + if err != nil { return err } @@ -1762,7 +1770,7 @@ func (p *Page) parse(reader io.Reader) error { meta, err := psr.Metadata() if err != nil { - return fmt.Errorf("failed to parse page metadata for %q: %s", p.File.Path(), err) + return _errors.Wrap(err, "error in front matter") } if meta == nil { // missing frontmatter equivalent to empty frontmatter @@ -2079,7 +2087,7 @@ func (p *Page) decodeRefArgs(args map[string]interface{}) (refArgs, *SiteInfo, e func (p *Page) Ref(argsm map[string]interface{}) (string, error) { args, s, err := p.decodeRefArgs(argsm) if err != nil { - return "", fmt.Errorf("invalid arguments to Ref: %s", err) + return "", _errors.Wrap(err, "invalid arguments to Ref") } if s == nil { @@ -2099,7 +2107,7 @@ func (p *Page) Ref(argsm map[string]interface{}) (string, error) { func (p *Page) RelRef(argsm map[string]interface{}) (string, error) { args, s, err := p.decodeRefArgs(argsm) if err != nil { - return "", fmt.Errorf("invalid arguments to Ref: %s", err) + return "", _errors.Wrap(err, "invalid arguments to Ref") } if s == nil { @@ -2303,8 +2311,13 @@ func (p *Page) setValuesForKind(s *Site) { // Used in error logs. func (p *Page) pathOrTitle() string { - if p.Path() != "" { - return p.Path() + if p.Filename() != "" { + // Make a path relative to the working dir if possible. + filename := strings.TrimPrefix(p.Filename(), p.s.WorkingDir) + if filename != p.Filename() { + filename = strings.TrimPrefix(filename, helpers.FilePathSeparator) + } + return filename } return p.title } diff --git a/hugolib/page_bundler.go b/hugolib/page_bundler.go index f3aaef826..62ef2b52b 100644 --- a/hugolib/page_bundler.go +++ b/hugolib/page_bundler.go @@ -19,6 +19,8 @@ import ( "math" "runtime" + _errors "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) @@ -145,7 +147,7 @@ func (s *siteContentProcessor) process(ctx context.Context) error { for _, file := range files { f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) if err != nil { - return fmt.Errorf("failed to open assets file: %s", err) + return _errors.Wrap(err, "failed to open assets file") } err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) f.Close() diff --git a/hugolib/page_bundler_capture.go b/hugolib/page_bundler_capture.go index fbfad0103..c152262cc 100644 --- a/hugolib/page_bundler_capture.go +++ b/hugolib/page_bundler_capture.go @@ -20,6 +20,10 @@ import ( "path" "path/filepath" "runtime" + + "github.com/gohugoio/hugo/common/loggers" + _errors "github.com/pkg/errors" + "sort" "strings" "sync" @@ -33,7 +37,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/source" - jww "github.com/spf13/jwalterweatherman" ) var errSkipCyclicDir = errors.New("skip potential cyclic dir") @@ -47,7 +50,7 @@ type capturer struct { sourceSpec *source.SourceSpec fs afero.Fs - logger *jww.Notepad + logger *loggers.Logger // Filenames limits the content to process to a list of filenames/directories. // This is used for partial building in server mode. @@ -61,7 +64,7 @@ type capturer struct { } func newCapturer( - logger *jww.Notepad, + logger *loggers.Logger, sourceSpec *source.SourceSpec, handler captureResultHandler, contentChanges *contentChangeMap, @@ -701,13 +704,13 @@ func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(path) if err != nil { - return fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err) + return _errors.Wrapf(err, "Cannot read symbolic link %q, error was:", path) } // This is a file on the outside of any base fs, so we have to use the os package. sfi, err := os.Stat(link) if err != nil { - return fmt.Errorf("Cannot stat %q, error was: %s", link, err) + return _errors.Wrapf(err, "Cannot stat %q, error was:", link) } // TODO(bep) improve all of this. diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index ace96b633..d6128352c 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -22,8 +22,6 @@ import ( "github.com/gohugoio/hugo/common/loggers" - jww "github.com/spf13/jwalterweatherman" - "runtime" "strings" "sync" @@ -100,9 +98,6 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) { assert.NoError(c.capture()) - // Symlink back to content skipped to prevent infinite recursion. - assert.Equal(uint64(3), logger.LogCountForLevelsGreaterThanorEqualTo(jww.LevelWarn)) - expected := ` F: /base/a/page_s.md diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index cfbec04b7..39de49663 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -132,7 +132,7 @@ func TestPageBundlerSiteRegular(t *testing.T) { assert.Len(pageResources, 2) firstPage := pageResources[0].(*Page) secondPage := pageResources[1].(*Page) - assert.Equal(filepath.FromSlash("b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle()) + assert.Equal(filepath.FromSlash("base/b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle()) assert.Contains(firstPage.content(), "TheContent") assert.Equal(6, len(leafBundle1.Resources)) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 8dedf9cab..33588a201 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1361,23 +1361,6 @@ func TestPagePaths(t *testing.T) { } } -var pageWithDraftAndPublished = `--- -title: broken -published: false -draft: true ---- -some content -` - -func TestDraftAndPublishedFrontMatterError(t *testing.T) { - t.Parallel() - s := newTestSite(t) - _, err := s.NewPageFrom(strings.NewReader(pageWithDraftAndPublished), "content/post/broken.md") - if err != ErrHasDraftAndPublished { - t.Errorf("expected ErrHasDraftAndPublished, was %#v", err) - } -} - var pagesWithPublishedFalse = `--- title: okay published: false diff --git a/hugolib/pagemeta/page_frontmatter.go b/hugolib/pagemeta/page_frontmatter.go index c1139bd90..88f6f3a11 100644 --- a/hugolib/pagemeta/page_frontmatter.go +++ b/hugolib/pagemeta/page_frontmatter.go @@ -14,17 +14,14 @@ package pagemeta import ( - "io/ioutil" - "log" - "os" "strings" "time" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/config" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" ) // FrontMatterHandler maps front matter into Page fields and .Params. @@ -40,7 +37,7 @@ type FrontMatterHandler struct { // A map of all date keys configured, including any custom. allDateKeys map[string]bool - logger *jww.Notepad + logger *loggers.Logger } // FrontMatterDescriptor describes how to handle front matter for a given Page. @@ -263,10 +260,10 @@ func toLowerSlice(in interface{}) []string { // NewFrontmatterHandler creates a new FrontMatterHandler with the given logger and configuration. // If no logger is provided, one will be created. -func NewFrontmatterHandler(logger *jww.Notepad, cfg config.Provider) (FrontMatterHandler, error) { +func NewFrontmatterHandler(logger *loggers.Logger, cfg config.Provider) (FrontMatterHandler, error) { if logger == nil { - logger = jww.NewNotepad(jww.LevelWarn, jww.LevelWarn, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + logger = loggers.NewWarningLogger() } frontMatterConfig, err := newFrontmatterConfig(cfg) diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 625d68ec6..05d5019d2 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -20,6 +20,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/langs" + "github.com/pkg/errors" "github.com/gohugoio/hugo/hugofs" ) @@ -83,13 +84,13 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) { baseURL, err := newBaseURLFromString(baseURLstr) if err != nil { - return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) + return nil, errors.Wrapf(err, "Failed to create baseURL from %q:", baseURLstr) } - contentDir := cfg.GetString("contentDir") - workingDir := cfg.GetString("workingDir") - resourceDir := cfg.GetString("resourceDir") - publishDir := cfg.GetString("publishDir") + contentDir := filepath.Clean(cfg.GetString("contentDir")) + workingDir := filepath.Clean(cfg.GetString("workingDir")) + resourceDir := filepath.Clean(cfg.GetString("resourceDir")) + publishDir := filepath.Clean(cfg.GetString("publishDir")) if contentDir == "" { return nil, fmt.Errorf("contentDir not set") diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index c2fcf1b8d..d0268d8c4 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -21,6 +21,9 @@ import ( "reflect" "regexp" "sort" + + _errors "github.com/pkg/errors" + "strings" "sync" @@ -278,7 +281,7 @@ func prepareShortcodeForPage(placeholder string, sc *shortcode, parent *Shortcod // The most specific template will win. key := newScKeyFromLangAndOutputFormat(lang, f, placeholder) m[key] = func() (string, error) { - return renderShortcode(key, sc, nil, p), nil + return renderShortcode(key, sc, nil, p) } } @@ -289,12 +292,12 @@ func renderShortcode( tmplKey scKey, sc *shortcode, parent *ShortcodeWithPage, - p *PageWithoutContent) string { + p *PageWithoutContent) (string, error) { tmpl := getShortcodeTemplateForTemplateKey(tmplKey, sc.name, p.s.Tmpl) if tmpl == nil { p.s.Log.ERROR.Printf("Unable to locate template for shortcode %q in page %q", sc.name, p.Path()) - return "" + return "", nil } data := &ShortcodeWithPage{Ordinal: sc.ordinal, Params: sc.params, Page: p, Parent: parent} @@ -309,11 +312,15 @@ func renderShortcode( case string: inner += innerData.(string) case *shortcode: - inner += renderShortcode(tmplKey, innerData.(*shortcode), data, p) + s, err := renderShortcode(tmplKey, innerData.(*shortcode), data, p) + if err != nil { + return "", err + } + inner += s default: p.s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", sc.name, p.Path(), reflect.TypeOf(innerData)) - return "" + return "", nil } } @@ -441,7 +448,7 @@ func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) erro render := s.contentShortcodesDelta.getShortcodeRenderer(k) renderedShortcode, err := render() if err != nil { - return fmt.Errorf("Failed to execute shortcode in page %q: %s", p.Path(), err) + return _errors.Wrapf(err, "Failed to execute shortcode in page %q:", p.Path()) } s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode @@ -479,6 +486,16 @@ func (s *shortcodeHandler) extractShortcode(ordinal int, pt *pageTokens, p *Page var cnt = 0 var nestedOrdinal = 0 + // TODO(bep) 2errors revisit after https://github.com/gohugoio/hugo/issues/5324 + msgf := func(i item, format string, args ...interface{}) string { + format = format + ":%d:" + c1 := strings.Count(pt.lexer.input[:i.pos], "\n") + 1 + c2 := bytes.Count(p.frontmatter, []byte{'\n'}) + args = append(args, c1+c2) + return fmt.Sprintf(format, args...) + + } + Loop: for { currItem = pt.next() @@ -524,7 +541,7 @@ Loop: // return that error, more specific continue } - return sc, fmt.Errorf("Shortcode '%s' in page '%s' has no .Inner, yet a closing tag was provided", next.val, p.FullFilePath()) + return sc, errors.New(msgf(next, "shortcode %q has no .Inner, yet a closing tag was provided", next.val)) } if next.typ == tRightDelimScWithMarkup || next.typ == tRightDelimScNoMarkup { // self-closing @@ -542,13 +559,13 @@ Loop: // if more than one. It is "all inner or no inner". tmpl := getShortcodeTemplateForTemplateKey(scKey{}, sc.name, p.s.Tmpl) if tmpl == nil { - return sc, fmt.Errorf("Unable to locate template for shortcode %q in page %q", sc.name, p.Path()) + return sc, errors.New(msgf(currItem, "unable to locate template for shortcode %q", sc.name)) } var err error isInner, err = isInnerShortcode(tmpl.(tpl.TemplateExecutor)) if err != nil { - return sc, fmt.Errorf("Failed to handle template for shortcode %q for page %q: %s", sc.name, p.Path(), err) + return sc, _errors.Wrap(err, msgf(currItem, "failed to handle template for shortcode %q", sc.name)) } case tScParam: @@ -651,8 +668,8 @@ Loop: case tEOF: break Loop case tError: - err := fmt.Errorf("%s:%d: %s", - p.FullFilePath(), (p.lineNumRawContentStart() + pt.lexer.lineNum() - 1), currItem) + err := fmt.Errorf("%s:shortcode:%d: %s", + p.pathOrTitle(), (p.lineNumRawContentStart() + pt.lexer.lineNum() - 1), currItem) currShortcode.err = err return result.String(), err } @@ -750,7 +767,7 @@ func getShortcodeTemplateForTemplateKey(key scKey, shortcodeName string, t tpl.T return nil } -func renderShortcodeWithPage(tmpl tpl.Template, data *ShortcodeWithPage) string { +func renderShortcodeWithPage(tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { buffer := bp.GetBuffer() defer bp.PutBuffer(buffer) @@ -758,7 +775,7 @@ func renderShortcodeWithPage(tmpl tpl.Template, data *ShortcodeWithPage) string err := tmpl.Execute(buffer, data) isInnerShortcodeCache.RUnlock() if err != nil { - data.Page.s.Log.ERROR.Printf("error processing shortcode %q for page %q: %s", tmpl.Name(), data.Page.Path(), err) + return "", data.Page.errorf(err, "failed to process shortcode") } - return buffer.String() + return buffer.String(), nil } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index df7b7103f..3385d31f0 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -24,8 +24,6 @@ import ( "github.com/spf13/viper" - jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/afero" "github.com/gohugoio/hugo/output" @@ -367,11 +365,11 @@ func TestExtractShortcodes(t *testing.T) { expectErrorMsg string }{ {"text", "Some text.", "map[]", "Some text.", ""}, - {"invalid right delim", "{{< tag }}", "", false, "simple.md:4:.*unrecognized character.*}"}, - {"invalid close", "\n{{< /tag >}}", "", false, "simple.md:5:.*got closing shortcode, but none is open"}, - {"invalid close2", "\n\n{{< tag >}}{{< /anotherTag >}}", "", false, "simple.md:6: closing tag for shortcode 'anotherTag' does not match start tag"}, - {"unterminated quote 1", `{{< figure src="im caption="S" >}}`, "", false, "simple.md:4:.got pos.*"}, - {"unterminated quote 1", `{{< figure src="im" caption="S >}}`, "", false, "simple.md:4:.*unterm.*}"}, + {"invalid right delim", "{{< tag }}", "", false, ":4:.*unrecognized character.*}"}, + {"invalid close", "\n{{< /tag >}}", "", false, ":5:.*got closing shortcode, but none is open"}, + {"invalid close2", "\n\n{{< tag >}}{{< /anotherTag >}}", "", false, ":6: closing tag for shortcode 'anotherTag' does not match start tag"}, + {"unterminated quote 1", `{{< figure src="im caption="S" >}}`, "", false, ":4:.got pos.*"}, + {"unterminated quote 1", `{{< figure src="im" caption="S >}}`, "", false, ":4:.*unterm.*}"}, {"one shortcode, no markup", "{{< tag >}}", "", testScPlaceholderRegexp, ""}, {"one shortcode, markup", "{{% tag %}}", "", testScPlaceholderRegexp, ""}, {"one pos param", "{{% tag param1 %}}", `tag([\"param1\"], true){[]}"]`, testScPlaceholderRegexp, ""}, @@ -384,7 +382,7 @@ func TestExtractShortcodes(t *testing.T) { // issue #934 {"inner self-closing", `Some text. {{< inner />}}. Some more text.`, `inner([], false){[]}`, fmt.Sprintf("Some text. %s. Some more text.", testScPlaceholderRegexp), ""}, - {"close, but not inner", "{{< tag >}}foo{{< /tag >}}", "", false, "Shortcode 'tag' in page 'simple.md' has no .Inner.*"}, + {"close, but not inner", "{{< tag >}}foo{{< /tag >}}", "", false, `shortcode "tag" has no .Inner, yet a closing tag was provided`}, {"nested inner", `Inner->{{< inner >}}Inner Content->{{% inner2 param1 %}}inner2txt{{% /inner2 %}}Inner close->{{< / inner >}}<-done`, `inner([], false){[Inner Content-> inner2([\"param1\"], true){[inner2txt]} Inner close->]}`, fmt.Sprintf("Inner->%s<-done", testScPlaceholderRegexp), ""}, @@ -434,7 +432,7 @@ func TestExtractShortcodes(t *testing.T) { } else { r, _ := regexp.Compile(this.expectErrorMsg) if !r.MatchString(err.Error()) { - t.Fatalf("[%d] %s: ExtractShortcodes didn't return an expected error message, got %s but expected %s", + t.Fatalf("[%d] %s: ExtractShortcodes didn't return an expected error message, got\n%s but expected\n%s", i, this.name, err.Error(), this.expectErrorMsg) } } @@ -777,7 +775,7 @@ NotFound: {{< thisDoesNotExist >}} "thisDoesNotExist", ) - require.Equal(t, uint64(1), s.Log.LogCountForLevel(jww.LevelError)) + require.Equal(t, uint64(1), s.Log.ErrorCounter.Count()) } diff --git a/hugolib/site.go b/hugolib/site.go index 1196496d3..687c6338c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -15,7 +15,6 @@ package hugolib import ( "context" - "errors" "fmt" "html/template" "io" @@ -29,6 +28,9 @@ import ( "strings" "time" + _errors "github.com/pkg/errors" + + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/resource" @@ -754,8 +756,6 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { return whatChanged{}, err } - s.TemplateHandler().PrintErrors() - for i := 1; i < len(sites); i++ { site := sites[i] var err error @@ -861,7 +861,7 @@ func (s *Site) handleDataFile(r source.ReadableFile) error { f, err := r.Open() if err != nil { - return fmt.Errorf("Failed to open data file %q: %s", r.LogicalName(), err) + return _errors.Wrapf(err, "Failed to open data file %q:", r.LogicalName()) } defer f.Close() @@ -942,7 +942,7 @@ func (s *Site) handleDataFile(r source.ReadableFile) error { func (s *Site) readData(f source.ReadableFile) (interface{}, error) { file, err := f.Open() if err != nil { - return nil, fmt.Errorf("readData: failed to open data file: %s", err) + return nil, _errors.Wrap(err, "readData: failed to open data file") } defer file.Close() content := helpers.ReaderToBytes(file) @@ -1558,26 +1558,52 @@ func (s *Site) preparePages() error { } } - if len(errors) != 0 { - return fmt.Errorf("Prepare pages failed: %.100q…", errors) + return s.pickOneAndLogTheRest(errors) +} + +func (s *Site) errorCollator(results <-chan error, errs chan<- error) { + var errors []error + for e := range results { + errors = append(errors, e) } - return nil + errs <- s.pickOneAndLogTheRest(errors) + + close(errs) } -func errorCollator(results <-chan error, errs chan<- error) { - errMsgs := []string{} - for err := range results { - if err != nil { - errMsgs = append(errMsgs, err.Error()) +func (s *Site) pickOneAndLogTheRest(errors []error) error { + if len(errors) == 0 { + return nil + } + + var i int + + for j, err := range errors { + // If this is in server mode, we want to return an error to the client + // with a file context, if possible. + if herrors.UnwrapErrorWithFileContext(err) != nil { + i = j + break } } - if len(errMsgs) == 0 { - errs <- nil - } else { - errs <- errors.New(strings.Join(errMsgs, "\n")) + + // Log the rest, but add a threshold to avoid flooding the log. + const errLogThreshold = 5 + + for j, err := range errors { + if j == i { + continue + } + + if j >= errLogThreshold { + break + } + + s.Log.ERROR.Println(err) } - close(errs) + + return errors[i] } func (s *Site) appendThemeTemplates(in []string) []string { @@ -1650,8 +1676,7 @@ func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath st renderBuffer.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n") if err := s.renderForLayouts(name, d, renderBuffer, layouts...); err != nil { - helpers.DistinctWarnLog.Println(err) - return nil + return err } var path string @@ -1684,8 +1709,8 @@ func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath s defer bp.PutBuffer(renderBuffer) if err := s.renderForLayouts(p.Kind, p, renderBuffer, layouts...); err != nil { - helpers.DistinctWarnLog.Println(err) - return nil + + return err } if renderBuffer.Len() == 0 { @@ -1735,46 +1760,18 @@ func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath s func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts ...string) (err error) { var templ tpl.Template - defer func() { - if r := recover(); r != nil { - templName := "" - if templ != nil { - templName = templ.Name() - } - s.DistinctErrorLog.Printf("Failed to render %q: %s", templName, r) - s.DistinctErrorLog.Printf("Stack Trace:\n%s", stackTrace(1200)) - - // TOD(bep) we really need to fix this. Also see below. - if !s.running() && !testMode { - os.Exit(-1) - } - } - }() - templ = s.findFirstTemplate(layouts...) if templ == nil { - return fmt.Errorf("[%s] Unable to locate layout for %q: %s\n", s.Language.Lang, name, layouts) + s.Log.WARN.Printf("[%s] Unable to locate layout for %q: %s\n", s.Language.Lang, name, layouts) + return nil } if err = templ.Execute(w, d); err != nil { - // Behavior here should be dependent on if running in server or watch mode. if p, ok := d.(*PageOutput); ok { - if p.File != nil { - s.DistinctErrorLog.Printf("Error while rendering %q in %q: %s", name, p.File.Dir(), err) - } else { - s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) - } - } else { - s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) - } - if !s.running() && !testMode { - // TODO(bep) check if this can be propagated - os.Exit(-1) - } else if testMode { - return + return p.errorf(err, "render of %q failed", name) } + return _errors.Wrapf(err, "render of %q failed", name) } - return } diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 1f2c5c708..13fbb43cd 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -19,6 +19,8 @@ import ( "strings" "sync" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/output" ) @@ -30,7 +32,7 @@ func (s *Site) renderPages(cfg *BuildCfg) error { pages := make(chan *Page) errs := make(chan error) - go errorCollator(results, errs) + go s.errorCollator(results, errs) numWorkers := getGoMaxProcs() * 4 @@ -60,7 +62,7 @@ func (s *Site) renderPages(cfg *BuildCfg) error { err := <-errs if err != nil { - return fmt.Errorf("Error(s) rendering pages: %s", err) + return errors.Wrap(err, "failed to render pages") } return nil } @@ -132,6 +134,7 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa if shouldRender { if err := pageOutput.renderResources(); err != nil { + // TODO(bep) 2errors s.Log.ERROR.Printf("Failed to render resources for page %q: %s", page, err) continue } diff --git a/hugolib/site_test.go b/hugolib/site_test.go index f775b0e79..a5688c78e 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -54,7 +54,7 @@ func TestRenderWithInvalidTemplate(t *testing.T) { withTemplate := createWithTemplateFromNameValues("missing", templateMissingFunc) - buildSingleSiteExpected(t, true, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}, BuildCfg{}) + buildSingleSiteExpected(t, true, false, deps.DepsCfg{Fs: fs, Cfg: cfg, WithTemplate: withTemplate}, BuildCfg{}) } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 27edf3fdd..70c9263b3 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -14,7 +14,6 @@ import ( "github.com/gohugoio/hugo/langs" "github.com/sanity-io/litter" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -26,6 +25,7 @@ import ( "os" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/hugofs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,7 +38,7 @@ type sitesBuilder struct { Fs *hugofs.Fs T testing.TB - logger *jww.Notepad + logger *loggers.Logger dumper litter.Options @@ -103,7 +103,7 @@ func (s *sitesBuilder) Running() *sitesBuilder { return s } -func (s *sitesBuilder) WithLogger(logger *jww.Notepad) *sitesBuilder { +func (s *sitesBuilder) WithLogger(logger *loggers.Logger) *sitesBuilder { s.logger = logger return s } @@ -312,6 +312,14 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) * } func (s *sitesBuilder) CreateSites() *sitesBuilder { + if err := s.CreateSitesE(); err != nil { + s.Fatalf("Failed to create sites: %s", err) + } + + return s +} + +func (s *sitesBuilder) CreateSitesE() error { s.addDefaults() s.writeFilePairs("content", s.contentFilePairs) s.writeFilePairs("content", s.contentFilePairsAdded) @@ -325,7 +333,7 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { if s.Cfg == nil { cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat}) if err != nil { - s.Fatalf("Failed to load config: %s", err) + return err } // TODO(bep) /* expectedConfigs := 1 @@ -339,11 +347,19 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder { sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running}) if err != nil { - s.Fatalf("Failed to create sites: %s", err) + return err } s.H = sites - return s + return nil +} + +func (s *sitesBuilder) BuildE(cfg BuildCfg) error { + if s.H == nil { + s.CreateSites() + } + + return s.H.Build(cfg) } func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder { @@ -360,6 +376,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder { } err := s.H.Build(cfg) + if err == nil { logErrorCount := s.H.NumLogErrors() if logErrorCount > 0 { @@ -639,13 +656,19 @@ func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ } func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { - return buildSingleSiteExpected(t, false, depsCfg, buildCfg) + return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg) } -func buildSingleSiteExpected(t testing.TB, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { +func buildSingleSiteExpected(t testing.TB, expectSiteInitEror, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site { h, err := NewHugoSites(depsCfg) - require.NoError(t, err) + if expectSiteInitEror { + require.Error(t, err) + return nil + } else { + require.NoError(t, err) + } + require.Len(t, h.Sites, 1) if expectBuildError { diff --git a/i18n/i18n.go b/i18n/i18n.go index 73417fb32..09f4b63f0 100644 --- a/i18n/i18n.go +++ b/i18n/i18n.go @@ -14,8 +14,10 @@ package i18n import ( + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" + "github.com/nicksnyder/go-i18n/i18n/bundle" jww "github.com/spf13/jwalterweatherman" ) @@ -28,11 +30,11 @@ var ( type Translator struct { translateFuncs map[string]bundle.TranslateFunc cfg config.Provider - logger *jww.Notepad + logger *loggers.Logger } // NewTranslator creates a new Translator for the given language bundle and configuration. -func NewTranslator(b *bundle.Bundle, cfg config.Provider, logger *jww.Notepad) Translator { +func NewTranslator(b *bundle.Bundle, cfg config.Provider, logger *loggers.Logger) Translator { t := Translator{cfg: cfg, logger: logger, translateFuncs: make(map[string]bundle.TranslateFunc)} t.initFuncs(b) return t diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 5075839ff..84b7384d0 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -19,24 +19,19 @@ import ( "github.com/gohugoio/hugo/tpl/tplimpl" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" "github.com/gohugoio/hugo/deps" - "io/ioutil" - "os" - - "log" - "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) -var logger = jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) +var logger = loggers.NewErrorLogger() type i18nTest struct { data map[string][]byte diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 5f90895aa..4e937c5a1 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -15,14 +15,13 @@ package i18n import ( "errors" - "fmt" - - "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/source" "github.com/nicksnyder/go-i18n/i18n/bundle" "github.com/nicksnyder/go-i18n/i18n/language" + _errors "github.com/pkg/errors" ) // TranslationProvider provides translation handling, i.e. loading @@ -82,12 +81,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error { func addTranslationFile(bundle *bundle.Bundle, r source.ReadableFile) error { f, err := r.Open() if err != nil { - return fmt.Errorf("Failed to open translations file %q: %s", r.LogicalName(), err) + return _errors.Wrapf(err, "Failed to open translations file %q:", r.LogicalName()) } defer f.Close() err = bundle.ParseTranslationFileBytes(r.LogicalName(), helpers.ReaderToBytes(f)) if err != nil { - return fmt.Errorf("Failed to load translations in file %q: %s", r.LogicalName(), err) + return _errors.Wrapf(err, "Failed to load translations in file %q:", r.LogicalName()) } return nil } diff --git a/releaser/releaser.go b/releaser/releaser.go index 7f7bc8db0..0738fb81c 100644 --- a/releaser/releaser.go +++ b/releaser/releaser.go @@ -16,7 +16,6 @@ package releaser import ( - "errors" "fmt" "io/ioutil" "log" @@ -26,6 +25,8 @@ import ( "regexp" "strings" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/helpers" ) @@ -255,7 +256,7 @@ func (r *ReleaseHandler) release(releaseNotesFile string) error { cmd.Stderr = os.Stderr err := cmd.Run() if err != nil { - return fmt.Errorf("goreleaser failed: %s", err) + return errors.Wrap(err, "goreleaser failed") } return nil } diff --git a/resource/image.go b/resource/image.go index e8b87cdb4..d9a1dd47d 100644 --- a/resource/image.go +++ b/resource/image.go @@ -26,6 +26,8 @@ import ( "strings" "sync" + _errors "github.com/pkg/errors" + "github.com/disintegration/imaging" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" @@ -430,7 +432,7 @@ func (i *Image) initConfig() error { }) if err != nil { - return fmt.Errorf("failed to load image config: %s", err) + return _errors.Wrap(err, "failed to load image config") } return nil @@ -439,7 +441,7 @@ func (i *Image) initConfig() error { func (i *Image) decodeSource() (image.Image, error) { f, err := i.ReadSeekCloser() if err != nil { - return nil, fmt.Errorf("failed to open image for decode: %s", err) + return nil, _errors.Wrap(err, "failed to open image for decode") } defer f.Close() img, _, err := image.Decode(f) diff --git a/resource/postcss/postcss.go b/resource/postcss/postcss.go index 202b4c06b..ec73543dd 100644 --- a/resource/postcss/postcss.go +++ b/resource/postcss/postcss.go @@ -14,19 +14,18 @@ package postcss import ( - "fmt" "io" "path/filepath" "github.com/gohugoio/hugo/hugofs" + "github.com/pkg/errors" - "github.com/mitchellh/mapstructure" - // "io/ioutil" "os" "os/exec" - "github.com/gohugoio/hugo/common/errors" + "github.com/mitchellh/mapstructure" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/resource" ) @@ -111,7 +110,7 @@ func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCt binary = binaryName if _, err := exec.LookPath(binary); err != nil { // This may be on a CI server etc. Will fall back to pre-built assets. - return errors.ErrFeatureNotAvailable + return herrors.ErrFeatureNotAvailable } } @@ -134,7 +133,7 @@ func (t *postcssTransformation) Transform(ctx *resource.ResourceTransformationCt if err != nil { if t.options.Config != "" { // Only fail if the user specificed config file is not found. - return fmt.Errorf("postcss config %q not found: %s", configFile, err) + return errors.Wrapf(err, "postcss config %q not found:", configFile) } configFile = "" } else { diff --git a/resource/resource.go b/resource/resource.go index dd9cbbd41..be3ebdb8b 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -14,7 +14,6 @@ package resource import ( - "errors" "fmt" "io" "io/ioutil" @@ -27,13 +26,12 @@ import ( "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/tpl" + "github.com/pkg/errors" "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/common/loggers" - jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/afero" "github.com/gobwas/glob" @@ -273,7 +271,7 @@ type Spec struct { MediaTypes media.Types OutputFormats output.Formats - Logger *jww.Notepad + Logger *loggers.Logger TextTemplates tpl.TemplateParseFinder @@ -287,7 +285,7 @@ type Spec struct { GenAssetsPath string } -func NewSpec(s *helpers.PathSpec, logger *jww.Notepad, outputFormats output.Formats, mimeTypes media.Types) (*Spec, error) { +func NewSpec(s *helpers.PathSpec, logger *loggers.Logger, outputFormats output.Formats, mimeTypes media.Types) (*Spec, error) { imaging, err := decodeImaging(s.Cfg.GetStringMap("imaging")) if err != nil { @@ -542,7 +540,7 @@ type resourceHash struct { type publishOnce struct { publisherInit sync.Once publisherErr error - logger *jww.Notepad + logger *loggers.Logger } func (l *publishOnce) publish(s Source) error { @@ -660,7 +658,7 @@ func (l *genericResource) initHash() error { var f hugio.ReadSeekCloser f, err = l.ReadSeekCloser() if err != nil { - err = fmt.Errorf("failed to open source file: %s", err) + err = errors.Wrap(err, "failed to open source file") return } defer f.Close() diff --git a/resource/resource_metadata.go b/resource/resource_metadata.go index 2c82aeaf6..20c4f130b 100644 --- a/resource/resource_metadata.go +++ b/resource/resource_metadata.go @@ -17,6 +17,7 @@ import ( "fmt" "strconv" + "github.com/pkg/errors" "github.com/spf13/cast" "strings" @@ -69,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) er glob, err := getGlob(srcKey) if err != nil { - return fmt.Errorf("failed to match resource with metadata: %s", err) + return errors.Wrap(err, "failed to match resource with metadata") } match := glob.Match(resourceSrcKey) diff --git a/resource/templates/execute_as_template.go b/resource/templates/execute_as_template.go index dee9d0d9a..a126b26c9 100644 --- a/resource/templates/execute_as_template.go +++ b/resource/templates/execute_as_template.go @@ -15,11 +15,10 @@ package templates import ( - "fmt" - "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/tpl" + "github.com/pkg/errors" ) // Client contains methods to perform template processing of Resource objects. @@ -55,7 +54,7 @@ func (t *executeAsTemplateTransform) Transform(ctx *resource.ResourceTransformat tplStr := helpers.ReaderToString(ctx.From) templ, err := t.textTemplate.Parse(ctx.InPath, tplStr) if err != nil { - return fmt.Errorf("failed to parse Resource %q as Template: %s", ctx.InPath, err) + return errors.Wrapf(err, "failed to parse Resource %q as Template:", ctx.InPath) } ctx.OutPath = t.targetPath diff --git a/resource/tocss/scss/tocss.go b/resource/tocss/scss/tocss.go index 5ba7793c0..984e14fc2 100644 --- a/resource/tocss/scss/tocss.go +++ b/resource/tocss/scss/tocss.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resource" + "github.com/pkg/errors" ) // Used in tests. This feature requires Hugo to be built with the extended tag. @@ -165,7 +166,7 @@ func (c *Client) toCSS(options scss.Options, dst io.Writer, src io.Reader) (tocs res, err = transpiler.Execute(dst, src) if err != nil { - return res, fmt.Errorf("SCSS processing failed: %s", err) + return res, errors.Wrap(err, "SCSS processing failed") } return res, nil diff --git a/resource/tocss/scss/tocss_notavailable.go b/resource/tocss/scss/tocss_notavailable.go index 2ec5a4832..df918b368 100644 --- a/resource/tocss/scss/tocss_notavailable.go +++ b/resource/tocss/scss/tocss_notavailable.go @@ -16,7 +16,7 @@ package scss import ( - "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/resource" ) @@ -26,5 +26,5 @@ func Supports() bool { } func (t *toCSSTransformation) Transform(ctx *resource.ResourceTransformationCtx) error { - return errors.ErrFeatureNotAvailable + return herrors.ErrFeatureNotAvailable } diff --git a/resource/transform.go b/resource/transform.go index 01b05b73e..d3a215467 100644 --- a/resource/transform.go +++ b/resource/transform.go @@ -20,7 +20,7 @@ import ( "strings" "github.com/gohugoio/hugo/common/collections" - "github.com/gohugoio/hugo/common/errors" + "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" "github.com/mitchellh/hashstructure" @@ -390,7 +390,7 @@ func (r *transformedResource) transform(setContent bool) (err error) { } if err := tr.transformation.Transform(tctx); err != nil { - if err == errors.ErrFeatureNotAvailable { + if err == herrors.ErrFeatureNotAvailable { // This transformation is not available in this // Hugo installation (scss not compiled in, PostCSS not available etc.) // If a prepared bundle for this transformation chain is available, use that. diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index c8a7207ea..b3ac8a8f2 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -17,20 +17,17 @@ import ( "errors" "fmt" "html/template" - "io/ioutil" - "log" "math/rand" - "os" "reflect" "testing" "time" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -856,7 +853,7 @@ func newDeps(cfg config.Provider) *deps.Deps { Cfg: cfg, Fs: hugofs.NewMem(l), ContentSpec: cs, - Log: jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime), + Log: loggers.NewErrorLogger(), } } diff --git a/tpl/data/data.go b/tpl/data/data.go index 14a4975a5..3f87eda31 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -18,12 +18,12 @@ import ( "encoding/csv" "encoding/json" "errors" - "fmt" "net/http" "strings" "time" "github.com/gohugoio/hugo/deps" + _errors "github.com/pkg/errors" ) // New returns a new instance of the data-namespaced template functions. @@ -59,7 +59,7 @@ func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err e var req *http.Request req, err = http.NewRequest("GET", url, nil) if err != nil { - return nil, fmt.Errorf("Failed to create request for getCSV for resource %s: %s", url, err) + return nil, _errors.Wrapf(err, "Failed to create request for getCSV for resource %s:", url) } req.Header.Add("Accept", "text/csv") @@ -103,7 +103,7 @@ func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) { var req *http.Request req, err = http.NewRequest("GET", url, nil) if err != nil { - return nil, fmt.Errorf("Failed to create request for getJSON resource %s: %s", url, err) + return nil, _errors.Wrapf(err, "Failed to create request for getJSON resource %s:", url) } req.Header.Add("Accept", "application/json") diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go index 6bee0d524..9ef969244 100644 --- a/tpl/data/data_test.go +++ b/tpl/data/data_test.go @@ -113,11 +113,11 @@ func TestGetCSV(t *testing.T) { require.NoError(t, err, msg) if _, ok := test.expect.(bool); ok { - require.Equal(t, 1, int(ns.deps.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError))) + require.Equal(t, 1, int(ns.deps.Log.ErrorCounter.Count())) require.Nil(t, got) continue } - require.Equal(t, 0, int(ns.deps.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError))) + require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count())) require.NotNil(t, got, msg) assert.EqualValues(t, test.expect, got, msg) @@ -198,14 +198,14 @@ func TestGetJSON(t *testing.T) { continue } - if errLevel, ok := test.expect.(jww.Threshold); ok { - logCount := ns.deps.Log.LogCountForLevelsGreaterThanorEqualTo(errLevel) + if errLevel, ok := test.expect.(jww.Threshold); ok && errLevel >= jww.LevelError { + logCount := ns.deps.Log.ErrorCounter.Count() require.True(t, logCount >= 1, fmt.Sprintf("got log count %d", logCount)) continue } require.NoError(t, err, msg) - require.Equal(t, 0, int(ns.deps.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError)), msg) + require.Equal(t, 0, int(ns.deps.Log.ErrorCounter.Count()), msg) require.NotNil(t, got, msg) assert.EqualValues(t, test.expect, got, msg) diff --git a/tpl/fmt/fmt.go b/tpl/fmt/fmt.go index 0f4f906c2..09e4f5a40 100644 --- a/tpl/fmt/fmt.go +++ b/tpl/fmt/fmt.go @@ -16,12 +16,13 @@ package fmt import ( _fmt "fmt" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" ) // New returns a new instance of the fmt-namespaced template functions. -func New() *Namespace { - return &Namespace{helpers.NewDistinctErrorLogger()} +func New(d *deps.Deps) *Namespace { + return &Namespace{helpers.NewDistinctLogger(d.Log.ERROR)} } // Namespace provides template functions for the "fmt" namespace. diff --git a/tpl/fmt/init.go b/tpl/fmt/init.go index 76c68957a..117055801 100644 --- a/tpl/fmt/init.go +++ b/tpl/fmt/init.go @@ -22,7 +22,7 @@ const name = "fmt" func init() { f := func(d *deps.Deps) *internal.TemplateFuncsNamespace { - ctx := New() + ctx := New(d) ns := &internal.TemplateFuncsNamespace{ Name: name, diff --git a/tpl/fmt/init_test.go b/tpl/fmt/init_test.go index 01eb2fa69..b693ffa2b 100644 --- a/tpl/fmt/init_test.go +++ b/tpl/fmt/init_test.go @@ -16,6 +16,7 @@ package fmt import ( "testing" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/internal" "github.com/stretchr/testify/require" @@ -26,7 +27,7 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{Log: loggers.NewErrorLogger()}) if ns.Name == name { found = true break diff --git a/tpl/partials/init_test.go b/tpl/partials/init_test.go index 4832e6b66..0513f1572 100644 --- a/tpl/partials/init_test.go +++ b/tpl/partials/init_test.go @@ -16,6 +16,7 @@ package partials import ( "testing" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/internal" "github.com/stretchr/testify/require" @@ -28,6 +29,7 @@ func TestInit(t *testing.T) { for _, nsf := range internal.TemplateFuncsNamespaceRegistry { ns = nsf(&deps.Deps{ BuildStartListeners: &deps.Listeners{}, + Log: loggers.NewErrorLogger(), }) if ns.Name == name { found = true diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 883afbcd7..c24cd2b42 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -18,6 +18,8 @@ import ( "fmt" "path/filepath" + _errors "github.com/pkg/errors" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/resource" "github.com/gohugoio/hugo/resource/bundler" @@ -256,7 +258,7 @@ func (ns *Namespace) resolveArgs(args []interface{}) (resource.Resource, map[str m, err := cast.ToStringMapE(args[0]) if err != nil { - return nil, nil, fmt.Errorf("invalid options type: %s", err) + return nil, nil, _errors.Wrap(err, "invalid options type") } return r, m, nil diff --git a/tpl/strings/strings.go b/tpl/strings/strings.go index 9b8409ed6..1853cd727 100644 --- a/tpl/strings/strings.go +++ b/tpl/strings/strings.go @@ -20,6 +20,8 @@ import ( _strings "strings" "unicode/utf8" + _errors "github.com/pkg/errors" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" @@ -44,7 +46,7 @@ type Namespace struct { func (ns *Namespace) CountRunes(s interface{}) (int, error) { ss, err := cast.ToStringE(s) if err != nil { - return 0, fmt.Errorf("Failed to convert content to string: %s", err) + return 0, _errors.Wrap(err, "Failed to convert content to string") } counter := 0 @@ -61,7 +63,7 @@ func (ns *Namespace) CountRunes(s interface{}) (int, error) { func (ns *Namespace) RuneCount(s interface{}) (int, error) { ss, err := cast.ToStringE(s) if err != nil { - return 0, fmt.Errorf("Failed to convert content to string: %s", err) + return 0, _errors.Wrap(err, "Failed to convert content to string") } return utf8.RuneCountInString(ss), nil } @@ -70,7 +72,7 @@ func (ns *Namespace) RuneCount(s interface{}) (int, error) { func (ns *Namespace) CountWords(s interface{}) (int, error) { ss, err := cast.ToStringE(s) if err != nil { - return 0, fmt.Errorf("Failed to convert content to string: %s", err) + return 0, _errors.Wrap(err, "Failed to convert content to string") } counter := 0 diff --git a/tpl/template.go b/tpl/template.go index 2cef92bb2..02b2d4a9b 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,16 +14,26 @@ package tpl import ( + "fmt" "io" + "path/filepath" + "regexp" + "strings" "time" - "text/template/parse" + "github.com/gohugoio/hugo/common/herrors" + + "github.com/gohugoio/hugo/hugofs" + + "github.com/spf13/afero" "html/template" texttemplate "text/template" + "text/template/parse" bp "github.com/gohugoio/hugo/bufferpool" "github.com/gohugoio/hugo/metrics" + "github.com/pkg/errors" ) var ( @@ -35,8 +45,7 @@ type TemplateHandler interface { TemplateFinder AddTemplate(name, tpl string) error AddLateTemplate(name, tpl string) error - LoadTemplates(prefix string) - PrintErrors() + LoadTemplates(prefix string) error NewTextTemplate() TemplateParseFinder @@ -82,16 +91,122 @@ type TemplateDebugger interface { type TemplateAdapter struct { Template Metrics metrics.Provider + + // The filesystem where the templates are stored. + Fs afero.Fs + + // Maps to base template if relevant. + NameBaseTemplateName map[string]string +} + +var baseOfRe = regexp.MustCompile("template: (.*?):") + +func extractBaseOf(err string) string { + m := baseOfRe.FindStringSubmatch(err) + if len(m) == 2 { + return m[1] + } + return "" } // Execute executes the current template. The actual execution is performed // by the embedded text or html template, but we add an implementation here so // we can add a timer for some metrics. -func (t *TemplateAdapter) Execute(w io.Writer, data interface{}) error { +func (t *TemplateAdapter) Execute(w io.Writer, data interface{}) (execErr error) { + defer func() { + // Panics in templates are a little bit too common (nil pointers etc.) + if r := recover(); r != nil { + execErr = t.addFileContext(t.Name(), fmt.Errorf("panic in Execute: %s", r)) + } + }() + if t.Metrics != nil { defer t.Metrics.MeasureSince(t.Name(), time.Now()) } - return t.Template.Execute(w, data) + + execErr = t.Template.Execute(w, data) + if execErr != nil { + execErr = t.addFileContext(t.Name(), execErr) + } + + return +} + +var identifiersRe = regexp.MustCompile("at \\<(.*?)\\>:") + +func (t *TemplateAdapter) extractIdentifiers(line string) []string { + m := identifiersRe.FindAllStringSubmatch(line, -1) + identifiers := make([]string, len(m)) + for i := 0; i < len(m); i++ { + identifiers[i] = m[i][1] + } + return identifiers +} + +func (t *TemplateAdapter) addFileContext(name string, inerr error) error { + f, realFilename, err := t.fileAndFilename(t.Name()) + if err != nil { + return err + } + defer f.Close() + + master, hasMaster := t.NameBaseTemplateName[name] + + ferr := errors.Wrapf(inerr, "execute of template %q failed", realFilename) + + // Since this can be a composite of multiple template files (single.html + baseof.html etc.) + // we potentially need to look in both -- and cannot rely on line number alone. + lineMatcher := func(le herrors.FileError, lineNumber int, line string) bool { + if le.LineNumber() != lineNumber { + return false + } + if !hasMaster { + return true + } + + identifiers := t.extractIdentifiers(le.Error()) + + for _, id := range identifiers { + if strings.Contains(line, id) { + return true + } + } + return false + } + + // TODO(bep) 2errors text vs HTML + fe, ok := herrors.WithFileContext(ferr, f, "go-html-template", lineMatcher) + if ok || !hasMaster { + return fe + } + + // Try the base template if relevant + f, realFilename, err = t.fileAndFilename(master) + if err != nil { + return err + } + defer f.Close() + + ferr = errors.Wrapf(inerr, "execute of template %q failed", realFilename) + fe, _ = herrors.WithFileContext(ferr, f, "go-html-template", lineMatcher) + return fe + +} + +func (t *TemplateAdapter) fileAndFilename(name string) (afero.File, string, error) { + fs := t.Fs + filename := filepath.FromSlash(name) + + fi, err := fs.Stat(filename) + if err != nil { + return nil, "", errors.Wrapf(err, "failed to Stat %q", filename) + } + f, err := fs.Open(filename) + if err != nil { + return nil, "", errors.Wrapf(err, "failed to open template file %q:", filename) + } + + return f, fi.(hugofs.RealFilenameInfo).RealFilename(), nil } // ExecuteToString executes the current template and returns the result as a diff --git a/tpl/template_test.go b/tpl/template_test.go new file mode 100644 index 000000000..73e9640be --- /dev/null +++ b/tpl/template_test.go @@ -0,0 +1,31 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tpl + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractBaseof(t *testing.T) { + assert := require.New(t) + + replaced := extractBaseOf(`failed: template: _default/baseof.html:37:11: executing "_default/baseof.html" at <.Parents>: can't evaluate field Parents in type *hugolib.PageOutput`) + + assert.Equal("_default/baseof.html", replaced) + assert.Equal("", extractBaseOf("not baseof for you")) + assert.Equal("blog/baseof.html", extractBaseOf("template: blog/baseof.html:23:11:")) + assert.Equal("blog/baseof.ace", extractBaseOf("template: blog/baseof.ace:23:11:")) +} diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go index f19c312ec..fc77bb1af 100644 --- a/tpl/tplimpl/template.go +++ b/tpl/tplimpl/template.go @@ -1,4 +1,4 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2018 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,7 +20,9 @@ import ( "strings" texttemplate "text/template" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/tpl/tplimpl/embedded" + "github.com/pkg/errors" "github.com/eknkc/amber" @@ -64,7 +66,7 @@ type templateErr struct { } type templateLoader interface { - handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error + handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error addTemplate(name, tpl string) error addLateTemplate(name, tpl string) error } @@ -114,22 +116,11 @@ func (t *templateHandler) NewTextTemplate() tpl.TemplateParseFinder { } -func (t *templateHandler) addError(name string, err error) { - t.errors = append(t.errors, &templateErr{name, err}) -} - func (t *templateHandler) Debug() { fmt.Println("HTML templates:\n", t.html.t.DefinedTemplates()) fmt.Println("\n\nText templates:\n", t.text.t.DefinedTemplates()) } -// PrintErrors prints the accumulated errors as ERROR to the log. -func (t *templateHandler) PrintErrors() { - for _, e := range t.errors { - t.Log.ERROR.Println(e.name, ":", e.err) - } -} - // Lookup tries to find a template with the given name in both template // collections: First HTML, then the plain text template collection. func (t *templateHandler) Lookup(name string) (tpl.Template, bool) { @@ -156,8 +147,8 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler { c := &templateHandler{ Deps: d, layoutsFs: d.BaseFs.Layouts.Fs, - html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)}, - text: &textTemplates{textTemplate: &textTemplate{t: texttemplate.Must(t.text.t.Clone())}, overlays: make(map[string]*texttemplate.Template)}, + html: &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template), templatesCommon: t.html.templatesCommon}, + text: &textTemplates{textTemplate: &textTemplate{t: texttemplate.Must(t.text.t.Clone())}, overlays: make(map[string]*texttemplate.Template), templatesCommon: t.text.templatesCommon}, errors: make([]*templateErr, 0), } @@ -187,15 +178,21 @@ func (t *templateHandler) clone(d *deps.Deps) *templateHandler { } func newTemplateAdapter(deps *deps.Deps) *templateHandler { + common := &templatesCommon{ + nameBaseTemplateName: make(map[string]string), + } + htmlT := &htmlTemplates{ - t: template.New(""), - overlays: make(map[string]*template.Template), + t: template.New(""), + overlays: make(map[string]*template.Template), + templatesCommon: common, } textT := &textTemplates{ - textTemplate: &textTemplate{t: texttemplate.New("")}, - overlays: make(map[string]*texttemplate.Template), + textTemplate: &textTemplate{t: texttemplate.New("")}, + overlays: make(map[string]*texttemplate.Template), + templatesCommon: common, } - return &templateHandler{ + h := &templateHandler{ Deps: deps, layoutsFs: deps.BaseFs.Layouts.Fs, html: htmlT, @@ -203,11 +200,23 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler { errors: make([]*templateErr, 0), } + common.handler = h + + return h + } -type htmlTemplates struct { +// Shared by both HTML and text templates. +type templatesCommon struct { + handler *templateHandler funcster *templateFuncster + // Used to get proper filenames in errors + nameBaseTemplateName map[string]string +} +type htmlTemplates struct { + *templatesCommon + t *template.Template // This looks, and is, strange. @@ -231,7 +240,8 @@ func (t *htmlTemplates) Lookup(name string) (tpl.Template, bool) { if templ == nil { return nil, false } - return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics}, true + + return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics, Fs: t.handler.layoutsFs, NameBaseTemplateName: t.nameBaseTemplateName}, true } func (t *htmlTemplates) lookup(name string) *template.Template { @@ -259,8 +269,8 @@ func (t *textTemplates) setTemplateFuncster(f *templateFuncster) { } type textTemplates struct { + *templatesCommon *textTemplate - funcster *templateFuncster clone *texttemplate.Template cloneClone *texttemplate.Template @@ -272,7 +282,7 @@ func (t *textTemplates) Lookup(name string) (tpl.Template, bool) { if templ == nil { return nil, false } - return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics}, true + return &tpl.TemplateAdapter{Template: templ, Metrics: t.funcster.Deps.Metrics, Fs: t.handler.layoutsFs, NameBaseTemplateName: t.nameBaseTemplateName}, true } func (t *textTemplates) lookup(name string) *texttemplate.Template { @@ -321,8 +331,8 @@ func (t *textTemplates) setFuncs(funcMap map[string]interface{}) { // LoadTemplates loads the templates from the layouts filesystem. // A prefix can be given to indicate a template namespace to load the templates // into, i.e. "_internal" etc. -func (t *templateHandler) LoadTemplates(prefix string) { - t.loadTemplates(prefix) +func (t *templateHandler) LoadTemplates(prefix string) error { + return t.loadTemplates(prefix) } @@ -423,7 +433,6 @@ func (t *templateHandler) addLateTemplate(name, tpl string) error { func (t *templateHandler) AddLateTemplate(name, tpl string) error { h := t.getTemplateHandler(name) if err := h.addLateTemplate(name, tpl); err != nil { - t.addError(name, err) return err } return nil @@ -435,7 +444,6 @@ func (t *templateHandler) AddLateTemplate(name, tpl string) error { func (t *templateHandler) AddTemplate(name, tpl string) error { h := t.getTemplateHandler(name) if err := h.addTemplate(name, tpl); err != nil { - t.addError(name, err) return err } return nil @@ -458,14 +466,19 @@ func (t *templateHandler) MarkReady() { // RebuildClone rebuilds the cloned templates. Used for live-reloads. func (t *templateHandler) RebuildClone() { - t.html.clone = template.Must(t.html.cloneClone.Clone()) - t.text.clone = texttemplate.Must(t.text.cloneClone.Clone()) + if t.html != nil && t.html.cloneClone != nil { + t.html.clone = template.Must(t.html.cloneClone.Clone()) + } + if t.text != nil && t.text.cloneClone != nil { + t.text.clone = texttemplate.Must(t.text.cloneClone.Clone()) + } } -func (t *templateHandler) loadTemplates(prefix string) { +func (t *templateHandler) loadTemplates(prefix string) error { + walker := func(path string, fi os.FileInfo, err error) error { if err != nil || fi.IsDir() { - return nil + return err } if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) { @@ -490,21 +503,25 @@ func (t *templateHandler) loadTemplates(prefix string) { tplID, err := output.CreateTemplateNames(descriptor) if err != nil { t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err) - return nil } if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil { - t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err) + return err } return nil } if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil { - t.Log.ERROR.Printf("Failed to load templates: %s", err) + if !os.IsNotExist(err) { + return err + } + return nil } + return nil + } func (t *templateHandler) initFuncs() { @@ -553,12 +570,12 @@ func (t *templateHandler) getTemplateHandler(name string) templateLoader { return t.html } -func (t *templateHandler) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { +func (t *templateHandler) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error { h := t.getTemplateHandler(name) return h.handleMaster(name, overlayFilename, masterFilename, onMissing) } -func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { +func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error { masterTpl := t.lookup(masterFilename) @@ -568,9 +585,9 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin return err } - masterTpl, err = t.t.New(overlayFilename).Parse(templ) + masterTpl, err = t.t.New(overlayFilename).Parse(templ.template) if err != nil { - return err + return templ.errWithFileContext("parse master failed", err) } } @@ -579,9 +596,9 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin return err } - overlayTpl, err := template.Must(masterTpl.Clone()).Parse(templ) + overlayTpl, err := template.Must(masterTpl.Clone()).Parse(templ.template) if err != nil { - return err + return templ.errWithFileContext("parse failed", err) } // The extra lookup is a workaround, see @@ -593,12 +610,13 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin } t.overlays[name] = overlayTpl + t.nameBaseTemplateName[name] = masterFilename return err } -func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error { +func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (templateInfo, error)) error { name = strings.TrimPrefix(name, textTmplNamePrefix) masterTpl := t.lookup(masterFilename) @@ -609,10 +627,11 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin return err } - masterTpl, err = t.t.New(overlayFilename).Parse(templ) + masterTpl, err = t.t.New(masterFilename).Parse(templ.template) if err != nil { - return err + return errors.Wrapf(err, "failed to parse %q:", templ.filename) } + t.nameBaseTemplateName[masterFilename] = templ.filename } templ, err := onMissing(overlayFilename) @@ -620,9 +639,9 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin return err } - overlayTpl, err := texttemplate.Must(masterTpl.Clone()).Parse(templ) + overlayTpl, err := texttemplate.Must(masterTpl.Clone()).Parse(templ.template) if err != nil { - return err + return errors.Wrapf(err, "failed to parse %q:", templ.filename) } overlayTpl = overlayTpl.Lookup(overlayTpl.Name()) @@ -630,6 +649,7 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin return err } t.overlays[name] = overlayTpl + t.nameBaseTemplateName[name] = templ.filename return err @@ -640,14 +660,22 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e t.Log.DEBUG.Printf("Add template file: name %q, baseTemplatePath %q, path %q", name, baseTemplatePath, path) - getTemplate := func(filename string) (string, error) { - b, err := afero.ReadFile(t.Layouts.Fs, filename) + getTemplate := func(filename string) (templateInfo, error) { + fs := t.Layouts.Fs + b, err := afero.ReadFile(fs, filename) if err != nil { - return "", err + return templateInfo{filename: filename, fs: fs}, err } s := string(b) - return s, nil + realFilename := filename + if fi, err := fs.Stat(filename); err == nil { + if fir, ok := fi.(hugofs.RealFilenameInfo); ok { + realFilename = fir.RealFilename() + } + } + + return templateInfo{template: s, filename: filename, realFilename: realFilename, fs: fs}, nil } // get the suffix and switch on that @@ -712,7 +740,11 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e return err } - return t.AddTemplate(name, templ) + err = t.AddTemplate(name, templ.template) + if err != nil { + return templ.errWithFileContext("parse failed", err) + } + return nil } } @@ -720,19 +752,24 @@ var embeddedTemplatesAliases = map[string][]string{ "shortcodes/twitter.html": []string{"shortcodes/tweet.html"}, } -func (t *templateHandler) loadEmbedded() { +func (t *templateHandler) loadEmbedded() error { for _, kv := range embedded.EmbeddedTemplates { - // TODO(bep) error handling name, templ := kv[0], kv[1] - t.addInternalTemplate(name, templ) + if err := t.addInternalTemplate(name, templ); err != nil { + return err + } if aliases, found := embeddedTemplatesAliases[name]; found { for _, alias := range aliases { - t.addInternalTemplate(alias, templ) + if err := t.addInternalTemplate(alias, templ); err != nil { + return err + } } } } + return nil + } func (t *templateHandler) addInternalTemplate(name, tpl string) error { diff --git a/tpl/tplimpl/templateProvider.go b/tpl/tplimpl/templateProvider.go index df44e81a6..3a803f2da 100644 --- a/tpl/tplimpl/templateProvider.go +++ b/tpl/tplimpl/templateProvider.go @@ -33,12 +33,15 @@ func (*TemplateProvider) Update(deps *deps.Deps) error { deps.TextTmpl = newTmpl.NewTextTemplate() newTmpl.initFuncs() - newTmpl.loadEmbedded() + + if err := newTmpl.loadEmbedded(); err != nil { + return err + } if deps.WithTemplate != nil { err := deps.WithTemplate(newTmpl) if err != nil { - newTmpl.addError("init", err) + return err } } diff --git a/tpl/tplimpl/template_errors.go b/tpl/tplimpl/template_errors.go new file mode 100644 index 000000000..a422d77f1 --- /dev/null +++ b/tpl/tplimpl/template_errors.go @@ -0,0 +1,46 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tplimpl + +import ( + "github.com/gohugoio/hugo/common/herrors" + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +type templateInfo struct { + template string + + // Used to create some error context in error situations + fs afero.Fs + + // The filename relative to the fs above. + filename string + + // The real filename (if possible). Used for logging. + realFilename string +} + +func (info templateInfo) errWithFileContext(what string, err error) error { + err = errors.Wrapf(err, "file %q: %s:", info.realFilename, what) + + err, _ = herrors.WithFileContextForFile( + err, + info.filename, + info.fs, + "go-html-template", + herrors.SimpleLineMatcher) + + return err +} diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index 8594c67a4..04bb4941a 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -21,10 +21,7 @@ import ( "testing" "time" - "io/ioutil" - "log" - "os" - + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" @@ -35,13 +32,12 @@ import ( "github.com/gohugoio/hugo/tpl/internal" "github.com/gohugoio/hugo/tpl/partials" "github.com/spf13/afero" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) var ( - logger = jww.NewNotepad(jww.LevelFatal, jww.LevelFatal, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + logger = loggers.NewErrorLogger() ) func newTestConfig() config.Provider { diff --git a/tpl/urls/urls.go b/tpl/urls/urls.go index 54c94ec17..8f6f92f3d 100644 --- a/tpl/urls/urls.go +++ b/tpl/urls/urls.go @@ -17,11 +17,12 @@ import ( "errors" "fmt" - "github.com/russross/blackfriday" - "html/template" "net/url" + _errors "github.com/pkg/errors" + "github.com/russross/blackfriday" + "github.com/gohugoio/hugo/deps" "github.com/spf13/cast" ) @@ -55,7 +56,7 @@ func (ns *Namespace) AbsURL(a interface{}) (template.HTML, error) { func (ns *Namespace) Parse(rawurl interface{}) (*url.URL, error) { s, err := cast.ToStringE(rawurl) if err != nil { - return nil, fmt.Errorf("Error in Parse: %s", err) + return nil, _errors.Wrap(err, "Error in Parse") } return url.Parse(s) |