Add Markdown diagrams and render hooks for code blocks

You can now create custom hook templates for code blocks, either one for all (`render-codeblock.html`) or for a given code language (e.g. `render-codeblock-go.html`).

We also used this new hook to add support for diagrams in Hugo:

* Goat (Go ASCII Tool) is built-in and enabled by default; just create a fenced code block with the language `goat` and start draw your Ascii diagrams.
* Another popular alternative for diagrams in Markdown, Mermaid (supported by GitHub), can also be implemented with a simple template. See the Hugo documentation for more information.

Updates #7765
Closes #9538
Fixes #9553
Fixes #8520
Fixes #6702
Fixes #9558

10 files changed, 136 insertions, 151 deletions

diff --git a/hugolib/content_render_hooks_test.go b/hugolib/content_render_hooks_test.go
index edfeaa82a..33ebe1f41 100644
--- a/hugolib/content_render_hooks_test.go
+++ b/hugolib/content_render_hooks_test.go
@@ -231,8 +231,8 @@ SHORT3|
 	b.AssertFileContent("public/blog/p3/index.html", `PARTIAL3`)
 	// We may add type template support later, keep this for then. b.AssertFileContent("public/docs/docs1/index.html", `Link docs section: Docs 1|END`)
 	b.AssertFileContent("public/blog/p4/index.html", `<p>IMAGE: Cool Page With Image||/images/Dragster.jpg|Title: image title|Text: Drag Racing|END</p>`)
-	// The regular markdownify func currently gets regular links.
-	b.AssertFileContent("public/blog/p5/index.html", "Inner Link: <a href=\"https://www.google.com\" title=\"Google's Homepage\">Inner Link</a>\n</div>")
+	// markdownify
+	b.AssertFileContent("public/blog/p5/index.html", "Inner Link: |https://www.google.com|Title: Google's Homepage|Text: Inner Link|END")
 
 	b.AssertFileContent("public/blog/p6/index.html",
 		"Inner Inline: Inner Link: With RenderString|https://www.gohugo.io|Title: Hugo's Homepage|Text: Inner Link|END",
diff --git a/hugolib/integrationtest_builder.go b/hugolib/integrationtest_builder.go
index 7ec7a1503..ed68783a1 100644
--- a/hugolib/integrationtest_builder.go
+++ b/hugolib/integrationtest_builder.go
@@ -125,7 +125,7 @@ func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...s
 			if match == "" || strings.HasPrefix(match, "#") {
 				continue
 			}
-			s.Assert(content, qt.Contains, match, qt.Commentf(content))
+			s.Assert(content, qt.Contains, match, qt.Commentf(m))
 		}
 	}
 }
@@ -164,7 +164,7 @@ func (s *IntegrationTestBuilder) AssertRenderCountPage(count int) {
 func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder {
 	s.Helper()
 	_, err := s.BuildE()
-	if s.Cfg.Verbose {
+	if s.Cfg.Verbose || err != nil {
 		fmt.Println(s.logBuff.String())
 	}
 	s.Assert(err, qt.IsNil)
diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go
index 117fdfb14..9a7a78e7e 100644
--- a/hugolib/language_content_dir_test.go
+++ b/hugolib/language_content_dir_test.go
@@ -314,7 +314,7 @@ Content.
 	nnSect := nnSite.getPage(page.KindSection, "sect")
 	c.Assert(nnSect, qt.Not(qt.IsNil))
 	c.Assert(len(nnSect.Pages()), qt.Equals, 12)
-	nnHome, _ := nnSite.Info.Home()
+	nnHome := nnSite.Info.Home()
 	c.Assert(nnHome.RelPermalink(), qt.Equals, "/nn/")
 }
 
diff --git a/hugolib/page.go b/hugolib/page.go
index 83b654cc0..7101af814 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -22,6 +22,8 @@ import (
 	"sort"
 	"strings"
 
+	"go.uber.org/atomic"
+
 	"github.com/gohugoio/hugo/identity"
 
 	"github.com/gohugoio/hugo/markup/converter"
@@ -47,7 +49,6 @@ import (
 
 	"github.com/gohugoio/hugo/common/collections"
 	"github.com/gohugoio/hugo/common/text"
-	"github.com/gohugoio/hugo/markup/converter/hooks"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/gohugoio/hugo/resources/resource"
@@ -118,6 +119,9 @@ type pageState struct {
 	// formats (for all sites).
 	pageOutputs []*pageOutput
 
+	// Used to determine if we can reuse content across output formats.
+	pageOutputTemplateVariationsState *atomic.Uint32
+
 	// This will be shifted out when we start to render a new output format.
 	*pageOutput
 
@@ -125,6 +129,10 @@ type pageState struct {
 	*pageCommon
 }
 
+func (p *pageState) reusePageOutputContent() bool {
+	return p.pageOutputTemplateVariationsState.Load() == 1
+}
+
 func (p *pageState) Err() error {
 	return nil
 }
@@ -394,56 +402,6 @@ func (ps *pageState) initCommonProviders(pp pagePaths) error {
 	return nil
 }
 
-func (p *pageState) createRenderHooks(f output.Format) (hooks.Renderers, error) {
-	layoutDescriptor := p.getLayoutDescriptor()
-	layoutDescriptor.RenderingHook = true
-	layoutDescriptor.LayoutOverride = false
-	layoutDescriptor.Layout = ""
-
-	var renderers hooks.Renderers
-
-	layoutDescriptor.Kind = "render-link"
-	templ, templFound, err := p.s.Tmpl().LookupLayout(layoutDescriptor, f)
-	if err != nil {
-		return renderers, err
-	}
-	if templFound {
-		renderers.LinkRenderer = hookRenderer{
-			templateHandler: p.s.Tmpl(),
-			SearchProvider:  templ.(identity.SearchProvider),
-			templ:           templ,
-		}
-	}
-
-	layoutDescriptor.Kind = "render-image"
-	templ, templFound, err = p.s.Tmpl().LookupLayout(layoutDescriptor, f)
-	if err != nil {
-		return renderers, err
-	}
-	if templFound {
-		renderers.ImageRenderer = hookRenderer{
-			templateHandler: p.s.Tmpl(),
-			SearchProvider:  templ.(identity.SearchProvider),
-			templ:           templ,
-		}
-	}
-
-	layoutDescriptor.Kind = "render-heading"
-	templ, templFound, err = p.s.Tmpl().LookupLayout(layoutDescriptor, f)
-	if err != nil {
-		return renderers, err
-	}
-	if templFound {
-		renderers.HeadingRenderer = hookRenderer{
-			templateHandler: p.s.Tmpl(),
-			SearchProvider:  templ.(identity.SearchProvider),
-			templ:           templ,
-		}
-	}
-
-	return renderers, nil
-}
-
 func (p *pageState) getLayoutDescriptor() output.LayoutDescriptor {
 	p.layoutDescriptorInit.Do(func() {
 		var section string
@@ -867,7 +825,7 @@ func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error {
 
 	if isRenderingSite {
 		cp := p.pageOutput.cp
-		if cp == nil {
+		if cp == nil && p.reusePageOutputContent() {
 			// Look for content to reuse.
 			for i := 0; i < len(p.pageOutputs); i++ {
 				if i == idx {
@@ -875,7 +833,7 @@ func (p *pageState) shiftToOutputFormat(isRenderingSite bool, idx int) error {
 				}
 				po := p.pageOutputs[i]
 
-				if po.cp != nil && po.cp.reuse {
+				if po.cp != nil {
 					cp = po.cp
 					break
 				}
diff --git a/hugolib/page__new.go b/hugolib/page__new.go
index 918477843..b8395d5ca 100644
--- a/hugolib/page__new.go
+++ b/hugolib/page__new.go
@@ -17,6 +17,8 @@ import (
 	"html/template"
 	"strings"
 
+	"go.uber.org/atomic"
+
 	"github.com/gohugoio/hugo/common/hugo"
 
 	"github.com/gohugoio/hugo/common/maps"
@@ -36,7 +38,8 @@ func newPageBase(metaProvider *pageMeta) (*pageState, error) {
 	s := metaProvider.s
 
 	ps := &pageState{
-		pageOutput: nopPageOutput,
+		pageOutput:                        nopPageOutput,
+		pageOutputTemplateVariationsState: atomic.NewUint32(0),
 		pageCommon: &pageCommon{
 			FileProvider:            metaProvider,
 			AuthorProvider:          metaProvider,
diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go
index bd4e35a5b..29beb672e 100644
--- a/hugolib/page__per_output.go
+++ b/hugolib/page__per_output.go
@@ -32,6 +32,7 @@ import (
 
 	"github.com/gohugoio/hugo/markup/converter"
 
+	"github.com/alecthomas/chroma/lexers"
 	"github.com/gohugoio/hugo/lazy"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
@@ -109,16 +110,8 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 			return err
 		}
 
-		enableReuse := !(hasShortcodeVariants || cp.renderHooksHaveVariants)
-
-		if enableReuse {
-			// Reuse this for the other output formats.
-			// We may improve on this, but we really want to avoid re-rendering the content
-			// to all output formats.
-			// The current rule is that if you need output format-aware shortcodes or
-			// content rendering hooks, create a output format-specific template, e.g.
-			// myshortcode.amp.html.
-			cp.enableReuse()
+		if hasShortcodeVariants {
+			p.pageOutputTemplateVariationsState.Store(2)
 		}
 
 		cp.workContent = p.contentToRender(cp.contentPlaceholders)
@@ -199,19 +192,10 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 		return nil
 	}
 
-	// Recursive loops can only happen in content files with template code (shortcodes etc.)
-	// Avoid creating new goroutines if we don't have to.
-	needTimeout := p.shortcodeState.hasShortcodes() || cp.renderHooks != nil
-
-	if needTimeout {
-		cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) {
-			return nil, initContent()
-		})
-	} else {
-		cp.initMain = parent.Branch(func() (interface{}, error) {
-			return nil, initContent()
-		})
-	}
+	// There may be recursive loops in shortcodes and render hooks.
+	cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (interface{}, error) {
+		return nil, initContent()
+	})
 
 	cp.initPlain = cp.initMain.Branch(func() (interface{}, error) {
 		cp.plain = helpers.StripHTML(string(cp.content))
@@ -229,18 +213,14 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 }
 
 type renderHooks struct {
-	hooks hooks.Renderers
-	init  sync.Once
+	getRenderer hooks.GetRendererFunc
+	init        sync.Once
 }
 
 // pageContentOutput represents the Page content for a given output format.
 type pageContentOutput struct {
 	f output.Format
 
-	// If we can reuse this for other output formats.
-	reuse     bool
-	reuseInit sync.Once
-
 	p *pageState
 
 	// Lazy load dependencies
@@ -250,13 +230,9 @@ type pageContentOutput struct {
 	placeholdersEnabled     bool
 	placeholdersEnabledInit sync.Once
 
+	// Renders Markdown hooks.
 	renderHooks *renderHooks
 
-	// Set if there are more than one output format variant
-	renderHooksHaveVariants bool // TODO(bep) reimplement this in another way, consolidate with shortcodes
-
-	// Content state
-
 	workContent       []byte
 	dependencyTracker identity.Manager // Set in server mode.
 
@@ -440,55 +416,107 @@ func (p *pageContentOutput) initRenderHooks() error {
 		return nil
 	}
 
-	var initErr error
-
 	p.renderHooks.init.Do(func() {
-		ps := p.p
-
-		c := ps.getContentConverter()
-		if c == nil || !c.Supports(converter.FeatureRenderHooks) {
-			return
+		if p.p.pageOutputTemplateVariationsState.Load() == 0 {
+			p.p.pageOutputTemplateVariationsState.Store(1)
 		}
 
-		h, err := ps.createRenderHooks(p.f)
-		if err != nil {
-			initErr = err
-			return
+		type cacheKey struct {
+			tp hooks.RendererType
+			id interface{}
+			f  output.Format
 		}
-		p.renderHooks.hooks = h
-
-		if !p.renderHooksHaveVariants || h.IsZero() {
-			// Check if there is a different render hooks template
-			// for any of the other page output formats.
-			// If not, we can reuse this.
-			for _, po := range ps.pageOutputs {
-				if po.f.Name != p.f.Name {
-					h2, err := ps.createRenderHooks(po.f)
-					if err != nil {
-						initErr = err
-						return
-					}
 
-					if h2.IsZero() {
-						continue
-					}
+		renderCache := make(map[cacheKey]interface{})
+		var renderCacheMu sync.Mutex
+
+		p.renderHooks.getRenderer = func(tp hooks.RendererType, id interface{}) interface{} {
+			renderCacheMu.Lock()
+			defer renderCacheMu.Unlock()
+
+			key := cacheKey{tp: tp, id: id, f: p.f}
+			if r, ok := renderCache[key]; ok {
+				return r
+			}
 
-					if p.renderHooks.hooks.IsZero() {
-						p.renderHooks.hooks = h2
+			layoutDescriptor := p.p.getLayoutDescriptor()
+			layoutDescriptor.RenderingHook = true
+			layoutDescriptor.LayoutOverride = false
+			layoutDescriptor.Layout = ""
+
+			switch tp {
+			case hooks.LinkRendererType:
+				layoutDescriptor.Kind = "render-link"
+			case hooks.ImageRendererType:
+				layoutDescriptor.Kind = "render-image"
+			case hooks.HeadingRendererType:
+				layoutDescriptor.Kind = "render-heading"
+			case hooks.CodeBlockRendererType:
+				layoutDescriptor.Kind = "render-codeblock"
+				if id != nil {
+					lang := id.(string)
+					lexer := lexers.Get(lang)
+					if lexer != nil {
+						layoutDescriptor.KindVariants = strings.Join(lexer.Config().Aliases, ",")
+					} else {
+						layoutDescriptor.KindVariants = lang
 					}
+				}
+			}
 
-					p.renderHooksHaveVariants = !h2.Eq(p.renderHooks.hooks)
+			getHookTemplate := func(f output.Format) (tpl.Template, bool) {
+				templ, found, err := p.p.s.Tmpl().LookupLayout(layoutDescriptor, f)
+				if err != nil {
+					panic(err)
+				}
+				return templ, found
+			}
+
+			templ, found1 := getHookTemplate(p.f)
 
-					if p.renderHooksHaveVariants {
-						break
+			if p.p.reusePageOutputContent() {
+				// Check if some of the other output formats would give a different template.
+				for _, f := range p.p.s.renderFormats {
+					if f.Name == p.f.Name {
+						continue
+					}
+					templ2, found2 := getHookTemplate(f)
+					if found2 {
+						if !found1 {
+							templ = templ2
+							found1 = true
+							break
+						}
+
+						if templ != templ2 {
+							p.p.pageOutputTemplateVariationsState.Store(2)
+							break
+						}
 					}
+				}
+			}
 
+			if !found1 {
+				if tp == hooks.CodeBlockRendererType {
+					// No user provided tempplate for code blocks, so we use the native Go code version -- which is also faster.
+					r := p.p.s.ContentSpec.Converters.GetHighlighter()
+					renderCache[key] = r
+					return r
 				}
+				return nil
 			}
+
+			r := hookRendererTemplate{
+				templateHandler: p.p.s.Tmpl(),
+				SearchProvider:  templ.(identity.SearchProvider),
+				templ:           templ,
+			}
+			renderCache[key] = r
+			return r
 		}
 	})
 
-	return initErr
+	return nil
 }
 
 func (p *pageContentOutput) setAutoSummary() error {
@@ -512,6 +540,9 @@ func (p *pageContentOutput) setAutoSummary() error {
 }
 
 func (cp *pageContentOutput) renderContent(content []byte, renderTOC bool) (converter.Result, error) {
+	if err := cp.initRenderHooks(); err != nil {
+		return nil, err
+	}
 	c := cp.p.getContentConverter()
 	return cp.renderContentWithConverter(c, content, renderTOC)
 }
@@ -521,7 +552,7 @@ func (cp *pageContentOutput) renderContentWithConverter(c converter.Converter, c
 		converter.RenderContext{
 			Src:         content,
 			RenderTOC:   renderTOC,
-			RenderHooks: cp.renderHooks.hooks,
+			GetRenderer: cp.renderHooks.getRenderer,
 		})
 
 	if err == nil {
@@ -570,12 +601,6 @@ func (p *pageContentOutput) enablePlaceholders() {
 	})
 }
 
-func (p *pageContentOutput) enableReuse() {
-	p.reuseInit.Do(func() {
-		p.reuse = true
-	})
-}
-
 // these will be shifted out when rendering a given output format.
 type pagePerOutputProviders interface {
 	targetPather
diff --git a/hugolib/page_test.go b/hugolib/page_test.go
index c281ad36c..04ca696c8 100644
--- a/hugolib/page_test.go
+++ b/hugolib/page_test.go
@@ -428,8 +428,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
 
 			assertFunc(t, e.ext, s.RegularPages())
 
-			home, err := s.Info.Home()
-			b.Assert(err, qt.IsNil)
+			home := s.Info.Home()
 			b.Assert(home, qt.Not(qt.IsNil))
 			b.Assert(home.File().Path(), qt.Equals, homePath)
 			b.Assert(content(home), qt.Contains, "Home Page Content")
@@ -1286,7 +1285,7 @@ func TestTranslationKey(t *testing.T) {
 
 	c.Assert(len(s.RegularPages()), qt.Equals, 2)
 
-	home, _ := s.Info.Home()
+	home := s.Info.Home()
 	c.Assert(home, qt.Not(qt.IsNil))
 	c.Assert(home.TranslationKey(), qt.Equals, "home")
 	c.Assert(s.RegularPages()[0].TranslationKey(), qt.Equals, "page/k1")
diff --git a/hugolib/pagebundler_test.go b/hugolib/pagebundler_test.go
index 1694b02ee..238c725bd 100644
--- a/hugolib/pagebundler_test.go
+++ b/hugolib/pagebundler_test.go
@@ -150,7 +150,7 @@ func TestPageBundlerSiteRegular(t *testing.T) {
 						c.Assert(leafBundle1.Section(), qt.Equals, "b")
 						sectionB := s.getPage(page.KindSection, "b")
 						c.Assert(sectionB, qt.Not(qt.IsNil))
-						home, _ := s.Info.Home()
+						home := s.Info.Home()
 						c.Assert(home.BundleType(), qt.Equals, files.ContentClassBranch)
 
 						// This is a root bundle and should live in the "home section"
@@ -290,7 +290,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) {
 
 				c.Assert(len(s.RegularPages()), qt.Equals, 8)
 				c.Assert(len(s.Pages()), qt.Equals, 16)
-				//dumpPages(s.AllPages()...)
+				// dumpPages(s.AllPages()...)
 
 				c.Assert(len(s.AllPages()), qt.Equals, 31)
 
diff --git a/hugolib/site.go b/hugolib/site.go
index 02380a6e7..57821ee93 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -30,6 +30,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/modules"
 	"golang.org/x/text/unicode/norm"
@@ -54,12 +55,11 @@ import (
 
 	"github.com/gohugoio/hugo/common/maps"
 
-	"github.com/pkg/errors"
-
 	"github.com/gohugoio/hugo/common/text"
 
 	"github.com/gohugoio/hugo/common/hugo"
 	"github.com/gohugoio/hugo/publisher"
+	"github.com/pkg/errors"
 	_errors "github.com/pkg/errors"
 
 	"github.com/gohugoio/hugo/langs"
@@ -1773,19 +1773,23 @@ var infoOnMissingLayout = map[string]bool{
 	"404": true,
 }
 
-// hookRenderer is the canonical implementation of all hooks.ITEMRenderer,
+// hookRendererTemplate is the canonical implementation of all hooks.ITEMRenderer,
 // where ITEM is the thing being hooked.
-type hookRenderer struct {
+type hookRendererTemplate struct {
 	templateHandler tpl.TemplateHandler
 	identity.SearchProvider
 	templ tpl.Template
 }
 
-func (hr hookRenderer) RenderLink(w io.Writer, ctx hooks.LinkContext) error {
+func (hr hookRendererTemplate) RenderLink(w io.Writer, ctx hooks.LinkContext) error {
+	return hr.templateHandler.Execute(hr.templ, w, ctx)
+}
+
+func (hr hookRendererTemplate) RenderHeading(w io.Writer, ctx hooks.HeadingContext) error {
 	return hr.templateHandler.Execute(hr.templ, w, ctx)
 }
 
-func (hr hookRenderer) RenderHeading(w io.Writer, ctx hooks.HeadingContext) error {
+func (hr hookRendererTemplate) RenderCodeblock(w hugio.FlexiWriter, ctx hooks.CodeblockContext) error {
 	return hr.templateHandler.Execute(hr.templ, w, ctx)
 }
 
diff --git a/hugolib/site_sections.go b/hugolib/site_sections.go
index ae343716e..50dfe6ffa 100644
--- a/hugolib/site_sections.go
+++ b/hugolib/site_sections.go
@@ -19,14 +19,10 @@ import (
 
 // Sections returns the top level sections.
 func (s *SiteInfo) Sections() page.Pages {
-	home, err := s.Home()
-	if err == nil {
-		return home.Sections()
-	}
-	return nil
+	return s.Home().Sections()
 }
 
 // Home is a shortcut to the home page, equivalent to .Site.GetPage "home".
-func (s *SiteInfo) Home() (page.Page, error) {
-	return s.s.home, nil
+func (s *SiteInfo) Home() page.Page {
+	return s.s.home
 }