Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/gohugoio/hugo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/markup
diff options
context:
space:
mode:
authorBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2019-08-16 16:55:03 +0300
committerBjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>2019-11-06 21:09:08 +0300
commit5f6b6ec68936ebbbf590894c02a1a3ecad30735f (patch)
treef6c91e225a3f24f51af1bde5cfb5b88515d0665d /markup
parent366ee4d8da1c2b0c1751e9bf6d54638439735296 (diff)
Prepare for Goldmark
This commmit prepares for the addition of Goldmark as the new Markdown renderer in Hugo. This introduces a new `markup` package with some common interfaces and each implementation in its own package. See #5963
Diffstat (limited to 'markup')
-rw-r--r--markup/asciidoc/convert.go97
-rw-r--r--markup/asciidoc/convert_test.go38
-rw-r--r--markup/blackfriday/convert.go224
-rw-r--r--markup/blackfriday/convert_test.go194
-rw-r--r--markup/blackfriday/renderer.go85
-rw-r--r--markup/converter/converter.go83
-rw-r--r--markup/internal/blackfriday.go108
-rw-r--r--markup/internal/external.go52
-rw-r--r--markup/markup.go83
-rw-r--r--markup/markup_test.go41
-rw-r--r--markup/mmark/convert.go143
-rw-r--r--markup/mmark/convert_test.go77
-rw-r--r--markup/mmark/renderer.go44
-rw-r--r--markup/org/convert.go69
-rw-r--r--markup/org/convert_test.go35
-rw-r--r--markup/pandoc/convert.go76
-rw-r--r--markup/pandoc/convert_test.go38
-rw-r--r--markup/rst/convert.go109
-rw-r--r--markup/rst/convert_test.go38
19 files changed, 1634 insertions, 0 deletions
diff --git a/markup/asciidoc/convert.go b/markup/asciidoc/convert.go
new file mode 100644
index 000000000..9e63911d8
--- /dev/null
+++ b/markup/asciidoc/convert.go
@@ -0,0 +1,97 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package asciidoc converts Asciidoc to HTML using Asciidoc or Asciidoctor
+// external binaries.
+package asciidoc
+
+import (
+ "os/exec"
+
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ return &asciidocConverter{
+ ctx: ctx,
+ cfg: cfg,
+ }, nil
+ }
+ return n, nil
+}
+
+type asciidocConverter struct {
+ ctx converter.DocumentContext
+ cfg converter.ProviderConfig
+}
+
+func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ return converter.Bytes(a.getAsciidocContent(ctx.Src, a.ctx)), nil
+}
+
+// getAsciidocContent calls asciidoctor or asciidoc as an external helper
+// to convert AsciiDoc content to HTML.
+func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte {
+ var isAsciidoctor bool
+ path := getAsciidoctorExecPath()
+ if path == "" {
+ path = getAsciidocExecPath()
+ if path == "" {
+ a.cfg.Logger.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
+ " Leaving AsciiDoc content unrendered.")
+ return src
+ }
+ } else {
+ isAsciidoctor = true
+ }
+
+ a.cfg.Logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+ args := []string{"--no-header-footer", "--safe"}
+ if isAsciidoctor {
+ // asciidoctor-specific arg to show stack traces on errors
+ args = append(args, "--trace")
+ }
+ args = append(args, "-")
+ return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args)
+}
+
+func getAsciidocExecPath() string {
+ path, err := exec.LookPath("asciidoc")
+ if err != nil {
+ return ""
+ }
+ return path
+}
+
+func getAsciidoctorExecPath() string {
+ path, err := exec.LookPath("asciidoctor")
+ if err != nil {
+ return ""
+ }
+ return path
+}
+
+// Supports returns whether Asciidoc or Asciidoctor is installed on this computer.
+func Supports() bool {
+ return (getAsciidoctorExecPath() != "" ||
+ getAsciidocExecPath() != "")
+}
diff --git a/markup/asciidoc/convert_test.go b/markup/asciidoc/convert_test.go
new file mode 100644
index 000000000..1c53f4f25
--- /dev/null
+++ b/markup/asciidoc/convert_test.go
@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package asciidoc
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+ if !Supports() {
+ t.Skip("asciidoc/asciidoctor not installed")
+ }
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"paragraph\">\n<p>testContent</p>\n</div>\n")
+}
diff --git a/markup/blackfriday/convert.go b/markup/blackfriday/convert.go
new file mode 100644
index 000000000..f9d957a4e
--- /dev/null
+++ b/markup/blackfriday/convert.go
@@ -0,0 +1,224 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package blackfriday converts Markdown to HTML using Blackfriday v1.
+package blackfriday
+
+import (
+ "github.com/gohugoio/hugo/markup/converter"
+ "github.com/gohugoio/hugo/markup/internal"
+ "github.com/russross/blackfriday"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ defaultExtensions := getMarkdownExtensions(defaultBlackFriday)
+
+ pygmentsCodeFences := cfg.Cfg.GetBool("pygmentsCodeFences")
+ pygmentsCodeFencesGuessSyntax := cfg.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")
+ pygmentsOptions := cfg.Cfg.GetString("pygmentsOptions")
+
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ b := defaultBlackFriday
+ extensions := defaultExtensions
+
+ if ctx.ConfigOverrides != nil {
+ var err error
+ b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+ if err != nil {
+ return nil, err
+ }
+ extensions = getMarkdownExtensions(b)
+ }
+
+ return &blackfridayConverter{
+ ctx: ctx,
+ bf: b,
+ extensions: extensions,
+ cfg: cfg,
+
+ pygmentsCodeFences: pygmentsCodeFences,
+ pygmentsCodeFencesGuessSyntax: pygmentsCodeFencesGuessSyntax,
+ pygmentsOptions: pygmentsOptions,
+ }, nil
+ }
+
+ return n, nil
+
+}
+
+type blackfridayConverter struct {
+ ctx converter.DocumentContext
+ bf *internal.BlackFriday
+ extensions int
+
+ pygmentsCodeFences bool
+ pygmentsCodeFencesGuessSyntax bool
+ pygmentsOptions string
+
+ cfg converter.ProviderConfig
+}
+
+func (c *blackfridayConverter) AnchorSuffix() string {
+ if c.bf.PlainIDAnchors {
+ return ""
+ }
+ return ":" + c.ctx.DocumentID
+}
+
+func (c *blackfridayConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ r := c.getHTMLRenderer(ctx.RenderTOC)
+
+ return converter.Bytes(blackfriday.Markdown(ctx.Src, r, c.extensions)), nil
+
+}
+
+func (c *blackfridayConverter) getHTMLRenderer(renderTOC bool) blackfriday.Renderer {
+ flags := getFlags(renderTOC, c.bf)
+
+ documentID := c.ctx.DocumentID
+
+ renderParameters := blackfriday.HtmlRendererParameters{
+ FootnoteAnchorPrefix: c.bf.FootnoteAnchorPrefix,
+ FootnoteReturnLinkContents: c.bf.FootnoteReturnLinkContents,
+ }
+
+ if documentID != "" && !c.bf.PlainIDAnchors {
+ renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+ renderParameters.HeaderIDSuffix = ":" + documentID
+ }
+
+ return &hugoHTMLRenderer{
+ c: c,
+ Renderer: blackfriday.HtmlRendererWithParameters(flags, "", "", renderParameters),
+ }
+}
+
+func getFlags(renderTOC bool, cfg *internal.BlackFriday) int {
+
+ var flags int
+
+ if renderTOC {
+ flags = blackfriday.HTML_TOC
+ }
+
+ flags |= blackfriday.HTML_USE_XHTML
+ flags |= blackfriday.HTML_FOOTNOTE_RETURN_LINKS
+
+ if cfg.Smartypants {
+ flags |= blackfriday.HTML_USE_SMARTYPANTS
+ }
+
+ if cfg.SmartypantsQuotesNBSP {
+ flags |= blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP
+ }
+
+ if cfg.AngledQuotes {
+ flags |= blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES
+ }
+
+ if cfg.Fractions {
+ flags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
+ }
+
+ if cfg.HrefTargetBlank {
+ flags |= blackfriday.HTML_HREF_TARGET_BLANK
+ }
+
+ if cfg.NofollowLinks {
+ flags |= blackfriday.HTML_NOFOLLOW_LINKS
+ }
+
+ if cfg.NoreferrerLinks {
+ flags |= blackfriday.HTML_NOREFERRER_LINKS
+ }
+
+ if cfg.SmartDashes {
+ flags |= blackfriday.HTML_SMARTYPANTS_DASHES
+ }
+
+ if cfg.LatexDashes {
+ flags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
+ }
+
+ if cfg.SkipHTML {
+ flags |= blackfriday.HTML_SKIP_HTML
+ }
+
+ return flags
+}
+
+func getMarkdownExtensions(cfg *internal.BlackFriday) int {
+ // Default Blackfriday common extensions
+ commonExtensions := 0 |
+ blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
+ blackfriday.EXTENSION_TABLES |
+ blackfriday.EXTENSION_FENCED_CODE |
+ blackfriday.EXTENSION_AUTOLINK |
+ blackfriday.EXTENSION_STRIKETHROUGH |
+ blackfriday.EXTENSION_SPACE_HEADERS |
+ blackfriday.EXTENSION_HEADER_IDS |
+ blackfriday.EXTENSION_BACKSLASH_LINE_BREAK |
+ blackfriday.EXTENSION_DEFINITION_LISTS
+
+ // Extra Blackfriday extensions that Hugo enables by default
+ flags := commonExtensions |
+ blackfriday.EXTENSION_AUTO_HEADER_IDS |
+ blackfriday.EXTENSION_FOOTNOTES
+
+ for _, extension := range cfg.Extensions {
+ if flag, ok := blackfridayExtensionMap[extension]; ok {
+ flags |= flag
+ }
+ }
+ for _, extension := range cfg.ExtensionsMask {
+ if flag, ok := blackfridayExtensionMap[extension]; ok {
+ flags &= ^flag
+ }
+ }
+ return flags
+}
+
+var blackfridayExtensionMap = map[string]int{
+ "noIntraEmphasis": blackfriday.EXTENSION_NO_INTRA_EMPHASIS,
+ "tables": blackfriday.EXTENSION_TABLES,
+ "fencedCode": blackfriday.EXTENSION_FENCED_CODE,
+ "autolink": blackfriday.EXTENSION_AUTOLINK,
+ "strikethrough": blackfriday.EXTENSION_STRIKETHROUGH,
+ "laxHtmlBlocks": blackfriday.EXTENSION_LAX_HTML_BLOCKS,
+ "spaceHeaders": blackfriday.EXTENSION_SPACE_HEADERS,
+ "hardLineBreak": blackfriday.EXTENSION_HARD_LINE_BREAK,
+ "tabSizeEight": blackfriday.EXTENSION_TAB_SIZE_EIGHT,
+ "footnotes": blackfriday.EXTENSION_FOOTNOTES,
+ "noEmptyLineBeforeBlock": blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+ "headerIds": blackfriday.EXTENSION_HEADER_IDS,
+ "titleblock": blackfriday.EXTENSION_TITLEBLOCK,
+ "autoHeaderIds": blackfriday.EXTENSION_AUTO_HEADER_IDS,
+ "backslashLineBreak": blackfriday.EXTENSION_BACKSLASH_LINE_BREAK,
+ "definitionLists": blackfriday.EXTENSION_DEFINITION_LISTS,
+ "joinLines": blackfriday.EXTENSION_JOIN_LINES,
+}
+
+var (
+ _ converter.DocumentInfo = (*blackfridayConverter)(nil)
+)
diff --git a/markup/blackfriday/convert_test.go b/markup/blackfriday/convert_test.go
new file mode 100644
index 000000000..094edf35f
--- /dev/null
+++ b/markup/blackfriday/convert_test.go
@@ -0,0 +1,194 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blackfriday
+
+import (
+ "testing"
+
+ "github.com/spf13/viper"
+
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+ "github.com/russross/blackfriday"
+)
+
+func TestGetMarkdownExtensionsMasksAreRemovedFromExtensions(t *testing.T) {
+ c := qt.New(t)
+ b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+ c.Assert(err, qt.IsNil)
+
+ b.Extensions = []string{"headerId"}
+ b.ExtensionsMask = []string{"noIntraEmphasis"}
+
+ actualFlags := getMarkdownExtensions(b)
+ if actualFlags&blackfriday.EXTENSION_NO_INTRA_EMPHASIS == blackfriday.EXTENSION_NO_INTRA_EMPHASIS {
+ t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_NO_INTRA_EMPHASIS)
+ }
+}
+
+func TestGetMarkdownExtensionsByDefaultAllExtensionsAreEnabled(t *testing.T) {
+ type data struct {
+ testFlag int
+ }
+
+ c := qt.New(t)
+ b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+ c.Assert(err, qt.IsNil)
+
+ b.Extensions = []string{""}
+ b.ExtensionsMask = []string{""}
+ allExtensions := []data{
+ {blackfriday.EXTENSION_NO_INTRA_EMPHASIS},
+ {blackfriday.EXTENSION_TABLES},
+ {blackfriday.EXTENSION_FENCED_CODE},
+ {blackfriday.EXTENSION_AUTOLINK},
+ {blackfriday.EXTENSION_STRIKETHROUGH},
+ // {blackfriday.EXTENSION_LAX_HTML_BLOCKS},
+ {blackfriday.EXTENSION_SPACE_HEADERS},
+ // {blackfriday.EXTENSION_HARD_LINE_BREAK},
+ // {blackfriday.EXTENSION_TAB_SIZE_EIGHT},
+ {blackfriday.EXTENSION_FOOTNOTES},
+ // {blackfriday.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+ {blackfriday.EXTENSION_HEADER_IDS},
+ // {blackfriday.EXTENSION_TITLEBLOCK},
+ {blackfriday.EXTENSION_AUTO_HEADER_IDS},
+ {blackfriday.EXTENSION_BACKSLASH_LINE_BREAK},
+ {blackfriday.EXTENSION_DEFINITION_LISTS},
+ }
+
+ actualFlags := getMarkdownExtensions(b)
+ for _, e := range allExtensions {
+ if actualFlags&e.testFlag != e.testFlag {
+ t.Errorf("Flag %v was not found in the list of extensions.", e)
+ }
+ }
+}
+
+func TestGetMarkdownExtensionsAddingFlagsThroughRenderingContext(t *testing.T) {
+ c := qt.New(t)
+ b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+ c.Assert(err, qt.IsNil)
+
+ b.Extensions = []string{"definitionLists"}
+ b.ExtensionsMask = []string{""}
+
+ actualFlags := getMarkdownExtensions(b)
+ if actualFlags&blackfriday.EXTENSION_DEFINITION_LISTS != blackfriday.EXTENSION_DEFINITION_LISTS {
+ t.Errorf("Masked out flag {%v} found amongst returned extensions.", blackfriday.EXTENSION_DEFINITION_LISTS)
+ }
+}
+
+func TestGetFlags(t *testing.T) {
+ c := qt.New(t)
+ cfg := converter.ProviderConfig{Cfg: viper.New()}
+ b, err := internal.NewBlackfriday(cfg)
+ c.Assert(err, qt.IsNil)
+ flags := getFlags(false, b)
+ if flags&blackfriday.HTML_USE_XHTML != blackfriday.HTML_USE_XHTML {
+ t.Errorf("Test flag: %d was not found amongs set flags:%d; Result: %d", blackfriday.HTML_USE_XHTML, flags, flags&blackfriday.HTML_USE_XHTML)
+ }
+}
+
+func TestGetAllFlags(t *testing.T) {
+ c := qt.New(t)
+ cfg := converter.ProviderConfig{Cfg: viper.New()}
+ b, err := internal.NewBlackfriday(cfg)
+ c.Assert(err, qt.IsNil)
+
+ type data struct {
+ testFlag int
+ }
+
+ allFlags := []data{
+ {blackfriday.HTML_USE_XHTML},
+ {blackfriday.HTML_FOOTNOTE_RETURN_LINKS},
+ {blackfriday.HTML_USE_SMARTYPANTS},
+ {blackfriday.HTML_SMARTYPANTS_QUOTES_NBSP},
+ {blackfriday.HTML_SMARTYPANTS_ANGLED_QUOTES},
+ {blackfriday.HTML_SMARTYPANTS_FRACTIONS},
+ {blackfriday.HTML_HREF_TARGET_BLANK},
+ {blackfriday.HTML_NOFOLLOW_LINKS},
+ {blackfriday.HTML_NOREFERRER_LINKS},
+ {blackfriday.HTML_SMARTYPANTS_DASHES},
+ {blackfriday.HTML_SMARTYPANTS_LATEX_DASHES},
+ }
+
+ b.AngledQuotes = true
+ b.Fractions = true
+ b.HrefTargetBlank = true
+ b.NofollowLinks = true
+ b.NoreferrerLinks = true
+ b.LatexDashes = true
+ b.PlainIDAnchors = true
+ b.SmartDashes = true
+ b.Smartypants = true
+ b.SmartypantsQuotesNBSP = true
+
+ actualFlags := getFlags(false, b)
+
+ var expectedFlags int
+ //OR-ing flags together...
+ for _, d := range allFlags {
+ expectedFlags |= d.testFlag
+ }
+ if expectedFlags != actualFlags {
+ t.Errorf("Expected flags (%d) did not equal actual (%d) flags.", expectedFlags, actualFlags)
+ }
+}
+
+func TestConvert(t *testing.T) {
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{
+ Cfg: viper.New(),
+ })
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
+
+func TestGetHTMLRendererAnchors(t *testing.T) {
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{
+ Cfg: viper.New(),
+ })
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{
+ DocumentID: "testid",
+ ConfigOverrides: map[string]interface{}{
+ "plainIDAnchors": false,
+ "footnotes": true,
+ },
+ })
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte(`# Header
+
+This is a footnote.[^1] And then some.
+
+
+[^1]: Footnote text.
+
+`)})
+
+ c.Assert(err, qt.IsNil)
+ s := string(b.Bytes())
+ c.Assert(s, qt.Contains, "<h1 id=\"header:testid\">Header</h1>")
+ c.Assert(s, qt.Contains, "This is a footnote.<sup class=\"footnote-ref\" id=\"fnref:testid:1\"><a href=\"#fn:testid:1\">1</a></sup>")
+ c.Assert(s, qt.Contains, "<a class=\"footnote-return\" href=\"#fnref:testid:1\"><sup>[return]</sup></a>")
+}
diff --git a/markup/blackfriday/renderer.go b/markup/blackfriday/renderer.go
new file mode 100644
index 000000000..9f4d44e02
--- /dev/null
+++ b/markup/blackfriday/renderer.go
@@ -0,0 +1,85 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blackfriday
+
+import (
+ "bytes"
+ "strings"
+
+ "github.com/russross/blackfriday"
+)
+
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// adding some custom behaviour.
+type hugoHTMLRenderer struct {
+ c *blackfridayConverter
+ blackfriday.Renderer
+}
+
+// BlockCode renders a given text as a block of code.
+// Pygments is used if it is setup to handle code fences.
+func (r *hugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
+ if r.c.pygmentsCodeFences && (lang != "" || r.c.pygmentsCodeFencesGuessSyntax) {
+ opts := r.c.pygmentsOptions
+ str := strings.Trim(string(text), "\n\r")
+ highlighted, _ := r.c.cfg.Highlight(str, lang, opts)
+ out.WriteString(highlighted)
+ } else {
+ r.Renderer.BlockCode(out, text, lang)
+ }
+}
+
+// ListItem adds task list support to the Blackfriday renderer.
+func (r *hugoHTMLRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {
+ if !r.c.bf.TaskLists {
+ r.Renderer.ListItem(out, text, flags)
+ return
+ }
+
+ switch {
+ case bytes.HasPrefix(text, []byte("[ ] ")):
+ text = append([]byte(`<label><input type="checkbox" disabled class="task-list-item">`), text[3:]...)
+ text = append(text, []byte(`</label>`)...)
+
+ case bytes.HasPrefix(text, []byte("[x] ")) || bytes.HasPrefix(text, []byte("[X] ")):
+ text = append([]byte(`<label><input type="checkbox" checked disabled class="task-list-item">`), text[3:]...)
+ text = append(text, []byte(`</label>`)...)
+ }
+
+ r.Renderer.ListItem(out, text, flags)
+}
+
+// List adds task list support to the Blackfriday renderer.
+func (r *hugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
+ if !r.c.bf.TaskLists {
+ r.Renderer.List(out, text, flags)
+ return
+ }
+ marker := out.Len()
+ r.Renderer.List(out, text, flags)
+ if out.Len() > marker {
+ list := out.Bytes()[marker:]
+ if bytes.Contains(list, []byte("task-list-item")) {
+ // Find the index of the first >, it might be 3 or 4 depending on whether
+ // there is a new line at the start, but this is safer than just hardcoding it.
+ closingBracketIndex := bytes.Index(list, []byte(">"))
+ // Rewrite the buffer from the marker
+ out.Truncate(marker)
+ // Safely assuming closingBracketIndex won't be -1 since there is a list
+ // May be either dl, ul or ol
+ list := append(list[:closingBracketIndex], append([]byte(` class="task-list"`), list[closingBracketIndex:]...)...)
+ out.Write(list)
+ }
+ }
+}
diff --git a/markup/converter/converter.go b/markup/converter/converter.go
new file mode 100644
index 000000000..809efca8e
--- /dev/null
+++ b/markup/converter/converter.go
@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter
+
+import (
+ "github.com/gohugoio/hugo/common/loggers"
+ "github.com/gohugoio/hugo/config"
+ "github.com/spf13/afero"
+)
+
+// ProviderConfig configures a new Provider.
+type ProviderConfig struct {
+ Cfg config.Provider // Site config
+ ContentFs afero.Fs
+ Logger *loggers.Logger
+ Highlight func(code, lang, optsStr string) (string, error)
+}
+
+// NewProvider creates converter providers.
+type NewProvider interface {
+ New(cfg ProviderConfig) (Provider, error)
+}
+
+// Provider creates converters.
+type Provider interface {
+ New(ctx DocumentContext) (Converter, error)
+}
+
+// NewConverter is an adapter that can be used as a ConverterProvider.
+type NewConverter func(ctx DocumentContext) (Converter, error)
+
+// New creates a new Converter for the given ctx.
+func (n NewConverter) New(ctx DocumentContext) (Converter, error) {
+ return n(ctx)
+}
+
+// Converter wraps the Convert method that converts some markup into
+// another format, e.g. Markdown to HTML.
+type Converter interface {
+ Convert(ctx RenderContext) (Result, error)
+}
+
+// Result represents the minimum returned from Convert.
+type Result interface {
+ Bytes() []byte
+}
+
+// DocumentInfo holds additional information provided by some converters.
+type DocumentInfo interface {
+ AnchorSuffix() string
+}
+
+// Bytes holds a byte slice and implements the Result interface.
+type Bytes []byte
+
+// Bytes returns itself
+func (b Bytes) Bytes() []byte {
+ return b
+}
+
+// DocumentContext holds contextual information about the document to convert.
+type DocumentContext struct {
+ DocumentID string
+ DocumentName string
+ ConfigOverrides map[string]interface{}
+}
+
+// RenderContext holds contextual information about the content to render.
+type RenderContext struct {
+ Src []byte
+ RenderTOC bool
+}
diff --git a/markup/internal/blackfriday.go b/markup/internal/blackfriday.go
new file mode 100644
index 000000000..373df0c50
--- /dev/null
+++ b/markup/internal/blackfriday.go
@@ -0,0 +1,108 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package helpers implements general utility functions that work with
+// and on content. The helper functions defined here lay down the
+// foundation of how Hugo works with files and filepaths, and perform
+// string operations on content.
+
+package internal
+
+import (
+ "github.com/gohugoio/hugo/common/maps"
+ "github.com/gohugoio/hugo/markup/converter"
+ "github.com/mitchellh/mapstructure"
+ "github.com/pkg/errors"
+)
+
+// BlackFriday holds configuration values for BlackFriday rendering.
+// It is kept here because it's used in several packages.
+type BlackFriday struct {
+ Smartypants bool
+ SmartypantsQuotesNBSP bool
+ AngledQuotes bool
+ Fractions bool
+ HrefTargetBlank bool
+ NofollowLinks bool
+ NoreferrerLinks bool
+ SmartDashes bool
+ LatexDashes bool
+ TaskLists bool
+ PlainIDAnchors bool
+ Extensions []string
+ ExtensionsMask []string
+ SkipHTML bool
+
+ FootnoteAnchorPrefix string
+ FootnoteReturnLinkContents string
+}
+
+func UpdateBlackFriday(old *BlackFriday, m map[string]interface{}) (*BlackFriday, error) {
+ // Create a copy so we can modify it.
+ bf := *old
+ if err := mapstructure.Decode(m, &bf); err != nil {
+ return nil, errors.WithMessage(err, "failed to decode rendering config")
+ }
+ return &bf, nil
+}
+
+// NewBlackfriday creates a new Blackfriday filled with site config or some sane defaults.
+func NewBlackfriday(cfg converter.ProviderConfig) (*BlackFriday, error) {
+ var siteConfig map[string]interface{}
+ if cfg.Cfg != nil {
+ siteConfig = cfg.Cfg.GetStringMap("blackfriday")
+ }
+
+ defaultParam := map[string]interface{}{
+ "smartypants": true,
+ "angledQuotes": false,
+ "smartypantsQuotesNBSP": false,
+ "fractions": true,
+ "hrefTargetBlank": false,
+ "nofollowLinks": false,
+ "noreferrerLinks": false,
+ "smartDashes": true,
+ "latexDashes": true,
+ "plainIDAnchors": true,
+ "taskLists": true,
+ "skipHTML": false,
+ }
+
+ maps.ToLower(defaultParam)
+
+ config := make(map[string]interface{})
+
+ for k, v := range defaultParam {
+ config[k] = v
+ }
+
+ for k, v := range siteConfig {
+ config[k] = v
+ }
+
+ combinedConfig := &BlackFriday{}
+ if err := mapstructure.Decode(config, combinedConfig); err != nil {
+ return nil, errors.Errorf("failed to decode Blackfriday config: %s", err)
+ }
+
+ // TODO(bep) update/consolidate docs
+ if combinedConfig.FootnoteAnchorPrefix == "" {
+ combinedConfig.FootnoteAnchorPrefix = cfg.Cfg.GetString("footnoteAnchorPrefix")
+ }
+
+ if combinedConfig.FootnoteReturnLinkContents == "" {
+ combinedConfig.FootnoteReturnLinkContents = cfg.Cfg.GetString("footnoteReturnLinkContents")
+ }
+
+ return combinedConfig, nil
+}
diff --git a/markup/internal/external.go b/markup/internal/external.go
new file mode 100644
index 000000000..2105e7cff
--- /dev/null
+++ b/markup/internal/external.go
@@ -0,0 +1,52 @@
+package internal
+
+import (
+ "bytes"
+ "os/exec"
+ "strings"
+
+ "github.com/gohugoio/hugo/markup/converter"
+)
+
+func ExternallyRenderContent(
+ cfg converter.ProviderConfig,
+ ctx converter.DocumentContext,
+ content []byte, path string, args []string) []byte {
+
+ logger := cfg.Logger
+ cmd := exec.Command(path, args...)
+ cmd.Stdin = bytes.NewReader(content)
+ var out, cmderr bytes.Buffer
+ cmd.Stdout = &out
+ cmd.Stderr = &cmderr
+ err := cmd.Run()
+ // Most external helpers exit w/ non-zero exit code only if severe, i.e.
+ // halting errors occurred. -> log stderr output regardless of state of err
+ for _, item := range strings.Split(cmderr.String(), "\n") {
+ item := strings.TrimSpace(item)
+ if item != "" {
+ logger.ERROR.Printf("%s: %s", ctx.DocumentName, item)
+ }
+ }
+ if err != nil {
+ logger.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
+ }
+
+ return normalizeExternalHelperLineFeeds(out.Bytes())
+}
+
+// Strips carriage returns from third-party / external processes (useful for Windows)
+func normalizeExternalHelperLineFeeds(content []byte) []byte {
+ return bytes.Replace(content, []byte("\r"), []byte(""), -1)
+}
+
+func GetPythonExecPath() string {
+ path, err := exec.LookPath("python")
+ if err != nil {
+ path, err = exec.LookPath("python.exe")
+ if err != nil {
+ return ""
+ }
+ }
+ return path
+}
diff --git a/markup/markup.go b/markup/markup.go
new file mode 100644
index 000000000..54193aba3
--- /dev/null
+++ b/markup/markup.go
@@ -0,0 +1,83 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+ "strings"
+
+ "github.com/gohugoio/hugo/markup/org"
+
+ "github.com/gohugoio/hugo/markup/asciidoc"
+ "github.com/gohugoio/hugo/markup/blackfriday"
+ "github.com/gohugoio/hugo/markup/converter"
+ "github.com/gohugoio/hugo/markup/mmark"
+ "github.com/gohugoio/hugo/markup/pandoc"
+ "github.com/gohugoio/hugo/markup/rst"
+)
+
+func NewConverterProvider(cfg converter.ProviderConfig) (ConverterProvider, error) {
+ converters := make(map[string]converter.Provider)
+
+ add := func(p converter.NewProvider, aliases ...string) error {
+ c, err := p.New(cfg)
+ if err != nil {
+ return err
+ }
+ addConverter(converters, c, aliases...)
+ return nil
+ }
+
+ if err := add(blackfriday.Provider, "md", "markdown", "blackfriday"); err != nil {
+ return nil, err
+ }
+ if err := add(mmark.Provider, "mmark"); err != nil {
+ return nil, err
+ }
+ if err := add(asciidoc.Provider, "asciidoc"); err != nil {
+ return nil, err
+ }
+ if err := add(rst.Provider, "rst"); err != nil {
+ return nil, err
+ }
+ if err := add(pandoc.Provider, "pandoc"); err != nil {
+ return nil, err
+ }
+ if err := add(org.Provider, "org"); err != nil {
+ return nil, err
+ }
+
+ return &converterRegistry{converters: converters}, nil
+}
+
+type ConverterProvider interface {
+ Get(name string) converter.Provider
+}
+
+type converterRegistry struct {
+ // Maps name (md, markdown, blackfriday etc.) to a converter provider.
+ // Note that this is also used for aliasing, so the same converter
+ // may be registered multiple times.
+ // All names are lower case.
+ converters map[string]converter.Provider
+}
+
+func (r *converterRegistry) Get(name string) converter.Provider {
+ return r.converters[strings.ToLower(name)]
+}
+
+func addConverter(m map[string]converter.Provider, c converter.Provider, aliases ...string) {
+ for _, alias := range aliases {
+ m[alias] = c
+ }
+}
diff --git a/markup/markup_test.go b/markup/markup_test.go
new file mode 100644
index 000000000..c4c1ee032
--- /dev/null
+++ b/markup/markup_test.go
@@ -0,0 +1,41 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package markup
+
+import (
+ "testing"
+
+ "github.com/spf13/viper"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestConverterRegistry(t *testing.T) {
+ c := qt.New(t)
+
+ r, err := NewConverterProvider(converter.ProviderConfig{Cfg: viper.New()})
+
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(r.Get("foo"), qt.IsNil)
+ c.Assert(r.Get("markdown"), qt.Not(qt.IsNil))
+ c.Assert(r.Get("mmark"), qt.Not(qt.IsNil))
+ c.Assert(r.Get("asciidoc"), qt.Not(qt.IsNil))
+ c.Assert(r.Get("rst"), qt.Not(qt.IsNil))
+ c.Assert(r.Get("pandoc"), qt.Not(qt.IsNil))
+ c.Assert(r.Get("org"), qt.Not(qt.IsNil))
+
+}
diff --git a/markup/mmark/convert.go b/markup/mmark/convert.go
new file mode 100644
index 000000000..a0da346c1
--- /dev/null
+++ b/markup/mmark/convert.go
@@ -0,0 +1,143 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package mmark converts Markdown to HTML using MMark v1.
+package mmark
+
+import (
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+ "github.com/miekg/mmark"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ defaultBlackFriday, err := internal.NewBlackfriday(cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ defaultExtensions := getMmarkExtensions(defaultBlackFriday)
+
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ b := defaultBlackFriday
+ extensions := defaultExtensions
+
+ if ctx.ConfigOverrides != nil {
+ var err error
+ b, err = internal.UpdateBlackFriday(b, ctx.ConfigOverrides)
+ if err != nil {
+ return nil, err
+ }
+ extensions = getMmarkExtensions(b)
+ }
+
+ return &mmarkConverter{
+ ctx: ctx,
+ b: b,
+ extensions: extensions,
+ cfg: cfg,
+ }, nil
+ }
+
+ return n, nil
+
+}
+
+type mmarkConverter struct {
+ ctx converter.DocumentContext
+ extensions int
+ b *internal.BlackFriday
+ cfg converter.ProviderConfig
+}
+
+func (c *mmarkConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ r := getHTMLRenderer(c.ctx, c.b, c.cfg)
+ return mmark.Parse(ctx.Src, r, c.extensions), nil
+}
+
+func getHTMLRenderer(
+ ctx converter.DocumentContext,
+ cfg *internal.BlackFriday,
+ pcfg converter.ProviderConfig) mmark.Renderer {
+
+ var (
+ flags int
+ documentID string
+ )
+
+ documentID = ctx.DocumentID
+
+ renderParameters := mmark.HtmlRendererParameters{
+ FootnoteAnchorPrefix: cfg.FootnoteAnchorPrefix,
+ FootnoteReturnLinkContents: cfg.FootnoteReturnLinkContents,
+ }
+
+ if documentID != "" && !cfg.PlainIDAnchors {
+ renderParameters.FootnoteAnchorPrefix = documentID + ":" + renderParameters.FootnoteAnchorPrefix
+ }
+
+ htmlFlags := flags
+ htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
+
+ return &mmarkRenderer{
+ Config: cfg,
+ Cfg: pcfg.Cfg,
+ highlight: pcfg.Highlight,
+ Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
+ }
+
+}
+
+func getMmarkExtensions(cfg *internal.BlackFriday) int {
+ flags := 0
+ flags |= mmark.EXTENSION_TABLES
+ flags |= mmark.EXTENSION_FENCED_CODE
+ flags |= mmark.EXTENSION_AUTOLINK
+ flags |= mmark.EXTENSION_SPACE_HEADERS
+ flags |= mmark.EXTENSION_CITATION
+ flags |= mmark.EXTENSION_TITLEBLOCK_TOML
+ flags |= mmark.EXTENSION_HEADER_IDS
+ flags |= mmark.EXTENSION_AUTO_HEADER_IDS
+ flags |= mmark.EXTENSION_UNIQUE_HEADER_IDS
+ flags |= mmark.EXTENSION_FOOTNOTES
+ flags |= mmark.EXTENSION_SHORT_REF
+ flags |= mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK
+ flags |= mmark.EXTENSION_INCLUDE
+
+ for _, extension := range cfg.Extensions {
+ if flag, ok := mmarkExtensionMap[extension]; ok {
+ flags |= flag
+ }
+ }
+ return flags
+}
+
+var mmarkExtensionMap = map[string]int{
+ "tables": mmark.EXTENSION_TABLES,
+ "fencedCode": mmark.EXTENSION_FENCED_CODE,
+ "autolink": mmark.EXTENSION_AUTOLINK,
+ "laxHtmlBlocks": mmark.EXTENSION_LAX_HTML_BLOCKS,
+ "spaceHeaders": mmark.EXTENSION_SPACE_HEADERS,
+ "hardLineBreak": mmark.EXTENSION_HARD_LINE_BREAK,
+ "footnotes": mmark.EXTENSION_FOOTNOTES,
+ "noEmptyLineBeforeBlock": mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK,
+ "headerIds": mmark.EXTENSION_HEADER_IDS,
+ "autoHeaderIds": mmark.EXTENSION_AUTO_HEADER_IDS,
+}
diff --git a/markup/mmark/convert_test.go b/markup/mmark/convert_test.go
new file mode 100644
index 000000000..d015ee94c
--- /dev/null
+++ b/markup/mmark/convert_test.go
@@ -0,0 +1,77 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+ "testing"
+
+ "github.com/spf13/viper"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/miekg/mmark"
+
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestGetMmarkExtensions(t *testing.T) {
+ c := qt.New(t)
+ b, err := internal.NewBlackfriday(converter.ProviderConfig{Cfg: viper.New()})
+ c.Assert(err, qt.IsNil)
+
+ //TODO: This is doing the same just with different marks...
+ type data struct {
+ testFlag int
+ }
+
+ b.Extensions = []string{"tables"}
+ b.ExtensionsMask = []string{""}
+ allExtensions := []data{
+ {mmark.EXTENSION_TABLES},
+ {mmark.EXTENSION_FENCED_CODE},
+ {mmark.EXTENSION_AUTOLINK},
+ {mmark.EXTENSION_SPACE_HEADERS},
+ {mmark.EXTENSION_CITATION},
+ {mmark.EXTENSION_TITLEBLOCK_TOML},
+ {mmark.EXTENSION_HEADER_IDS},
+ {mmark.EXTENSION_AUTO_HEADER_IDS},
+ {mmark.EXTENSION_UNIQUE_HEADER_IDS},
+ {mmark.EXTENSION_FOOTNOTES},
+ {mmark.EXTENSION_SHORT_REF},
+ {mmark.EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK},
+ {mmark.EXTENSION_INCLUDE},
+ }
+
+ actualFlags := getMmarkExtensions(b)
+ for _, e := range allExtensions {
+ if actualFlags&e.testFlag != e.testFlag {
+ t.Errorf("Flag %v was not found in the list of extensions.", e)
+ }
+ }
+}
+
+func TestConvert(t *testing.T) {
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{Cfg: viper.New(), Logger: loggers.NewErrorLogger()})
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
diff --git a/markup/mmark/renderer.go b/markup/mmark/renderer.go
new file mode 100644
index 000000000..07fe71c95
--- /dev/null
+++ b/markup/mmark/renderer.go
@@ -0,0 +1,44 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mmark
+
+import (
+ "bytes"
+ "strings"
+
+ "github.com/miekg/mmark"
+
+ "github.com/gohugoio/hugo/config"
+ "github.com/gohugoio/hugo/markup/internal"
+)
+
+// hugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
+// adding some custom behaviour.
+type mmarkRenderer struct {
+ Cfg config.Provider
+ Config *internal.BlackFriday
+ highlight func(code, lang, optsStr string) (string, error)
+ mmark.Renderer
+}
+
+// BlockCode renders a given text as a block of code.
+func (r *mmarkRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
+ if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
+ str := strings.Trim(string(text), "\n\r")
+ highlighted, _ := r.highlight(str, lang, "")
+ out.WriteString(highlighted)
+ } else {
+ r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
+ }
+}
diff --git a/markup/org/convert.go b/markup/org/convert.go
new file mode 100644
index 000000000..a951e6fe1
--- /dev/null
+++ b/markup/org/convert.go
@@ -0,0 +1,69 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package org converts Emacs Org-Mode to HTML.
+package org
+
+import (
+ "bytes"
+
+ "github.com/gohugoio/hugo/markup/converter"
+ "github.com/niklasfasching/go-org/org"
+ "github.com/spf13/afero"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provide{}
+
+type provide struct {
+}
+
+func (p provide) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ return &orgConverter{
+ ctx: ctx,
+ cfg: cfg,
+ }, nil
+ }
+ return n, nil
+}
+
+type orgConverter struct {
+ ctx converter.DocumentContext
+ cfg converter.ProviderConfig
+}
+
+func (c *orgConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ logger := c.cfg.Logger
+ config := org.New()
+ config.Log = logger.WARN
+ config.ReadFile = func(filename string) ([]byte, error) {
+ return afero.ReadFile(c.cfg.ContentFs, filename)
+ }
+ writer := org.NewHTMLWriter()
+ writer.HighlightCodeBlock = func(source, lang string) string {
+ highlightedSource, err := c.cfg.Highlight(source, lang, "")
+ if err != nil {
+ logger.ERROR.Printf("Could not highlight source as lang %s. Using raw source.", lang)
+ return source
+ }
+ return highlightedSource
+ }
+
+ html, err := config.Parse(bytes.NewReader(ctx.Src), c.ctx.DocumentName).Write(writer)
+ if err != nil {
+ logger.ERROR.Printf("Could not render org: %s. Using unrendered content.", err)
+ return converter.Bytes(ctx.Src), nil
+ }
+ return converter.Bytes([]byte(html)), nil
+}
diff --git a/markup/org/convert_test.go b/markup/org/convert_test.go
new file mode 100644
index 000000000..94fcdf836
--- /dev/null
+++ b/markup/org/convert_test.go
@@ -0,0 +1,35 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<p>\ntestContent\n</p>\n")
+}
diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go
new file mode 100644
index 000000000..4deab0b46
--- /dev/null
+++ b/markup/pandoc/convert.go
@@ -0,0 +1,76 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package pandoc converts content to HTML using Pandoc as an external helper.
+package pandoc
+
+import (
+ "os/exec"
+
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ return &pandocConverter{
+ ctx: ctx,
+ cfg: cfg,
+ }, nil
+ }
+ return n, nil
+
+}
+
+type pandocConverter struct {
+ ctx converter.DocumentContext
+ cfg converter.ProviderConfig
+}
+
+func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil
+}
+
+// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
+func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte {
+ logger := c.cfg.Logger
+ path := getPandocExecPath()
+ if path == "" {
+ logger.ERROR.Println("pandoc not found in $PATH: Please install.\n",
+ " Leaving pandoc content unrendered.")
+ return src
+ }
+ args := []string{"--mathjax"}
+ return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+}
+
+func getPandocExecPath() string {
+ path, err := exec.LookPath("pandoc")
+ if err != nil {
+ return ""
+ }
+
+ return path
+}
+
+// Supports returns whether Pandoc is installed on this computer.
+func Supports() bool {
+ return getPandocExecPath() != ""
+}
diff --git a/markup/pandoc/convert_test.go b/markup/pandoc/convert_test.go
new file mode 100644
index 000000000..bd6ca19e6
--- /dev/null
+++ b/markup/pandoc/convert_test.go
@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pandoc
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+ if !Supports() {
+ t.Skip("pandoc not installed")
+ }
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
diff --git a/markup/rst/convert.go b/markup/rst/convert.go
new file mode 100644
index 000000000..e12e34f6d
--- /dev/null
+++ b/markup/rst/convert.go
@@ -0,0 +1,109 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package rst converts content to HTML using the RST external helper.
+package rst
+
+import (
+ "bytes"
+ "os/exec"
+ "runtime"
+
+ "github.com/gohugoio/hugo/markup/internal"
+
+ "github.com/gohugoio/hugo/markup/converter"
+)
+
+// Provider is the package entry point.
+var Provider converter.NewProvider = provider{}
+
+type provider struct {
+}
+
+func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) {
+ var n converter.NewConverter = func(ctx converter.DocumentContext) (converter.Converter, error) {
+ return &rstConverter{
+ ctx: ctx,
+ cfg: cfg,
+ }, nil
+ }
+ return n, nil
+
+}
+
+type rstConverter struct {
+ ctx converter.DocumentContext
+ cfg converter.ProviderConfig
+}
+
+func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
+ return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil
+}
+
+// getRstContent calls the Python script rst2html as an external helper
+// to convert reStructuredText content to HTML.
+func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte {
+ logger := c.cfg.Logger
+ path := getRstExecPath()
+
+ if path == "" {
+ logger.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
+ " Leaving reStructuredText content unrendered.")
+ return src
+ }
+ logger.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
+ var result []byte
+ // certain *nix based OSs wrap executables in scripted launchers
+ // invoking binaries on these OSs via python interpreter causes SyntaxError
+ // invoke directly so that shebangs work as expected
+ // handle Windows manually because it doesn't do shebangs
+ if runtime.GOOS == "windows" {
+ python := internal.GetPythonExecPath()
+ args := []string{path, "--leave-comments", "--initial-header-level=2"}
+ result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args)
+ } else {
+ args := []string{"--leave-comments", "--initial-header-level=2"}
+ result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
+ }
+ // TODO(bep) check if rst2html has a body only option.
+ bodyStart := bytes.Index(result, []byte("<body>\n"))
+ if bodyStart < 0 {
+ bodyStart = -7 //compensate for length
+ }
+
+ bodyEnd := bytes.Index(result, []byte("\n</body>"))
+ if bodyEnd < 0 || bodyEnd >= len(result) {
+ bodyEnd = len(result) - 1
+ if bodyEnd < 0 {
+ bodyEnd = 0
+ }
+ }
+
+ return result[bodyStart+7 : bodyEnd]
+}
+
+func getRstExecPath() string {
+ path, err := exec.LookPath("rst2html")
+ if err != nil {
+ path, err = exec.LookPath("rst2html.py")
+ if err != nil {
+ return ""
+ }
+ }
+ return path
+}
+
+// Supports returns whether rst is installed on this computer.
+func Supports() bool {
+ return getRstExecPath() != ""
+}
diff --git a/markup/rst/convert_test.go b/markup/rst/convert_test.go
new file mode 100644
index 000000000..269d92caa
--- /dev/null
+++ b/markup/rst/convert_test.go
@@ -0,0 +1,38 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rst
+
+import (
+ "testing"
+
+ "github.com/gohugoio/hugo/common/loggers"
+
+ "github.com/gohugoio/hugo/markup/converter"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestConvert(t *testing.T) {
+ if !Supports() {
+ t.Skip("rst not installed")
+ }
+ c := qt.New(t)
+ p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
+ c.Assert(err, qt.IsNil)
+ conv, err := p.New(converter.DocumentContext{})
+ c.Assert(err, qt.IsNil)
+ b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+ c.Assert(err, qt.IsNil)
+ c.Assert(string(b.Bytes()), qt.Equals, "<div class=\"document\">\n\n\n<p>testContent</p>\n</div>")
+}