Fix case issues with Params

There are currently several Params and case related issues floating around in Hugo.

This is very confusing for users and one of the most common support questions on the forum.

And while there have been done some great leg work in Viper etc., this is of limited value since this and similar doesn't work:

`Params.myCamelCasedParam`

Hugo has control over all the template method invocations, and can take care of all the lower-casing of the map lookup keys.

But that doesn't help with direct template lookups of type `Site.Params.TWITTER_CONFIG.USER_ID`.

This commit solves that by doing some carefully crafted modifications of the templates' AST -- lowercasing the params keys.

This is low-level work, but it's not like the template API wil change -- and this is important enough to defend such "bit fiddling".

Tests are added for all the template engines: Go templates, Ace and Amber.

Fixes #2615
Fixes #1129
Fixes #2590

3 files changed, 465 insertions, 9 deletions

diff --git a/tpl/template.go b/tpl/template.go
index 2d8ed2943..275ec6b8f 100644
--- a/tpl/template.go
+++ b/tpl/template.go
@@ -213,11 +213,16 @@ func (t *GoHTMLTemplate) AddInternalShortcode(name, content string) error {
 
 func (t *GoHTMLTemplate) AddTemplate(name, tpl string) error {
 	t.checkState()
-	_, err := t.New(name).Parse(tpl)
+	templ, err := t.New(name).Parse(tpl)
 	if err != nil {
 		t.errors = append(t.errors, &templateErr{name: name, err: err})
+		return err
 	}
-	return err
+	if err := applyTemplateTransformers(templ); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (t *GoHTMLTemplate) AddTemplateFileWithMaster(name, overlayFilename, masterFilename string) error {
@@ -264,7 +269,11 @@ func (t *GoHTMLTemplate) AddTemplateFileWithMaster(name, overlayFilename, master
 		// The extra lookup is a workaround, see
 		// * https://github.com/golang/go/issues/16101
 		// * https://github.com/spf13/hugo/issues/2549
-		t.overlays[name] = overlayTpl.Lookup(overlayTpl.Name())
+		overlayTpl = overlayTpl.Lookup(overlayTpl.Name())
+		if err := applyTemplateTransformers(overlayTpl); err != nil {
+			return err
+		}
+		t.overlays[name] = overlayTpl
 	}
 
 	return err
@@ -291,11 +300,12 @@ func (t *GoHTMLTemplate) AddAceTemplate(name, basePath, innerPath string, baseCo
 		t.errors = append(t.errors, &templateErr{name: name, err: err})
 		return err
 	}
-	_, err = ace.CompileResultWithTemplate(t.New(name), parsed, nil)
+	templ, err := ace.CompileResultWithTemplate(t.New(name), parsed, nil)
 	if err != nil {
 		t.errors = append(t.errors, &templateErr{name: name, err: err})
+		return err
 	}
-	return err
+	return applyTemplateTransformers(templ)
 }
 
 func (t *GoHTMLTemplate) AddTemplateFile(name, baseTemplatePath, path string) error {
@@ -317,9 +327,12 @@ func (t *GoHTMLTemplate) AddTemplateFile(name, baseTemplatePath, path string) er
 			return err
 		}
 
-		if _, err := compiler.CompileWithTemplate(t.New(templateName)); err != nil {
+		templ, err := compiler.CompileWithTemplate(t.New(templateName))
+		if err != nil {
 			return err
 		}
+
+		return applyTemplateTransformers(templ)
 	case ".ace":
 		var innerContent, baseContent []byte
 		innerContent, err := afero.ReadFile(hugofs.Source(), path)
@@ -353,8 +366,6 @@ func (t *GoHTMLTemplate) AddTemplateFile(name, baseTemplatePath, path string) er
 		return t.AddTemplate(name, string(b))
 	}
 
-	return nil
-
 }
 
 func (t *GoHTMLTemplate) GenerateTemplateNameFrom(base, path string) string {
@@ -467,7 +478,7 @@ func (t *GoHTMLTemplate) loadTemplates(absPath string, prefix string) {
 			}
 
 			if err := t.AddTemplateFile(tplName, baseTemplatePath, path); err != nil {
-				jww.ERROR.Printf("Failed to add template %s: %s", tplName, err)
+				jww.ERROR.Printf("Failed to add template %s in path %s: %s", tplName, path, err)
 			}
 
 		}
diff --git a/tpl/template_ast_transformers.go b/tpl/template_ast_transformers.go
new file mode 100644
index 000000000..105ecc13d
--- /dev/null
+++ b/tpl/template_ast_transformers.go
@@ -0,0 +1,227 @@
+// Copyright 2016 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tpl
+
+import (
+	"errors"
+	"html/template"
+	"strings"
+	"text/template/parse"
+)
+
+// decl keeps track of the variable mappings, i.e. $mysite => .Site etc.
+type decl map[string]string
+
+type templateContext struct {
+	decl  decl
+	templ *template.Template
+}
+
+func newTemplateContext(templ *template.Template) *templateContext {
+	return &templateContext{templ: templ, decl: make(map[string]string)}
+
+}
+
+func applyTemplateTransformers(templ *template.Template) error {
+	if templ == nil || templ.Tree == nil {
+		return errors.New("expected template, but none provided")
+	}
+
+	c := newTemplateContext(templ)
+
+	c.paramsKeysToLower(templ.Tree.Root)
+
+	return nil
+}
+
+// paramsKeysToLower is made purposely non-generic to make it not so tempting
+// to do more of these hard-to-maintain AST transformations.
+func (c *templateContext) paramsKeysToLower(n parse.Node) {
+
+	var nodes []parse.Node
+
+	switch x := n.(type) {
+	case *parse.ListNode:
+		if x != nil {
+			nodes = append(nodes, x.Nodes...)
+		}
+	case *parse.ActionNode:
+		nodes = append(nodes, x.Pipe)
+	case *parse.IfNode:
+		nodes = append(nodes, x.Pipe, x.List, x.ElseList)
+	case *parse.WithNode:
+		nodes = append(nodes, x.Pipe, x.List, x.ElseList)
+	case *parse.RangeNode:
+		nodes = append(nodes, x.Pipe, x.List, x.ElseList)
+	case *parse.TemplateNode:
+		subTempl := c.templ.Lookup(x.Name)
+		if subTempl != nil {
+			nodes = append(nodes, subTempl.Tree.Root)
+		}
+	case *parse.PipeNode:
+		for i, elem := range x.Decl {
+			if len(x.Cmds) > i {
+				// maps $site => .Site etc.
+				c.decl[elem.Ident[0]] = x.Cmds[i].String()
+			}
+		}
+		for _, c := range x.Cmds {
+			nodes = append(nodes, c)
+		}
+
+	case *parse.CommandNode:
+		for _, elem := range x.Args {
+			switch an := elem.(type) {
+			case *parse.FieldNode:
+				c.updateIdentsIfNeeded(an.Ident)
+			case *parse.VariableNode:
+				c.updateIdentsIfNeeded(an.Ident)
+			}
+		}
+	}
+
+	for _, n := range nodes {
+		c.paramsKeysToLower(n)
+	}
+}
+
+func (c *templateContext) updateIdentsIfNeeded(idents []string) {
+	index := c.decl.indexOfReplacementStart(idents)
+
+	if index == -1 {
+		return
+	}
+
+	for i := index; i < len(idents); i++ {
+		idents[i] = strings.ToLower(idents[i])
+	}
+}
+
+// indexOfReplacementStart will return the index of where to start doing replacement,
+// -1 if none needed.
+func (d decl) indexOfReplacementStart(idents []string) int {
+
+	if len(idents) == 0 {
+		return -1
+	}
+
+	var (
+		resolvedIdents []string
+		replacements   []string
+		replaced       []string
+	)
+
+	// An Ident can start out as one of
+	// [Params] [$blue] [$colors.Blue]
+	// We need to resolve the variables, so
+	// $blue => [Params Colors Blue]
+	// etc.
+	replacements = strings.Split(idents[0], ".")
+
+	// Loop until there are no more $vars to resolve.
+	for i := 0; i < len(replacements); i++ {
+
+		potentialVar := replacements[i]
+
+		if potentialVar == "$" {
+			continue
+		}
+
+		if potentialVar == "" || potentialVar[0] != '$' {
+			// leave it as is
+			replaced = append(replaced, strings.Split(potentialVar, ".")...)
+			continue
+		}
+
+		replacement, ok := d[potentialVar]
+
+		if !ok {
+			// Temporary range vars. We do not care about those.
+			return -1
+		}
+
+		replacement = strings.TrimPrefix(replacement, ".")
+
+		if replacement == "" {
+			continue
+		}
+
+		if replacement[0] == '$' {
+			// Needs further expansion
+			replacements = append(replacements, strings.Split(replacement, ".")...)
+		} else {
+			replaced = append(replaced, strings.Split(replacement, ".")...)
+		}
+	}
+
+	resolvedIdents = append(replaced, idents[1:]...)
+
+	paramsPaths := [][]string{
+		{"Params"},
+		{"Site", "Params"},
+
+		// Site and Pag referenced from shortcodes
+		{"Page", "Site", "Params"},
+		{"Page", "Params"},
+
+		{"Site", "Language", "Params"},
+	}
+
+	for _, paramPath := range paramsPaths {
+		if index := indexOfFirstRealIdentAfterWords(resolvedIdents, idents, paramPath...); index != -1 {
+			return index
+		}
+	}
+
+	return -1
+
+}
+
+func indexOfFirstRealIdentAfterWords(resolvedIdents, idents []string, words ...string) int {
+	if !sliceStartsWith(resolvedIdents, words...) {
+		return -1
+	}
+
+	for i, ident := range idents {
+		if ident == "" || ident[0] == '$' {
+			continue
+		}
+		found := true
+		for _, word := range words {
+			if ident == word {
+				found = false
+				break
+			}
+		}
+		if found {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func sliceStartsWith(slice []string, words ...string) bool {
+
+	if len(slice) < len(words) {
+		return false
+	}
+
+	for i, word := range words {
+		if word != slice[i] {
+			return false
+		}
+	}
+	return true
+}
diff --git a/tpl/template_ast_transformers_test.go b/tpl/template_ast_transformers_test.go
new file mode 100644
index 000000000..b62282b00
--- /dev/null
+++ b/tpl/template_ast_transformers_test.go
@@ -0,0 +1,218 @@
+// Copyright 2016 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package tpl
+
+import (
+	"bytes"
+	"testing"
+
+	"html/template"
+
+	jww "github.com/spf13/jwalterweatherman"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParamsKeysToLower(t *testing.T) {
+	var (
+		ctx = map[string]interface{}{
+			"Slice": []int{1, 3},
+			"Params": map[string]interface{}{
+				"lower": "P1L",
+			},
+			"Site": map[string]interface{}{
+				"Params": map[string]interface{}{
+					"lower": "P2L",
+					"slice": []int{1, 3},
+				},
+				"Language": map[string]interface{}{
+					"Params": map[string]interface{}{
+						"lower": "P22L",
+					},
+				},
+				"Data": map[string]interface{}{
+					"Params": map[string]interface{}{
+						"NOLOW": "P3H",
+					},
+				},
+			},
+		}
+
+		paramsTempl = `
+{{ $page := . }}
+{{ $pageParams := .Params }}
+{{ $site := .Site }}
+{{ $siteParams := .Site.Params }}
+{{ $data := .Site.Data }}
+
+P1: {{ .Params.LOWER }}
+P1_2: {{ $.Params.LOWER }}
+P1_3: {{ $page.Params.LOWER }}
+P1_4: {{ $pageParams.LOWER }}
+P2: {{ .Site.Params.LOWER }}
+P2_2: {{ $.Site.Params.LOWER }}
+P2_3: {{ $site.Params.LOWER }}
+P2_4: {{ $siteParams.LOWER }}
+P22: {{ .Site.Language.Params.LOWER }}
+P3: {{ .Site.Data.Params.NOLOW }}
+P3_2: {{ $.Site.Data.Params.NOLOW }}
+P3_3: {{ $site.Data.Params.NOLOW }}
+P3_4: {{ $data.Params.NOLOW }}
+P4: {{ range $i, $e := .Site.Params.SLICE }}{{ $e }}{{ end }}
+P5: {{ Echo .Params.LOWER }}
+P5_2: {{ Echo $site.Params.LOWER }}
+{{ if .Params.LOWER }}
+IF: {{ .Params.LOWER }}
+{{ end }}
+{{ if .Params.NOT_EXIST }}
+{{ else }}
+ELSE: {{ .Params.LOWER }}
+{{ end }}
+
+
+{{ with .Params.LOWER }}
+WITH: {{ . }}
+{{ end }}
+
+
+{{ range .Slice }}
+RANGE: {{ . }}: {{ $.Params.LOWER }}
+{{ end }}
+`
+	)
+
+	require.Error(t, applyTemplateTransformers(nil))
+
+	var funcs = map[string]interface{}{
+		"Echo": func(v interface{}) interface{} { return v },
+	}
+
+	templ, err := template.New("foo").Funcs(funcs).Parse(paramsTempl)
+
+	require.NoError(t, err)
+
+	c := newTemplateContext(templ)
+
+	require.Equal(t, -1, c.decl.indexOfReplacementStart([]string{}))
+
+	c.paramsKeysToLower(templ.Tree.Root)
+
+	var b bytes.Buffer
+
+	require.NoError(t, templ.Execute(&b, ctx))
+
+	result := b.String()
+
+	require.Contains(t, result, "P1: P1L")
+	require.Contains(t, result, "P1_2: P1L")
+	require.Contains(t, result, "P1_3: P1L")
+	require.Contains(t, result, "P1_4: P1L")
+	require.Contains(t, result, "P2: P2L")
+	require.Contains(t, result, "P2_2: P2L")
+	require.Contains(t, result, "P2_3: P2L")
+	require.Contains(t, result, "P2_4: P2L")
+	require.Contains(t, result, "P22: P22L")
+	require.Contains(t, result, "P3: P3H")
+	require.Contains(t, result, "P3_2: P3H")
+	require.Contains(t, result, "P3_3: P3H")
+	require.Contains(t, result, "P3_4: P3H")
+	require.Contains(t, result, "P4: 13")
+	require.Contains(t, result, "P5: P1L")
+	require.Contains(t, result, "P5_2: P2L")
+
+	require.Contains(t, result, "IF: P1L")
+	require.Contains(t, result, "ELSE: P1L")
+
+	require.Contains(t, result, "WITH: P1L")
+
+	require.Contains(t, result, "RANGE: 3: P1L")
+}
+
+func TestParamsKeysToLowerVars(t *testing.T) {
+	var (
+		ctx = map[string]interface{}{
+			"Params": map[string]interface{}{
+				"colors": map[string]interface{}{
+					"blue": "Amber",
+				},
+			},
+		}
+
+		// This is how Amber behaves:
+		paramsTempl = `
+{{$__amber_1 := .Params.Colors}}
+{{$__amber_2 := $__amber_1.Blue}}
+Color: {{$__amber_2}}
+Blue: {{ $__amber_1.Blue}}
+`
+	)
+
+	templ, err := template.New("foo").Parse(paramsTempl)
+
+	require.NoError(t, err)
+
+	c := newTemplateContext(templ)
+
+	c.paramsKeysToLower(templ.Tree.Root)
+
+	var b bytes.Buffer
+
+	require.NoError(t, templ.Execute(&b, ctx))
+
+	result := b.String()
+
+	require.Contains(t, result, "Color: Amber")
+
+}
+
+func TestParamsKeysToLowerInBlockTemplate(t *testing.T) {
+
+	var (
+		ctx = map[string]interface{}{
+			"Params": map[string]interface{}{
+				"lower": "P1L",
+			},
+		}
+
+		master = `
+P1: {{ .Params.LOWER }}
+{{ block "main" . }}DEFAULT{{ end }}`
+		overlay = `
+{{ define "main" }}
+P2: {{ .Params.LOWER }}
+{{ end }}`
+	)
+
+	masterTpl, err := template.New("foo").Parse(master)
+	require.NoError(t, err)
+
+	overlayTpl, err := template.Must(masterTpl.Clone()).Parse(overlay)
+	require.NoError(t, err)
+	overlayTpl = overlayTpl.Lookup(overlayTpl.Name())
+
+	c := newTemplateContext(overlayTpl)
+
+	c.paramsKeysToLower(overlayTpl.Tree.Root)
+
+	var b bytes.Buffer
+
+	require.NoError(t, overlayTpl.Execute(&b, ctx))
+
+	result := b.String()
+
+	require.Contains(t, result, "P1: P1L")
+	require.Contains(t, result, "P2: P1L")
+}
+
+func init() {
+	jww.SetStdoutThreshold(jww.LevelCritical)
+}