parser/metadecoders: Consolidate the metadata decoders

See #5324

6 files changed, 432 insertions, 159 deletions

diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go
index 280361a84..e409b7665 100644
--- a/parser/metadecoders/decoder.go
+++ b/parser/metadecoders/decoder.go
@@ -15,81 +15,139 @@ package metadecoders
 
 import (
 	"encoding/json"
+	"fmt"
 
 	"github.com/BurntSushi/toml"
 	"github.com/chaseadamsio/goorgeous"
-	"github.com/gohugoio/hugo/parser/pageparser"
 	"github.com/pkg/errors"
+	"github.com/spf13/cast"
 	yaml "gopkg.in/yaml.v2"
 )
 
-type Format string
-
-const (
-	// These are the supported metdata  formats in Hugo. Most of these are also
-	// supported as /data formats.
-	ORG  Format = "org"
-	JSON Format = "json"
-	TOML Format = "toml"
-	YAML Format = "yaml"
-)
-
-// FormatFromFrontMatterType will return empty if not supported.
-func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
-	switch typ {
-	case pageparser.TypeFrontMatterJSON:
-		return JSON
-	case pageparser.TypeFrontMatterORG:
-		return ORG
-	case pageparser.TypeFrontMatterTOML:
-		return TOML
-	case pageparser.TypeFrontMatterYAML:
-		return YAML
-	default:
-		return ""
-	}
-}
-
 // UnmarshalToMap will unmarshall data in format f into a new map. This is
 // what's needed for Hugo's front matter decoding.
 func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
 	m := make(map[string]interface{})
-
 	if data == nil {
 		return m, nil
 	}
 
+	err := unmarshal(data, f, &m)
+
+	return m, err
+
+}
+
+// Unmarshal will unmarshall data in format f into an interface{}.
+// This is what's needed for Hugo's /data handling.
+func Unmarshal(data []byte, f Format) (interface{}, error) {
+	if data == nil {
+		return make(map[string]interface{}), nil
+	}
+	var v interface{}
+	err := unmarshal(data, f, &v)
+
+	return v, err
+}
+
+// unmarshal unmarshals data in format f into v.
+func unmarshal(data []byte, f Format, v interface{}) error {
+
 	var err error
 
 	switch f {
 	case ORG:
-		m, err = goorgeous.OrgHeaders(data)
+		vv, err := goorgeous.OrgHeaders(data)
+		if err != nil {
+			return err
+		}
+		switch v.(type) {
+		case *map[string]interface{}:
+			*v.(*map[string]interface{}) = vv
+		default:
+			*v.(*interface{}) = vv
+		}
 	case JSON:
-		err = json.Unmarshal(data, &m)
+		err = json.Unmarshal(data, v)
 	case TOML:
-		_, err = toml.Decode(string(data), &m)
+		err = toml.Unmarshal(data, v)
 	case YAML:
-		err = yaml.Unmarshal(data, &m)
+		err = yaml.Unmarshal(data, v)
 
-		// To support boolean keys, the `yaml` package unmarshals maps to
+		// To support boolean keys, the YAML package unmarshals maps to
 		// map[interface{}]interface{}. Here we recurse through the result
 		// and change all maps to map[string]interface{} like we would've
 		// gotten from `json`.
-		if err == nil {
-			for k, v := range m {
-				if vv, changed := stringifyMapKeys(v); changed {
-					m[k] = vv
-				}
+		var ptr interface{}
+		switch v.(type) {
+		case *map[string]interface{}:
+			ptr = *v.(*map[string]interface{})
+		case *interface{}:
+			ptr = *v.(*interface{})
+		default:
+			return errors.Errorf("unknown type %T in YAML unmarshal", v)
+		}
+
+		if mm, changed := stringifyMapKeys(ptr); changed {
+			switch v.(type) {
+			case *map[string]interface{}:
+				*v.(*map[string]interface{}) = mm.(map[string]interface{})
+			case *interface{}:
+				*v.(*interface{}) = mm
 			}
 		}
 	default:
-		return nil, errors.Errorf("unmarshal of format %q is not supported", f)
+		return errors.Errorf("unmarshal of format %q is not supported", f)
 	}
 
-	if err != nil {
-		return nil, errors.Wrapf(err, "unmarshal failed for format %q", f)
-	}
+	return err
 
-	return m, nil
+}
+
+// stringifyMapKeys recurses into in and changes all instances of
+// map[interface{}]interface{} to map[string]interface{}. This is useful to
+// work around the impedence mismatch between JSON and YAML unmarshaling that's
+// described here: https://github.com/go-yaml/yaml/issues/139
+//
+// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
+func stringifyMapKeys(in interface{}) (interface{}, bool) {
+
+	switch in := in.(type) {
+	case []interface{}:
+		for i, v := range in {
+			if vv, replaced := stringifyMapKeys(v); replaced {
+				in[i] = vv
+			}
+		}
+	case map[string]interface{}:
+		for k, v := range in {
+			if vv, changed := stringifyMapKeys(v); changed {
+				in[k] = vv
+			}
+		}
+	case map[interface{}]interface{}:
+		res := make(map[string]interface{})
+		var (
+			ok  bool
+			err error
+		)
+		for k, v := range in {
+			var ks string
+
+			if ks, ok = k.(string); !ok {
+				ks, err = cast.ToStringE(k)
+				if err != nil {
+					ks = fmt.Sprintf("%v", k)
+				}
+			}
+			if vv, replaced := stringifyMapKeys(v); replaced {
+				res[ks] = vv
+			} else {
+				res[ks] = v
+			}
+		}
+		return res, true
+	}
 
+	return nil, false
 }
diff --git a/parser/metadecoders/decoder_test.go b/parser/metadecoders/decoder_test.go
new file mode 100644
index 000000000..94cfd5a9a
--- /dev/null
+++ b/parser/metadecoders/decoder_test.go
@@ -0,0 +1,207 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalToMap(t *testing.T) {
+	assert := require.New(t)
+
+	expect := map[string]interface{}{"a": "b"}
+
+	for i, test := range []struct {
+		data   string
+		format Format
+		expect interface{}
+	}{
+		{`a = "b"`, TOML, expect},
+		{`a: "b"`, YAML, expect},
+		// Make sure we get all string keys, even for YAML
+		{"a: Easy!\nb:\n  c: 2\n  d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
+		{"a:\n  true: 1\n  false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}},
+		{`{ "a": "b" }`, JSON, expect},
+		{`#+a: b`, ORG, expect},
+		// errors
+		{`a = b`, TOML, false},
+	} {
+		msg := fmt.Sprintf("%d: %s", i, test.format)
+		m, err := UnmarshalToMap([]byte(test.data), test.format)
+		if b, ok := test.expect.(bool); ok && !b {
+			assert.Error(err, msg)
+		} else {
+			assert.NoError(err, msg)
+			assert.Equal(test.expect, m, msg)
+		}
+	}
+}
+
+func TestUnmarshalToInterface(t *testing.T) {
+	assert := require.New(t)
+
+	expect := map[string]interface{}{"a": "b"}
+
+	for i, test := range []struct {
+		data   string
+		format Format
+		expect interface{}
+	}{
+		{`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}},
+		{`{ "a": "b" }`, JSON, expect},
+		{`#+a: b`, ORG, expect},
+		{`a = "b"`, TOML, expect},
+		{`a: "b"`, YAML, expect},
+		{"a: Easy!\nb:\n  c: 2\n  d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
+		// errors
+		{`a = "`, TOML, false},
+	} {
+		msg := fmt.Sprintf("%d: %s", i, test.format)
+		m, err := Unmarshal([]byte(test.data), test.format)
+		if b, ok := test.expect.(bool); ok && !b {
+			assert.Error(err, msg)
+		} else {
+			assert.NoError(err, msg)
+			assert.Equal(test.expect, m, msg)
+		}
+
+	}
+
+}
+
+func TestStringifyYAMLMapKeys(t *testing.T) {
+	cases := []struct {
+		input    interface{}
+		want     interface{}
+		replaced bool
+	}{
+		{
+			map[interface{}]interface{}{"a": 1, "b": 2},
+			map[string]interface{}{"a": 1, "b": 2},
+			true,
+		},
+		{
+			map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
+			map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
+			true,
+		},
+		{
+			map[interface{}]interface{}{true: 1, "b": false},
+			map[string]interface{}{"true": 1, "b": false},
+			true,
+		},
+		{
+			map[interface{}]interface{}{1: "a", 2: "b"},
+			map[string]interface{}{"1": "a", "2": "b"},
+			true,
+		},
+		{
+			map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			true,
+		},
+		{
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			map[string]interface{}{"a": map[string]interface{}{"b": 1}},
+			false,
+		},
+		{
+			[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
+			[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
+			false,
+		},
+	}
+
+	for i, c := range cases {
+		res, replaced := stringifyMapKeys(c.input)
+
+		if c.replaced != replaced {
+			t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
+		}
+		if !c.replaced {
+			res = c.input
+		}
+		if !reflect.DeepEqual(res, c.want) {
+			t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
+		}
+	}
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
+	maps := make([]map[interface{}]interface{}, b.N)
+	for i := 0; i < b.N; i++ {
+		maps[i] = map[interface{}]interface{}{
+			"a": map[interface{}]interface{}{
+				"b": 32,
+				"c": 43,
+				"d": map[interface{}]interface{}{
+					"b": 32,
+					"c": 43,
+				},
+			},
+			"b": []interface{}{"a", "b"},
+			"c": "d",
+		}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(maps[i])
+	}
+}
+
+func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
+	m := map[string]interface{}{
+		"a": map[string]interface{}{
+			"b": 32,
+			"c": 43,
+			"d": map[string]interface{}{
+				"b": 32,
+				"c": 43,
+			},
+		},
+		"b": []interface{}{"a", "b"},
+		"c": "d",
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(m)
+	}
+}
+
+func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
+	maps := make([]map[interface{}]interface{}, b.N)
+	for i := 0; i < b.N; i++ {
+		maps[i] = map[interface{}]interface{}{
+			1: map[interface{}]interface{}{
+				4: 32,
+				5: 43,
+				6: map[interface{}]interface{}{
+					7: 32,
+					8: 43,
+				},
+			},
+			2: []interface{}{"a", "b"},
+			3: "d",
+		}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		stringifyMapKeys(maps[i])
+	}
+}
diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go
new file mode 100644
index 000000000..b9f7f6919
--- /dev/null
+++ b/parser/metadecoders/format.go
@@ -0,0 +1,66 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"strings"
+
+	"github.com/gohugoio/hugo/parser/pageparser"
+)
+
+type Format string
+
+const (
+	// These are the supported metdata  formats in Hugo. Most of these are also
+	// supported as /data formats.
+	ORG  Format = "org"
+	JSON Format = "json"
+	TOML Format = "toml"
+	YAML Format = "yaml"
+)
+
+// FormatFromString turns formatStr, typically a file extension without any ".",
+// into a Format. It returns an empty string for unknown formats.
+func FormatFromString(formatStr string) Format {
+	formatStr = strings.ToLower(formatStr)
+	switch formatStr {
+	case "yaml", "yml":
+		return YAML
+	case "json":
+		return JSON
+	case "toml":
+		return TOML
+	case "org":
+		return ORG
+	}
+
+	return ""
+
+}
+
+// FormatFromFrontMatterType will return empty if not supported.
+func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
+	switch typ {
+	case pageparser.TypeFrontMatterJSON:
+		return JSON
+	case pageparser.TypeFrontMatterORG:
+		return ORG
+	case pageparser.TypeFrontMatterTOML:
+		return TOML
+	case pageparser.TypeFrontMatterYAML:
+		return YAML
+	default:
+		return ""
+	}
+}
diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go
new file mode 100644
index 000000000..46b4e434a
--- /dev/null
+++ b/parser/metadecoders/format_test.go
@@ -0,0 +1,57 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadecoders
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/gohugoio/hugo/parser/pageparser"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFormatFromString(t *testing.T) {
+	assert := require.New(t)
+	for i, test := range []struct {
+		s      string
+		expect Format
+	}{
+		{"json", JSON},
+		{"yaml", YAML},
+		{"yml", YAML},
+		{"toml", TOML},
+		{"tOMl", TOML},
+		{"org", ORG},
+		{"foo", ""},
+	} {
+		assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i))
+	}
+}
+
+func TestFormatFromFrontMatterType(t *testing.T) {
+	assert := require.New(t)
+	for i, test := range []struct {
+		typ    pageparser.ItemType
+		expect Format
+	}{
+		{pageparser.TypeFrontMatterJSON, JSON},
+		{pageparser.TypeFrontMatterTOML, TOML},
+		{pageparser.TypeFrontMatterYAML, YAML},
+		{pageparser.TypeFrontMatterORG, ORG},
+		{pageparser.TypeIgnore, ""},
+	} {
+		assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
+	}
+}
diff --git a/parser/metadecoders/json.go b/parser/metadecoders/json.go
deleted file mode 100644
index 21ca8a3b9..000000000
--- a/parser/metadecoders/json.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package metadecoders
-
-import "encoding/json"
-
-// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleJSONData(datum []byte) (interface{}, error) {
-	if datum == nil {
-		// Package json returns on error on nil input.
-		// Return an empty map to be consistent with our other supported
-		// formats.
-		return make(map[string]interface{}), nil
-	}
-
-	var f interface{}
-	err := json.Unmarshal(datum, &f)
-	return f, err
-}
diff --git a/parser/metadecoders/yaml.go b/parser/metadecoders/yaml.go
deleted file mode 100644
index 21b23a9fd..000000000
--- a/parser/metadecoders/yaml.go
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The metadecoders package contains functions to decode metadata (e.g. page front matter)
-// from different formats: TOML, YAML, JSON.
-package metadecoders
-
-import (
-	"fmt"
-
-	"github.com/spf13/cast"
-	yaml "gopkg.in/yaml.v2"
-)
-
-// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
-// representing the encoded data structure.
-func HandleYAMLData(datum []byte) (interface{}, error) {
-	var m interface{}
-	err := yaml.Unmarshal(datum, &m)
-	if err != nil {
-		return nil, err
-	}
-
-	// To support boolean keys, the `yaml` package unmarshals maps to
-	// map[interface{}]interface{}. Here we recurse through the result
-	// and change all maps to map[string]interface{} like we would've
-	// gotten from `json`.
-	if mm, changed := stringifyMapKeys(m); changed {
-		return mm, nil
-	}
-
-	return m, nil
-}
-
-// stringifyMapKeys recurses into in and changes all instances of
-// map[interface{}]interface{} to map[string]interface{}. This is useful to
-// work around the impedence mismatch between JSON and YAML unmarshaling that's
-// described here: https://github.com/go-yaml/yaml/issues/139
-//
-// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
-func stringifyMapKeys(in interface{}) (interface{}, bool) {
-	switch in := in.(type) {
-	case []interface{}:
-		for i, v := range in {
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				in[i] = vv
-			}
-		}
-	case map[interface{}]interface{}:
-		res := make(map[string]interface{})
-		var (
-			ok  bool
-			err error
-		)
-		for k, v := range in {
-			var ks string
-
-			if ks, ok = k.(string); !ok {
-				ks, err = cast.ToStringE(k)
-				if err != nil {
-					ks = fmt.Sprintf("%v", k)
-				}
-			}
-			if vv, replaced := stringifyMapKeys(v); replaced {
-				res[ks] = vv
-			} else {
-				res[ks] = v
-			}
-		}
-		return res, true
-	}
-
-	return nil, false
-}