Misc config loading fixes

The main motivation behind this is simplicity and correctnes, but the new small config library is also faster:

```
BenchmarkDefaultConfigProvider/Viper-16         	  252418	      4546 ns/op	    2720 B/op	      30 allocs/op
BenchmarkDefaultConfigProvider/Custom-16        	  450756	      2651 ns/op	    1008 B/op	       6 allocs/op
```

Fixes #8633
Fixes #8618
Fixes #8630
Updates #8591
Closes #6680
Closes #5192

4 files changed, 280 insertions, 34 deletions

diff --git a/common/maps/maps.go b/common/maps/maps.go
index 41d9b6e15..5fb079009 100644
--- a/common/maps/maps.go
+++ b/common/maps/maps.go
@@ -18,53 +18,65 @@ import (
 	"strings"
 
 	"github.com/gobwas/glob"
-
 	"github.com/spf13/cast"
 )
 
-// ToLower makes all the keys in the given map lower cased and will do so
-// recursively.
-// Notes:
-// * This will modify the map given.
-// * Any nested map[interface{}]interface{} will be converted to Params.
-func ToLower(m Params) {
-	for k, v := range m {
-		var retyped bool
-		switch v.(type) {
-		case map[interface{}]interface{}:
-			var p Params = cast.ToStringMap(v)
-			v = p
-			ToLower(p)
-			retyped = true
-		case map[string]interface{}:
-			var p Params = v.(map[string]interface{})
-			v = p
-			ToLower(p)
-			retyped = true
+// ToStringMapE converts in to map[string]interface{}.
+func ToStringMapE(in interface{}) (map[string]interface{}, error) {
+	switch vv := in.(type) {
+	case Params:
+		return vv, nil
+	case map[string]string:
+		var m = map[string]interface{}{}
+		for k, v := range vv {
+			m[k] = v
 		}
+		return m, nil
 
-		lKey := strings.ToLower(k)
-		if retyped || k != lKey {
-			delete(m, k)
-			m[lKey] = v
-		}
+	default:
+		return cast.ToStringMapE(in)
 	}
 }
 
-func ToStringMapE(in interface{}) (map[string]interface{}, error) {
-	switch in.(type) {
-	case Params:
-		return in.(Params), nil
-	default:
-		return cast.ToStringMapE(in)
+// ToParamsAndPrepare converts in to Params and prepares it for use.
+// See PrepareParams.
+func ToParamsAndPrepare(in interface{}) (Params, bool) {
+	m, err := ToStringMapE(in)
+	if err != nil {
+		return nil, false
 	}
+	PrepareParams(m)
+	return m, true
 }
 
+// ToStringMap converts in to map[string]interface{}.
 func ToStringMap(in interface{}) map[string]interface{} {
 	m, _ := ToStringMapE(in)
 	return m
 }
 
+// ToStringMapStringE converts in to map[string]string.
+func ToStringMapStringE(in interface{}) (map[string]string, error) {
+	m, err := ToStringMapE(in)
+	if err != nil {
+		return nil, err
+	}
+	return cast.ToStringMapStringE(m)
+}
+
+// ToStringMapString converts in to map[string]string.
+func ToStringMapString(in interface{}) map[string]string {
+	m, _ := ToStringMapStringE(in)
+	return m
+}
+
+// ToStringMapBool converts in to bool.
+func ToStringMapBool(in interface{}) map[string]bool {
+	m, _ := ToStringMapE(in)
+	return cast.ToStringMapBool(m)
+}
+
+// ToSliceStringMap converts in to []map[string]interface{}.
 func ToSliceStringMap(in interface{}) ([]map[string]interface{}, error) {
 	switch v := in.(type) {
 	case []map[string]interface{}:
@@ -127,9 +139,8 @@ func (KeyRenamer) keyPath(k1, k2 string) string {
 	k1, k2 = strings.ToLower(k1), strings.ToLower(k2)
 	if k1 == "" {
 		return k2
-	} else {
-		return k1 + "/" + k2
 	}
+	return k1 + "/" + k2
 }
 
 func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]interface{}) {
diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go
index 7e527aac5..dbe97a15a 100644
--- a/common/maps/maps_test.go
+++ b/common/maps/maps_test.go
@@ -67,7 +67,7 @@ func TestToLower(t *testing.T) {
 	for i, test := range tests {
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
 			// ToLower modifies input.
-			ToLower(test.input)
+			PrepareParams(test.input)
 			if !reflect.DeepEqual(test.expected, test.input) {
 				t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
 			}
diff --git a/common/maps/params.go b/common/maps/params.go
index 4c881093c..7e94d593b 100644
--- a/common/maps/params.go
+++ b/common/maps/params.go
@@ -14,6 +14,7 @@
 package maps
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/spf13/cast"
@@ -29,6 +30,95 @@ func (p Params) Get(indices ...string) interface{} {
 	return v
 }
 
+// Set overwrites values in p with values in pp for common or new keys.
+// This is done recursively.
+func (p Params) Set(pp Params) {
+	for k, v := range pp {
+		vv, found := p[k]
+		if !found {
+			p[k] = v
+		} else {
+			switch vvv := vv.(type) {
+			case Params:
+				if pv, ok := v.(Params); ok {
+					vvv.Set(pv)
+				} else {
+					p[k] = v
+				}
+			default:
+				p[k] = v
+			}
+		}
+	}
+}
+
+// Merge transfers values from pp to p for new keys.
+// This is done recursively.
+func (p Params) Merge(pp Params) {
+	p.merge("", pp)
+}
+
+func (p Params) merge(ps ParamsMergeStrategy, pp Params) {
+	ns, found := p.GetMergeStrategy()
+
+	var ms = ns
+	if !found && ps != "" {
+		ms = ps
+	}
+
+	noUpdate := ms == ParamsMergeStrategyNone
+	noUpdate = noUpdate || (ps != "" && ps == ParamsMergeStrategyShallow)
+
+	for k, v := range pp {
+
+		if k == mergeStrategyKey {
+			continue
+		}
+		vv, found := p[k]
+
+		if found {
+			// Key matches, if both sides are Params, we try to merge.
+			if vvv, ok := vv.(Params); ok {
+				if pv, ok := v.(Params); ok {
+					vvv.merge(ms, pv)
+				}
+
+			}
+
+		} else if !noUpdate {
+			p[k] = v
+
+		}
+
+	}
+}
+
+func (p Params) GetMergeStrategy() (ParamsMergeStrategy, bool) {
+	if v, found := p[mergeStrategyKey]; found {
+		if s, ok := v.(ParamsMergeStrategy); ok {
+			return s, true
+		}
+	}
+	return ParamsMergeStrategyShallow, false
+}
+
+func (p Params) DeleteMergeStrategy() bool {
+	if _, found := p[mergeStrategyKey]; found {
+		delete(p, mergeStrategyKey)
+		return true
+	}
+	return false
+}
+
+func (p Params) SetDefaultMergeStrategy(s ParamsMergeStrategy) {
+	switch s {
+	case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow:
+	default:
+		panic(fmt.Sprintf("invalid merge strategy %q", s))
+	}
+	p[mergeStrategyKey] = s
+}
+
 func getNested(m map[string]interface{}, indices []string) (interface{}, string, map[string]interface{}) {
 	if len(indices) == 0 {
 		return nil, "", nil
@@ -108,3 +198,61 @@ func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interf
 
 	return nil, "", nil, nil
 }
+
+// ParamsMergeStrategy tells what strategy to use in Params.Merge.
+type ParamsMergeStrategy string
+
+const (
+	// Do not merge.
+	ParamsMergeStrategyNone ParamsMergeStrategy = "none"
+	// Only add new keys.
+	ParamsMergeStrategyShallow ParamsMergeStrategy = "shallow"
+	// Add new keys, merge existing.
+	ParamsMergeStrategyDeep ParamsMergeStrategy = "deep"
+
+	mergeStrategyKey = "_merge"
+)
+
+func toMergeStrategy(v interface{}) ParamsMergeStrategy {
+	s := ParamsMergeStrategy(cast.ToString(v))
+	switch s {
+	case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow:
+		return s
+	default:
+		return ParamsMergeStrategyDeep
+	}
+}
+
+// PrepareParams
+// * makes all the keys in the given map lower cased and will do so
+// * This will modify the map given.
+// * Any nested map[interface{}]interface{} will be converted to Params.
+// * Any _merge value will be converted to proper type and value.
+func PrepareParams(m Params) {
+	for k, v := range m {
+		var retyped bool
+		lKey := strings.ToLower(k)
+		if lKey == mergeStrategyKey {
+			v = toMergeStrategy(v)
+			retyped = true
+		} else {
+			switch v.(type) {
+			case map[interface{}]interface{}:
+				var p Params = cast.ToStringMap(v)
+				v = p
+				PrepareParams(p)
+				retyped = true
+			case map[string]interface{}:
+				var p Params = v.(map[string]interface{})
+				v = p
+				PrepareParams(p)
+				retyped = true
+			}
+		}
+
+		if retyped || k != lKey {
+			delete(m, k)
+			m[lKey] = v
+		}
+	}
+}
diff --git a/common/maps/params_test.go b/common/maps/params_test.go
index df8cbf8d6..8859bb86b 100644
--- a/common/maps/params_test.go
+++ b/common/maps/params_test.go
@@ -69,3 +69,90 @@ func TestGetNestedParamFnNestedNewKey(t *testing.T) {
 	c.Assert(nestedKey, qt.Equals, "new")
 	c.Assert(owner, qt.DeepEquals, nested)
 }
+
+func TestParamsSetAndMerge(t *testing.T) {
+	c := qt.New(t)
+
+	createParamsPair := func() (Params, Params) {
+		p1 := Params{"a": "av", "c": "cv", "nested": Params{"al2": "al2v", "cl2": "cl2v"}}
+		p2 := Params{"b": "bv", "a": "abv", "nested": Params{"bl2": "bl2v", "al2": "al2bv"}, mergeStrategyKey: ParamsMergeStrategyDeep}
+		return p1, p2
+	}
+
+	p1, p2 := createParamsPair()
+
+	p1.Set(p2)
+
+	c.Assert(p1, qt.DeepEquals, Params{
+		"a": "abv",
+		"c": "cv",
+		"nested": Params{
+			"al2": "al2bv",
+			"cl2": "cl2v",
+			"bl2": "bl2v",
+		},
+		"b":              "bv",
+		mergeStrategyKey: ParamsMergeStrategyDeep,
+	})
+
+	p1, p2 = createParamsPair()
+
+	p1.Merge(p2)
+
+	// Default is to do a shallow merge.
+	c.Assert(p1, qt.DeepEquals, Params{
+		"c": "cv",
+		"nested": Params{
+			"al2": "al2v",
+			"cl2": "cl2v",
+		},
+		"b": "bv",
+		"a": "av",
+	})
+
+	p1, p2 = createParamsPair()
+	p1.SetDefaultMergeStrategy(ParamsMergeStrategyNone)
+	p1.Merge(p2)
+	p1.DeleteMergeStrategy()
+
+	c.Assert(p1, qt.DeepEquals, Params{
+		"a": "av",
+		"c": "cv",
+		"nested": Params{
+			"al2": "al2v",
+			"cl2": "cl2v",
+		},
+	})
+
+	p1, p2 = createParamsPair()
+	p1.SetDefaultMergeStrategy(ParamsMergeStrategyShallow)
+	p1.Merge(p2)
+	p1.DeleteMergeStrategy()
+
+	c.Assert(p1, qt.DeepEquals, Params{
+		"a": "av",
+		"c": "cv",
+		"nested": Params{
+			"al2": "al2v",
+			"cl2": "cl2v",
+		},
+		"b": "bv",
+	})
+
+	p1, p2 = createParamsPair()
+	p1.SetDefaultMergeStrategy(ParamsMergeStrategyDeep)
+	p1.Merge(p2)
+	p1.DeleteMergeStrategy()
+
+	c.Assert(p1, qt.DeepEquals, Params{
+		"nested": Params{
+			"al2": "al2v",
+			"cl2": "cl2v",
+			"bl2": "bl2v",
+		},
+		"b": "bv",
+		"a": "av",
+		"c": "cv",
+	})
+
+}