Add resources.GetRemote

In Hugo 0.89 we added remote support to `resources.Get`.

In hindsight that was not a great idea, as a poll from many Hugo users showed. See Issue #9285 for more details.

After this commit `resources.Get` only supports local resource lookups. If you want to support both, you need to use a construct similar to:

Also improve some option case handling.

```
{{ resource := "" }}
{{ if (urls.Parse $url).IsAbs }}
{{ $resource = resources.GetRemote $url }}
{{ else }}
{{ $resource = resources.Get $url }}
{{ end }}
```

Fixes #9285
Fixes #9296

3 files changed, 315 insertions, 213 deletions

diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go
index f7d0efe64..f8e7e18db 100644
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -16,15 +16,7 @@
 package create
 
 import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"mime"
 	"net/http"
-	"net/http/httputil"
-	"net/url"
 	"path"
 	"path/filepath"
 	"strings"
@@ -36,13 +28,8 @@ import (
 
 	"github.com/gohugoio/hugo/cache/filecache"
 	"github.com/gohugoio/hugo/common/hugio"
-	"github.com/gohugoio/hugo/common/maps"
-	"github.com/gohugoio/hugo/common/types"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
-
-	"github.com/pkg/errors"
 )
 
 // Client contains methods to create Resource objects.
@@ -150,203 +137,3 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
 			})
 	})
 }
-
-// FromRemote expects one or n-parts of a URL to a resource
-// If you provide multiple parts they will be joined together to the final URL.
-func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
-	if err := c.validateFromRemoteArgs(uri, options); err != nil {
-		return nil, err
-	}
-	rURL, err := url.Parse(uri)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
-	}
-
-	resourceID := helpers.HashString(uri, options)
-
-	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
-		method, reqBody, err := getMethodAndBody(options)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
-		}
-
-		req, err := http.NewRequest(method, uri, reqBody)
-		if err != nil {
-			return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
-		}
-		addDefaultHeaders(req)
-
-		if _, ok := options["headers"]; ok {
-			headers, err := maps.ToStringMapE(options["headers"])
-			if err != nil {
-				return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri)
-			}
-			addUserProvidedHeaders(headers, req)
-		}
-
-		res, err := c.httpClient.Do(req)
-		if err != nil {
-			return nil, err
-		}
-
-		if res.StatusCode != http.StatusNotFound {
-			if res.StatusCode < 200 || res.StatusCode > 299 {
-				return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
-			}
-		}
-
-		httpResponse, err := httputil.DumpResponse(res, true)
-		if err != nil {
-			return nil, err
-		}
-
-		return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	defer httpResponse.Close()
-
-	res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if res.StatusCode == http.StatusNotFound {
-		// Not found. This matches how looksup for local resources work.
-		return nil, nil
-	}
-
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
-	}
-
-	filename := path.Base(rURL.Path)
-	if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
-		if _, ok := params["filename"]; ok {
-			filename = params["filename"]
-		}
-	}
-
-	var extension string
-	if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
-		extension = arr[0]
-	}
-
-	// If extension was not determined by header, look for a file extention
-	if extension == "" {
-		if ext := path.Ext(filename); ext != "" {
-			extension = ext
-		}
-	}
-
-	// If extension was not determined by header or file extention, try using content itself
-	if extension == "" {
-		if ct := http.DetectContentType(body); ct != "application/octet-stream" {
-			if ct == "image/jpeg" {
-				extension = ".jpg"
-			} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
-				extension = arr[0]
-			}
-		}
-	}
-
-	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension
-
-	return c.rs.New(
-		resources.ResourceSourceDescriptor{
-			LazyPublish: true,
-			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
-				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
-			},
-			RelTargetFilename: filepath.Clean(resourceID),
-		})
-
-}
-
-func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error {
-	if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
-		return err
-	}
-
-	if method, ok := options["method"].(string); ok {
-		if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func addDefaultHeaders(req *http.Request, accepts ...string) {
-	for _, accept := range accepts {
-		if !hasHeaderValue(req.Header, "Accept", accept) {
-			req.Header.Add("Accept", accept)
-		}
-	}
-	if !hasHeaderKey(req.Header, "User-Agent") {
-		req.Header.Add("User-Agent", "Hugo Static Site Generator")
-	}
-}
-
-func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
-	if headers == nil {
-		return
-	}
-	for key, val := range headers {
-		vals := types.ToStringSlicePreserveString(val)
-		for _, s := range vals {
-			req.Header.Add(key, s)
-		}
-	}
-}
-
-func hasHeaderValue(m http.Header, key, value string) bool {
-	var s []string
-	var ok bool
-
-	if s, ok = m[key]; !ok {
-		return false
-	}
-
-	for _, v := range s {
-		if v == value {
-			return true
-		}
-	}
-	return false
-}
-
-func hasHeaderKey(m http.Header, key string) bool {
-	_, ok := m[key]
-	return ok
-}
-
-func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) {
-	if options == nil {
-		return "GET", nil, nil
-	}
-
-	if method, ok := options["method"].(string); ok {
-		method = strings.ToUpper(method)
-		switch method {
-		case "GET", "DELETE", "HEAD", "OPTIONS":
-			return method, nil, nil
-		case "POST", "PUT", "PATCH":
-			var body []byte
-			if _, ok := options["body"]; ok {
-				switch b := options["body"].(type) {
-				case string:
-					body = []byte(b)
-				case []byte:
-					body = b
-				}
-			}
-			return method, bytes.NewBuffer(body), nil
-		}
-
-		return "", nil, fmt.Errorf("invalid HTTP method %q", method)
-	}
-
-	return "GET", nil, nil
-}
diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go
new file mode 100644
index 000000000..53e77bc5e
--- /dev/null
+++ b/resources/resource_factories/create/remote.go
@@ -0,0 +1,230 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package create
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/resources"
+	"github.com/gohugoio/hugo/resources/resource"
+	"github.com/mitchellh/mapstructure"
+	"github.com/pkg/errors"
+)
+
+// FromRemote expects one or n-parts of a URL to a resource
+// If you provide multiple parts they will be joined together to the final URL.
+func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resource.Resource, error) {
+	rURL, err := url.Parse(uri)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
+	}
+
+	resourceID := helpers.HashString(uri, optionsm)
+
+	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
+		options, err := decodeRemoteOptions(optionsm)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to decode options for resource %s", uri)
+		}
+		if err := c.validateFromRemoteArgs(uri, options); err != nil {
+			return nil, err
+		}
+
+		req, err := http.NewRequest(options.Method, uri, options.BodyReader())
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
+		}
+		addDefaultHeaders(req)
+
+		if options.Headers != nil {
+			addUserProvidedHeaders(options.Headers, req)
+		}
+
+		res, err := c.httpClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		if res.StatusCode != http.StatusNotFound {
+			if res.StatusCode < 200 || res.StatusCode > 299 {
+				return nil, errors.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode))
+			}
+		}
+
+		httpResponse, err := httputil.DumpResponse(res, true)
+		if err != nil {
+			return nil, err
+		}
+
+		return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer httpResponse.Close()
+
+	res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode == http.StatusNotFound {
+		// Not found. This matches how looksup for local resources work.
+		return nil, nil
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+	}
+
+	filename := path.Base(rURL.Path)
+	if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+		if _, ok := params["filename"]; ok {
+			filename = params["filename"]
+		}
+	}
+
+	var extension string
+	if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
+		extension = arr[0]
+	}
+
+	// If extension was not determined by header, look for a file extention
+	if extension == "" {
+		if ext := path.Ext(filename); ext != "" {
+			extension = ext
+		}
+	}
+
+	// If extension was not determined by header or file extention, try using content itself
+	if extension == "" {
+		if ct := http.DetectContentType(body); ct != "application/octet-stream" {
+			if ct == "image/jpeg" {
+				extension = ".jpg"
+			} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
+				extension = arr[0]
+			}
+		}
+	}
+
+	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension
+
+	return c.rs.New(
+		resources.ResourceSourceDescriptor{
+			LazyPublish: true,
+			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+			},
+			RelTargetFilename: filepath.Clean(resourceID),
+		})
+
+}
+
+func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
+	if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
+		return err
+	}
+
+	if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(options.Method); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func addDefaultHeaders(req *http.Request, accepts ...string) {
+	for _, accept := range accepts {
+		if !hasHeaderValue(req.Header, "Accept", accept) {
+			req.Header.Add("Accept", accept)
+		}
+	}
+	if !hasHeaderKey(req.Header, "User-Agent") {
+		req.Header.Add("User-Agent", "Hugo Static Site Generator")
+	}
+}
+
+func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
+	if headers == nil {
+		return
+	}
+	for key, val := range headers {
+		vals := types.ToStringSlicePreserveString(val)
+		for _, s := range vals {
+			req.Header.Add(key, s)
+		}
+	}
+}
+
+func hasHeaderValue(m http.Header, key, value string) bool {
+	var s []string
+	var ok bool
+
+	if s, ok = m[key]; !ok {
+		return false
+	}
+
+	for _, v := range s {
+		if v == value {
+			return true
+		}
+	}
+	return false
+}
+
+func hasHeaderKey(m http.Header, key string) bool {
+	_, ok := m[key]
+	return ok
+}
+
+type fromRemoteOptions struct {
+	Method  string
+	Headers map[string]interface{}
+	Body    []byte
+}
+
+func (o fromRemoteOptions) BodyReader() io.Reader {
+	if o.Body == nil {
+		return nil
+	}
+	return bytes.NewBuffer(o.Body)
+}
+
+func decodeRemoteOptions(optionsm map[string]interface{}) (fromRemoteOptions, error) {
+	var options = fromRemoteOptions{
+		Method: "GET",
+	}
+
+	err := mapstructure.WeakDecode(optionsm, &options)
+	if err != nil {
+		return options, err
+	}
+	options.Method = strings.ToUpper(options.Method)
+
+	return options, nil
+
+}
diff --git a/resources/resource_factories/create/remote_test.go b/resources/resource_factories/create/remote_test.go
new file mode 100644
index 000000000..42e3fc890
--- /dev/null
+++ b/resources/resource_factories/create/remote_test.go
@@ -0,0 +1,85 @@
+// Copyright 2021 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package create
+
+import (
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestDecodeRemoteOptions(t *testing.T) {
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		name    string
+		args    map[string]interface{}
+		want    fromRemoteOptions
+		wantErr bool
+	}{
+		{
+			"POST",
+			map[string]interface{}{
+				"meThod": "PoST",
+				"headers": map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+			fromRemoteOptions{
+				Method: "POST",
+				Headers: map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+			false,
+		},
+		{
+			"Body",
+			map[string]interface{}{
+				"meThod": "POST",
+				"body":   []byte("foo"),
+			},
+			fromRemoteOptions{
+				Method: "POST",
+				Body:   []byte("foo"),
+			},
+			false,
+		},
+		{
+			"Body, string",
+			map[string]interface{}{
+				"meThod": "POST",
+				"body":   "foo",
+			},
+			fromRemoteOptions{
+				Method: "POST",
+				Body:   []byte("foo"),
+			},
+			false,
+		},
+	} {
+		c.Run(test.name, func(c *qt.C) {
+			got, err := decodeRemoteOptions(test.args)
+			isErr := qt.IsNil
+			if test.wantErr {
+				isErr = qt.IsNotNil
+			}
+
+			c.Assert(err, isErr)
+			c.Assert(got, qt.DeepEquals, test.want)
+		})
+
+	}
+
+}