diff options
58 files changed, 1701 insertions, 329 deletions
diff --git a/common/collections/slice.go b/common/collections/slice.go index 38ca86b08..07ad48eb3 100644 --- a/common/collections/slice.go +++ b/common/collections/slice.go @@ -64,3 +64,13 @@ func Slice(args ...interface{}) interface{} { } return slice.Interface() } + +// StringSliceToInterfaceSlice converts ss to []interface{}. +func StringSliceToInterfaceSlice(ss []string) []interface{} { + result := make([]interface{}, len(ss)) + for i, s := range ss { + result[i] = s + } + return result + +} diff --git a/common/herrors/errors.go b/common/herrors/errors.go index fded30b1a..00aed1eb6 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -88,3 +88,10 @@ func GetGID() uint64 { // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, // and this error is used to signal those situations. var ErrFeatureNotAvailable = errors.New("this feature is not available in your current Hugo version, see https://goo.gl/YMrWcn for more information") + +// Must panics if err != nil. +func Must(err error) { + if err != nil { + panic(err) + } +} diff --git a/common/hexec/exec.go b/common/hexec/exec.go new file mode 100644 index 000000000..a8bdd1bb7 --- /dev/null +++ b/common/hexec/exec.go @@ -0,0 +1,276 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hexec + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "os" + "os/exec" + + "github.com/cli/safeexec" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" +) + +var WithDir = func(dir string) func(c *commandeer) { + return func(c *commandeer) { + c.dir = dir + } +} + +var WithContext = func(ctx context.Context) func(c *commandeer) { + return func(c *commandeer) { + c.ctx = ctx + } +} + +var WithStdout = func(w io.Writer) func(c *commandeer) { + return func(c *commandeer) { + c.stdout = w + } +} + +var WithStderr = func(w io.Writer) func(c *commandeer) { + return func(c *commandeer) { + c.stderr = w + } +} + +var WithStdin = func(r io.Reader) func(c *commandeer) { + return func(c *commandeer) { + c.stdin = r + } +} + +var WithEnviron = func(env []string) func(c *commandeer) { + return func(c *commandeer) { + setOrAppend := func(s string) { + k1, _ := config.SplitEnvVar(s) + var found bool + for i, v := range c.env { + k2, _ := config.SplitEnvVar(v) + if k1 == k2 { + found = true + c.env[i] = s + } + } + + if !found { + c.env = append(c.env, s) + } + } + + for _, s := range env { + setOrAppend(s) + } + } +} + +// New creates a new Exec using the provided security config. +func New(cfg security.Config) *Exec { + var baseEnviron []string + for _, v := range os.Environ() { + k, _ := config.SplitEnvVar(v) + if cfg.Exec.OsEnv.Accept(k) { + baseEnviron = append(baseEnviron, v) + } + } + + return &Exec{ + sc: cfg, + baseEnviron: baseEnviron, + } +} + +// IsNotFound reports whether this is an error about a binary not found. +func IsNotFound(err error) bool { + var notFoundErr *NotFoundError + return errors.As(err, ¬FoundErr) +} + +// SafeCommand is a wrapper around os/exec Command which uses a LookPath +// implementation that does not search in current directory before looking in PATH. +// See https://github.com/cli/safeexec and the linked issues. +func SafeCommand(name string, arg ...string) (*exec.Cmd, error) { + bin, err := safeexec.LookPath(name) + if err != nil { + return nil, err + } + + return exec.Command(bin, arg...), nil +} + +// Exec encorces a security policy for commands run via os/exec. +type Exec struct { + sc security.Config + + // os.Environ filtered by the Exec.OsEnviron whitelist filter. + baseEnviron []string +} + +// New will fail if name is not allowed according to the configured security policy. +// Else a configured Runner will be returned ready to be Run. +func (e *Exec) New(name string, arg ...interface{}) (Runner, error) { + if err := e.sc.CheckAllowedExec(name); err != nil { + return nil, err + } + + env := make([]string, len(e.baseEnviron)) + copy(env, e.baseEnviron) + + cm := &commandeer{ + name: name, + env: env, + } + + return cm.command(arg...) + +} + +// Npx is a convenience method to create a Runner running npx --no-install <name> <args. +func (e *Exec) Npx(name string, arg ...interface{}) (Runner, error) { + arg = append(arg[:0], append([]interface{}{"--no-install", name}, arg[0:]...)...) + return e.New("npx", arg...) +} + +// Sec returns the security policies this Exec is configured with. +func (e *Exec) Sec() security.Config { + return e.sc +} + +type NotFoundError struct { + name string +} + +func (e *NotFoundError) Error() string { + return fmt.Sprintf("binary with name %q not found", e.name) +} + +// Runner wraps a *os.Cmd. +type Runner interface { + Run() error + StdinPipe() (io.WriteCloser, error) +} + +type cmdWrapper struct { + name string + c *exec.Cmd + + outerr *bytes.Buffer +} + +var notFoundRe = regexp.MustCompile(`(?s)not found:|could not determine executable`) + +func (c *cmdWrapper) Run() error { + err := c.c.Run() + if err == nil { + return nil + } + if notFoundRe.MatchString(c.outerr.String()) { + return &NotFoundError{name: c.name} + } + return fmt.Errorf("failed to execute binary %q with args %v: %s", c.name, c.c.Args[1:], c.outerr.String()) +} + +func (c *cmdWrapper) StdinPipe() (io.WriteCloser, error) { + return c.c.StdinPipe() +} + +type commandeer struct { + stdout io.Writer + stderr io.Writer + stdin io.Reader + dir string + ctx context.Context + + name string + env []string +} + +func (c *commandeer) command(arg ...interface{}) (*cmdWrapper, error) { + if c == nil { + return nil, nil + } + + var args []string + for _, a := range arg { + switch v := a.(type) { + case string: + args = append(args, v) + case func(*commandeer): + v(c) + default: + return nil, fmt.Errorf("invalid argument to command: %T", a) + } + } + + bin, err := safeexec.LookPath(c.name) + if err != nil { + return nil, &NotFoundError{ + name: c.name, + } + } + + outerr := &bytes.Buffer{} + if c.stderr == nil { + c.stderr = outerr + } else { + c.stderr = io.MultiWriter(c.stderr, outerr) + } + + var cmd *exec.Cmd + + if c.ctx != nil { + cmd = exec.CommandContext(c.ctx, bin, args...) + } else { + cmd = exec.Command(bin, args...) + } + + cmd.Stdin = c.stdin + cmd.Stderr = c.stderr + cmd.Stdout = c.stdout + cmd.Env = c.env + cmd.Dir = c.dir + + return &cmdWrapper{outerr: outerr, c: cmd, name: c.name}, nil +} + +// InPath reports whether binaryName is in $PATH. +func InPath(binaryName string) bool { + if strings.Contains(binaryName, "/") { + panic("binary name should not contain any slash") + } + _, err := safeexec.LookPath(binaryName) + return err == nil +} + +// LookPath finds the path to binaryName in $PATH. +// Returns "" if not found. +func LookPath(binaryName string) string { + if strings.Contains(binaryName, "/") { + panic("binary name should not contain any slash") + } + s, err := safeexec.LookPath(binaryName) + if err != nil { + return "" + } + return s +} diff --git a/common/hexec/safeCommand.go b/common/hexec/safeCommand.go deleted file mode 100644 index 6d5c73982..000000000 --- a/common/hexec/safeCommand.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hexec - -import ( - "context" - - "os/exec" - - "github.com/cli/safeexec" -) - -// SafeCommand is a wrapper around os/exec Command which uses a LookPath -// implementation that does not search in current directory before looking in PATH. -// See https://github.com/cli/safeexec and the linked issues. -func SafeCommand(name string, arg ...string) (*exec.Cmd, error) { - bin, err := safeexec.LookPath(name) - if err != nil { - return nil, err - } - - return exec.Command(bin, arg...), nil -} - -// SafeCommandContext wraps CommandContext -// See SafeCommand for more context. -func SafeCommandContext(ctx context.Context, name string, arg ...string) (*exec.Cmd, error) { - bin, err := safeexec.LookPath(name) - if err != nil { - return nil, err - } - - return exec.CommandContext(ctx, bin, arg...), nil -} diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index 4548de93a..d8f92e298 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -89,8 +89,10 @@ func NewInfo(environment string) Info { } } +// GetExecEnviron creates and gets the common os/exec environment used in the +// external programs we interact with via os/exec, e.g. postcss. func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string { - env := os.Environ() + var env []string nodepath := filepath.Join(workDir, "node_modules") if np := os.Getenv("NODE_PATH"); np != "" { nodepath = workDir + string(os.PathListSeparator) + np @@ -98,12 +100,15 @@ func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string { config.SetEnvVars(&env, "NODE_PATH", nodepath) config.SetEnvVars(&env, "PWD", workDir) config.SetEnvVars(&env, "HUGO_ENVIRONMENT", cfg.GetString("environment")) - fis, err := afero.ReadDir(fs, files.FolderJSConfig) - if err == nil { - for _, fi := range fis { - key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_")) - value := fi.(hugofs.FileMetaInfo).Meta().Filename - config.SetEnvVars(&env, key, value) + + if fs != nil { + fis, err := afero.ReadDir(fs, files.FolderJSConfig) + if err == nil { + for _, fi := range fis { + key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_")) + value := fi.(hugofs.FileMetaInfo).Meta().Filename + config.SetEnvVars(&env, key, value) + } } } diff --git a/config/defaultConfigProvider.go b/config/defaultConfigProvider.go index 0a10d5cc6..7701e765a 100644 --- a/config/defaultConfigProvider.go +++ b/config/defaultConfigProvider.go @@ -44,6 +44,8 @@ var ( "permalinks": true, "related": true, "sitemap": true, + "privacy": true, + "security": true, "taxonomies": true, } diff --git a/config/security/docshelper.go b/config/security/docshelper.go new file mode 100644 index 000000000..ade03560e --- /dev/null +++ b/config/security/docshelper.go @@ -0,0 +1,26 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "github.com/gohugoio/hugo/docshelper" +) + +func init() { + docsProvider := func() docshelper.DocProvider { + + return docshelper.DocProvider{"config": DefaultConfig.ToSecurityMap()} + } + docshelper.AddDocProviderFunc(docsProvider) +} diff --git a/config/security/securityConfig.go b/config/security/securityConfig.go new file mode 100644 index 000000000..09c5cb625 --- /dev/null +++ b/config/security/securityConfig.go @@ -0,0 +1,227 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/mitchellh/mapstructure" +) + +const securityConfigKey = "security" + +// DefaultConfig holds the default security policy. +var DefaultConfig = Config{ + Exec: Exec{ + Allow: NewWhitelist( + "^dart-sass-embedded$", + "^go$", // for Go Modules + "^npx$", // used by all Node tools (Babel, PostCSS). + "^postcss$", + ), + // These have been tested to work with Hugo's external programs + // on Windows, Linux and MacOS. + OsEnv: NewWhitelist("(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$"), + }, + Funcs: Funcs{ + Getenv: NewWhitelist("^HUGO_"), + }, + HTTP: HTTP{ + URLs: NewWhitelist(".*"), + Methods: NewWhitelist("(?i)GET|POST"), + }, +} + +// Config is the top level security config. +type Config struct { + // Restricts access to os.Exec. + Exec Exec `json:"exec"` + + // Restricts access to certain template funcs. + Funcs Funcs `json:"funcs"` + + // Restricts access to resources.Get, getJSON, getCSV. + HTTP HTTP `json:"http"` + + // Allow inline shortcodes + EnableInlineShortcodes bool `json:"enableInlineShortcodes"` +} + +// Exec holds os/exec policies. +type Exec struct { + Allow Whitelist `json:"allow"` + OsEnv Whitelist `json:"osEnv"` +} + +// Funcs holds template funcs policies. +type Funcs struct { + // OS env keys allowed to query in os.Getenv. + Getenv Whitelist `json:"getenv"` +} + +type HTTP struct { + // URLs to allow in remote HTTP (resources.Get, getJSON, getCSV). + URLs Whitelist `json:"urls"` + + // HTTP methods to allow. + Methods Whitelist `json:"methods"` +} + +// ToTOML converts c to TOML with [security] as the root. +func (c Config) ToTOML() string { + sec := c.ToSecurityMap() + + var b bytes.Buffer + + if err := parser.InterfaceToConfig(sec, metadecoders.TOML, &b); err != nil { + panic(err) + } + + return strings.TrimSpace(b.String()) +} + +func (c Config) CheckAllowedExec(name string) error { + if !c.Exec.Allow.Accept(name) { + return &AccessDeniedError{ + name: name, + path: "security.exec.allow", + policies: c.ToTOML(), + } + } + return nil + +} + +func (c Config) CheckAllowedGetEnv(name string) error { + if !c.Funcs.Getenv.Accept(name) { + return &AccessDeniedError{ + name: name, + path: "security.funcs.getenv", + policies: c.ToTOML(), + } + } + return nil +} + +func (c Config) CheckAllowedHTTPURL(url string) error { + if !c.HTTP.URLs.Accept(url) { + return &AccessDeniedError{ + name: url, + path: "security.http.urls", + policies: c.ToTOML(), + } + } + return nil +} + +func (c Config) CheckAllowedHTTPMethod(method string) error { + if !c.HTTP.Methods.Accept(method) { + return &AccessDeniedError{ + name: method, + path: "security.http.method", + policies: c.ToTOML(), + } + } + return nil +} + +// ToSecurityMap converts c to a map with 'security' as the root key. +func (c Config) ToSecurityMap() map[string]interface{} { + // Take it to JSON and back to get proper casing etc. + asJson, err := json.Marshal(c) + herrors.Must(err) + m := make(map[string]interface{}) + herrors.Must(json.Unmarshal(asJson, &m)) + + // Add the root + sec := map[string]interface{}{ + "security": m, + } + return sec + +} + +// DecodeConfig creates a privacy Config from a given Hugo configuration. +func DecodeConfig(cfg config.Provider) (Config, error) { + sc := DefaultConfig + if cfg.IsSet(securityConfigKey) { + m := cfg.GetStringMap(securityConfigKey) + dec, err := mapstructure.NewDecoder( + &mapstructure.DecoderConfig{ + WeaklyTypedInput: true, + Result: &sc, + DecodeHook: stringSliceToWhitelistHook(), + }, + ) + if err != nil { + return sc, err + } + + if err = dec.Decode(m); err != nil { + return sc, err + } + } + + if !sc.EnableInlineShortcodes { + // Legacy + sc.EnableInlineShortcodes = cfg.GetBool("enableInlineShortcodes") + } + + return sc, nil + +} + +func stringSliceToWhitelistHook() mapstructure.DecodeHookFuncType { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + + if t != reflect.TypeOf(Whitelist{}) { + return data, nil + } + + wl := types.ToStringSlicePreserveString(data) + + return NewWhitelist(wl...), nil + + } +} + +// AccessDeniedError represents a security policy conflict. +type AccessDeniedError struct { + path string + name string + policies string +} + +func (e *AccessDeniedError) Error() string { + return fmt.Sprintf("access denied: %q is not whitelisted in policy %q; the current security configuration is:\n\n%s\n\n", e.name, e.path, e.policies) +} + +// IsAccessDenied reports whether err is an AccessDeniedError +func IsAccessDenied(err error) bool { + var notFoundErr *AccessDeniedError + return errors.As(err, ¬FoundErr) +} diff --git a/config/security/securityonfig_test.go b/config/security/securityonfig_test.go new file mode 100644 index 000000000..d0416a20d --- /dev/null +++ b/config/security/securityonfig_test.go @@ -0,0 +1,166 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" +) + +func TestDecodeConfigFromTOML(t *testing.T) { + c := qt.New(t) + + c.Run("Slice whitelist", func(c *qt.C) { + c.Parallel() + tomlConfig := ` + + +someOtherValue = "bar" + +[security] +enableInlineShortcodes=true +[security.exec] +allow=["a", "b"] +osEnv=["a", "b", "c"] +[security.funcs] +getEnv=["a", "b"] + +` + + cfg, err := config.FromConfigString(tomlConfig, "toml") + c.Assert(err, qt.IsNil) + + pc, err := DecodeConfig(cfg) + c.Assert(err, qt.IsNil) + c.Assert(pc, qt.Not(qt.IsNil)) + c.Assert(pc.EnableInlineShortcodes, qt.IsTrue) + c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue) + c.Assert(pc.Exec.Allow.Accept("d"), qt.IsFalse) + c.Assert(pc.Exec.OsEnv.Accept("a"), qt.IsTrue) + c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse) + c.Assert(pc.Funcs.Getenv.Accept("a"), qt.IsTrue) + c.Assert(pc.Funcs.Getenv.Accept("c"), qt.IsFalse) + + }) + + c.Run("String whitelist", func(c *qt.C) { + c.Parallel() + tomlConfig := ` + + +someOtherValue = "bar" + +[security] +[security.exec] +allow="a" +osEnv="b" + +` + + cfg, err := config.FromConfigString(tomlConfig, "toml") + c.Assert(err, qt.IsNil) + + pc, err := DecodeConfig(cfg) + c.Assert(err, qt.IsNil) + c.Assert(pc, qt.Not(qt.IsNil)) + c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue) + c.Assert(pc.Exec.Allow.Accept("d"), qt.IsFalse) + c.Assert(pc.Exec.OsEnv.Accept("b"), qt.IsTrue) + c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse) + + }) + + c.Run("Default exec.osEnv", func(c *qt.C) { + c.Parallel() + tomlConfig := ` + + +someOtherValue = "bar" + +[security] +[security.exec] +allow="a" + +` + + cfg, err := config.FromConfigString(tomlConfig, "toml") + c.Assert(err, qt.IsNil) + + pc, err := DecodeConfig(cfg) + c.Assert(err, qt.IsNil) + c.Assert(pc, qt.Not(qt.IsNil)) + c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue) + c.Assert(pc.Exec.OsEnv.Accept("PATH"), qt.IsTrue) + c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse) + + }) + + c.Run("Enable inline shortcodes, legacy", func(c *qt.C) { + c.Parallel() + tomlConfig := ` + + +someOtherValue = "bar" +enableInlineShortcodes=true + +[security] +[security.exec] +allow="a" +osEnv="b" + +` + + cfg, err := config.FromConfigString(tomlConfig, "toml") + c.Assert(err, qt.IsNil) + + pc, err := DecodeConfig(cfg) + c.Assert(err, qt.IsNil) + c.Assert(pc.EnableInlineShortcodes, qt.IsTrue) + + }) + +} + +func TestToTOML(t *testing.T) { + c := qt.New(t) + + got := DefaultConfig.ToTOML() + + c.Assert(got, qt.Equals, + "[security]\n enableInlineShortcodes = false\n [security.exec]\n allow = ['^dart-sass-embedded$', '^go$', '^npx$', '^postcss$']\n osEnv = ['(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$']\n\n [security.funcs]\n getenv = ['^HUGO_']\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']", + ) +} + +func TestDecodeConfigDefault(t *testing.T) { + t.Parallel() + c := qt.New(t) + + pc, err := DecodeConfig(config.New()) + c.Assert(err, qt.IsNil) + c.Assert(pc, qt.Not(qt.IsNil)) + c.Assert(pc.Exec.Allow.Accept("a"), qt.IsFalse) + c.Assert(pc.Exec.Allow.Accept("npx"), qt.IsTrue) + c.Assert(pc.Exec.Allow.Accept("Npx"), qt.IsFalse) + c.Assert(pc.Exec.OsEnv.Accept("a"), qt.IsFalse) + c.Assert(pc.Exec.OsEnv.Accept("PATH"), qt.IsTrue) + c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse) + + c.Assert(pc.HTTP.URLs.Accept("https://example.org"), qt.IsTrue) + c.Assert(pc.HTTP.Methods.Accept("POST"), qt.IsTrue) + c.Assert(pc.HTTP.Methods.Accept("GET"), qt.IsTrue) + c.Assert(pc.HTTP.Methods.Accept("get"), qt.IsTrue) + c.Assert(pc.HTTP.Methods.Accept("DELETE"), qt.IsFalse) +} diff --git a/config/security/whitelist.go b/config/security/whitelist.go new file mode 100644 index 000000000..0d2c187c6 --- /dev/null +++ b/config/security/whitelist.go @@ -0,0 +1,102 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" +) + +const ( + acceptNoneKeyword = "none" +) + +// Whitelist holds a whitelist. +type Whitelist struct { + acceptNone bool + patterns []*regexp.Regexp + + // Store this for debugging/error reporting + patternsStrings []string +} + +func (w Whitelist) MarshalJSON() ([]byte, error) { + if w.acceptNone { + return json.Marshal(acceptNoneKeyword) + } + + return json.Marshal(w.patternsStrings) +} + +// NewWhitelist creates a new Whitelist from zero or more patterns. +// An empty patterns list or a pattern with the value 'none' will create +// a whitelist that will Accept noone. +func NewWhitelist(patterns ...string) Whitelist { + if len(patterns) == 0 { + return Whitelist{acceptNone: true} + } + + var acceptSome bool + var patternsStrings []string + + for _, p := range patterns { + if p == acceptNoneKeyword { + acceptSome = false + break + } + + if ps := strings.TrimSpace(p); ps != "" { + acceptSome = true + patternsStrings = append(patternsStrings, ps) + } + } + + if !acceptSome { + return Whitelist{ + acceptNone: true, + } + } + + var patternsr []*regexp.Regexp + + for i := 0; i < len(patterns); i++ { + p := strings.TrimSpace(patterns[i]) + if p == "" { + continue + } + patternsr = append(patternsr, regexp.MustCompile(p)) + } + + return Whitelist{patterns: patternsr, patternsStrings: patternsStrings} +} + +// Accepted reports whether name is whitelisted. +func (w Whitelist) Accept(name string) bool { + if w.acceptNone { + return false + } + + for _, p := range w.patterns { + if p.MatchString(name) { + return true + } + } + return false +} + +func (w Whitelist) String() string { + return fmt.Sprint(w.patternsStrings) +} diff --git a/config/security/whitelist_test.go b/config/security/whitelist_test.go new file mode 100644 index 000000000..5c4196dff --- /dev/null +++ b/config/security/whitelist_test.go @@ -0,0 +1,47 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package security + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestWhitelist(t *testing.T) { + t.Parallel() + c := qt.New(t) + + c.Run("none", func(c *qt.C) { + c.Assert(NewWhitelist("none", "foo").Accept("foo"), qt.IsFalse) + c.Assert(NewWhitelist().Accept("foo"), qt.IsFalse) + c.Assert(NewWhitelist("").Accept("foo"), qt.IsFalse) + c.Assert(NewWhitelist(" ", " ").Accept("foo"), qt.IsFalse) + c.Assert(Whitelist{}.Accept("foo"), qt.IsFalse) + }) + + c.Run("One", func(c *qt.C) { + w := NewWhitelist("^foo.*") + c.Assert(w.Accept("foo"), qt.IsTrue) + c.Assert(w.Accept("mfoo"), qt.IsFalse) + }) + + c.Run("Multiple", func(c *qt.C) { + w := NewWhitelist("^foo.*", "^bar.*") + c.Assert(w.Accept("foo"), qt.IsTrue) + c.Assert(w.Accept("bar"), qt.IsTrue) + c.Assert(w.Accept("mbar"), qt.IsFalse) + }) + +} diff --git a/create/content.go b/create/content.go index ce5934e4a..6ae912882 100644 --- a/create/content.go +++ b/create/content.go @@ -24,11 +24,11 @@ import ( "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/paths" "github.com/pkg/errors" - "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs" @@ -344,16 +344,18 @@ func (b *contentBuilder) openInEditorIfConfigured(filename string) error { } b.h.Log.Printf("Editing %q with %q ...\n", filename, editor) + cmd, err := b.h.Deps.ExecHelper.New( + editor, + filename, + hexec.WithStdin(os.Stdin), + hexec.WithStderr(os.Stderr), + hexec.WithStdout(os.Stdout), + ) - cmd, err := hexec.SafeCommand(editor, filename) if err != nil { return err } - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() } diff --git a/deps/deps.go b/deps/deps.go index 6b9da21fe..191193b9b 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -8,8 +8,10 @@ import ( "github.com/pkg/errors" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" @@ -36,6 +38,8 @@ type Deps struct { // Used to log errors that may repeat itself many times. LogDistinct loggers.Logger + ExecHelper *hexec.Exec + // The templates to use. This will usually implement the full tpl.TemplateManager. tmpl tpl.TemplateHandler @@ -230,6 +234,12 @@ func New(cfg DepsCfg) (*Deps, error) { cfg.OutputFormats = output.DefaultFormats } + securityConfig, err := security.DecodeConfig(cfg.Cfg) + if err != nil { + return nil, errors.WithMessage(err, "failed to create security config from configuration") + } + execHelper := hexec.New(securityConfig) + ps, err := helpers.NewPathSpec(fs, cfg.Language, logger) if err != nil { return nil, errors.Wrap(err, "create PathSpec") @@ -243,12 +253,12 @@ func New(cfg DepsCfg) (*Deps, error) { errorHandler := &globalErrHandler{} buildState := &BuildState{} - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes) + resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, execHelper, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } - contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs) + contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs, execHelper) if err != nil { return nil, err } @@ -269,6 +279,7 @@ func New(cfg DepsCfg) (*Deps, error) { Fs: fs, Log: ignorableLogger, LogDistinct: logDistinct, + ExecHelper: execHelper, templateProvider: cfg.TemplateProvider, translationProvider: cfg.TranslationProvider, WithTemplate: cfg.WithTemplate, @@ -311,7 +322,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er return nil, err } - d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs) + d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs, d.ExecHelper) if err != nil { return nil, err } @@ -322,7 +333,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er // TODO(bep) clean up these inits. resourceCache := d.ResourceSpec.ResourceCache postBuildAssets := d.ResourceSpec.PostBuildAssets - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes) + d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, d.ExecHelper, cfg.OutputFormats, cfg.MediaTypes) if err != nil { return nil, err } diff --git a/docs/config/_default/security.toml b/docs/config/_default/security.toml new file mode 100644 index 000000000..73af66164 --- /dev/null +++ b/docs/config/_default/security.toml @@ -0,0 +1,13 @@ + + enableInlineShortcodes = false + + [exec] + allow = ['^go$'] + osEnv = ['^PATH$'] + + [funcs] + getenv = ['^HUGO_', '^REPOSITORY_URL$'] + + [http] + methods = ['(?i)GET|POST'] + urls = ['.*']
\ No newline at end of file diff --git a/docs/content/en/about/security-model/index.md b/docs/content/en/about/security-model/index.md index 7a7841131..aed925d49 100644 --- a/docs/content/en/about/security-model/index.md +++ b/docs/content/en/about/security-model/index.md @@ -21,14 +21,29 @@ Hugo produces static output, so once built, the runtime is the browser (assuming But when developing and building your site, the runtime is the `hugo` executable. Securing a runtime can be [a real challenge](https://blog.logrocket.com/how-to-protect-your-node-js-applications-from-malicious-dependencies-5f2e60ea08f9/). -**Hugo's main approach is that of sandboxing:** +**Hugo's main approach is that of sandboxing and a security policy with strict defaults:** * Hugo has a virtual file system and only the main project (not third-party components) is allowed to mount directories or files outside the project root. * Only the main project can walk symbolic links. * User-defined components have only read-access to the filesystem. -* We shell out to some external binaries to support [Asciidoctor](/content-management/formats/#list-of-content-formats) and similar, but those binaries and their flags are predefined. General functions to run arbitrary external OS commands have been [discussed](https://github.com/gohugoio/hugo/issues/796), but not implemented because of security concerns. +* We shell out to some external binaries to support [Asciidoctor](/content-management/formats/#list-of-content-formats) and similar, but those binaries and their flags are predefined and disabled by default (see [Security Policy](#security-policy)). General functions to run arbitrary external OS commands have been [discussed](https://github.com/gohugoio/hugo/issues/796), but not implemented because of security concerns. -Hugo will soon introduce a concept of _Content Source Plugins_ (AKA _Pages from Data_), but the above will still hold true. + +## Security Policy + +{{< new-in "0.91.0" >}} + +Hugo has a built-in security policy that restricts access to [os/exec](https://pkg.go.dev/os/exec), remote communication and similar. + +The defdault configuration is listed below. And build using features not whitelisted in the security policy will faill with a detailed message about what needs to be done. Most of these settings are whitelists (string or slice, [Regular Expressions](https://pkg.go.dev/regexp) or `none` which matches nothing). + +{{< code-toggle config="security" />}} + +Note that these and other config settings in Hugo can be overridden by the OS environment. If you want to block all remote HTTP fetching of data: + +``` +HUGO_SECURITY_HTTP_URLS=none hugo +``` ## Dependency Security diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 0f48c39e3..2123558d9 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -381,6 +381,10 @@ Maximum number of items in the RSS feed. ### sectionPagesMenu See ["Section Menu for Lazy Bloggers"](/templates/menu-templates/#section-menu-for-lazy-bloggers). +### security + +See [Security Policy](/about/security-model/#security-policy) + ### sitemap Default [sitemap configuration](/templates/sitemap-template/#configure-sitemapxml). diff --git a/docs/data/docs.json b/docs/data/docs.json index 70a2eafb4..8f8950dc4 100644 --- a/docs/data/docs.json +++ b/docs/data/docs.json @@ -1775,9 +1775,15 @@ "permalinks": { "_merge": "none" }, + "privacy": { + "_merge": "none" + }, "related": { "_merge": "none" }, + "security": { + "_merge": "none" + }, "sitemap": { "_merge": "none" }, @@ -1822,6 +1828,32 @@ "keepWhitespace": false } } + }, + "security": { + "enableInlineShortcodes": false, + "exec": { + "allow": [ + "^go$", + "^npx$", + "^postcss$" + ], + "osEnv": [ + "(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$" + ] + }, + "funcs": { + "getenv": [ + "^HUGO_" + ] + }, + "http": { + "methods": [ + "(?i)GET|POST" + ], + "urls": [ + ".*" + ] + } } }, "media": { @@ -1966,7 +1998,10 @@ "string": "image/jpeg", "suffixes": [ "jpg", - "jpeg" + "jpeg", + "jpe", + "jif", + "jfif" ] }, { diff --git a/helpers/content.go b/helpers/content.go index 161b14e76..2d26a0c48 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -24,6 +24,7 @@ import ( "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" @@ -64,7 +65,7 @@ type ContentSpec struct { // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) { +func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs, ex *hexec.Exec) (*ContentSpec, error) { spec := &ContentSpec{ summaryLength: cfg.GetInt("summaryLength"), BuildFuture: cfg.GetBool("buildFuture"), @@ -78,6 +79,7 @@ func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero. Cfg: cfg, ContentFs: contentFs, Logger: logger, + Exec: ex, }) if err != nil { return nil, err diff --git a/helpers/content_test.go b/helpers/content_test.go index 515b788f1..c1ff5c1d2 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -110,7 +110,7 @@ func TestNewContentSpec(t *testing.T) { cfg.Set("buildExpired", true) cfg.Set("buildDrafts", true) - spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) c.Assert(err, qt.IsNil) c.Assert(spec.summaryLength, qt.Equals, 32) diff --git a/helpers/general_test.go b/helpers/general_test.go index bfabcbef4..db8cb30a8 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -30,7 +30,7 @@ import ( func TestResolveMarkup(t *testing.T) { c := qt.New(t) cfg := config.New() - spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) c.Assert(err, qt.IsNil) for i, this := range []struct { diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 7d63e4d88..e9502acc0 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -50,7 +50,7 @@ func newTestCfg() config.Provider { func newTestContentSpec() *ContentSpec { v := config.New() - spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index 20722f092..fa3f29c44 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -115,7 +115,7 @@ func IsGitHubAction() bool { // SupportsAll reports whether the running system supports all Hugo features, // e.g. Asciidoc, Pandoc etc. func SupportsAll() bool { - return IsGitHubAction() + return IsGitHubAction() || os.Getenv("CI_LOCAL") != "" } // GoMinorVersion returns the minor version of the current Go version, diff --git a/hugolib/config.go b/hugolib/config.go index 3b5ade598..e79899b94 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/common/maps" @@ -41,6 +42,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" @@ -377,6 +379,12 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide return nil, nil, err } + secConfig, err := security.DecodeConfig(v1) + if err != nil { + return nil, nil, err + } + ex := hexec.New(secConfig) + v1.Set("filecacheConfigs", filecacheConfigs) var configFilenames []string @@ -405,6 +413,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide modulesClient := modules.NewClient(modules.ClientConfig{ Fs: l.Fs, Logger: l.Logger, + Exec: ex, HookBeforeFinalize: hook, WorkingDir: workingDir, ThemesDir: themesDir, diff --git a/hugolib/js_test.go b/hugolib/js_test.go index 66c284d8b..69f528758 100644 --- a/hugolib/js_test.go +++ b/hugolib/js_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/htesting" @@ -123,10 +122,9 @@ TS2: {{ template "print" $ts2 }} b.WithSourceFile("assets/js/included.js", includedJS) - cmd, err := hexec.SafeCommand("npm", "install") + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) - out, err := cmd.CombinedOutput() - b.Assert(err, qt.IsNil, qt.Commentf(string(out))) b.Build(BuildCfg{}) @@ -195,8 +193,8 @@ require github.com/gohugoio/hugoTestProjectJSModImports v0.9.0 // indirect }`) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, _ := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 7a1ff6c4e..50263d483 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -24,8 +24,6 @@ import ( "github.com/gohugoio/hugo/htesting" - "github.com/gohugoio/hugo/markup/rst" - "github.com/gohugoio/hugo/markup/asciidocext" "github.com/gohugoio/hugo/config" @@ -370,6 +368,7 @@ func normalizeExpected(ext, str string) string { func testAllMarkdownEnginesForPages(t *testing.T, assertFunc func(t *testing.T, ext string, pages page.Pages), settings map[string]interface{}, pageSources ...string) { + engines := []struct { ext string shouldExecute func() bool @@ -377,7 +376,7 @@ func testAllMarkdownEnginesForPages(t *testing.T, {"md", func() bool { return true }}, {"mmark", func() bool { return true }}, {"ad", func() bool { return asciidocext.Supports() }}, - {"rst", func() bool { return rst.Supports() }}, + {"rst", func() bool { return true }}, } for _, e := range engines { @@ -385,47 +384,57 @@ func testAllMarkdownEnginesForPages(t *testing.T, continue } - cfg, fs := newTestCfg(func(cfg config.Provider) error { - for k, v := range settings { - cfg.Set(k, v) + t.Run(e.ext, func(t *testing.T) { + + cfg, fs := newTestCfg(func(cfg config.Provider) error { + for k, v := range settings { + cfg.Set(k, v) + } + return nil + }) + + contentDir := "content" + + if s := cfg.GetString("contentDir"); s != "" { + contentDir = s } - return nil - }) - contentDir := "content" + cfg.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"}, + }, + }) - if s := cfg.GetString("contentDir"); s != "" { - contentDir = s - } + var fileSourcePairs []string - var fileSourcePairs []string + for i, source := range pageSources { + fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source) + } - for i, source := range pageSources { - fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source) - } + for i := 0; i < len(fileSourcePairs); i += 2 { + writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1]) + } - for i := 0; i < len(fileSourcePairs); i += 2 { - writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1]) - } + // Add a content page for the home page + homePath := fmt.Sprintf("_index.%s", e.ext) + writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) - // Add a content page for the home page - homePath := fmt.Sprintf("_index.%s", e.ext) - writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{}) - b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() - b.Build(BuildCfg{SkipRender: true}) + s := b.H.Sites[0] - s := b.H.Sites[0] + b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources)) - b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources)) + assertFunc(t, e.ext, s.RegularPages()) - assertFunc(t, e.ext, s.RegularPages()) + home, err := s.Info.Home() + b.Assert(err, qt.IsNil) + b.Assert(home, qt.Not(qt.IsNil)) + b.Assert(home.File().Path(), qt.Equals, homePath) + b.Assert(content(home), qt.Contains, "Home Page Content") - home, err := s.Info.Home() - b.Assert(err, qt.IsNil) - b.Assert(home, qt.Not(qt.IsNil)) - b.Assert(home.File().Path(), qt.Equals, homePath) - b.Assert(content(home), qt.Contains, "Home Page Content") + }) } } diff --git a/hugolib/resource_chain_babel_test.go b/hugolib/resource_chain_babel_test.go index 5cca22ba1..7a97e820a 100644 --- a/hugolib/resource_chain_babel_test.go +++ b/hugolib/resource_chain_babel_test.go @@ -21,8 +21,6 @@ import ( "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/common/hexec" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/htesting" @@ -51,7 +49,7 @@ func TestResourceChainBabel(t *testing.T) { "devDependencies": { "@babel/cli": "7.8.4", - "@babel/core": "7.9.0", + "@babel/core": "7.9.0", "@babel/preset-env": "7.9.5" } } @@ -94,6 +92,12 @@ class Car2 { v := config.New() v.Set("workingDir", workDir) v.Set("disableKinds", []string{"taxonomy", "term", "page"}) + v.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^npx$", "^babel$"}, + }, + }) + b := newTestSitesBuilder(t).WithLogger(logger) // Need to use OS fs for this. @@ -123,8 +127,8 @@ Transpiled3: {{ $transpiled.Permalink }} b.WithSourceFile("babel.config.js", babelConfig) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, _ := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 214dda216..0a5b9177c 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -32,8 +32,6 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/common/hexec" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/common/herrors" @@ -387,8 +385,6 @@ T1: {{ $r.Content }} } func TestResourceChainBasic(t *testing.T) { - t.Parallel() - ts := httptest.NewServer(http.FileServer(http.Dir("testdata/"))) t.Cleanup(func() { ts.Close() @@ -1184,8 +1180,8 @@ class-in-b { b.WithSourceFile("postcss.config.js", postcssConfig) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, err := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/securitypolicies_test.go b/hugolib/securitypolicies_test.go new file mode 100644 index 000000000..297f49479 --- /dev/null +++ b/hugolib/securitypolicies_test.go @@ -0,0 +1,202 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/markup/asciidocext" + "github.com/gohugoio/hugo/markup/pandoc" + "github.com/gohugoio/hugo/markup/rst" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/dartsass" +) + +func TestSecurityPolicies(t *testing.T) { + c := qt.New(t) + + testVariant := func(c *qt.C, withBuilder func(b *sitesBuilder), expectErr string) { + c.Helper() + b := newTestSitesBuilder(c) + withBuilder(b) + + if expectErr != "" { + err := b.BuildE(BuildCfg{}) + b.Assert(err, qt.IsNotNil) + b.Assert(err, qt.ErrorMatches, expectErr) + } else { + b.Build(BuildCfg{}) + } + + } + + httpTestVariant := func(c *qt.C, templ, expectErr string, withBuilder func(b *sitesBuilder)) { + ts := httptest.NewServer(http.FileServer(http.Dir("testdata/"))) + c.Cleanup(func() { + ts.Close() + }) + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", fmt.Sprintf(templ, ts.URL)) + if withBuilder != nil { + withBuilder(b) + } + } + testVariant(c, cb, expectErr) + } + + c.Run("os.GetEnv, denied", func(c *qt.C) { + c.Parallel() + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ os.Getenv "FOOBAR" }}`) + } + testVariant(c, cb, `(?s).*"FOOBAR" is not whitelisted in policy "security\.funcs\.getenv".*`) + }) + + c.Run("os.GetEnv, OK", func(c *qt.C) { + c.Parallel() + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ os.Getenv "HUGO_FOO" }}`) + } + testVariant(c, cb, "") + }) + + c.Run("Asciidoc, denied", func(c *qt.C) { + c.Parallel() + if !asciidocext.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.ad", "foo") + } + + testVariant(c, cb, `(?s).*"asciidoctor" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("RST, denied", func(c *qt.C) { + c.Parallel() + if !rst.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.rst", "foo") + } + + if runtime.GOOS == "windows" { + testVariant(c, cb, `(?s).*python(\.exe)?" is not whitelisted in policy "security\.exec\.allow".*`) + } else { + testVariant(c, cb, `(?s).*"rst2html(\.py)?" is not whitelisted in policy "security\.exec\.allow".*`) + + } + + }) + + c.Run("Pandoc, denied", func(c *qt.C) { + c.Parallel() + if !pandoc.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.pdc", "foo") + } + + testVariant(c, cb, `"(?s).*pandoc" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("Dart SASS, OK", func(c *qt.C) { + c.Parallel() + if !dartsass.Supports() { + c.Skip() + } + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`) + } + testVariant(c, cb, "") + }) + + c.Run("Dart SASS, denied", func(c *qt.C) { + c.Parallel() + if !dartsass.Supports() { + c.Skip() + } + cb := func(b *sitesBuilder) { + b.WithConfigFile("toml", ` + [security] + [security.exec] + allow="none" + + `) + b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`) + } + testVariant(c, cb, `(?s).*"dart-sass-embedded" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("resources.Get, OK", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil) + }) + + c.Run("resources.Get, denied method", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil) + }) + + c.Run("resources.Get, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + + c.Run("getJSON, OK", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil) + }) + + c.Run("getJSON, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + + c.Run("getCSV, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $d := getCSV ";" "%[1]s/cities.csv" }}{{ $d.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 21d65de32..ec3a4a01b 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -257,7 +257,7 @@ func newShortcodeHandler(p *pageState, s *Site, placeholderFunc func() string) * sh := &shortcodeHandler{ p: p, s: s, - enableInlineShortcodes: s.enableInlineShortcodes, + enableInlineShortcodes: s.ExecHelper.Sec().EnableInlineShortcodes, shortcodes: make([]*shortcode, 0, 4), nameSet: make(map[string]bool), } @@ -287,7 +287,7 @@ func renderShortcode( var hasVariants bool if sc.isInline { - if !p.s.enableInlineShortcodes { + if !p.s.ExecHelper.Sec().EnableInlineShortcodes { return "", false, nil } templName := path.Join("_inline_shortcode", p.File().Path(), sc.name) diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 6ef110c9b..6316afc98 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -619,6 +619,12 @@ title: "Foo" cfg.Set("uglyURLs", false) cfg.Set("verbose", true) + cfg.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"}, + }, + }) + cfg.Set("markup.highlight.noClasses", false) cfg.Set("markup.highlight.codeFences", true) cfg.Set("markup", map[string]interface{}{ diff --git a/hugolib/site.go b/hugolib/site.go index 96cf0b93c..dce4b8d25 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -120,8 +120,6 @@ type Site struct { disabledKinds map[string]bool - enableInlineShortcodes bool - // Output formats defined in site config per Page Kind, or some defaults // if not set. // Output formats defined in Page front matter will override these. @@ -378,25 +376,24 @@ func (s *Site) isEnabled(kind string) bool { // reset returns a new Site prepared for rebuild. func (s *Site) reset() *Site { return &Site{ - Deps: s.Deps, - disabledKinds: s.disabledKinds, - titleFunc: s.titleFunc, - relatedDocsHandler: s.relatedDocsHandler.Clone(), - siteRefLinker: s.siteRefLinker, - outputFormats: s.outputFormats, - rc: s.rc, - outputFormatsConfig: s.outputFormatsConfig, - frontmatterHandler: s.frontmatterHandler, - mediaTypesConfig: s.mediaTypesConfig, - language: s.language, - siteBucket: s.siteBucket, - h: s.h, - publisher: s.publisher, - siteConfigConfig: s.siteConfigConfig, - enableInlineShortcodes: s.enableInlineShortcodes, - init: s.init, - PageCollections: s.PageCollections, - siteCfg: s.siteCfg, + Deps: s.Deps, + disabledKinds: s.disabledKinds, + titleFunc: s.titleFunc, + relatedDocsHandler: s.relatedDocsHandler.Clone(), + siteRefLinker: s.siteRefLinker, + outputFormats: s.outputFormats, + rc: s.rc, + outputFormatsConfig: s.outputFormatsConfig, + frontmatterHandler: s.frontmatterHandler, + mediaTypesConfig: s.mediaTypesConfig, + language: s.language, + siteBucket: s.siteBucket, + h: s.h, + publisher: s.publisher, + siteConfigConfig: s.siteConfigConfig, + init: s.init, + PageCollections: s.PageCollections, + siteCfg: s.siteCfg, } } @@ -564,8 +561,7 @@ But this also means that your site configuration may not do what you expect. If outputFormatsConfig: siteOutputFormatsConfig, mediaTypesConfig: siteMediaTypesConfig, - enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"), - siteCfg: siteConfig, + siteCfg: siteConfig, titleFunc: titleFunc, diff --git a/hugolib/testdata/cities.csv b/hugolib/testdata/cities.csv new file mode 100644 index 000000000..ee6b058b6 --- /dev/null +++ b/hugolib/testdata/cities.csv @@ -0,0 +1,130 @@ +"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State" + 41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH + 42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD + 46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA + 42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA + 43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI + 36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC + 49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB + 39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA + 34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC + 39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE + 48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND + 41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA + 37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV + 33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX + 37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS + 40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV + 26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL + 47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA + 41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA + 31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA + 44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI + 42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL + 44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD + 43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY + 42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA + 41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT + 38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC + 41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA + 46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA + 31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX + 38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN + 28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX + 32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS + 49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC + 46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND + 30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA + 43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY + 39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA + 32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX + 42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID + 33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL + 34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS + 36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK + 32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ + 37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO + 40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ + 44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI + 43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON + 39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS + 41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH + 33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX + 39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN + 27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL + 30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL + 47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA + 43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY + 32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA + 33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC + 40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA + 37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA + 44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI + 40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH + 40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO + 38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA + 39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH + 37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO + 42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA + 39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL + 47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA + 41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN + 43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD + 42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA + 32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA + 33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX + 44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY + 35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK + 32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL + 38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO + 47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA + 41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA + 41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB + 42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY + 32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA + 46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI + 27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL + 38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA + 35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM + 34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA + 33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA + 37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA + 37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA + 41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH + 32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA + 34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA + 29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX + 31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX + 40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT + 38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD + 36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA + 38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS + 38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO + 44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR + 44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN + 38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO + 39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO + 42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI + 44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT + 45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN + 29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL + 43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI + 38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA + 43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT + 33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM + 35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC + 41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY + 42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL + 43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY + 44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN + 37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA + 37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA + 39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN + 38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT + 45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI + 39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV + 50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA + 40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA + 40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA + 41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH + diff --git a/hugolib/testdata/fruits.json b/hugolib/testdata/fruits.json new file mode 100644 index 000000000..3bb802a16 --- /dev/null +++ b/hugolib/testdata/fruits.json @@ -0,0 +1,5 @@ +{ + "fruit": "Apple", + "size": "Large", + "color": "Red" +} diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index ba3965675..72e22ed1d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -18,6 +18,7 @@ import ( "time" "unicode/utf8" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/output" @@ -30,6 +31,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -791,6 +793,16 @@ func (s *sitesBuilder) GetPageRel(p page.Page, ref string) page.Page { return p } +func (s *sitesBuilder) NpmInstall() hexec.Runner { + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("npm") + ex := hexec.New(sc) + command, err := ex.New("npm", "install") + s.Assert(err, qt.IsNil) + return command + +} + func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper { return testHelper{ Cfg: cfg, diff --git a/markup/asciidocext/convert.go b/markup/asciidocext/convert.go index ff843cb6e..4c83e0e95 100644 --- a/markup/asciidocext/convert.go +++ b/markup/asciidocext/convert.go @@ -21,10 +21,9 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" - "github.com/cli/safeexec" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config" "github.com/gohugoio/hugo/markup/converter" @@ -67,7 +66,11 @@ type asciidocConverter struct { } func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - content, toc, err := a.extractTOC(a.getAsciidocContent(ctx.Src, a.ctx)) + b, err := a.getAsciidocContent(ctx.Src, a.ctx) + if err != nil { + return nil, err + } + content, toc, err := a.extractTOC(b) if err != nil { return nil, err } @@ -83,20 +86,19 @@ func (a *asciidocConverter) Supports(_ identity.Identity) bool { // getAsciidocContent calls asciidoctor as an external helper // to convert AsciiDoc content to HTML. -func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte { - path := getAsciidoctorExecPath() - if path == "" { +func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { + if !hasAsciiDoc() { a.cfg.Logger.Errorln("asciidoctor not found in $PATH: Please install.\n", " Leaving AsciiDoc content unrendered.") - return src + return src, nil } args := a.parseArgs(ctx) args = append(args, "-") - a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, "with", path, "using asciidoctor args", args, "...") + a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, " using asciidoctor args", args, "...") - return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args) + return internal.ExternallyRenderContent(a.cfg, ctx, src, asciiDocBinaryName, args) } func (a *asciidocConverter) parseArgs(ctx converter.DocumentContext) []string { @@ -195,12 +197,10 @@ func (a *asciidocConverter) appendArg(args []string, option, value, defaultValue return args } -func getAsciidoctorExecPath() string { - path, err := safeexec.LookPath("asciidoctor") - if err != nil { - return "" - } - return path +const asciiDocBinaryName = "asciidoctor" + +func hasAsciiDoc() bool { + return hexec.InPath(asciiDocBinaryName) } // extractTOC extracts the toc from the given src html. @@ -311,8 +311,12 @@ func nodeContent(node *html.Node) string { // Supports returns whether Asciidoctor is installed on this computer. func Supports() bool { + hasBin := hasAsciiDoc() if htesting.SupportsAll() { + if !hasBin { + panic("asciidoctor not installed") + } return true } - return getAsciidoctorExecPath() != "" + return hasBin } diff --git a/markup/asciidocext/convert_test.go b/markup/asciidocext/convert_test.go index acc525c3b..3a350c5ce 100644 --- a/markup/asciidocext/convert_test.go +++ b/markup/asciidocext/convert_test.go @@ -21,8 +21,10 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" @@ -280,20 +282,28 @@ func TestAsciidoctorAttributes(t *testing.T) { c.Assert(args[4], qt.Equals, "--no-header-footer") } -func TestConvert(t *testing.T) { - if !Supports() { - t.Skip("asciidoctor not installed") - } - c := qt.New(t) +func getProvider(c *qt.C, mconf markup_config.Config) converter.Provider { + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("asciidoctor") - mconf := markup_config.Default p, err := Provider.New( converter.ProviderConfig{ MarkupConfig: mconf, Logger: loggers.NewErrorLogger(), + Exec: hexec.New(sc), }, ) c.Assert(err, qt.IsNil) + return p +} + +func TestConvert(t *testing.T) { + if !Supports() { + t.Skip("asciidoctor not installed") + } + c := qt.New(t) + + p := getProvider(c, markup_config.Default) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) @@ -308,14 +318,8 @@ func TestTableOfContents(t *testing.T) { t.Skip("asciidoctor not installed") } c := qt.New(t) - mconf := markup_config.Default - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, markup_config.Default) + conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: macro @@ -390,14 +394,7 @@ func TestTableOfContentsWithCode(t *testing.T) { t.Skip("asciidoctor not installed") } c := qt.New(t) - mconf := markup_config.Default - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, markup_config.Default) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: auto @@ -433,13 +430,8 @@ func TestTableOfContentsPreserveTOC(t *testing.T) { c := qt.New(t) mconf := markup_config.Default mconf.AsciidocExt.PreserveTOC = true - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, mconf) + conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: diff --git a/markup/converter/converter.go b/markup/converter/converter.go index 3fa3bea39..180208a7b 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -16,6 +16,7 @@ package converter import ( "bytes" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" @@ -32,6 +33,7 @@ type ProviderConfig struct { Cfg config.Provider // Site config ContentFs afero.Fs Logger loggers.Logger + Exec *hexec.Exec Highlight func(code, lang, optsStr string) (string, error) } diff --git a/markup/internal/external.go b/markup/internal/external.go index 0937afa34..97cf5cc7d 100644 --- a/markup/internal/external.go +++ b/markup/internal/external.go @@ -2,42 +2,56 @@ package internal import ( "bytes" + "fmt" "strings" - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" - "github.com/gohugoio/hugo/markup/converter" ) func ExternallyRenderContent( cfg converter.ProviderConfig, ctx converter.DocumentContext, - content []byte, path string, args []string) []byte { + content []byte, binaryName string, args []string) ([]byte, error) { logger := cfg.Logger - cmd, err := hexec.SafeCommand(path, args...) - if err != nil { - logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err) - return nil + + if strings.Contains(binaryName, "/") { + panic(fmt.Sprintf("should be no slash in %q", binaryName)) } - cmd.Stdin = bytes.NewReader(content) + + argsv := collections.StringSliceToInterfaceSlice(args) + var out, cmderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &cmderr + argsv = append(argsv, hexec.WithStdout(&out)) + argsv = append(argsv, hexec.WithStderr(&cmderr)) + argsv = append(argsv, hexec.WithStdin(bytes.NewReader(content))) + + cmd, err := cfg.Exec.New(binaryName, argsv...) + if err != nil { + return nil, err + } + err = cmd.Run() + // Most external helpers exit w/ non-zero exit code only if severe, i.e. // halting errors occurred. -> log stderr output regardless of state of err for _, item := range strings.Split(cmderr.String(), "\n") { item := strings.TrimSpace(item) if item != "" { - logger.Errorf("%s: %s", ctx.DocumentName, item) + if err == nil { + logger.Warnf("%s: %s", ctx.DocumentName, item) + } else { + logger.Errorf("%s: %s", ctx.DocumentName, item) + } } } + if err != nil { - logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err) + logger.Errorf("%s rendering %s: %v", binaryName, ctx.DocumentName, err) } - return normalizeExternalHelperLineFeeds(out.Bytes()) + return normalizeExternalHelperLineFeeds(out.Bytes()), nil } // Strips carriage returns from third-party / external processes (useful for Windows) @@ -45,13 +59,13 @@ func normalizeExternalHelperLineFeeds(content []byte) []byte { return bytes.Replace(content, []byte("\r"), []byte(""), -1) } -func GetPythonExecPath() string { - path, err := safeexec.LookPath("python") - if err != nil { - path, err = safeexec.LookPath("python.exe") - if err != nil { - return "" +var pythonBinaryCandidates = []string{"python", "python.exe"} + +func GetPythonBinaryAndExecPath() (string, string) { + for _, p := range pythonBinaryCandidates { + if pth := hexec.LookPath(p); pth != "" { + return p, pth } } - return path + return "", "" } diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index 1c25e41d2..ae90cf417 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -15,7 +15,7 @@ package pandoc import ( - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" @@ -44,7 +44,11 @@ type pandocConverter struct { } func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil + b, err := c.getPandocContent(ctx.Src, c.ctx) + if err != nil { + return nil, err + } + return converter.Bytes(b), nil } func (c *pandocConverter) Supports(feature identity.Identity) bool { @@ -52,31 +56,35 @@ func (c *pandocConverter) Supports(feature identity.Identity) bool { } // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. -func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte { +func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { logger := c.cfg.Logger - path := getPandocExecPath() - if path == "" { + binaryName := getPandocBinaryName() + if binaryName == "" { logger.Println("pandoc not found in $PATH: Please install.\n", " Leaving pandoc content unrendered.") - return src + return src, nil } args := []string{"--mathjax"} - return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) + return internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args) } -func getPandocExecPath() string { - path, err := safeexec.LookPath("pandoc") - if err != nil { - return "" - } +const pandocBinary = "pandoc" - return path +func getPandocBinaryName() string { + if hexec.InPath(pandocBinary) { + return pandocBinary + } + return "" } // Supports returns whether Pandoc is installed on this computer. func Supports() bool { + hasBin := getPandocBinaryName() != "" if htesting.SupportsAll() { + if !hasBin { + panic("pandoc not installed") + } return true } - return getPandocExecPath() != "" + return hasBin } diff --git a/markup/pandoc/convert_test.go b/markup/pandoc/convert_test.go index bd6ca19e6..f549d5f4f 100644 --- a/markup/pandoc/convert_test.go +++ b/markup/pandoc/convert_test.go @@ -16,7 +16,9 @@ package pandoc import ( "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" @@ -28,7 +30,9 @@ func TestConvert(t *testing.T) { t.Skip("pandoc not installed") } c := qt.New(t) - p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("pandoc") + p, err := Provider.New(converter.ProviderConfig{Exec: hexec.New(sc), Logger: loggers.NewErrorLogger()}) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) diff --git a/markup/rst/convert.go b/markup/rst/convert.go index 4c11c4be8..b86b35f1b 100644 --- a/markup/rst/convert.go +++ b/markup/rst/convert.go @@ -18,7 +18,7 @@ import ( "bytes" "runtime" - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/identity" @@ -48,7 +48,11 @@ type rstConverter struct { } func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil + b, err := c.getRstContent(ctx.Src, c.ctx) + if err != nil { + return nil, err + } + return converter.Bytes(b), nil } func (c *rstConverter) Supports(feature identity.Identity) bool { @@ -57,31 +61,38 @@ func (c *rstConverter) Supports(feature identity.Identity) bool { // getRstContent calls the Python script rst2html as an external helper // to convert reStructuredText content to HTML. -func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte { +func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { logger := c.cfg.Logger - path := getRstExecPath() + binaryName, binaryPath := getRstBinaryNameAndPath() - if path == "" { + if binaryName == "" { logger.Println("rst2html / rst2html.py not found in $PATH: Please install.\n", " Leaving reStructuredText content unrendered.") - return src + return src, nil } - logger.Infoln("Rendering", ctx.DocumentName, "with", path, "...") + logger.Infoln("Rendering", ctx.DocumentName, "with", binaryName, "...") var result []byte + var err error + // certain *nix based OSs wrap executables in scripted launchers // invoking binaries on these OSs via python interpreter causes SyntaxError // invoke directly so that shebangs work as expected // handle Windows manually because it doesn't do shebangs if runtime.GOOS == "windows" { - python := internal.GetPythonExecPath() - args := []string{path, "--leave-comments", "--initial-header-level=2"} - result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args) + pythonBinary, _ := internal.GetPythonBinaryAndExecPath() + args := []string{binaryPath, "--leave-comments", "--initial-header-level=2"} + result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, pythonBinary, args) } else { args := []string{"--leave-comments", "--initial-header-level=2"} - result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) + result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args) + } + + if err != nil { + return nil, err } + // TODO(bep) check if rst2html has a body only option. bodyStart := bytes.Index(result, []byte("<body>\n")) if bodyStart < 0 { @@ -96,24 +107,29 @@ func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) } } - return result[bodyStart+7 : bodyEnd] + return result[bodyStart+7 : bodyEnd], err } -func getRstExecPath() string { - path, err := safeexec.LookPath("rst2html") - if err != nil { - path, err = safeexec.LookPath("rst2html.py") - if err != nil { - return "" +var rst2Binaries = []string{"rst2html", "rst2html.py"} + +func getRstBinaryNameAndPath() (string, string) { + for _, candidate := range rst2Binaries { + if pth := hexec.LookPath(candidate); pth != "" { + return candidate, pth } } - return path + return "", "" } -// Supports returns whether rst is installed on this computer. +// Supports returns whether rst is (or should be) installed on this computer. func Supports() bool { + name, _ := getRstBinaryNameAndPath() + hasBin := name != "" if htesting.SupportsAll() { + if !hasBin { + panic("rst not installed") + } return true } - return getRstExecPath() != "" + return hasBin } diff --git a/markup/rst/convert_test.go b/markup/rst/convert_test.go index 269d92caa..5d2882de1 100644 --- a/markup/rst/convert_test.go +++ b/markup/rst/convert_test.go @@ -16,7 +16,9 @@ package rst import ( "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" @@ -28,7 +30,14 @@ func TestConvert(t *testing.T) { t.Skip("rst not installed") } c := qt.New(t) - p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("rst", "python") + + p, err := Provider.New( + converter.ProviderConfig{ + Logger: loggers.NewErrorLogger(), + Exec: hexec.New(sc), + }) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) diff --git a/modules/client.go b/modules/client.go index fcb5957c3..1924cd5b4 100644 --- a/modules/client.go +++ b/modules/client.go @@ -28,6 +28,7 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" hglob "github.com/gohugoio/hugo/hugofs/glob" @@ -79,7 +80,7 @@ func NewClient(cfg ClientConfig) *Client { goModFilename = n } - env := os.Environ() + var env []string mcfg := cfg.ModuleConfig config.SetEnvVars(&env, @@ -87,12 +88,9 @@ func NewClient(cfg ClientConfig) *Client { "GO111MODULE", "on", "GOPROXY", mcfg.Proxy, "GOPRIVATE", mcfg.Private, - "GONOPROXY", mcfg.NoProxy) - - if cfg.CacheDir != "" { - // Module cache stored below $GOPATH/pkg - config.SetEnvVars(&env, "GOPATH", cfg.CacheDir) - } + "GONOPROXY", mcfg.NoProxy, + "GOPATH", cfg.CacheDir, + ) logger := cfg.Logger if logger == nil { @@ -609,16 +607,19 @@ func (c *Client) runGo( } stderr := new(bytes.Buffer) - cmd, err := hexec.SafeCommandContext(ctx, "go", args...) + + argsv := collections.StringSliceToInterfaceSlice(args) + argsv = append(argsv, hexec.WithEnviron(c.environ)) + argsv = append(argsv, hexec.WithStderr(io.MultiWriter(stderr, os.Stderr))) + argsv = append(argsv, hexec.WithStdout(stdout)) + argsv = append(argsv, hexec.WithDir(c.ccfg.WorkingDir)) + argsv = append(argsv, hexec.WithContext(ctx)) + + cmd, err := c.ccfg.Exec.New("go", argsv...) if err != nil { return err } - cmd.Env = c.environ - cmd.Dir = c.ccfg.WorkingDir - cmd.Stdout = stdout - cmd.Stderr = io.MultiWriter(stderr, os.Stderr) - if err := cmd.Run(); err != nil { if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { c.goBinaryStatus = goBinaryStatusNotFound @@ -727,6 +728,8 @@ type ClientConfig struct { // Eg. "production" Environment string + Exec *hexec.Exec + CacheDir string // Module cache ModuleConfig Config } diff --git a/modules/client_test.go b/modules/client_test.go index f801af07d..75e3c2b08 100644 --- a/modules/client_test.go +++ b/modules/client_test.go @@ -21,6 +21,8 @@ import ( "sync/atomic" "testing" + "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/htesting" @@ -53,7 +55,9 @@ github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/h ccfg := ClientConfig{ Fs: hugofs.Os, WorkingDir: workingDir, + CacheDir: filepath.Join(workingDir, "modcache"), ThemesDir: themesDir, + Exec: hexec.New(security.DefaultConfig), } withConfig(&ccfg) diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 64e2f95a6..f7d0efe64 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -154,6 +154,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro // FromRemote expects one or n-parts of a URL to a resource // If you provide multiple parts they will be joined together to the final URL. func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) { + if err := c.validateFromRemoteArgs(uri, options); err != nil { + return nil, err + } rURL, err := url.Parse(uri) if err != nil { return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) @@ -262,6 +265,19 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc } +func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error { + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil { + return err + } + + if method, ok := options["method"].(string); ok { + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil { + return err + } + } + return nil +} + func addDefaultHeaders(req *http.Request, accepts ...string) { for _, accept := range accepts { if !hasHeaderValue(req.Header, "Accept", accept) { diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 156def363..897c1bbaa 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -26,6 +26,7 @@ import ( "github.com/gohugoio/hugo/resources/jsconfig" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" @@ -51,6 +52,7 @@ func NewSpec( incr identity.Incrementer, logger loggers.Logger, errorHandler herrors.ErrorSender, + execHelper *hexec.Exec, outputFormats output.Formats, mimeTypes media.Types) (*Spec, error) { imgConfig, err := images.DecodeConfig(s.Cfg.GetStringMap("imaging")) @@ -81,6 +83,7 @@ func NewSpec( Logger: logger, ErrorSender: errorHandler, imaging: imaging, + ExecHelper: execHelper, incr: incr, MediaTypes: mimeTypes, OutputFormats: outputFormats, @@ -120,6 +123,8 @@ type Spec struct { // Holds default filter settings etc. imaging *images.ImageProcessor + ExecHelper *hexec.Exec + incr identity.Incrementer imageCache *imageCache ResourceCache *ResourceCache diff --git a/resources/resource_transformers/babel/babel.go b/resources/resource_transformers/babel/babel.go index e291b210b..c20a131f6 100644 --- a/resources/resource_transformers/babel/babel.go +++ b/resources/resource_transformers/babel/babel.go @@ -23,7 +23,6 @@ import ( "regexp" "strconv" - "github.com/cli/safeexec" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" @@ -59,8 +58,8 @@ func DecodeOptions(m map[string]interface{}) (opts Options, err error) { return } -func (opts Options) toArgs() []string { - var args []string +func (opts Options) toArgs() []interface{} { + var args []interface{} // external is not a known constant on the babel command line // .sourceMaps must be a boolean, "inline", "both", or undefined @@ -115,21 +114,12 @@ func (t *babelTransformation) Key() internal.ResourceTransformationKey { // npm install -g @babel/preset-env // Instead of installing globally, you can also install everything as a dev-dependency (--save-dev instead of -g) func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - const localBabelPath = "node_modules/.bin/" const binaryName = "babel" - // Try first in the project's node_modules. - csiBinPath := filepath.Join(t.rs.WorkingDir, localBabelPath, binaryName) + ex := t.rs.ExecHelper - binary := csiBinPath - - if _, err := safeexec.LookPath(binary); err != nil { - // Try PATH - binary = binaryName - if _, err := safeexec.LookPath(binary); err != nil { - // This may be on a CI server etc. Will fall back to pre-built assets. - return herrors.ErrFeatureNotAvailable - } + if err := ex.Sec().CheckAllowedExec(binaryName); err != nil { + return err } var configFile string @@ -157,11 +147,11 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx ctx.ReplaceOutPathExtension(".js") - var cmdArgs []string + var cmdArgs []interface{} if configFile != "" { logger.Infoln("babel: use config file", configFile) - cmdArgs = []string{"--config-file", configFile} + cmdArgs = []interface{}{"--config-file", configFile} } if optArgs := t.options.toArgs(); len(optArgs) > 0 { @@ -178,18 +168,27 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx } cmdArgs = append(cmdArgs, "--out-file="+compileOutput.Name()) + stderr := io.MultiWriter(infoW, &errBuf) + cmdArgs = append(cmdArgs, hexec.WithStderr(stderr)) + cmdArgs = append(cmdArgs, hexec.WithStdout(stderr)) + cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs))) + defer os.Remove(compileOutput.Name()) - cmd, err := hexec.SafeCommand(binary, cmdArgs...) + // ARGA [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel812882892/babel.config.js --source-maps --filename=js/main2.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-2237820197.js] + // [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel332846848/babel.config.js --filename=js/main.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-1451390834.js 0x10304ee60 0x10304ed60 0x10304f060] + cmd, err := ex.Npx(binaryName, cmdArgs...) + if err != nil { + if hexec.IsNotFound(err) { + // This may be on a CI server etc. Will fall back to pre-built assets. + return herrors.ErrFeatureNotAvailable + } return err } - cmd.Stderr = io.MultiWriter(infoW, &errBuf) - cmd.Stdout = cmd.Stderr - cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) - stdin, err := cmd.StdinPipe() + if err != nil { return err } @@ -201,6 +200,9 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx err = cmd.Run() if err != nil { + if hexec.IsNotFound(err) { + return herrors.ErrFeatureNotAvailable + } return errors.Wrap(err, errBuf.String()) } diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 21333eccb..674101f03 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -51,7 +51,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index 8104d0336..56cbea156 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -25,8 +25,7 @@ import ( "strconv" "strings" - "github.com/cli/safeexec" - + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/hugo" @@ -142,22 +141,9 @@ func (t *postcssTransformation) Key() internal.ResourceTransformationKey { // npm install -g postcss-cli // npm install -g autoprefixer func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - const localPostCSSPath = "node_modules/.bin/" const binaryName = "postcss" - // Try first in the project's node_modules. - csiBinPath := filepath.Join(t.rs.WorkingDir, localPostCSSPath, binaryName) - - binary := csiBinPath - - if _, err := safeexec.LookPath(binary); err != nil { - // Try PATH - binary = binaryName - if _, err := safeexec.LookPath(binary); err != nil { - // This may be on a CI server etc. Will fall back to pre-built assets. - return herrors.ErrFeatureNotAvailable - } - } + ex := t.rs.ExecHelper var configFile string logger := t.rs.Logger @@ -179,29 +165,33 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC } } - var cmdArgs []string + var cmdArgs []interface{} if configFile != "" { logger.Infoln("postcss: use config file", configFile) - cmdArgs = []string{"--config", configFile} + cmdArgs = []interface{}{"--config", configFile} } if optArgs := t.options.toArgs(); len(optArgs) > 0 { - cmdArgs = append(cmdArgs, optArgs...) - } - - cmd, err := hexec.SafeCommand(binary, cmdArgs...) - if err != nil { - return err + cmdArgs = append(cmdArgs, collections.StringSliceToInterfaceSlice(optArgs)...) } var errBuf bytes.Buffer infoW := loggers.LoggerToWriterWithPrefix(logger.Info(), "postcss") - cmd.Stdout = ctx.To - cmd.Stderr = io.MultiWriter(infoW, &errBuf) + stderr := io.MultiWriter(infoW, &errBuf) + cmdArgs = append(cmdArgs, hexec.WithStderr(stderr)) + cmdArgs = append(cmdArgs, hexec.WithStdout(ctx.To)) + cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs))) - cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) + cmd, err := ex.Npx(binaryName, cmdArgs...) + if err != nil { + if hexec.IsNotFound(err) { + // This may be on a CI server etc. Will fall back to pre-built assets. + return herrors.ErrFeatureNotAvailable + } + return err + } stdin, err := cmd.StdinPipe() if err != nil { @@ -231,6 +221,9 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC err = cmd.Run() if err != nil { + if hexec.IsNotFound(err) { + return herrors.ErrFeatureNotAvailable + } return imp.toFileError(errBuf.String()) } diff --git a/resources/resource_transformers/tocss/dartsass/client.go b/resources/resource_transformers/tocss/dartsass/client.go index 1d8250dc5..c2a572d9b 100644 --- a/resources/resource_transformers/tocss/dartsass/client.go +++ b/resources/resource_transformers/tocss/dartsass/client.go @@ -33,8 +33,13 @@ const transformationName = "tocss-dart" func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) { if !Supports() { - return &Client{dartSassNoAvailable: true}, nil + return &Client{dartSassNotAvailable: true}, nil } + + if err := rs.ExecHelper.Sec().CheckAllowedExec(dartSassEmbeddedBinaryName); err != nil { + return nil, err + } + transpiler, err := godartsass.Start(godartsass.Options{}) if err != nil { return nil, err @@ -43,15 +48,15 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) } type Client struct { - dartSassNoAvailable bool - rs *resources.Spec - sfs *filesystems.SourceFilesystem - workFs afero.Fs - transpiler *godartsass.Transpiler + dartSassNotAvailable bool + rs *resources.Spec + sfs *filesystems.SourceFilesystem + workFs afero.Fs + transpiler *godartsass.Transpiler } func (c *Client) ToCSS(res resources.ResourceTransformer, args map[string]interface{}) (resource.Resource, error) { - if c.dartSassNoAvailable { + if c.dartSassNotAvailable { return res.Transform(resources.NewFeatureNotAvailableTransformer(transformationName, args)) } return res.Transform(&transform{c: c, optsm: args}) diff --git a/resources/resource_transformers/tocss/dartsass/transform.go b/resources/resource_transformers/tocss/dartsass/transform.go index d70d2e6e0..57d9feadb 100644 --- a/resources/resource_transformers/tocss/dartsass/transform.go +++ b/resources/resource_transformers/tocss/dartsass/transform.go @@ -21,9 +21,8 @@ import ( "path/filepath" "strings" - "github.com/cli/safeexec" - "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/media" @@ -38,16 +37,18 @@ import ( "github.com/bep/godartsass" ) -// See https://github.com/sass/dart-sass-embedded/issues/24 -const stdinPlaceholder = "HUGOSTDIN" +const ( + // See https://github.com/sass/dart-sass-embedded/issues/24 + stdinPlaceholder = "HUGOSTDIN" + dartSassEmbeddedBinaryName = "dart-sass-embedded" +) // Supports returns whether dart-sass-embedded is found in $PATH. func Supports() bool { if htesting.SupportsAll() { return true } - p, err := safeexec.LookPath("dart-sass-embedded") - return err == nil && p != "" + return hexec.InPath(dartSassEmbeddedBinaryName) } type transform struct { diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 12dc8efe8..14d431644 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -87,7 +87,7 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -126,7 +126,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index 3faf46930..8cced6fe5 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -32,7 +32,6 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" - ) type tstNoStringer struct{} @@ -973,7 +972,7 @@ func ToTstXIs(slice interface{}) []TstXI { func newDeps(cfg config.Provider) *deps.Deps { l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") - cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) } diff --git a/tpl/data/data.go b/tpl/data/data.go index e993ed140..cfd847474 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/common/types" @@ -88,6 +89,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err err = ns.getResource(cache, unmarshal, req) if err != nil { + if security.IsAccessDenied(err) { + return nil, err + } ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err) return nil, nil } @@ -121,6 +125,9 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { err = ns.getResource(cache, unmarshal, req) if err != nil { + if security.IsAccessDenied(err) { + return nil, err + } ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err) return nil, nil } diff --git a/tpl/data/resources.go b/tpl/data/resources.go index b38b2784a..b4b310bcc 100644 --- a/tpl/data/resources.go +++ b/tpl/data/resources.go @@ -38,6 +38,13 @@ var ( // getRemote loads the content of a remote file. This method is thread safe. func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error { url := req.URL.String() + if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil { + return err + } + if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil { + return err + } + var headers bytes.Buffer req.Header.Write(&headers) id := helpers.MD5String(url + headers.String()) diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 8425bf87a..e825c2be1 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -22,12 +22,14 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/helpers" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -193,8 +195,10 @@ func newDeps(cfg config.Provider) *deps.Deps { } cfg.Set("allModules", modules.Modules{mod}) + ex := hexec.New(security.DefaultConfig) + logger := loggers.NewIgnorableLogger(loggers.NewErrorLogger(), "none") - cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs(), ex) if err != nil { panic(err) } @@ -215,6 +219,7 @@ func newDeps(cfg config.Provider) *deps.Deps { Cfg: cfg, Fs: fs, FileCaches: fileCaches, + ExecHelper: ex, ContentSpec: cs, Log: logger, LogDistinct: helpers.NewDistinctLogger(logger), diff --git a/tpl/os/os.go b/tpl/os/os.go index e729b810b..43c42f5e1 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -56,6 +56,10 @@ func (ns *Namespace) Getenv(key interface{}) (string, error) { return "", nil } + if err = ns.deps.ExecHelper.Sec().CheckAllowedGetEnv(skey); err != nil { + return "", err + } + return _os.Getenv(skey), nil } diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 2b0c69d09..260de5f83 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -241,7 +241,7 @@ func newDeps(cfg config.Provider) *deps.Deps { l := langs.NewLanguage("en", cfg) - cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) } |