Add support for redirects24-add-redirects

Fix https://gitlab.com/gitlab-org/gitlab-pages/-/issues/24

3 files changed, 565 insertions, 0 deletions

diff --git a/internal/redirects/redirects.go b/internal/redirects/redirects.go
new file mode 100644
index 00000000..f6ec766e
--- /dev/null
+++ b/internal/redirects/redirects.go
@@ -0,0 +1,200 @@
+// Package redirects provides functions for parsing and rewriting URLs
+// according to Netlify style _redirects syntax
+package redirects
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+
+	netlifyRedirects "github.com/tj/go-redirects"
+
+	"gitlab.com/gitlab-org/labkit/log"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/vfs"
+)
+
+const (
+	// ConfigFile is the default name of the file containing the redirect rules.
+	// It follows Netlify's syntax but we don't support the special options yet like splats, placeholders, query parameters
+	//  - https://docs.netlify.com/routing/redirects/
+	//  - https://docs.netlify.com/routing/redirects/redirect-options/
+	ConfigFile    = "_redirects"
+	maxConfigSize = 32 * 1024
+)
+
+var (
+	// ErrNoRedirect is the error thrown when a no redirect rule matches while trying to Rewrite URL.
+	// This means that no redirect applies to the URL and you can fallback to serving actual content instead.
+	ErrNoRedirect                      = errors.New("no redirect found")
+	errConfigNotFound                  = errors.New("_redirects file not found")
+	errNeedRegularFile                 = errors.New("_redirects needs to be a regular file (not a directory)")
+	errFileTooLarge                    = errors.New("_redirects file too large")
+	errFailedToOpenConfig              = errors.New("unable to open _redirects file")
+	errFailedToParseConfig             = errors.New("failed to parse _redirects file")
+	errFailedToParseURL                = errors.New("unable to parse URL")
+	errNoDomainLevelRedirects          = errors.New("no domain-level redirects to outside sites")
+	errNoStartingForwardSlashInURLPath = errors.New("url path must start with forward slash /")
+	errNoSplats                        = errors.New("splats are not supported")
+	errNoPlaceholders                  = errors.New("placeholders are not supported")
+	errNoParams                        = errors.New("params not supported")
+	errUnsupportedStatus               = errors.New("status not supported")
+	errNoForce                         = errors.New("force! not supported")
+	regexpPlaceholder                  = regexp.MustCompile(`(?i)/:[a-z]+`)
+)
+
+type Redirects struct {
+	rules []netlifyRedirects.Rule
+	error error
+}
+
+// Status maps over each redirect rule and returns any error message
+func (r *Redirects) Status() string {
+	if r.error != nil {
+		return fmt.Sprintf("parse error: %s", r.error.Error())
+	}
+
+	messages := make([]string, 0, len(r.rules)+1)
+	messages = append(messages, fmt.Sprintf("%d rules", len(r.rules)))
+
+	for i, rule := range r.rules {
+		if err := validateRule(rule); err != nil {
+			messages = append(messages, fmt.Sprintf("rule %d: error: %s", i+1, err.Error()))
+		} else {
+			messages = append(messages, fmt.Sprintf("rule %d: valid", i+1))
+		}
+	}
+
+	return strings.Join(messages, "\n")
+}
+
+func validateURL(urlText string) error {
+	url, err := url.Parse(urlText)
+	if err != nil {
+		return errFailedToParseURL
+	}
+
+	// No support for domain-level redirects to outside sites:
+	// - `https://google.com`
+	// - `//google.com`
+	if url.Host != "" || url.Scheme != "" {
+		return errNoDomainLevelRedirects
+	}
+
+	// No parent traversing relative URL's with `./` or `../`
+	// No ambiguous URLs like bare domains `GitLab.com`
+	if !strings.HasPrefix(url.Path, "/") {
+		return errNoStartingForwardSlashInURLPath
+	}
+
+	// No support for splats, https://docs.netlify.com/routing/redirects/redirect-options/#splats
+	if strings.Contains(url.Path, "*") {
+		return errNoSplats
+	}
+
+	// No support for placeholders, https://docs.netlify.com/routing/redirects/redirect-options/#placeholders
+	if regexpPlaceholder.MatchString(url.Path) {
+		return errNoPlaceholders
+	}
+
+	return nil
+}
+
+func validateRule(r netlifyRedirects.Rule) error {
+	if err := validateURL(r.From); err != nil {
+		return err
+	}
+
+	if err := validateURL(r.To); err != nil {
+		return err
+	}
+
+	// No support for query parameters, https://docs.netlify.com/routing/redirects/redirect-options/#query-parameters
+	if r.Params != nil {
+		return errNoParams
+	}
+
+	// We strictly validate return status codes
+	switch r.Status {
+	case http.StatusMovedPermanently, http.StatusFound:
+		// noop
+	default:
+		return errUnsupportedStatus
+	}
+
+	// No support for rules that use ! force
+	if r.Force {
+		return errNoForce
+	}
+
+	return nil
+}
+
+func normalizePath(path string) string {
+	return strings.TrimSuffix(path, "/") + "/"
+}
+
+func (r *Redirects) match(url *url.URL) *netlifyRedirects.Rule {
+	for _, rule := range r.rules {
+		// TODO: Likely this should include host comparison once we have domain-level redirects
+		if normalizePath(rule.From) == normalizePath(url.Path) && validateRule(rule) == nil {
+			return &rule
+		}
+	}
+
+	return nil
+}
+
+// Rewrite takes in a URL and uses the parsed Netlify rules to rewrite
+// the URL to the new location if it matches any rule
+func (r *Redirects) Rewrite(url *url.URL) (*url.URL, int, error) {
+	rule := r.match(url)
+	if rule == nil {
+		return nil, 0, ErrNoRedirect
+	}
+
+	newURL, err := url.Parse(rule.To)
+	log.WithFields(log.Fields{
+		"url":         url,
+		"newURL":      newURL,
+		"err":         err,
+		"rule.From":   rule.From,
+		"rule.To":     rule.To,
+		"rule.Status": rule.Status,
+	}).Debug("Rewrite")
+	return newURL, rule.Status, err
+}
+
+// ParseRedirects decodes Netlify style redirects from the projects `.../public/_redirects`
+// https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file
+func ParseRedirects(ctx context.Context, root vfs.Root) *Redirects {
+	fi, err := root.Lstat(ctx, ConfigFile)
+	if err != nil {
+		return &Redirects{error: errConfigNotFound}
+	}
+
+	if !fi.Mode().IsRegular() {
+		return &Redirects{error: errNeedRegularFile}
+	}
+
+	if fi.Size() > maxConfigSize {
+		return &Redirects{error: errFileTooLarge}
+	}
+
+	reader, err := root.Open(ctx, ConfigFile)
+	if err != nil {
+		return &Redirects{error: errFailedToOpenConfig}
+	}
+	defer reader.Close()
+
+	redirectRules, err := netlifyRedirects.Parse(reader)
+	if err != nil {
+		return &Redirects{error: errFailedToParseConfig}
+	}
+
+	return &Redirects{rules: redirectRules}
+}
diff --git a/internal/redirects/redirects_benchmark_test.go b/internal/redirects/redirects_benchmark_test.go
new file mode 100644
index 00000000..1a05a00c
--- /dev/null
+++ b/internal/redirects/redirects_benchmark_test.go
@@ -0,0 +1,69 @@
+package redirects
+
+import (
+	"context"
+	"io/ioutil"
+	"net/url"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
+)
+
+func generateRedirectsFile(dirPath string, count int) error {
+	content := strings.Repeat("/goto.html /target.html 301\n", count)
+	content = content + "/entrance.html /exit.html 301\n"
+
+	return ioutil.WriteFile(path.Join(dirPath, ConfigFile), []byte(content), 0600)
+}
+
+func benchmarkRedirectsRewrite(b *testing.B, redirectsCount int) {
+	ctx := context.Background()
+
+	root, tmpDir, cleanup := testhelpers.TmpDir(nil, "ParseRedirects_benchmarks")
+	defer cleanup()
+
+	err := generateRedirectsFile(tmpDir, redirectsCount)
+	require.NoError(b, err)
+
+	url, err := url.Parse("/entrance.html")
+	require.NoError(b, err)
+
+	redirects := ParseRedirects(ctx, root)
+	require.NoError(b, redirects.error)
+
+	for i := 0; i < b.N; i++ {
+		_, _, err := redirects.Rewrite(url)
+		require.NoError(b, err)
+	}
+}
+
+func BenchmarkRedirectsRewrite(b *testing.B) {
+	b.Run("10 redirects", func(b *testing.B) { benchmarkRedirectsRewrite(b, 10) })
+	b.Run("100 redirects", func(b *testing.B) { benchmarkRedirectsRewrite(b, 100) })
+	b.Run("1000 redirects", func(b *testing.B) { benchmarkRedirectsRewrite(b, 1000) })
+}
+
+func benchmarkRedirectsParseRedirects(b *testing.B, redirectsCount int) {
+	ctx := context.Background()
+
+	root, tmpDir, cleanup := testhelpers.TmpDir(nil, "ParseRedirects_benchmarks")
+	defer cleanup()
+
+	err := generateRedirectsFile(tmpDir, redirectsCount)
+	require.NoError(b, err)
+
+	for i := 0; i < b.N; i++ {
+		redirects := ParseRedirects(ctx, root)
+		require.NoError(b, redirects.error)
+	}
+}
+
+func BenchmarkRedirectsParseRedirects(b *testing.B) {
+	b.Run("10 redirects", func(b *testing.B) { benchmarkRedirectsParseRedirects(b, 10) })
+	b.Run("100 redirects", func(b *testing.B) { benchmarkRedirectsParseRedirects(b, 100) })
+	b.Run("1000 redirects", func(b *testing.B) { benchmarkRedirectsParseRedirects(b, 1000) })
+}
diff --git a/internal/redirects/redirects_test.go b/internal/redirects/redirects_test.go
new file mode 100644
index 00000000..f538fc4b
--- /dev/null
+++ b/internal/redirects/redirects_test.go
@@ -0,0 +1,296 @@
+package redirects
+
+import (
+	"context"
+	"io/ioutil"
+	"net/url"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	netlifyRedirects "github.com/tj/go-redirects"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
+)
+
+func TestRedirectsValidateUrl(t *testing.T) {
+	tests := []struct {
+		name        string
+		url         string
+		expectedErr string
+	}{
+		{
+			name:        "Valid url",
+			url:         "/goto.html",
+			expectedErr: "",
+		},
+		{
+			name:        "No domain-level redirects",
+			url:         "https://GitLab.com",
+			expectedErr: errNoDomainLevelRedirects.Error(),
+		},
+		{
+			name:        "No Schema-less URL domain-level redirects",
+			url:         "//GitLab.com/pages.html",
+			expectedErr: errNoDomainLevelRedirects.Error(),
+		},
+		{
+			name:        "No bare domain-level redirects",
+			url:         "GitLab.com",
+			expectedErr: errNoStartingForwardSlashInURLPath.Error(),
+		},
+		{
+			name:        "No parent traversing relative URL",
+			url:         "../target.html",
+			expectedErr: errNoStartingForwardSlashInURLPath.Error(),
+		},
+		{
+			name:        "No splats",
+			url:         "/blog/*",
+			expectedErr: errNoSplats.Error(),
+		},
+		{
+			name:        "No Placeholders",
+			url:         "/news/:year/:month/:date/:slug",
+			expectedErr: errNoPlaceholders.Error(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateURL(tt.url)
+			if tt.expectedErr != "" {
+				require.EqualError(t, err, tt.expectedErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestRedirectsValidateRule(t *testing.T) {
+	tests := []struct {
+		name        string
+		rule        string
+		expectedErr string
+	}{
+		{
+			name:        "valid rule",
+			rule:        "/goto.html /target.html 301",
+			expectedErr: "",
+		},
+		{
+			name:        "invalid From URL",
+			rule:        "invalid.com /teapot.html 302",
+			expectedErr: errNoStartingForwardSlashInURLPath.Error(),
+		},
+		{
+			name:        "invalid To URL",
+			rule:        "/goto.html invalid.com",
+			expectedErr: errNoStartingForwardSlashInURLPath.Error(),
+		},
+		{
+			name: "No parameters",
+			rule: "/	/something	302	foo=bar",
+			expectedErr: errNoParams.Error(),
+		},
+		{
+			name:        "Invalid status",
+			rule:        "/goto.html /target.html 418",
+			expectedErr: errUnsupportedStatus.Error(),
+		},
+		{
+			name:        "Force not supported",
+			rule:        "/goto.html /target.html 302!",
+			expectedErr: errNoForce.Error(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rules, err := netlifyRedirects.ParseString(tt.rule)
+			require.NoError(t, err)
+
+			err = validateRule(rules[0])
+			if tt.expectedErr != "" {
+				require.EqualError(t, err, tt.expectedErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestRedirectsRewrite(t *testing.T) {
+	tests := []struct {
+		name           string
+		url            string
+		rule           string
+		expectedURL    string
+		expectedStatus int
+		expectedErr    string
+	}{
+		{
+			name:           "No rules given",
+			url:            "/no-redirect/",
+			rule:           "",
+			expectedURL:    "",
+			expectedStatus: 0,
+			expectedErr:    ErrNoRedirect.Error(),
+		},
+		{
+			name:           "No matching rules",
+			url:            "/no-redirect/",
+			rule:           "/cake-portal.html  /still-alive.html 301",
+			expectedURL:    "",
+			expectedStatus: 0,
+			expectedErr:    ErrNoRedirect.Error(),
+		},
+		{
+			name:           "Matching rule redirects",
+			url:            "/cake-portal.html",
+			rule:           "/cake-portal.html  /still-alive.html 301",
+			expectedURL:    "/still-alive.html",
+			expectedStatus: 301,
+			expectedErr:    "",
+		},
+		{
+			name:           "Does not redirect to invalid rule",
+			url:            "/goto.html",
+			rule:           "/goto.html GitLab.com 301",
+			expectedURL:    "",
+			expectedStatus: 0,
+			expectedErr:    ErrNoRedirect.Error(),
+		},
+		{
+			name:           "Matches trailing slash rule to no trailing slash URL",
+			url:            "/cake-portal",
+			rule:           "/cake-portal/  /still-alive/ 301",
+			expectedURL:    "/still-alive/",
+			expectedStatus: 301,
+			expectedErr:    "",
+		},
+		{
+			name:           "Matches trailing slash rule to trailing slash URL",
+			url:            "/cake-portal/",
+			rule:           "/cake-portal/  /still-alive/ 301",
+			expectedURL:    "/still-alive/",
+			expectedStatus: 301,
+			expectedErr:    "",
+		},
+		{
+			name:           "Matches no trailing slash rule to no trailing slash URL",
+			url:            "/cake-portal",
+			rule:           "/cake-portal  /still-alive 301",
+			expectedURL:    "/still-alive",
+			expectedStatus: 301,
+			expectedErr:    "",
+		},
+		{
+			name:           "Matches no trailing slash rule to trailing slash URL",
+			url:            "/cake-portal/",
+			rule:           "/cake-portal  /still-alive 301",
+			expectedURL:    "/still-alive",
+			expectedStatus: 301,
+			expectedErr:    "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Redirects{}
+
+			if tt.rule != "" {
+				rules, err := netlifyRedirects.ParseString(tt.rule)
+				require.NoError(t, err)
+				r.rules = rules
+			}
+
+			url, err := url.Parse(tt.url)
+			require.NoError(t, err)
+
+			toURL, status, err := r.Rewrite(url)
+
+			if tt.expectedURL != "" {
+				require.Equal(t, tt.expectedURL, toURL.String())
+			} else {
+				require.Nil(t, toURL)
+			}
+
+			require.Equal(t, tt.expectedStatus, status)
+
+			if tt.expectedErr != "" {
+				require.EqualError(t, err, tt.expectedErr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestRedirectsParseRedirects(t *testing.T) {
+	ctx := context.Background()
+
+	root, tmpDir, cleanup := testhelpers.TmpDir(t, "ParseRedirects_tests")
+	defer cleanup()
+
+	tests := []struct {
+		name          string
+		redirectsFile string
+		expectedRules int
+		expectedErr   string
+	}{
+		{
+			name:          "No `_redirects` file present",
+			redirectsFile: "",
+			expectedRules: 0,
+			expectedErr:   errConfigNotFound.Error(),
+		},
+		{
+			name:          "Everything working as expected",
+			redirectsFile: `/goto.html /target.html 301`,
+			expectedRules: 1,
+			expectedErr:   "",
+		},
+		{
+			name:          "Invalid _redirects syntax gives no rules",
+			redirectsFile: `foobar::baz`,
+			expectedRules: 0,
+			expectedErr:   "",
+		},
+		{
+			name:          "Config file too big",
+			redirectsFile: strings.Repeat("a", 2*maxConfigSize),
+			expectedRules: 0,
+			expectedErr:   errFileTooLarge.Error(),
+		},
+		// In future versions of `github.com/tj/go-redirects`,
+		// this may not throw a parsing error and this test could be removed
+		{
+			name:          "Parsing error is caught",
+			redirectsFile: "/store id=:id  /blog/:id  301",
+			expectedRules: 0,
+			expectedErr:   errFailedToParseConfig.Error(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.redirectsFile != "" {
+				err := ioutil.WriteFile(path.Join(tmpDir, ConfigFile), []byte(tt.redirectsFile), 0600)
+				require.NoError(t, err)
+			}
+
+			redirects := ParseRedirects(ctx, root)
+
+			if tt.expectedErr != "" {
+				require.EqualError(t, redirects.error, tt.expectedErr)
+			} else {
+				require.NoError(t, redirects.error)
+			}
+
+			require.Len(t, redirects.rules, tt.expectedRules)
+		})
+	}
+}