Merge branch '485-add-more-tests-to-serve-from-file' into 'master'

Add extra tests for serving zip archives from disk

Closes #485

See merge request gitlab-org/gitlab-pages!430

3 files changed, 230 insertions, 28 deletions

diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go
index d3b4f7b9..1ece8148 100644
--- a/test/acceptance/helpers_test.go
+++ b/test/acceptance/helpers_test.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,10 +20,11 @@ import (
 	"testing"
 	"time"
 
-	proxyproto "github.com/pires/go-proxyproto"
+	"github.com/pires/go-proxyproto"
 	"github.com/stretchr/testify/require"
 
 	"gitlab.com/gitlab-org/gitlab-pages/internal/request"
+	"gitlab.com/gitlab-org/gitlab-pages/test/acceptance/testdata"
 )
 
 // The HTTPS certificate isn't signed by anyone. This http client is set up
@@ -347,7 +349,7 @@ func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraAr
 		args = append(args, "-root-key", key, "-root-cert", cert)
 	}
 
-	if !contains(args, "pages-root") {
+	if !contains(extraArgs, "-pages-root") {
 		args = append(args, "-pages-root", "../../shared/pages")
 	}
 
@@ -541,13 +543,13 @@ type stubOpts struct {
 	statusHandler       http.HandlerFunc
 	pagesHandler        http.HandlerFunc
 	pagesStatusResponse int
+	pagesRoot           string
 }
 
 func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 	t.Helper()
 	require.NotNil(t, opts)
 
-	opts.apiCalled = false
 	currentStatusCount := 0
 
 	mux := http.NewServeMux()
@@ -566,7 +568,36 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 
 	mux.HandleFunc("/api/v4/internal/pages/status", statusHandler)
 
-	pagesHandler := func(w http.ResponseWriter, r *http.Request) {
+	pagesHandler := defaultAPIHandler(t, opts)
+	if opts.pagesHandler != nil {
+		pagesHandler = opts.pagesHandler
+	}
+
+	mux.HandleFunc("/api/v4/internal/pages", pagesHandler)
+
+	return httptest.NewServer(mux)
+}
+
+func lookupFromFile(t *testing.T, domain string, w http.ResponseWriter) {
+	fixture, err := os.Open("../../shared/lookups/" + domain + ".json")
+	if os.IsNotExist(err) {
+		w.WriteHeader(http.StatusNoContent)
+
+		t.Logf("GitLab domain %s source stub served 204", domain)
+		return
+	}
+
+	defer fixture.Close()
+	require.NoError(t, err)
+
+	_, err = io.Copy(w, fixture)
+	require.NoError(t, err)
+
+	t.Logf("GitLab domain %s source stub served lookup", domain)
+}
+
+func defaultAPIHandler(t *testing.T, opts *stubOpts) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
 		domain := r.URL.Query().Get("host")
 		if domain == "127.0.0.1" {
 			// shortcut for healthy checkup done by WaitUntilRequestSucceeds
@@ -581,31 +612,16 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 			return
 		}
 
-		path := "../../shared/lookups/" + domain + ".json"
-
-		fixture, err := os.Open(path)
-		if os.IsNotExist(err) {
-			w.WriteHeader(http.StatusNoContent)
-
-			t.Logf("GitLab domain %s source stub served 204", domain)
+		// check if predefined response exists
+		if responseFn, ok := testdata.DomainResponses[domain]; ok {
+			err := json.NewEncoder(w).Encode(responseFn(opts.pagesRoot))
+			require.NoError(t, err)
 			return
 		}
 
-		defer fixture.Close()
-		require.NoError(t, err)
-
-		_, err = io.Copy(w, fixture)
-		require.NoError(t, err)
-
-		t.Logf("GitLab domain %s source stub served lookup", domain)
-	}
-	if opts.pagesHandler != nil {
-		pagesHandler = opts.pagesHandler
+		// serve lookup from files
+		lookupFromFile(t, domain, w)
 	}
-
-	mux.HandleFunc("/api/v4/internal/pages", pagesHandler)
-
-	return httptest.NewServer(mux)
 }
 
 func newConfigFile(t *testing.T, configs ...string) string {
diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go
new file mode 100644
index 00000000..a79d2130
--- /dev/null
+++ b/test/acceptance/testdata/api_responses.go
@@ -0,0 +1,81 @@
+package testdata
+
+import (
+	"fmt"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api"
+)
+
+type responseFn func(string) api.VirtualDomain
+
+// DomainResponses holds the predefined API responses for certain domains
+// that can be used with the GitLab API stub in acceptance tests
+var DomainResponses = map[string]responseFn{
+	"zip-from-disk.gitlab.io":           ZipFromFile,
+	"zip-from-disk-not-found.gitlab.io": ZipFromFileNotFound,
+	"zip-not-allowed-path.gitlab.io":    ZipFromNotAllowedPath,
+}
+
+// ZipFromFile response for zip.gitlab.io
+func ZipFromFile(wd string) api.VirtualDomain {
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type:       "zip",
+					Path:       fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip", wd),
+					Serverless: api.Serverless{},
+				},
+			},
+		},
+	}
+}
+
+// ZipFromFileNotFound response for zip-from-disk-not-found.gitlab.io
+func ZipFromFileNotFound(wd string) api.VirtualDomain {
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type:       "zip",
+					Path:       fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/unknown.zip", wd),
+					Serverless: api.Serverless{},
+				},
+			},
+		},
+	}
+}
+
+// ZipFromNotAllowedPath response for zip-not-allowed-path.gitlab.io
+func ZipFromNotAllowedPath(wd string) api.VirtualDomain {
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type: "zip",
+					// path outside of `pages-root`
+					Path:       "file:///some/random/path/public.zip",
+					Serverless: api.Serverless{},
+				},
+			},
+		},
+	}
+}
diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go
index 6257458e..a7e82d27 100644
--- a/test/acceptance/zip_test.go
+++ b/test/acceptance/zip_test.go
@@ -5,14 +5,20 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
 )
 
 func TestZipServing(t *testing.T) {
 	skipUnlessEnabled(t)
 
+	_, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
+	defer cleanup()
+
 	source := NewGitlabDomainsSourceStub(t, &stubOpts{})
 	defer source.Close()
 
@@ -22,9 +28,6 @@ func TestZipServing(t *testing.T) {
 	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...)
 	defer teardown()
 
-	_, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
-	defer cleanup()
-
 	tests := map[string]struct {
 		host               string
 		urlSuffix          string
@@ -103,6 +106,108 @@ func TestZipServing(t *testing.T) {
 	}
 }
 
+func TestZipServingFromDisk(t *testing.T) {
+	skipUnlessEnabled(t, "not-inplace-chroot")
+
+	chdir := false
+	defer testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)()
+
+	_, cleanup := newZipFileServerURL(t, "shared/pages/group/zip.gitlab.io/public.zip")
+	defer cleanup()
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	source := NewGitlabDomainsSourceStub(t, &stubOpts{
+		pagesRoot: wd,
+	})
+
+	defer source.Close()
+
+	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
+
+	pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", "-pages-root", wd}
+	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...)
+	defer teardown()
+
+	tests := map[string]struct {
+		host               string
+		urlSuffix          string
+		expectedStatusCode int
+		expectedContent    string
+	}{
+		"base_domain_no_suffix": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/index.html\n",
+		},
+		"file_exists": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/index.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/index.html\n",
+		},
+		"file_exists_in_subdir": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/subdir/hello.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/subdir/hello.html\n",
+		},
+		"file_exists_symlink": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/symlink.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "symlink.html->subdir/linked.html\n",
+		},
+		"dir": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/subdir/",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"file_does_not_exist": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/unknown.html",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"bad_symlink": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/bad-symlink.html",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"with_not_found_zip": {
+			host:               "zip-from-disk-not-found.gitlab.io",
+			urlSuffix:          "/",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "The page you're looking for could not be found",
+		},
+		"file_not_allowed_in_path": {
+			host:               "zip-not-allowed-path.gitlab.io",
+			urlSuffix:          "/",
+			expectedStatusCode: http.StatusInternalServerError,
+			expectedContent:    "Whoops, something went wrong on our end.",
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			response, err := GetPageFromListener(t, httpListener, tt.host, tt.urlSuffix)
+			require.NoError(t, err)
+			defer response.Body.Close()
+
+			require.Equal(t, tt.expectedStatusCode, response.StatusCode)
+
+			body, err := ioutil.ReadAll(response.Body)
+			require.NoError(t, err)
+
+			require.Contains(t, string(body), tt.expectedContent, "content mismatch")
+		})
+	}
+}
+
 func TestZipServingConfigShortTimeout(t *testing.T) {
 	skipUnlessEnabled(t)