Merge branch '571-refactor-acceptance-part-2' into 'master'

Use gitlab stub server in most tests in serving_test.go

See merge request gitlab-org/gitlab-pages!493

8 files changed, 93 insertions, 52 deletions

diff --git a/shared/pages/group.404/domain.404/public.zip b/shared/pages/group.404/domain.404/public.zip
new file mode 100644
index 00000000..2e72c112
--- /dev/null
+++ b/shared/pages/group.404/domain.404/public.zip
diff --git a/shared/pages/group.404/group.404.gitlab-example.com/public.zip b/shared/pages/group.404/group.404.gitlab-example.com/public.zip
new file mode 100644
index 00000000..66d27243
--- /dev/null
+++ b/shared/pages/group.404/group.404.gitlab-example.com/public.zip
diff --git a/shared/pages/group.404/project.404/public.zip b/shared/pages/group.404/project.404/public.zip
new file mode 100644
index 00000000..7f291f05
--- /dev/null
+++ b/shared/pages/group.404/project.404/public.zip
diff --git a/shared/pages/group.404/project.no.404/public.zip b/shared/pages/group.404/project.no.404/public.zip
new file mode 100644
index 00000000..ae3b8b76
--- /dev/null
+++ b/shared/pages/group.404/project.no.404/public.zip
diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go
index c434ce37..97387786 100644
--- a/test/acceptance/helpers_test.go
+++ b/test/acceptance/helpers_test.go
@@ -25,6 +25,7 @@ import (
 	"golang.org/x/net/nettest"
 
 	"gitlab.com/gitlab-org/gitlab-pages/internal/request"
+	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
 	"gitlab.com/gitlab-org/gitlab-pages/test/acceptance/testdata"
 )
 
@@ -232,16 +233,27 @@ func RunPagesProcessWithOutput(t *testing.T, pagesBinary string, listeners []Lis
 	return runPagesProcess(t, true, pagesBinary, listeners, promPort, nil, extraArgs...)
 }
 
-func RunPagesProcessWithStubGitLabServer(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, promPort string, envs []string, extraArgs ...string) (*LogCaptureBuffer, func()) {
-	source := NewGitlabDomainsSourceStub(t, &stubOpts{})
+func RunPagesProcessWithStubGitLabServer(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, envs []string, extraArgs ...string) (*LogCaptureBuffer, func()) {
+	chdir := false
+	chdirCleanup := testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	opts := &stubOpts{
+		pagesRoot: wd,
+	}
+
+	source := NewGitlabDomainsSourceStub(t, opts)
 
 	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
-	pagesArgs := append([]string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab"}, extraArgs...)
+	pagesArgs := append([]string{"-pages-root", wd, "-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab"}, extraArgs...)
 
-	logBuf, cleanup := runPagesProcess(t, wait, pagesBinary, listeners, promPort, envs, pagesArgs...)
+	logBuf, cleanup := runPagesProcess(t, wait, pagesBinary, listeners, "", envs, pagesArgs...)
 
 	return logBuf, func() {
 		source.Close()
+		chdirCleanup()
 		cleanup()
 	}
 }
diff --git a/test/acceptance/metrics_test.go b/test/acceptance/metrics_test.go
index 1b58e081..b1166bc7 100644
--- a/test/acceptance/metrics_test.go
+++ b/test/acceptance/metrics_test.go
@@ -14,7 +14,7 @@ func TestPrometheusMetricsCanBeScraped(t *testing.T) {
 	_, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
 	defer cleanup()
 
-	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), ":42345", []string{}, "-max-conns=10")
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{}, "-max-conns=10", "-metrics-address=:42345")
 	defer teardown()
 
 	// need to call an actual resource to populate certain metrics e.g. gitlab_pages_domains_source_api_requests_total
diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go
index 59c53510..ba0069b5 100644
--- a/test/acceptance/serving_test.go
+++ b/test/acceptance/serving_test.go
@@ -11,13 +11,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-
-	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
 )
 
 func TestUnknownHostReturnsNotFound(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -31,7 +30,8 @@ func TestUnknownHostReturnsNotFound(t *testing.T) {
 
 func TestUnknownProjectReturnsNotFound(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, []ListenSpec{httpListener}, []string{})
 	defer teardown()
 
 	rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "/nonexistent/")
@@ -42,34 +42,25 @@ func TestUnknownProjectReturnsNotFound(t *testing.T) {
 
 func TestGroupDomainReturns200(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "/")
 	require.NoError(t, err)
 	defer rsp.Body.Close()
 	require.Equal(t, http.StatusOK, rsp.StatusCode)
-}
-
-func TestKnownHostReturns200(t *testing.T) {
-	skipUnlessEnabled(t)
-	chdir := false
-	defer testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)()
 
-	wd, err := os.Getwd()
+	body, err := ioutil.ReadAll(rsp.Body)
 	require.NoError(t, err)
 
-	opts := &stubOpts{
-		pagesRoot: wd,
-	}
-
-	source := NewGitlabDomainsSourceStub(t, opts)
-	defer source.Close()
+	require.Equal(t, string(body), "OK\n")
+}
 
-	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
+func TestKnownHostReturns200(t *testing.T) {
+	skipUnlessEnabled(t)
 
-	pagesArgs := []string{"-pages-root", wd, "-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab"}
-	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, supportedListeners(), "", []string{}, pagesArgs...)
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	tests := []struct {
@@ -127,6 +118,7 @@ func TestKnownHostReturns200(t *testing.T) {
 	}
 }
 
+// TODO: remove along with https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382
 func TestNestedSubgroups(t *testing.T) {
 	skipUnlessEnabled(t)
 
@@ -237,7 +229,8 @@ func TestCustom404(t *testing.T) {
 
 func TestCORSWhenDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "", "-disable-cross-origin-requests")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{}, "-disable-cross-origin-requests")
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -253,7 +246,8 @@ func TestCORSWhenDisabled(t *testing.T) {
 
 func TestCORSAllowsGET(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -270,7 +264,7 @@ func TestCORSAllowsGET(t *testing.T) {
 func TestCORSForbidsPOST(t *testing.T) {
 	skipUnlessEnabled(t)
 
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -284,8 +278,7 @@ func TestCORSForbidsPOST(t *testing.T) {
 
 func TestCustomHeaders(t *testing.T) {
 	skipUnlessEnabled(t)
-
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "", "-header", "X-Test1:Testing1", "-header", "X-Test2:Testing2")
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{}, "-header", "X-Test1:Testing1", "-header", "X-Test2:Testing2")
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -300,7 +293,7 @@ func TestCustomHeaders(t *testing.T) {
 func TestKnownHostWithPortReturns200(t *testing.T) {
 	skipUnlessEnabled(t)
 
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	for _, spec := range supportedListeners() {
@@ -314,7 +307,8 @@ func TestKnownHostWithPortReturns200(t *testing.T) {
 
 func TestHttpToHttpsRedirectDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "project/")
@@ -330,7 +324,8 @@ func TestHttpToHttpsRedirectDisabled(t *testing.T) {
 
 func TestHttpToHttpsRedirectEnabled(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "", "-redirect-http=true")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{}, "-redirect-http=true")
 	defer teardown()
 
 	rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "project/")
@@ -348,9 +343,12 @@ func TestHttpToHttpsRedirectEnabled(t *testing.T) {
 
 func TestHttpsOnlyGroupEnabled(t *testing.T) {
 	skipUnlessEnabled(t)
+
 	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
 	defer teardown()
 
+	// TODO: allow configuring HTTPS responses from stub https://gitlab.com/gitlab-org/gitlab-pages/-/issues/571
+	// Related MR in progress https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/498
 	rsp, err := GetRedirectPage(t, httpListener, "group.https-only.gitlab-example.com", "project1/")
 	require.NoError(t, err)
 	defer rsp.Body.Close()
@@ -359,6 +357,7 @@ func TestHttpsOnlyGroupEnabled(t *testing.T) {
 
 func TestHttpsOnlyGroupDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
+
 	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
 	defer teardown()
 
@@ -370,6 +369,7 @@ func TestHttpsOnlyGroupDisabled(t *testing.T) {
 
 func TestHttpsOnlyProjectEnabled(t *testing.T) {
 	skipUnlessEnabled(t)
+
 	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
 	defer teardown()
 
@@ -381,6 +381,7 @@ func TestHttpsOnlyProjectEnabled(t *testing.T) {
 
 func TestHttpsOnlyProjectDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
+
 	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
 	defer teardown()
 
@@ -392,6 +393,7 @@ func TestHttpsOnlyProjectDisabled(t *testing.T) {
 
 func TestHttpsOnlyDomainDisabled(t *testing.T) {
 	skipUnlessEnabled(t)
+
 	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
 	defer teardown()
 
@@ -585,19 +587,14 @@ func TestGitLabSourceBecomesUnauthorized(t *testing.T) {
 func TestKnownHostInReverseProxySetupReturns200(t *testing.T) {
 	skipUnlessEnabled(t)
 
-	specs := []ListenSpec{
-		proxyListener,
-	}
-	teardown := RunPagesProcess(t, *pagesBinary, specs, "")
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, []ListenSpec{proxyListener}, []string{})
 	defer teardown()
 
-	for _, spec := range specs {
-		rsp, err := GetProxiedPageFromListener(t, spec, "localhost", "group.gitlab-example.com", "project/")
+	rsp, err := GetProxiedPageFromListener(t, proxyListener, "localhost", "group.gitlab-example.com", "project/")
 
-		require.NoError(t, err)
-		rsp.Body.Close()
-		require.Equal(t, http.StatusOK, rsp.StatusCode)
-	}
+	require.NoError(t, err)
+	rsp.Body.Close()
+	require.Equal(t, http.StatusOK, rsp.StatusCode)
 }
 
 func TestDomainResolverError(t *testing.T) {
@@ -696,7 +693,8 @@ func doCrossOriginRequest(t *testing.T, spec ListenSpec, method, reqMethod, url
 
 func TestQueryStringPersistedInSlashRewrite(t *testing.T) {
 	skipUnlessEnabled(t)
-	teardown := RunPagesProcess(t, *pagesBinary, supportedListeners(), "")
+
+	_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, supportedListeners(), []string{})
 	defer teardown()
 
 	rsp, err := GetRedirectPage(t, httpsListener, "group.gitlab-example.com", "project?q=test")
@@ -733,8 +731,7 @@ func TestServerRepliesWithHeaders(t *testing.T) {
 	for name, test := range tests {
 		testFn := func(envArgs, headerArgs []string) func(*testing.T) {
 			return func(t *testing.T) {
-				teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, []ListenSpec{httpListener}, "", envArgs, headerArgs...)
-
+				_, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, []ListenSpec{httpListener}, envArgs, headerArgs...)
 				defer teardown()
 
 				rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "/")
@@ -780,7 +777,7 @@ func TestServerRepliesWithHeaders(t *testing.T) {
 func TestDiskDisabledFailsToServeFileAndLocalContent(t *testing.T) {
 	skipUnlessEnabled(t)
 
-	logBuf, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, []ListenSpec{httpListener}, "", nil, "-enable-disk=false")
+	logBuf, teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, []ListenSpec{httpListener}, nil, "-enable-disk=false")
 	defer teardown()
 
 	for host, suffix := range map[string]string{
diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go
index bf81e090..a1c99955 100644
--- a/test/acceptance/testdata/api_responses.go
+++ b/test/acceptance/testdata/api_responses.go
@@ -21,8 +21,10 @@ var DomainResponses = map[string]responseFn{
 	"zip-from-disk-not-found.gitlab.io": ZipFromFileNotFound,
 	"zip-not-allowed-path.gitlab.io":    ZipFromNotAllowedPath,
 	// test assume the working dir is inside shared/pages/
-	"group.gitlab-example.com":        GenerateVirtualDomainFromDir("group"),
-	"CapitalGroup.gitlab-example.com": GenerateVirtualDomainFromDir("CapitalGroup"),
+	"group.gitlab-example.com":        GenerateVirtualDomainFromDir("group", "group.gitlab-example.com"),
+	"CapitalGroup.gitlab-example.com": GenerateVirtualDomainFromDir("CapitalGroup", "CapitalGroup.gitlab-example.com"),
+	"group.404.gitlab-example.com":    GenerateVirtualDomainFromDir("group.404", "group.404.gitlab-example.com"),
+	"domain.404.com":                  domain404,
 	// NOTE: before adding more domains here, generate the zip archive by running (per project)
 	// make zip PROJECT_SUBDIR=group/serving
 	// make zip PROJECT_SUBDIR=group/project2
@@ -97,7 +99,7 @@ func ZipFromNotAllowedPath(t *testing.T, wd string) api.VirtualDomain {
 
 // GenerateVirtualDomainFromDir walks the subdirectory inside of shared/pages/ to find any zip archives.
 // It works for subdomains of pages-domain but not for custom domains (yet)
-func GenerateVirtualDomainFromDir(dir string) responseFn {
+func GenerateVirtualDomainFromDir(dir, rootDomain string) responseFn {
 	return func(t *testing.T, wd string) api.VirtualDomain {
 		t.Helper()
 
@@ -131,8 +133,15 @@ func GenerateVirtualDomainFromDir(dir string) responseFn {
 			prefix := strings.TrimPrefix(project, dir)
 			prefix = strings.TrimSuffix(prefix, "/"+filepath.Base(project))
 
+			// use / as prefix when the current prefix matches the rootDomain, e.g.
+			// if request is group.gitlab-example.com/ and group/group.gitlab-example.com/public.zip exists
+			if prefix == "/"+rootDomain {
+				prefix = "/"
+			}
+
 			lookupPath := api.LookupPath{
-				// TODO: find a way to configure this
+				// TODO: allow configuring response
+				// Related MR in progress https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/498
 				ProjectID:     123,
 				AccessControl: false,
 				HTTPSOnly:     false,
@@ -151,3 +160,26 @@ func GenerateVirtualDomainFromDir(dir string) responseFn {
 		}
 	}
 }
+
+// domain404 hardcoding for now, will implement a better solution in a follow up MR
+//TODO: remove hardcoded custom domains: https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/498
+func domain404(t *testing.T, wd string) api.VirtualDomain {
+	t.Helper()
+
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type: "zip",
+					Path: fmt.Sprintf("file://%s/group.404/domain.404/public.zip", wd),
+				},
+			},
+		},
+	}
+}