Add extra tests for serving zip archives from disk

6 files changed, 252 insertions, 41 deletions

diff --git a/internal/testhelpers/testhelpers.go b/internal/testhelpers/testhelpers.go
index 422a3d9a..f731166b 100644
--- a/internal/testhelpers/testhelpers.go
+++ b/internal/testhelpers/testhelpers.go
@@ -1,10 +1,12 @@
 package testhelpers
 
 import (
+	"fmt"
 	"mime"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"testing"
 
 	"github.com/sirupsen/logrus"
@@ -55,3 +57,14 @@ func AssertLogContains(t *testing.T, wantLogEntry string, entries []*logrus.Entr
 		require.Contains(t, messages, wantLogEntry)
 	}
 }
+
+// ToFileProtocol appends the file:// protocol to the current os.Getwd
+// and formats path to be a full filepath
+func ToFileProtocol(t *testing.T, path string) string {
+	t.Helper()
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	return fmt.Sprintf("file://%s/%s", wd, path)
+}
diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go
index e7dd3017..658228d9 100644
--- a/internal/vfs/zip/archive_test.go
+++ b/internal/vfs/zip/archive_test.go
@@ -33,9 +33,11 @@ var (
 )
 
 func TestOpen(t *testing.T) {
-	zip, cleanup := openZipArchive(t, nil)
-	defer cleanup()
+	t.Run("open_from_server", runZipTest(t, testOpen, false))
+	t.Run("open_from_disk", runZipTest(t, testOpen, true))
+}
 
+func testOpen(t *testing.T, zip *zipArchive) {
 	tests := map[string]struct {
 		file            string
 		expectedContent string
@@ -190,9 +192,11 @@ func TestOpenCached(t *testing.T) {
 }
 
 func TestLstat(t *testing.T) {
-	zip, cleanup := openZipArchive(t, nil)
-	defer cleanup()
+	t.Run("lstat_from_server", runZipTest(t, testLstat, false))
+	t.Run("lstat_from_disk", runZipTest(t, testLstat, true))
+}
 
+func testLstat(t *testing.T, zip *zipArchive) {
 	tests := map[string]struct {
 		file         string
 		isDir        bool
@@ -270,9 +274,11 @@ func TestLstat(t *testing.T) {
 }
 
 func TestReadLink(t *testing.T) {
-	zip, cleanup := openZipArchive(t, nil)
-	defer cleanup()
+	t.Run("read_link_from_server", runZipTest(t, testReadLink, false))
+	t.Run("read_link_from_disk", runZipTest(t, testReadLink, true))
+}
 
+func testReadLink(t *testing.T, zip *zipArchive) {
 	tests := map[string]struct {
 		file        string
 		expectedErr error
@@ -314,7 +320,7 @@ func TestReadLink(t *testing.T) {
 
 func TestReadlinkCached(t *testing.T) {
 	var requests int64
-	zip, cleanup := openZipArchive(t, &requests)
+	zip, cleanup := openZipArchive(t, &requests, false)
 	defer cleanup()
 
 	t.Run("readlink first time", func(t *testing.T) {
@@ -370,7 +376,7 @@ func TestReadArchiveFails(t *testing.T) {
 	require.EqualError(t, err, os.ErrNotExist.Error())
 }
 
-func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) {
+func openZipArchive(t *testing.T, requests *int64, fromDisk bool) (*zipArchive, func()) {
 	t.Helper()
 
 	if requests == nil {
@@ -382,14 +388,20 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) {
 	fs := New(zipCfg).(*zipVFS)
 	zip := newArchive(fs, time.Second)
 
-	err := zip.openArchive(context.Background(), testServerURL+"/public.zip")
-	require.NoError(t, err)
+	if fromDisk {
+		fileName := testhelpers.ToFileProtocol(t, "group/zip.gitlab.io/public-without-dirs.zip")
+		err := zip.openArchive(context.Background(), fileName)
+		require.NoError(t, err)
+	} else {
+		err := zip.openArchive(context.Background(), testServerURL+"/public.zip")
+		require.NoError(t, err)
+		require.Equal(t, int64(3), atomic.LoadInt64(requests), "we expect three requests to open ZIP archive: size and two to seek central directory")
+	}
 
 	// public/ public/index.html public/404.html public/symlink.html
 	// public/subdir/ public/subdir/hello.html public/subdir/linked.html
 	// public/bad_symlink.html public/subdir/2bp3Qzs...
 	require.NotZero(t, zip.files)
-	require.Equal(t, int64(3), atomic.LoadInt64(requests), "we expect three requests to open ZIP archive: size and two to seek central directory")
 
 	return zip, func() {
 		cleanup()
@@ -401,8 +413,10 @@ func newZipFileServerURL(t *testing.T, zipFilePath string, requests *int64) (str
 
 	chdir := testhelpers.ChdirInPath(t, "../../../shared/pages", &chdirSet)
 
-	// TODO: add more tests about serving from file
-	httprange.InitClient("")
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	httprange.InitClient(wd)
 
 	m := http.NewServeMux()
 	m.HandleFunc("/public.zip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -475,3 +489,14 @@ func BenchmarkArchiveRead(b *testing.B) {
 		})
 	}
 }
+
+func runZipTest(t *testing.T, runTest func(t *testing.T, zip *zipArchive), fromDisk bool) func(t *testing.T) {
+	t.Helper()
+
+	return func(t *testing.T) {
+		zip, cleanup := openZipArchive(t, nil, fromDisk)
+		defer cleanup()
+
+		runTest(t, zip)
+	}
+}
diff --git a/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip b/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip
new file mode 100644
index 00000000..f1278bce
--- /dev/null
+++ b/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip
diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go
index 5c380938..c3432a78 100644
--- a/test/acceptance/helpers_test.go
+++ b/test/acceptance/helpers_test.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,10 +20,11 @@ import (
 	"testing"
 	"time"
 
-	proxyproto "github.com/pires/go-proxyproto"
+	"github.com/pires/go-proxyproto"
 	"github.com/stretchr/testify/require"
 
 	"gitlab.com/gitlab-org/gitlab-pages/internal/request"
+	"gitlab.com/gitlab-org/gitlab-pages/test/acceptance/testdata"
 )
 
 // The HTTPS certificate isn't signed by anyone. This http client is set up
@@ -347,7 +349,7 @@ func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraAr
 		args = append(args, "-root-key", key, "-root-cert", cert)
 	}
 
-	if !contains(args, "pages-root") {
+	if !contains(extraArgs, "-pages-root") {
 		args = append(args, "-pages-root", "../../shared/pages")
 	}
 
@@ -541,13 +543,13 @@ type stubOpts struct {
 	statusHandler       http.HandlerFunc
 	pagesHandler        http.HandlerFunc
 	pagesStatusResponse int
+	pagesRoot           string
 }
 
 func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 	t.Helper()
 	require.NotNil(t, opts)
 
-	opts.apiCalled = false
 	currentStatusCount := 0
 
 	mux := http.NewServeMux()
@@ -566,7 +568,36 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 
 	mux.HandleFunc("/api/v4/internal/pages/status", statusHandler)
 
-	pagesHandler := func(w http.ResponseWriter, r *http.Request) {
+	pagesHandler := defaultAPIHandler(t, opts)
+	if opts.pagesHandler != nil {
+		pagesHandler = opts.pagesHandler
+	}
+
+	mux.HandleFunc("/api/v4/internal/pages", pagesHandler)
+
+	return httptest.NewServer(mux)
+}
+
+func lookupFromFile(t *testing.T, domain string, w http.ResponseWriter) {
+	fixture, err := os.Open("../../shared/lookups/" + domain + ".json")
+	if os.IsNotExist(err) {
+		w.WriteHeader(http.StatusNoContent)
+
+		t.Logf("GitLab domain %s source stub served 204", domain)
+		return
+	}
+
+	defer fixture.Close()
+	require.NoError(t, err)
+
+	_, err = io.Copy(w, fixture)
+	require.NoError(t, err)
+
+	t.Logf("GitLab domain %s source stub served lookup", domain)
+}
+
+func defaultAPIHandler(t *testing.T, opts *stubOpts) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
 		domain := r.URL.Query().Get("host")
 		if domain == "127.0.0.1" {
 			// shortcut for healthy checkup done by WaitUntilRequestSucceeds
@@ -581,31 +612,16 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server {
 			return
 		}
 
-		path := "../../shared/lookups/" + domain + ".json"
-
-		fixture, err := os.Open(path)
-		if os.IsNotExist(err) {
-			w.WriteHeader(http.StatusNoContent)
-
-			t.Logf("GitLab domain %s source stub served 204", domain)
+		// check if predefined response exists
+		if responseFn, ok := testdata.DomainResponses[domain]; ok {
+			err := json.NewEncoder(w).Encode(responseFn(opts.pagesRoot))
+			require.NoError(t, err)
 			return
 		}
 
-		defer fixture.Close()
-		require.NoError(t, err)
-
-		_, err = io.Copy(w, fixture)
-		require.NoError(t, err)
-
-		t.Logf("GitLab domain %s source stub served lookup", domain)
-	}
-	if opts.pagesHandler != nil {
-		pagesHandler = opts.pagesHandler
+		// serve lookup from files
+		lookupFromFile(t, domain, w)
 	}
-
-	mux.HandleFunc("/api/v4/internal/pages", pagesHandler)
-
-	return httptest.NewServer(mux)
 }
 
 func newConfigFile(t *testing.T, configs ...string) string {
diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go
new file mode 100644
index 00000000..c7f6d9a1
--- /dev/null
+++ b/test/acceptance/testdata/api_responses.go
@@ -0,0 +1,58 @@
+package testdata
+
+import (
+	"fmt"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api"
+)
+
+type responseFn func(string) api.VirtualDomain
+
+// DomainResponses holds the predefined API responses for certain domains
+// that can be used with the GitLab API stub in acceptance tests
+var DomainResponses = map[string]responseFn{
+	"zip-from-disk.gitlab.io":           ZipFromFile,
+	"zip-from-disk-not-found.gitlab.io": ZipFromFileNotFound,
+}
+
+// ZipFromFile response for zip.gitlab.io
+func ZipFromFile(wd string) api.VirtualDomain {
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type:       "zip",
+					Path:       fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip", wd),
+					Serverless: api.Serverless{},
+				},
+			},
+		},
+	}
+}
+
+// ZipFromFile response for zip.gitlab.io
+func ZipFromFileNotFound(wd string) api.VirtualDomain {
+	return api.VirtualDomain{
+		Certificate: "",
+		Key:         "",
+		LookupPaths: []api.LookupPath{
+			{
+				ProjectID:     123,
+				AccessControl: false,
+				HTTPSOnly:     false,
+				Prefix:        "/",
+				Source: api.Source{
+					Type:       "zip",
+					Path:       fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/unknown.zip", wd),
+					Serverless: api.Serverless{},
+				},
+			},
+		},
+	}
+}
diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go
index 6257458e..0f55e891 100644
--- a/test/acceptance/zip_test.go
+++ b/test/acceptance/zip_test.go
@@ -5,14 +5,20 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
 )
 
 func TestZipServing(t *testing.T) {
 	skipUnlessEnabled(t)
 
+	_, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
+	defer cleanup()
+
 	source := NewGitlabDomainsSourceStub(t, &stubOpts{})
 	defer source.Close()
 
@@ -22,9 +28,6 @@ func TestZipServing(t *testing.T) {
 	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...)
 	defer teardown()
 
-	_, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip")
-	defer cleanup()
-
 	tests := map[string]struct {
 		host               string
 		urlSuffix          string
@@ -103,6 +106,102 @@ func TestZipServing(t *testing.T) {
 	}
 }
 
+func TestZipServingFromDisk(t *testing.T) {
+	skipUnlessEnabled(t, "not-inplace-chroot")
+
+	chdir := false
+	defer testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)()
+
+	_, cleanup := newZipFileServerURL(t, "shared/pages/group/zip.gitlab.io/public.zip")
+	defer cleanup()
+
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	source := NewGitlabDomainsSourceStub(t, &stubOpts{
+		pagesRoot: wd,
+	})
+
+	defer source.Close()
+
+	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
+
+	pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", "-pages-root", wd}
+	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...)
+	defer teardown()
+
+	tests := map[string]struct {
+		host               string
+		urlSuffix          string
+		expectedStatusCode int
+		expectedContent    string
+	}{
+		"base_domain_no_suffix": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/index.html\n",
+		},
+		"file_exists": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/index.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/index.html\n",
+		},
+		"file_exists_in_subdir": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/subdir/hello.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "zip.gitlab.io/project/subdir/hello.html\n",
+		},
+		"file_exists_symlink": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/symlink.html",
+			expectedStatusCode: http.StatusOK,
+			expectedContent:    "symlink.html->subdir/linked.html\n",
+		},
+		"dir": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/subdir/",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"file_does_not_exist": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/unknown.html",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"bad_symlink": {
+			host:               "zip-from-disk.gitlab.io",
+			urlSuffix:          "/bad-symlink.html",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "zip.gitlab.io/project/404.html\n",
+		},
+		"with_not_found_zip": {
+			host:               "zip-from-disk-not-found.gitlab.io",
+			urlSuffix:          "/",
+			expectedStatusCode: http.StatusNotFound,
+			expectedContent:    "The page you're looking for could not be found",
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+			response, err := GetPageFromListener(t, httpListener, tt.host, tt.urlSuffix)
+			require.NoError(t, err)
+			defer response.Body.Close()
+
+			require.Equal(t, tt.expectedStatusCode, response.StatusCode)
+
+			body, err := ioutil.ReadAll(response.Body)
+			require.NoError(t, err)
+
+			require.Contains(t, string(body), tt.expectedContent, "content mismatch")
+		})
+	}
+}
+
 func TestZipServingConfigShortTimeout(t *testing.T) {
 	skipUnlessEnabled(t)