From 586317c827f5878444bcc86bf953b21e7e31ee10 Mon Sep 17 00:00:00 2001 From: Jaime Martinez Date: Wed, 3 Feb 2021 17:06:02 +1100 Subject: Add extra tests for serving zip archives from disk --- internal/testhelpers/testhelpers.go | 13 +++ internal/vfs/zip/archive_test.go | 51 +++++++--- .../pages_deployments/01/artifacts.zip | Bin 0 -> 2415 bytes test/acceptance/helpers_test.go | 66 ++++++++----- test/acceptance/testdata/api_responses.go | 58 ++++++++++++ test/acceptance/zip_test.go | 105 ++++++++++++++++++++- 6 files changed, 252 insertions(+), 41 deletions(-) create mode 100644 shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip create mode 100644 test/acceptance/testdata/api_responses.go diff --git a/internal/testhelpers/testhelpers.go b/internal/testhelpers/testhelpers.go index 422a3d9a..f731166b 100644 --- a/internal/testhelpers/testhelpers.go +++ b/internal/testhelpers/testhelpers.go @@ -1,10 +1,12 @@ package testhelpers import ( + "fmt" "mime" "net/http" "net/http/httptest" "net/url" + "os" "testing" "github.com/sirupsen/logrus" @@ -55,3 +57,14 @@ func AssertLogContains(t *testing.T, wantLogEntry string, entries []*logrus.Entr require.Contains(t, messages, wantLogEntry) } } + +// ToFileProtocol appends the file:// protocol to the current os.Getwd +// and formats path to be a full filepath +func ToFileProtocol(t *testing.T, path string) string { + t.Helper() + + wd, err := os.Getwd() + require.NoError(t, err) + + return fmt.Sprintf("file://%s/%s", wd, path) +} diff --git a/internal/vfs/zip/archive_test.go b/internal/vfs/zip/archive_test.go index e7dd3017..658228d9 100644 --- a/internal/vfs/zip/archive_test.go +++ b/internal/vfs/zip/archive_test.go @@ -33,9 +33,11 @@ var ( ) func TestOpen(t *testing.T) { - zip, cleanup := openZipArchive(t, nil) - defer cleanup() + t.Run("open_from_server", runZipTest(t, testOpen, false)) + t.Run("open_from_disk", runZipTest(t, testOpen, true)) +} +func testOpen(t *testing.T, zip *zipArchive) { tests := map[string]struct { file string expectedContent string @@ -190,9 +192,11 @@ func TestOpenCached(t *testing.T) { } func TestLstat(t *testing.T) { - zip, cleanup := openZipArchive(t, nil) - defer cleanup() + t.Run("lstat_from_server", runZipTest(t, testLstat, false)) + t.Run("lstat_from_disk", runZipTest(t, testLstat, true)) +} +func testLstat(t *testing.T, zip *zipArchive) { tests := map[string]struct { file string isDir bool @@ -270,9 +274,11 @@ func TestLstat(t *testing.T) { } func TestReadLink(t *testing.T) { - zip, cleanup := openZipArchive(t, nil) - defer cleanup() + t.Run("read_link_from_server", runZipTest(t, testReadLink, false)) + t.Run("read_link_from_disk", runZipTest(t, testReadLink, true)) +} +func testReadLink(t *testing.T, zip *zipArchive) { tests := map[string]struct { file string expectedErr error @@ -314,7 +320,7 @@ func TestReadLink(t *testing.T) { func TestReadlinkCached(t *testing.T) { var requests int64 - zip, cleanup := openZipArchive(t, &requests) + zip, cleanup := openZipArchive(t, &requests, false) defer cleanup() t.Run("readlink first time", func(t *testing.T) { @@ -370,7 +376,7 @@ func TestReadArchiveFails(t *testing.T) { require.EqualError(t, err, os.ErrNotExist.Error()) } -func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) { +func openZipArchive(t *testing.T, requests *int64, fromDisk bool) (*zipArchive, func()) { t.Helper() if requests == nil { @@ -382,14 +388,20 @@ func openZipArchive(t *testing.T, requests *int64) (*zipArchive, func()) { fs := New(zipCfg).(*zipVFS) zip := newArchive(fs, time.Second) - err := zip.openArchive(context.Background(), testServerURL+"/public.zip") - require.NoError(t, err) + if fromDisk { + fileName := testhelpers.ToFileProtocol(t, "group/zip.gitlab.io/public-without-dirs.zip") + err := zip.openArchive(context.Background(), fileName) + require.NoError(t, err) + } else { + err := zip.openArchive(context.Background(), testServerURL+"/public.zip") + require.NoError(t, err) + require.Equal(t, int64(3), atomic.LoadInt64(requests), "we expect three requests to open ZIP archive: size and two to seek central directory") + } // public/ public/index.html public/404.html public/symlink.html // public/subdir/ public/subdir/hello.html public/subdir/linked.html // public/bad_symlink.html public/subdir/2bp3Qzs... require.NotZero(t, zip.files) - require.Equal(t, int64(3), atomic.LoadInt64(requests), "we expect three requests to open ZIP archive: size and two to seek central directory") return zip, func() { cleanup() @@ -401,8 +413,10 @@ func newZipFileServerURL(t *testing.T, zipFilePath string, requests *int64) (str chdir := testhelpers.ChdirInPath(t, "../../../shared/pages", &chdirSet) - // TODO: add more tests about serving from file - httprange.InitClient("") + wd, err := os.Getwd() + require.NoError(t, err) + + httprange.InitClient(wd) m := http.NewServeMux() m.HandleFunc("/public.zip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -475,3 +489,14 @@ func BenchmarkArchiveRead(b *testing.B) { }) } } + +func runZipTest(t *testing.T, runTest func(t *testing.T, zip *zipArchive), fromDisk bool) func(t *testing.T) { + t.Helper() + + return func(t *testing.T) { + zip, cleanup := openZipArchive(t, nil, fromDisk) + defer cleanup() + + runTest(t, zip) + } +} diff --git a/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip b/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip new file mode 100644 index 00000000..f1278bce Binary files /dev/null and b/shared/pages/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip differ diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index 5c380938..c3432a78 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/json" "fmt" "io" "io/ioutil" @@ -19,10 +20,11 @@ import ( "testing" "time" - proxyproto "github.com/pires/go-proxyproto" + "github.com/pires/go-proxyproto" "github.com/stretchr/testify/require" "gitlab.com/gitlab-org/gitlab-pages/internal/request" + "gitlab.com/gitlab-org/gitlab-pages/test/acceptance/testdata" ) // The HTTPS certificate isn't signed by anyone. This http client is set up @@ -347,7 +349,7 @@ func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraAr args = append(args, "-root-key", key, "-root-cert", cert) } - if !contains(args, "pages-root") { + if !contains(extraArgs, "-pages-root") { args = append(args, "-pages-root", "../../shared/pages") } @@ -541,13 +543,13 @@ type stubOpts struct { statusHandler http.HandlerFunc pagesHandler http.HandlerFunc pagesStatusResponse int + pagesRoot string } func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server { t.Helper() require.NotNil(t, opts) - opts.apiCalled = false currentStatusCount := 0 mux := http.NewServeMux() @@ -566,7 +568,36 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server { mux.HandleFunc("/api/v4/internal/pages/status", statusHandler) - pagesHandler := func(w http.ResponseWriter, r *http.Request) { + pagesHandler := defaultAPIHandler(t, opts) + if opts.pagesHandler != nil { + pagesHandler = opts.pagesHandler + } + + mux.HandleFunc("/api/v4/internal/pages", pagesHandler) + + return httptest.NewServer(mux) +} + +func lookupFromFile(t *testing.T, domain string, w http.ResponseWriter) { + fixture, err := os.Open("../../shared/lookups/" + domain + ".json") + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + + t.Logf("GitLab domain %s source stub served 204", domain) + return + } + + defer fixture.Close() + require.NoError(t, err) + + _, err = io.Copy(w, fixture) + require.NoError(t, err) + + t.Logf("GitLab domain %s source stub served lookup", domain) +} + +func defaultAPIHandler(t *testing.T, opts *stubOpts) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { domain := r.URL.Query().Get("host") if domain == "127.0.0.1" { // shortcut for healthy checkup done by WaitUntilRequestSucceeds @@ -581,31 +612,16 @@ func NewGitlabDomainsSourceStub(t *testing.T, opts *stubOpts) *httptest.Server { return } - path := "../../shared/lookups/" + domain + ".json" - - fixture, err := os.Open(path) - if os.IsNotExist(err) { - w.WriteHeader(http.StatusNoContent) - - t.Logf("GitLab domain %s source stub served 204", domain) + // check if predefined response exists + if responseFn, ok := testdata.DomainResponses[domain]; ok { + err := json.NewEncoder(w).Encode(responseFn(opts.pagesRoot)) + require.NoError(t, err) return } - defer fixture.Close() - require.NoError(t, err) - - _, err = io.Copy(w, fixture) - require.NoError(t, err) - - t.Logf("GitLab domain %s source stub served lookup", domain) - } - if opts.pagesHandler != nil { - pagesHandler = opts.pagesHandler + // serve lookup from files + lookupFromFile(t, domain, w) } - - mux.HandleFunc("/api/v4/internal/pages", pagesHandler) - - return httptest.NewServer(mux) } func newConfigFile(t *testing.T, configs ...string) string { diff --git a/test/acceptance/testdata/api_responses.go b/test/acceptance/testdata/api_responses.go new file mode 100644 index 00000000..c7f6d9a1 --- /dev/null +++ b/test/acceptance/testdata/api_responses.go @@ -0,0 +1,58 @@ +package testdata + +import ( + "fmt" + + "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api" +) + +type responseFn func(string) api.VirtualDomain + +// DomainResponses holds the predefined API responses for certain domains +// that can be used with the GitLab API stub in acceptance tests +var DomainResponses = map[string]responseFn{ + "zip-from-disk.gitlab.io": ZipFromFile, + "zip-from-disk-not-found.gitlab.io": ZipFromFileNotFound, +} + +// ZipFromFile response for zip.gitlab.io +func ZipFromFile(wd string) api.VirtualDomain { + return api.VirtualDomain{ + Certificate: "", + Key: "", + LookupPaths: []api.LookupPath{ + { + ProjectID: 123, + AccessControl: false, + HTTPSOnly: false, + Prefix: "/", + Source: api.Source{ + Type: "zip", + Path: fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/artifacts.zip", wd), + Serverless: api.Serverless{}, + }, + }, + }, + } +} + +// ZipFromFile response for zip.gitlab.io +func ZipFromFileNotFound(wd string) api.VirtualDomain { + return api.VirtualDomain{ + Certificate: "", + Key: "", + LookupPaths: []api.LookupPath{ + { + ProjectID: 123, + AccessControl: false, + HTTPSOnly: false, + Prefix: "/", + Source: api.Source{ + Type: "zip", + Path: fmt.Sprintf("file://%s/@hashed/67/06/670671cd97404156226e507973f2ab8330d3022ca96e0c93bdbdb320c41adcaf/pages_deployments/01/unknown.zip", wd), + Serverless: api.Serverless{}, + }, + }, + }, + } +} diff --git a/test/acceptance/zip_test.go b/test/acceptance/zip_test.go index 6257458e..0f55e891 100644 --- a/test/acceptance/zip_test.go +++ b/test/acceptance/zip_test.go @@ -5,14 +5,20 @@ import ( "net" "net/http" "net/http/httptest" + "os" "testing" "github.com/stretchr/testify/require" + + "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" ) func TestZipServing(t *testing.T) { skipUnlessEnabled(t) + _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + source := NewGitlabDomainsSourceStub(t, &stubOpts{}) defer source.Close() @@ -22,9 +28,6 @@ func TestZipServing(t *testing.T) { teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) defer teardown() - _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") - defer cleanup() - tests := map[string]struct { host string urlSuffix string @@ -103,6 +106,102 @@ func TestZipServing(t *testing.T) { } } +func TestZipServingFromDisk(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + + chdir := false + defer testhelpers.ChdirInPath(t, "../../shared/pages", &chdir)() + + _, cleanup := newZipFileServerURL(t, "shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + + wd, err := os.Getwd() + require.NoError(t, err) + + source := NewGitlabDomainsSourceStub(t, &stubOpts{ + pagesRoot: wd, + }) + + defer source.Close() + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab", "-pages-root", wd} + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) + defer teardown() + + tests := map[string]struct { + host string + urlSuffix string + expectedStatusCode int + expectedContent string + }{ + "base_domain_no_suffix": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/index.html\n", + }, + "file_exists": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/index.html", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/index.html\n", + }, + "file_exists_in_subdir": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/subdir/hello.html", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/subdir/hello.html\n", + }, + "file_exists_symlink": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/symlink.html", + expectedStatusCode: http.StatusOK, + expectedContent: "symlink.html->subdir/linked.html\n", + }, + "dir": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/subdir/", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "file_does_not_exist": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/unknown.html", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "bad_symlink": { + host: "zip-from-disk.gitlab.io", + urlSuffix: "/bad-symlink.html", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "with_not_found_zip": { + host: "zip-from-disk-not-found.gitlab.io", + urlSuffix: "/", + expectedStatusCode: http.StatusNotFound, + expectedContent: "The page you're looking for could not be found", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + response, err := GetPageFromListener(t, httpListener, tt.host, tt.urlSuffix) + require.NoError(t, err) + defer response.Body.Close() + + require.Equal(t, tt.expectedStatusCode, response.StatusCode) + + body, err := ioutil.ReadAll(response.Body) + require.NoError(t, err) + + require.Contains(t, string(body), tt.expectedContent, "content mismatch") + }) + } +} + func TestZipServingConfigShortTimeout(t *testing.T) { skipUnlessEnabled(t) -- cgit v1.2.3