diff options
author | Jaime Martinez <jmartinez@gitlab.com> | 2020-11-12 03:44:12 +0300 |
---|---|---|
committer | Jaime Martinez <jmartinez@gitlab.com> | 2020-11-12 03:44:12 +0300 |
commit | 40c602322cb6a4872bb69835d97647a68fe4da65 (patch) | |
tree | 47905df65431cbee73f7ecfeb585ea021b578b68 /test/acceptance | |
parent | 623e10999c8cfc79deeda453ff5f42d44eac9f9e (diff) |
Move all acceptance tests into the test/acceptance/ dir
Diffstat (limited to 'test/acceptance')
-rw-r--r-- | test/acceptance/acceptance_test.go | 2103 | ||||
-rw-r--r-- | test/acceptance/helpers_test.go | 581 |
2 files changed, 2684 insertions, 0 deletions
diff --git a/test/acceptance/acceptance_test.go b/test/acceptance/acceptance_test.go new file mode 100644 index 00000000..35815fc1 --- /dev/null +++ b/test/acceptance/acceptance_test.go @@ -0,0 +1,2103 @@ +// +build acceptance + +package acceptance_test + +import ( + "crypto/tls" + "fmt" + "io/ioutil" + "mime" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "regexp" + "testing" + "time" + + "github.com/namsral/flag" + "github.com/stretchr/testify/require" +) + +var pagesBinary = flag.String("gitlab-pages-binary", "../../gitlab-pages", "Path to the gitlab-pages binary") + +const ( + objectStorageMockServer = "127.0.0.1:37003" +) + +// TODO: Use TCP port 0 everywhere to avoid conflicts. The binary could output +// the actual port (and type of listener) for us to read in place of the +// hardcoded values below. +var listeners = []ListenSpec{ + {"http", "127.0.0.1", "37000"}, + {"http", "::1", "37000"}, + {"https", "127.0.0.1", "37001"}, + {"https", "::1", "37001"}, + {"proxy", "127.0.0.1", "37002"}, + {"proxy", "::1", "37002"}, +} + +var ( + httpListener = listeners[0] + httpsListener = listeners[2] + proxyListener = listeners[4] +) + +func skipUnlessEnabled(t *testing.T, conditions ...string) { + t.Helper() + + if testing.Short() { + t.Log("Acceptance tests disabled") + t.SkipNow() + } + + if _, err := os.Stat(*pagesBinary); os.IsNotExist(err) { + t.Errorf("Couldn't find gitlab-pages binary at %s", *pagesBinary) + t.FailNow() + } + + for _, condition := range conditions { + switch condition { + case "not-inplace-chroot": + if os.Getenv("TEST_DAEMONIZE") == "inplace" { + t.Log("Not supported with -daemon-inplace-chroot") + t.SkipNow() + } + default: + t.Error("Unknown condition:", condition) + t.FailNow() + } + } +} + +func TestUnknownHostReturnsNotFound(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, "invalid.invalid", "") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + } +} + +func TestUnknownProjectReturnsNotFound(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "/nonexistent/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusNotFound, rsp.StatusCode) +} + +func TestGroupDomainReturns200(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestKnownHostReturns200(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + tests := []struct { + name string + host string + path string + }{ + { + name: "lower case", + host: "group.gitlab-example.com", + path: "project/", + }, + { + name: "capital project", + host: "group.gitlab-example.com", + path: "CapitalProject/", + }, + { + name: "capital group", + host: "CapitalGroup.gitlab-example.com", + path: "project/", + }, + { + name: "capital group and project", + host: "CapitalGroup.gitlab-example.com", + path: "CapitalProject/", + }, + { + name: "subgroup", + host: "group.gitlab-example.com", + path: "subgroup/project/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, tt.host, tt.path) + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + } + }) + } +} + +func TestNestedSubgroups(t *testing.T) { + skipUnlessEnabled(t) + + maxNestedSubgroup := 21 + + pagesRoot, err := ioutil.TempDir("", "pages-root") + require.NoError(t, err) + defer os.RemoveAll(pagesRoot) + + makeProjectIndex := func(subGroupPath string) { + projectPath := path.Join(pagesRoot, "nested", subGroupPath, "project", "public") + require.NoError(t, os.MkdirAll(projectPath, 0755)) + + projectIndex := path.Join(projectPath, "index.html") + require.NoError(t, ioutil.WriteFile(projectIndex, []byte("index"), 0644)) + } + makeProjectIndex("") + + paths := []string{""} + for i := 1; i < maxNestedSubgroup*2; i++ { + subGroupPath := fmt.Sprintf("%ssub%d/", paths[i-1], i) + paths = append(paths, subGroupPath) + + makeProjectIndex(subGroupPath) + } + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-root", pagesRoot) + defer teardown() + + for nestingLevel, path := range paths { + t.Run(fmt.Sprintf("nested level %d", nestingLevel), func(t *testing.T) { + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, "nested.gitlab-example.com", path+"project/") + + require.NoError(t, err) + rsp.Body.Close() + if nestingLevel <= maxNestedSubgroup { + require.Equal(t, http.StatusOK, rsp.StatusCode) + } else { + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + } + } + }) + } +} + +func TestCustom404(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + tests := []struct { + host string + path string + content string + }{ + { + host: "group.404.gitlab-example.com", + path: "project.404/not/existing-file", + content: "Custom 404 project page", + }, + { + host: "group.404.gitlab-example.com", + path: "project.404/", + content: "Custom 404 project page", + }, + { + host: "group.404.gitlab-example.com", + path: "not/existing-file", + content: "Custom 404 group page", + }, + { + host: "group.404.gitlab-example.com", + path: "not-existing-file", + content: "Custom 404 group page", + }, + { + host: "group.404.gitlab-example.com", + content: "Custom 404 group page", + }, + { + host: "domain.404.com", + content: "Custom domain.404 page", + }, + { + host: "group.404.gitlab-example.com", + path: "project.no.404/not/existing-file", + content: "The page you're looking for could not be found.", + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s/%s", test.host, test.path), func(t *testing.T) { + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, test.host, test.path) + + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + + page, err := ioutil.ReadAll(rsp.Body) + require.NoError(t, err) + require.Contains(t, string(page), test.content) + } + }) + } +} + +func TestCORSWhenDisabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-disable-cross-origin-requests") + defer teardown() + + for _, spec := range listeners { + for _, method := range []string{"GET", "OPTIONS"} { + rsp := doCrossOriginRequest(t, method, method, spec.URL("project/")) + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "", rsp.Header.Get("Access-Control-Allow-Origin")) + require.Equal(t, "", rsp.Header.Get("Access-Control-Allow-Credentials")) + } + } +} + +func TestCORSAllowsGET(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, spec := range listeners { + for _, method := range []string{"GET", "OPTIONS"} { + rsp := doCrossOriginRequest(t, method, method, spec.URL("project/")) + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "*", rsp.Header.Get("Access-Control-Allow-Origin")) + require.Equal(t, "", rsp.Header.Get("Access-Control-Allow-Credentials")) + } + } +} + +func TestCORSForbidsPOST(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, spec := range listeners { + rsp := doCrossOriginRequest(t, "OPTIONS", "POST", spec.URL("project/")) + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "", rsp.Header.Get("Access-Control-Allow-Origin")) + require.Equal(t, "", rsp.Header.Get("Access-Control-Allow-Credentials")) + } +} + +func TestCustomHeaders(t *testing.T) { + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-header", "X-Test1:Testing1", "-header", "X-Test2:Testing2") + defer teardown() + + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, "group.gitlab-example.com:", "project/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, "Testing1", rsp.Header.Get("X-Test1")) + require.Equal(t, "Testing2", rsp.Header.Get("X-Test2")) + } +} + +func doCrossOriginRequest(t *testing.T, method, reqMethod, url string) *http.Response { + req, err := http.NewRequest(method, url, nil) + require.NoError(t, err) + + req.Host = "group.gitlab-example.com" + req.Header.Add("Origin", "example.com") + req.Header.Add("Access-Control-Request-Method", reqMethod) + + var rsp *http.Response + err = fmt.Errorf("no request was made") + for start := time.Now(); time.Since(start) < 1*time.Second; { + rsp, err = DoPagesRequest(t, req) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + require.NoError(t, err) + + rsp.Body.Close() + return rsp +} + +func TestKnownHostWithPortReturns200(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, spec := range listeners { + rsp, err := GetPageFromListener(t, spec, "group.gitlab-example.com:"+spec.Port, "project/") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + } +} + +func TestHttpToHttpsRedirectDisabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "project/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + + rsp, err = GetPageFromListener(t, httpsListener, "group.gitlab-example.com", "project/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestHttpToHttpsRedirectEnabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-redirect-http=true") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "project/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusTemporaryRedirect, rsp.StatusCode) + require.Equal(t, 1, len(rsp.Header["Location"])) + require.Equal(t, "https://group.gitlab-example.com/project/", rsp.Header.Get("Location")) + + rsp, err = GetPageFromListener(t, httpsListener, "group.gitlab-example.com", "project/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestHttpsOnlyGroupEnabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "group.https-only.gitlab-example.com", "project1/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusMovedPermanently, rsp.StatusCode) +} + +func TestHttpsOnlyGroupDisabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.https-only.gitlab-example.com", "project2/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestHttpsOnlyProjectEnabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "test.my-domain.com", "/index.html") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusMovedPermanently, rsp.StatusCode) +} + +func TestHttpsOnlyProjectDisabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "test2.my-domain.com", "/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestHttpsOnlyDomainDisabled(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "no.cert.com", "/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestPrometheusMetricsCanBeScraped(t *testing.T) { + skipUnlessEnabled(t) + + _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + + teardown := RunPagesProcessWithStubGitLabServer(t, true, *pagesBinary, listeners, ":42345", []string{}) + defer teardown() + + // need to call an actual resource to populate certain metrics e.g. gitlab_pages_domains_source_api_requests_total + res, err := GetPageFromListener(t, httpListener, "zip.gitlab.io", + "/symlink.html") + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + + resp, err := http.Get("http://localhost:42345/metrics") + require.NoError(t, err) + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + require.Contains(t, string(body), "gitlab_pages_http_in_flight_requests 0") + // TODO: remove metrics for disk source https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 + require.Contains(t, string(body), "gitlab_pages_served_domains 0") + require.Contains(t, string(body), "gitlab_pages_domains_failed_total 0") + require.Contains(t, string(body), "gitlab_pages_domains_updated_total 0") + require.Contains(t, string(body), "gitlab_pages_last_domain_update_seconds gauge") + require.Contains(t, string(body), "gitlab_pages_domains_configuration_update_duration gauge") + // end TODO + require.Contains(t, string(body), "gitlab_pages_domains_source_cache_hit") + require.Contains(t, string(body), "gitlab_pages_domains_source_cache_miss") + require.Contains(t, string(body), "gitlab_pages_domains_source_failures_total") + require.Contains(t, string(body), "gitlab_pages_serverless_requests 0") + require.Contains(t, string(body), "gitlab_pages_serverless_latency_sum 0") + require.Contains(t, string(body), "gitlab_pages_disk_serving_file_size_bytes_sum") + require.Contains(t, string(body), "gitlab_pages_serving_time_seconds_sum") + require.Contains(t, string(body), `gitlab_pages_domains_source_api_requests_total{status_code="200"}`) + require.Contains(t, string(body), `gitlab_pages_domains_source_api_call_duration_bucket`) + require.Contains(t, string(body), `gitlab_pages_domains_source_api_trace_duration`) + // httprange + require.Contains(t, string(body), `gitlab_pages_httprange_requests_total{status_code="206"}`) + require.Contains(t, string(body), "gitlab_pages_httprange_requests_duration_bucket") + require.Contains(t, string(body), "gitlab_pages_httprange_trace_duration") + require.Contains(t, string(body), "gitlab_pages_httprange_open_requests") + // zip archives + require.Contains(t, string(body), "gitlab_pages_zip_opened") + require.Contains(t, string(body), "gitlab_pages_zip_cache_requests") + require.Contains(t, string(body), "gitlab_pages_zip_cached_entries") + require.Contains(t, string(body), "gitlab_pages_zip_archive_entries_cached") + require.Contains(t, string(body), "gitlab_pages_zip_opened_entries_count") +} + +func TestDisabledRedirects(t *testing.T) { + skipUnlessEnabled(t) + + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{"FF_ENABLE_REDIRECTS=false"}) + defer teardown() + + // Test that redirects status page is forbidden + rsp, err := GetPageFromListener(t, httpListener, "group.redirects.gitlab-example.com", "/project-redirects/_redirects") + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusForbidden, rsp.StatusCode) + + // Test that redirects are disabled + rsp, err = GetRedirectPage(t, httpListener, "group.redirects.gitlab-example.com", "/project-redirects/redirect-portal.html") + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusNotFound, rsp.StatusCode) +} + +func TestRedirectStatusPage(t *testing.T) { + skipUnlessEnabled(t) + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.redirects.gitlab-example.com", "/project-redirects/_redirects") + require.NoError(t, err) + + body, err := ioutil.ReadAll(rsp.Body) + require.NoError(t, err) + defer rsp.Body.Close() + + require.Contains(t, string(body), "11 rules") + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestRedirect(t *testing.T) { + skipUnlessEnabled(t) + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + // Test that serving a file still works with redirects enabled + rsp, err := GetRedirectPage(t, httpListener, "group.redirects.gitlab-example.com", "/project-redirects/index.html") + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + + tests := []struct { + host string + path string + expectedStatus int + expectedLocation string + }{ + // Project domain + { + host: "group.redirects.gitlab-example.com", + path: "/project-redirects/redirect-portal.html", + expectedStatus: http.StatusFound, + expectedLocation: "/project-redirects/magic-land.html", + }, + // Make sure invalid rule does not redirect + { + host: "group.redirects.gitlab-example.com", + path: "/project-redirects/goto-domain.html", + expectedStatus: http.StatusNotFound, + expectedLocation: "", + }, + // Actual file on disk should override any redirects that match + { + host: "group.redirects.gitlab-example.com", + path: "/project-redirects/file-override.html", + expectedStatus: http.StatusOK, + expectedLocation: "", + }, + // Group-level domain + { + host: "group.redirects.gitlab-example.com", + path: "/redirect-portal.html", + expectedStatus: http.StatusFound, + expectedLocation: "/magic-land.html", + }, + // Custom domain + { + host: "redirects.custom-domain.com", + path: "/redirect-portal.html", + expectedStatus: http.StatusFound, + expectedLocation: "/magic-land.html", + }, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s%s -> %s (%d)", tt.host, tt.path, tt.expectedLocation, tt.expectedStatus), func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, tt.host, tt.path) + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, tt.expectedLocation, rsp.Header.Get("Location")) + require.Equal(t, tt.expectedStatus, rsp.StatusCode) + }) + } +} + +func TestStatusPage(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-status=/@statuscheck") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "@statuscheck") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestStatusNotYetReady(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "", "-pages-status=/@statuscheck", "-pages-root=../../shared/invalid-pages") + defer teardown() + + waitForRoundtrips(t, listeners, 5*time.Second) + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "@statuscheck") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) +} + +func TestPageNotAvailableIfNotLoaded(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "", "-pages-root=../../shared/invalid-pages") + defer teardown() + waitForRoundtrips(t, listeners, 5*time.Second) + + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "index.html") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) +} + +func TestMIMETypes(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "") + defer teardown() + + require.NoError(t, httpListener.WaitUntilRequestSucceeds(nil)) + + tests := map[string]struct { + file string + expectedContentType string + }{ + "manifest_json": { + file: "file.webmanifest", + expectedContentType: "application/manifest+json", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "project/"+tt.file) + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + mt, _, err := mime.ParseMediaType(rsp.Header.Get("Content-Type")) + require.NoError(t, err) + require.Equal(t, tt.expectedContentType, mt) + }) + } +} + +func TestCompressedEncoding(t *testing.T) { + skipUnlessEnabled(t) + + tests := []struct { + name string + host string + path string + encoding string + }{ + { + "gzip encoding", + "group.gitlab-example.com", + "index.html", + "gzip", + }, + { + "brotli encoding", + "group.gitlab-example.com", + "index.html", + "br", + }, + } + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rsp, err := GetCompressedPageFromListener(t, httpListener, "group.gitlab-example.com", "index.html", tt.encoding) + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + require.Equal(t, tt.encoding, rsp.Header.Get("Content-Encoding")) + }) + } +} + +func TestArtifactProxyRequest(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + + transport := (TestHTTPSClient.Transport).(*http.Transport) + defer func(t time.Duration) { + transport.ResponseHeaderTimeout = t + }(transport.ResponseHeaderTimeout) + transport.ResponseHeaderTimeout = 5 * time.Second + + content := "<!DOCTYPE html><html><head><title>Title of the document</title></head><body></body></html>" + contentLength := int64(len(content)) + testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.RawPath { + case "/api/v4/projects/group%2Fproject/jobs/1/artifacts/delayed_200.html": + time.Sleep(2 * time.Second) + fallthrough + case "/api/v4/projects/group%2Fproject/jobs/1/artifacts/200.html", + "/api/v4/projects/group%2Fsubgroup%2Fproject/jobs/1/artifacts/200.html": + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprint(w, content) + case "/api/v4/projects/group%2Fproject/jobs/1/artifacts/500.html": + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, content) + default: + t.Logf("Unexpected r.URL.RawPath: %q", r.URL.RawPath) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, content) + } + })) + + keyFile, certFile := CreateHTTPSFixtureFiles(t) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + defer os.Remove(keyFile) + defer os.Remove(certFile) + + testServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + testServer.StartTLS() + defer testServer.Close() + + tests := []struct { + name string + host string + path string + status int + binaryOption string + content string + length int64 + cacheControl string + contentType string + }{ + { + name: "basic proxied request", + host: "group.gitlab-example.com", + path: "/-/project/-/jobs/1/artifacts/200.html", + status: http.StatusOK, + binaryOption: "", + content: content, + length: contentLength, + cacheControl: "max-age=3600", + contentType: "text/html; charset=utf-8", + }, + { + name: "basic proxied request for subgroup", + host: "group.gitlab-example.com", + path: "/-/subgroup/project/-/jobs/1/artifacts/200.html", + status: http.StatusOK, + binaryOption: "", + content: content, + length: contentLength, + cacheControl: "max-age=3600", + contentType: "text/html; charset=utf-8", + }, + { + name: "502 error while attempting to proxy", + host: "group.gitlab-example.com", + path: "/-/project/-/jobs/1/artifacts/delayed_200.html", + status: http.StatusBadGateway, + binaryOption: "-artifacts-server-timeout=1", + content: "", + length: 0, + cacheControl: "", + contentType: "text/html; charset=utf-8", + }, + { + name: "Proxying 404 from server", + host: "group.gitlab-example.com", + path: "/-/project/-/jobs/1/artifacts/404.html", + status: http.StatusNotFound, + binaryOption: "", + content: "", + length: 0, + cacheControl: "", + contentType: "text/html; charset=utf-8", + }, + { + name: "Proxying 500 from server", + host: "group.gitlab-example.com", + path: "/-/project/-/jobs/1/artifacts/500.html", + status: http.StatusInternalServerError, + binaryOption: "", + content: "", + length: 0, + cacheControl: "", + contentType: "text/html; charset=utf-8", + }, + } + + // Ensure the IP address is used in the URL, as we're relying on IP SANs to + // validate + artifactServerURL := testServer.URL + "/api/v4" + t.Log("Artifact server URL", artifactServerURL) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + teardown := RunPagesProcessWithSSLCertFile( + t, + *pagesBinary, + listeners, + "", + certFile, + "-artifacts-server="+artifactServerURL, + tt.binaryOption, + ) + defer teardown() + + resp, err := GetPageFromListener(t, httpListener, tt.host, tt.path) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, tt.status, resp.StatusCode) + require.Equal(t, tt.contentType, resp.Header.Get("Content-Type")) + + if !((tt.status == http.StatusBadGateway) || (tt.status == http.StatusNotFound) || (tt.status == http.StatusInternalServerError)) { + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, tt.content, string(body)) + require.Equal(t, tt.length, resp.ContentLength) + require.Equal(t, tt.cacheControl, resp.Header.Get("Cache-Control")) + } + }) + } +} + +func TestPrivateArtifactProxyRequest(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + + setupTransport(t) + + testServer := makeGitLabPagesAccessStub(t) + + keyFile, certFile := CreateHTTPSFixtureFiles(t) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + defer os.Remove(keyFile) + defer os.Remove(certFile) + + testServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + testServer.StartTLS() + defer testServer.Close() + + tests := []struct { + name string + host string + path string + status int + binaryOption string + }{ + { + name: "basic proxied request for private project", + host: "group.gitlab-example.com", + path: "/-/private/-/jobs/1/artifacts/200.html", + status: http.StatusOK, + binaryOption: "", + }, + { + name: "basic proxied request for subgroup", + host: "group.gitlab-example.com", + path: "/-/subgroup/private/-/jobs/1/artifacts/200.html", + status: http.StatusOK, + binaryOption: "", + }, + { + name: "502 error while attempting to proxy", + host: "group.gitlab-example.com", + path: "/-/private/-/jobs/1/artifacts/delayed_200.html", + status: http.StatusBadGateway, + binaryOption: "artifacts-server-timeout=1", + }, + { + name: "Proxying 404 from server", + host: "group.gitlab-example.com", + path: "/-/private/-/jobs/1/artifacts/404.html", + status: http.StatusNotFound, + binaryOption: "", + }, + { + name: "Proxying 500 from server", + host: "group.gitlab-example.com", + path: "/-/private/-/jobs/1/artifacts/500.html", + status: http.StatusInternalServerError, + binaryOption: "", + }, + } + + // Ensure the IP address is used in the URL, as we're relying on IP SANs to + // validate + artifactServerURL := testServer.URL + "/api/v4" + t.Log("Artifact server URL", artifactServerURL) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configFile, cleanup := defaultConfigFileWith(t, + "artifacts-server="+artifactServerURL, + "auth-server="+testServer.URL, + "auth-redirect-uri=https://projects.gitlab-example.com/auth", + tt.binaryOption) + defer cleanup() + + teardown := RunPagesProcessWithSSLCertFile( + t, + *pagesBinary, + listeners, + "", + certFile, + "-config="+configFile, + ) + defer teardown() + + resp, err := GetRedirectPage(t, httpListener, tt.host, tt.path) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusFound, resp.StatusCode) + + cookie := resp.Header.Get("Set-Cookie") + + // Redirects to the projects under gitlab pages domain for authentication flow + url, err := url.Parse(resp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, "projects.gitlab-example.com", url.Host) + require.Equal(t, "/auth", url.Path) + state := url.Query().Get("state") + + resp, err = GetRedirectPage(t, httpsListener, url.Host, url.Path+"?"+url.RawQuery) + + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusFound, resp.StatusCode) + pagesDomainCookie := resp.Header.Get("Set-Cookie") + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetRedirectPageWithCookie(t, httpsListener, "projects.gitlab-example.com", "/auth?code=1&state="+ + state, pagesDomainCookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will redirect auth callback to correct host + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, tt.host, url.Host) + require.Equal(t, "/auth", url.Path) + + // Request auth callback in project domain + authrsp, err = GetRedirectPageWithCookie(t, httpsListener, url.Host, url.Path+"?"+url.RawQuery, cookie) + require.NoError(t, err) + + // server returns the ticket, user will be redirected to the project page + require.Equal(t, http.StatusFound, authrsp.StatusCode) + cookie = authrsp.Header.Get("Set-Cookie") + resp, err = GetRedirectPageWithCookie(t, httpsListener, tt.host, tt.path, cookie) + + require.Equal(t, tt.status, resp.StatusCode) + + require.NoError(t, err) + defer resp.Body.Close() + }) + } +} + +func TestEnvironmentVariablesConfig(t *testing.T) { + skipUnlessEnabled(t) + os.Setenv("LISTEN_HTTP", net.JoinHostPort(httpListener.Host, httpListener.Port)) + defer func() { os.Unsetenv("LISTEN_HTTP") }() + + teardown := RunPagesProcessWithoutWait(t, *pagesBinary, []ListenSpec{}, "") + defer teardown() + require.NoError(t, httpListener.WaitUntilRequestSucceeds(nil)) + + rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com:", "project/") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) +} + +func TestMixedConfigSources(t *testing.T) { + skipUnlessEnabled(t) + os.Setenv("LISTEN_HTTP", net.JoinHostPort(httpListener.Host, httpListener.Port)) + defer func() { os.Unsetenv("LISTEN_HTTP") }() + + teardown := RunPagesProcessWithoutWait(t, *pagesBinary, []ListenSpec{httpsListener}, "") + defer teardown() + + for _, listener := range []ListenSpec{httpListener, httpsListener} { + require.NoError(t, listener.WaitUntilRequestSucceeds(nil)) + rsp, err := GetPageFromListener(t, listener, "group.gitlab-example.com", "project/") + require.NoError(t, err) + rsp.Body.Close() + + require.Equal(t, http.StatusOK, rsp.StatusCode) + } +} + +func TestMultiFlagEnvironmentVariables(t *testing.T) { + skipUnlessEnabled(t) + listenSpecs := []ListenSpec{{"http", "127.0.0.1", "37001"}, {"http", "127.0.0.1", "37002"}} + envVarValue := fmt.Sprintf("%s,%s", net.JoinHostPort("127.0.0.1", "37001"), net.JoinHostPort("127.0.0.1", "37002")) + + os.Setenv("LISTEN_HTTP", envVarValue) + defer func() { os.Unsetenv("LISTEN_HTTP") }() + + teardown := RunPagesProcess(t, *pagesBinary, []ListenSpec{}, "") + defer teardown() + + for _, listener := range listenSpecs { + require.NoError(t, listener.WaitUntilRequestSucceeds(nil)) + rsp, err := GetPageFromListener(t, listener, "group.gitlab-example.com", "project/") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + } +} + +func TestKnownHostInReverseProxySetupReturns200(t *testing.T) { + skipUnlessEnabled(t) + + var listeners = []ListenSpec{ + {"proxy", "127.0.0.1", "37002"}, + {"proxy", "::1", "37002"}, + } + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + for _, spec := range listeners { + rsp, err := GetProxiedPageFromListener(t, spec, "localhost", "group.gitlab-example.com", "project/") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + } +} + +func TestWhenAuthIsDisabledPrivateIsNotAccessible(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpListener, "group.auth.gitlab-example.com", "private.project/") + + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusInternalServerError, rsp.StatusCode) +} + +func TestWhenAuthIsEnabledPrivateWillRedirectToAuthorize(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpsListener, "group.auth.gitlab-example.com", "private.project/") + + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusFound, rsp.StatusCode) + require.Equal(t, 1, len(rsp.Header["Location"])) + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + rsp, err = GetRedirectPage(t, httpsListener, url.Host, url.Path+"?"+url.RawQuery) + require.NoError(t, err) + + require.Equal(t, http.StatusFound, rsp.StatusCode) + require.Equal(t, 1, len(rsp.Header["Location"])) + + url, err = url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + require.Equal(t, "https", url.Scheme) + require.Equal(t, "gitlab-auth.com", url.Host) + require.Equal(t, "/oauth/authorize", url.Path) + require.Equal(t, "clientID", url.Query().Get("client_id")) + require.Equal(t, "https://projects.gitlab-example.com/auth", url.Query().Get("redirect_uri")) + require.NotEqual(t, "", url.Query().Get("state")) +} + +func TestWhenAuthDeniedWillCauseUnauthorized(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetPageFromListener(t, httpsListener, "projects.gitlab-example.com", "/auth?error=access_denied") + + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, rsp.StatusCode) +} +func TestWhenLoginCallbackWithWrongStateShouldFail(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpsListener, "group.auth.gitlab-example.com", "private.project/") + + require.NoError(t, err) + defer rsp.Body.Close() + + // Go to auth page with wrong state will cause failure + authrsp, err := GetPageFromListener(t, httpsListener, "projects.gitlab-example.com", "/auth?code=0&state=0") + + require.NoError(t, err) + defer authrsp.Body.Close() + + require.Equal(t, http.StatusUnauthorized, authrsp.StatusCode) +} + +func TestWhenLoginCallbackWithCorrectStateWithoutEndpoint(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpsListener, "group.auth.gitlab-example.com", "private.project/") + + require.NoError(t, err) + defer rsp.Body.Close() + + cookie := rsp.Header.Get("Set-Cookie") + + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetPageFromListenerWithCookie(t, httpsListener, "projects.gitlab-example.com", "/auth?code=1&state="+ + url.Query().Get("state"), cookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will cause 503 because token endpoint is not available + require.Equal(t, http.StatusServiceUnavailable, authrsp.StatusCode) +} + +// makeGitLabPagesAccessStub provides a stub *httptest.Server to check pages_access API call. +// the result is based on the project id. +// +// Project IDs must be 4 digit long and the following rules applies: +// 1000-1999: Ok +// 2000-2999: Unauthorized +// 3000-3999: Invalid token +func makeGitLabPagesAccessStub(t *testing.T) *httptest.Server { + return httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/oauth/token": + require.Equal(t, "POST", r.Method) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, "{\"access_token\":\"abc\"}") + case "/api/v4/user": + require.Equal(t, "Bearer abc", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusOK) + default: + if handleAccessControlArtifactRequests(t, w, r) { + return + } + handleAccessControlRequests(t, w, r) + } + })) +} + +var existingAcmeTokenPath = "/.well-known/acme-challenge/existingtoken" +var notexistingAcmeTokenPath = "/.well-known/acme-challenge/notexistingtoken" + +func TestAcmeChallengesWhenItIsConfigured(t *testing.T) { + skipUnlessEnabled(t) + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-gitlab-server=https://gitlab-acme.com") + defer teardown() + + t.Run("When domain folder contains requested acme challenge it responds with it", func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, "withacmechallenge.domain.com", + existingAcmeTokenPath) + + defer rsp.Body.Close() + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + body, _ := ioutil.ReadAll(rsp.Body) + require.Equal(t, "this is token\n", string(body)) + }) + + t.Run("When domain folder doesn't contains requested acme challenge it redirects to GitLab", + func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, "withacmechallenge.domain.com", + notexistingAcmeTokenPath) + + defer rsp.Body.Close() + require.NoError(t, err) + require.Equal(t, http.StatusTemporaryRedirect, rsp.StatusCode) + + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + require.Equal(t, url.String(), "https://gitlab-acme.com/-/acme-challenge?domain=withacmechallenge.domain.com&token=notexistingtoken") + }, + ) +} + +func TestAcmeChallengesWhenItIsNotConfigured(t *testing.T) { + skipUnlessEnabled(t) + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "") + defer teardown() + + t.Run("When domain folder contains requested acme challenge it responds with it", func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, "withacmechallenge.domain.com", + existingAcmeTokenPath) + + defer rsp.Body.Close() + require.NoError(t, err) + require.Equal(t, http.StatusOK, rsp.StatusCode) + body, _ := ioutil.ReadAll(rsp.Body) + require.Equal(t, "this is token\n", string(body)) + }) + + t.Run("When domain folder doesn't contains requested acme challenge it returns 404", + func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, "withacmechallenge.domain.com", + notexistingAcmeTokenPath) + + defer rsp.Body.Close() + require.NoError(t, err) + require.Equal(t, http.StatusNotFound, rsp.StatusCode) + }, + ) +} + +func handleAccessControlArtifactRequests(t *testing.T, w http.ResponseWriter, r *http.Request) bool { + authorization := r.Header.Get("Authorization") + + switch { + case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/delayed_200.html`).MatchString(r.URL.Path): + sleepIfAuthorized(t, authorization, w) + return true + case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/404.html`).MatchString(r.URL.Path): + w.WriteHeader(http.StatusNotFound) + return true + case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/500.html`).MatchString(r.URL.Path): + returnIfAuthorized(t, authorization, w, http.StatusInternalServerError) + return true + case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/200.html`).MatchString(r.URL.Path): + returnIfAuthorized(t, authorization, w, http.StatusOK) + return true + case regexp.MustCompile(`/api/v4/projects/group/subgroup/private/jobs/\d+/artifacts/200.html`).MatchString(r.URL.Path): + returnIfAuthorized(t, authorization, w, http.StatusOK) + return true + default: + return false + } +} + +func handleAccessControlRequests(t *testing.T, w http.ResponseWriter, r *http.Request) { + allowedProjects := regexp.MustCompile(`/api/v4/projects/1\d{3}/pages_access`) + deniedProjects := regexp.MustCompile(`/api/v4/projects/2\d{3}/pages_access`) + invalidTokenProjects := regexp.MustCompile(`/api/v4/projects/3\d{3}/pages_access`) + + switch { + case allowedProjects.MatchString(r.URL.Path): + require.Equal(t, "Bearer abc", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusOK) + case deniedProjects.MatchString(r.URL.Path): + require.Equal(t, "Bearer abc", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusUnauthorized) + case invalidTokenProjects.MatchString(r.URL.Path): + require.Equal(t, "Bearer abc", r.Header.Get("Authorization")) + w.WriteHeader(http.StatusUnauthorized) + fmt.Fprint(w, "{\"error\":\"invalid_token\"}") + default: + t.Logf("Unexpected r.URL.RawPath: %q", r.URL.Path) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusNotFound) + } +} + +func returnIfAuthorized(t *testing.T, authorization string, w http.ResponseWriter, status int) { + if authorization != "" { + require.Equal(t, "Bearer abc", authorization) + w.WriteHeader(status) + } else { + w.WriteHeader(http.StatusNotFound) + } +} + +func sleepIfAuthorized(t *testing.T, authorization string, w http.ResponseWriter) { + if authorization != "" { + require.Equal(t, "Bearer abc", authorization) + time.Sleep(2 * time.Second) + } else { + w.WriteHeader(http.StatusNotFound) + } +} + +func TestAccessControlUnderCustomDomain(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + + testServer := makeGitLabPagesAccessStub(t) + testServer.Start() + defer testServer.Close() + + teardown := RunPagesProcessWithAuthServer(t, *pagesBinary, listeners, "", testServer.URL) + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "private.domain.com", "/") + require.NoError(t, err) + defer rsp.Body.Close() + + cookie := rsp.Header.Get("Set-Cookie") + + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + state := url.Query().Get("state") + require.Equal(t, url.Query().Get("domain"), "http://private.domain.com") + + pagesrsp, err := GetRedirectPage(t, httpListener, url.Host, url.Path+"?"+url.RawQuery) + require.NoError(t, err) + defer pagesrsp.Body.Close() + + pagescookie := pagesrsp.Header.Get("Set-Cookie") + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetRedirectPageWithCookie(t, httpListener, "projects.gitlab-example.com", "/auth?code=1&state="+ + state, pagescookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain + require.Equal(t, "private.domain.com", url.Host) + require.Equal(t, "1", url.Query().Get("code")) + require.Equal(t, state, url.Query().Get("state")) + + // Run auth callback in custom domain + authrsp, err = GetRedirectPageWithCookie(t, httpListener, "private.domain.com", "/auth?code=1&state="+ + state, cookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will redirect to the page + cookie = authrsp.Header.Get("Set-Cookie") + require.Equal(t, http.StatusFound, authrsp.StatusCode) + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain + require.Equal(t, "http://private.domain.com/", url.String()) + + // Fetch page in custom domain + authrsp, err = GetRedirectPageWithCookie(t, httpListener, "private.domain.com", "/", cookie) + require.NoError(t, err) + require.Equal(t, http.StatusOK, authrsp.StatusCode) +} + +func TestCustomErrorPageWithAuth(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + testServer := makeGitLabPagesAccessStub(t) + testServer.Start() + defer testServer.Close() + + teardown := RunPagesProcessWithAuthServer(t, *pagesBinary, listeners, "", testServer.URL) + defer teardown() + + tests := []struct { + name string + domain string + path string + expectedErrorPage string + }{ + { + name: "private_project_authorized", + domain: "group.404.gitlab-example.com", + path: "/private_project/unknown", + expectedErrorPage: "Private custom 404 error page", + }, + { + name: "public_namespace_with_private_unauthorized_project", + domain: "group.404.gitlab-example.com", + // /private_unauthorized/config.json resolves project ID to 2000 which will cause a 401 from the mock GitLab testServer + path: "/private_unauthorized/unknown", + expectedErrorPage: "Custom 404 group page", + }, + { + name: "private_namespace_authorized", + domain: "group.auth.gitlab-example.com", + path: "/unknown", + expectedErrorPage: "group.auth.gitlab-example.com namespace custom 404", + }, + { + name: "private_namespace_with_private_project_auth_failed", + domain: "group.auth.gitlab-example.com", + // project ID is 2000 + path: "/private.project.1/unknown", + expectedErrorPage: "The page you're looking for could not be found.", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rsp, err := GetRedirectPage(t, httpListener, tt.domain, tt.path) + require.NoError(t, err) + defer rsp.Body.Close() + + cookie := rsp.Header.Get("Set-Cookie") + + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + state := url.Query().Get("state") + require.Equal(t, "http://"+tt.domain, url.Query().Get("domain")) + + pagesrsp, err := GetRedirectPage(t, httpListener, url.Host, url.Path+"?"+url.RawQuery) + require.NoError(t, err) + defer pagesrsp.Body.Close() + + pagescookie := pagesrsp.Header.Get("Set-Cookie") + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetRedirectPageWithCookie(t, httpListener, "projects.gitlab-example.com", "/auth?code=1&state="+ + state, pagescookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain + require.Equal(t, tt.domain, url.Host) + require.Equal(t, "1", url.Query().Get("code")) + require.Equal(t, state, url.Query().Get("state")) + + // Run auth callback in custom domain + authrsp, err = GetRedirectPageWithCookie(t, httpListener, tt.domain, "/auth?code=1&state="+ + state, cookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will redirect to the page + groupCookie := authrsp.Header.Get("Set-Cookie") + require.Equal(t, http.StatusFound, authrsp.StatusCode) + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain error page + require.Equal(t, "http://"+tt.domain+tt.path, url.String()) + + // Fetch page in custom domain + anotherResp, err := GetRedirectPageWithCookie(t, httpListener, tt.domain, tt.path, groupCookie) + require.NoError(t, err) + + require.Equal(t, http.StatusNotFound, anotherResp.StatusCode) + + page, err := ioutil.ReadAll(anotherResp.Body) + require.NoError(t, err) + require.Contains(t, string(page), tt.expectedErrorPage) + }) + } +} + +func TestAccessControlUnderCustomDomainWithHTTPSProxy(t *testing.T) { + skipUnlessEnabled(t, "not-inplace-chroot") + + testServer := makeGitLabPagesAccessStub(t) + testServer.Start() + defer testServer.Close() + + teardown := RunPagesProcessWithAuthServer(t, *pagesBinary, listeners, "", testServer.URL) + defer teardown() + + rsp, err := GetProxyRedirectPageWithCookie(t, proxyListener, "private.domain.com", "/", "", true) + require.NoError(t, err) + defer rsp.Body.Close() + + cookie := rsp.Header.Get("Set-Cookie") + + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + state := url.Query().Get("state") + require.Equal(t, url.Query().Get("domain"), "https://private.domain.com") + pagesrsp, err := GetProxyRedirectPageWithCookie(t, proxyListener, url.Host, url.Path+"?"+url.RawQuery, "", true) + require.NoError(t, err) + defer pagesrsp.Body.Close() + + pagescookie := pagesrsp.Header.Get("Set-Cookie") + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetProxyRedirectPageWithCookie(t, proxyListener, + "projects.gitlab-example.com", "/auth?code=1&state="+state, + pagescookie, true) + + require.NoError(t, err) + defer authrsp.Body.Close() + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain + require.Equal(t, "private.domain.com", url.Host) + require.Equal(t, "1", url.Query().Get("code")) + require.Equal(t, state, url.Query().Get("state")) + + // Run auth callback in custom domain + authrsp, err = GetProxyRedirectPageWithCookie(t, proxyListener, "private.domain.com", + "/auth?code=1&state="+state, cookie, true) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will redirect to the page + cookie = authrsp.Header.Get("Set-Cookie") + require.Equal(t, http.StatusFound, authrsp.StatusCode) + + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + + // Will redirect to custom domain + require.Equal(t, "https://private.domain.com/", url.String()) + // Fetch page in custom domain + authrsp, err = GetProxyRedirectPageWithCookie(t, proxyListener, "private.domain.com", "/", + cookie, true) + require.NoError(t, err) + require.Equal(t, http.StatusOK, authrsp.StatusCode) +} + +func TestAccessControlGroupDomain404RedirectsAuth(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "/nonexistent/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusFound, rsp.StatusCode) + // Redirects to the projects under gitlab pages domain for authentication flow + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, "projects.gitlab-example.com", url.Host) + require.Equal(t, "/auth", url.Path) +} +func TestAccessControlProject404DoesNotRedirect(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcessWithAuth(t, *pagesBinary, listeners, "") + defer teardown() + + rsp, err := GetRedirectPage(t, httpListener, "group.gitlab-example.com", "/project/nonexistent/") + require.NoError(t, err) + defer rsp.Body.Close() + require.Equal(t, http.StatusNotFound, rsp.StatusCode) +} + +func setupTransport(t *testing.T) { + transport := (TestHTTPSClient.Transport).(*http.Transport) + defer func(t time.Duration) { + transport.ResponseHeaderTimeout = t + }(transport.ResponseHeaderTimeout) + transport.ResponseHeaderTimeout = 5 * time.Second +} + +type runPagesFunc func(t *testing.T, pagesPath string, listeners []ListenSpec, promPort string, sslCertFile string, authServer string) func() + +func testAccessControl(t *testing.T, runPages runPagesFunc) { + skipUnlessEnabled(t, "not-inplace-chroot") + + setupTransport(t) + + keyFile, certFile := CreateHTTPSFixtureFiles(t) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + require.NoError(t, err) + defer os.Remove(keyFile) + defer os.Remove(certFile) + + testServer := makeGitLabPagesAccessStub(t) + testServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + testServer.StartTLS() + defer testServer.Close() + + tests := []struct { + host string + path string + status int + redirectBack bool + name string + }{ + { + name: "project with access", + host: "group.auth.gitlab-example.com", + path: "/private.project/", + status: http.StatusOK, + redirectBack: false, + }, + { + name: "project without access", + host: "group.auth.gitlab-example.com", + path: "/private.project.1/", + status: http.StatusNotFound, // Do not expose project existed + redirectBack: false, + }, + { + name: "invalid token test should redirect back", + host: "group.auth.gitlab-example.com", + path: "/private.project.2/", + status: http.StatusFound, + redirectBack: true, + }, + { + name: "no project should redirect to login and then return 404", + host: "group.auth.gitlab-example.com", + path: "/nonexistent/", + status: http.StatusNotFound, + redirectBack: false, + }, + { + name: "no project should redirect to login and then return 404", + host: "nonexistent.gitlab-example.com", + path: "/nonexistent/", + status: http.StatusNotFound, + redirectBack: false, + }, // subgroups + { + name: "[subgroup] project with access", + host: "group.auth.gitlab-example.com", + path: "/subgroup/private.project/", + status: http.StatusOK, + redirectBack: false, + }, + { + name: "[subgroup] project without access", + host: "group.auth.gitlab-example.com", + path: "/subgroup/private.project.1/", + status: http.StatusNotFound, // Do not expose project existed + redirectBack: false, + }, + { + name: "[subgroup] invalid token test should redirect back", + host: "group.auth.gitlab-example.com", + path: "/subgroup/private.project.2/", + status: http.StatusFound, + redirectBack: true, + }, + { + name: "[subgroup] no project should redirect to login and then return 404", + host: "group.auth.gitlab-example.com", + path: "/subgroup/nonexistent/", + status: http.StatusNotFound, + redirectBack: false, + }, + { + name: "[subgroup] no project should redirect to login and then return 404", + host: "nonexistent.gitlab-example.com", + path: "/subgroup/nonexistent/", + status: http.StatusNotFound, + redirectBack: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + teardown := runPages(t, *pagesBinary, listeners, "", certFile, testServer.URL) + defer teardown() + + rsp, err := GetRedirectPage(t, httpsListener, tt.host, tt.path) + + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusFound, rsp.StatusCode) + cookie := rsp.Header.Get("Set-Cookie") + + // Redirects to the projects under gitlab pages domain for authentication flow + url, err := url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, "projects.gitlab-example.com", url.Host) + require.Equal(t, "/auth", url.Path) + state := url.Query().Get("state") + + rsp, err = GetRedirectPage(t, httpsListener, url.Host, url.Path+"?"+url.RawQuery) + + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, http.StatusFound, rsp.StatusCode) + pagesDomainCookie := rsp.Header.Get("Set-Cookie") + + // Go to auth page with correct state will cause fetching the token + authrsp, err := GetRedirectPageWithCookie(t, httpsListener, "projects.gitlab-example.com", "/auth?code=1&state="+ + state, pagesDomainCookie) + + require.NoError(t, err) + defer authrsp.Body.Close() + + // Will redirect auth callback to correct host + url, err = url.Parse(authrsp.Header.Get("Location")) + require.NoError(t, err) + require.Equal(t, tt.host, url.Host) + require.Equal(t, "/auth", url.Path) + + // Request auth callback in project domain + authrsp, err = GetRedirectPageWithCookie(t, httpsListener, url.Host, url.Path+"?"+url.RawQuery, cookie) + require.NoError(t, err) + + // server returns the ticket, user will be redirected to the project page + require.Equal(t, http.StatusFound, authrsp.StatusCode) + cookie = authrsp.Header.Get("Set-Cookie") + rsp, err = GetRedirectPageWithCookie(t, httpsListener, tt.host, tt.path, cookie) + + require.NoError(t, err) + defer rsp.Body.Close() + + require.Equal(t, tt.status, rsp.StatusCode) + require.Equal(t, "", rsp.Header.Get("Cache-Control")) + + if tt.redirectBack { + url, err = url.Parse(rsp.Header.Get("Location")) + require.NoError(t, err) + + require.Equal(t, "https", url.Scheme) + require.Equal(t, tt.host, url.Host) + require.Equal(t, tt.path, url.Path) + } + }) + } +} + +func TestAccessControlWithSSLCertFile(t *testing.T) { + testAccessControl(t, RunPagesProcessWithAuthServerWithSSLCertFile) +} + +func TestAccessControlWithSSLCertDir(t *testing.T) { + testAccessControl(t, RunPagesProcessWithAuthServerWithSSLCertDir) +} + +func TestAcceptsSupportedCiphers(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + tlsConfig := &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + }, + } + client, cleanup := ClientWithConfig(tlsConfig) + defer cleanup() + + rsp, err := client.Get(httpsListener.URL("/")) + + if rsp != nil { + rsp.Body.Close() + } + + require.NoError(t, err) +} + +func tlsConfigWithInsecureCiphersOnly() *tls.Config { + return &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + }, + MaxVersion: tls.VersionTLS12, // ciphers for TLS1.3 are not configurable and will work if enabled + } +} + +func TestRejectsUnsupportedCiphers(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "") + defer teardown() + + client, cleanup := ClientWithConfig(tlsConfigWithInsecureCiphersOnly()) + defer cleanup() + + rsp, err := client.Get(httpsListener.URL("/")) + + if rsp != nil { + rsp.Body.Close() + } + + require.Error(t, err) + require.Nil(t, rsp) +} + +func TestEnableInsecureCiphers(t *testing.T) { + skipUnlessEnabled(t) + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-insecure-ciphers") + defer teardown() + + client, cleanup := ClientWithConfig(tlsConfigWithInsecureCiphersOnly()) + defer cleanup() + + rsp, err := client.Get(httpsListener.URL("/")) + + if rsp != nil { + rsp.Body.Close() + } + + require.NoError(t, err) +} + +func TestTLSVersions(t *testing.T) { + skipUnlessEnabled(t) + + tests := map[string]struct { + tlsMin string + tlsMax string + tlsClient uint16 + expectError bool + }{ + "client version not supported": {tlsMin: "tls1.1", tlsMax: "tls1.2", tlsClient: tls.VersionTLS10, expectError: true}, + "client version supported": {tlsMin: "tls1.1", tlsMax: "tls1.2", tlsClient: tls.VersionTLS12, expectError: false}, + "client and server using default settings": {tlsMin: "", tlsMax: "", tlsClient: 0, expectError: false}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + args := []string{} + if tc.tlsMin != "" { + args = append(args, "-tls-min-version", tc.tlsMin) + } + if tc.tlsMax != "" { + args = append(args, "-tls-max-version", tc.tlsMax) + } + + teardown := RunPagesProcess(t, *pagesBinary, listeners, "", args...) + defer teardown() + + tlsConfig := &tls.Config{} + if tc.tlsClient != 0 { + tlsConfig.MinVersion = tc.tlsClient + tlsConfig.MaxVersion = tc.tlsClient + } + client, cleanup := ClientWithConfig(tlsConfig) + defer cleanup() + + rsp, err := client.Get(httpsListener.URL("/")) + + if rsp != nil { + rsp.Body.Close() + } + + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestDomainsSource(t *testing.T) { + skipUnlessEnabled(t) + + type args struct { + configSource string + domain string + urlSuffix string + } + type want struct { + statusCode int + content string + apiCalled bool + } + tests := []struct { + name string + args args + want want + }{ + { + name: "gitlab_source_domain_exists", + args: args{ + configSource: "gitlab", + domain: "new-source-test.gitlab.io", + urlSuffix: "/my/pages/project/", + }, + want: want{ + statusCode: http.StatusOK, + content: "New Pages GitLab Source TEST OK\n", + apiCalled: true, + }, + }, + { + name: "gitlab_source_domain_does_not_exist", + args: args{ + configSource: "gitlab", + domain: "non-existent-domain.gitlab.io", + }, + want: want{ + statusCode: http.StatusNotFound, + apiCalled: true, + }, + }, + { + name: "disk_source_domain_exists", + args: args{ + configSource: "disk", + // test.domain.com sourced from disk configuration + domain: "test.domain.com", + urlSuffix: "/", + }, + want: want{ + statusCode: http.StatusOK, + content: "main-dir\n", + apiCalled: false, + }, + }, + { + name: "disk_source_domain_does_not_exist", + args: args{ + configSource: "disk", + domain: "non-existent-domain.gitlab.io", + }, + want: want{ + statusCode: http.StatusNotFound, + apiCalled: false, + }, + }, + { + name: "disk_source_domain_should_not_exist_under_hashed_dir", + args: args{ + configSource: "disk", + domain: "hashed.com", + }, + want: want{ + statusCode: http.StatusNotFound, + apiCalled: false, + }, + }, + // TODO: modify mock so we can test domain-config-source=auto when API/disk is not ready https://gitlab.com/gitlab-org/gitlab/-/issues/218358 + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var apiCalled bool + source := NewGitlabDomainsSourceStub(t, &apiCalled) + defer source.Close() + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", tt.args.configSource} + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) + defer teardown() + + response, err := GetPageFromListener(t, httpListener, tt.args.domain, tt.args.urlSuffix) + require.NoError(t, err) + + require.Equal(t, tt.want.statusCode, response.StatusCode) + if tt.want.statusCode == http.StatusOK { + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + require.NoError(t, err) + + require.Equal(t, tt.want.content, string(body), "content mismatch") + } + + require.Equal(t, tt.want.apiCalled, apiCalled, "api called mismatch") + }) + } +} + +func TestZipServing(t *testing.T) { + skipUnlessEnabled(t) + + var apiCalled bool + source := NewGitlabDomainsSourceStub(t, &apiCalled) + defer source.Close() + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab"} + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{}, pagesArgs...) + defer teardown() + + _, cleanup := newZipFileServerURL(t, "../../shared/pages/group/zip.gitlab.io/public.zip") + defer cleanup() + + tests := map[string]struct { + host string + urlSuffix string + expectedStatusCode int + expectedContent string + }{ + "base_domain_no_suffix": { + host: "zip.gitlab.io", + urlSuffix: "/", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/index.html\n", + }, + "file_exists": { + host: "zip.gitlab.io", + urlSuffix: "/index.html", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/index.html\n", + }, + "file_exists_in_subdir": { + host: "zip.gitlab.io", + urlSuffix: "/subdir/hello.html", + expectedStatusCode: http.StatusOK, + expectedContent: "zip.gitlab.io/project/subdir/hello.html\n", + }, + "file_exists_symlink": { + host: "zip.gitlab.io", + urlSuffix: "/symlink.html", + expectedStatusCode: http.StatusOK, + expectedContent: "symlink.html->subdir/linked.html\n", + }, + "dir": { + host: "zip.gitlab.io", + urlSuffix: "/subdir/", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "file_does_not_exist": { + host: "zip.gitlab.io", + urlSuffix: "/unknown.html", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "bad_symlink": { + host: "zip.gitlab.io", + urlSuffix: "/bad-symlink.html", + expectedStatusCode: http.StatusNotFound, + expectedContent: "zip.gitlab.io/project/404.html\n", + }, + "with_not_found_zip": { + host: "zip-not-found.gitlab.io", + urlSuffix: "/", + expectedStatusCode: http.StatusNotFound, + expectedContent: "The page you're looking for could not be found", + }, + "with_malformed_zip": { + host: "zip-malformed.gitlab.io", + urlSuffix: "/", + expectedStatusCode: http.StatusInternalServerError, + expectedContent: "Something went wrong (500)", + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + response, err := GetPageFromListener(t, httpListener, tt.host, tt.urlSuffix) + require.NoError(t, err) + defer response.Body.Close() + + require.Equal(t, tt.expectedStatusCode, response.StatusCode) + + body, err := ioutil.ReadAll(response.Body) + require.NoError(t, err) + + require.Contains(t, string(body), tt.expectedContent, "content mismatch") + }) + } +} diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go new file mode 100644 index 00000000..6cd0f812 --- /dev/null +++ b/test/acceptance/helpers_test.go @@ -0,0 +1,581 @@ +// +build acceptance + +package acceptance_test + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" + "gitlab.com/gitlab-org/gitlab-pages/internal/request" +) + +type tWriter struct { + t *testing.T +} + +func (t *tWriter) Write(b []byte) (int, error) { + t.t.Log(string(bytes.TrimRight(b, "\r\n"))) + + return len(b), nil +} + +// The HTTPS certificate isn't signed by anyone. This http client is set up +// so it can talk to servers using it. +var ( + TestHTTPSClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: TestCertPool}, + }, + } + + // Use HTTP with a very short timeout to repeatedly check for the server to be + // up. Again, ignore HTTP + QuickTimeoutHTTPSClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: TestCertPool}, + ResponseHeaderTimeout: 100 * time.Millisecond, + }, + } + + TestCertPool = x509.NewCertPool() +) + +func init() { + if ok := TestCertPool.AppendCertsFromPEM([]byte(fixture.Certificate)); !ok { + fmt.Println("Failed to load cert!") + } +} + +func CreateHTTPSFixtureFiles(t *testing.T) (key string, cert string) { + keyfile, err := ioutil.TempFile("", "https-fixture") + require.NoError(t, err) + key = keyfile.Name() + keyfile.Close() + + certfile, err := ioutil.TempFile("", "https-fixture") + require.NoError(t, err) + cert = certfile.Name() + certfile.Close() + + require.NoError(t, ioutil.WriteFile(key, []byte(fixture.Key), 0644)) + require.NoError(t, ioutil.WriteFile(cert, []byte(fixture.Certificate), 0644)) + + return keyfile.Name(), certfile.Name() +} + +func CreateGitLabAPISecretKeyFixtureFile(t *testing.T) (filepath string) { + secretfile, err := ioutil.TempFile("", "gitlab-api-secret") + require.NoError(t, err) + secretfile.Close() + + require.NoError(t, ioutil.WriteFile(secretfile.Name(), []byte(fixture.GitLabAPISecretKey), 0644)) + + return secretfile.Name() +} + +// ListenSpec is used to point at a gitlab-pages http server, preserving the +// type of port it is (http, https, proxy) +type ListenSpec struct { + Type string + Host string + Port string +} + +func (l ListenSpec) URL(suffix string) string { + scheme := request.SchemeHTTP + if l.Type == request.SchemeHTTPS { + scheme = request.SchemeHTTPS + } + + suffix = strings.TrimPrefix(suffix, "/") + + return fmt.Sprintf("%s://%s/%s", scheme, l.JoinHostPort(), suffix) +} + +// Returns only once this spec points at a working TCP server +func (l ListenSpec) WaitUntilRequestSucceeds(done chan struct{}) error { + timeout := 5 * time.Second + for start := time.Now(); time.Since(start) < timeout; { + select { + case <-done: + return fmt.Errorf("server has shut down already") + default: + } + + req, err := http.NewRequest("GET", l.URL("/"), nil) + if err != nil { + return err + } + + response, err := QuickTimeoutHTTPSClient.Transport.RoundTrip(req) + if err != nil { + time.Sleep(100 * time.Millisecond) + continue + } + response.Body.Close() + + if code := response.StatusCode; code >= 200 && code < 500 { + return nil + } + + time.Sleep(100 * time.Millisecond) + } + + return fmt.Errorf("timed out after %v waiting for listener %v", timeout, l) +} + +func (l ListenSpec) JoinHostPort() string { + return net.JoinHostPort(l.Host, l.Port) +} + +// RunPagesProcess will start a gitlab-pages process with the specified listeners +// and return a function you can call to shut it down again. Use +// GetPageFromProcess to do a HTTP GET against a listener. +// +// If run as root via sudo, the gitlab-pages process will drop privileges +func RunPagesProcess(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, extraArgs ...string) (teardown func()) { + return runPagesProcess(t, true, pagesBinary, listeners, promPort, nil, extraArgs...) +} + +func RunPagesProcessWithoutWait(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, extraArgs ...string) (teardown func()) { + return runPagesProcess(t, false, pagesBinary, listeners, promPort, nil, extraArgs...) +} + +func RunPagesProcessWithSSLCertFile(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, sslCertFile string, extraArgs ...string) (teardown func()) { + return runPagesProcess(t, true, pagesBinary, listeners, promPort, []string{"SSL_CERT_FILE=" + sslCertFile}, extraArgs...) +} + +func RunPagesProcessWithEnvs(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, promPort string, envs []string, extraArgs ...string) (teardown func()) { + return runPagesProcess(t, wait, pagesBinary, listeners, promPort, envs, extraArgs...) +} + +func RunPagesProcessWithStubGitLabServer(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, promPort string, envs []string, extraArgs ...string) (teardown func()) { + var apiCalled bool + source := NewGitlabDomainsSourceStub(t, &apiCalled) + + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + pagesArgs := append([]string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", "gitlab"}, extraArgs...) + + cleanup := runPagesProcess(t, wait, pagesBinary, listeners, promPort, envs, pagesArgs...) + + return func() { + source.Close() + cleanup() + } +} + +func RunPagesProcessWithAuth(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string) func() { + configFile, cleanup := defaultConfigFileWith(t, + "auth-server=https://gitlab-auth.com", + "auth-redirect-uri=https://projects.gitlab-example.com/auth") + defer cleanup() + + return runPagesProcess(t, true, pagesBinary, listeners, promPort, nil, + "-config="+configFile, + ) +} + +func RunPagesProcessWithAuthServer(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, authServer string) func() { + return runPagesProcessWithAuthServer(t, pagesBinary, listeners, promPort, nil, authServer) +} + +func RunPagesProcessWithAuthServerWithSSLCertFile(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, sslCertFile string, authServer string) func() { + return runPagesProcessWithAuthServer(t, pagesBinary, listeners, promPort, + []string{"SSL_CERT_FILE=" + sslCertFile}, authServer) +} + +func RunPagesProcessWithAuthServerWithSSLCertDir(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, sslCertFile string, authServer string) func() { + // Create temporary cert dir + sslCertDir, err := ioutil.TempDir("", "pages-test-SSL_CERT_DIR") + require.NoError(t, err) + + // Copy sslCertFile into temp cert dir + err = copyFile(sslCertDir+"/"+path.Base(sslCertFile), sslCertFile) + require.NoError(t, err) + + innerCleanup := runPagesProcessWithAuthServer(t, pagesBinary, listeners, promPort, + []string{"SSL_CERT_DIR=" + sslCertDir}, authServer) + + return func() { + innerCleanup() + os.RemoveAll(sslCertDir) + } +} + +func runPagesProcessWithAuthServer(t *testing.T, pagesBinary string, listeners []ListenSpec, promPort string, extraEnv []string, authServer string) func() { + configFile, cleanup := defaultConfigFileWith(t, + "auth-server="+authServer, + "auth-redirect-uri=https://projects.gitlab-example.com/auth") + defer cleanup() + + return runPagesProcess(t, true, pagesBinary, listeners, promPort, extraEnv, + "-config="+configFile) +} + +func runPagesProcess(t *testing.T, wait bool, pagesBinary string, listeners []ListenSpec, promPort string, extraEnv []string, extraArgs ...string) (teardown func()) { + t.Helper() + + _, err := os.Stat(pagesBinary) + require.NoError(t, err) + + args, tempfiles := getPagesArgs(t, listeners, promPort, extraArgs) + cmd := exec.Command(pagesBinary, args...) + cmd.Env = append(os.Environ(), extraEnv...) + cmd.Stdout = &tWriter{t} + cmd.Stderr = &tWriter{t} + require.NoError(t, cmd.Start()) + t.Logf("Running %s %v", pagesBinary, args) + + waitCh := make(chan struct{}) + go func() { + cmd.Wait() + for _, tempfile := range tempfiles { + os.Remove(tempfile) + } + close(waitCh) + }() + + cleanup := func() { + cmd.Process.Signal(os.Interrupt) + <-waitCh + } + + if wait { + for _, spec := range listeners { + if err := spec.WaitUntilRequestSucceeds(waitCh); err != nil { + cleanup() + t.Fatal(err) + } + } + } + + return cleanup +} + +func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraArgs []string) (args, tempfiles []string) { + var hasHTTPS bool + + args = append(args, "-log-verbose=true") + + for _, spec := range listeners { + args = append(args, "-listen-"+spec.Type, spec.JoinHostPort()) + + if spec.Type == request.SchemeHTTPS { + hasHTTPS = true + } + } + + if hasHTTPS { + key, cert := CreateHTTPSFixtureFiles(t) + tempfiles = []string{key, cert} + args = append(args, "-root-key", key, "-root-cert", cert) + } + + if !contains(args, "pages-root") { + args = append(args, "-pages-root", "../../shared/pages") + } + + if promPort != "" { + args = append(args, "-metrics-address", promPort) + } + + args = append(args, getPagesDaemonArgs(t)...) + args = append(args, extraArgs...) + + return +} + +func contains(slice []string, s string) bool { + for _, e := range slice { + if e == s { + return true + } + } + return false +} +func getPagesDaemonArgs(t *testing.T) []string { + mode := os.Getenv("TEST_DAEMONIZE") + if mode == "" { + return nil + } + + if os.Geteuid() != 0 { + t.Log("Privilege-dropping requested but not running as root!") + t.FailNow() + return nil + } + + out := []string{} + + switch mode { + case "tmpdir": + out = append(out, "-daemon-inplace-chroot=false") + case "inplace": + out = append(out, "-daemon-inplace-chroot=true") + default: + t.Log("Unknown daemonize mode", mode) + t.FailNow() + return nil + } + + t.Log("Running pages as a daemon") + + // This triggers the drop-privileges-and-chroot code in the pages daemon + out = append(out, "-daemon-uid", "0") + out = append(out, "-daemon-gid", "65534") + + return out +} + +// Does a HTTP(S) GET against the listener specified, setting a fake +// Host: and constructing the URL from the listener and the URL suffix. +func GetPageFromListener(t *testing.T, spec ListenSpec, host, urlsuffix string) (*http.Response, error) { + return GetPageFromListenerWithCookie(t, spec, host, urlsuffix, "") +} + +func GetPageFromListenerWithCookie(t *testing.T, spec ListenSpec, host, urlsuffix string, cookie string) (*http.Response, error) { + url := spec.URL(urlsuffix) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + if cookie != "" { + req.Header.Set("Cookie", cookie) + } + + req.Host = host + + return DoPagesRequest(t, req) +} + +func GetCompressedPageFromListener(t *testing.T, spec ListenSpec, host, urlsuffix string, encoding string) (*http.Response, error) { + url := spec.URL(urlsuffix) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Host = host + req.Header.Set("Accept-Encoding", encoding) + + return DoPagesRequest(t, req) +} + +func GetProxiedPageFromListener(t *testing.T, spec ListenSpec, host, xForwardedHost, urlsuffix string) (*http.Response, error) { + url := spec.URL(urlsuffix) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Host = host + req.Header.Set("X-Forwarded-Host", xForwardedHost) + + return DoPagesRequest(t, req) +} + +func DoPagesRequest(t *testing.T, req *http.Request) (*http.Response, error) { + t.Logf("curl -X %s -H'Host: %s' %s", req.Method, req.Host, req.URL) + + return TestHTTPSClient.Do(req) +} + +func GetRedirectPage(t *testing.T, spec ListenSpec, host, urlsuffix string) (*http.Response, error) { + return GetRedirectPageWithCookie(t, spec, host, urlsuffix, "") +} + +func GetProxyRedirectPageWithCookie(t *testing.T, spec ListenSpec, host string, urlsuffix string, cookie string, https bool) (*http.Response, error) { + schema := request.SchemeHTTP + if https { + schema = request.SchemeHTTPS + } + header := http.Header{ + "X-Forwarded-Proto": []string{schema}, + "X-Forwarded-Host": []string{host}, + "cookie": []string{cookie}, + } + + return GetRedirectPageWithHeaders(t, spec, host, urlsuffix, header) +} + +func GetRedirectPageWithCookie(t *testing.T, spec ListenSpec, host, urlsuffix string, cookie string) (*http.Response, error) { + return GetRedirectPageWithHeaders(t, spec, host, urlsuffix, http.Header{"cookie": []string{cookie}}) +} + +func GetRedirectPageWithHeaders(t *testing.T, spec ListenSpec, host, urlsuffix string, header http.Header) (*http.Response, error) { + url := spec.URL(urlsuffix) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header = header + + req.Host = host + + return TestHTTPSClient.Transport.RoundTrip(req) +} + +func ClientWithConfig(tlsConfig *tls.Config) (*http.Client, func()) { + tlsConfig.RootCAs = TestCertPool + tr := &http.Transport{TLSClientConfig: tlsConfig} + client := &http.Client{Transport: tr} + + return client, tr.CloseIdleConnections +} + +func waitForRoundtrips(t *testing.T, listeners []ListenSpec, timeout time.Duration) { + nListening := 0 + start := time.Now() + for _, spec := range listeners { + for time.Since(start) < timeout { + req, err := http.NewRequest("GET", spec.URL("/"), nil) + if err != nil { + t.Fatal(err) + } + + if response, err := QuickTimeoutHTTPSClient.Transport.RoundTrip(req); err == nil { + nListening++ + response.Body.Close() + break + } + + time.Sleep(100 * time.Millisecond) + } + } + + require.Equal(t, len(listeners), nListening, "all listeners must be accepting TCP connections") +} + +func NewGitlabDomainsSourceStub(t *testing.T, apiCalled *bool) *httptest.Server { + *apiCalled = false + mux := http.NewServeMux() + mux.HandleFunc("/api/v4/internal/pages/status", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + handler := func(w http.ResponseWriter, r *http.Request) { + *apiCalled = true + domain := r.URL.Query().Get("host") + path := "../../shared/lookups/" + domain + ".json" + + fixture, err := os.Open(path) + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNoContent) + + t.Logf("GitLab domain %s source stub served 204", domain) + return + } + + defer fixture.Close() + require.NoError(t, err) + + _, err = io.Copy(w, fixture) + require.NoError(t, err) + + t.Logf("GitLab domain %s source stub served lookup", domain) + } + mux.HandleFunc("/api/v4/internal/pages", handler) + + return httptest.NewServer(mux) +} + +func newConfigFile(configs ...string) (string, error) { + f, err := ioutil.TempFile(os.TempDir(), "gitlab-pages-config") + if err != nil { + return "", err + } + defer f.Close() + + for _, config := range configs { + _, err := fmt.Fprintf(f, "%s\n", config) + if err != nil { + return "", err + } + } + + return f.Name(), nil +} + +func defaultConfigFileWith(t *testing.T, configs ...string) (string, func()) { + configs = append(configs, "auth-client-id=clientID", + "auth-client-secret=clientSecret", + "auth-secret=authSecret") + + name, err := newConfigFile(configs...) + require.NoError(t, err) + + cleanup := func() { + err := os.Remove(name) + require.NoError(t, err) + } + + return name, cleanup +} + +func copyFile(dest, src string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + srcInfo, err := srcFile.Stat() + if err != nil { + return err + } + + destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, srcInfo.Mode()) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, srcFile) + return err +} + +func newZipFileServerURL(t *testing.T, zipFilePath string) (string, func()) { + t.Helper() + + m := http.NewServeMux() + m.HandleFunc("/public.zip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, zipFilePath) + })) + m.HandleFunc("/malformed.zip", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + + // create a listener with the desired port. + l, err := net.Listen("tcp", objectStorageMockServer) + require.NoError(t, err) + + testServer := httptest.NewUnstartedServer(m) + + // NewUnstartedServer creates a listener. Close that listener and replace + // with the one we created. + testServer.Listener.Close() + testServer.Listener = l + + // Start the server. + testServer.Start() + + return testServer.URL, func() { + // Cleanup. + testServer.Close() + } +} |