Extract gitlab stub server in a separate package

5 files changed, 523 insertions, 0 deletions

diff --git a/test/gitlabstub/api_responses.go b/test/gitlabstub/api_responses.go
new file mode 100644
index 00000000..3ebf4e1f
--- /dev/null
+++ b/test/gitlabstub/api_responses.go
@@ -0,0 +1,241 @@
+package gitlabstub
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab/api"
+)
+
+type responseFn func(string) api.VirtualDomain
+
+// domainResponses holds the predefined API responses for certain domains
+// that can be used with the GitLab API stub in acceptance tests
+// Assume the working dir is inside shared/pages/
+var domainResponses = map[string]responseFn{
+	"zip-from-disk.gitlab.io": customDomain(projectConfig{
+		projectID:  123,
+		pathOnDisk: "@hashed/zip-from-disk.gitlab.io",
+	}),
+	"zip-from-disk-not-found.gitlab.io": customDomain(projectConfig{}),
+	// outside of working dir
+	"zip-not-allowed-path.gitlab.io":  customDomain(projectConfig{pathOnDisk: "../../../../"}),
+	"group.gitlab-example.com":        generateVirtualDomainFromDir("group", "group.gitlab-example.com", nil),
+	"CapitalGroup.gitlab-example.com": generateVirtualDomainFromDir("CapitalGroup", "CapitalGroup.gitlab-example.com", nil),
+	"group.404.gitlab-example.com": generateVirtualDomainFromDir("group.404", "group.404.gitlab-example.com", map[string]projectConfig{
+		"/private_project": {
+			projectID:     1300,
+			accessControl: true,
+		},
+		"/private_unauthorized": {
+			projectID:     2000,
+			accessControl: true,
+		},
+	}),
+	"group.https-only.gitlab-example.com": generateVirtualDomainFromDir("group.https-only", "group.https-only.gitlab-example.com", map[string]projectConfig{
+		"/project1": {
+			projectID: 1000,
+			https:     true,
+		},
+		"/project2": {
+			projectID: 1100,
+			https:     false,
+		},
+	}),
+	"domain.404.com": customDomain(projectConfig{
+		projectID:  1000,
+		pathOnDisk: "group.404/domain.404",
+	}),
+	"withacmechallenge.domain.com": customDomain(projectConfig{
+		projectID:  1234,
+		pathOnDisk: "group.acme/with.acme.challenge",
+	}),
+	"group.redirects.gitlab-example.com": generateVirtualDomainFromDir("group.redirects", "group.redirects.gitlab-example.com", nil),
+	"redirects.custom-domain.com": customDomain(projectConfig{
+		projectID:  1001,
+		pathOnDisk: "group.redirects/custom-domain",
+	}),
+	"test.my-domain.com": customDomain(projectConfig{
+		projectID:  1002,
+		https:      true,
+		pathOnDisk: "group.https-only/project3",
+	}),
+	"test2.my-domain.com": customDomain(projectConfig{
+		projectID:  1003,
+		https:      false,
+		pathOnDisk: "group.https-only/project4",
+	}),
+	"no.cert.com": customDomain(projectConfig{
+		projectID:  1004,
+		https:      true,
+		pathOnDisk: "group.https-only/project5",
+	}),
+	"group.auth.gitlab-example.com": generateVirtualDomainFromDir("group.auth", "group.auth.gitlab-example.com", map[string]projectConfig{
+		"/": {
+			projectID:     1005,
+			accessControl: true,
+		},
+		"/private.project": {
+			projectID:     1006,
+			accessControl: true,
+		},
+		"/private.project.1": {
+			projectID:     2006,
+			accessControl: true,
+		},
+		"/private.project.2": {
+			projectID:     3006,
+			accessControl: true,
+		},
+		"/subgroup/private.project": {
+			projectID:     1007,
+			accessControl: true,
+		},
+		"/subgroup/private.project.1": {
+			projectID:     2007,
+			accessControl: true,
+		},
+		"/subgroup/private.project.2": {
+			projectID:     3007,
+			accessControl: true,
+		},
+	}),
+	"private.domain.com": customDomain(projectConfig{
+		projectID:     1007,
+		accessControl: true,
+		pathOnDisk:    "group.auth/private.project",
+	}),
+	// NOTE: before adding more domains here, generate the zip archive by running (per project)
+	// make zip PROJECT_SUBDIR=group/serving
+	// make zip PROJECT_SUBDIR=group/project2
+}
+
+// generateVirtualDomainFromDir walks the subdirectory inside of shared/pages/ to find any zip archives.
+// It works for subdomains of pages-domain but not for custom domains (yet)
+func generateVirtualDomainFromDir(dir, rootDomain string, perPrefixConfig map[string]projectConfig) responseFn {
+	return func(wd string) api.VirtualDomain {
+		var foundZips []string
+
+		// walk over dir and save any paths containing a `.zip` file
+		// $(GITLAB_PAGES_DIR)/shared/pages + "/" + group
+
+		cleanDir := filepath.Join(wd, dir)
+
+		// make sure resolved path inside dir is under wd to avoid https://securego.io/docs/rules/g304.html
+		if !strings.HasPrefix(cleanDir, wd) {
+			log.Fatalf("path %q outside of wd %q", cleanDir, wd)
+		}
+
+		walkErr := filepath.Walk(cleanDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if strings.HasSuffix(info.Name(), ".zip") {
+				project := strings.TrimPrefix(path, wd+"/"+dir)
+				foundZips = append(foundZips, project)
+			}
+
+			return nil
+		})
+
+		if walkErr != nil {
+			log.Fatal(walkErr)
+		}
+
+		lookupPaths := make([]api.LookupPath, 0, len(foundZips))
+		// generate lookup paths
+		for _, project := range foundZips {
+			// if project = "group/subgroup/project/public.zip
+			// trim prefix group and suffix /public.zip
+			// so prefix = "/subgroup/project"
+			prefix := strings.TrimPrefix(project, dir)
+			prefix = strings.TrimSuffix(prefix, "/"+filepath.Base(project))
+
+			// use / as prefix when the current prefix matches the rootDomain, e.g.
+			// if request is group.gitlab-example.com/ and group/group.gitlab-example.com/public.zip exists
+			if prefix == "/"+rootDomain {
+				prefix = "/"
+			}
+
+			cfg, ok := perPrefixConfig[prefix]
+			if !ok {
+				cfg = projectConfig{}
+			}
+
+			sourcePath := fmt.Sprintf("file://%s", wd+"/"+dir+project)
+			sum := sha256.Sum256([]byte(sourcePath))
+			sha := hex.EncodeToString(sum[:])
+
+			lookupPath := api.LookupPath{
+				ProjectID:     cfg.projectID,
+				AccessControl: cfg.accessControl,
+				HTTPSOnly:     cfg.https,
+				// gitlab.Resolve logic expects prefix to have ending slash
+				Prefix: ensureEndingSlash(prefix),
+				Source: api.Source{
+					Type:   "zip",
+					Path:   sourcePath,
+					SHA256: sha,
+				},
+			}
+
+			lookupPaths = append(lookupPaths, lookupPath)
+		}
+
+		return api.VirtualDomain{
+			LookupPaths: lookupPaths,
+		}
+	}
+}
+
+type projectConfig struct {
+	// refer to makeGitLabPagesAccessStub for custom HTTP responses per projectID
+	projectID     int
+	accessControl bool
+	https         bool
+	pathOnDisk    string
+}
+
+// customDomain with per project config
+func customDomain(config projectConfig) responseFn {
+	return func(wd string) api.VirtualDomain {
+		sourcePath := fmt.Sprintf("file://%s/%s/public.zip", wd, config.pathOnDisk)
+		sum := sha256.Sum256([]byte(sourcePath))
+		sha := hex.EncodeToString(sum[:])
+
+		return api.VirtualDomain{
+			Certificate: "",
+			Key:         "",
+			LookupPaths: []api.LookupPath{
+				{
+					ProjectID:     config.projectID,
+					AccessControl: config.accessControl,
+					HTTPSOnly:     config.https,
+					// prefix should always be `/` for custom domains, otherwise `resolvePath` will try
+					// to look for files under public/prefix/ when serving content instead of just public/
+					// see internal/serving/disk/ for details
+					Prefix: "/",
+					Source: api.Source{
+						Type:   "zip",
+						SHA256: sha,
+						Path:   sourcePath,
+					},
+				},
+			},
+		}
+	}
+}
+
+func ensureEndingSlash(path string) string {
+	if strings.HasSuffix(path, "/") {
+		return path
+	}
+
+	return path + "/"
+}
diff --git a/test/gitlabstub/cmd/server/main.go b/test/gitlabstub/cmd/server/main.go
new file mode 100644
index 00000000..3e33daaa
--- /dev/null
+++ b/test/gitlabstub/cmd/server/main.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"gitlab.com/gitlab-org/gitlab-pages/test/gitlabstub"
+)
+
+var (
+	pagesRoot = flag.String("pages-root", "shared/pages", "The directory where pages are stored")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := os.Chdir(*pagesRoot); err != nil {
+		log.Fatalf("error chdir in %s: %v", *pagesRoot, err)
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("error getting current dir: %v", err)
+	}
+
+	server, err := gitlabstub.NewUnstartedServer(gitlabstub.WithPagesRoot(wd))
+	if err != nil {
+		log.Fatalf("error starting the server: %v", err)
+	}
+
+	server.Start()
+
+	log.Printf("listening on %s\n", server.URL)
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
+
+	<-sigChan
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := server.Config.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) {
+		log.Fatalf("error shutting down %v", err)
+	}
+}
diff --git a/test/gitlabstub/handlers.go b/test/gitlabstub/handlers.go
new file mode 100644
index 00000000..df09d914
--- /dev/null
+++ b/test/gitlabstub/handlers.go
@@ -0,0 +1,154 @@
+package gitlabstub
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"time"
+)
+
+func defaultAPIHandler(delay time.Duration, pagesRoot string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		domain := r.URL.Query().Get("host")
+		if domain == "127.0.0.1" {
+			// shortcut for healthy checkup done by WaitUntilRequestSucceeds
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		// to test slow responses from the API
+		if delay > 0 {
+			time.Sleep(delay)
+		}
+
+		// check if predefined response exists
+		if responseFn, ok := domainResponses[domain]; ok {
+			if err := json.NewEncoder(w).Encode(responseFn(pagesRoot)); err != nil {
+				log.Fatal(err)
+			}
+			return
+		}
+
+		// serve lookup from files
+		lookupFromFile(domain, w)
+	}
+}
+
+func defaultAuthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+
+		err := json.NewEncoder(w).Encode(struct {
+			AccessToken string `json:"access_token"`
+		}{
+			AccessToken: "abc",
+		})
+
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func defaultUserHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer abc" {
+			w.WriteHeader(http.StatusForbidden)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+	}
+}
+
+func lookupFromFile(domain string, w http.ResponseWriter) {
+	fixture, err := os.Open("../../shared/lookups/" + domain + ".json")
+	if errors.Is(err, fs.ErrNotExist) {
+		w.WriteHeader(http.StatusNoContent)
+
+		log.Printf("GitLab domain %s source stub served 204", domain)
+		return
+	}
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer fixture.Close()
+
+	if _, err = io.Copy(w, fixture); err != nil {
+		log.Fatal(err)
+	}
+
+	log.Printf("GitLab domain %s source stub served lookup", domain)
+}
+
+func handleAccessControlArtifactRequests(w http.ResponseWriter, r *http.Request) {
+	authorization := r.Header.Get("Authorization")
+
+	switch {
+	case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/delayed_200.html`).MatchString(r.URL.Path):
+		sleepIfAuthorized(authorization, w)
+	case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/404.html`).MatchString(r.URL.Path):
+		w.WriteHeader(http.StatusNotFound)
+	case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/500.html`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusInternalServerError)
+	case regexp.MustCompile(`/api/v4/projects/group/private/jobs/\d+/artifacts/200.html`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusOK)
+	case regexp.MustCompile(`/api/v4/projects/group/subgroup/private/jobs/\d+/artifacts/200.html`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusOK)
+	default:
+		log.Printf("Unexpected r.URL.RawPath: %q", r.URL.Path)
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.WriteHeader(http.StatusTeapot)
+	}
+}
+
+func handleAccessControlRequests(w http.ResponseWriter, r *http.Request) {
+	authorization := r.Header.Get("Authorization")
+
+	switch {
+	case regexp.MustCompile(`/api/v4/projects/1\d{3}/pages_access`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusOK)
+	case regexp.MustCompile(`/api/v4/projects/2\d{3}/pages_access`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusUnauthorized)
+	case regexp.MustCompile(`/api/v4/projects/3\d{3}/pages_access`).MatchString(r.URL.Path):
+		returnIfAuthorized(authorization, w, http.StatusUnauthorized)
+		fmt.Fprint(w, "{\"error\":\"invalid_token\"}")
+	default:
+		log.Printf("Unexpected r.URL.RawPath: %q", r.URL.Path)
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		w.WriteHeader(http.StatusTeapot)
+	}
+}
+
+func returnIfAuthorized(authorization string, w http.ResponseWriter, status int) {
+	if authorization != "" {
+		checkAuth(authorization)
+		w.WriteHeader(status)
+	} else {
+		w.WriteHeader(http.StatusNotFound)
+	}
+}
+
+func sleepIfAuthorized(authorization string, w http.ResponseWriter) {
+	if authorization != "" {
+		checkAuth(authorization)
+		time.Sleep(2 * time.Second)
+	} else {
+		w.WriteHeader(http.StatusNotFound)
+	}
+}
+
+func checkAuth(authorization string) {
+	if authorization != "Bearer abc" {
+		log.Fatalf("expected bearer abc but go %s", authorization)
+	}
+}
diff --git a/test/gitlabstub/option.go b/test/gitlabstub/option.go
new file mode 100644
index 00000000..d55abec2
--- /dev/null
+++ b/test/gitlabstub/option.go
@@ -0,0 +1,32 @@
+package gitlabstub
+
+import (
+	"net/http"
+	"time"
+)
+
+type config struct {
+	pagesHandler http.HandlerFunc
+	pagesRoot    string
+	delay        time.Duration
+}
+
+type Option func(*config)
+
+func WithPagesHandler(ph http.HandlerFunc) Option {
+	return func(sc *config) {
+		sc.pagesHandler = ph
+	}
+}
+
+func WithPagesRoot(pagesRoot string) Option {
+	return func(sc *config) {
+		sc.pagesRoot = pagesRoot
+	}
+}
+
+func WithDelay(delay time.Duration) Option {
+	return func(sc *config) {
+		sc.delay = delay
+	}
+}
diff --git a/test/gitlabstub/server.go b/test/gitlabstub/server.go
new file mode 100644
index 00000000..5cf3dacf
--- /dev/null
+++ b/test/gitlabstub/server.go
@@ -0,0 +1,43 @@
+package gitlabstub
+
+import (
+	"net/http/httptest"
+	"os"
+
+	"github.com/gorilla/mux"
+)
+
+func NewUnstartedServer(opts ...Option) (*httptest.Server, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+
+	conf := &config{
+		pagesRoot: wd,
+	}
+
+	for _, so := range opts {
+		so(conf)
+	}
+
+	if conf.pagesHandler == nil {
+		conf.pagesHandler = defaultAPIHandler(conf.delay, conf.pagesRoot)
+	}
+
+	router := mux.NewRouter()
+
+	router.HandleFunc("/api/v4/internal/pages", conf.pagesHandler)
+
+	authHandler := defaultAuthHandler()
+	router.HandleFunc("/oauth/token", authHandler)
+
+	userHandler := defaultUserHandler()
+	router.HandleFunc("/api/v4/user", userHandler)
+
+	router.HandleFunc("/api/v4/projects/{project_id:[0-9]+}/pages_access", handleAccessControlRequests)
+
+	router.PathPrefix("/").HandlerFunc(handleAccessControlArtifactRequests)
+
+	return httptest.NewUnstartedServer(router), nil
+}