diff options
author | Jaime Martinez <jmartinez@gitlab.com> | 2021-08-26 03:08:30 +0300 |
---|---|---|
committer | Jaime Martinez <jmartinez@gitlab.com> | 2021-08-26 03:08:30 +0300 |
commit | 0ef975db7aaa3595ebf56fde4a91351aa0174f11 (patch) | |
tree | fffa9bffca13d109d8508a3fa57d3171e0332bf3 | |
parent | c27027696e0745654d38fac79ab3adff12a8f72e (diff) | |
parent | 33985bb325b4b21c84d47c644c6d4616b9c3e8ed (diff) |
Merge branch 'remove/disk-source' into 'master'
refactor: remove support for disk configuration source
Closes #103, #158, #68, and #382
See merge request gitlab-org/gitlab-pages!541
-rw-r--r-- | app.go | 6 | ||||
-rw-r--r-- | app_test.go | 24 | ||||
-rw-r--r-- | go.mod | 2 | ||||
-rw-r--r-- | go.sum | 4 | ||||
-rw-r--r-- | internal/config/config.go | 21 | ||||
-rw-r--r-- | internal/config/flags.go | 4 | ||||
-rw-r--r-- | internal/source/disk/config.go | 57 | ||||
-rw-r--r-- | internal/source/disk/config_test.go | 65 | ||||
-rw-r--r-- | internal/source/disk/custom.go | 37 | ||||
-rw-r--r-- | internal/source/disk/disk.go | 56 | ||||
-rw-r--r-- | internal/source/disk/domain_test.go | 507 | ||||
-rw-r--r-- | internal/source/disk/group.go | 104 | ||||
-rw-r--r-- | internal/source/disk/group_test.go | 97 | ||||
-rw-r--r-- | internal/source/disk/map.go | 308 | ||||
-rw-r--r-- | internal/source/disk/map_test.go | 253 | ||||
-rw-r--r-- | internal/source/domains.go | 106 | ||||
-rw-r--r-- | internal/source/domains_test.go | 84 | ||||
-rw-r--r-- | internal/source/gitlab/gitlab.go | 5 | ||||
-rw-r--r-- | internal/source/source.go | 1 | ||||
-rw-r--r-- | test/acceptance/config_test.go | 27 | ||||
-rw-r--r-- | test/acceptance/helpers_test.go | 11 | ||||
-rw-r--r-- | test/acceptance/serving_test.go | 85 |
22 files changed, 53 insertions, 1811 deletions
@@ -59,7 +59,7 @@ type theApp struct { } func (a *theApp) isReady() bool { - return a.domains.IsReady() + return true } func (a *theApp) ServeTLS(ch *cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) { @@ -413,8 +413,6 @@ func (a *theApp) Run() { a.listenMetricsFD(&wg, a.config.ListenMetrics) } - a.domains.Read(a.config.General.Domain) - wg.Wait() } @@ -495,7 +493,7 @@ func (a *theApp) listenMetricsFD(wg *sync.WaitGroup, fd uintptr) { } func runApp(config *cfg.Config) { - domains, err := source.NewDomains(config.General.DomainConfigurationSource, &config.GitLab) + domains, err := source.NewDomains(&config.GitLab) if err != nil { log.WithError(err).Fatal("could not create domains config source") } diff --git a/app_test.go b/app_test.go index 48a6013b..483ffb39 100644 --- a/app_test.go +++ b/app_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/require" @@ -79,20 +80,29 @@ func TestHealthCheckMiddleware(t *testing.T) { { name: "Healthcheck request", path: "/-/healthcheck", - status: http.StatusServiceUnavailable, - body: "not yet ready\n", + status: http.StatusOK, + body: "success\n", }, } - cfg, err := config.LoadConfig() - require.NoError(t, err) - cfg.General.StatusPath = "/-/healthcheck" + validCfg := config.GitLab{ + InternalServer: "server", + APISecretKey: []byte("secret"), + ClientHTTPTimeout: time.Second, + JWTTokenExpiration: time.Second, + } - domains, err := source.NewDomains("auto", &cfg.GitLab) + domains, err := source.NewDomains(&validCfg) require.NoError(t, err) + cfg := config.Config{ + General: config.General{ + StatusPath: "/-/healthcheck", + }, + } + app := theApp{ - config: cfg, + config: &cfg, domains: domains, } @@ -3,7 +3,6 @@ module gitlab.com/gitlab-org/gitlab-pages go 1.16 require ( - github.com/andybalholm/brotli v1.0.3 github.com/golang-jwt/jwt/v4 v4.0.0 github.com/golang/mock v1.3.1 github.com/gorilla/handlers v1.4.2 @@ -12,7 +11,6 @@ require ( github.com/gorilla/sessions v1.2.0 github.com/hashicorp/go-multierror v1.1.1 github.com/karlseguin/ccache/v2 v2.0.6 - github.com/karrick/godirwalk v1.10.12 github.com/namsral/flag v1.7.4-pre github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pires/go-proxyproto v0.2.0 @@ -27,8 +27,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= -github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -164,8 +162,6 @@ github.com/karlseguin/ccache/v2 v2.0.6 h1:jFCLz4bF4EPfuCcvESAgYNClkEb31LV3WzyOwL github.com/karlseguin/ccache/v2 v2.0.6/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= -github.com/karrick/godirwalk v1.10.12 h1:BqUm+LuJcXjGv1d2mj3gBiQyrQ57a0rYoAmhvJQ7RDU= -github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= diff --git a/internal/config/config.go b/internal/config/config.go index 64dc4666..c57eb15d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -45,15 +45,14 @@ type Config struct { // General groups settings that are general to GitLab Pages and can not // be categorized under other head. type General struct { - Domain string - DomainConfigurationSource string - MaxConns int - MetricsAddress string - RedirectHTTP bool - RootCertificate []byte - RootDir string - RootKey []byte - StatusPath string + Domain string + MaxConns int + MetricsAddress string + RedirectHTTP bool + RootCertificate []byte + RootDir string + RootKey []byte + StatusPath string DisableCrossOriginRequests bool InsecureCiphers bool @@ -187,7 +186,6 @@ func loadConfig() (*Config, error) { config := &Config{ General: General{ Domain: strings.ToLower(*pagesDomain), - DomainConfigurationSource: *domainConfigSource, MaxConns: *maxConns, MetricsAddress: *metricsAddress, RedirectHTTP: *redirectHTTP, @@ -227,7 +225,7 @@ func loadConfig() (*Config, error) { UID: *daemonUID, GID: *daemonGID, InplaceChroot: *daemonInplaceChroot, - EnableJail: *daemonEnableJail || *domainConfigSource == "disk", + EnableJail: *daemonEnableJail, }, Log: Log{ Format: *logFormat, @@ -325,7 +323,6 @@ func LogConfig(config *Config) { "gitlab-server": config.GitLab.PublicServer, "internal-gitlab-server": config.GitLab.InternalServer, "api-secret-key": *gitLabAPISecretKey, - "domain-config-source": config.General.DomainConfigurationSource, "enable-disk": config.GitLab.EnableDisk, "auth-redirect-uri": config.Authentication.RedirectURI, "auth-scope": config.Authentication.Scope, diff --git a/internal/config/flags.go b/internal/config/flags.go index 28a318c4..63c52631 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -41,8 +41,8 @@ var ( gitlabRetrievalInterval = flag.Duration("gitlab-retrieval-interval", time.Second, "The interval to wait before retrying to resolve a domain's configuration via the GitLab API") gitlabRetrievalRetries = flag.Int("gitlab-retrieval-retries", 3, "The maximum number of times to retry to resolve a domain's configuration via the API") - domainConfigSource = flag.String("domain-config-source", "gitlab", "Domain configuration source 'disk', 'auto' or 'gitlab' (default: 'gitlab'). DEPRECATED: gitlab-pages will use the API-based configuration starting from 14.3 see https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993") - enableDisk = flag.Bool("enable-disk", true, "Enable disk access, shall be disabled in environments where shared disk storage isn't available") + _ = flag.String("domain-config-source", "gitlab", "DEPRECATED and has not affect, see https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/541") + enableDisk = flag.Bool("enable-disk", true, "Enable disk access, shall be disabled in environments where shared disk storage isn't available") clientID = flag.String("auth-client-id", "", "GitLab application Client ID") clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret") diff --git a/internal/source/disk/config.go b/internal/source/disk/config.go deleted file mode 100644 index d2e6c123..00000000 --- a/internal/source/disk/config.go +++ /dev/null @@ -1,57 +0,0 @@ -package disk - -import ( - "encoding/json" - "os" - "path/filepath" - "strings" -) - -// DomainConfig represents a custom domain config -type domainConfig struct { - Domain string - Certificate string - Key string - HTTPSOnly bool `json:"https_only"` - ID uint64 `json:"id"` - AccessControl bool `json:"access_control"` -} - -// MultiDomainConfig represents a group of custom domain configs -type multiDomainConfig struct { - Domains []domainConfig - HTTPSOnly bool `json:"https_only"` - ID uint64 `json:"id"` - AccessControl bool `json:"access_control"` -} - -// ProjectConfig is a project-level configuration -type projectConfig struct { - NamespaceProject bool - HTTPSOnly bool - AccessControl bool - ID uint64 -} - -// Valid validates a custom domain config for a root domain -func (c *domainConfig) Valid(rootDomain string) bool { - if c.Domain == "" { - return false - } - - // TODO: better sanitize domain - domain := strings.ToLower(c.Domain) - rootDomain = "." + rootDomain - return !strings.HasSuffix(domain, rootDomain) -} - -// Read reads a multi domain config and decodes it from a `config.json` -func (c *multiDomainConfig) Read(group, project string) error { - configFile, err := os.Open(filepath.Join(group, project, "config.json")) - if err != nil { - return err - } - defer configFile.Close() - - return json.NewDecoder(configFile).Decode(c) -} diff --git a/internal/source/disk/config_test.go b/internal/source/disk/config_test.go deleted file mode 100644 index 1bb2364a..00000000 --- a/internal/source/disk/config_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package disk - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -const configFile = "test-group/test-project/config.json" -const invalidConfig = `{"Domains":{}}` -const validConfig = `{"Domains":[{"Domain":"test"}]}` - -func TestDomainConfigValidness(t *testing.T) { - d := domainConfig{} - require.False(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test"} - require.True(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test"} - require.True(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test.gitlab.io"} - require.False(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test.test.gitlab.io"} - require.False(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test.testgitlab.io"} - require.True(t, d.Valid("gitlab.io")) - - d = domainConfig{Domain: "test.GitLab.Io"} - require.False(t, d.Valid("gitlab.io")) -} - -func TestDomainConfigRead(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - d := multiDomainConfig{} - err := d.Read("test-group", "test-project") - require.Error(t, err) - - os.MkdirAll(filepath.Dir(configFile), 0700) - defer os.RemoveAll("test-group") - - d = multiDomainConfig{} - err = d.Read("test-group", "test-project") - require.Error(t, err) - - err = ioutil.WriteFile(configFile, []byte(invalidConfig), 0600) - require.NoError(t, err) - d = multiDomainConfig{} - err = d.Read("test-group", "test-project") - require.Error(t, err) - - err = ioutil.WriteFile(configFile, []byte(validConfig), 0600) - require.NoError(t, err) - d = multiDomainConfig{} - err = d.Read("test-group", "test-project") - require.NoError(t, err) -} diff --git a/internal/source/disk/custom.go b/internal/source/disk/custom.go deleted file mode 100644 index 037abfee..00000000 --- a/internal/source/disk/custom.go +++ /dev/null @@ -1,37 +0,0 @@ -package disk - -import ( - "net/http" - - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/serving" - "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/local" -) - -type customProjectResolver struct { - config *domainConfig - - path string -} - -func (p *customProjectResolver) Resolve(r *http.Request) (*serving.Request, error) { - if p.config == nil { - return nil, domain.ErrDomainDoesNotExist - } - - lookupPath := &serving.LookupPath{ - ServingType: "file", - Prefix: "/", - Path: p.path, - IsNamespaceProject: false, - IsHTTPSOnly: p.config.HTTPSOnly, - HasAccessControl: p.config.AccessControl, - ProjectID: p.config.ID, - } - - return &serving.Request{ - Serving: local.Instance(), - LookupPath: lookupPath, - SubPath: r.URL.Path, - }, nil -} diff --git a/internal/source/disk/disk.go b/internal/source/disk/disk.go deleted file mode 100644 index 3596af29..00000000 --- a/internal/source/disk/disk.go +++ /dev/null @@ -1,56 +0,0 @@ -package disk - -import ( - "context" - "strings" - "sync" - "time" - - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" -) - -// Disk struct represents a map of all domains supported by pages that are -// stored on a disk with corresponding `config.json`. -type Disk struct { - dm Map - lock *sync.RWMutex -} - -// New is a factory method for the Disk source. It is initializing a mutex. It -// should not initialize `dm` as we later check the readiness by comparing it -// with a nil value. -func New() *Disk { - return &Disk{ - lock: &sync.RWMutex{}, - } -} - -// GetDomain returns a domain from the domains map if it exists -func (d *Disk) GetDomain(ctx context.Context, host string) (*domain.Domain, error) { - host = strings.ToLower(host) - - d.lock.RLock() - defer d.lock.RUnlock() - - return d.dm[host], nil -} - -// IsReady checks if the domains source is ready for work. The disk source is -// ready after traversing entire filesystem and reading all domains' -// configuration files. -func (d *Disk) IsReady() bool { - return d.dm != nil -} - -// Read starts the domain source, in this case it is reading domains from -// groups on disk concurrently. -func (d *Disk) Read(rootDomain string) { - go Watch(rootDomain, d.updateDomains, time.Second) -} - -func (d *Disk) updateDomains(dm Map) { - d.lock.Lock() - defer d.lock.Unlock() - - d.dm = dm -} diff --git a/internal/source/disk/domain_test.go b/internal/source/disk/domain_test.go deleted file mode 100644 index abffddb4..00000000 --- a/internal/source/disk/domain_test.go +++ /dev/null @@ -1,507 +0,0 @@ -package disk - -import ( - "compress/gzip" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "strconv" - "testing" - "time" - - "github.com/andybalholm/brotli" - "github.com/stretchr/testify/require" - - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/fixture" - "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" -) - -func serveFileOrNotFound(domain *domain.Domain) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if !domain.ServeFileHTTP(w, r) { - domain.ServeNotFoundHTTP(w, r) - } - } -} - -func testGroupServeHTTPHost(t *testing.T, host string) { - testGroup := &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: map[string]*projectConfig{ - "group.test.io": &projectConfig{}, - "group.gitlab-example.com": &projectConfig{}, - "project": &projectConfig{}, - "project2": &projectConfig{}, - }, - }, - } - - makeURL := func(path string) string { - return "http://" + host + path - } - - serve := serveFileOrNotFound(testGroup) - - require.HTTPBodyContains(t, serve, "GET", makeURL("/"), nil, "main-dir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/index"), nil, "main-dir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/index.html"), nil, "main-dir") - testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project"), nil, "//"+host+"/project/") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index/"), nil, "project-subdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index.html"), nil, "project-subdir") - testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project/subdir"), nil, "//"+host+"/project/subdir/") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project/subdir/"), nil, "project-subsubdir") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/"), nil, "project2-main") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index"), nil, "project2-main") - require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index.html"), nil, "project2-main") - require.HTTPError(t, serve, "GET", makeURL("/private.project/"), nil) - require.HTTPError(t, serve, "GET", makeURL("//about.gitlab.com/%2e%2e"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink/index.html"), nil) - require.HTTPError(t, serve, "GET", makeURL("/symlink/subdir/"), nil) - require.HTTPError(t, serve, "GET", makeURL("/project/fifo"), nil) - require.HTTPError(t, serve, "GET", makeURL("/not-existing-file"), nil) - require.HTTPRedirect(t, serve, "GET", makeURL("/project//about.gitlab.com/%2e%2e"), nil) -} - -func TestGroupServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - t.Run("group.test.io", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io") }) - t.Run("group.test.io:8080", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io:8080") }) -} - -func TestDomainServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testDomain := &domain.Domain{ - Name: "test.domain.com", - Resolver: &customProjectResolver{ - path: "group/project2/public", - config: &domainConfig{}, - }, - } - - require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/", nil, "project2-main") - require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/index.html", nil, "project2-main") - require.HTTPRedirect(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil) - require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil, - `<a href="/subdir/">Found</a>`) - require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/", nil, "project2-subdir") - require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/index.html", nil, "project2-subdir") - require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "//about.gitlab.com/%2e%2e", nil) - require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "/not-existing-file", nil) -} - -func TestIsHTTPSOnly(t *testing.T) { - tests := []struct { - name string - domain *domain.Domain - url string - expected bool - }{ - { - name: "Default group domain with HTTPS-only enabled", - domain: &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}}, - }, - }, - url: "http://test-domain", - expected: true, - }, - { - name: "Default group domain with HTTPS-only disabled", - domain: &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: projects{"test-domain": &projectConfig{HTTPSOnly: false}}, - }, - }, - url: "http://test-domain", - expected: false, - }, - { - name: "Case-insensitive default group domain with HTTPS-only enabled", - domain: &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}}, - }, - }, - url: "http://Test-domain", - expected: true, - }, - { - name: "Other group domain with HTTPS-only enabled", - domain: &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: projects{"project": &projectConfig{HTTPSOnly: true}}, - }, - }, - url: "http://test-domain/project", - expected: true, - }, - { - name: "Other group domain with HTTPS-only disabled", - domain: &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: projects{"project": &projectConfig{HTTPSOnly: false}}, - }, - }, - url: "http://test-domain/project", - expected: false, - }, - { - name: "Unknown project", - domain: &domain.Domain{ - Resolver: &Group{}, - }, - url: "http://test-domain/project", - expected: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - req, _ := http.NewRequest(http.MethodGet, test.url, nil) - require.Equal(t, test.expected, test.domain.IsHTTPSOnly(req)) - }) - } -} - -func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) { - w := httptest.NewRecorder() - req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) - require.NoError(t, err) - if acceptEncoding != "" { - req.Header.Add("Accept-Encoding", acceptEncoding) - } - handler(w, req) - - if expectCompressed { - contentLength := w.Header().Get("Content-Length") - require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length") - - contentEncoding := w.Header().Get("Content-Encoding") - require.Equal(t, "gzip", contentEncoding, "Content-Encoding") - - reader, err := gzip.NewReader(w.Body) - require.NoError(t, err) - defer reader.Close() - - bytes, err := ioutil.ReadAll(reader) - require.NoError(t, err) - require.Contains(t, string(bytes), str) - } else { - require.Contains(t, w.Body.String(), str) - } - - require.Equal(t, contentType, w.Header().Get("Content-Type")) -} - -func TestGroupServeHTTPGzip(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: map[string]*projectConfig{ - "group.test.io": &projectConfig{}, - "group.gitlab-example.com": &projectConfig{}, - "project": &projectConfig{}, - "project2": &projectConfig{}, - }, - }, - } - - testSet := []struct { - mode string // HTTP mode - url string // Test URL - acceptEncoding string // Accept encoding header - body interface{} // Expected body at above URL - contentType string // Expected content-type - expectCompressed bool // Expect the response to be gzipped? - }{ - // No gzip encoding requested - {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "gzip; q=0", "main-dir", "text/html; charset=utf-8", false}, - // gzip encoding requested, - {"GET", "/index.html", "identity, gzip", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=0.9", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=0.9, gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=0, gzip; q=1", "main-dir", "text/html; charset=utf-8", true}, - // fallback to gzip because .br is missing - {"GET", "/index2.html", "*", "main-dir", "text/html; charset=utf-8", true}, - // gzip encoding requested, but url does not have compressed content on disk - {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "identity, gzip", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=1", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=0.9", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=1, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "gzip; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false}, - // malformed headers - {"GET", "/index.html", ";; gzip", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "gzip; quality=1", "main-dir", "text/html; charset=utf-8", false}, - // Symlinked .gz files are not supported - {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with text content - {"GET", "/text.unknown", "gzip", "hello", "text/plain; charset=utf-8", true}, - {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with PNG content - {"GET", "/image.unknown", "gzip", "GIF89a", "image/gif", true}, - {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false}, - } - - for _, tt := range testSet { - t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) { - URL := "http://group.test.io" + tt.url - testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed) - }) - } -} - -func testHTTPBrotli(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) { - w := httptest.NewRecorder() - req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil) - require.NoError(t, err) - if acceptEncoding != "" { - req.Header.Add("Accept-Encoding", acceptEncoding) - } - handler(w, req) - - if expectCompressed { - contentLength := w.Header().Get("Content-Length") - require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length") - - contentEncoding := w.Header().Get("Content-Encoding") - require.Equal(t, "br", contentEncoding, "Content-Encoding") - - reader := brotli.NewReader(w.Body) - bytes, err := ioutil.ReadAll(reader) - require.NoError(t, err) - require.Contains(t, string(bytes), str) - } else { - require.Contains(t, w.Body.String(), str) - } - - require.Equal(t, contentType, w.Header().Get("Content-Type")) -} - -func TestGroupServeHTTPBrotli(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: map[string]*projectConfig{ - "group.test.io": &projectConfig{}, - "group.gitlab-example.com": &projectConfig{}, - "project": &projectConfig{}, - "project2": &projectConfig{}, - }, - }, - } - - testSet := []struct { - mode string // HTTP mode - url string // Test URL - acceptEncoding string // Accept encoding header - body interface{} // Expected body at above URL - contentType string // Expected content-type - expectCompressed bool // Expect the response to be br compressed? - }{ - // No br encoding requested - {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "br; q=0", "main-dir", "text/html; charset=utf-8", false}, - // br encoding requested, - {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "identity, br", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=1", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=0.9", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=1, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "br; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true}, - {"GET", "/index.html", "gzip; q=0.5, br; q=1", "main-dir", "text/html; charset=utf-8", true}, - // br encoding requested, but url does not have compressed content on disk - {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "identity, br", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br; q=1", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br; q=0.9", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br; q=1, deflate", "project2-main", "text/html; charset=utf-8", false}, - {"GET", "/project2/index.html", "br; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false}, - // malformed headers - {"GET", "/index.html", ";; br", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false}, - {"GET", "/index.html", "br; quality=1", "main-dir", "text/html; charset=utf-8", false}, - // Symlinked .br files are not supported - {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with text content - {"GET", "/text.unknown", "*", "hello", "text/plain; charset=utf-8", true}, - {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false}, - // Unknown file-extension, with PNG content - {"GET", "/image.unknown", "*", "GIF89a", "image/gif", true}, - {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false}, - } - - for _, tt := range testSet { - t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) { - URL := "http://group.test.io" + tt.url - testHTTPBrotli(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed) - }) - } -} - -func TestGroup404ServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &domain.Domain{ - Resolver: &Group{ - name: "group.404", - projects: map[string]*projectConfig{ - "domain.404": &projectConfig{}, - "group.404.test.io": &projectConfig{}, - "project.404": &projectConfig{}, - "project.404.symlink": &projectConfig{}, - "project.no.404": &projectConfig{}, - }, - }, - } - - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page") - require.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page") - - // Ensure the namespace project's custom 404.html is not used by projects - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.") -} - -func TestDomain404ServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testDomain := &domain.Domain{ - Resolver: &customProjectResolver{ - path: "group.404/domain.404/public/", - config: &domainConfig{Domain: "domain.404.com"}, - }, - } - - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom domain.404 page") - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/", nil, "Custom domain.404 page") -} - -func TestPredefined404ServeHTTP(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testDomain := domain.New("", "", "", &customProjectResolver{}) - - testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found") -} - -func TestGroupCertificate(t *testing.T) { - testGroup := &domain.Domain{} - - tls, err := testGroup.EnsureCertificate() - require.Nil(t, tls) - require.Error(t, err) -} - -func TestDomainNoCertificate(t *testing.T) { - testDomain := &domain.Domain{ - Resolver: &customProjectResolver{ - path: "group/project2/public", - config: &domainConfig{Domain: "test.domain.com"}, - }, - } - - tls, err := testDomain.EnsureCertificate() - require.Nil(t, tls) - require.Error(t, err) - - _, err2 := testDomain.EnsureCertificate() - require.Error(t, err) - require.Equal(t, err, err2) -} - -func TestDomainCertificate(t *testing.T) { - testDomain := &domain.Domain{ - Name: "test.domain.com", - CertificateCert: fixture.Certificate, - CertificateKey: fixture.Key, - Resolver: &customProjectResolver{ - path: "group/project2/public", - }, - } - - tls, err := testDomain.EnsureCertificate() - require.NotNil(t, tls) - require.NoError(t, err) -} - -func TestCacheControlHeaders(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - testGroup := &domain.Domain{ - Resolver: &Group{ - name: "group", - projects: map[string]*projectConfig{ - "group.test.io": &projectConfig{}, - }, - }, - } - w := httptest.NewRecorder() - req, err := http.NewRequest("GET", "http://group.test.io/", nil) - require.NoError(t, err) - - now := time.Now() - serveFileOrNotFound(testGroup)(w, req) - - require.Equal(t, http.StatusOK, w.Code) - require.Equal(t, "max-age=600", w.Header().Get("Cache-Control")) - - expires := w.Header().Get("Expires") - require.NotEmpty(t, expires) - - expiresTime, err := time.Parse(time.RFC1123, expires) - require.NoError(t, err) - - require.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute) -} - -var chdirSet = false - -func setUpTests(t *testing.T) func() { - t.Helper() - return testhelpers.ChdirInPath(t, "../../../shared/pages", &chdirSet) -} diff --git a/internal/source/disk/group.go b/internal/source/disk/group.go deleted file mode 100644 index 9499df46..00000000 --- a/internal/source/disk/group.go +++ /dev/null @@ -1,104 +0,0 @@ -package disk - -import ( - "net/http" - "path" - "path/filepath" - "strings" - - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/host" - "gitlab.com/gitlab-org/gitlab-pages/internal/serving" - "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/local" -) - -const ( - subgroupScanLimit int = 21 - // maxProjectDepth is set to the maximum nested project depth in gitlab (21) plus 3. - // One for the project, one for the first empty element of the split (URL.Path starts with /), - // and one for the real file path - maxProjectDepth int = subgroupScanLimit + 3 -) - -// Group represents a GitLab group with project configs and subgroups -type Group struct { - name string - - // nested groups - subgroups subgroups - - // group domains: - projects projects -} - -type projects map[string]*projectConfig -type subgroups map[string]*Group - -func (g *Group) digProjectWithSubpath(parentPath string, keys []string) (*projectConfig, string, string) { - if len(keys) >= 1 { - head := keys[0] - tail := keys[1:] - currentPath := path.Join(parentPath, head) - search := strings.ToLower(head) - - if project := g.projects[search]; project != nil { - return project, currentPath, path.Join(tail...) - } - - if subgroup := g.subgroups[search]; subgroup != nil { - return subgroup.digProjectWithSubpath(currentPath, tail) - } - } - - return nil, "", "" -} - -// Look up a project inside the domain based on the host and path. Returns the -// project and its name (if applicable) -func (g *Group) getProjectConfigWithSubpath(r *http.Request) (*projectConfig, string, string, string) { - // Check for a project specified in the URL: http://group.gitlab.io/projectA - // If present, these projects shadow the group domain. - split := strings.SplitN(r.URL.Path, "/", maxProjectDepth) - if len(split) >= 2 { - projectConfig, projectPath, urlPath := g.digProjectWithSubpath("", split[1:]) - if projectConfig != nil { - return projectConfig, "/" + projectPath, projectPath, urlPath - } - } - - // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io), - // return the group project if it exists. - if host := host.FromRequest(r); host != "" { - if groupProject := g.projects[host]; groupProject != nil { - return groupProject, "/", host, strings.Join(split[1:], "/") - } - } - - return nil, "", "", "" -} - -// Resolve tries to find project and its config recursively for a given request -// to a group domain -func (g *Group) Resolve(r *http.Request) (*serving.Request, error) { - projectConfig, prefix, projectPath, subPath := g.getProjectConfigWithSubpath(r) - - if projectConfig == nil { - return nil, domain.ErrDomainDoesNotExist - } - - lookupPath := &serving.LookupPath{ - ServingType: "file", - Prefix: prefix, - Path: filepath.Join(g.name, projectPath, "public") + "/", - IsNamespaceProject: projectConfig.NamespaceProject, - IsHTTPSOnly: projectConfig.HTTPSOnly, - HasAccessControl: projectConfig.AccessControl, - ProjectID: projectConfig.ID, - } - - return &serving.Request{ - Serving: local.Instance(), - LookupPath: lookupPath, - SubPath: subPath, - }, nil -} diff --git a/internal/source/disk/group_test.go b/internal/source/disk/group_test.go deleted file mode 100644 index d0fb49bd..00000000 --- a/internal/source/disk/group_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package disk - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestGroupDig(t *testing.T) { - matchingProject := &projectConfig{ID: 1} - - tests := []struct { - name string - g Group - path string - expectedProject *projectConfig - expectedProjectPath string - expectedPath string - }{ - { - name: "empty group", - path: "projectb/demo/features.html", - g: Group{}, - }, - { - name: "group with project", - path: "projectb/demo/features.html", - g: Group{ - projects: projects{"projectb": matchingProject}, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with project and no path in URL", - path: "projectb", - g: Group{ - projects: projects{"projectb": matchingProject}, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - }, - { - name: "group with subgroup and project", - path: "projectb/demo/features.html", - g: Group{ - projects: projects{"projectb": matchingProject}, - subgroups: subgroups{ - "sub1": &Group{ - projects: projects{"another": &projectConfig{}}, - }, - }, - }, - expectedProject: matchingProject, - expectedProjectPath: "projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with project inside a subgroup", - path: "sub1/projectb/demo/features.html", - g: Group{ - subgroups: subgroups{ - "sub1": &Group{ - projects: projects{"projectb": matchingProject}, - }, - }, - projects: projects{"another": &projectConfig{}}, - }, - expectedProject: matchingProject, - expectedProjectPath: "sub1/projectb", - expectedPath: "demo/features.html", - }, - { - name: "group with matching subgroup but no project", - path: "sub1/projectb/demo/features.html", - g: Group{ - subgroups: subgroups{ - "sub1": &Group{ - projects: projects{"another": &projectConfig{}}, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - project, projectPath, urlPath := test.g.digProjectWithSubpath("", strings.Split(test.path, "/")) - - require.Equal(t, test.expectedProject, project) - require.Equal(t, test.expectedProjectPath, projectPath) - require.Equal(t, test.expectedPath, urlPath) - }) - } -} diff --git a/internal/source/disk/map.go b/internal/source/disk/map.go deleted file mode 100644 index 05ab4c30..00000000 --- a/internal/source/disk/map.go +++ /dev/null @@ -1,308 +0,0 @@ -package disk - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/karrick/godirwalk" - "github.com/sirupsen/logrus" - "gitlab.com/gitlab-org/labkit/log" - - "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/metrics" -) - -// preventive measure to skip `@hashed` dir for new zip deployments when sourcing config from disk -// https://gitlab.com/gitlab-org/gitlab-pages/-/issues/468 -const skipHashedDir = "@hashed" - -// Map maps domain names to Domain instances. -type Map map[string]*domain.Domain - -type domainsUpdater func(Map) - -func (dm Map) updateDomainMap(domainName string, domain *domain.Domain) { - if _, ok := dm[domainName]; ok { - log.WithFields(log.Fields{ - "domain_name": domainName, - }).Error("Duplicate domain") - } - - dm[domainName] = domain -} - -func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) { - newDomain := domain.New( - strings.ToLower(config.Domain), - config.Certificate, - config.Key, - &customProjectResolver{ - config: config, - path: filepath.Join(groupName, projectName, "public"), - }, - ) - - dm.updateDomainMap(newDomain.Name, newDomain) -} - -func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, httpsOnly bool, accessControl bool, id uint64) { - domainName := strings.ToLower(groupName + "." + rootDomain) - groupDomain := dm[domainName] - - if groupDomain == nil { - groupResolver := &Group{ - name: groupName, - projects: make(projects), - subgroups: make(subgroups), - } - - groupDomain = domain.New(domainName, "", "", groupResolver) - } - - split := strings.SplitN(strings.ToLower(projectPath), "/", maxProjectDepth) - projectName := split[len(split)-1] - g := groupDomain.Resolver.(*Group) - - for i := 0; i < len(split)-1; i++ { - subgroupName := split[i] - subgroup := g.subgroups[subgroupName] - if subgroup == nil { - subgroup = &Group{ - name: subgroupName, - projects: make(projects), - subgroups: make(subgroups), - } - g.subgroups[subgroupName] = subgroup - } - - g = subgroup - } - - g.projects[projectName] = &projectConfig{ - NamespaceProject: domainName == projectName, - HTTPSOnly: httpsOnly, - AccessControl: accessControl, - ID: id, - } - - dm[domainName] = groupDomain -} - -func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *multiDomainConfig) { - if config == nil { - // This is necessary to preserve the previous behaviour where a - // group domain is created even if no config.json files are - // loaded successfully. Is it safe to remove this? - dm.updateGroupDomain(rootDomain, group, projectName, false, false, 0) - return - } - - dm.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly, config.AccessControl, config.ID) - - for _, domainConfig := range config.Domains { - config := domainConfig // domainConfig is reused for each loop iteration - if domainConfig.Valid(rootDomain) { - dm.addDomain(rootDomain, group, projectName, &config) - } - } -} - -func readProject(group, parent, projectName string, level int, fanIn chan<- jobResult) { - if strings.HasPrefix(projectName, ".") { - return - } - - // Ignore projects that have .deleted in name - if strings.HasSuffix(projectName, ".deleted") { - return - } - - projectPath := filepath.Join(parent, projectName) - if _, err := os.Lstat(filepath.Join(group, projectPath, "public")); err != nil { - // maybe it's a subgroup - if level <= subgroupScanLimit { - buf := make([]byte, 2*os.Getpagesize()) - readProjects(group, projectPath, level+1, buf, fanIn) - } - - return - } - - // We read the config.json file _before_ fanning in, because it does disk - // IO and it does not need access to the domains map. - config := &multiDomainConfig{} - if err := config.Read(group, projectPath); err != nil { - config = nil - } - - fanIn <- jobResult{group: group, project: projectPath, config: config} -} - -func readProjects(group, parent string, level int, buf []byte, fanIn chan<- jobResult) { - subgroup := filepath.Join(group, parent) - fis, err := godirwalk.ReadDirents(subgroup, buf) - if err != nil { - log.WithError(err).WithFields(log.Fields{ - "group": group, - "parent": parent, - }).Print("readdir failed") - return - } - - for _, project := range fis { - // Ignore non directories - if !project.IsDir() { - continue - } - - readProject(group, parent, project.Name(), level, fanIn) - } -} - -type jobResult struct { - group string - project string - config *multiDomainConfig -} - -// ReadGroups walks the pages directory and populates dm with all the domains it finds. -func (dm Map) ReadGroups(rootDomain string, fis godirwalk.Dirents) { - fanOutGroups := make(chan string) - fanIn := make(chan jobResult) - wg := &sync.WaitGroup{} - for i := 0; i < 4; i++ { - wg.Add(1) - - go func() { - buf := make([]byte, 2*os.Getpagesize()) - - for group := range fanOutGroups { - if group == skipHashedDir { - continue - } - - started := time.Now() - - readProjects(group, "", 0, buf, fanIn) - - log.WithFields(log.Fields{ - "group": group, - "duration": time.Since(started).Seconds(), - }).Debug("Loaded projects for group") - } - - wg.Done() - }() - } - - go func() { - wg.Wait() - close(fanIn) - }() - - done := make(chan struct{}) - go func() { - for result := range fanIn { - dm.readProjectConfig(rootDomain, result.group, result.project, result.config) - } - - close(done) - }() - - for _, group := range fis { - if !group.IsDir() { - continue - } - if strings.HasPrefix(group.Name(), ".") { - continue - } - fanOutGroups <- group.Name() - } - close(fanOutGroups) - - <-done -} - -const ( - updateFile = ".update" -) - -// Watch polls the filesystem and kicks off a new domain directory scan when needed. -func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) { - lastUpdate := []byte("no-update") - - for { - // Read the update file - update, err := ioutil.ReadFile(updateFile) - if err != nil && !os.IsNotExist(err) { - log.WithError(err).Print("failed to read update timestamp") - time.Sleep(interval) - continue - } - - // If it's the same ignore - if bytes.Equal(lastUpdate, update) { - time.Sleep(interval) - continue - } - lastUpdate = update - - started := time.Now() - dm := make(Map) - - fis, err := godirwalk.ReadDirents(".", nil) - if err != nil { - log.WithError(err).Warn("domain scan failed") - metrics.DomainFailedUpdates.Inc() - continue - } - - dm.ReadGroups(rootDomain, fis) - duration := time.Since(started).Seconds() - - var hash string - if len(update) < 1 { - hash = "<empty>" - } else { - hash = strings.TrimSpace(string(update)) - } - - logConfiguredDomains(dm) - - log.WithFields(log.Fields{ - "count(domains)": len(dm), - "duration": duration, - "hash": hash, - }).Info("Updated all domains") - - if updater != nil { - updater(dm) - } - - // Update prometheus metrics - metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix())) - metrics.DomainsServed.Set(float64(len(dm))) - metrics.DomainsConfigurationUpdateDuration.Set(duration) - metrics.DomainUpdates.Inc() - - time.Sleep(interval) - } -} - -func logConfiguredDomains(dm Map) { - if logrus.GetLevel() != logrus.DebugLevel { - return - } - - for h, d := range dm { - log.WithFields(log.Fields{ - "domain": d, - "host": h, - }).Debug("Configured domain") - } -} diff --git a/internal/source/disk/map_test.go b/internal/source/disk/map_test.go deleted file mode 100644 index 2a5fd828..00000000 --- a/internal/source/disk/map_test.go +++ /dev/null @@ -1,253 +0,0 @@ -package disk - -import ( - "crypto/rand" - "fmt" - "io/ioutil" - "os" - "strings" - "testing" - "time" - - "github.com/karrick/godirwalk" - "github.com/stretchr/testify/require" - - "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers" -) - -func getEntries(t require.TestingT) godirwalk.Dirents { - fis, err := godirwalk.ReadDirents(".", nil) - - require.NoError(t, err) - - return fis -} - -func TestReadProjects(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - dm := make(Map) - dm.ReadGroups("test.io", getEntries(t)) - - var domains []string - for d := range dm { - domains = append(domains, d) - } - - expectedDomains := []string{ - "group.test.io", - "group.internal.test.io", - "test.domain.com", // from config.json - "other.domain.com", - "domain.404.com", - "group.404.test.io", - "group.https-only.test.io", - "test.my-domain.com", - "test2.my-domain.com", - "no.cert.com", - "private.domain.com", - "group.auth.test.io", - "group.acme.test.io", - "withacmechallenge.domain.com", - "capitalgroup.test.io", - "group.404.gitlab-example.com", - "group.redirects.test.io", - "redirects.custom-domain.com", - } - - for _, expected := range domains { - require.Contains(t, domains, expected) - } - - for _, actual := range domains { - require.Contains(t, expectedDomains, actual) - } - - // Check that multiple domains in the same project are recorded faithfully - require.Equal(t, "test.domain.com", dm["test.domain.com"].Name) - require.Equal(t, "other.domain.com", dm["other.domain.com"].Name) - require.Equal(t, "test", dm["other.domain.com"].CertificateCert) - require.Equal(t, "key", dm["other.domain.com"].CertificateKey) - - // check subgroups - domain, ok := dm["group.test.io"] - require.True(t, ok, "missing group.test.io domain") - subgroup, ok := domain.Resolver.(*Group).subgroups["subgroup"] - require.True(t, ok, "missing group.test.io subgroup") - _, ok = subgroup.projects["project"] - require.True(t, ok, "missing project for subgroup in group.test.io domain") -} - -func TestReadProjectsMaxDepth(t *testing.T) { - nGroups := 3 - levels := subgroupScanLimit + 5 - cleanup := buildFakeDomainsDirectory(t, nGroups, levels) - defer cleanup() - - defaultDomain := "test.io" - dm := make(Map) - dm.ReadGroups(defaultDomain, getEntries(t)) - - var domains []string - for d := range dm { - domains = append(domains, d) - } - - var expectedDomains []string - for i := 0; i < nGroups; i++ { - expectedDomains = append(expectedDomains, fmt.Sprintf("group-%d.%s", i, defaultDomain)) - } - - for _, expected := range domains { - require.Contains(t, domains, expected) - } - - for _, actual := range domains { - // we are not checking config.json domains here - if !strings.HasSuffix(actual, defaultDomain) { - continue - } - require.Contains(t, expectedDomains, actual) - } - - // check subgroups - domain, ok := dm["group-0.test.io"] - require.True(t, ok, "missing group-0.test.io domain") - subgroup := domain.Resolver.(*Group) - for i := 0; i < levels; i++ { - subgroup, ok = subgroup.subgroups["sub"] - if i <= subgroupScanLimit { - require.True(t, ok, "missing group-0.test.io subgroup at level %d", i) - _, ok = subgroup.projects["project-0"] - require.True(t, ok, "missing project for subgroup in group-0.test.io domain at level %d", i) - } else { - require.False(t, ok, "subgroup level %d. Maximum allowed nesting level is %d", i, subgroupScanLimit) - break - } - } -} - -// This write must be atomic, otherwise we cannot predict the state of the -// domain watcher goroutine. We cannot use ioutil.WriteFile because that -// has a race condition where the file is empty, which can get picked up -// by the domain watcher. -func writeRandomTimestamp(t *testing.T) { - b := make([]byte, 10) - n, _ := rand.Read(b) - require.True(t, n > 0, "read some random bytes") - - temp, err := ioutil.TempFile(".", "TestWatch") - require.NoError(t, err) - _, err = temp.Write(b) - require.NoError(t, err, "write to tempfile") - require.NoError(t, temp.Close(), "close tempfile") - - require.NoError(t, os.Rename(temp.Name(), updateFile), "rename tempfile") -} - -func TestWatch(t *testing.T) { - cleanup := setUpTests(t) - defer cleanup() - - require.NoError(t, os.RemoveAll(updateFile)) - - update := make(chan Map) - go Watch("gitlab.io", func(dm Map) { - update <- dm - }, time.Microsecond*50) - - defer os.Remove(updateFile) - - domains := recvTimeout(t, update) - require.NotNil(t, domains, "if the domains are fetched on start") - - writeRandomTimestamp(t) - domains = recvTimeout(t, update) - require.NotNil(t, domains, "if the domains are updated after the creation") - - writeRandomTimestamp(t) - domains = recvTimeout(t, update) - require.NotNil(t, domains, "if the domains are updated after the timestamp change") -} - -func recvTimeout(t *testing.T, ch <-chan Map) Map { - timeout := 5 * time.Second - - select { - case dm := <-ch: - return dm - case <-time.After(timeout): - t.Fatalf("timeout after %v waiting for domain update", timeout) - return nil - } -} - -func buildFakeDomainsDirectory(t testing.TB, nGroups, levels int) func() { - testRoot, err := ioutil.TempDir("", "gitlab-pages-test") - require.NoError(t, err) - - for i := 0; i < nGroups; i++ { - parent := fmt.Sprintf("%s/group-%d", testRoot, i) - domain := fmt.Sprintf("%d.example.io", i) - buildFakeProjectsDirectory(t, parent, domain) - for j := 0; j < levels; j++ { - parent = fmt.Sprintf("%s/sub", parent) - domain = fmt.Sprintf("%d.%s", j, domain) - buildFakeProjectsDirectory(t, parent, domain) - } - if testing.Verbose() && i%100 == 0 { - fmt.Print(".") - } - } - - cleanup := testhelpers.ChdirInPath(t, testRoot, &chdirSet) - - return func() { - defer cleanup() - - if testing.Verbose() { - fmt.Printf("cleaning up test directory %s\n", testRoot) - } - - os.RemoveAll(testRoot) - } -} - -func buildFakeProjectsDirectory(t require.TestingT, groupPath, domain string) { - for j := 0; j < 5; j++ { - dir := fmt.Sprintf("%s/project-%d", groupPath, j) - require.NoError(t, os.MkdirAll(dir+"/public", 0755)) - - fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%s","Certificate":"bar","Key":"baz"}]}`, j, domain) - require.NoError(t, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644)) - } -} - -// this is a safeguard against compiler optimizations -// we use this package variable to make sure the benchmarkReadGroups loop -// has side effects outside of the loop. -// Without this the compiler (with the optimizations enabled) may remove the whole loop -var result int - -func benchmarkReadGroups(b *testing.B, groups, levels int) { - cleanup := buildFakeDomainsDirectory(b, groups, levels) - defer cleanup() - - b.ResetTimer() - - domainsCnt := 0 - for i := 0; i < b.N; i++ { - dm := make(Map) - dm.ReadGroups("example.com", getEntries(b)) - domainsCnt = len(dm) - } - result = domainsCnt -} - -func BenchmarkReadGroups(b *testing.B) { - b.Run("10 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 10, 3) }) - b.Run("100 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 100, 3) }) - b.Run("1000 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 1000, 3) }) - b.Run("10000 groups 1 levels", func(b *testing.B) { benchmarkReadGroups(b, 10000, 1) }) -} diff --git a/internal/source/domains.go b/internal/source/domains.go index 8b7e2be9..efa613e1 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -2,41 +2,24 @@ package source import ( "context" - "fmt" - - "gitlab.com/gitlab-org/labkit/log" "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab" ) -type configSource int - -const ( - sourceGitlab configSource = iota - // Disk source is deprecated and support will be removed in 14.3 - // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 - sourceDisk - sourceAuto -) - -// Domains struct represents a map of all domains supported by pages. It is -// currently using two sources during the transition to the new GitLab domains -// source. +// Domains struct wraps the GitLab client to fetch a domain configuration +// TODO: remove/refactor this package https://gitlab.com/gitlab-org/gitlab-pages/-/issues/608 type Domains struct { - configSource configSource - gitlab Source - disk *disk.Disk // legacy disk source + gitlab Source } // NewDomains is a factory method for domains initializing a mutex. It should // not initialize `dm` as we later check the readiness by comparing it with a // nil value. -func NewDomains(source string, cfg *config.GitLab) (*Domains, error) { +func NewDomains(cfg *config.GitLab) (*Domains, error) { domains := &Domains{} - if err := domains.setConfigSource(source, cfg); err != nil { + if err := domains.setConfigSource(cfg); err != nil { return nil, err } @@ -44,43 +27,17 @@ func NewDomains(source string, cfg *config.GitLab) (*Domains, error) { } // setConfigSource and initialize gitlab source -// returns error if -domain-config-source is not valid -// returns error if -domain-config-source=gitlab and init fails -func (d *Domains) setConfigSource(source string, cfg *config.GitLab) error { - switch source { - case "gitlab": - d.configSource = sourceGitlab - return d.setGitLabClient(cfg) - case "auto": - d.configSource = sourceAuto - // enable disk for auto for now - d.disk = disk.New() - return d.setGitLabClient(cfg) - case "disk": - // TODO: disable domains.disk https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 - d.configSource = sourceDisk - d.disk = disk.New() - default: - return fmt.Errorf("invalid option for -domain-config-source: %q", source) - } - - return nil +func (d *Domains) setConfigSource(cfg *config.GitLab) error { + return d.setGitLabClient(cfg) } -// setGitLabClient when domain-config-source is `gitlab` or `auto`, only return error for `gitlab` source func (d *Domains) setGitLabClient(cfg *config.GitLab) error { // We want to notify users about any API issues // Creating a glClient will start polling connectivity in the background // and spam errors in log glClient, err := gitlab.New(cfg) if err != nil { - if d.configSource == sourceGitlab { - return err - } - - log.WithError(err).Warn("failed to initialize GitLab client for `-domain-config-source=auto`") - - return nil + return err } d.gitlab = glClient @@ -93,50 +50,5 @@ func (d *Domains) setGitLabClient(cfg *config.GitLab) error { // for some subset of domains, to test / PoC the new GitLab Domains Source that // we plan to use to replace the disk source. func (d *Domains) GetDomain(ctx context.Context, name string) (*domain.Domain, error) { - return d.source(name).GetDomain(ctx, name) -} - -// Read starts the disk domain source. It is DEPRECATED, because we want to -// remove it entirely when disk source gets removed. -func (d *Domains) Read(rootDomain string) { - // start disk.Read for sourceDisk and sourceAuto - if d.configSource != sourceGitlab { - d.disk.Read(rootDomain) - } -} - -// IsReady checks if the disk domain source managed to traverse entire pages -// filesystem and is ready for use. It is DEPRECATED, because we want to remove -// it entirely when disk source gets removed. -func (d *Domains) IsReady() bool { - switch d.configSource { - case sourceGitlab: - return d.gitlab.IsReady() - case sourceDisk: - return d.disk.IsReady() - case sourceAuto: - // if gitlab is configured and is ready - if d.gitlab != nil && d.gitlab.IsReady() { - return true - } - - return d.disk.IsReady() - default: - return false - } -} - -func (d *Domains) source(domain string) Source { - switch d.configSource { - case sourceDisk: - return d.disk - case sourceGitlab: - return d.gitlab - default: - if d.gitlab != nil && d.gitlab.IsReady() { - return d.gitlab - } - - return d.disk - } + return d.gitlab.GetDomain(ctx, name) } diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index 8618df38..c3f0d2cc 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -9,7 +9,6 @@ import ( "gitlab.com/gitlab-org/gitlab-pages/internal/config" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" - "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" ) func TestNewDomains(t *testing.T) { @@ -21,51 +20,16 @@ func TestNewDomains(t *testing.T) { } tests := []struct { - name string - source string - config config.GitLab - expectedErr string - expectGitlabNil bool - expectDiskNil bool + name string + config config.GitLab + expectedErr string }{ { - name: "no_source_config", - source: "", - expectedErr: "invalid option for -domain-config-source: \"\"", + name: "gitlab_source_success", + config: validCfg, }, { - name: "invalid_source_config", - source: "invalid", - expectedErr: "invalid option for -domain-config-source: \"invalid\"", - }, - { - name: "disk_source", - source: "disk", - expectGitlabNil: true, - expectDiskNil: false, - }, - { - name: "auto_without_api_config", - source: "auto", - expectGitlabNil: true, - expectDiskNil: false, - }, - { - name: "auto_with_api_config", - source: "auto", - config: validCfg, - expectGitlabNil: false, - expectDiskNil: false, - }, - { - name: "gitlab_source_success", - source: "gitlab", - config: validCfg, - expectDiskNil: true, - }, - { - name: "gitlab_source_no_url", - source: "gitlab", + name: "gitlab_source_no_url", config: func() config.GitLab { cfg := validCfg cfg.InternalServer = "" @@ -75,8 +39,7 @@ func TestNewDomains(t *testing.T) { expectedErr: "GitLab API URL or API secret has not been provided", }, { - name: "gitlab_source_no_secret", - source: "gitlab", + name: "gitlab_source_no_secret", config: func() config.GitLab { cfg := validCfg cfg.APISecretKey = []byte{} @@ -89,15 +52,13 @@ func TestNewDomains(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - domains, err := NewDomains(tt.source, &tt.config) + domains, err := NewDomains(&tt.config) if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return } require.NoError(t, err) - - require.Equal(t, tt.expectGitlabNil, domains.gitlab == nil, "mismatch gitlab nil") - require.Equal(t, tt.expectDiskNil, domains.disk == nil, "mismatch disk nil") + require.NotNil(t, domains.gitlab) }) } } @@ -112,24 +73,7 @@ func TestGetDomain(t *testing.T) { Once() defer newSource.AssertExpectations(t) - domains := newTestDomains(t, newSource, sourceGitlab) - - domain, err := domains.GetDomain(context.Background(), testDomain) - require.NoError(t, err) - require.NotNil(t, domain) - }) - - t.Run("when requesting an existing domain for auto source", func(t *testing.T) { - testDomain := "new-source-test.gitlab.io" - - newSource := NewMockSource() - newSource.On("GetDomain", testDomain). - Return(&domain.Domain{Name: testDomain}, nil). - Once() - newSource.On("IsReady").Return(true).Once() - defer newSource.AssertExpectations(t) - - domains := newTestDomains(t, newSource, sourceAuto) + domains := newTestDomains(t, newSource) domain, err := domains.GetDomain(context.Background(), testDomain) require.NoError(t, err) @@ -144,7 +88,7 @@ func TestGetDomain(t *testing.T) { defer newSource.AssertExpectations(t) - domains := newTestDomains(t, newSource, sourceGitlab) + domains := newTestDomains(t, newSource) domain, err := domains.GetDomain(context.Background(), "does-not-exist.test.io") require.NoError(t, err) @@ -152,12 +96,10 @@ func TestGetDomain(t *testing.T) { }) } -func newTestDomains(t *testing.T, gitlabSource *MockSource, config configSource) *Domains { +func newTestDomains(t *testing.T, gitlabSource *MockSource) *Domains { t.Helper() return &Domains{ - configSource: config, - gitlab: gitlabSource, - disk: disk.New(), + gitlab: gitlabSource, } } diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go index b79b434f..d0233028 100644 --- a/internal/source/gitlab/gitlab.go +++ b/internal/source/gitlab/gitlab.go @@ -110,8 +110,3 @@ func sortLookupsByPrefixLengthDesc(lookups []api.LookupPath) { return len(lookups[i].Prefix) > len(lookups[j].Prefix) }) } - -// IsReady returns the value of Gitlab `isReady` which is updated by `Poll`. -func (g *Gitlab) IsReady() bool { - return true -} diff --git a/internal/source/source.go b/internal/source/source.go index cbe949ef..9a247b1b 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -9,5 +9,4 @@ import ( // Source represents an abstract interface of a domains configuration source. type Source interface { GetDomain(context.Context, string) (*domain.Domain, error) - IsReady() bool } diff --git a/test/acceptance/config_test.go b/test/acceptance/config_test.go index 82b7fbb1..07697c48 100644 --- a/test/acceptance/config_test.go +++ b/test/acceptance/config_test.go @@ -5,7 +5,6 @@ import ( "net" "net/http" "testing" - "time" "github.com/stretchr/testify/require" ) @@ -65,29 +64,3 @@ func TestMultipleListenersFromEnvironmentVariables(t *testing.T) { require.Equal(t, http.StatusOK, rsp.StatusCode) } } - -// TODO: remove along chroot https://gitlab.com/gitlab-org/gitlab-pages/-/issues/561 -func TestEnableJailFromEnvironment(t *testing.T) { - out, teardown := runPagesProcess(t, - true, - *pagesBinary, - []ListenSpec{httpListener}, - "", - []string{ - "DAEMON_ENABLE_JAIL=true", - }, - "-domain-config-source", "disk", - ) - t.Cleanup(teardown) - - require.Eventually(t, func() bool { - require.Contains(t, out.String(), "\"daemon-enable-jail\":true") - return true - }, time.Second, 10*time.Millisecond) - - rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "project/") - - require.NoError(t, err) - rsp.Body.Close() - require.Equal(t, http.StatusOK, rsp.StatusCode) -} diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index c1074230..c8852976 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -242,7 +242,6 @@ func RunPagesProcess(t *testing.T, opts ...processOption) *LogCaptureBuffer { "-pages-root", wd, "-internal-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, - "-domain-config-source", "gitlab", ) logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) @@ -352,16 +351,6 @@ func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraAr args = append(args, "-metrics-address", promPort) } - // most of our acceptance tests still work only with disk source - // TODO: remove this with -domain-config-source flag itself along with daemon-enable-jail: - // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 - // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/561 - if !contains(extraArgs, "-domain-config-source") { - args = append(args, - "-domain-config-source", "disk", - ) - } - args = append(args, getPagesDaemonArgs(t)...) args = append(args, extraArgs...) diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go index c42866ee..8c4ab461 100644 --- a/test/acceptance/serving_test.go +++ b/test/acceptance/serving_test.go @@ -4,8 +4,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" - "path" "strings" "testing" "time" @@ -106,51 +104,6 @@ func TestKnownHostReturns200(t *testing.T) { } } -// TODO: remove along with support for disk configuration https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382 -func TestNestedSubgroups(t *testing.T) { - maxNestedSubgroup := 21 - - pagesRoot, err := ioutil.TempDir("", "pages-root") - require.NoError(t, err) - defer os.RemoveAll(pagesRoot) - - makeProjectIndex := func(subGroupPath string) { - projectPath := path.Join(pagesRoot, "nested", subGroupPath, "project", "public") - require.NoError(t, os.MkdirAll(projectPath, 0755)) - - projectIndex := path.Join(projectPath, "index.html") - require.NoError(t, ioutil.WriteFile(projectIndex, []byte("index"), 0644)) - } - makeProjectIndex("") - - paths := []string{""} - for i := 1; i < maxNestedSubgroup*2; i++ { - subGroupPath := fmt.Sprintf("%ssub%d/", paths[i-1], i) - paths = append(paths, subGroupPath) - - makeProjectIndex(subGroupPath) - } - - teardown := RunPagesProcessWithoutGitLabStub(t, *pagesBinary, supportedListeners(), "", "-pages-root", pagesRoot) - defer teardown() - - for nestingLevel, path := range paths { - t.Run(fmt.Sprintf("nested level %d", nestingLevel), func(t *testing.T) { - for _, spec := range supportedListeners() { - rsp, err := GetPageFromListener(t, spec, "nested.gitlab-example.com", path+"project/") - - require.NoError(t, err) - rsp.Body.Close() - if nestingLevel <= maxNestedSubgroup { - require.Equal(t, http.StatusOK, rsp.StatusCode) - } else { - require.Equal(t, http.StatusNotFound, rsp.StatusCode) - } - } - }) - } -} - func TestCustom404(t *testing.T) { RunPagesProcess(t) @@ -425,42 +378,6 @@ func TestDomainsSource(t *testing.T) { apiCalled: true, }, }, - { - name: "disk_source_domain_exists", - args: args{ - configSource: "disk", - // test.domain.com sourced from disk configuration - domain: "test.domain.com", - urlSuffix: "/", - }, - want: want{ - statusCode: http.StatusOK, - content: "main-dir\n", - apiCalled: false, - }, - }, - { - name: "disk_source_domain_does_not_exist", - args: args{ - configSource: "disk", - domain: "non-existent-domain.gitlab.io", - }, - want: want{ - statusCode: http.StatusNotFound, - apiCalled: false, - }, - }, - { - name: "disk_source_domain_should_not_exist_under_hashed_dir", - args: args{ - configSource: "disk", - domain: "hashed.com", - }, - want: want{ - statusCode: http.StatusNotFound, - apiCalled: false, - }, - }, } for _, tt := range tests { @@ -474,7 +391,7 @@ func TestDomainsSource(t *testing.T) { gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) - pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", tt.args.configSource} + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey} teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, []ListenSpec{httpListener}, "", []string{}, pagesArgs...) defer teardown() |