Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-pages.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJaime Martinez <jmartinez@gitlab.com>2021-08-26 03:08:30 +0300
committerJaime Martinez <jmartinez@gitlab.com>2021-08-26 03:08:30 +0300
commit0ef975db7aaa3595ebf56fde4a91351aa0174f11 (patch)
treefffa9bffca13d109d8508a3fa57d3171e0332bf3
parentc27027696e0745654d38fac79ab3adff12a8f72e (diff)
parent33985bb325b4b21c84d47c644c6d4616b9c3e8ed (diff)
Merge branch 'remove/disk-source' into 'master'
refactor: remove support for disk configuration source Closes #103, #158, #68, and #382 See merge request gitlab-org/gitlab-pages!541
-rw-r--r--app.go6
-rw-r--r--app_test.go24
-rw-r--r--go.mod2
-rw-r--r--go.sum4
-rw-r--r--internal/config/config.go21
-rw-r--r--internal/config/flags.go4
-rw-r--r--internal/source/disk/config.go57
-rw-r--r--internal/source/disk/config_test.go65
-rw-r--r--internal/source/disk/custom.go37
-rw-r--r--internal/source/disk/disk.go56
-rw-r--r--internal/source/disk/domain_test.go507
-rw-r--r--internal/source/disk/group.go104
-rw-r--r--internal/source/disk/group_test.go97
-rw-r--r--internal/source/disk/map.go308
-rw-r--r--internal/source/disk/map_test.go253
-rw-r--r--internal/source/domains.go106
-rw-r--r--internal/source/domains_test.go84
-rw-r--r--internal/source/gitlab/gitlab.go5
-rw-r--r--internal/source/source.go1
-rw-r--r--test/acceptance/config_test.go27
-rw-r--r--test/acceptance/helpers_test.go11
-rw-r--r--test/acceptance/serving_test.go85
22 files changed, 53 insertions, 1811 deletions
diff --git a/app.go b/app.go
index 43f50d9c..18fe20ae 100644
--- a/app.go
+++ b/app.go
@@ -59,7 +59,7 @@ type theApp struct {
}
func (a *theApp) isReady() bool {
- return a.domains.IsReady()
+ return true
}
func (a *theApp) ServeTLS(ch *cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
@@ -413,8 +413,6 @@ func (a *theApp) Run() {
a.listenMetricsFD(&wg, a.config.ListenMetrics)
}
- a.domains.Read(a.config.General.Domain)
-
wg.Wait()
}
@@ -495,7 +493,7 @@ func (a *theApp) listenMetricsFD(wg *sync.WaitGroup, fd uintptr) {
}
func runApp(config *cfg.Config) {
- domains, err := source.NewDomains(config.General.DomainConfigurationSource, &config.GitLab)
+ domains, err := source.NewDomains(&config.GitLab)
if err != nil {
log.WithError(err).Fatal("could not create domains config source")
}
diff --git a/app_test.go b/app_test.go
index 48a6013b..483ffb39 100644
--- a/app_test.go
+++ b/app_test.go
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
+ "time"
"github.com/stretchr/testify/require"
@@ -79,20 +80,29 @@ func TestHealthCheckMiddleware(t *testing.T) {
{
name: "Healthcheck request",
path: "/-/healthcheck",
- status: http.StatusServiceUnavailable,
- body: "not yet ready\n",
+ status: http.StatusOK,
+ body: "success\n",
},
}
- cfg, err := config.LoadConfig()
- require.NoError(t, err)
- cfg.General.StatusPath = "/-/healthcheck"
+ validCfg := config.GitLab{
+ InternalServer: "server",
+ APISecretKey: []byte("secret"),
+ ClientHTTPTimeout: time.Second,
+ JWTTokenExpiration: time.Second,
+ }
- domains, err := source.NewDomains("auto", &cfg.GitLab)
+ domains, err := source.NewDomains(&validCfg)
require.NoError(t, err)
+ cfg := config.Config{
+ General: config.General{
+ StatusPath: "/-/healthcheck",
+ },
+ }
+
app := theApp{
- config: cfg,
+ config: &cfg,
domains: domains,
}
diff --git a/go.mod b/go.mod
index 91131c18..5f17d253 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,6 @@ module gitlab.com/gitlab-org/gitlab-pages
go 1.16
require (
- github.com/andybalholm/brotli v1.0.3
github.com/golang-jwt/jwt/v4 v4.0.0
github.com/golang/mock v1.3.1
github.com/gorilla/handlers v1.4.2
@@ -12,7 +11,6 @@ require (
github.com/gorilla/sessions v1.2.0
github.com/hashicorp/go-multierror v1.1.1
github.com/karlseguin/ccache/v2 v2.0.6
- github.com/karrick/godirwalk v1.10.12
github.com/namsral/flag v1.7.4-pre
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/pires/go-proxyproto v0.2.0
diff --git a/go.sum b/go.sum
index aa77da2f..f004e86f 100644
--- a/go.sum
+++ b/go.sum
@@ -27,8 +27,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM=
-github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -164,8 +162,6 @@ github.com/karlseguin/ccache/v2 v2.0.6 h1:jFCLz4bF4EPfuCcvESAgYNClkEb31LV3WzyOwL
github.com/karlseguin/ccache/v2 v2.0.6/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
-github.com/karrick/godirwalk v1.10.12 h1:BqUm+LuJcXjGv1d2mj3gBiQyrQ57a0rYoAmhvJQ7RDU=
-github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk=
github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
diff --git a/internal/config/config.go b/internal/config/config.go
index 64dc4666..c57eb15d 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -45,15 +45,14 @@ type Config struct {
// General groups settings that are general to GitLab Pages and can not
// be categorized under other head.
type General struct {
- Domain string
- DomainConfigurationSource string
- MaxConns int
- MetricsAddress string
- RedirectHTTP bool
- RootCertificate []byte
- RootDir string
- RootKey []byte
- StatusPath string
+ Domain string
+ MaxConns int
+ MetricsAddress string
+ RedirectHTTP bool
+ RootCertificate []byte
+ RootDir string
+ RootKey []byte
+ StatusPath string
DisableCrossOriginRequests bool
InsecureCiphers bool
@@ -187,7 +186,6 @@ func loadConfig() (*Config, error) {
config := &Config{
General: General{
Domain: strings.ToLower(*pagesDomain),
- DomainConfigurationSource: *domainConfigSource,
MaxConns: *maxConns,
MetricsAddress: *metricsAddress,
RedirectHTTP: *redirectHTTP,
@@ -227,7 +225,7 @@ func loadConfig() (*Config, error) {
UID: *daemonUID,
GID: *daemonGID,
InplaceChroot: *daemonInplaceChroot,
- EnableJail: *daemonEnableJail || *domainConfigSource == "disk",
+ EnableJail: *daemonEnableJail,
},
Log: Log{
Format: *logFormat,
@@ -325,7 +323,6 @@ func LogConfig(config *Config) {
"gitlab-server": config.GitLab.PublicServer,
"internal-gitlab-server": config.GitLab.InternalServer,
"api-secret-key": *gitLabAPISecretKey,
- "domain-config-source": config.General.DomainConfigurationSource,
"enable-disk": config.GitLab.EnableDisk,
"auth-redirect-uri": config.Authentication.RedirectURI,
"auth-scope": config.Authentication.Scope,
diff --git a/internal/config/flags.go b/internal/config/flags.go
index 28a318c4..63c52631 100644
--- a/internal/config/flags.go
+++ b/internal/config/flags.go
@@ -41,8 +41,8 @@ var (
gitlabRetrievalInterval = flag.Duration("gitlab-retrieval-interval", time.Second, "The interval to wait before retrying to resolve a domain's configuration via the GitLab API")
gitlabRetrievalRetries = flag.Int("gitlab-retrieval-retries", 3, "The maximum number of times to retry to resolve a domain's configuration via the API")
- domainConfigSource = flag.String("domain-config-source", "gitlab", "Domain configuration source 'disk', 'auto' or 'gitlab' (default: 'gitlab'). DEPRECATED: gitlab-pages will use the API-based configuration starting from 14.3 see https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/5993")
- enableDisk = flag.Bool("enable-disk", true, "Enable disk access, shall be disabled in environments where shared disk storage isn't available")
+ _ = flag.String("domain-config-source", "gitlab", "DEPRECATED and has not affect, see https://gitlab.com/gitlab-org/gitlab-pages/-/merge_requests/541")
+ enableDisk = flag.Bool("enable-disk", true, "Enable disk access, shall be disabled in environments where shared disk storage isn't available")
clientID = flag.String("auth-client-id", "", "GitLab application Client ID")
clientSecret = flag.String("auth-client-secret", "", "GitLab application Client Secret")
diff --git a/internal/source/disk/config.go b/internal/source/disk/config.go
deleted file mode 100644
index d2e6c123..00000000
--- a/internal/source/disk/config.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package disk
-
-import (
- "encoding/json"
- "os"
- "path/filepath"
- "strings"
-)
-
-// DomainConfig represents a custom domain config
-type domainConfig struct {
- Domain string
- Certificate string
- Key string
- HTTPSOnly bool `json:"https_only"`
- ID uint64 `json:"id"`
- AccessControl bool `json:"access_control"`
-}
-
-// MultiDomainConfig represents a group of custom domain configs
-type multiDomainConfig struct {
- Domains []domainConfig
- HTTPSOnly bool `json:"https_only"`
- ID uint64 `json:"id"`
- AccessControl bool `json:"access_control"`
-}
-
-// ProjectConfig is a project-level configuration
-type projectConfig struct {
- NamespaceProject bool
- HTTPSOnly bool
- AccessControl bool
- ID uint64
-}
-
-// Valid validates a custom domain config for a root domain
-func (c *domainConfig) Valid(rootDomain string) bool {
- if c.Domain == "" {
- return false
- }
-
- // TODO: better sanitize domain
- domain := strings.ToLower(c.Domain)
- rootDomain = "." + rootDomain
- return !strings.HasSuffix(domain, rootDomain)
-}
-
-// Read reads a multi domain config and decodes it from a `config.json`
-func (c *multiDomainConfig) Read(group, project string) error {
- configFile, err := os.Open(filepath.Join(group, project, "config.json"))
- if err != nil {
- return err
- }
- defer configFile.Close()
-
- return json.NewDecoder(configFile).Decode(c)
-}
diff --git a/internal/source/disk/config_test.go b/internal/source/disk/config_test.go
deleted file mode 100644
index 1bb2364a..00000000
--- a/internal/source/disk/config_test.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package disk
-
-import (
- "io/ioutil"
- "os"
- "path/filepath"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-const configFile = "test-group/test-project/config.json"
-const invalidConfig = `{"Domains":{}}`
-const validConfig = `{"Domains":[{"Domain":"test"}]}`
-
-func TestDomainConfigValidness(t *testing.T) {
- d := domainConfig{}
- require.False(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test"}
- require.True(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test"}
- require.True(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test.gitlab.io"}
- require.False(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test.test.gitlab.io"}
- require.False(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test.testgitlab.io"}
- require.True(t, d.Valid("gitlab.io"))
-
- d = domainConfig{Domain: "test.GitLab.Io"}
- require.False(t, d.Valid("gitlab.io"))
-}
-
-func TestDomainConfigRead(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- d := multiDomainConfig{}
- err := d.Read("test-group", "test-project")
- require.Error(t, err)
-
- os.MkdirAll(filepath.Dir(configFile), 0700)
- defer os.RemoveAll("test-group")
-
- d = multiDomainConfig{}
- err = d.Read("test-group", "test-project")
- require.Error(t, err)
-
- err = ioutil.WriteFile(configFile, []byte(invalidConfig), 0600)
- require.NoError(t, err)
- d = multiDomainConfig{}
- err = d.Read("test-group", "test-project")
- require.Error(t, err)
-
- err = ioutil.WriteFile(configFile, []byte(validConfig), 0600)
- require.NoError(t, err)
- d = multiDomainConfig{}
- err = d.Read("test-group", "test-project")
- require.NoError(t, err)
-}
diff --git a/internal/source/disk/custom.go b/internal/source/disk/custom.go
deleted file mode 100644
index 037abfee..00000000
--- a/internal/source/disk/custom.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package disk
-
-import (
- "net/http"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/internal/serving"
- "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/local"
-)
-
-type customProjectResolver struct {
- config *domainConfig
-
- path string
-}
-
-func (p *customProjectResolver) Resolve(r *http.Request) (*serving.Request, error) {
- if p.config == nil {
- return nil, domain.ErrDomainDoesNotExist
- }
-
- lookupPath := &serving.LookupPath{
- ServingType: "file",
- Prefix: "/",
- Path: p.path,
- IsNamespaceProject: false,
- IsHTTPSOnly: p.config.HTTPSOnly,
- HasAccessControl: p.config.AccessControl,
- ProjectID: p.config.ID,
- }
-
- return &serving.Request{
- Serving: local.Instance(),
- LookupPath: lookupPath,
- SubPath: r.URL.Path,
- }, nil
-}
diff --git a/internal/source/disk/disk.go b/internal/source/disk/disk.go
deleted file mode 100644
index 3596af29..00000000
--- a/internal/source/disk/disk.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package disk
-
-import (
- "context"
- "strings"
- "sync"
- "time"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/domain"
-)
-
-// Disk struct represents a map of all domains supported by pages that are
-// stored on a disk with corresponding `config.json`.
-type Disk struct {
- dm Map
- lock *sync.RWMutex
-}
-
-// New is a factory method for the Disk source. It is initializing a mutex. It
-// should not initialize `dm` as we later check the readiness by comparing it
-// with a nil value.
-func New() *Disk {
- return &Disk{
- lock: &sync.RWMutex{},
- }
-}
-
-// GetDomain returns a domain from the domains map if it exists
-func (d *Disk) GetDomain(ctx context.Context, host string) (*domain.Domain, error) {
- host = strings.ToLower(host)
-
- d.lock.RLock()
- defer d.lock.RUnlock()
-
- return d.dm[host], nil
-}
-
-// IsReady checks if the domains source is ready for work. The disk source is
-// ready after traversing entire filesystem and reading all domains'
-// configuration files.
-func (d *Disk) IsReady() bool {
- return d.dm != nil
-}
-
-// Read starts the domain source, in this case it is reading domains from
-// groups on disk concurrently.
-func (d *Disk) Read(rootDomain string) {
- go Watch(rootDomain, d.updateDomains, time.Second)
-}
-
-func (d *Disk) updateDomains(dm Map) {
- d.lock.Lock()
- defer d.lock.Unlock()
-
- d.dm = dm
-}
diff --git a/internal/source/disk/domain_test.go b/internal/source/disk/domain_test.go
deleted file mode 100644
index abffddb4..00000000
--- a/internal/source/disk/domain_test.go
+++ /dev/null
@@ -1,507 +0,0 @@
-package disk
-
-import (
- "compress/gzip"
- "io/ioutil"
- "net/http"
- "net/http/httptest"
- "net/url"
- "strconv"
- "testing"
- "time"
-
- "github.com/andybalholm/brotli"
- "github.com/stretchr/testify/require"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/internal/fixture"
- "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
-)
-
-func serveFileOrNotFound(domain *domain.Domain) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- if !domain.ServeFileHTTP(w, r) {
- domain.ServeNotFoundHTTP(w, r)
- }
- }
-}
-
-func testGroupServeHTTPHost(t *testing.T, host string) {
- testGroup := &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: map[string]*projectConfig{
- "group.test.io": &projectConfig{},
- "group.gitlab-example.com": &projectConfig{},
- "project": &projectConfig{},
- "project2": &projectConfig{},
- },
- },
- }
-
- makeURL := func(path string) string {
- return "http://" + host + path
- }
-
- serve := serveFileOrNotFound(testGroup)
-
- require.HTTPBodyContains(t, serve, "GET", makeURL("/"), nil, "main-dir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/index"), nil, "main-dir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/index.html"), nil, "main-dir")
- testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project"), nil, "//"+host+"/project/")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project/"), nil, "project-subdir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index"), nil, "project-subdir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index/"), nil, "project-subdir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project/index.html"), nil, "project-subdir")
- testhelpers.AssertRedirectTo(t, serve, "GET", makeURL("/project/subdir"), nil, "//"+host+"/project/subdir/")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project/subdir/"), nil, "project-subsubdir")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/"), nil, "project2-main")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index"), nil, "project2-main")
- require.HTTPBodyContains(t, serve, "GET", makeURL("/project2/index.html"), nil, "project2-main")
- require.HTTPError(t, serve, "GET", makeURL("/private.project/"), nil)
- require.HTTPError(t, serve, "GET", makeURL("//about.gitlab.com/%2e%2e"), nil)
- require.HTTPError(t, serve, "GET", makeURL("/symlink"), nil)
- require.HTTPError(t, serve, "GET", makeURL("/symlink/index.html"), nil)
- require.HTTPError(t, serve, "GET", makeURL("/symlink/subdir/"), nil)
- require.HTTPError(t, serve, "GET", makeURL("/project/fifo"), nil)
- require.HTTPError(t, serve, "GET", makeURL("/not-existing-file"), nil)
- require.HTTPRedirect(t, serve, "GET", makeURL("/project//about.gitlab.com/%2e%2e"), nil)
-}
-
-func TestGroupServeHTTP(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- t.Run("group.test.io", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io") })
- t.Run("group.test.io:8080", func(t *testing.T) { testGroupServeHTTPHost(t, "group.test.io:8080") })
-}
-
-func TestDomainServeHTTP(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testDomain := &domain.Domain{
- Name: "test.domain.com",
- Resolver: &customProjectResolver{
- path: "group/project2/public",
- config: &domainConfig{},
- },
- }
-
- require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/", nil, "project2-main")
- require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/index.html", nil, "project2-main")
- require.HTTPRedirect(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil)
- require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir", nil,
- `<a href="/subdir/">Found</a>`)
- require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/", nil, "project2-subdir")
- require.HTTPBodyContains(t, serveFileOrNotFound(testDomain), "GET", "/subdir/index.html", nil, "project2-subdir")
- require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "//about.gitlab.com/%2e%2e", nil)
- require.HTTPError(t, serveFileOrNotFound(testDomain), "GET", "/not-existing-file", nil)
-}
-
-func TestIsHTTPSOnly(t *testing.T) {
- tests := []struct {
- name string
- domain *domain.Domain
- url string
- expected bool
- }{
- {
- name: "Default group domain with HTTPS-only enabled",
- domain: &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}},
- },
- },
- url: "http://test-domain",
- expected: true,
- },
- {
- name: "Default group domain with HTTPS-only disabled",
- domain: &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: projects{"test-domain": &projectConfig{HTTPSOnly: false}},
- },
- },
- url: "http://test-domain",
- expected: false,
- },
- {
- name: "Case-insensitive default group domain with HTTPS-only enabled",
- domain: &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: projects{"test-domain": &projectConfig{HTTPSOnly: true}},
- },
- },
- url: "http://Test-domain",
- expected: true,
- },
- {
- name: "Other group domain with HTTPS-only enabled",
- domain: &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: projects{"project": &projectConfig{HTTPSOnly: true}},
- },
- },
- url: "http://test-domain/project",
- expected: true,
- },
- {
- name: "Other group domain with HTTPS-only disabled",
- domain: &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: projects{"project": &projectConfig{HTTPSOnly: false}},
- },
- },
- url: "http://test-domain/project",
- expected: false,
- },
- {
- name: "Unknown project",
- domain: &domain.Domain{
- Resolver: &Group{},
- },
- url: "http://test-domain/project",
- expected: false,
- },
- }
-
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- req, _ := http.NewRequest(http.MethodGet, test.url, nil)
- require.Equal(t, test.expected, test.domain.IsHTTPSOnly(req))
- })
- }
-}
-
-func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) {
- w := httptest.NewRecorder()
- req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)
- require.NoError(t, err)
- if acceptEncoding != "" {
- req.Header.Add("Accept-Encoding", acceptEncoding)
- }
- handler(w, req)
-
- if expectCompressed {
- contentLength := w.Header().Get("Content-Length")
- require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length")
-
- contentEncoding := w.Header().Get("Content-Encoding")
- require.Equal(t, "gzip", contentEncoding, "Content-Encoding")
-
- reader, err := gzip.NewReader(w.Body)
- require.NoError(t, err)
- defer reader.Close()
-
- bytes, err := ioutil.ReadAll(reader)
- require.NoError(t, err)
- require.Contains(t, string(bytes), str)
- } else {
- require.Contains(t, w.Body.String(), str)
- }
-
- require.Equal(t, contentType, w.Header().Get("Content-Type"))
-}
-
-func TestGroupServeHTTPGzip(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testGroup := &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: map[string]*projectConfig{
- "group.test.io": &projectConfig{},
- "group.gitlab-example.com": &projectConfig{},
- "project": &projectConfig{},
- "project2": &projectConfig{},
- },
- },
- }
-
- testSet := []struct {
- mode string // HTTP mode
- url string // Test URL
- acceptEncoding string // Accept encoding header
- body interface{} // Expected body at above URL
- contentType string // Expected content-type
- expectCompressed bool // Expect the response to be gzipped?
- }{
- // No gzip encoding requested
- {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "gzip; q=0", "main-dir", "text/html; charset=utf-8", false},
- // gzip encoding requested,
- {"GET", "/index.html", "identity, gzip", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip; q=1", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip; q=0.9", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip; q=1, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=0.9, gzip; q=1", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=0, gzip; q=1", "main-dir", "text/html; charset=utf-8", true},
- // fallback to gzip because .br is missing
- {"GET", "/index2.html", "*", "main-dir", "text/html; charset=utf-8", true},
- // gzip encoding requested, but url does not have compressed content on disk
- {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "identity, gzip", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip; q=1", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip; q=0.9", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip, deflate", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip; q=1, deflate", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "gzip; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false},
- // malformed headers
- {"GET", "/index.html", ";; gzip", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "gzip; quality=1", "main-dir", "text/html; charset=utf-8", false},
- // Symlinked .gz files are not supported
- {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false},
- // Unknown file-extension, with text content
- {"GET", "/text.unknown", "gzip", "hello", "text/plain; charset=utf-8", true},
- {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false},
- // Unknown file-extension, with PNG content
- {"GET", "/image.unknown", "gzip", "GIF89a", "image/gif", true},
- {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false},
- }
-
- for _, tt := range testSet {
- t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) {
- URL := "http://group.test.io" + tt.url
- testHTTPGzip(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed)
- })
- }
-}
-
-func testHTTPBrotli(t *testing.T, handler http.HandlerFunc, mode, url string, values url.Values, acceptEncoding string, str interface{}, contentType string, expectCompressed bool) {
- w := httptest.NewRecorder()
- req, err := http.NewRequest(mode, url+"?"+values.Encode(), nil)
- require.NoError(t, err)
- if acceptEncoding != "" {
- req.Header.Add("Accept-Encoding", acceptEncoding)
- }
- handler(w, req)
-
- if expectCompressed {
- contentLength := w.Header().Get("Content-Length")
- require.Equal(t, strconv.Itoa(w.Body.Len()), contentLength, "Content-Length")
-
- contentEncoding := w.Header().Get("Content-Encoding")
- require.Equal(t, "br", contentEncoding, "Content-Encoding")
-
- reader := brotli.NewReader(w.Body)
- bytes, err := ioutil.ReadAll(reader)
- require.NoError(t, err)
- require.Contains(t, string(bytes), str)
- } else {
- require.Contains(t, w.Body.String(), str)
- }
-
- require.Equal(t, contentType, w.Header().Get("Content-Type"))
-}
-
-func TestGroupServeHTTPBrotli(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testGroup := &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: map[string]*projectConfig{
- "group.test.io": &projectConfig{},
- "group.gitlab-example.com": &projectConfig{},
- "project": &projectConfig{},
- "project2": &projectConfig{},
- },
- },
- }
-
- testSet := []struct {
- mode string // HTTP mode
- url string // Test URL
- acceptEncoding string // Accept encoding header
- body interface{} // Expected body at above URL
- contentType string // Expected content-type
- expectCompressed bool // Expect the response to be br compressed?
- }{
- // No br encoding requested
- {"GET", "/index.html", "", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "identity", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "br; q=0", "main-dir", "text/html; charset=utf-8", false},
- // br encoding requested,
- {"GET", "/index.html", "*", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "identity, br", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=1", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=0.9", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=1, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "br; q=0.9, deflate", "main-dir", "text/html; charset=utf-8", true},
- {"GET", "/index.html", "gzip; q=0.5, br; q=1", "main-dir", "text/html; charset=utf-8", true},
- // br encoding requested, but url does not have compressed content on disk
- {"GET", "/project2/index.html", "*", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "identity, br", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br; q=1", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br; q=0.9", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br, deflate", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br; q=1, deflate", "project2-main", "text/html; charset=utf-8", false},
- {"GET", "/project2/index.html", "br; q=0.9, deflate", "project2-main", "text/html; charset=utf-8", false},
- // malformed headers
- {"GET", "/index.html", ";; br", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "middle-out", "main-dir", "text/html; charset=utf-8", false},
- {"GET", "/index.html", "br; quality=1", "main-dir", "text/html; charset=utf-8", false},
- // Symlinked .br files are not supported
- {"GET", "/gz-symlink", "*", "data", "text/plain; charset=utf-8", false},
- // Unknown file-extension, with text content
- {"GET", "/text.unknown", "*", "hello", "text/plain; charset=utf-8", true},
- {"GET", "/text-nogzip.unknown", "*", "hello", "text/plain; charset=utf-8", false},
- // Unknown file-extension, with PNG content
- {"GET", "/image.unknown", "*", "GIF89a", "image/gif", true},
- {"GET", "/image-nogzip.unknown", "*", "GIF89a", "image/gif", false},
- }
-
- for _, tt := range testSet {
- t.Run(tt.url+" acceptEncoding: "+tt.acceptEncoding, func(t *testing.T) {
- URL := "http://group.test.io" + tt.url
- testHTTPBrotli(t, serveFileOrNotFound(testGroup), tt.mode, URL, nil, tt.acceptEncoding, tt.body, tt.contentType, tt.expectCompressed)
- })
- }
-}
-
-func TestGroup404ServeHTTP(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testGroup := &domain.Domain{
- Resolver: &Group{
- name: "group.404",
- projects: map[string]*projectConfig{
- "domain.404": &projectConfig{},
- "group.404.test.io": &projectConfig{},
- "project.404": &projectConfig{},
- "project.404.symlink": &projectConfig{},
- "project.no.404": &projectConfig{},
- },
- },
- }
-
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/not/existing-file", nil, "Custom 404 project page")
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404/", nil, "Custom 404 project page")
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not/existing-file", nil, "Custom 404 group page")
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom 404 group page")
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/", nil, "Custom 404 group page")
- require.HTTPBodyNotContains(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.404.symlink/not/existing-file", nil, "Custom 404 project page")
-
- // Ensure the namespace project's custom 404.html is not used by projects
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testGroup), "GET", "http://group.404.test.io/project.no.404/not/existing-file", nil, "The page you're looking for could not be found.")
-}
-
-func TestDomain404ServeHTTP(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testDomain := &domain.Domain{
- Resolver: &customProjectResolver{
- path: "group.404/domain.404/public/",
- config: &domainConfig{Domain: "domain.404.com"},
- },
- }
-
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/not-existing-file", nil, "Custom domain.404 page")
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.404.test.io/", nil, "Custom domain.404 page")
-}
-
-func TestPredefined404ServeHTTP(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testDomain := domain.New("", "", "", &customProjectResolver{})
-
- testhelpers.AssertHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found")
-}
-
-func TestGroupCertificate(t *testing.T) {
- testGroup := &domain.Domain{}
-
- tls, err := testGroup.EnsureCertificate()
- require.Nil(t, tls)
- require.Error(t, err)
-}
-
-func TestDomainNoCertificate(t *testing.T) {
- testDomain := &domain.Domain{
- Resolver: &customProjectResolver{
- path: "group/project2/public",
- config: &domainConfig{Domain: "test.domain.com"},
- },
- }
-
- tls, err := testDomain.EnsureCertificate()
- require.Nil(t, tls)
- require.Error(t, err)
-
- _, err2 := testDomain.EnsureCertificate()
- require.Error(t, err)
- require.Equal(t, err, err2)
-}
-
-func TestDomainCertificate(t *testing.T) {
- testDomain := &domain.Domain{
- Name: "test.domain.com",
- CertificateCert: fixture.Certificate,
- CertificateKey: fixture.Key,
- Resolver: &customProjectResolver{
- path: "group/project2/public",
- },
- }
-
- tls, err := testDomain.EnsureCertificate()
- require.NotNil(t, tls)
- require.NoError(t, err)
-}
-
-func TestCacheControlHeaders(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- testGroup := &domain.Domain{
- Resolver: &Group{
- name: "group",
- projects: map[string]*projectConfig{
- "group.test.io": &projectConfig{},
- },
- },
- }
- w := httptest.NewRecorder()
- req, err := http.NewRequest("GET", "http://group.test.io/", nil)
- require.NoError(t, err)
-
- now := time.Now()
- serveFileOrNotFound(testGroup)(w, req)
-
- require.Equal(t, http.StatusOK, w.Code)
- require.Equal(t, "max-age=600", w.Header().Get("Cache-Control"))
-
- expires := w.Header().Get("Expires")
- require.NotEmpty(t, expires)
-
- expiresTime, err := time.Parse(time.RFC1123, expires)
- require.NoError(t, err)
-
- require.WithinDuration(t, now.UTC().Add(10*time.Minute), expiresTime.UTC(), time.Minute)
-}
-
-var chdirSet = false
-
-func setUpTests(t *testing.T) func() {
- t.Helper()
- return testhelpers.ChdirInPath(t, "../../../shared/pages", &chdirSet)
-}
diff --git a/internal/source/disk/group.go b/internal/source/disk/group.go
deleted file mode 100644
index 9499df46..00000000
--- a/internal/source/disk/group.go
+++ /dev/null
@@ -1,104 +0,0 @@
-package disk
-
-import (
- "net/http"
- "path"
- "path/filepath"
- "strings"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/internal/host"
- "gitlab.com/gitlab-org/gitlab-pages/internal/serving"
- "gitlab.com/gitlab-org/gitlab-pages/internal/serving/disk/local"
-)
-
-const (
- subgroupScanLimit int = 21
- // maxProjectDepth is set to the maximum nested project depth in gitlab (21) plus 3.
- // One for the project, one for the first empty element of the split (URL.Path starts with /),
- // and one for the real file path
- maxProjectDepth int = subgroupScanLimit + 3
-)
-
-// Group represents a GitLab group with project configs and subgroups
-type Group struct {
- name string
-
- // nested groups
- subgroups subgroups
-
- // group domains:
- projects projects
-}
-
-type projects map[string]*projectConfig
-type subgroups map[string]*Group
-
-func (g *Group) digProjectWithSubpath(parentPath string, keys []string) (*projectConfig, string, string) {
- if len(keys) >= 1 {
- head := keys[0]
- tail := keys[1:]
- currentPath := path.Join(parentPath, head)
- search := strings.ToLower(head)
-
- if project := g.projects[search]; project != nil {
- return project, currentPath, path.Join(tail...)
- }
-
- if subgroup := g.subgroups[search]; subgroup != nil {
- return subgroup.digProjectWithSubpath(currentPath, tail)
- }
- }
-
- return nil, "", ""
-}
-
-// Look up a project inside the domain based on the host and path. Returns the
-// project and its name (if applicable)
-func (g *Group) getProjectConfigWithSubpath(r *http.Request) (*projectConfig, string, string, string) {
- // Check for a project specified in the URL: http://group.gitlab.io/projectA
- // If present, these projects shadow the group domain.
- split := strings.SplitN(r.URL.Path, "/", maxProjectDepth)
- if len(split) >= 2 {
- projectConfig, projectPath, urlPath := g.digProjectWithSubpath("", split[1:])
- if projectConfig != nil {
- return projectConfig, "/" + projectPath, projectPath, urlPath
- }
- }
-
- // Since the URL doesn't specify a project (e.g. http://mydomain.gitlab.io),
- // return the group project if it exists.
- if host := host.FromRequest(r); host != "" {
- if groupProject := g.projects[host]; groupProject != nil {
- return groupProject, "/", host, strings.Join(split[1:], "/")
- }
- }
-
- return nil, "", "", ""
-}
-
-// Resolve tries to find project and its config recursively for a given request
-// to a group domain
-func (g *Group) Resolve(r *http.Request) (*serving.Request, error) {
- projectConfig, prefix, projectPath, subPath := g.getProjectConfigWithSubpath(r)
-
- if projectConfig == nil {
- return nil, domain.ErrDomainDoesNotExist
- }
-
- lookupPath := &serving.LookupPath{
- ServingType: "file",
- Prefix: prefix,
- Path: filepath.Join(g.name, projectPath, "public") + "/",
- IsNamespaceProject: projectConfig.NamespaceProject,
- IsHTTPSOnly: projectConfig.HTTPSOnly,
- HasAccessControl: projectConfig.AccessControl,
- ProjectID: projectConfig.ID,
- }
-
- return &serving.Request{
- Serving: local.Instance(),
- LookupPath: lookupPath,
- SubPath: subPath,
- }, nil
-}
diff --git a/internal/source/disk/group_test.go b/internal/source/disk/group_test.go
deleted file mode 100644
index d0fb49bd..00000000
--- a/internal/source/disk/group_test.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package disk
-
-import (
- "strings"
- "testing"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestGroupDig(t *testing.T) {
- matchingProject := &projectConfig{ID: 1}
-
- tests := []struct {
- name string
- g Group
- path string
- expectedProject *projectConfig
- expectedProjectPath string
- expectedPath string
- }{
- {
- name: "empty group",
- path: "projectb/demo/features.html",
- g: Group{},
- },
- {
- name: "group with project",
- path: "projectb/demo/features.html",
- g: Group{
- projects: projects{"projectb": matchingProject},
- },
- expectedProject: matchingProject,
- expectedProjectPath: "projectb",
- expectedPath: "demo/features.html",
- },
- {
- name: "group with project and no path in URL",
- path: "projectb",
- g: Group{
- projects: projects{"projectb": matchingProject},
- },
- expectedProject: matchingProject,
- expectedProjectPath: "projectb",
- },
- {
- name: "group with subgroup and project",
- path: "projectb/demo/features.html",
- g: Group{
- projects: projects{"projectb": matchingProject},
- subgroups: subgroups{
- "sub1": &Group{
- projects: projects{"another": &projectConfig{}},
- },
- },
- },
- expectedProject: matchingProject,
- expectedProjectPath: "projectb",
- expectedPath: "demo/features.html",
- },
- {
- name: "group with project inside a subgroup",
- path: "sub1/projectb/demo/features.html",
- g: Group{
- subgroups: subgroups{
- "sub1": &Group{
- projects: projects{"projectb": matchingProject},
- },
- },
- projects: projects{"another": &projectConfig{}},
- },
- expectedProject: matchingProject,
- expectedProjectPath: "sub1/projectb",
- expectedPath: "demo/features.html",
- },
- {
- name: "group with matching subgroup but no project",
- path: "sub1/projectb/demo/features.html",
- g: Group{
- subgroups: subgroups{
- "sub1": &Group{
- projects: projects{"another": &projectConfig{}},
- },
- },
- },
- },
- }
-
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- project, projectPath, urlPath := test.g.digProjectWithSubpath("", strings.Split(test.path, "/"))
-
- require.Equal(t, test.expectedProject, project)
- require.Equal(t, test.expectedProjectPath, projectPath)
- require.Equal(t, test.expectedPath, urlPath)
- })
- }
-}
diff --git a/internal/source/disk/map.go b/internal/source/disk/map.go
deleted file mode 100644
index 05ab4c30..00000000
--- a/internal/source/disk/map.go
+++ /dev/null
@@ -1,308 +0,0 @@
-package disk
-
-import (
- "bytes"
- "io/ioutil"
- "os"
- "path/filepath"
- "strings"
- "sync"
- "time"
-
- "github.com/karrick/godirwalk"
- "github.com/sirupsen/logrus"
- "gitlab.com/gitlab-org/labkit/log"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/metrics"
-)
-
-// preventive measure to skip `@hashed` dir for new zip deployments when sourcing config from disk
-// https://gitlab.com/gitlab-org/gitlab-pages/-/issues/468
-const skipHashedDir = "@hashed"
-
-// Map maps domain names to Domain instances.
-type Map map[string]*domain.Domain
-
-type domainsUpdater func(Map)
-
-func (dm Map) updateDomainMap(domainName string, domain *domain.Domain) {
- if _, ok := dm[domainName]; ok {
- log.WithFields(log.Fields{
- "domain_name": domainName,
- }).Error("Duplicate domain")
- }
-
- dm[domainName] = domain
-}
-
-func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) {
- newDomain := domain.New(
- strings.ToLower(config.Domain),
- config.Certificate,
- config.Key,
- &customProjectResolver{
- config: config,
- path: filepath.Join(groupName, projectName, "public"),
- },
- )
-
- dm.updateDomainMap(newDomain.Name, newDomain)
-}
-
-func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, httpsOnly bool, accessControl bool, id uint64) {
- domainName := strings.ToLower(groupName + "." + rootDomain)
- groupDomain := dm[domainName]
-
- if groupDomain == nil {
- groupResolver := &Group{
- name: groupName,
- projects: make(projects),
- subgroups: make(subgroups),
- }
-
- groupDomain = domain.New(domainName, "", "", groupResolver)
- }
-
- split := strings.SplitN(strings.ToLower(projectPath), "/", maxProjectDepth)
- projectName := split[len(split)-1]
- g := groupDomain.Resolver.(*Group)
-
- for i := 0; i < len(split)-1; i++ {
- subgroupName := split[i]
- subgroup := g.subgroups[subgroupName]
- if subgroup == nil {
- subgroup = &Group{
- name: subgroupName,
- projects: make(projects),
- subgroups: make(subgroups),
- }
- g.subgroups[subgroupName] = subgroup
- }
-
- g = subgroup
- }
-
- g.projects[projectName] = &projectConfig{
- NamespaceProject: domainName == projectName,
- HTTPSOnly: httpsOnly,
- AccessControl: accessControl,
- ID: id,
- }
-
- dm[domainName] = groupDomain
-}
-
-func (dm Map) readProjectConfig(rootDomain string, group, projectName string, config *multiDomainConfig) {
- if config == nil {
- // This is necessary to preserve the previous behaviour where a
- // group domain is created even if no config.json files are
- // loaded successfully. Is it safe to remove this?
- dm.updateGroupDomain(rootDomain, group, projectName, false, false, 0)
- return
- }
-
- dm.updateGroupDomain(rootDomain, group, projectName, config.HTTPSOnly, config.AccessControl, config.ID)
-
- for _, domainConfig := range config.Domains {
- config := domainConfig // domainConfig is reused for each loop iteration
- if domainConfig.Valid(rootDomain) {
- dm.addDomain(rootDomain, group, projectName, &config)
- }
- }
-}
-
-func readProject(group, parent, projectName string, level int, fanIn chan<- jobResult) {
- if strings.HasPrefix(projectName, ".") {
- return
- }
-
- // Ignore projects that have .deleted in name
- if strings.HasSuffix(projectName, ".deleted") {
- return
- }
-
- projectPath := filepath.Join(parent, projectName)
- if _, err := os.Lstat(filepath.Join(group, projectPath, "public")); err != nil {
- // maybe it's a subgroup
- if level <= subgroupScanLimit {
- buf := make([]byte, 2*os.Getpagesize())
- readProjects(group, projectPath, level+1, buf, fanIn)
- }
-
- return
- }
-
- // We read the config.json file _before_ fanning in, because it does disk
- // IO and it does not need access to the domains map.
- config := &multiDomainConfig{}
- if err := config.Read(group, projectPath); err != nil {
- config = nil
- }
-
- fanIn <- jobResult{group: group, project: projectPath, config: config}
-}
-
-func readProjects(group, parent string, level int, buf []byte, fanIn chan<- jobResult) {
- subgroup := filepath.Join(group, parent)
- fis, err := godirwalk.ReadDirents(subgroup, buf)
- if err != nil {
- log.WithError(err).WithFields(log.Fields{
- "group": group,
- "parent": parent,
- }).Print("readdir failed")
- return
- }
-
- for _, project := range fis {
- // Ignore non directories
- if !project.IsDir() {
- continue
- }
-
- readProject(group, parent, project.Name(), level, fanIn)
- }
-}
-
-type jobResult struct {
- group string
- project string
- config *multiDomainConfig
-}
-
-// ReadGroups walks the pages directory and populates dm with all the domains it finds.
-func (dm Map) ReadGroups(rootDomain string, fis godirwalk.Dirents) {
- fanOutGroups := make(chan string)
- fanIn := make(chan jobResult)
- wg := &sync.WaitGroup{}
- for i := 0; i < 4; i++ {
- wg.Add(1)
-
- go func() {
- buf := make([]byte, 2*os.Getpagesize())
-
- for group := range fanOutGroups {
- if group == skipHashedDir {
- continue
- }
-
- started := time.Now()
-
- readProjects(group, "", 0, buf, fanIn)
-
- log.WithFields(log.Fields{
- "group": group,
- "duration": time.Since(started).Seconds(),
- }).Debug("Loaded projects for group")
- }
-
- wg.Done()
- }()
- }
-
- go func() {
- wg.Wait()
- close(fanIn)
- }()
-
- done := make(chan struct{})
- go func() {
- for result := range fanIn {
- dm.readProjectConfig(rootDomain, result.group, result.project, result.config)
- }
-
- close(done)
- }()
-
- for _, group := range fis {
- if !group.IsDir() {
- continue
- }
- if strings.HasPrefix(group.Name(), ".") {
- continue
- }
- fanOutGroups <- group.Name()
- }
- close(fanOutGroups)
-
- <-done
-}
-
-const (
- updateFile = ".update"
-)
-
-// Watch polls the filesystem and kicks off a new domain directory scan when needed.
-func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) {
- lastUpdate := []byte("no-update")
-
- for {
- // Read the update file
- update, err := ioutil.ReadFile(updateFile)
- if err != nil && !os.IsNotExist(err) {
- log.WithError(err).Print("failed to read update timestamp")
- time.Sleep(interval)
- continue
- }
-
- // If it's the same ignore
- if bytes.Equal(lastUpdate, update) {
- time.Sleep(interval)
- continue
- }
- lastUpdate = update
-
- started := time.Now()
- dm := make(Map)
-
- fis, err := godirwalk.ReadDirents(".", nil)
- if err != nil {
- log.WithError(err).Warn("domain scan failed")
- metrics.DomainFailedUpdates.Inc()
- continue
- }
-
- dm.ReadGroups(rootDomain, fis)
- duration := time.Since(started).Seconds()
-
- var hash string
- if len(update) < 1 {
- hash = "<empty>"
- } else {
- hash = strings.TrimSpace(string(update))
- }
-
- logConfiguredDomains(dm)
-
- log.WithFields(log.Fields{
- "count(domains)": len(dm),
- "duration": duration,
- "hash": hash,
- }).Info("Updated all domains")
-
- if updater != nil {
- updater(dm)
- }
-
- // Update prometheus metrics
- metrics.DomainLastUpdateTime.Set(float64(time.Now().UTC().Unix()))
- metrics.DomainsServed.Set(float64(len(dm)))
- metrics.DomainsConfigurationUpdateDuration.Set(duration)
- metrics.DomainUpdates.Inc()
-
- time.Sleep(interval)
- }
-}
-
-func logConfiguredDomains(dm Map) {
- if logrus.GetLevel() != logrus.DebugLevel {
- return
- }
-
- for h, d := range dm {
- log.WithFields(log.Fields{
- "domain": d,
- "host": h,
- }).Debug("Configured domain")
- }
-}
diff --git a/internal/source/disk/map_test.go b/internal/source/disk/map_test.go
deleted file mode 100644
index 2a5fd828..00000000
--- a/internal/source/disk/map_test.go
+++ /dev/null
@@ -1,253 +0,0 @@
-package disk
-
-import (
- "crypto/rand"
- "fmt"
- "io/ioutil"
- "os"
- "strings"
- "testing"
- "time"
-
- "github.com/karrick/godirwalk"
- "github.com/stretchr/testify/require"
-
- "gitlab.com/gitlab-org/gitlab-pages/internal/testhelpers"
-)
-
-func getEntries(t require.TestingT) godirwalk.Dirents {
- fis, err := godirwalk.ReadDirents(".", nil)
-
- require.NoError(t, err)
-
- return fis
-}
-
-func TestReadProjects(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- dm := make(Map)
- dm.ReadGroups("test.io", getEntries(t))
-
- var domains []string
- for d := range dm {
- domains = append(domains, d)
- }
-
- expectedDomains := []string{
- "group.test.io",
- "group.internal.test.io",
- "test.domain.com", // from config.json
- "other.domain.com",
- "domain.404.com",
- "group.404.test.io",
- "group.https-only.test.io",
- "test.my-domain.com",
- "test2.my-domain.com",
- "no.cert.com",
- "private.domain.com",
- "group.auth.test.io",
- "group.acme.test.io",
- "withacmechallenge.domain.com",
- "capitalgroup.test.io",
- "group.404.gitlab-example.com",
- "group.redirects.test.io",
- "redirects.custom-domain.com",
- }
-
- for _, expected := range domains {
- require.Contains(t, domains, expected)
- }
-
- for _, actual := range domains {
- require.Contains(t, expectedDomains, actual)
- }
-
- // Check that multiple domains in the same project are recorded faithfully
- require.Equal(t, "test.domain.com", dm["test.domain.com"].Name)
- require.Equal(t, "other.domain.com", dm["other.domain.com"].Name)
- require.Equal(t, "test", dm["other.domain.com"].CertificateCert)
- require.Equal(t, "key", dm["other.domain.com"].CertificateKey)
-
- // check subgroups
- domain, ok := dm["group.test.io"]
- require.True(t, ok, "missing group.test.io domain")
- subgroup, ok := domain.Resolver.(*Group).subgroups["subgroup"]
- require.True(t, ok, "missing group.test.io subgroup")
- _, ok = subgroup.projects["project"]
- require.True(t, ok, "missing project for subgroup in group.test.io domain")
-}
-
-func TestReadProjectsMaxDepth(t *testing.T) {
- nGroups := 3
- levels := subgroupScanLimit + 5
- cleanup := buildFakeDomainsDirectory(t, nGroups, levels)
- defer cleanup()
-
- defaultDomain := "test.io"
- dm := make(Map)
- dm.ReadGroups(defaultDomain, getEntries(t))
-
- var domains []string
- for d := range dm {
- domains = append(domains, d)
- }
-
- var expectedDomains []string
- for i := 0; i < nGroups; i++ {
- expectedDomains = append(expectedDomains, fmt.Sprintf("group-%d.%s", i, defaultDomain))
- }
-
- for _, expected := range domains {
- require.Contains(t, domains, expected)
- }
-
- for _, actual := range domains {
- // we are not checking config.json domains here
- if !strings.HasSuffix(actual, defaultDomain) {
- continue
- }
- require.Contains(t, expectedDomains, actual)
- }
-
- // check subgroups
- domain, ok := dm["group-0.test.io"]
- require.True(t, ok, "missing group-0.test.io domain")
- subgroup := domain.Resolver.(*Group)
- for i := 0; i < levels; i++ {
- subgroup, ok = subgroup.subgroups["sub"]
- if i <= subgroupScanLimit {
- require.True(t, ok, "missing group-0.test.io subgroup at level %d", i)
- _, ok = subgroup.projects["project-0"]
- require.True(t, ok, "missing project for subgroup in group-0.test.io domain at level %d", i)
- } else {
- require.False(t, ok, "subgroup level %d. Maximum allowed nesting level is %d", i, subgroupScanLimit)
- break
- }
- }
-}
-
-// This write must be atomic, otherwise we cannot predict the state of the
-// domain watcher goroutine. We cannot use ioutil.WriteFile because that
-// has a race condition where the file is empty, which can get picked up
-// by the domain watcher.
-func writeRandomTimestamp(t *testing.T) {
- b := make([]byte, 10)
- n, _ := rand.Read(b)
- require.True(t, n > 0, "read some random bytes")
-
- temp, err := ioutil.TempFile(".", "TestWatch")
- require.NoError(t, err)
- _, err = temp.Write(b)
- require.NoError(t, err, "write to tempfile")
- require.NoError(t, temp.Close(), "close tempfile")
-
- require.NoError(t, os.Rename(temp.Name(), updateFile), "rename tempfile")
-}
-
-func TestWatch(t *testing.T) {
- cleanup := setUpTests(t)
- defer cleanup()
-
- require.NoError(t, os.RemoveAll(updateFile))
-
- update := make(chan Map)
- go Watch("gitlab.io", func(dm Map) {
- update <- dm
- }, time.Microsecond*50)
-
- defer os.Remove(updateFile)
-
- domains := recvTimeout(t, update)
- require.NotNil(t, domains, "if the domains are fetched on start")
-
- writeRandomTimestamp(t)
- domains = recvTimeout(t, update)
- require.NotNil(t, domains, "if the domains are updated after the creation")
-
- writeRandomTimestamp(t)
- domains = recvTimeout(t, update)
- require.NotNil(t, domains, "if the domains are updated after the timestamp change")
-}
-
-func recvTimeout(t *testing.T, ch <-chan Map) Map {
- timeout := 5 * time.Second
-
- select {
- case dm := <-ch:
- return dm
- case <-time.After(timeout):
- t.Fatalf("timeout after %v waiting for domain update", timeout)
- return nil
- }
-}
-
-func buildFakeDomainsDirectory(t testing.TB, nGroups, levels int) func() {
- testRoot, err := ioutil.TempDir("", "gitlab-pages-test")
- require.NoError(t, err)
-
- for i := 0; i < nGroups; i++ {
- parent := fmt.Sprintf("%s/group-%d", testRoot, i)
- domain := fmt.Sprintf("%d.example.io", i)
- buildFakeProjectsDirectory(t, parent, domain)
- for j := 0; j < levels; j++ {
- parent = fmt.Sprintf("%s/sub", parent)
- domain = fmt.Sprintf("%d.%s", j, domain)
- buildFakeProjectsDirectory(t, parent, domain)
- }
- if testing.Verbose() && i%100 == 0 {
- fmt.Print(".")
- }
- }
-
- cleanup := testhelpers.ChdirInPath(t, testRoot, &chdirSet)
-
- return func() {
- defer cleanup()
-
- if testing.Verbose() {
- fmt.Printf("cleaning up test directory %s\n", testRoot)
- }
-
- os.RemoveAll(testRoot)
- }
-}
-
-func buildFakeProjectsDirectory(t require.TestingT, groupPath, domain string) {
- for j := 0; j < 5; j++ {
- dir := fmt.Sprintf("%s/project-%d", groupPath, j)
- require.NoError(t, os.MkdirAll(dir+"/public", 0755))
-
- fakeConfig := fmt.Sprintf(`{"Domains":[{"Domain":"foo.%d.%s","Certificate":"bar","Key":"baz"}]}`, j, domain)
- require.NoError(t, ioutil.WriteFile(dir+"/config.json", []byte(fakeConfig), 0644))
- }
-}
-
-// this is a safeguard against compiler optimizations
-// we use this package variable to make sure the benchmarkReadGroups loop
-// has side effects outside of the loop.
-// Without this the compiler (with the optimizations enabled) may remove the whole loop
-var result int
-
-func benchmarkReadGroups(b *testing.B, groups, levels int) {
- cleanup := buildFakeDomainsDirectory(b, groups, levels)
- defer cleanup()
-
- b.ResetTimer()
-
- domainsCnt := 0
- for i := 0; i < b.N; i++ {
- dm := make(Map)
- dm.ReadGroups("example.com", getEntries(b))
- domainsCnt = len(dm)
- }
- result = domainsCnt
-}
-
-func BenchmarkReadGroups(b *testing.B) {
- b.Run("10 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 10, 3) })
- b.Run("100 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 100, 3) })
- b.Run("1000 groups 3 levels", func(b *testing.B) { benchmarkReadGroups(b, 1000, 3) })
- b.Run("10000 groups 1 levels", func(b *testing.B) { benchmarkReadGroups(b, 10000, 1) })
-}
diff --git a/internal/source/domains.go b/internal/source/domains.go
index 8b7e2be9..efa613e1 100644
--- a/internal/source/domains.go
+++ b/internal/source/domains.go
@@ -2,41 +2,24 @@ package source
import (
"context"
- "fmt"
-
- "gitlab.com/gitlab-org/labkit/log"
"gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk"
"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab"
)
-type configSource int
-
-const (
- sourceGitlab configSource = iota
- // Disk source is deprecated and support will be removed in 14.3
- // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382
- sourceDisk
- sourceAuto
-)
-
-// Domains struct represents a map of all domains supported by pages. It is
-// currently using two sources during the transition to the new GitLab domains
-// source.
+// Domains struct wraps the GitLab client to fetch a domain configuration
+// TODO: remove/refactor this package https://gitlab.com/gitlab-org/gitlab-pages/-/issues/608
type Domains struct {
- configSource configSource
- gitlab Source
- disk *disk.Disk // legacy disk source
+ gitlab Source
}
// NewDomains is a factory method for domains initializing a mutex. It should
// not initialize `dm` as we later check the readiness by comparing it with a
// nil value.
-func NewDomains(source string, cfg *config.GitLab) (*Domains, error) {
+func NewDomains(cfg *config.GitLab) (*Domains, error) {
domains := &Domains{}
- if err := domains.setConfigSource(source, cfg); err != nil {
+ if err := domains.setConfigSource(cfg); err != nil {
return nil, err
}
@@ -44,43 +27,17 @@ func NewDomains(source string, cfg *config.GitLab) (*Domains, error) {
}
// setConfigSource and initialize gitlab source
-// returns error if -domain-config-source is not valid
-// returns error if -domain-config-source=gitlab and init fails
-func (d *Domains) setConfigSource(source string, cfg *config.GitLab) error {
- switch source {
- case "gitlab":
- d.configSource = sourceGitlab
- return d.setGitLabClient(cfg)
- case "auto":
- d.configSource = sourceAuto
- // enable disk for auto for now
- d.disk = disk.New()
- return d.setGitLabClient(cfg)
- case "disk":
- // TODO: disable domains.disk https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382
- d.configSource = sourceDisk
- d.disk = disk.New()
- default:
- return fmt.Errorf("invalid option for -domain-config-source: %q", source)
- }
-
- return nil
+func (d *Domains) setConfigSource(cfg *config.GitLab) error {
+ return d.setGitLabClient(cfg)
}
-// setGitLabClient when domain-config-source is `gitlab` or `auto`, only return error for `gitlab` source
func (d *Domains) setGitLabClient(cfg *config.GitLab) error {
// We want to notify users about any API issues
// Creating a glClient will start polling connectivity in the background
// and spam errors in log
glClient, err := gitlab.New(cfg)
if err != nil {
- if d.configSource == sourceGitlab {
- return err
- }
-
- log.WithError(err).Warn("failed to initialize GitLab client for `-domain-config-source=auto`")
-
- return nil
+ return err
}
d.gitlab = glClient
@@ -93,50 +50,5 @@ func (d *Domains) setGitLabClient(cfg *config.GitLab) error {
// for some subset of domains, to test / PoC the new GitLab Domains Source that
// we plan to use to replace the disk source.
func (d *Domains) GetDomain(ctx context.Context, name string) (*domain.Domain, error) {
- return d.source(name).GetDomain(ctx, name)
-}
-
-// Read starts the disk domain source. It is DEPRECATED, because we want to
-// remove it entirely when disk source gets removed.
-func (d *Domains) Read(rootDomain string) {
- // start disk.Read for sourceDisk and sourceAuto
- if d.configSource != sourceGitlab {
- d.disk.Read(rootDomain)
- }
-}
-
-// IsReady checks if the disk domain source managed to traverse entire pages
-// filesystem and is ready for use. It is DEPRECATED, because we want to remove
-// it entirely when disk source gets removed.
-func (d *Domains) IsReady() bool {
- switch d.configSource {
- case sourceGitlab:
- return d.gitlab.IsReady()
- case sourceDisk:
- return d.disk.IsReady()
- case sourceAuto:
- // if gitlab is configured and is ready
- if d.gitlab != nil && d.gitlab.IsReady() {
- return true
- }
-
- return d.disk.IsReady()
- default:
- return false
- }
-}
-
-func (d *Domains) source(domain string) Source {
- switch d.configSource {
- case sourceDisk:
- return d.disk
- case sourceGitlab:
- return d.gitlab
- default:
- if d.gitlab != nil && d.gitlab.IsReady() {
- return d.gitlab
- }
-
- return d.disk
- }
+ return d.gitlab.GetDomain(ctx, name)
}
diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go
index 8618df38..c3f0d2cc 100644
--- a/internal/source/domains_test.go
+++ b/internal/source/domains_test.go
@@ -9,7 +9,6 @@ import (
"gitlab.com/gitlab-org/gitlab-pages/internal/config"
"gitlab.com/gitlab-org/gitlab-pages/internal/domain"
- "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk"
)
func TestNewDomains(t *testing.T) {
@@ -21,51 +20,16 @@ func TestNewDomains(t *testing.T) {
}
tests := []struct {
- name string
- source string
- config config.GitLab
- expectedErr string
- expectGitlabNil bool
- expectDiskNil bool
+ name string
+ config config.GitLab
+ expectedErr string
}{
{
- name: "no_source_config",
- source: "",
- expectedErr: "invalid option for -domain-config-source: \"\"",
+ name: "gitlab_source_success",
+ config: validCfg,
},
{
- name: "invalid_source_config",
- source: "invalid",
- expectedErr: "invalid option for -domain-config-source: \"invalid\"",
- },
- {
- name: "disk_source",
- source: "disk",
- expectGitlabNil: true,
- expectDiskNil: false,
- },
- {
- name: "auto_without_api_config",
- source: "auto",
- expectGitlabNil: true,
- expectDiskNil: false,
- },
- {
- name: "auto_with_api_config",
- source: "auto",
- config: validCfg,
- expectGitlabNil: false,
- expectDiskNil: false,
- },
- {
- name: "gitlab_source_success",
- source: "gitlab",
- config: validCfg,
- expectDiskNil: true,
- },
- {
- name: "gitlab_source_no_url",
- source: "gitlab",
+ name: "gitlab_source_no_url",
config: func() config.GitLab {
cfg := validCfg
cfg.InternalServer = ""
@@ -75,8 +39,7 @@ func TestNewDomains(t *testing.T) {
expectedErr: "GitLab API URL or API secret has not been provided",
},
{
- name: "gitlab_source_no_secret",
- source: "gitlab",
+ name: "gitlab_source_no_secret",
config: func() config.GitLab {
cfg := validCfg
cfg.APISecretKey = []byte{}
@@ -89,15 +52,13 @@ func TestNewDomains(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- domains, err := NewDomains(tt.source, &tt.config)
+ domains, err := NewDomains(&tt.config)
if tt.expectedErr != "" {
require.EqualError(t, err, tt.expectedErr)
return
}
require.NoError(t, err)
-
- require.Equal(t, tt.expectGitlabNil, domains.gitlab == nil, "mismatch gitlab nil")
- require.Equal(t, tt.expectDiskNil, domains.disk == nil, "mismatch disk nil")
+ require.NotNil(t, domains.gitlab)
})
}
}
@@ -112,24 +73,7 @@ func TestGetDomain(t *testing.T) {
Once()
defer newSource.AssertExpectations(t)
- domains := newTestDomains(t, newSource, sourceGitlab)
-
- domain, err := domains.GetDomain(context.Background(), testDomain)
- require.NoError(t, err)
- require.NotNil(t, domain)
- })
-
- t.Run("when requesting an existing domain for auto source", func(t *testing.T) {
- testDomain := "new-source-test.gitlab.io"
-
- newSource := NewMockSource()
- newSource.On("GetDomain", testDomain).
- Return(&domain.Domain{Name: testDomain}, nil).
- Once()
- newSource.On("IsReady").Return(true).Once()
- defer newSource.AssertExpectations(t)
-
- domains := newTestDomains(t, newSource, sourceAuto)
+ domains := newTestDomains(t, newSource)
domain, err := domains.GetDomain(context.Background(), testDomain)
require.NoError(t, err)
@@ -144,7 +88,7 @@ func TestGetDomain(t *testing.T) {
defer newSource.AssertExpectations(t)
- domains := newTestDomains(t, newSource, sourceGitlab)
+ domains := newTestDomains(t, newSource)
domain, err := domains.GetDomain(context.Background(), "does-not-exist.test.io")
require.NoError(t, err)
@@ -152,12 +96,10 @@ func TestGetDomain(t *testing.T) {
})
}
-func newTestDomains(t *testing.T, gitlabSource *MockSource, config configSource) *Domains {
+func newTestDomains(t *testing.T, gitlabSource *MockSource) *Domains {
t.Helper()
return &Domains{
- configSource: config,
- gitlab: gitlabSource,
- disk: disk.New(),
+ gitlab: gitlabSource,
}
}
diff --git a/internal/source/gitlab/gitlab.go b/internal/source/gitlab/gitlab.go
index b79b434f..d0233028 100644
--- a/internal/source/gitlab/gitlab.go
+++ b/internal/source/gitlab/gitlab.go
@@ -110,8 +110,3 @@ func sortLookupsByPrefixLengthDesc(lookups []api.LookupPath) {
return len(lookups[i].Prefix) > len(lookups[j].Prefix)
})
}
-
-// IsReady returns the value of Gitlab `isReady` which is updated by `Poll`.
-func (g *Gitlab) IsReady() bool {
- return true
-}
diff --git a/internal/source/source.go b/internal/source/source.go
index cbe949ef..9a247b1b 100644
--- a/internal/source/source.go
+++ b/internal/source/source.go
@@ -9,5 +9,4 @@ import (
// Source represents an abstract interface of a domains configuration source.
type Source interface {
GetDomain(context.Context, string) (*domain.Domain, error)
- IsReady() bool
}
diff --git a/test/acceptance/config_test.go b/test/acceptance/config_test.go
index 82b7fbb1..07697c48 100644
--- a/test/acceptance/config_test.go
+++ b/test/acceptance/config_test.go
@@ -5,7 +5,6 @@ import (
"net"
"net/http"
"testing"
- "time"
"github.com/stretchr/testify/require"
)
@@ -65,29 +64,3 @@ func TestMultipleListenersFromEnvironmentVariables(t *testing.T) {
require.Equal(t, http.StatusOK, rsp.StatusCode)
}
}
-
-// TODO: remove along chroot https://gitlab.com/gitlab-org/gitlab-pages/-/issues/561
-func TestEnableJailFromEnvironment(t *testing.T) {
- out, teardown := runPagesProcess(t,
- true,
- *pagesBinary,
- []ListenSpec{httpListener},
- "",
- []string{
- "DAEMON_ENABLE_JAIL=true",
- },
- "-domain-config-source", "disk",
- )
- t.Cleanup(teardown)
-
- require.Eventually(t, func() bool {
- require.Contains(t, out.String(), "\"daemon-enable-jail\":true")
- return true
- }, time.Second, 10*time.Millisecond)
-
- rsp, err := GetPageFromListener(t, httpListener, "group.gitlab-example.com", "project/")
-
- require.NoError(t, err)
- rsp.Body.Close()
- require.Equal(t, http.StatusOK, rsp.StatusCode)
-}
diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go
index c1074230..c8852976 100644
--- a/test/acceptance/helpers_test.go
+++ b/test/acceptance/helpers_test.go
@@ -242,7 +242,6 @@ func RunPagesProcess(t *testing.T, opts ...processOption) *LogCaptureBuffer {
"-pages-root", wd,
"-internal-gitlab-server", source.URL,
"-api-secret-key", gitLabAPISecretKey,
- "-domain-config-source", "gitlab",
)
logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...)
@@ -352,16 +351,6 @@ func getPagesArgs(t *testing.T, listeners []ListenSpec, promPort string, extraAr
args = append(args, "-metrics-address", promPort)
}
- // most of our acceptance tests still work only with disk source
- // TODO: remove this with -domain-config-source flag itself along with daemon-enable-jail:
- // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382
- // https://gitlab.com/gitlab-org/gitlab-pages/-/issues/561
- if !contains(extraArgs, "-domain-config-source") {
- args = append(args,
- "-domain-config-source", "disk",
- )
- }
-
args = append(args, getPagesDaemonArgs(t)...)
args = append(args, extraArgs...)
diff --git a/test/acceptance/serving_test.go b/test/acceptance/serving_test.go
index c42866ee..8c4ab461 100644
--- a/test/acceptance/serving_test.go
+++ b/test/acceptance/serving_test.go
@@ -4,8 +4,6 @@ import (
"fmt"
"io/ioutil"
"net/http"
- "os"
- "path"
"strings"
"testing"
"time"
@@ -106,51 +104,6 @@ func TestKnownHostReturns200(t *testing.T) {
}
}
-// TODO: remove along with support for disk configuration https://gitlab.com/gitlab-org/gitlab-pages/-/issues/382
-func TestNestedSubgroups(t *testing.T) {
- maxNestedSubgroup := 21
-
- pagesRoot, err := ioutil.TempDir("", "pages-root")
- require.NoError(t, err)
- defer os.RemoveAll(pagesRoot)
-
- makeProjectIndex := func(subGroupPath string) {
- projectPath := path.Join(pagesRoot, "nested", subGroupPath, "project", "public")
- require.NoError(t, os.MkdirAll(projectPath, 0755))
-
- projectIndex := path.Join(projectPath, "index.html")
- require.NoError(t, ioutil.WriteFile(projectIndex, []byte("index"), 0644))
- }
- makeProjectIndex("")
-
- paths := []string{""}
- for i := 1; i < maxNestedSubgroup*2; i++ {
- subGroupPath := fmt.Sprintf("%ssub%d/", paths[i-1], i)
- paths = append(paths, subGroupPath)
-
- makeProjectIndex(subGroupPath)
- }
-
- teardown := RunPagesProcessWithoutGitLabStub(t, *pagesBinary, supportedListeners(), "", "-pages-root", pagesRoot)
- defer teardown()
-
- for nestingLevel, path := range paths {
- t.Run(fmt.Sprintf("nested level %d", nestingLevel), func(t *testing.T) {
- for _, spec := range supportedListeners() {
- rsp, err := GetPageFromListener(t, spec, "nested.gitlab-example.com", path+"project/")
-
- require.NoError(t, err)
- rsp.Body.Close()
- if nestingLevel <= maxNestedSubgroup {
- require.Equal(t, http.StatusOK, rsp.StatusCode)
- } else {
- require.Equal(t, http.StatusNotFound, rsp.StatusCode)
- }
- }
- })
- }
-}
-
func TestCustom404(t *testing.T) {
RunPagesProcess(t)
@@ -425,42 +378,6 @@ func TestDomainsSource(t *testing.T) {
apiCalled: true,
},
},
- {
- name: "disk_source_domain_exists",
- args: args{
- configSource: "disk",
- // test.domain.com sourced from disk configuration
- domain: "test.domain.com",
- urlSuffix: "/",
- },
- want: want{
- statusCode: http.StatusOK,
- content: "main-dir\n",
- apiCalled: false,
- },
- },
- {
- name: "disk_source_domain_does_not_exist",
- args: args{
- configSource: "disk",
- domain: "non-existent-domain.gitlab.io",
- },
- want: want{
- statusCode: http.StatusNotFound,
- apiCalled: false,
- },
- },
- {
- name: "disk_source_domain_should_not_exist_under_hashed_dir",
- args: args{
- configSource: "disk",
- domain: "hashed.com",
- },
- want: want{
- statusCode: http.StatusNotFound,
- apiCalled: false,
- },
- },
}
for _, tt := range tests {
@@ -474,7 +391,7 @@ func TestDomainsSource(t *testing.T) {
gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
- pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey, "-domain-config-source", tt.args.configSource}
+ pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey}
teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, []ListenSpec{httpListener}, "", []string{}, pagesArgs...)
defer teardown()