diff options
author | Nick Thomas <nick@gitlab.com> | 2020-01-13 14:27:42 +0300 |
---|---|---|
committer | Nick Thomas <nick@gitlab.com> | 2020-01-13 14:27:42 +0300 |
commit | b5290fda3d9c054feeac87e6a6ed64172f40a865 (patch) | |
tree | ad3d8db7c85cea98b453b76823d95492e9f98925 | |
parent | 1e9f978d74ae0e1d536f1984635021b368802ccb (diff) | |
parent | e0ba15e77cfe922ece762f487269cb4e626ca290 (diff) |
Merge branch '266-flip-config-source' into 'master'
Watch a file to configure which domains should use the new gitlab config source
See merge request gitlab-org/gitlab-pages!211
-rw-r--r-- | acceptance_test.go | 39 | ||||
-rw-r--r-- | go.mod | 1 | ||||
-rw-r--r-- | helpers_test.go | 22 | ||||
-rw-r--r-- | internal/source/domains.go | 23 | ||||
-rw-r--r-- | internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go | 85 | ||||
-rw-r--r-- | internal/source/domains_test.go | 6 |
6 files changed, 142 insertions, 34 deletions
diff --git a/acceptance_test.go b/acceptance_test.go index ae1bf2b6..49d391b1 100644 --- a/acceptance_test.go +++ b/acceptance_test.go @@ -432,20 +432,6 @@ func TestPageNotAvailableIfNotLoaded(t *testing.T) { require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode) } -func TestPageNotAvailableInDomainSource(t *testing.T) { - skipUnlessEnabled(t) - - brokenDomain := "GITLAB_NEW_SOURCE_BROKEN_DOMAIN=pages-broken-poc.gitlab.io" - teardown := RunPagesProcessWithEnvs(t, false, *pagesBinary, listeners, "", []string{brokenDomain}, "-pages-root=shared/invalid-pages") - defer teardown() - waitForRoundtrips(t, listeners, 5*time.Second) - - rsp, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html") - require.NoError(t, err) - defer rsp.Body.Close() - require.Equal(t, http.StatusBadGateway, rsp.StatusCode) -} - func TestObscureMIMEType(t *testing.T) { skipUnlessEnabled(t) teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "") @@ -1534,10 +1520,22 @@ func TestGitlabDomainsSource(t *testing.T) { source := NewGitlabDomainsSourceStub(t) defer source.Close() - newSourceDomains := "GITLAB_NEW_SOURCE_DOMAINS=new-source-test.gitlab.io,non-existent-domain.gitlab.io" + gitlabSourceConfig := ` +domains: + enabled: + - new-source-test.gitlab.io + broken: pages-broken-poc.gitlab.io +` + gitlabSourceConfigFile, cleanupGitlabSourceConfigFile := CreateGitlabSourceConfigFixtureFile(t, gitlabSourceConfig) + defer cleanupGitlabSourceConfigFile() + + gitlabSourceConfigFile = "GITLAB_SOURCE_CONFIG_FILE=" + gitlabSourceConfigFile + gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t) + pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey} - teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{newSourceDomains}, pagesArgs...) + + teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{gitlabSourceConfigFile}, pagesArgs...) defer teardown() t.Run("when a domain exists", func(t *testing.T) { @@ -1558,4 +1556,13 @@ func TestGitlabDomainsSource(t *testing.T) { require.Equal(t, http.StatusNotFound, response.StatusCode) }) + + t.Run("broken domain is requested", func(t *testing.T) { + response, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html") + require.NoError(t, err) + + defer response.Body.Close() + + require.Equal(t, http.StatusBadGateway, response.StatusCode) + }) } @@ -29,4 +29,5 @@ require ( golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b golang.org/x/tools v0.0.0-20191010201905-e5ffc44a6fee gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/helpers_test.go b/helpers_test.go index bcc43d63..b1f5e1b2 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "os" "os/exec" + "path/filepath" "strings" "testing" "time" @@ -85,6 +86,27 @@ func CreateGitLabAPISecretKeyFixtureFile(t *testing.T) (filepath string) { return secretfile.Name() } +func CreateGitlabSourceConfigFixtureFile(t *testing.T, domains string) (filename string, cleanup func()) { + configfile, err := ioutil.TempFile("shared/pages", "gitlab-source-config-*") + require.NoError(t, err) + configfile.Close() + + cleanup = func() { + os.RemoveAll(configfile.Name()) + } + + require.NoError(t, ioutil.WriteFile(configfile.Name(), []byte(domains), 0644)) + + filename, err = filepath.Abs(configfile.Name()) + require.NoError(t, err) + + if os.Getenv("TEST_DAEMONIZE") != "" { + filename = filepath.Base(filename) + } + + return filename, cleanup +} + // ListenSpec is used to point at a gitlab-pages http server, preserving the // type of port it is (http, https, proxy) type ListenSpec struct { diff --git a/internal/source/domains.go b/internal/source/domains.go index 85d06152..22187855 100644 --- a/internal/source/domains.go +++ b/internal/source/domains.go @@ -2,27 +2,20 @@ package source import ( "errors" - "os" - "strings" + "time" "gitlab.com/gitlab-org/gitlab-pages/internal/domain" "gitlab.com/gitlab-org/gitlab-pages/internal/source/disk" + "gitlab.com/gitlab-org/gitlab-pages/internal/source/domains/gitlabsourceconfig" "gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab" ) -var newSourceDomains []string -var brokenSourceDomain string +var gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig func init() { - testDomains := os.Getenv("GITLAB_NEW_SOURCE_DOMAINS") - if testDomains != "" { - newSourceDomains = strings.Split(testDomains, ",") - } - - brokenDomain := os.Getenv("GITLAB_NEW_SOURCE_BROKEN_DOMAIN") - if brokenDomain != "" { - brokenSourceDomain = brokenDomain - } + // Start watching the config file for domains that will use the new `gitlab` source, + // to be removed once we switch completely to using it. + go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute) } // Domains struct represents a map of all domains supported by pages. It is @@ -57,7 +50,7 @@ func NewDomains(config Config) (*Domains, error) { // for some subset of domains, to test / PoC the new GitLab Domains Source that // we plan to use to replace the disk source. func (d *Domains) GetDomain(name string) (*domain.Domain, error) { - if name == brokenSourceDomain { + if name == gitlabSourceConfig.Domains.Broken { return nil, errors.New("broken test domain used") } @@ -82,7 +75,7 @@ func (d *Domains) source(domain string) Source { return d.disk } - for _, name := range newSourceDomains { + for _, name := range gitlabSourceConfig.Domains.Enabled { if domain == name { return d.gitlab } diff --git a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go new file mode 100644 index 00000000..4eda31aa --- /dev/null +++ b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go @@ -0,0 +1,85 @@ +package gitlabsourceconfig + +import ( + "bytes" + "io/ioutil" + "os" + "time" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// GitlabSourceDomains holds the domains to be used with the gitlab source +type GitlabSourceDomains struct { + Enabled []string + Broken string +} + +// GitlabSourceConfig holds the configuration for the gitlab source +type GitlabSourceConfig struct { + Domains GitlabSourceDomains +} + +// UpdateFromYaml updates the config +// We use new variable here (instead of using `config` directly) +// because if `content` is empty `yaml.Unmarshal` does not update +// the fields already set. +func (config *GitlabSourceConfig) UpdateFromYaml(content []byte) error { + updated := GitlabSourceConfig{} + + err := yaml.Unmarshal(content, &updated) + if err != nil { + return err + } + + *config = updated + + log.WithFields(log.Fields{ + "Enabled domains": config.Domains.Enabled, + "Broken domain": config.Domains.Broken, + }).Info("gitlab source config updated") + + return nil +} + +// WatchForGitlabSourceConfigChange polls the filesystem and updates test domains if needed. +func WatchForGitlabSourceConfigChange(config *GitlabSourceConfig, interval time.Duration) { + var lastContent []byte + + gitlabSourceConfigFile := os.Getenv("GITLAB_SOURCE_CONFIG_FILE") + if gitlabSourceConfigFile == "" { + gitlabSourceConfigFile = ".gitlab-source-config.yml" + } + + for { + content, err := readConfig(gitlabSourceConfigFile) + if err != nil { + log.WithError(err).Warn("Failed to read gitlab source config file") + + time.Sleep(interval) + continue + } + + if !bytes.Equal(lastContent, content) { + lastContent = content + + err = config.UpdateFromYaml(content) + if err != nil { + log.WithError(err).Warn("Failed to update gitlab source config") + } + } + + time.Sleep(interval) + } +} + +func readConfig(configfile string) ([]byte, error) { + content, err := ioutil.ReadFile(configfile) + + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + return content, nil +} diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go index 378f8c89..5fea6ae3 100644 --- a/internal/source/domains_test.go +++ b/internal/source/domains_test.go @@ -41,11 +41,11 @@ func TestDomainSources(t *testing.T) { } func TestGetDomain(t *testing.T) { - newSourceDomains = []string{"new-source-test.gitlab.io"} - brokenSourceDomain = "pages-broken-poc.gitlab.io" + gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"} + gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io" t.Run("when requesting a test domain", func(t *testing.T) { - testDomain := newSourceDomains[0] + testDomain := gitlabSourceConfig.Domains.Enabled[0] newSource := NewMockSource() newSource.On("GetDomain", testDomain). |