Watch a file to configure which domains should use new gitlab source

Instead of passing domains once in an ENV variable we now watcn a config
file (specified with `GITLAB_SOURCE_CONFIG_FILE`, defaults to
`.gitlab-source-config.yml` and update ednabled/broken domains when it's
content change.

This way we can control this without having to restart Pages.

Related to https://gitlab.com/gitlab-org/gitlab-pages/issues/266.

6 files changed, 142 insertions, 34 deletions

diff --git a/acceptance_test.go b/acceptance_test.go
index ae1bf2b6..49d391b1 100644
--- a/acceptance_test.go
+++ b/acceptance_test.go
@@ -432,20 +432,6 @@ func TestPageNotAvailableIfNotLoaded(t *testing.T) {
 	require.Equal(t, http.StatusServiceUnavailable, rsp.StatusCode)
 }
 
-func TestPageNotAvailableInDomainSource(t *testing.T) {
-	skipUnlessEnabled(t)
-
-	brokenDomain := "GITLAB_NEW_SOURCE_BROKEN_DOMAIN=pages-broken-poc.gitlab.io"
-	teardown := RunPagesProcessWithEnvs(t, false, *pagesBinary, listeners, "", []string{brokenDomain}, "-pages-root=shared/invalid-pages")
-	defer teardown()
-	waitForRoundtrips(t, listeners, 5*time.Second)
-
-	rsp, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html")
-	require.NoError(t, err)
-	defer rsp.Body.Close()
-	require.Equal(t, http.StatusBadGateway, rsp.StatusCode)
-}
-
 func TestObscureMIMEType(t *testing.T) {
 	skipUnlessEnabled(t)
 	teardown := RunPagesProcessWithoutWait(t, *pagesBinary, listeners, "")
@@ -1534,10 +1520,22 @@ func TestGitlabDomainsSource(t *testing.T) {
 	source := NewGitlabDomainsSourceStub(t)
 	defer source.Close()
 
-	newSourceDomains := "GITLAB_NEW_SOURCE_DOMAINS=new-source-test.gitlab.io,non-existent-domain.gitlab.io"
+	gitlabSourceConfig := `
+domains:
+  enabled:
+    - new-source-test.gitlab.io
+  broken: pages-broken-poc.gitlab.io
+`
+	gitlabSourceConfigFile, cleanupGitlabSourceConfigFile := CreateGitlabSourceConfigFixtureFile(t, gitlabSourceConfig)
+	defer cleanupGitlabSourceConfigFile()
+
+	gitlabSourceConfigFile = "GITLAB_SOURCE_CONFIG_FILE=" + gitlabSourceConfigFile
+
 	gitLabAPISecretKey := CreateGitLabAPISecretKeyFixtureFile(t)
+
 	pagesArgs := []string{"-gitlab-server", source.URL, "-api-secret-key", gitLabAPISecretKey}
-	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{newSourceDomains}, pagesArgs...)
+
+	teardown := RunPagesProcessWithEnvs(t, true, *pagesBinary, listeners, "", []string{gitlabSourceConfigFile}, pagesArgs...)
 	defer teardown()
 
 	t.Run("when a domain exists", func(t *testing.T) {
@@ -1558,4 +1556,13 @@ func TestGitlabDomainsSource(t *testing.T) {
 
 		require.Equal(t, http.StatusNotFound, response.StatusCode)
 	})
+
+	t.Run("broken domain is requested", func(t *testing.T) {
+		response, err := GetPageFromListener(t, httpListener, "pages-broken-poc.gitlab.io", "index.html")
+		require.NoError(t, err)
+
+		defer response.Body.Close()
+
+		require.Equal(t, http.StatusBadGateway, response.StatusCode)
+	})
 }
diff --git a/go.mod b/go.mod
index e9a50332..2ae4178c 100644
--- a/go.mod
+++ b/go.mod
@@ -29,4 +29,5 @@ require (
 	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
 	golang.org/x/tools v0.0.0-20191010201905-e5ffc44a6fee
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/yaml.v2 v2.2.2
 )
diff --git a/helpers_test.go b/helpers_test.go
index bcc43d63..b1f5e1b2 100644
--- a/helpers_test.go
+++ b/helpers_test.go
@@ -12,6 +12,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -85,6 +86,27 @@ func CreateGitLabAPISecretKeyFixtureFile(t *testing.T) (filepath string) {
 	return secretfile.Name()
 }
 
+func CreateGitlabSourceConfigFixtureFile(t *testing.T, domains string) (filename string, cleanup func()) {
+	configfile, err := ioutil.TempFile("shared/pages", "gitlab-source-config-*")
+	require.NoError(t, err)
+	configfile.Close()
+
+	cleanup = func() {
+		os.RemoveAll(configfile.Name())
+	}
+
+	require.NoError(t, ioutil.WriteFile(configfile.Name(), []byte(domains), 0644))
+
+	filename, err = filepath.Abs(configfile.Name())
+	require.NoError(t, err)
+
+	if os.Getenv("TEST_DAEMONIZE") != "" {
+		filename = filepath.Base(filename)
+	}
+
+	return filename, cleanup
+}
+
 // ListenSpec is used to point at a gitlab-pages http server, preserving the
 // type of port it is (http, https, proxy)
 type ListenSpec struct {
diff --git a/internal/source/domains.go b/internal/source/domains.go
index 85d06152..22187855 100644
--- a/internal/source/domains.go
+++ b/internal/source/domains.go
@@ -2,27 +2,20 @@ package source
 
 import (
 	"errors"
-	"os"
-	"strings"
+	"time"
 
 	"gitlab.com/gitlab-org/gitlab-pages/internal/domain"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/source/disk"
+	"gitlab.com/gitlab-org/gitlab-pages/internal/source/domains/gitlabsourceconfig"
 	"gitlab.com/gitlab-org/gitlab-pages/internal/source/gitlab"
 )
 
-var newSourceDomains []string
-var brokenSourceDomain string
+var gitlabSourceConfig gitlabsourceconfig.GitlabSourceConfig
 
 func init() {
-	testDomains := os.Getenv("GITLAB_NEW_SOURCE_DOMAINS")
-	if testDomains != "" {
-		newSourceDomains = strings.Split(testDomains, ",")
-	}
-
-	brokenDomain := os.Getenv("GITLAB_NEW_SOURCE_BROKEN_DOMAIN")
-	if brokenDomain != "" {
-		brokenSourceDomain = brokenDomain
-	}
+	// Start watching the config file for domains that will use the new `gitlab` source,
+	// to be removed once we switch completely to using it.
+	go gitlabsourceconfig.WatchForGitlabSourceConfigChange(&gitlabSourceConfig, 1*time.Minute)
 }
 
 // Domains struct represents a map of all domains supported by pages. It is
@@ -57,7 +50,7 @@ func NewDomains(config Config) (*Domains, error) {
 // for some subset of domains, to test / PoC the new GitLab Domains Source that
 // we plan to use to replace the disk source.
 func (d *Domains) GetDomain(name string) (*domain.Domain, error) {
-	if name == brokenSourceDomain {
+	if name == gitlabSourceConfig.Domains.Broken {
 		return nil, errors.New("broken test domain used")
 	}
 
@@ -82,7 +75,7 @@ func (d *Domains) source(domain string) Source {
 		return d.disk
 	}
 
-	for _, name := range newSourceDomains {
+	for _, name := range gitlabSourceConfig.Domains.Enabled {
 		if domain == name {
 			return d.gitlab
 		}
diff --git a/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go
new file mode 100644
index 00000000..4eda31aa
--- /dev/null
+++ b/internal/source/domains/gitlabsourceconfig/gitlabsourceconfig.go
@@ -0,0 +1,85 @@
+package gitlabsourceconfig
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+// GitlabSourceDomains holds the domains to be used with the gitlab source
+type GitlabSourceDomains struct {
+	Enabled []string
+	Broken  string
+}
+
+// GitlabSourceConfig holds the configuration for the gitlab source
+type GitlabSourceConfig struct {
+	Domains GitlabSourceDomains
+}
+
+// UpdateFromYaml updates the config
+// We use new variable here (instead of using `config` directly)
+// because if `content` is empty `yaml.Unmarshal` does not update
+// the fields already set.
+func (config *GitlabSourceConfig) UpdateFromYaml(content []byte) error {
+	updated := GitlabSourceConfig{}
+
+	err := yaml.Unmarshal(content, &updated)
+	if err != nil {
+		return err
+	}
+
+	*config = updated
+
+	log.WithFields(log.Fields{
+		"Enabled domains": config.Domains.Enabled,
+		"Broken domain":   config.Domains.Broken,
+	}).Info("gitlab source config updated")
+
+	return nil
+}
+
+// WatchForGitlabSourceConfigChange polls the filesystem and updates test domains if needed.
+func WatchForGitlabSourceConfigChange(config *GitlabSourceConfig, interval time.Duration) {
+	var lastContent []byte
+
+	gitlabSourceConfigFile := os.Getenv("GITLAB_SOURCE_CONFIG_FILE")
+	if gitlabSourceConfigFile == "" {
+		gitlabSourceConfigFile = ".gitlab-source-config.yml"
+	}
+
+	for {
+		content, err := readConfig(gitlabSourceConfigFile)
+		if err != nil {
+			log.WithError(err).Warn("Failed to read gitlab source config file")
+
+			time.Sleep(interval)
+			continue
+		}
+
+		if !bytes.Equal(lastContent, content) {
+			lastContent = content
+
+			err = config.UpdateFromYaml(content)
+			if err != nil {
+				log.WithError(err).Warn("Failed to update gitlab source config")
+			}
+		}
+
+		time.Sleep(interval)
+	}
+}
+
+func readConfig(configfile string) ([]byte, error) {
+	content, err := ioutil.ReadFile(configfile)
+
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	return content, nil
+}
diff --git a/internal/source/domains_test.go b/internal/source/domains_test.go
index 378f8c89..5fea6ae3 100644
--- a/internal/source/domains_test.go
+++ b/internal/source/domains_test.go
@@ -41,11 +41,11 @@ func TestDomainSources(t *testing.T) {
 }
 
 func TestGetDomain(t *testing.T) {
-	newSourceDomains = []string{"new-source-test.gitlab.io"}
-	brokenSourceDomain = "pages-broken-poc.gitlab.io"
+	gitlabSourceConfig.Domains.Enabled = []string{"new-source-test.gitlab.io"}
+	gitlabSourceConfig.Domains.Broken = "pages-broken-poc.gitlab.io"
 
 	t.Run("when requesting a test domain", func(t *testing.T) {
-		testDomain := newSourceDomains[0]
+		testDomain := gitlabSourceConfig.Domains.Enabled[0]
 
 		newSource := NewMockSource()
 		newSource.On("GetDomain", testDomain).