Merge branch 'sh-abort-upon-domain-scan-error' into 'master'

Abort domain scan if a failure is encountered

Closes gitlab-com/infrastructure#4749

See merge request gitlab-org/gitlab-pages!102

3 files changed, 35 insertions, 13 deletions

diff --git a/internal/domain/map.go b/internal/domain/map.go
index 943f5c20..30a8a74d 100644
--- a/internal/domain/map.go
+++ b/internal/domain/map.go
@@ -119,12 +119,7 @@ type jobResult struct {
 }
 
 // ReadGroups walks the pages directory and populates dm with all the domains it finds.
-func (dm Map) ReadGroups(rootDomain string) error {
-	fis, err := godirwalk.ReadDirents(".", nil)
-	if err != nil {
-		return err
-	}
-
+func (dm Map) ReadGroups(rootDomain string, fis godirwalk.Dirents) {
 	fanOutGroups := make(chan string)
 	fanIn := make(chan jobResult)
 	wg := &sync.WaitGroup{}
@@ -175,7 +170,6 @@ func (dm Map) ReadGroups(rootDomain string) error {
 	close(fanOutGroups)
 
 	<-done
-	return nil
 }
 
 const (
@@ -204,9 +198,15 @@ func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) {
 
 		started := time.Now()
 		dm := make(Map)
-		if err := dm.ReadGroups(rootDomain); err != nil {
+
+		fis, err := godirwalk.ReadDirents(".", nil)
+		if err != nil {
 			log.WithError(err).Warn("domain scan failed")
+			metrics.FailedDomainUpdates.Inc()
+			continue
 		}
+
+		dm.ReadGroups(rootDomain, fis)
 		duration := time.Since(started).Seconds()
 
 		var hash string
diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go
index f20f98bd..21d547ac 100644
--- a/internal/domain/map_test.go
+++ b/internal/domain/map_test.go
@@ -8,16 +8,32 @@ import (
 	"testing"
 	"time"
 
+	"github.com/karrick/godirwalk"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func getEntries(t *testing.T) godirwalk.Dirents {
+	fis, err := godirwalk.ReadDirents(".", nil)
+
+	require.NoError(t, err)
+
+	return fis
+}
+
+func getEntriesForBenchmark(t *testing.B) godirwalk.Dirents {
+	fis, err := godirwalk.ReadDirents(".", nil)
+
+	require.NoError(t, err)
+
+	return fis
+}
+
 func TestReadProjects(t *testing.T) {
 	setUpTests()
 
 	dm := make(Map)
-	err := dm.ReadGroups("test.io")
-	require.NoError(t, err)
+	dm.ReadGroups("test.io", getEntries(t))
 
 	var domains []string
 	for d := range dm {
@@ -141,7 +157,7 @@ func BenchmarkReadGroups(b *testing.B) {
 		var dm Map
 		for i := 0; i < 2; i++ {
 			dm = make(Map)
-			require.NoError(b, dm.ReadGroups("example.com"))
+			dm.ReadGroups("example.com", getEntriesForBenchmark(b))
 		}
 		b.Logf("found %d domains", len(dm))
 	})
diff --git a/metrics/metrics.go b/metrics/metrics.go
index b15c6711..44350ae5 100644
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@@ -11,10 +11,16 @@ var (
 		Help: "The total number of sites served by this Pages app",
 	})
 
-	// DomainUpdates counts the number of site updates processed
+	// FailedDomainUpdates counts the number of failed site updates
+	FailedDomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "gitlab_pages_domains_failed_total",
+		Help: "The total number of site updates that have failed since daemon start",
+	})
+
+	// DomainUpdates counts the number of site updates successfully processed
 	DomainUpdates = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "gitlab_pages_domains_updated_total",
-		Help: "The total number of site updates processed since daemon start",
+		Help: "The total number of site updates successfully processed since daemon start",
 	})
 
 	// DomainLastUpdateTime is the UNIX timestamp of the last update