diff options
author | Stan Hu <stanhu@gmail.com> | 2018-08-15 16:59:24 +0300 |
---|---|---|
committer | Stan Hu <stanhu@gmail.com> | 2018-08-15 16:59:24 +0300 |
commit | 661e3d8dfc0b1ac28f8ae062d2e2faf1324f223f (patch) | |
tree | 404e4d87d87d877aadcd60d450931f58d9dc393c /internal/domain | |
parent | 24204ec7eed6359aa3aaaca4073cbbe2ced4817c (diff) |
Make it explicit that when ReadGroups fails initially, an error is logged
Diffstat (limited to 'internal/domain')
-rw-r--r-- | internal/domain/map.go | 13 | ||||
-rw-r--r-- | internal/domain/map_test.go | 21 |
2 files changed, 25 insertions, 9 deletions
diff --git a/internal/domain/map.go b/internal/domain/map.go index d4a9764c..a2c49317 100644 --- a/internal/domain/map.go +++ b/internal/domain/map.go @@ -119,12 +119,7 @@ type jobResult struct { } // ReadGroups walks the pages directory and populates dm with all the domains it finds. -func (dm Map) ReadGroups(rootDomain string) error { - fis, err := godirwalk.ReadDirents(".", nil) - if err != nil { - return err - } - +func (dm Map) ReadGroups(rootDomain string, fis godirwalk.Dirents) error { fanOutGroups := make(chan string) fanIn := make(chan jobResult) wg := &sync.WaitGroup{} @@ -204,11 +199,15 @@ func Watch(rootDomain string, updater domainsUpdater, interval time.Duration) { started := time.Now() dm := make(Map) - if err := dm.ReadGroups(rootDomain); err != nil { + + fis, err := godirwalk.ReadDirents(".", nil) + if err != nil { log.WithError(err).Warn("domain scan failed") metrics.FailedDomainUpdates.Inc() continue } + + dm.ReadGroups(rootDomain, fis) duration := time.Since(started).Seconds() var hash string diff --git a/internal/domain/map_test.go b/internal/domain/map_test.go index f20f98bd..96dab65f 100644 --- a/internal/domain/map_test.go +++ b/internal/domain/map_test.go @@ -8,15 +8,32 @@ import ( "testing" "time" + "github.com/karrick/godirwalk" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func getEntries(t *testing.T) godirwalk.Dirents { + fis, err := godirwalk.ReadDirents(".", nil) + + require.NoError(t, err) + + return fis +} + +func getEntriesForBenchmark(t *testing.B) godirwalk.Dirents { + fis, err := godirwalk.ReadDirents(".", nil) + + require.NoError(t, err) + + return fis +} + func TestReadProjects(t *testing.T) { setUpTests() dm := make(Map) - err := dm.ReadGroups("test.io") + err := dm.ReadGroups("test.io", getEntries(t)) require.NoError(t, err) var domains []string @@ -141,7 +158,7 @@ func BenchmarkReadGroups(b *testing.B) { var dm Map for i := 0; i < 2; i++ { dm = make(Map) - require.NoError(b, dm.ReadGroups("example.com")) + require.NoError(b, dm.ReadGroups("example.com", getEntriesForBenchmark(b))) } b.Logf("found %d domains", len(dm)) }) |