Introduce a tree map for all content

This commit introduces a new data structure to store pages and their resources.

This data structure is backed by radix trees.

This simplies tree operations, makes all pages a bundle,  and paves the way for #6310.

It also solves a set of annoying issues (see list below).

Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use.

```
name                                   old time/op    new time/op    delta
SiteNew/Bundle_with_image/Edit-16        1.32ms ± 8%    1.00ms ± 9%  -24.42%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16    1.28ms ± 0%    0.94ms ± 0%  -26.26%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16      33.9ms ± 2%    21.8ms ± 1%  -35.67%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16            40.6ms ± 1%    37.7ms ± 3%   -7.20%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16        56.7ms ± 0%    51.7ms ± 1%   -8.82%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16      19.9ms ± 2%    18.3ms ± 3%   -7.64%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16         37.9ms ± 4%    34.0ms ± 2%  -10.28%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16             10.7ms ± 0%    10.6ms ± 0%   -1.15%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16         10.8ms ± 0%    10.7ms ± 0%   -1.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16           43.2ms ± 1%    39.6ms ± 1%   -8.35%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                 47.6ms ± 1%    47.3ms ± 0%     ~     (p=0.057 n=4+4)
SiteNew/Deep_content_tree-16             73.0ms ± 1%    74.2ms ± 1%     ~     (p=0.114 n=4+4)
SiteNew/Many_HTML_templates-16           37.9ms ± 0%    38.1ms ± 1%     ~     (p=0.114 n=4+4)
SiteNew/Page_collections-16              53.6ms ± 1%    54.7ms ± 1%   +2.09%  (p=0.029 n=4+4)

name                                   old alloc/op   new alloc/op   delta
SiteNew/Bundle_with_image/Edit-16         486kB ± 0%     430kB ± 0%  -11.47%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16     265kB ± 0%     209kB ± 0%  -21.06%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16      13.6MB ± 0%     8.8MB ± 0%  -34.93%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16            66.5MB ± 0%    63.9MB ± 0%   -3.95%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16        28.8MB ± 0%    25.8MB ± 0%  -10.55%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16      6.16MB ± 0%    5.56MB ± 0%   -9.86%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16         16.9MB ± 0%    16.0MB ± 0%   -5.19%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16             2.28MB ± 0%    2.29MB ± 0%   +0.35%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16         2.07MB ± 0%    2.07MB ± 0%     ~     (p=0.114 n=4+4)
SiteNew/Tags_and_categories-16           14.3MB ± 0%    13.2MB ± 0%   -7.30%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                 69.1MB ± 0%    69.0MB ± 0%     ~     (p=0.343 n=4+4)
SiteNew/Deep_content_tree-16             31.3MB ± 0%    31.8MB ± 0%   +1.49%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16           10.8MB ± 0%    10.9MB ± 0%   +1.11%  (p=0.029 n=4+4)
SiteNew/Page_collections-16              21.4MB ± 0%    21.6MB ± 0%   +1.15%  (p=0.029 n=4+4)

name                                   old allocs/op  new allocs/op  delta
SiteNew/Bundle_with_image/Edit-16         4.74k ± 0%     3.86k ± 0%  -18.57%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file/Edit-16     4.73k ± 0%     3.85k ± 0%  -18.58%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories/Edit-16        301k ± 0%      198k ± 0%  -34.14%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs/Edit-16              389k ± 0%      373k ± 0%   -4.07%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree/Edit-16          338k ± 0%      262k ± 0%  -22.63%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates/Edit-16        102k ± 0%       88k ± 0%  -13.81%  (p=0.029 n=4+4)
SiteNew/Page_collections/Edit-16           176k ± 0%      152k ± 0%  -13.32%  (p=0.029 n=4+4)
SiteNew/Bundle_with_image-16              26.8k ± 0%     26.8k ± 0%   +0.05%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16          26.8k ± 0%     26.8k ± 0%   +0.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16             273k ± 0%      245k ± 0%  -10.36%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16                   396k ± 0%      398k ± 0%   +0.39%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16               317k ± 0%      325k ± 0%   +2.53%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16             146k ± 0%      147k ± 0%   +0.98%  (p=0.029 n=4+4)
SiteNew/Page_collections-16                210k ± 0%      215k ± 0%   +2.44%  (p=0.029 n=4+4)
```

Fixes #6312
Fixes #6087
Fixes #6738
Fixes #6412
Fixes #6743
Fixes #6875
Fixes #6034
Fixes #6902
Fixes #6173
Fixes #6590

1 files changed, 998 insertions, 0 deletions

diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go
new file mode 100644
index 000000000..3269abe12
--- /dev/null
+++ b/hugolib/content_map_page.go
@@ -0,0 +1,998 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/gohugoio/hugo/common/maps"
+
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/resources"
+
+	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/hugofs/files"
+	"github.com/gohugoio/hugo/parser/pageparser"
+	"github.com/gohugoio/hugo/resources/page"
+	"github.com/gohugoio/hugo/resources/resource"
+	"github.com/spf13/cast"
+
+	"github.com/gohugoio/hugo/common/para"
+	"github.com/pkg/errors"
+)
+
+func newPageMaps(h *HugoSites) *pageMaps {
+	mps := make([]*pageMap, len(h.Sites))
+	for i, s := range h.Sites {
+		mps[i] = s.pageMap
+	}
+	return &pageMaps{
+		workers: para.New(h.numWorkers),
+		pmaps:   mps,
+	}
+
+}
+
+type pageMap struct {
+	s *Site
+	*contentMap
+}
+
+func (m *pageMap) Len() int {
+	l := 0
+	for _, t := range m.contentMap.pageTrees {
+		l += t.Len()
+	}
+	return l
+}
+
+func (m *pageMap) createMissingTaxonomyNodes() error {
+	if m.cfg.taxonomyDisabled {
+		return nil
+	}
+	m.taxonomyEntries.Walk(func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+		vi := n.viewInfo
+		k := cleanTreeKey(vi.name.plural + "/" + vi.termKey)
+
+		if _, found := m.taxonomies.Get(k); !found {
+			vic := &contentBundleViewInfo{
+				name:       vi.name,
+				termKey:    vi.termKey,
+				termOrigin: vi.termOrigin,
+			}
+			m.taxonomies.Insert(k, &contentNode{viewInfo: vic})
+		}
+		return false
+	})
+
+	return nil
+}
+
+func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapBucket, owner *pageState) (*pageState, error) {
+	if n.fi == nil {
+		panic("FileInfo must (currently) be set")
+	}
+
+	f, err := newFileInfo(m.s.SourceSpec, n.fi)
+	if err != nil {
+		return nil, err
+	}
+
+	meta := n.fi.Meta()
+	content := func() (hugio.ReadSeekCloser, error) {
+		return meta.Open()
+	}
+
+	bundled := owner != nil
+	s := m.s
+
+	sections := s.sectionsFromFile(f)
+
+	kind := s.kindFromFileInfoOrSections(f, sections)
+	if kind == page.KindTaxonomy {
+		s.PathSpec.MakePathsSanitized(sections)
+	}
+
+	metaProvider := &pageMeta{kind: kind, sections: sections, bundled: bundled, s: s, f: f}
+
+	ps, err := newPageBase(metaProvider)
+	if err != nil {
+		return nil, err
+	}
+
+	if n.fi.Meta().GetBool(walkIsRootFileMetaKey) {
+		// Make sure that the bundle/section we start walking from is always
+		// rendered.
+		// This is only relevant in server fast render mode.
+		ps.forceRender = true
+	}
+
+	n.p = ps
+	if ps.IsNode() {
+		ps.bucket = newPageBucket(ps)
+	}
+
+	gi, err := s.h.gitInfoForPage(ps)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load Git data")
+	}
+	ps.gitInfo = gi
+
+	r, err := content()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	parseResult, err := pageparser.Parse(
+		r,
+		pageparser.Config{EnableEmoji: s.siteCfg.enableEmoji},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	ps.pageContent = pageContent{
+		source: rawPageContent{
+			parsed:         parseResult,
+			posMainContent: -1,
+			posSummaryEnd:  -1,
+			posBodyStart:   -1,
+		},
+	}
+
+	ps.shortcodeState = newShortcodeHandler(ps, ps.s, nil)
+
+	if err := ps.mapContent(parentBucket, metaProvider); err != nil {
+		return nil, ps.wrapError(err)
+	}
+
+	if err := metaProvider.applyDefaultValues(n); err != nil {
+		return nil, err
+	}
+
+	ps.init.Add(func() (interface{}, error) {
+		pp, err := newPagePaths(s, ps, metaProvider)
+		if err != nil {
+			return nil, err
+		}
+
+		outputFormatsForPage := ps.m.outputFormats()
+
+		if !ps.m.noRender() {
+			// Prepare output formats for all sites.
+			ps.pageOutputs = make([]*pageOutput, len(ps.s.h.renderFormats))
+			created := make(map[string]*pageOutput)
+
+			for i, f := range ps.s.h.renderFormats {
+				if po, found := created[f.Name]; found {
+					ps.pageOutputs[i] = po
+					continue
+				}
+
+				_, render := outputFormatsForPage.GetByName(f.Name)
+				po := newPageOutput(ps, pp, f, render)
+
+				// Create a content provider for the first,
+				// we may be able to reuse it.
+				if i == 0 {
+					contentProvider, err := newPageContentOutput(ps, po)
+					if err != nil {
+						return nil, err
+					}
+					po.initContentProvider(contentProvider)
+				}
+
+				ps.pageOutputs[i] = po
+				created[f.Name] = po
+			}
+		} else if ps.m.buildConfig.PublishResources {
+			// We need one output format for potential resources to publish.
+			po := newPageOutput(ps, pp, outputFormatsForPage[0], false)
+			contentProvider, err := newPageContentOutput(ps, po)
+			if err != nil {
+				return nil, err
+			}
+			po.initContentProvider(contentProvider)
+			ps.pageOutputs = []*pageOutput{po}
+		}
+
+		if err := ps.initCommonProviders(pp); err != nil {
+			return nil, err
+		}
+
+		return nil, nil
+	})
+
+	ps.parent = owner
+
+	return ps, nil
+}
+
+func (m *pageMap) newResource(fim hugofs.FileMetaInfo, owner *pageState) (resource.Resource, error) {
+
+	if owner == nil {
+		panic("owner is nil")
+	}
+	// TODO(bep) consolidate with multihost logic + clean up
+	outputFormats := owner.m.outputFormats()
+	seen := make(map[string]bool)
+	var targetBasePaths []string
+	// Make sure bundled resources are published to all of the ouptput formats'
+	// sub paths.
+	for _, f := range outputFormats {
+		p := f.Path
+		if seen[p] {
+			continue
+		}
+		seen[p] = true
+		targetBasePaths = append(targetBasePaths, p)
+
+	}
+
+	meta := fim.Meta()
+	r := func() (hugio.ReadSeekCloser, error) {
+		return meta.Open()
+	}
+
+	target := strings.TrimPrefix(meta.Path(), owner.File().Dir())
+
+	return owner.s.ResourceSpec.New(
+		resources.ResourceSourceDescriptor{
+			TargetPaths:        owner.getTargetPaths,
+			OpenReadSeekCloser: r,
+			FileInfo:           fim,
+			RelTargetFilename:  target,
+			TargetBasePaths:    targetBasePaths,
+		})
+}
+
+func (m *pageMap) createSiteTaxonomies() error {
+	m.s.taxonomies = make(TaxonomyList)
+	m.taxonomies.Walk(func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+		t := n.viewInfo
+
+		viewName := t.name
+
+		if t.termKey == "" {
+			m.s.taxonomies[viewName.plural] = make(Taxonomy)
+		} else {
+			taxonomy := m.s.taxonomies[viewName.plural]
+			m.taxonomyEntries.WalkPrefix(s+"/", func(ss string, v interface{}) bool {
+				b2 := v.(*contentNode)
+				info := b2.viewInfo
+				taxonomy.add(info.termKey, page.NewWeightedPage(info.weight, info.ref.p, n.p))
+
+				return false
+			})
+		}
+
+		return false
+	})
+
+	for _, taxonomy := range m.s.taxonomies {
+		for _, v := range taxonomy {
+			v.Sort()
+		}
+	}
+
+	return nil
+}
+
+func (m *pageMap) createListAllPages() page.Pages {
+	pages := make(page.Pages, 0)
+
+	m.contentMap.pageTrees.Walk(func(s string, n *contentNode) bool {
+		if n.p == nil {
+			panic(fmt.Sprintf("BUG: page not set for %q", s))
+		}
+		if contentTreeNoListFilter(s, n) {
+			return false
+		}
+		pages = append(pages, n.p)
+		return false
+	})
+
+	page.SortByDefault(pages)
+	return pages
+}
+
+func (m *pageMap) assemblePages() error {
+	m.taxonomyEntries.DeletePrefix("/")
+
+	if err := m.assembleSections(); err != nil {
+		return err
+	}
+
+	var err error
+
+	if err != nil {
+		return err
+	}
+
+	m.pages.Walk(func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+
+		var shouldBuild bool
+
+		defer func() {
+			// Make sure we always rebuild the view cache.
+			if shouldBuild && err == nil && n.p != nil {
+				m.attachPageToViews(s, n)
+			}
+		}()
+
+		if n.p != nil {
+			// A rebuild
+			shouldBuild = true
+			return false
+		}
+
+		var parent *contentNode
+		var parentBucket *pagesMapBucket
+
+		_, parent = m.getSection(s)
+		if parent == nil {
+			panic(fmt.Sprintf("BUG: parent not set for %q", s))
+		}
+		parentBucket = parent.p.bucket
+
+		n.p, err = m.newPageFromContentNode(n, parentBucket, nil)
+		if err != nil {
+			return true
+		}
+
+		shouldBuild = !(n.p.Kind() == page.KindPage && m.cfg.pageDisabled) && m.s.shouldBuild(n.p)
+		if !shouldBuild {
+			m.deletePage(s)
+			return false
+		}
+
+		n.p.treeRef = &contentTreeRef{
+			m:   m,
+			t:   m.pages,
+			n:   n,
+			key: s,
+		}
+
+		if err = m.assembleResources(s, n.p, parentBucket); err != nil {
+			return true
+		}
+
+		return false
+	})
+
+	m.deleteOrphanSections()
+
+	return err
+}
+
+func (m *pageMap) assembleResources(s string, p *pageState, parentBucket *pagesMapBucket) error {
+	var err error
+
+	m.resources.WalkPrefix(s, func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+		meta := n.fi.Meta()
+		classifier := meta.Classifier()
+		var r resource.Resource
+		switch classifier {
+		case files.ContentClassContent:
+			var rp *pageState
+			rp, err = m.newPageFromContentNode(n, parentBucket, p)
+			if err != nil {
+				return true
+			}
+			rp.m.resourcePath = filepath.ToSlash(strings.TrimPrefix(rp.Path(), p.File().Dir()))
+			r = rp
+
+		case files.ContentClassFile:
+			r, err = m.newResource(n.fi, p)
+			if err != nil {
+				return true
+			}
+		default:
+			panic(fmt.Sprintf("invalid classifier: %q", classifier))
+		}
+
+		p.resources = append(p.resources, r)
+		return false
+	})
+
+	return err
+}
+
+func (m *pageMap) assembleSections() error {
+
+	var sectionsToDelete []string
+	var err error
+
+	m.sections.Walk(func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+
+		var shouldBuild bool
+
+		defer func() {
+			// Make sure we always rebuild the view cache.
+			if shouldBuild && err == nil && n.p != nil {
+				m.attachPageToViews(s, n)
+				if n.p.IsHome() {
+					m.s.home = n.p
+				}
+			}
+		}()
+
+		sections := m.splitKey(s)
+
+		if n.p != nil {
+			if n.p.IsHome() {
+				m.s.home = n.p
+			}
+			shouldBuild = true
+			return false
+		}
+
+		var parent *contentNode
+		var parentBucket *pagesMapBucket
+
+		if s != "/" {
+			_, parent = m.getSection(s)
+			if parent == nil || parent.p == nil {
+				panic(fmt.Sprintf("BUG: parent not set for %q", s))
+			}
+		}
+
+		if parent != nil {
+			parentBucket = parent.p.bucket
+		}
+
+		kind := page.KindSection
+		if s == "/" {
+			kind = page.KindHome
+		}
+
+		if n.fi != nil {
+			n.p, err = m.newPageFromContentNode(n, parentBucket, nil)
+			if err != nil {
+				return true
+			}
+		} else {
+			n.p = m.s.newPage(n, parentBucket, kind, "", sections...)
+		}
+
+		shouldBuild = m.s.shouldBuild(n.p)
+		if !shouldBuild {
+			sectionsToDelete = append(sectionsToDelete, s)
+			return false
+		}
+
+		n.p.treeRef = &contentTreeRef{
+			m:   m,
+			t:   m.sections,
+			n:   n,
+			key: s,
+		}
+
+		if err = m.assembleResources(s+cmLeafSeparator, n.p, parentBucket); err != nil {
+			return true
+		}
+
+		return false
+	})
+
+	for _, s := range sectionsToDelete {
+		m.deleteSectionByPath(s)
+	}
+
+	return err
+}
+
+func (m *pageMap) assembleTaxonomies() error {
+
+	var taxonomiesToDelete []string
+	var err error
+
+	m.taxonomies.Walk(func(s string, v interface{}) bool {
+		n := v.(*contentNode)
+
+		if n.p != nil {
+			return false
+		}
+
+		kind := n.viewInfo.kind()
+		sections := n.viewInfo.sections()
+
+		_, parent := m.getTaxonomyParent(s)
+		if parent == nil || parent.p == nil {
+			panic(fmt.Sprintf("BUG: parent not set for %q", s))
+		}
+		parentBucket := parent.p.bucket
+
+		if n.fi != nil {
+			n.p, err = m.newPageFromContentNode(n, parent.p.bucket, nil)
+			if err != nil {
+				return true
+			}
+		} else {
+			title := ""
+			if kind == page.KindTaxonomy {
+				title = n.viewInfo.term()
+			}
+			n.p = m.s.newPage(n, parent.p.bucket, kind, title, sections...)
+		}
+
+		if !m.s.shouldBuild(n.p) {
+			taxonomiesToDelete = append(taxonomiesToDelete, s)
+			return false
+		}
+
+		n.p.treeRef = &contentTreeRef{
+			m:   m,
+			t:   m.taxonomies,
+			n:   n,
+			key: s,
+		}
+
+		if err = m.assembleResources(s+cmLeafSeparator, n.p, parentBucket); err != nil {
+			return true
+		}
+
+		return false
+	})
+
+	for _, s := range taxonomiesToDelete {
+		m.deleteTaxonomy(s)
+	}
+
+	return err
+
+}
+
+func (m *pageMap) attachPageToViews(s string, b *contentNode) {
+	if m.cfg.taxonomyDisabled {
+		return
+	}
+
+	for _, viewName := range m.cfg.taxonomyConfig {
+		vals := types.ToStringSlicePreserveString(getParam(b.p, viewName.plural, false))
+		if vals == nil {
+			continue
+		}
+
+		w := getParamToLower(b.p, viewName.plural+"_weight")
+		weight, err := cast.ToIntE(w)
+		if err != nil {
+			m.s.Log.ERROR.Printf("Unable to convert taxonomy weight %#v to int for %q", w, b.p.Path())
+			// weight will equal zero, so let the flow continue
+		}
+
+		for _, v := range vals {
+			termKey := m.s.getTaxonomyKey(v)
+
+			bv := &contentNode{
+				viewInfo: &contentBundleViewInfo{
+					name:       viewName,
+					termKey:    termKey,
+					termOrigin: v,
+					weight:     weight,
+					ref:        b,
+				},
+			}
+
+			if s == "/" {
+				// To avoid getting an empty key.
+				s = "home"
+			}
+			key := cleanTreeKey(path.Join(viewName.plural, termKey, s))
+			m.taxonomyEntries.Insert(key, bv)
+		}
+	}
+}
+
+func (m *pageMap) collectPages(prefix string, fn func(c *contentNode)) error {
+	m.pages.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+		fn(n)
+		return false
+	})
+	return nil
+}
+
+func (m *pageMap) collectPagesAndSections(prefix string, fn func(c *contentNode)) error {
+	if err := m.collectSections(prefix, fn); err != nil {
+		return err
+	}
+
+	if err := m.collectPages(prefix+cmBranchSeparator, fn); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *pageMap) collectSections(prefix string, fn func(c *contentNode)) error {
+	var level int
+	isHome := prefix == "/"
+
+	if !isHome {
+		level = strings.Count(prefix, "/")
+	}
+
+	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+		if s == prefix {
+			return false
+		}
+
+		if (strings.Count(s, "/") - level) != 1 {
+			return false
+		}
+
+		fn(c)
+
+		return false
+	})
+}
+
+func (m *pageMap) collectSectionsFn(prefix string, fn func(s string, c *contentNode) bool) error {
+	if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	m.sections.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+		return fn(s, n)
+	})
+
+	return nil
+}
+
+func (m *pageMap) collectSectionsRecursiveIncludingSelf(prefix string, fn func(c *contentNode)) error {
+	return m.collectSectionsFn(prefix, func(s string, c *contentNode) bool {
+		fn(c)
+		return false
+	})
+}
+
+func (m *pageMap) collectTaxonomies(prefix string, fn func(c *contentNode)) error {
+	m.taxonomies.WalkPrefixListable(prefix, func(s string, n *contentNode) bool {
+		fn(n)
+		return false
+	})
+	return nil
+}
+
+// withEveryBundlePage applies fn to every Page, including those bundled inside
+// leaf bundles.
+func (m *pageMap) withEveryBundlePage(fn func(p *pageState) bool) {
+	m.bundleTrees.Walk(func(s string, n *contentNode) bool {
+		if n.p != nil {
+			return fn(n.p)
+		}
+		return false
+	})
+}
+
+type pageMaps struct {
+	workers *para.Workers
+	pmaps   []*pageMap
+}
+
+// deleteSection deletes the entire section from s.
+func (m *pageMaps) deleteSection(s string) {
+	m.withMaps(func(pm *pageMap) error {
+		pm.deleteSectionByPath(s)
+		return nil
+	})
+}
+
+func (m *pageMaps) AssemblePages() error {
+	return m.withMaps(func(pm *pageMap) error {
+		if err := pm.CreateMissingNodes(); err != nil {
+			return err
+		}
+
+		if err := pm.assemblePages(); err != nil {
+			return err
+		}
+
+		if err := pm.createMissingTaxonomyNodes(); err != nil {
+			return err
+		}
+
+		// Handle any new sections created in the step above.
+		if err := pm.assembleSections(); err != nil {
+			return err
+		}
+
+		if err := pm.assembleTaxonomies(); err != nil {
+			return err
+		}
+
+		if err := pm.createSiteTaxonomies(); err != nil {
+			return err
+		}
+
+		a := (&sectionWalker{m: pm.contentMap}).applyAggregates()
+		_, mainSectionsSet := pm.s.s.Info.Params()["mainsections"]
+		if !mainSectionsSet && a.mainSection != "" {
+			mainSections := []string{a.mainSection}
+			pm.s.s.Info.Params()["mainSections"] = mainSections
+			pm.s.s.Info.Params()["mainsections"] = mainSections
+		}
+
+		pm.s.lastmod = a.datesAll.Lastmod()
+		if resource.IsZeroDates(pm.s.home) {
+			pm.s.home.m.Dates = a.datesAll
+		}
+
+		return nil
+	})
+}
+
+func (m *pageMaps) walkBundles(fn func(n *contentNode) bool) {
+	_ = m.withMaps(func(pm *pageMap) error {
+		pm.bundleTrees.Walk(func(s string, n *contentNode) bool {
+			return fn(n)
+		})
+		return nil
+	})
+}
+
+func (m *pageMaps) walkBranchesPrefix(prefix string, fn func(s string, n *contentNode) bool) {
+	_ = m.withMaps(func(pm *pageMap) error {
+		pm.branchTrees.WalkPrefix(prefix, func(s string, n *contentNode) bool {
+			return fn(s, n)
+		})
+		return nil
+	})
+}
+
+func (m *pageMaps) withMaps(fn func(pm *pageMap) error) error {
+	g, _ := m.workers.Start(context.Background())
+	for _, pm := range m.pmaps {
+		pm := pm
+		g.Run(func() error {
+			return fn(pm)
+		})
+	}
+	return g.Wait()
+}
+
+type pagesMapBucket struct {
+	// Cascading front matter.
+	cascade maps.Params
+
+	owner *pageState // The branch node
+
+	pagesInit sync.Once
+	pages     page.Pages
+
+	pagesAndSectionsInit sync.Once
+	pagesAndSections     page.Pages
+
+	sectionsInit sync.Once
+	sections     page.Pages
+}
+
+func (b *pagesMapBucket) getPages() page.Pages {
+	b.pagesInit.Do(func() {
+		b.pages = b.owner.treeRef.collectPages()
+		page.SortByDefault(b.pages)
+	})
+	return b.pages
+}
+
+func (b *pagesMapBucket) getPagesAndSections() page.Pages {
+	b.pagesAndSectionsInit.Do(func() {
+		b.pagesAndSections = b.owner.treeRef.collectPagesAndSections()
+	})
+	return b.pagesAndSections
+}
+
+func (b *pagesMapBucket) getSections() page.Pages {
+	b.sectionsInit.Do(func() {
+		b.sections = b.owner.treeRef.collectSections()
+	})
+
+	return b.sections
+}
+
+func (b *pagesMapBucket) getTaxonomies() page.Pages {
+	b.sectionsInit.Do(func() {
+		var pas page.Pages
+		ref := b.owner.treeRef
+		ref.m.collectTaxonomies(ref.key+"/", func(c *contentNode) {
+			pas = append(pas, c.p)
+		})
+		page.SortByDefault(pas)
+		b.sections = pas
+	})
+
+	return b.sections
+}
+
+type sectionAggregate struct {
+	datesAll             resource.Dates
+	datesSection         resource.Dates
+	pageCount            int
+	mainSection          string
+	mainSectionPageCount int
+}
+
+type sectionAggregateHandler struct {
+	sectionAggregate
+	sectionPageCount int
+
+	// Section
+	b *contentNode
+	s string
+}
+
+func (h *sectionAggregateHandler) isRootSection() bool {
+	return h.s != "/" && strings.Count(h.s, "/") == 1
+}
+
+func (h *sectionAggregateHandler) handleNested(v sectionWalkHandler) error {
+	nested := v.(*sectionAggregateHandler)
+	h.sectionPageCount += nested.pageCount
+	h.pageCount += h.sectionPageCount
+	h.datesAll.UpdateDateAndLastmodIfAfter(nested.datesAll)
+	h.datesSection.UpdateDateAndLastmodIfAfter(nested.datesAll)
+	return nil
+}
+
+func (h *sectionAggregateHandler) handlePage(s string, n *contentNode) error {
+	h.sectionPageCount++
+
+	var d resource.Dated
+	if n.p != nil {
+		d = n.p
+	} else if n.viewInfo != nil && n.viewInfo.ref != nil {
+		d = n.viewInfo.ref.p
+	} else {
+		return nil
+	}
+
+	h.datesAll.UpdateDateAndLastmodIfAfter(d)
+	h.datesSection.UpdateDateAndLastmodIfAfter(d)
+	return nil
+}
+
+func (h *sectionAggregateHandler) handleSectionPost() error {
+	if h.sectionPageCount > h.mainSectionPageCount && h.isRootSection() {
+		h.mainSectionPageCount = h.sectionPageCount
+		h.mainSection = strings.TrimPrefix(h.s, "/")
+	}
+
+	if resource.IsZeroDates(h.b.p) {
+		h.b.p.m.Dates = h.datesSection
+	}
+
+	h.datesSection = resource.Dates{}
+
+	return nil
+}
+
+func (h *sectionAggregateHandler) handleSectionPre(s string, b *contentNode) error {
+	h.s = s
+	h.b = b
+	h.sectionPageCount = 0
+	h.datesAll.UpdateDateAndLastmodIfAfter(b.p)
+	return nil
+}
+
+type sectionWalkHandler interface {
+	handleNested(v sectionWalkHandler) error
+	handlePage(s string, b *contentNode) error
+	handleSectionPost() error
+	handleSectionPre(s string, b *contentNode) error
+}
+
+type sectionWalker struct {
+	err error
+	m   *contentMap
+}
+
+func (w *sectionWalker) applyAggregates() *sectionAggregateHandler {
+	return w.walkLevel("/", func() sectionWalkHandler {
+		return &sectionAggregateHandler{}
+	}).(*sectionAggregateHandler)
+
+}
+
+func (w *sectionWalker) walkLevel(prefix string, createVisitor func() sectionWalkHandler) sectionWalkHandler {
+
+	level := strings.Count(prefix, "/")
+	visitor := createVisitor()
+
+	w.m.taxonomies.WalkPrefix(prefix, func(s string, v interface{}) bool {
+		currentLevel := strings.Count(s, "/")
+		if currentLevel > level {
+			return false
+		}
+
+		n := v.(*contentNode)
+
+		if w.err = visitor.handleSectionPre(s, n); w.err != nil {
+			return true
+		}
+
+		if currentLevel == 1 {
+			nested := w.walkLevel(s+"/", createVisitor)
+			if w.err = visitor.handleNested(nested); w.err != nil {
+				return true
+			}
+		} else {
+			w.m.taxonomyEntries.WalkPrefix(s, func(ss string, v interface{}) bool {
+				n := v.(*contentNode)
+				w.err = visitor.handlePage(ss, n)
+				return w.err != nil
+			})
+		}
+
+		w.err = visitor.handleSectionPost()
+
+		return w.err != nil
+	})
+
+	w.m.sections.WalkPrefix(prefix, func(s string, v interface{}) bool {
+		currentLevel := strings.Count(s, "/")
+		if currentLevel > level {
+			return false
+		}
+
+		n := v.(*contentNode)
+
+		if w.err = visitor.handleSectionPre(s, n); w.err != nil {
+			return true
+		}
+
+		w.m.pages.WalkPrefix(s+cmBranchSeparator, func(s string, v interface{}) bool {
+			w.err = visitor.handlePage(s, v.(*contentNode))
+			return w.err != nil
+		})
+
+		if w.err != nil {
+			return true
+		}
+
+		if s != "/" {
+			nested := w.walkLevel(s+"/", createVisitor)
+			if w.err = visitor.handleNested(nested); w.err != nil {
+				return true
+			}
+		}
+
+		w.err = visitor.handleSectionPost()
+
+		return w.err != nil
+	})
+
+	return visitor
+
+}
+
+type viewName struct {
+	singular string // e.g. "category"
+	plural   string // e.g. "categories"
+}
+
+func (v viewName) IsZero() bool {
+	return v.singular == ""
+}