Speedup and simplify page assembly for deeper content trees

This commit moves to a forked version go-radix (fork source has not had any updates in 3 years.), whith 2 notable changes:

* It's generic (using Go generics) and thus removes a lot of type conversions/assertions.
* It allows nodes to be replaced during walk, which allows to partition the tree for parallel processing without worrying about locking.

For this repo, this means:

* The assembly step now processes nested sections in parallel, which gives a speedup for deep content trees with a slight allocation penalty (see benchmarks below).
* Nodes that needs to be reinserted are inserted directly.
* Also, there are some drive-by fixes of some allocation issues, e.g. avoid wrapping mutexes in returned anonomous functions, a common source of hidden allocations.

```
                                                                                   │ master.bench │           perf-p3.bench            │
                                                                                   │    sec/op    │   sec/op     vs base               │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10     6.958m ± 3%   7.015m ± 3%        ~ (p=0.589 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10    14.25m ± 1%   14.56m ± 8%        ~ (p=0.394 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10    48.07m ± 3%   49.23m ± 3%        ~ (p=0.394 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10    66.66m ± 4%   66.47m ± 6%        ~ (p=0.485 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10    59.57m ± 4%   50.73m ± 5%  -14.85% (p=0.002 n=6)
geomean                                                                               28.54m        27.92m        -2.18%

                                                                                   │ master.bench │           perf-p3.bench            │
                                                                                   │     B/op     │     B/op      vs base              │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10    4.513Mi ± 0%   4.527Mi ± 0%  +0.33% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10   15.35Mi ± 0%   15.49Mi ± 0%  +0.94% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10   62.50Mi ± 0%   63.19Mi ± 0%  +1.10% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10   86.78Mi ± 0%   87.73Mi ± 0%  +1.09% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10   62.96Mi ± 0%   63.66Mi ± 0%  +1.12% (p=0.002 n=6)
geomean                                                                              29.84Mi        30.11Mi       +0.92%

                                                                                   │ master.bench │           perf-p3.bench           │
                                                                                   │  allocs/op   │  allocs/op   vs base              │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10     60.44k ± 0%   60.97k ± 0%  +0.87% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10    205.8k ± 0%   211.4k ± 0%  +2.70% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10    831.1k ± 0%   858.3k ± 0%  +3.27% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10    1.157M ± 0%   1.197M ± 0%  +3.41% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10    839.9k ± 0%   867.8k ± 0%  +3.31% (p=0.002 n=6)
geomean                                                                               398.5k        409.3k       +2.71%
```
This commit is contained in:
Bjørn Erik Pedersen
2025-11-21 14:20:59 +01:00
parent 3d21b0687b
commit 555dfa207a
20 changed files with 458 additions and 472 deletions

View File

@@ -97,6 +97,16 @@ func GetOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) {
return reCache.getOrCompileRegexp(pattern)
}
// HasAnyPrefix checks if the string s has any of the prefixes given.
func HasAnyPrefix(s string, prefixes ...string) bool {
for _, p := range prefixes {
if strings.HasPrefix(s, p) {
return true
}
}
return false
}
// InSlice checks if a string is an element of a slice of strings
// and returns a boolean value.
func InSlice(arr []string, el string) bool {

2
go.mod
View File

@@ -4,7 +4,6 @@ require (
github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69
github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0
github.com/alecthomas/chroma/v2 v2.20.0
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c
github.com/aws/aws-sdk-go-v2 v1.40.0
github.com/aws/aws-sdk-go-v2/service/cloudfront v1.57.0
github.com/bep/clocks v0.5.0
@@ -38,6 +37,7 @@ require (
github.com/gobwas/glob v0.2.3
github.com/goccy/go-yaml v1.18.0
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6
github.com/gohugoio/go-radix v1.2.0
github.com/gohugoio/hashstructure v0.6.0
github.com/gohugoio/httpcache v0.8.0
github.com/gohugoio/hugo-goldmark-extensions/extras v0.5.0

4
go.sum
View File

@@ -102,8 +102,6 @@ github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NT
github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
@@ -276,6 +274,8 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6 h1:pxlAea9eRwuAnt/zKbGqlFO2ZszpIe24YpOVLf+N+4I=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6/go.mod h1:m5hu1im5Qc7LDycVLvee6MPobJiRLBYHklypFJR0/aE=
github.com/gohugoio/go-radix v1.2.0 h1:D5GTk8jIoeXirBSc2P4E4NdHKDrenk9k9N0ctU5Yrhg=
github.com/gohugoio/go-radix v1.2.0/go.mod h1:k6vDa0ebpbpgtzSj9lPGJcA4AZwJ9xUNObUy2vczPFM=
github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=
github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=
github.com/gohugoio/httpcache v0.8.0 h1:hNdsmGSELztetYCsPVgjA960zSa4dfEqqF/SficorCU=

View File

@@ -30,7 +30,7 @@ import (
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs/hglob"
radix "github.com/armon/go-radix"
radix "github.com/gohugoio/go-radix"
"github.com/spf13/afero"
)
@@ -42,17 +42,12 @@ var _ ReverseLookupProvder = (*RootMappingFs)(nil)
// root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...*RootMapping) (*RootMappingFs, error) {
rootMapToReal := radix.New()
realMapToRoot := radix.New()
rootMapToReal := radix.New[[]*RootMapping]()
realMapToRoot := radix.New[[]*RootMapping]()
id := fmt.Sprintf("rfs-%d", rootMappingFsCounter.Add(1))
addMapping := func(key string, rm *RootMapping, to *radix.Tree) {
var mappings []*RootMapping
v, found := to.Get(key)
if found {
// There may be more than one language pointing to the same root.
mappings = v.([]*RootMapping)
}
addMapping := func(key string, rm *RootMapping, to *radix.Tree[[]*RootMapping]) {
mappings, _ := to.Get(key)
mappings = append(mappings, rm)
to.Insert(key, mappings)
}
@@ -232,8 +227,8 @@ var _ FilesystemUnwrapper = (*RootMappingFs)(nil)
type RootMappingFs struct {
id string
afero.Fs
rootMapToReal *radix.Tree
realMapToRoot *radix.Tree
rootMapToReal *radix.Tree[[]*RootMapping]
realMapToRoot *radix.Tree[[]*RootMapping]
}
var rootMappingFsCounter atomic.Int32
@@ -279,9 +274,8 @@ func (fs *RootMappingFs) UnwrapFilesystem() afero.Fs {
// Filter creates a copy of this filesystem with only mappings matching a filter.
func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
rootMapToReal := radix.New()
fs.rootMapToReal.Walk(func(b string, v any) bool {
rms := v.([]*RootMapping)
rootMapToReal := radix.New[[]*RootMapping]()
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
var nrms []*RootMapping
for _, rm := range rms {
if f(rm) {
@@ -291,8 +285,9 @@ func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
if len(nrms) != 0 {
rootMapToReal.Insert(b, nrms)
}
return false
})
return radix.WalkContinue, nil, nil
}
fs.rootMapToReal.Walk(walkFn)
fs.rootMapToReal = rootMapToReal
@@ -385,21 +380,18 @@ func (fs *RootMappingFs) ReverseLookupComponent(component, filename string) ([]C
func (fs *RootMappingFs) hasPrefix(prefix string) bool {
hasPrefix := false
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
hasPrefix = true
return true
})
return radix.WalkStop, nil, nil
}
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
return hasPrefix
}
func (fs *RootMappingFs) getRoot(key string) []*RootMapping {
v, found := fs.rootMapToReal.Get(key)
if !found {
return nil
}
return v.([]*RootMapping)
v, _ := fs.rootMapToReal.Get(key)
return v
}
func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
@@ -418,7 +410,7 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
break
}
for _, rm := range vv.([]*RootMapping) {
for _, rm := range vv {
if !seen[rm] {
seen[rm] = true
roots = append(roots, rm)
@@ -439,34 +431,33 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
func (fs *RootMappingFs) getRootsReverse(key string) (string, []*RootMapping) {
tree := fs.realMapToRoot
s, v, found := tree.LongestPrefix(key)
if !found {
return "", nil
}
return s, v.([]*RootMapping)
s, v, _ := tree.LongestPrefix(key)
return s, v
}
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []*RootMapping {
var roots []*RootMapping
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
roots = append(roots, v.([]*RootMapping)...)
return false
})
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
roots = append(roots, v...)
return radix.WalkContinue, nil, nil
}
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
return roots
}
func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
var roots []keyRootMappings
fs.rootMapToReal.WalkPath(prefix, func(s string, v any) bool {
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
if strings.HasPrefix(prefix, s+filepathSeparator) {
roots = append(roots, keyRootMappings{
key: s,
roots: v.([]*RootMapping),
roots: v,
})
}
return false
})
return radix.WalkContinue, nil, nil
}
fs.rootMapToReal.WalkPath(prefix, walkFn)
return roots
}
@@ -593,7 +584,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
// Next add any file mounts inside the given directory.
prefixInside := prefix + filepathSeparator
rfs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v any) bool {
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
if (strings.Count(s, filepathSeparator) - level) != 1 {
// This directory is not part of the current, but we
// need to include the first name part to make it
@@ -603,7 +594,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
name := parts[0]
if seen[name] {
return false
return radix.WalkContinue, nil, nil
}
seen[name] = true
opener := func() (afero.File, error) {
@@ -613,10 +604,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
fi := newDirNameOnlyFileInfo(name, nil, opener)
fis = append(fis, fi)
return false
return radix.WalkContinue, nil, nil
}
rms := v.([]*RootMapping)
for _, rm := range rms {
name := filepath.Base(rm.From)
if seen[name] {
@@ -630,8 +620,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
fis = append(fis, fi)
}
return false
})
return radix.WalkContinue, nil, nil
}
rfs.rootMapToReal.WalkPrefix(prefixInside, walkFn)
// Finally add any ancestor dirs with files in this directory.
ancestors := rfs.getAncestors(prefix)

View File

@@ -241,8 +241,8 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageSour
key := pi.Base()
tree := m.treeResources
commit := tree.Lock(true)
defer commit()
tree.Lock(true)
defer tree.Unlock(true)
if pi.IsContent() {
pm, err := h.newPageMetaSourceFromFile(fi)

View File

@@ -24,6 +24,7 @@ import (
"sync/atomic"
"github.com/bep/logg"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
@@ -195,16 +196,19 @@ func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, t
count := 0
prefix := section + "/"
level := strings.Count(prefix, "/")
tree.WalkPrefixRaw(prefix, func(s string, n contentNode) bool {
tree.WalkPrefixRaw(prefix, func(s string, n contentNode) (radix.WalkFlag, contentNode, error) {
if level != strings.Count(s, "/") {
return false
return radix.WalkContinue, nil, nil
}
cnh.toForEachIdentityProvider(n).ForEeachIdentity(func(id identity.Identity) bool {
ids = append(ids, id)
return false
})
count++
return count > maxSamples
if count > maxSamples {
return radix.WalkStop, nil, nil
}
return radix.WalkContinue, nil, nil
})
}
@@ -212,10 +216,10 @@ func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, t
}
func (t *pageTrees) DeletePageAndResourcesBelow(ss ...string) {
commit1 := t.resourceTrees.Lock(true)
defer commit1()
commit2 := t.treePages.Lock(true)
defer commit2()
t.resourceTrees.Lock(true)
defer t.resourceTrees.Unlock(true)
t.treePages.Lock(true)
defer t.treePages.Unlock(true)
for _, s := range ss {
t.resourceTrees.DeletePrefix(paths.AddTrailingSlash(s))
t.treePages.Delete(s)
@@ -287,13 +291,13 @@ func (m *pageMap) forEachPage(include predicate.PR[*pageState], fn func(p *pageS
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: m.treePages,
LockType: doctree.LockTypeRead,
Handle: func(key string, n contentNode) (bool, error) {
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
if p, ok := n.(*pageState); ok && include(p).OK() {
if terminate, err := fn(p); terminate || err != nil {
return terminate, err
return radix.WalkStop, err
}
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -314,13 +318,13 @@ func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.PR[*pageSt
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: m.treeResources,
LockType: doctree.LockTypeRead,
Handle: func(key string, n contentNode) (bool, error) {
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
if p, ok := n.(*pageState); ok && include(p).OK() {
if terminate, err := fn(p); terminate || err != nil {
return terminate, err
return radix.WalkStop, err
}
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -359,12 +363,12 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages {
Fallback: true,
}
w.Handle = func(key string, n contentNode) (bool, error) {
w.Handle = func(key string, n contentNode) (radix.WalkFlag, error) {
if q.Recursive {
if p, ok := n.(*pageState); ok && include(p) {
pas = append(pas, p)
}
return false, nil
return radix.WalkContinue, nil
}
if p, ok := n.(*pageState); ok && include(p) {
@@ -378,7 +382,7 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages {
}
otherBranch = currentBranch
}
return false, nil
return radix.WalkContinue, nil
}
err := w.Walk(context.Background())
@@ -537,13 +541,20 @@ func (m *pageMap) forEachResourceInPage(
}
}
rwr.Handle = func(resourceKey string, n contentNode) (terminate bool, err error) {
rwr.Handle = func(resourceKey string, n contentNode) (radix.WalkFlag, error) {
if transform == nil {
if ns := shouldSkipOrTerminate(resourceKey); ns >= doctree.NodeTransformStateSkip {
return ns == doctree.NodeTransformStateTerminate, nil
if ns == doctree.NodeTransformStateTerminate {
return radix.WalkStop, nil
}
return radix.WalkContinue, nil
}
}
return handle(resourceKey, n)
b, err := handle(resourceKey, n)
if b || err != nil {
return radix.WalkStop, err
}
return radix.WalkContinue, nil
}
return rwr.Walk(context.Background())
@@ -725,12 +736,12 @@ func newPageMap(s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: m.treePages,
LockType: doctree.LockTypeRead,
Handle: func(s string, n contentNode) (bool, error) {
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
p := n.(*pageState)
if p.PathInfo() != nil {
add(p.PathInfo().BaseNameNoIdentifier(), p)
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -779,10 +790,10 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) {
resourceWalker := pageWalker.Extend()
resourceWalker.Tree = m.treeResources
pageWalker.Handle = func(keyPage string, n contentNode) (bool, error) {
pageWalker.Handle = func(keyPage string, n contentNode) (radix.WalkFlag, error) {
level := strings.Count(keyPage, "/")
if level > maxLevel {
return false, nil
return radix.WalkContinue, nil
}
const indentStr = " "
p := n.(*pageState)
@@ -805,21 +816,25 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) {
isBranch := cnh.isBranchNode(n)
resourceWalker.Prefix = keyPage + "/"
resourceWalker.Handle = func(ss string, n contentNode) (bool, error) {
resourceWalker.Handle = func(ss string, n contentNode) (radix.WalkFlag, error) {
if isBranch {
ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, false, nil)
if ownerKey != keyPage {
// Stop walking downwards, someone else owns this resource.
pageWalker.SkipPrefix(ownerKey + "/")
return false, nil
return radix.WalkContinue, nil
}
}
fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+8))
fmt.Fprintln(w, ss+" (resource)")
return false, nil
return radix.WalkContinue, nil
}
return false, resourceWalker.Walk(context.Background())
if err := resourceWalker.Walk(context.Background()); err != nil {
return radix.WalkStop, err
}
return radix.WalkContinue, nil
}
err := pageWalker.Walk(context.Background())
@@ -1160,17 +1175,17 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) (page.TaxonomyList,
Tree: m.treePages,
Prefix: paths.AddTrailingSlash(key),
LockType: doctree.LockTypeRead,
Handle: func(s string, n contentNode) (bool, error) {
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
p := n.(*pageState)
switch p.Kind() {
case kinds.KindTerm:
if !p.m.shouldList(true) {
return false, nil
return radix.WalkContinue, nil
}
taxonomy := taxonomies[viewName.plural]
if taxonomy == nil {
return true, fmt.Errorf("missing taxonomy: %s", viewName.plural)
return radix.WalkStop, fmt.Errorf("missing taxonomy: %s", viewName.plural)
}
if p.m.term == "" {
panic("term is empty")
@@ -1186,14 +1201,14 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) (page.TaxonomyList,
},
)
if err != nil {
return true, err
return radix.WalkStop, err
}
default:
return false, nil
return radix.WalkContinue, nil
}
return false, nil
return radix.WalkContinue, nil
},
}

View File

@@ -21,11 +21,10 @@ import (
"strings"
"time"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
@@ -36,6 +35,7 @@ import (
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/spf13/cast"
"golang.org/x/sync/errgroup"
)
// Handles the flow of creating all the pages and resources for all sites.
@@ -45,7 +45,7 @@ type allPagesAssembler struct {
ctx context.Context
h *HugoSites
m *pageMap
r para.Runner
g errgroup.Group
assembleChanges *WhatChanged
assembleSectionsInParallel bool
@@ -133,18 +133,17 @@ func (a *allPagesAssembler) createAllPages() error {
a.h.previousSeenTerms = a.seenTerms
}()
}
workers := para.New(config.GetNumWorkerMultiplier())
a.r, _ = workers.Start(context.Background())
if err := cmp.Or(a.doCreatePages(""), a.r.Wait()); err != nil {
if err := cmp.Or(a.doCreatePages("", 0), a.g.Wait()); err != nil {
return err
}
if err := a.pwRoot.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
if err := a.pwRoot.WalkContext.HandleEventsAndHooks(); err != nil {
return err
}
return nil
}
func (a *allPagesAssembler) doCreatePages(prefix string) error {
func (a *allPagesAssembler) doCreatePages(prefix string, depth int) error {
var (
sites = a.h.sitesVersionsRolesMap
h = a.h
@@ -157,7 +156,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
pw *doctree.NodeShiftTreeWalker[contentNode] // walks pages.
rw *doctree.NodeShiftTreeWalker[contentNode] // walks resources.
isRootWalk = prefix == ""
isRootWalk = depth == 0
)
if isRootWalk {
@@ -170,9 +169,6 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
rw = a.rwRoot.Extend() // rw will get its prefix(es) set later.
pw.TransformDelayInsert = true
rw.TransformDelayInsert = true
}
resourceOwnerInfo := struct {
@@ -571,7 +567,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
}
// We need to wait until after the walk to have a complete set.
pw.WalkContext.HooksPost2().Push(
pw.WalkContext.HooksPost().Push(
func() error {
if i := len(missingVectorsForHomeOrRootSection); i > 0 {
// Pick one, the rest will be created later.
@@ -755,7 +751,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
}
// Create missing term pages.
pw.WalkContext.HooksPost2().Push(
pw.WalkContext.HooksPost().Push(
func() error {
for k, v := range a.seenTerms.All() {
viewTermKey := "/" + k.view.plural + "/" + k.term
@@ -825,13 +821,14 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
if err := rw.Walk(a.ctx); err != nil {
return nil, 0, err
}
if a.assembleSectionsInParallel && prefix == "" && s != "" && cnh.isBranchNode(n) && a.h.getFirstTaxonomyConfig(s).IsZero() {
// Handle this branch's descendants in its own goroutine.
const maxDepth = 3
if a.assembleSectionsInParallel && s != "" && depth < maxDepth && cnh.isBranchNode(n) && a.h.getFirstTaxonomyConfig(s).IsZero() {
// Handle this branch in its own goroutine.
pw.SkipPrefix(s + "/")
a.r.Run(func() error {
return a.doCreatePages(s)
a.g.Go(func() error {
return a.doCreatePages(s, depth+1)
})
}
return
@@ -860,12 +857,12 @@ func (sa *sitePagesAssembler) applyAggregates() error {
sa.s.lastmod = time.Time{}
rebuild := sa.s.h.isRebuild()
pw.Handle = func(keyPage string, n contentNode) (bool, error) {
pw.Handle = func(keyPage string, n contentNode) (radix.WalkFlag, error) {
pageBundle := n.(*pageState)
if pageBundle.Kind() == kinds.KindTerm {
// Delay this until they're created.
return false, nil
return radix.WalkContinue, nil
}
if pageBundle.IsPage() {
@@ -878,7 +875,7 @@ func (sa *sitePagesAssembler) applyAggregates() error {
oldDates := pageBundle.m.pageConfig.Dates
// We need to wait until after the walk to determine if any of the dates have changed.
pw.WalkContext.HooksPost2().Push(
pw.WalkContext.HooksPost().Push(
func() error {
if oldDates != pageBundle.m.pageConfig.Dates {
sa.assembleChanges.Add(pageBundle)
@@ -935,13 +932,13 @@ func (sa *sitePagesAssembler) applyAggregates() error {
}
}
rw.Handle = func(resourceKey string, n contentNode) (bool, error) {
rw.Handle = func(resourceKey string, n contentNode) (radix.WalkFlag, error) {
if isBranch {
ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, false, nil)
if ownerKey != keyPage {
// Stop walking downwards, someone else owns this resource.
rw.SkipPrefix(ownerKey + "/")
return false, nil
return radix.WalkContinue, nil
}
}
switch rs := n.(type) {
@@ -950,16 +947,19 @@ func (sa *sitePagesAssembler) applyAggregates() error {
rs.m.resourcePath = relPath
}
return false, nil
return radix.WalkContinue, nil
}
return false, rw.Walk(sa.ctx)
if err := rw.Walk(sa.ctx); err != nil {
return radix.WalkStop, err
}
return radix.WalkContinue, nil
}
if err := pw.Walk(sa.ctx); err != nil {
return err
}
if err := pw.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
if err := pw.WalkContext.HandleEventsAndHooks(); err != nil {
return err
}
@@ -989,12 +989,12 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash.
LockType: doctree.LockTypeRead,
WalkContext: walkContext,
Handle: func(s string, n contentNode) (bool, error) {
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
p := n.(*pageState)
if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy {
// Already handled.
return false, nil
return radix.WalkContinue, nil
}
const eventName = "dates"
@@ -1012,7 +1012,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
return false, nil
},
); err != nil {
return false, err
return radix.WalkStop, err
}
}
@@ -1030,7 +1030,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
})
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -1046,7 +1046,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
}
}
if err := walkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
if err := walkContext.HandleEventsAndHooks(); err != nil {
return err
}
@@ -1068,11 +1068,11 @@ func (sa *sitePagesAssembler) assembleTerms() error {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: pages,
LockType: lockType,
Handle: func(s string, n contentNode) (bool, error) {
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
ps := n.(*pageState)
if ps.m.noLink() {
return false, nil
return radix.WalkContinue, nil
}
for _, viewName := range views {
@@ -1118,7 +1118,7 @@ func (sa *sitePagesAssembler) assembleTerms() error {
}
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -1143,7 +1143,7 @@ func (sa *sitePagesAssembler) assembleResourcesAndSetHome() error {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: pagesTree,
LockType: lockType,
Handle: func(s string, n contentNode) (bool, error) {
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
ps := n.(*pageState)
if s == "" {
@@ -1253,7 +1253,7 @@ func (sa *sitePagesAssembler) assembleResourcesAndSetHome() error {
},
)
return false, err
return radix.WalkContinue, err
},
}

View File

@@ -22,6 +22,7 @@ import (
"testing"
qt "github.com/frankban/quicktest"
radix "github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/para"
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
@@ -86,7 +87,7 @@ func TestTreeData(t *testing.T) {
w := &doctree.NodeShiftTreeWalker[*testValue]{
Tree: tree,
WalkContext: ctx,
Handle: func(s string, t *testValue) (bool, error) {
Handle: func(s string, t *testValue) (radix.WalkFlag, error) {
ctx.Data().Insert(s, map[string]any{
"id": t.ID,
})
@@ -95,7 +96,7 @@ func TestTreeData(t *testing.T) {
p, v := ctx.Data().LongestPrefix(path.Dir(s))
values = append(values, fmt.Sprintf("%s:%s:%v", s, p, v))
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -128,7 +129,7 @@ func TestTreeEvents(t *testing.T) {
WalkContext: &doctree.WalkContext[*testValue]{},
}
w.Handle = func(s string, t *testValue) (bool, error) {
w.Handle = func(s string, t *testValue) (radix.WalkFlag, error) {
if t.IsBranch {
w.WalkContext.AddEventListener("weight", s, func(e *doctree.Event[*testValue]) {
if e.Source.Weight > t.Weight {
@@ -143,11 +144,11 @@ func TestTreeEvents(t *testing.T) {
w.WalkContext.SendEvent(&doctree.Event[*testValue]{Source: t, Path: s, Name: "weight"})
}
return false, nil
return radix.WalkContinue, nil
}
c.Assert(w.Walk(context.Background()), qt.IsNil)
c.Assert(w.WalkContext.HandleHooks1AndEventsAndHooks2(), qt.IsNil)
c.Assert(w.WalkContext.HandleEventsAndHooks(), qt.IsNil)
c.Assert(tree.Get("/a").Weight, eq, 9)
c.Assert(tree.Get("/a/s1").Weight, eq, 9)
@@ -197,8 +198,8 @@ func TestTreePara(t *testing.T) {
for i := range 8 {
r.Run(func() error {
a := &testValue{ID: "/a"}
lock := tree.Lock(true)
defer lock()
tree.Lock(true)
defer tree.Unlock(true)
tree.Insert("/a", a)
ab := &testValue{ID: "/a/b"}
tree.Insert("/a/b", ab)
@@ -350,8 +351,8 @@ func BenchmarkWalk(b *testing.B) {
return tree
}
handle := func(s string, t *testValue) (bool, error) {
return false, nil
handle := func(s string, t *testValue) (radix.WalkFlag, error) {
return radix.WalkContinue, nil
}
for _, numElements := range []int{1000, 10000, 100000} {

View File

@@ -17,10 +17,10 @@ import (
"context"
"fmt"
"path"
"strings"
"sync"
radix "github.com/armon/go-radix"
radix "github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/hstrings"
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
"github.com/gohugoio/hugo/resources/resource"
)
@@ -73,7 +73,7 @@ type (
// Note that multiplied shapes of the same tree is meant to be used concurrently,
// so use the applicable locking when needed.
type NodeShiftTree[T any] struct {
tree *radix.Tree
tree *radix.Tree[T]
// [language, version, role].
siteVector sitesmatrix.Vector
@@ -93,7 +93,7 @@ func New[T any](cfg Config[T]) *NodeShiftTree[T] {
shifter: cfg.Shifter,
transformerRaw: cfg.TransformerRaw,
tree: radix.New(),
tree: radix.New[T](),
}
}
@@ -118,7 +118,7 @@ func (r *NodeShiftTree[T]) DeleteFuncRaw(key string, f func(T) bool) (T, int) {
return lastDeleted, count
}
isEmpty := r.shifter.DeleteFunc(v.(T), func(n T) bool {
isEmpty := r.shifter.DeleteFunc(v, func(n T) bool {
if f(n) {
count++
lastDeleted = n
@@ -143,10 +143,12 @@ func (r *NodeShiftTree[T]) DeleteRaw(key string) {
func (r *NodeShiftTree[T]) DeletePrefix(prefix string) int {
count := 0
var keys []string
r.tree.WalkPrefix(prefix, func(key string, value any) bool {
var zero T
var walkFn radix.WalkFn[T] = func(key string, value T) (radix.WalkFlag, T, error) {
keys = append(keys, key)
return false
})
return radix.WalkContinue, zero, nil
}
r.tree.WalkPrefix(prefix, walkFn)
for _, key := range keys {
if _, ok := r.delete(key); ok {
count++
@@ -160,7 +162,7 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) {
var deleted T
if v, ok := r.tree.Get(key); ok {
var isEmpty bool
deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.siteVector)
deleted, wasDeleted, isEmpty = r.shifter.Delete(v, r.siteVector)
if isEmpty {
r.tree.Delete(key)
}
@@ -170,14 +172,15 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) {
func (t *NodeShiftTree[T]) DeletePrefixRaw(prefix string) int {
count := 0
t.tree.WalkPrefix(prefix, func(key string, value any) bool {
var zero T
var walkFn radix.WalkFn[T] = func(key string, value T) (radix.WalkFlag, T, error) {
if v, ok := t.tree.Delete(key); ok {
resource.MarkStale(v)
count++
}
return false
})
return radix.WalkContinue, zero, nil
}
t.tree.WalkPrefix(prefix, walkFn)
return count
}
@@ -192,27 +195,23 @@ func (r *NodeShiftTree[T]) Insert(s string, v T) (T, T, bool) {
existing T
)
if vv, ok := r.tree.Get(s); ok {
v, existing, updated = r.shifter.Insert(vv.(T), v)
v, existing, updated = r.shifter.Insert(vv, v)
}
r.insert(s, v)
return v, existing, updated
}
func (r *NodeShiftTree[T]) insert(s string, v any) (any, bool) {
if v == nil {
panic("nil value")
}
func (r *NodeShiftTree[T]) insert(s string, v T) (T, bool) {
n, updated := r.tree.Insert(s, v)
return n, updated
}
// InsertRaw inserts v into the tree at the given key without any shifting or transformation.
func (r *NodeShiftTree[T]) InsertRaw(s string, v any) (any, bool) {
func (r *NodeShiftTree[T]) InsertRaw(s string, v T) (T, bool) {
return r.insert(s, v)
}
func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) {
func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v T) (T, bool) {
r.mu.Lock()
defer r.mu.Unlock()
return r.insert(s, v)
@@ -237,19 +236,23 @@ func (t *NodeShiftTree[T]) CanLock() bool {
return ok
}
// Lock locks the data store for read or read/write access until commit is invoked.
// Lock locks the data store for read or read/write access.
// Note that NodeShiftTree is not thread-safe outside of this transaction construct.
func (t *NodeShiftTree[T]) Lock(writable bool) (commit func()) {
func (t *NodeShiftTree[T]) Lock(writable bool) {
if writable {
t.mu.Lock()
} else {
t.mu.RLock()
}
}
// Unlock unlocks the data store.
func (t *NodeShiftTree[T]) Unlock(writable bool) {
if writable {
return t.mu.Unlock
t.mu.Unlock()
} else {
t.mu.RUnlock()
}
return t.mu.RUnlock
}
// LongestPrefix finds the longest prefix of s that exists in the tree that also matches the predicate (if set).
@@ -259,7 +262,7 @@ func (r *NodeShiftTree[T]) LongestPrefix(s string, fallback bool, predicate func
longestPrefix, v, found := r.tree.LongestPrefix(s)
if found {
if v, ok := r.shift(v.(T), fallback); ok && (predicate == nil || predicate(v)) {
if v, ok := r.shift(v, fallback); ok && (predicate == nil || predicate(v)) {
return longestPrefix, v
}
}
@@ -282,7 +285,7 @@ func (r *NodeShiftTree[T]) LongestPrefiValueRaw(s string) (string, T) {
var t T
return s, t
}
return s, v.(T)
return s, v
}
// LongestPrefixRaw returns the longest prefix considering all dimensions.
@@ -298,7 +301,7 @@ func (r *NodeShiftTree[T]) GetRaw(s string) (T, bool) {
var t T
return t, false
}
return v.(T), true
return v, true
}
// AppendRaw appends ts to the node at the given key without any shifting or transformation.
@@ -315,11 +318,8 @@ func (r *NodeShiftTree[T]) AppendRaw(s string, ts ...T) (T, bool) {
}
// WalkPrefixRaw walks all nodes with the given prefix in the tree without any shifting or transformation.
func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, walker func(key string, value T) bool) {
walker2 := func(key string, value any) bool {
return walker(key, value.(T))
}
r.tree.WalkPrefix(prefix, walker2)
func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, fn radix.WalkFn[T]) error {
return r.tree.WalkPrefix(prefix, fn)
}
// Shape returns a new NodeShiftTree shaped to the given dimension.
@@ -344,7 +344,7 @@ func (r *NodeShiftTree[T]) ForEeachInDimension(s string, dims sitesmatrix.Vector
if !ok {
return
}
r.shifter.ForEeachInDimension(v.(T), dims, d, f)
r.shifter.ForEeachInDimension(v, dims, d, f)
}
func (r *NodeShiftTree[T]) ForEeachInAllDimensions(s string, f func(T) bool) {
@@ -353,7 +353,7 @@ func (r *NodeShiftTree[T]) ForEeachInAllDimensions(s string, f func(T) bool) {
if !ok {
return
}
r.shifter.ForEeachInAllDimensions(v.(T), f)
r.shifter.ForEeachInAllDimensions(v, f)
}
type WalkFunc[T any] func(string, T) (bool, error)
@@ -375,19 +375,11 @@ type NodeShiftTreeWalker[T any] struct {
Tree *NodeShiftTree[T]
// Transform will be called for each node in the main tree.
// v2 will replace v1 in the tree.
// The first bool indicates if the value was replaced and needs to be re-inserted into the tree.
// the second bool indicates if the walk should skip this node.
// the third bool indicates if the walk should terminate.
// v2 will replace v1 in the tree if state returned is NodeTransformStateReplaced.
Transform func(s string, v1 T) (v2 T, state NodeTransformState, err error)
// When set, will add inserts to WalkContext.HooksPost1 to be performed after the walk.
TransformDelayInsert bool
// Handle will be called for each node in the main tree.
// If the callback returns true, the walk will stop.
// The callback can optionally return a callback for the nested tree.
Handle func(s string, v T) (terminate bool, err error)
Handle func(s string, v T) (f radix.WalkFlag, err error)
// Optional prefix filter.
Prefix string
@@ -431,18 +423,17 @@ type NodeShiftTreeWalker[T any] struct {
// Any local state is reset.
func (r *NodeShiftTreeWalker[T]) Extend() *NodeShiftTreeWalker[T] {
return &NodeShiftTreeWalker[T]{
Tree: r.Tree,
Transform: r.Transform,
TransformDelayInsert: r.TransformDelayInsert,
Handle: r.Handle,
Prefix: r.Prefix,
IncludeFilter: r.IncludeFilter,
IncludeRawFilter: r.IncludeRawFilter,
LockType: r.LockType,
NoShift: r.NoShift,
Fallback: r.Fallback,
Debug: r.Debug,
WalkContext: r.WalkContext,
Tree: r.Tree,
Transform: r.Transform,
Handle: r.Handle,
Prefix: r.Prefix,
IncludeFilter: r.IncludeFilter,
IncludeRawFilter: r.IncludeRawFilter,
LockType: r.LockType,
NoShift: r.NoShift,
Fallback: r.Fallback,
Debug: r.Debug,
WalkContext: r.WalkContext,
}
}
@@ -458,13 +449,12 @@ func (r *NodeShiftTreeWalker[T]) SkipPrefix(prefix ...string) {
r.skipPrefixes = append(r.skipPrefixes, prefix...)
}
// ShouldSkip returns whether the given key should be skipped in the walk.
func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string, v T) bool {
for _, prefix := range r.skipPrefixes {
if strings.HasPrefix(s, prefix) {
return true
}
}
func (r *NodeShiftTreeWalker[T]) ShouldSkipKey(s string) bool {
return hstrings.HasAnyPrefix(s, r.skipPrefixes...)
}
// ShouldSkip returns whether the given value should be skipped in the walk.
func (r *NodeShiftTreeWalker[T]) ShouldSkipValue(s string, v T) bool {
if r.IncludeRawFilter != nil {
if !r.IncludeRawFilter(s, v) {
return true
@@ -473,106 +463,148 @@ func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string, v T) bool {
return false
}
func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
if r.Tree == nil {
panic("Tree is required")
}
type walkHandler[T any] struct {
r *NodeShiftTreeWalker[T]
handle func(s string, v T) (radix.WalkFlag, T, error)
}
func (h *walkHandler[T]) Check(s string) (radix.WalkFlag, error) {
if h.r == nil {
return radix.WalkContinue, nil
}
if h.r.ShouldSkipKey(s) {
return radix.WalkSkip, nil
}
return radix.WalkContinue, nil
}
func (h *walkHandler[T]) Handle(s string, v T) (radix.WalkFlag, T, error) {
if h.handle == nil {
return radix.WalkContinue, v, nil
}
return h.handle(s, v)
}
func (r *NodeShiftTreeWalker[T]) lock() {
switch r.LockType {
case LockTypeRead:
r.Tree.mu.RLock()
case LockTypeWrite:
r.Tree.mu.Lock()
default:
// No lock
}
}
func (r *NodeShiftTreeWalker[T]) unlock() {
switch r.LockType {
case LockTypeRead:
r.Tree.mu.RUnlock()
case LockTypeWrite:
r.Tree.mu.Unlock()
default:
// No lock
}
}
func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
var deletes []string
handleT := func(s string, t T) (ns NodeTransformState, err error) {
if r.IncludeFilter != nil && !r.IncludeFilter(s, t) {
return
}
if r.Transform != nil {
if !r.NoShift {
panic("Transform must be performed with NoShift=true")
}
if ns, err = func() (ns NodeTransformState, err error) {
t, ns, err = r.Transform(s, t)
if ns >= NodeTransformStateSkip || err != nil {
return
}
switch ns {
case NodeTransformStateReplaced:
// Delay insert until after the walk to
// avoid locking issues.
if r.TransformDelayInsert {
r.WalkContext.HooksPost1().Push(
func() error {
r.Tree.InsertRaw(s, t)
return nil
},
)
} else {
r.Tree.InsertRaw(s, t)
}
case NodeTransformStateDeleted:
// Delay delete until after the walk.
deletes = append(deletes, s)
ns = NodeTransformStateSkip
}
return
}(); ns >= NodeTransformStateSkip || err != nil {
return
}
}
if r.Handle != nil {
var terminate bool
terminate, err = r.Handle(s, t)
if terminate || err != nil {
return
}
}
return
}
return func() error {
if r.LockType > LockTypeNone {
unlock := r.Tree.Lock(r.LockType == LockTypeWrite)
defer unlock()
}
r.lock()
defer r.unlock()
r.resetLocalState()
main := r.Tree
var err error
handleV := func(s string, v any) (terminate bool) {
var handleV radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
var zero T
// Context cancellation check.
if ctx != nil && ctx.Err() != nil {
err = ctx.Err()
return true
return radix.WalkStop, zero, ctx.Err()
}
if r.ShouldSkip(s, v.(T)) {
return false
if r.ShouldSkipValue(s, v) {
return radix.WalkContinue, zero, nil
}
var t T
if r.NoShift {
t = v.(T)
t = v
} else {
var ok bool
t, ok = r.toT(r.Tree, v)
if !ok {
return false
return radix.WalkContinue, zero, nil
}
}
var ns NodeTransformState
ns, err = handleT(s, t)
if ns == NodeTransformStateTerminate || err != nil {
return true
var (
ns NodeTransformState
t2 T
)
if r.IncludeFilter != nil && !r.IncludeFilter(s, t) {
return radix.WalkContinue, zero, nil
}
return false
if r.Transform != nil {
if !r.NoShift {
panic("Transform must be performed with NoShift=true")
}
var err error
ns, err = func() (ns NodeTransformState, err error) {
t2, ns, err = r.Transform(s, t)
if ns >= NodeTransformStateSkip || err != nil {
return
}
switch ns {
case NodeTransformStateReplaced:
case NodeTransformStateDeleted:
// Delay delete until after the walk.
deletes = append(deletes, s)
ns = NodeTransformStateSkip
}
return
}()
if ns == NodeTransformStateTerminate || err != nil {
return radix.WalkStop, zero, err
}
if ns == NodeTransformStateSkip {
return radix.WalkContinue, zero, nil
}
}
if r.Handle != nil {
f, err := r.Handle(s, t)
if f.ShouldSet() {
panic("Handle cannot replace nodes in the tree, use Transform for that")
}
if f.ShouldStop() || err != nil {
return f, zero, err
}
}
if ns == NodeTransformStateTerminate {
return radix.WalkStop, zero, nil
}
if ns == NodeTransformStateReplaced {
return radix.WalkSet | radix.WalkContinue, t2, nil
}
return radix.WalkContinue, zero, nil
}
handle := &walkHandler[T]{
r: r,
handle: handleV,
}
if r.Prefix != "" {
main.tree.WalkPrefix(r.Prefix, handleV)
if err := main.tree.WalkPrefix(r.Prefix, handle); err != nil {
return err
}
} else {
main.tree.Walk(handleV)
if err := main.tree.Walk(handle); err != nil {
return err
}
}
// This is currently only performed with no shift.
@@ -580,7 +612,7 @@ func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
main.tree.Delete(s)
}
return err
return nil
}()
}
@@ -588,8 +620,8 @@ func (r *NodeShiftTreeWalker[T]) resetLocalState() {
r.skipPrefixes = nil
}
func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (T, bool) {
return tree.shift(v.(T), r.Fallback)
func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v T) (T, bool) {
return tree.shift(v, r.Fallback)
}
func (r *NodeShiftTree[T]) Has(s string) bool {
@@ -613,7 +645,7 @@ func (r *NodeShiftTree[T]) get(s string) (T, bool) {
var t T
return t, false
}
if v, ok := r.shift(v.(T), false); ok {
if v, ok := r.shift(v, false); ok {
return v, true
}

View File

@@ -17,7 +17,7 @@ import (
"iter"
"sync"
radix "github.com/armon/go-radix"
radix "github.com/gohugoio/go-radix"
)
// Tree is a non thread safe radix tree that holds T.
@@ -43,28 +43,24 @@ type TreeCommon[T any] interface {
}
func NewSimpleTree[T any]() *SimpleTree[T] {
return &SimpleTree[T]{tree: radix.New()}
return &SimpleTree[T]{tree: radix.New[T]()}
}
// SimpleTree is a radix tree that holds T.
// This tree is not thread safe.
type SimpleTree[T any] struct {
tree *radix.Tree
tree *radix.Tree[T]
zero T
}
func (tree *SimpleTree[T]) Get(s string) T {
if v, ok := tree.tree.Get(s); ok {
return v.(T)
}
return tree.zero
v, _ := tree.tree.Get(s)
return v
}
func (tree *SimpleTree[T]) LongestPrefix(s string) (string, T) {
if s, v, ok := tree.tree.LongestPrefix(s); ok {
return s, v.(T)
}
return "", tree.zero
s, v, _ := tree.tree.LongestPrefix(s)
return s, v
}
func (tree *SimpleTree[T]) Insert(s string, v T) T {
@@ -73,85 +69,79 @@ func (tree *SimpleTree[T]) Insert(s string, v T) T {
}
func (tree *SimpleTree[T]) Walk(f func(s string, v T) (bool, error)) error {
var err error
tree.tree.Walk(func(s string, v any) bool {
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
var b bool
b, err = f(s, v.(T))
if err != nil {
return true
b, err := f(s, v)
if b || err != nil {
return radix.WalkStop, tree.zero, err
}
return b
})
return err
return radix.WalkContinue, tree.zero, nil
}
return tree.tree.Walk(walkFn)
}
func (tree *SimpleTree[T]) WalkPrefix(s string, f func(s string, v T) (bool, error)) error {
var err error
tree.tree.WalkPrefix(s, func(s string, v any) bool {
var b bool
b, err = f(s, v.(T))
if err != nil {
return true
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
b, err := f(s, v)
if b || err != nil {
return radix.WalkStop, tree.zero, err
}
return b
})
return err
return radix.WalkContinue, tree.zero, nil
}
return tree.tree.WalkPrefix(s, walkFn)
}
func (tree *SimpleTree[T]) WalkPath(s string, f func(s string, v T) (bool, error)) error {
var err error
tree.tree.WalkPath(s, func(s string, v any) bool {
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
var b bool
b, err = f(s, v.(T))
if err != nil {
return true
b, err = f(s, v)
if b || err != nil {
return radix.WalkStop, tree.zero, err
}
return b
})
return radix.WalkContinue, tree.zero, nil
}
tree.tree.WalkPath(s, walkFn)
return err
}
func (tree *SimpleTree[T]) All() iter.Seq2[string, T] {
return func(yield func(s string, v T) bool) {
tree.tree.Walk(func(s string, v any) bool {
return !yield(s, v.(T))
})
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
if !yield(s, v) {
return radix.WalkStop, tree.zero, nil
}
return radix.WalkContinue, tree.zero, nil
}
tree.tree.Walk(walkFn)
}
}
// NewSimpleThreadSafeTree creates a new SimpleTree.
func NewSimpleThreadSafeTree[T any]() *SimpleThreadSafeTree[T] {
return &SimpleThreadSafeTree[T]{tree: radix.New(), mu: new(sync.RWMutex)}
return &SimpleThreadSafeTree[T]{tree: radix.New[T](), mu: new(sync.RWMutex)}
}
// SimpleThreadSafeTree is a thread safe radix tree that holds T.
type SimpleThreadSafeTree[T any] struct {
mu *sync.RWMutex
tree *radix.Tree
tree *radix.Tree[T]
zero T
}
var noopFunc = func() {}
func (tree *SimpleThreadSafeTree[T]) Get(s string) T {
tree.mu.RLock()
defer tree.mu.RUnlock()
if v, ok := tree.tree.Get(s); ok {
return v.(T)
}
return tree.zero
v, _ := tree.tree.Get(s)
return v
}
func (tree *SimpleThreadSafeTree[T]) LongestPrefix(s string) (string, T) {
tree.mu.RLock()
defer tree.mu.RUnlock()
if s, v, ok := tree.tree.LongestPrefix(s); ok {
return s, v.(T)
}
return "", tree.zero
s, v, _ := tree.tree.LongestPrefix(s)
return s, v
}
func (tree *SimpleThreadSafeTree[T]) Insert(s string, v T) T {
@@ -162,60 +152,65 @@ func (tree *SimpleThreadSafeTree[T]) Insert(s string, v T) T {
return v
}
func (tree *SimpleThreadSafeTree[T]) Lock(lockType LockType) func() {
func (tree *SimpleThreadSafeTree[T]) Lock(lockType LockType) {
switch lockType {
case LockTypeNone:
return noopFunc
case LockTypeRead:
tree.mu.RLock()
return tree.mu.RUnlock
case LockTypeWrite:
tree.mu.Lock()
return tree.mu.Unlock
}
return noopFunc
}
func (tree *SimpleThreadSafeTree[T]) Unlock(lockType LockType) {
switch lockType {
case LockTypeRead:
tree.mu.RUnlock()
case LockTypeWrite:
tree.mu.Unlock()
}
}
func (tree *SimpleThreadSafeTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
commit := tree.Lock(lockType)
defer commit()
var err error
tree.tree.WalkPrefix(s, func(s string, v any) bool {
tree.Lock(lockType)
defer tree.Unlock(lockType)
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
var b bool
b, err = f(s, v.(T))
if err != nil {
return true
b, err := f(s, v)
if b || err != nil {
return radix.WalkStop, tree.zero, err
}
return b
})
return err
return radix.WalkContinue, tree.zero, nil
}
return tree.tree.WalkPrefix(s, walkFn)
}
func (tree *SimpleThreadSafeTree[T]) WalkPath(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
commit := tree.Lock(lockType)
defer commit()
tree.Lock(lockType)
defer tree.Unlock(lockType)
var err error
tree.tree.WalkPath(s, func(s string, v any) bool {
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
var b bool
b, err = f(s, v.(T))
if err != nil {
return true
b, err = f(s, v)
if b || err != nil {
return radix.WalkStop, tree.zero, err
}
return b
})
return radix.WalkContinue, tree.zero, nil
}
tree.tree.WalkPath(s, walkFn)
return err
}
func (tree *SimpleThreadSafeTree[T]) All(lockType LockType) iter.Seq2[string, T] {
commit := tree.Lock(lockType)
defer commit()
return func(yield func(s string, v T) bool) {
tree.tree.Walk(func(s string, v any) bool {
return !yield(s, v.(T))
})
tree.Lock(lockType)
defer tree.Unlock(lockType)
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
if !yield(s, v) {
return radix.WalkStop, tree.zero, nil
}
return radix.WalkContinue, tree.zero, nil
}
tree.tree.Walk(walkFn)
}
}
// iter.Seq[*TemplWithBaseApplied]

View File

@@ -19,6 +19,7 @@ import (
"strings"
"sync"
radix "github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
@@ -139,23 +140,27 @@ type MutableTree interface {
DeleteRaw(key string)
DeletePrefix(prefix string) int
DeletePrefixRaw(prefix string) int
Lock(writable bool) (commit func())
Lock(writable bool)
Unlock(writable bool)
CanLock() bool // Used for troubleshooting only.
}
// WalkableTree is a tree that can be walked.
type WalkableTree[T any] interface {
WalkPrefixRaw(prefix string, walker func(key string, value T) bool)
WalkPrefixRaw(prefix string, walker radix.WalkFn[T]) error
}
var _ WalkableTree[any] = (*WalkableTrees[any])(nil)
type WalkableTrees[T any] []WalkableTree[T]
func (t WalkableTrees[T]) WalkPrefixRaw(prefix string, walker func(key string, value T) bool) {
func (t WalkableTrees[T]) WalkPrefixRaw(prefix string, walker radix.WalkFn[T]) error {
for _, tree := range t {
tree.WalkPrefixRaw(prefix, walker)
if err := tree.WalkPrefixRaw(prefix, walker); err != nil {
return err
}
}
return nil
}
var _ MutableTree = MutableTrees(nil)
@@ -184,15 +189,15 @@ func (t MutableTrees) DeletePrefixRaw(prefix string) int {
return count
}
func (t MutableTrees) Lock(writable bool) (commit func()) {
commits := make([]func(), len(t))
for i, tree := range t {
commits[i] = tree.Lock(writable)
func (t MutableTrees) Lock(writable bool) {
for _, tree := range t {
tree.Lock(writable)
}
return func() {
for _, commit := range commits {
commit()
}
}
func (t MutableTrees) Unlock(writable bool) {
for _, tree := range t {
tree.Unlock(writable)
}
}
@@ -218,11 +223,8 @@ type WalkContext[T any] struct {
eventMu sync.Mutex
events []*Event[T]
hooksPost1Init sync.Once
hooksPost1 *collections.Stack[func() error]
hooksPost2Init sync.Once
hooksPost2 *collections.Stack[func() error]
hooksPostInit sync.Once
hooksPost *collections.Stack[func() error]
}
type eventHandlers[T any] map[string][]func(*Event[T])
@@ -259,32 +261,19 @@ func (ctx *WalkContext[T]) HandleEvents() error {
return nil
}
func (ctx *WalkContext[T]) HooksPost1() *collections.Stack[func() error] {
ctx.hooksPost1Init.Do(func() {
ctx.hooksPost1 = collections.NewStack[func() error]()
func (ctx *WalkContext[T]) HooksPost() *collections.Stack[func() error] {
ctx.hooksPostInit.Do(func() {
ctx.hooksPost = collections.NewStack[func() error]()
})
return ctx.hooksPost1
return ctx.hooksPost
}
func (ctx *WalkContext[T]) HooksPost2() *collections.Stack[func() error] {
ctx.hooksPost2Init.Do(func() {
ctx.hooksPost2 = collections.NewStack[func() error]()
})
return ctx.hooksPost2
}
func (ctx *WalkContext[T]) HandleHooks1AndEventsAndHooks2() error {
for _, hook := range ctx.HooksPost1().All() {
if err := hook(); err != nil {
return err
}
}
func (ctx *WalkContext[T]) HandleEventsAndHooks() error {
if err := ctx.HandleEvents(); err != nil {
return err
}
for _, hook := range ctx.HooksPost2().All() {
for _, hook := range ctx.HooksPost().All() {
if err := hook(); err != nil {
return err
}

View File

@@ -93,8 +93,12 @@ func (t *TreeShiftTreeSlice[T]) Insert(s string, v T) T {
return t.tree().Insert(s, v)
}
func (t *TreeShiftTreeSlice[T]) Lock(lockType LockType) func() {
return t.tree().Lock(lockType)
func (t *TreeShiftTreeSlice[T]) Lock(lockType LockType) {
t.tree().Lock(lockType)
}
func (t *TreeShiftTreeSlice[T]) Unlock(lockType LockType) {
t.tree().Unlock(lockType)
}
func (t *TreeShiftTreeSlice[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {

View File

@@ -24,6 +24,7 @@ import (
"time"
"github.com/bep/logg"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/config/allconfig"
"github.com/gohugoio/hugo/hugofs/hglob"
@@ -536,8 +537,11 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: s.pageMap.treePages,
LockType: doctree.LockTypeRead,
Handle: func(s string, n contentNode) (bool, error) {
return fn(s, n.(*pageState)), nil
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
if fn(s, n.(*pageState)) {
return radix.WalkStop, nil
}
return radix.WalkContinue, nil
},
}
return w.Walk(context.Background())

View File

@@ -27,6 +27,7 @@ import (
"github.com/bep/debounce"
"github.com/bep/logg"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/hugofs"
@@ -261,9 +262,9 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error {
return errors.New("rebuild called when not in watch mode")
}
h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNode) bool {
h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNode) (radix.WalkFlag, contentNode, error) {
cnh.resetBuildState(n)
return false
return radix.WalkContinue, nil, nil
})
for _, s := range h.Sites {

View File

@@ -18,6 +18,7 @@ import (
"fmt"
"strings"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/hugolib/doctree"
@@ -167,9 +168,9 @@ func (pt pageTree) Sections() page.Pages {
Tree: tree,
Prefix: prefix,
}
w.Handle = func(ss string, n contentNode) (bool, error) {
w.Handle = func(ss string, n contentNode) (radix.WalkFlag, error) {
if !cnh.isBranchNode(n) {
return false, nil
return radix.WalkContinue, nil
}
if currentBranchPrefix == "" || !strings.HasPrefix(ss, currentBranchPrefix) {
if p, ok := n.(*pageState); ok && p.IsSection() && p.m.shouldList(false) && p.Parent() == pt.p {
@@ -179,7 +180,7 @@ func (pt pageTree) Sections() page.Pages {
}
}
currentBranchPrefix = ss + "/"
return false, nil
return radix.WalkContinue, nil
}
if err := w.Walk(context.Background()); err != nil {

View File

@@ -30,6 +30,7 @@ import (
"time"
"github.com/bep/logg"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/common/hstore"
"github.com/gohugoio/hugo/common/hsync"
@@ -928,7 +929,7 @@ func (s *Site) initRenderFormats() {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: s.pageMap.treePages,
Handle: func(key string, n contentNode) (bool, error) {
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
if p, ok := n.(*pageState); ok {
for _, f := range p.m.pageConfig.ConfiguredOutputFormats {
if !formatSet[f.Name] {
@@ -937,7 +938,7 @@ func (s *Site) initRenderFormats() {
}
}
}
return false, nil
return radix.WalkContinue, nil
},
}

View File

@@ -21,6 +21,7 @@ import (
"sync"
"github.com/bep/logg"
"github.com/gohugoio/go-radix"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/tpl/tplimpl"
@@ -87,18 +88,18 @@ func (s *Site) renderPages(ctx *siteRenderContext) error {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: s.pageMap.treePages,
Handle: func(key string, n contentNode) (bool, error) {
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
if p, ok := n.(*pageState); ok {
if cfg.shouldRender(ctx.infol, p) {
select {
case <-s.h.Done():
return true, nil
return radix.WalkStop, nil
default:
pages <- p
}
}
}
return false, nil
return radix.WalkContinue, nil
},
}
@@ -270,16 +271,16 @@ func (s *Site) renderPaginator(p *pageState, templ *tplimpl.TemplInfo) error {
func (s *Site) renderAliases() error {
w := &doctree.NodeShiftTreeWalker[contentNode]{
Tree: s.pageMap.treePages,
Handle: func(key string, n contentNode) (bool, error) {
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
p := n.(*pageState)
// We cannot alias a page that's not rendered.
if p.m.noLink() || p.skipRender() {
return false, nil
return radix.WalkContinue, nil
}
if len(p.Aliases()) == 0 {
return false, nil
return radix.WalkContinue, nil
}
pathSeen := make(map[string]bool)
@@ -324,11 +325,11 @@ func (s *Site) renderAliases() error {
err := s.writeDestAlias(a, plink, f, p)
if err != nil {
return true, err
return radix.WalkStop, err
}
}
}
return false, nil
return radix.WalkContinue, nil
},
}
return w.Walk(context.TODO())

View File

@@ -1,4 +1,3 @@
# TODO1 clean up when done.
go generate ./gen
javy compile js/greet.bundle.js -d -o wasm/greet.wasm
javy compile js/renderkatex.bundle.js -d -o wasm/renderkatex.wasm

View File

@@ -202,63 +202,6 @@ func (t *templateNamespace) templatesIn(in tpl.Template) iter.Seq[tpl.Template]
}
}
/*
func (t *templateHandler) applyBaseTemplate(overlay, base templateInfo) (tpl.Template, error) {
if overlay.isText {
var (
templ = t.main.getPrototypeText(prototypeCloneIDBaseof).New(overlay.name)
err error
)
if !base.IsZero() {
templ, err = templ.Parse(base.template)
if err != nil {
return nil, base.errWithFileContext("text: base: parse failed", err)
}
}
templ, err = texttemplate.Must(templ.Clone()).Parse(overlay.template)
if err != nil {
return nil, overlay.errWithFileContext("text: overlay: parse failed", err)
}
// The extra lookup is a workaround, see
// * https://github.com/golang/go/issues/16101
// * https://github.com/gohugoio/hugo/issues/2549
// templ = templ.Lookup(templ.Name())
return templ, nil
}
var (
templ = t.main.getPrototypeHTML(prototypeCloneIDBaseof).New(overlay.name)
err error
)
if !base.IsZero() {
templ, err = templ.Parse(base.template)
if err != nil {
return nil, base.errWithFileContext("html: base: parse failed", err)
}
}
templ, err = htmltemplate.Must(templ.Clone()).Parse(overlay.template)
if err != nil {
return nil, overlay.errWithFileContext("html: overlay: parse failed", err)
}
// The extra lookup is a workaround, see
// * https://github.com/golang/go/issues/16101
// * https://github.com/gohugoio/hugo/issues/2549
templ = templ.Lookup(templ.Name())
return templ, err
}
*/
var baseTemplateDefineRe = regexp.MustCompile(`^{{-?\s*define`)
// needsBaseTemplate returns true if the first non-comment template block is a

View File

@@ -269,14 +269,13 @@ func (ti *TemplInfo) SubCategory() SubCategory {
func (ti *TemplInfo) BaseVariantsSeq() iter.Seq[*TemplWithBaseApplied] {
return func(yield func(*TemplWithBaseApplied) bool) {
ti.baseVariants.Walk(func(key string, v map[TemplateDescriptor]*TemplWithBaseApplied) (bool, error) {
for _, v := range ti.baseVariants.All() {
for _, vv := range v {
if !yield(vv) {
return true, nil
return
}
}
return false, nil
})
}
}
}