mirror of
https://github.com/gohugoio/hugo.git
synced 2025-12-13 20:36:04 +01:00
Speedup and simplify page assembly for deeper content trees
This commit moves to a forked version go-radix (fork source has not had any updates in 3 years.), whith 2 notable changes:
* It's generic (using Go generics) and thus removes a lot of type conversions/assertions.
* It allows nodes to be replaced during walk, which allows to partition the tree for parallel processing without worrying about locking.
For this repo, this means:
* The assembly step now processes nested sections in parallel, which gives a speedup for deep content trees with a slight allocation penalty (see benchmarks below).
* Nodes that needs to be reinserted are inserted directly.
* Also, there are some drive-by fixes of some allocation issues, e.g. avoid wrapping mutexes in returned anonomous functions, a common source of hidden allocations.
```
│ master.bench │ perf-p3.bench │
│ sec/op │ sec/op vs base │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 6.958m ± 3% 7.015m ± 3% ~ (p=0.589 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 14.25m ± 1% 14.56m ± 8% ~ (p=0.394 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 48.07m ± 3% 49.23m ± 3% ~ (p=0.394 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 66.66m ± 4% 66.47m ± 6% ~ (p=0.485 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 59.57m ± 4% 50.73m ± 5% -14.85% (p=0.002 n=6)
geomean 28.54m 27.92m -2.18%
│ master.bench │ perf-p3.bench │
│ B/op │ B/op vs base │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 4.513Mi ± 0% 4.527Mi ± 0% +0.33% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 15.35Mi ± 0% 15.49Mi ± 0% +0.94% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 62.50Mi ± 0% 63.19Mi ± 0% +1.10% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 86.78Mi ± 0% 87.73Mi ± 0% +1.09% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 62.96Mi ± 0% 63.66Mi ± 0% +1.12% (p=0.002 n=6)
geomean 29.84Mi 30.11Mi +0.92%
│ master.bench │ perf-p3.bench │
│ allocs/op │ allocs/op vs base │
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=1/pagesPerSection=50-10 60.44k ± 0% 60.97k ± 0% +0.87% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=100-10 205.8k ± 0% 211.4k ± 0% +2.70% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=1/sectionsPerLevel=6/pagesPerSection=500-10 831.1k ± 0% 858.3k ± 0% +3.27% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=2/sectionsPerLevel=6/pagesPerSection=100-10 1.157M ± 0% 1.197M ± 0% +3.41% (p=0.002 n=6)
AssembleDeepSiteWithManySections/depth=4/sectionsPerLevel=2/pagesPerSection=100-10 839.9k ± 0% 867.8k ± 0% +3.31% (p=0.002 n=6)
geomean 398.5k 409.3k +2.71%
```
This commit is contained in:
@@ -97,6 +97,16 @@ func GetOrCompileRegexp(pattern string) (re *regexp.Regexp, err error) {
|
||||
return reCache.getOrCompileRegexp(pattern)
|
||||
}
|
||||
|
||||
// HasAnyPrefix checks if the string s has any of the prefixes given.
|
||||
func HasAnyPrefix(s string, prefixes ...string) bool {
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(s, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// InSlice checks if a string is an element of a slice of strings
|
||||
// and returns a boolean value.
|
||||
func InSlice(arr []string, el string) bool {
|
||||
|
||||
2
go.mod
2
go.mod
@@ -4,7 +4,6 @@ require (
|
||||
github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0
|
||||
github.com/alecthomas/chroma/v2 v2.20.0
|
||||
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0
|
||||
github.com/aws/aws-sdk-go-v2/service/cloudfront v1.57.0
|
||||
github.com/bep/clocks v0.5.0
|
||||
@@ -38,6 +37,7 @@ require (
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/goccy/go-yaml v1.18.0
|
||||
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6
|
||||
github.com/gohugoio/go-radix v1.2.0
|
||||
github.com/gohugoio/hashstructure v0.6.0
|
||||
github.com/gohugoio/httpcache v0.8.0
|
||||
github.com/gohugoio/hugo-goldmark-extensions/extras v0.5.0
|
||||
|
||||
4
go.sum
4
go.sum
@@ -102,8 +102,6 @@ github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NT
|
||||
github.com/alecthomas/chroma/v2 v2.20.0/go.mod h1:e7tViK0xh/Nf4BYHl00ycY6rV7b8iXBksI9E359yNmA=
|
||||
github.com/alecthomas/repr v0.5.1 h1:E3G4t2QbHTSNpPKBgMTln5KLkZHLOcU7r37J4pXBuIg=
|
||||
github.com/alecthomas/repr v0.5.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
|
||||
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
|
||||
github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
|
||||
@@ -276,6 +274,8 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6 h1:pxlAea9eRwuAnt/zKbGqlFO2ZszpIe24YpOVLf+N+4I=
|
||||
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20251018145728-cfcc22d823c6/go.mod h1:m5hu1im5Qc7LDycVLvee6MPobJiRLBYHklypFJR0/aE=
|
||||
github.com/gohugoio/go-radix v1.2.0 h1:D5GTk8jIoeXirBSc2P4E4NdHKDrenk9k9N0ctU5Yrhg=
|
||||
github.com/gohugoio/go-radix v1.2.0/go.mod h1:k6vDa0ebpbpgtzSj9lPGJcA4AZwJ9xUNObUy2vczPFM=
|
||||
github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=
|
||||
github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=
|
||||
github.com/gohugoio/httpcache v0.8.0 h1:hNdsmGSELztetYCsPVgjA960zSa4dfEqqF/SficorCU=
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/gohugoio/hugo/hugofs/files"
|
||||
"github.com/gohugoio/hugo/hugofs/hglob"
|
||||
|
||||
radix "github.com/armon/go-radix"
|
||||
radix "github.com/gohugoio/go-radix"
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
@@ -42,17 +42,12 @@ var _ ReverseLookupProvder = (*RootMappingFs)(nil)
|
||||
// root mappings with some optional metadata about the root.
|
||||
// Note that From represents a virtual root that maps to the actual filename in To.
|
||||
func NewRootMappingFs(fs afero.Fs, rms ...*RootMapping) (*RootMappingFs, error) {
|
||||
rootMapToReal := radix.New()
|
||||
realMapToRoot := radix.New()
|
||||
rootMapToReal := radix.New[[]*RootMapping]()
|
||||
realMapToRoot := radix.New[[]*RootMapping]()
|
||||
id := fmt.Sprintf("rfs-%d", rootMappingFsCounter.Add(1))
|
||||
|
||||
addMapping := func(key string, rm *RootMapping, to *radix.Tree) {
|
||||
var mappings []*RootMapping
|
||||
v, found := to.Get(key)
|
||||
if found {
|
||||
// There may be more than one language pointing to the same root.
|
||||
mappings = v.([]*RootMapping)
|
||||
}
|
||||
addMapping := func(key string, rm *RootMapping, to *radix.Tree[[]*RootMapping]) {
|
||||
mappings, _ := to.Get(key)
|
||||
mappings = append(mappings, rm)
|
||||
to.Insert(key, mappings)
|
||||
}
|
||||
@@ -232,8 +227,8 @@ var _ FilesystemUnwrapper = (*RootMappingFs)(nil)
|
||||
type RootMappingFs struct {
|
||||
id string
|
||||
afero.Fs
|
||||
rootMapToReal *radix.Tree
|
||||
realMapToRoot *radix.Tree
|
||||
rootMapToReal *radix.Tree[[]*RootMapping]
|
||||
realMapToRoot *radix.Tree[[]*RootMapping]
|
||||
}
|
||||
|
||||
var rootMappingFsCounter atomic.Int32
|
||||
@@ -279,9 +274,8 @@ func (fs *RootMappingFs) UnwrapFilesystem() afero.Fs {
|
||||
|
||||
// Filter creates a copy of this filesystem with only mappings matching a filter.
|
||||
func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
|
||||
rootMapToReal := radix.New()
|
||||
fs.rootMapToReal.Walk(func(b string, v any) bool {
|
||||
rms := v.([]*RootMapping)
|
||||
rootMapToReal := radix.New[[]*RootMapping]()
|
||||
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
|
||||
var nrms []*RootMapping
|
||||
for _, rm := range rms {
|
||||
if f(rm) {
|
||||
@@ -291,8 +285,9 @@ func (fs RootMappingFs) Filter(f func(m *RootMapping) bool) *RootMappingFs {
|
||||
if len(nrms) != 0 {
|
||||
rootMapToReal.Insert(b, nrms)
|
||||
}
|
||||
return false
|
||||
})
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
fs.rootMapToReal.Walk(walkFn)
|
||||
|
||||
fs.rootMapToReal = rootMapToReal
|
||||
|
||||
@@ -385,21 +380,18 @@ func (fs *RootMappingFs) ReverseLookupComponent(component, filename string) ([]C
|
||||
|
||||
func (fs *RootMappingFs) hasPrefix(prefix string) bool {
|
||||
hasPrefix := false
|
||||
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
|
||||
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
|
||||
hasPrefix = true
|
||||
return true
|
||||
})
|
||||
return radix.WalkStop, nil, nil
|
||||
}
|
||||
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
|
||||
|
||||
return hasPrefix
|
||||
}
|
||||
|
||||
func (fs *RootMappingFs) getRoot(key string) []*RootMapping {
|
||||
v, found := fs.rootMapToReal.Get(key)
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
|
||||
return v.([]*RootMapping)
|
||||
v, _ := fs.rootMapToReal.Get(key)
|
||||
return v
|
||||
}
|
||||
|
||||
func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
|
||||
@@ -418,7 +410,7 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
|
||||
break
|
||||
}
|
||||
|
||||
for _, rm := range vv.([]*RootMapping) {
|
||||
for _, rm := range vv {
|
||||
if !seen[rm] {
|
||||
seen[rm] = true
|
||||
roots = append(roots, rm)
|
||||
@@ -439,34 +431,33 @@ func (fs *RootMappingFs) getRoots(key string) (string, []*RootMapping) {
|
||||
|
||||
func (fs *RootMappingFs) getRootsReverse(key string) (string, []*RootMapping) {
|
||||
tree := fs.realMapToRoot
|
||||
s, v, found := tree.LongestPrefix(key)
|
||||
if !found {
|
||||
return "", nil
|
||||
}
|
||||
return s, v.([]*RootMapping)
|
||||
s, v, _ := tree.LongestPrefix(key)
|
||||
return s, v
|
||||
}
|
||||
|
||||
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []*RootMapping {
|
||||
var roots []*RootMapping
|
||||
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
|
||||
roots = append(roots, v.([]*RootMapping)...)
|
||||
return false
|
||||
})
|
||||
var walkFn radix.WalkFn[[]*RootMapping] = func(b string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
|
||||
roots = append(roots, v...)
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
fs.rootMapToReal.WalkPrefix(prefix, walkFn)
|
||||
|
||||
return roots
|
||||
}
|
||||
|
||||
func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
|
||||
var roots []keyRootMappings
|
||||
fs.rootMapToReal.WalkPath(prefix, func(s string, v any) bool {
|
||||
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, v []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
|
||||
if strings.HasPrefix(prefix, s+filepathSeparator) {
|
||||
roots = append(roots, keyRootMappings{
|
||||
key: s,
|
||||
roots: v.([]*RootMapping),
|
||||
roots: v,
|
||||
})
|
||||
}
|
||||
return false
|
||||
})
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
fs.rootMapToReal.WalkPath(prefix, walkFn)
|
||||
|
||||
return roots
|
||||
}
|
||||
@@ -593,7 +584,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
|
||||
|
||||
// Next add any file mounts inside the given directory.
|
||||
prefixInside := prefix + filepathSeparator
|
||||
rfs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v any) bool {
|
||||
var walkFn radix.WalkFn[[]*RootMapping] = func(s string, rms []*RootMapping) (radix.WalkFlag, []*RootMapping, error) {
|
||||
if (strings.Count(s, filepathSeparator) - level) != 1 {
|
||||
// This directory is not part of the current, but we
|
||||
// need to include the first name part to make it
|
||||
@@ -603,7 +594,7 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
|
||||
name := parts[0]
|
||||
|
||||
if seen[name] {
|
||||
return false
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
seen[name] = true
|
||||
opener := func() (afero.File, error) {
|
||||
@@ -613,10 +604,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
|
||||
fi := newDirNameOnlyFileInfo(name, nil, opener)
|
||||
fis = append(fis, fi)
|
||||
|
||||
return false
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
|
||||
rms := v.([]*RootMapping)
|
||||
for _, rm := range rms {
|
||||
name := filepath.Base(rm.From)
|
||||
if seen[name] {
|
||||
@@ -630,8 +620,9 @@ func (rfs *RootMappingFs) collectDirEntries(prefix string) ([]iofs.DirEntry, err
|
||||
fis = append(fis, fi)
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
rfs.rootMapToReal.WalkPrefix(prefixInside, walkFn)
|
||||
|
||||
// Finally add any ancestor dirs with files in this directory.
|
||||
ancestors := rfs.getAncestors(prefix)
|
||||
|
||||
@@ -241,8 +241,8 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, buildConfig *BuildCfg) (pageSour
|
||||
key := pi.Base()
|
||||
tree := m.treeResources
|
||||
|
||||
commit := tree.Lock(true)
|
||||
defer commit()
|
||||
tree.Lock(true)
|
||||
defer tree.Unlock(true)
|
||||
|
||||
if pi.IsContent() {
|
||||
pm, err := h.newPageMetaSourceFromFile(fi)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/cache/dynacache"
|
||||
"github.com/gohugoio/hugo/common/loggers"
|
||||
"github.com/gohugoio/hugo/common/paths"
|
||||
@@ -195,16 +196,19 @@ func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, t
|
||||
count := 0
|
||||
prefix := section + "/"
|
||||
level := strings.Count(prefix, "/")
|
||||
tree.WalkPrefixRaw(prefix, func(s string, n contentNode) bool {
|
||||
tree.WalkPrefixRaw(prefix, func(s string, n contentNode) (radix.WalkFlag, contentNode, error) {
|
||||
if level != strings.Count(s, "/") {
|
||||
return false
|
||||
return radix.WalkContinue, nil, nil
|
||||
}
|
||||
cnh.toForEachIdentityProvider(n).ForEeachIdentity(func(id identity.Identity) bool {
|
||||
ids = append(ids, id)
|
||||
return false
|
||||
})
|
||||
count++
|
||||
return count > maxSamples
|
||||
if count > maxSamples {
|
||||
return radix.WalkStop, nil, nil
|
||||
}
|
||||
return radix.WalkContinue, nil, nil
|
||||
})
|
||||
}
|
||||
|
||||
@@ -212,10 +216,10 @@ func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, t
|
||||
}
|
||||
|
||||
func (t *pageTrees) DeletePageAndResourcesBelow(ss ...string) {
|
||||
commit1 := t.resourceTrees.Lock(true)
|
||||
defer commit1()
|
||||
commit2 := t.treePages.Lock(true)
|
||||
defer commit2()
|
||||
t.resourceTrees.Lock(true)
|
||||
defer t.resourceTrees.Unlock(true)
|
||||
t.treePages.Lock(true)
|
||||
defer t.treePages.Unlock(true)
|
||||
for _, s := range ss {
|
||||
t.resourceTrees.DeletePrefix(paths.AddTrailingSlash(s))
|
||||
t.treePages.Delete(s)
|
||||
@@ -287,13 +291,13 @@ func (m *pageMap) forEachPage(include predicate.PR[*pageState], fn func(p *pageS
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: m.treePages,
|
||||
LockType: doctree.LockTypeRead,
|
||||
Handle: func(key string, n contentNode) (bool, error) {
|
||||
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
if p, ok := n.(*pageState); ok && include(p).OK() {
|
||||
if terminate, err := fn(p); terminate || err != nil {
|
||||
return terminate, err
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -314,13 +318,13 @@ func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.PR[*pageSt
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: m.treeResources,
|
||||
LockType: doctree.LockTypeRead,
|
||||
Handle: func(key string, n contentNode) (bool, error) {
|
||||
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
if p, ok := n.(*pageState); ok && include(p).OK() {
|
||||
if terminate, err := fn(p); terminate || err != nil {
|
||||
return terminate, err
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -359,12 +363,12 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages {
|
||||
Fallback: true,
|
||||
}
|
||||
|
||||
w.Handle = func(key string, n contentNode) (bool, error) {
|
||||
w.Handle = func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
if q.Recursive {
|
||||
if p, ok := n.(*pageState); ok && include(p) {
|
||||
pas = append(pas, p)
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
if p, ok := n.(*pageState); ok && include(p) {
|
||||
@@ -378,7 +382,7 @@ func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages {
|
||||
}
|
||||
otherBranch = currentBranch
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
err := w.Walk(context.Background())
|
||||
@@ -537,13 +541,20 @@ func (m *pageMap) forEachResourceInPage(
|
||||
}
|
||||
}
|
||||
|
||||
rwr.Handle = func(resourceKey string, n contentNode) (terminate bool, err error) {
|
||||
rwr.Handle = func(resourceKey string, n contentNode) (radix.WalkFlag, error) {
|
||||
if transform == nil {
|
||||
if ns := shouldSkipOrTerminate(resourceKey); ns >= doctree.NodeTransformStateSkip {
|
||||
return ns == doctree.NodeTransformStateTerminate, nil
|
||||
if ns == doctree.NodeTransformStateTerminate {
|
||||
return radix.WalkStop, nil
|
||||
}
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
}
|
||||
return handle(resourceKey, n)
|
||||
b, err := handle(resourceKey, n)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
return rwr.Walk(context.Background())
|
||||
@@ -725,12 +736,12 @@ func newPageMap(s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: m.treePages,
|
||||
LockType: doctree.LockTypeRead,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
p := n.(*pageState)
|
||||
if p.PathInfo() != nil {
|
||||
add(p.PathInfo().BaseNameNoIdentifier(), p)
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -779,10 +790,10 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) {
|
||||
resourceWalker := pageWalker.Extend()
|
||||
resourceWalker.Tree = m.treeResources
|
||||
|
||||
pageWalker.Handle = func(keyPage string, n contentNode) (bool, error) {
|
||||
pageWalker.Handle = func(keyPage string, n contentNode) (radix.WalkFlag, error) {
|
||||
level := strings.Count(keyPage, "/")
|
||||
if level > maxLevel {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
const indentStr = " "
|
||||
p := n.(*pageState)
|
||||
@@ -805,21 +816,25 @@ func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) {
|
||||
isBranch := cnh.isBranchNode(n)
|
||||
resourceWalker.Prefix = keyPage + "/"
|
||||
|
||||
resourceWalker.Handle = func(ss string, n contentNode) (bool, error) {
|
||||
resourceWalker.Handle = func(ss string, n contentNode) (radix.WalkFlag, error) {
|
||||
if isBranch {
|
||||
ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, false, nil)
|
||||
if ownerKey != keyPage {
|
||||
// Stop walking downwards, someone else owns this resource.
|
||||
pageWalker.SkipPrefix(ownerKey + "/")
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
}
|
||||
fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+8))
|
||||
fmt.Fprintln(w, ss+" (resource)")
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
return false, resourceWalker.Walk(context.Background())
|
||||
if err := resourceWalker.Walk(context.Background()); err != nil {
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
err := pageWalker.Walk(context.Background())
|
||||
@@ -1160,17 +1175,17 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) (page.TaxonomyList,
|
||||
Tree: m.treePages,
|
||||
Prefix: paths.AddTrailingSlash(key),
|
||||
LockType: doctree.LockTypeRead,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
p := n.(*pageState)
|
||||
|
||||
switch p.Kind() {
|
||||
case kinds.KindTerm:
|
||||
if !p.m.shouldList(true) {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
taxonomy := taxonomies[viewName.plural]
|
||||
if taxonomy == nil {
|
||||
return true, fmt.Errorf("missing taxonomy: %s", viewName.plural)
|
||||
return radix.WalkStop, fmt.Errorf("missing taxonomy: %s", viewName.plural)
|
||||
}
|
||||
if p.m.term == "" {
|
||||
panic("term is empty")
|
||||
@@ -1186,14 +1201,14 @@ func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) (page.TaxonomyList,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return true, err
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
|
||||
default:
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -21,11 +21,10 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/maps"
|
||||
"github.com/gohugoio/hugo/common/para"
|
||||
"github.com/gohugoio/hugo/common/paths"
|
||||
"github.com/gohugoio/hugo/common/types"
|
||||
"github.com/gohugoio/hugo/config"
|
||||
"github.com/gohugoio/hugo/hugofs/files"
|
||||
"github.com/gohugoio/hugo/hugolib/doctree"
|
||||
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
|
||||
@@ -36,6 +35,7 @@ import (
|
||||
"github.com/gohugoio/hugo/resources/page"
|
||||
"github.com/gohugoio/hugo/resources/page/pagemeta"
|
||||
"github.com/spf13/cast"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Handles the flow of creating all the pages and resources for all sites.
|
||||
@@ -45,7 +45,7 @@ type allPagesAssembler struct {
|
||||
ctx context.Context
|
||||
h *HugoSites
|
||||
m *pageMap
|
||||
r para.Runner
|
||||
g errgroup.Group
|
||||
|
||||
assembleChanges *WhatChanged
|
||||
assembleSectionsInParallel bool
|
||||
@@ -133,18 +133,17 @@ func (a *allPagesAssembler) createAllPages() error {
|
||||
a.h.previousSeenTerms = a.seenTerms
|
||||
}()
|
||||
}
|
||||
workers := para.New(config.GetNumWorkerMultiplier())
|
||||
a.r, _ = workers.Start(context.Background())
|
||||
if err := cmp.Or(a.doCreatePages(""), a.r.Wait()); err != nil {
|
||||
|
||||
if err := cmp.Or(a.doCreatePages("", 0), a.g.Wait()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.pwRoot.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
|
||||
if err := a.pwRoot.WalkContext.HandleEventsAndHooks(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
func (a *allPagesAssembler) doCreatePages(prefix string, depth int) error {
|
||||
var (
|
||||
sites = a.h.sitesVersionsRolesMap
|
||||
h = a.h
|
||||
@@ -157,7 +156,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
pw *doctree.NodeShiftTreeWalker[contentNode] // walks pages.
|
||||
rw *doctree.NodeShiftTreeWalker[contentNode] // walks resources.
|
||||
|
||||
isRootWalk = prefix == ""
|
||||
isRootWalk = depth == 0
|
||||
)
|
||||
|
||||
if isRootWalk {
|
||||
@@ -170,9 +169,6 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
|
||||
rw = a.rwRoot.Extend() // rw will get its prefix(es) set later.
|
||||
|
||||
pw.TransformDelayInsert = true
|
||||
rw.TransformDelayInsert = true
|
||||
|
||||
}
|
||||
|
||||
resourceOwnerInfo := struct {
|
||||
@@ -571,7 +567,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
}
|
||||
|
||||
// We need to wait until after the walk to have a complete set.
|
||||
pw.WalkContext.HooksPost2().Push(
|
||||
pw.WalkContext.HooksPost().Push(
|
||||
func() error {
|
||||
if i := len(missingVectorsForHomeOrRootSection); i > 0 {
|
||||
// Pick one, the rest will be created later.
|
||||
@@ -755,7 +751,7 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
}
|
||||
|
||||
// Create missing term pages.
|
||||
pw.WalkContext.HooksPost2().Push(
|
||||
pw.WalkContext.HooksPost().Push(
|
||||
func() error {
|
||||
for k, v := range a.seenTerms.All() {
|
||||
viewTermKey := "/" + k.view.plural + "/" + k.term
|
||||
@@ -825,13 +821,14 @@ func (a *allPagesAssembler) doCreatePages(prefix string) error {
|
||||
if err := rw.Walk(a.ctx); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if a.assembleSectionsInParallel && prefix == "" && s != "" && cnh.isBranchNode(n) && a.h.getFirstTaxonomyConfig(s).IsZero() {
|
||||
// Handle this branch's descendants in its own goroutine.
|
||||
const maxDepth = 3
|
||||
if a.assembleSectionsInParallel && s != "" && depth < maxDepth && cnh.isBranchNode(n) && a.h.getFirstTaxonomyConfig(s).IsZero() {
|
||||
// Handle this branch in its own goroutine.
|
||||
pw.SkipPrefix(s + "/")
|
||||
a.r.Run(func() error {
|
||||
return a.doCreatePages(s)
|
||||
a.g.Go(func() error {
|
||||
return a.doCreatePages(s, depth+1)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
return
|
||||
@@ -860,12 +857,12 @@ func (sa *sitePagesAssembler) applyAggregates() error {
|
||||
sa.s.lastmod = time.Time{}
|
||||
rebuild := sa.s.h.isRebuild()
|
||||
|
||||
pw.Handle = func(keyPage string, n contentNode) (bool, error) {
|
||||
pw.Handle = func(keyPage string, n contentNode) (radix.WalkFlag, error) {
|
||||
pageBundle := n.(*pageState)
|
||||
|
||||
if pageBundle.Kind() == kinds.KindTerm {
|
||||
// Delay this until they're created.
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
if pageBundle.IsPage() {
|
||||
@@ -878,7 +875,7 @@ func (sa *sitePagesAssembler) applyAggregates() error {
|
||||
oldDates := pageBundle.m.pageConfig.Dates
|
||||
|
||||
// We need to wait until after the walk to determine if any of the dates have changed.
|
||||
pw.WalkContext.HooksPost2().Push(
|
||||
pw.WalkContext.HooksPost().Push(
|
||||
func() error {
|
||||
if oldDates != pageBundle.m.pageConfig.Dates {
|
||||
sa.assembleChanges.Add(pageBundle)
|
||||
@@ -935,13 +932,13 @@ func (sa *sitePagesAssembler) applyAggregates() error {
|
||||
}
|
||||
}
|
||||
|
||||
rw.Handle = func(resourceKey string, n contentNode) (bool, error) {
|
||||
rw.Handle = func(resourceKey string, n contentNode) (radix.WalkFlag, error) {
|
||||
if isBranch {
|
||||
ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, false, nil)
|
||||
if ownerKey != keyPage {
|
||||
// Stop walking downwards, someone else owns this resource.
|
||||
rw.SkipPrefix(ownerKey + "/")
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
}
|
||||
switch rs := n.(type) {
|
||||
@@ -950,16 +947,19 @@ func (sa *sitePagesAssembler) applyAggregates() error {
|
||||
rs.m.resourcePath = relPath
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
return false, rw.Walk(sa.ctx)
|
||||
if err := rw.Walk(sa.ctx); err != nil {
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
if err := pw.Walk(sa.ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := pw.WalkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
|
||||
if err := pw.WalkContext.HandleEventsAndHooks(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -989,12 +989,12 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
|
||||
Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash.
|
||||
LockType: doctree.LockTypeRead,
|
||||
WalkContext: walkContext,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
p := n.(*pageState)
|
||||
|
||||
if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy {
|
||||
// Already handled.
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
const eventName = "dates"
|
||||
@@ -1012,7 +1012,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
|
||||
return false, nil
|
||||
},
|
||||
); err != nil {
|
||||
return false, err
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1030,7 +1030,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
|
||||
})
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1046,7 +1046,7 @@ func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := walkContext.HandleHooks1AndEventsAndHooks2(); err != nil {
|
||||
if err := walkContext.HandleEventsAndHooks(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1068,11 +1068,11 @@ func (sa *sitePagesAssembler) assembleTerms() error {
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: pages,
|
||||
LockType: lockType,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
ps := n.(*pageState)
|
||||
|
||||
if ps.m.noLink() {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
for _, viewName := range views {
|
||||
@@ -1118,7 +1118,7 @@ func (sa *sitePagesAssembler) assembleTerms() error {
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1143,7 +1143,7 @@ func (sa *sitePagesAssembler) assembleResourcesAndSetHome() error {
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: pagesTree,
|
||||
LockType: lockType,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
ps := n.(*pageState)
|
||||
|
||||
if s == "" {
|
||||
@@ -1253,7 +1253,7 @@ func (sa *sitePagesAssembler) assembleResourcesAndSetHome() error {
|
||||
},
|
||||
)
|
||||
|
||||
return false, err
|
||||
return radix.WalkContinue, err
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
radix "github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/para"
|
||||
"github.com/gohugoio/hugo/hugolib/doctree"
|
||||
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
|
||||
@@ -86,7 +87,7 @@ func TestTreeData(t *testing.T) {
|
||||
w := &doctree.NodeShiftTreeWalker[*testValue]{
|
||||
Tree: tree,
|
||||
WalkContext: ctx,
|
||||
Handle: func(s string, t *testValue) (bool, error) {
|
||||
Handle: func(s string, t *testValue) (radix.WalkFlag, error) {
|
||||
ctx.Data().Insert(s, map[string]any{
|
||||
"id": t.ID,
|
||||
})
|
||||
@@ -95,7 +96,7 @@ func TestTreeData(t *testing.T) {
|
||||
p, v := ctx.Data().LongestPrefix(path.Dir(s))
|
||||
values = append(values, fmt.Sprintf("%s:%s:%v", s, p, v))
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -128,7 +129,7 @@ func TestTreeEvents(t *testing.T) {
|
||||
WalkContext: &doctree.WalkContext[*testValue]{},
|
||||
}
|
||||
|
||||
w.Handle = func(s string, t *testValue) (bool, error) {
|
||||
w.Handle = func(s string, t *testValue) (radix.WalkFlag, error) {
|
||||
if t.IsBranch {
|
||||
w.WalkContext.AddEventListener("weight", s, func(e *doctree.Event[*testValue]) {
|
||||
if e.Source.Weight > t.Weight {
|
||||
@@ -143,11 +144,11 @@ func TestTreeEvents(t *testing.T) {
|
||||
w.WalkContext.SendEvent(&doctree.Event[*testValue]{Source: t, Path: s, Name: "weight"})
|
||||
}
|
||||
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
c.Assert(w.Walk(context.Background()), qt.IsNil)
|
||||
c.Assert(w.WalkContext.HandleHooks1AndEventsAndHooks2(), qt.IsNil)
|
||||
c.Assert(w.WalkContext.HandleEventsAndHooks(), qt.IsNil)
|
||||
|
||||
c.Assert(tree.Get("/a").Weight, eq, 9)
|
||||
c.Assert(tree.Get("/a/s1").Weight, eq, 9)
|
||||
@@ -197,8 +198,8 @@ func TestTreePara(t *testing.T) {
|
||||
for i := range 8 {
|
||||
r.Run(func() error {
|
||||
a := &testValue{ID: "/a"}
|
||||
lock := tree.Lock(true)
|
||||
defer lock()
|
||||
tree.Lock(true)
|
||||
defer tree.Unlock(true)
|
||||
tree.Insert("/a", a)
|
||||
ab := &testValue{ID: "/a/b"}
|
||||
tree.Insert("/a/b", ab)
|
||||
@@ -350,8 +351,8 @@ func BenchmarkWalk(b *testing.B) {
|
||||
return tree
|
||||
}
|
||||
|
||||
handle := func(s string, t *testValue) (bool, error) {
|
||||
return false, nil
|
||||
handle := func(s string, t *testValue) (radix.WalkFlag, error) {
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
for _, numElements := range []int{1000, 10000, 100000} {
|
||||
|
||||
@@ -17,10 +17,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
radix "github.com/armon/go-radix"
|
||||
radix "github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/hstrings"
|
||||
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
|
||||
"github.com/gohugoio/hugo/resources/resource"
|
||||
)
|
||||
@@ -73,7 +73,7 @@ type (
|
||||
// Note that multiplied shapes of the same tree is meant to be used concurrently,
|
||||
// so use the applicable locking when needed.
|
||||
type NodeShiftTree[T any] struct {
|
||||
tree *radix.Tree
|
||||
tree *radix.Tree[T]
|
||||
|
||||
// [language, version, role].
|
||||
siteVector sitesmatrix.Vector
|
||||
@@ -93,7 +93,7 @@ func New[T any](cfg Config[T]) *NodeShiftTree[T] {
|
||||
|
||||
shifter: cfg.Shifter,
|
||||
transformerRaw: cfg.TransformerRaw,
|
||||
tree: radix.New(),
|
||||
tree: radix.New[T](),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,7 +118,7 @@ func (r *NodeShiftTree[T]) DeleteFuncRaw(key string, f func(T) bool) (T, int) {
|
||||
return lastDeleted, count
|
||||
}
|
||||
|
||||
isEmpty := r.shifter.DeleteFunc(v.(T), func(n T) bool {
|
||||
isEmpty := r.shifter.DeleteFunc(v, func(n T) bool {
|
||||
if f(n) {
|
||||
count++
|
||||
lastDeleted = n
|
||||
@@ -143,10 +143,12 @@ func (r *NodeShiftTree[T]) DeleteRaw(key string) {
|
||||
func (r *NodeShiftTree[T]) DeletePrefix(prefix string) int {
|
||||
count := 0
|
||||
var keys []string
|
||||
r.tree.WalkPrefix(prefix, func(key string, value any) bool {
|
||||
var zero T
|
||||
var walkFn radix.WalkFn[T] = func(key string, value T) (radix.WalkFlag, T, error) {
|
||||
keys = append(keys, key)
|
||||
return false
|
||||
})
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
r.tree.WalkPrefix(prefix, walkFn)
|
||||
for _, key := range keys {
|
||||
if _, ok := r.delete(key); ok {
|
||||
count++
|
||||
@@ -160,7 +162,7 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) {
|
||||
var deleted T
|
||||
if v, ok := r.tree.Get(key); ok {
|
||||
var isEmpty bool
|
||||
deleted, wasDeleted, isEmpty = r.shifter.Delete(v.(T), r.siteVector)
|
||||
deleted, wasDeleted, isEmpty = r.shifter.Delete(v, r.siteVector)
|
||||
if isEmpty {
|
||||
r.tree.Delete(key)
|
||||
}
|
||||
@@ -170,14 +172,15 @@ func (r *NodeShiftTree[T]) delete(key string) (T, bool) {
|
||||
|
||||
func (t *NodeShiftTree[T]) DeletePrefixRaw(prefix string) int {
|
||||
count := 0
|
||||
|
||||
t.tree.WalkPrefix(prefix, func(key string, value any) bool {
|
||||
var zero T
|
||||
var walkFn radix.WalkFn[T] = func(key string, value T) (radix.WalkFlag, T, error) {
|
||||
if v, ok := t.tree.Delete(key); ok {
|
||||
resource.MarkStale(v)
|
||||
count++
|
||||
}
|
||||
return false
|
||||
})
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
t.tree.WalkPrefix(prefix, walkFn)
|
||||
|
||||
return count
|
||||
}
|
||||
@@ -192,27 +195,23 @@ func (r *NodeShiftTree[T]) Insert(s string, v T) (T, T, bool) {
|
||||
existing T
|
||||
)
|
||||
if vv, ok := r.tree.Get(s); ok {
|
||||
v, existing, updated = r.shifter.Insert(vv.(T), v)
|
||||
v, existing, updated = r.shifter.Insert(vv, v)
|
||||
}
|
||||
r.insert(s, v)
|
||||
return v, existing, updated
|
||||
}
|
||||
|
||||
func (r *NodeShiftTree[T]) insert(s string, v any) (any, bool) {
|
||||
if v == nil {
|
||||
panic("nil value")
|
||||
}
|
||||
func (r *NodeShiftTree[T]) insert(s string, v T) (T, bool) {
|
||||
n, updated := r.tree.Insert(s, v)
|
||||
|
||||
return n, updated
|
||||
}
|
||||
|
||||
// InsertRaw inserts v into the tree at the given key without any shifting or transformation.
|
||||
func (r *NodeShiftTree[T]) InsertRaw(s string, v any) (any, bool) {
|
||||
func (r *NodeShiftTree[T]) InsertRaw(s string, v T) (T, bool) {
|
||||
return r.insert(s, v)
|
||||
}
|
||||
|
||||
func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v any) (any, bool) {
|
||||
func (r *NodeShiftTree[T]) InsertRawWithLock(s string, v T) (T, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.insert(s, v)
|
||||
@@ -237,19 +236,23 @@ func (t *NodeShiftTree[T]) CanLock() bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// Lock locks the data store for read or read/write access until commit is invoked.
|
||||
// Lock locks the data store for read or read/write access.
|
||||
// Note that NodeShiftTree is not thread-safe outside of this transaction construct.
|
||||
func (t *NodeShiftTree[T]) Lock(writable bool) (commit func()) {
|
||||
func (t *NodeShiftTree[T]) Lock(writable bool) {
|
||||
if writable {
|
||||
t.mu.Lock()
|
||||
} else {
|
||||
t.mu.RLock()
|
||||
}
|
||||
}
|
||||
|
||||
// Unlock unlocks the data store.
|
||||
func (t *NodeShiftTree[T]) Unlock(writable bool) {
|
||||
if writable {
|
||||
return t.mu.Unlock
|
||||
t.mu.Unlock()
|
||||
} else {
|
||||
t.mu.RUnlock()
|
||||
}
|
||||
return t.mu.RUnlock
|
||||
}
|
||||
|
||||
// LongestPrefix finds the longest prefix of s that exists in the tree that also matches the predicate (if set).
|
||||
@@ -259,7 +262,7 @@ func (r *NodeShiftTree[T]) LongestPrefix(s string, fallback bool, predicate func
|
||||
longestPrefix, v, found := r.tree.LongestPrefix(s)
|
||||
|
||||
if found {
|
||||
if v, ok := r.shift(v.(T), fallback); ok && (predicate == nil || predicate(v)) {
|
||||
if v, ok := r.shift(v, fallback); ok && (predicate == nil || predicate(v)) {
|
||||
return longestPrefix, v
|
||||
}
|
||||
}
|
||||
@@ -282,7 +285,7 @@ func (r *NodeShiftTree[T]) LongestPrefiValueRaw(s string) (string, T) {
|
||||
var t T
|
||||
return s, t
|
||||
}
|
||||
return s, v.(T)
|
||||
return s, v
|
||||
}
|
||||
|
||||
// LongestPrefixRaw returns the longest prefix considering all dimensions.
|
||||
@@ -298,7 +301,7 @@ func (r *NodeShiftTree[T]) GetRaw(s string) (T, bool) {
|
||||
var t T
|
||||
return t, false
|
||||
}
|
||||
return v.(T), true
|
||||
return v, true
|
||||
}
|
||||
|
||||
// AppendRaw appends ts to the node at the given key without any shifting or transformation.
|
||||
@@ -315,11 +318,8 @@ func (r *NodeShiftTree[T]) AppendRaw(s string, ts ...T) (T, bool) {
|
||||
}
|
||||
|
||||
// WalkPrefixRaw walks all nodes with the given prefix in the tree without any shifting or transformation.
|
||||
func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, walker func(key string, value T) bool) {
|
||||
walker2 := func(key string, value any) bool {
|
||||
return walker(key, value.(T))
|
||||
}
|
||||
r.tree.WalkPrefix(prefix, walker2)
|
||||
func (r *NodeShiftTree[T]) WalkPrefixRaw(prefix string, fn radix.WalkFn[T]) error {
|
||||
return r.tree.WalkPrefix(prefix, fn)
|
||||
}
|
||||
|
||||
// Shape returns a new NodeShiftTree shaped to the given dimension.
|
||||
@@ -344,7 +344,7 @@ func (r *NodeShiftTree[T]) ForEeachInDimension(s string, dims sitesmatrix.Vector
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.shifter.ForEeachInDimension(v.(T), dims, d, f)
|
||||
r.shifter.ForEeachInDimension(v, dims, d, f)
|
||||
}
|
||||
|
||||
func (r *NodeShiftTree[T]) ForEeachInAllDimensions(s string, f func(T) bool) {
|
||||
@@ -353,7 +353,7 @@ func (r *NodeShiftTree[T]) ForEeachInAllDimensions(s string, f func(T) bool) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.shifter.ForEeachInAllDimensions(v.(T), f)
|
||||
r.shifter.ForEeachInAllDimensions(v, f)
|
||||
}
|
||||
|
||||
type WalkFunc[T any] func(string, T) (bool, error)
|
||||
@@ -375,19 +375,11 @@ type NodeShiftTreeWalker[T any] struct {
|
||||
Tree *NodeShiftTree[T]
|
||||
|
||||
// Transform will be called for each node in the main tree.
|
||||
// v2 will replace v1 in the tree.
|
||||
// The first bool indicates if the value was replaced and needs to be re-inserted into the tree.
|
||||
// the second bool indicates if the walk should skip this node.
|
||||
// the third bool indicates if the walk should terminate.
|
||||
// v2 will replace v1 in the tree if state returned is NodeTransformStateReplaced.
|
||||
Transform func(s string, v1 T) (v2 T, state NodeTransformState, err error)
|
||||
|
||||
// When set, will add inserts to WalkContext.HooksPost1 to be performed after the walk.
|
||||
TransformDelayInsert bool
|
||||
|
||||
// Handle will be called for each node in the main tree.
|
||||
// If the callback returns true, the walk will stop.
|
||||
// The callback can optionally return a callback for the nested tree.
|
||||
Handle func(s string, v T) (terminate bool, err error)
|
||||
Handle func(s string, v T) (f radix.WalkFlag, err error)
|
||||
|
||||
// Optional prefix filter.
|
||||
Prefix string
|
||||
@@ -431,18 +423,17 @@ type NodeShiftTreeWalker[T any] struct {
|
||||
// Any local state is reset.
|
||||
func (r *NodeShiftTreeWalker[T]) Extend() *NodeShiftTreeWalker[T] {
|
||||
return &NodeShiftTreeWalker[T]{
|
||||
Tree: r.Tree,
|
||||
Transform: r.Transform,
|
||||
TransformDelayInsert: r.TransformDelayInsert,
|
||||
Handle: r.Handle,
|
||||
Prefix: r.Prefix,
|
||||
IncludeFilter: r.IncludeFilter,
|
||||
IncludeRawFilter: r.IncludeRawFilter,
|
||||
LockType: r.LockType,
|
||||
NoShift: r.NoShift,
|
||||
Fallback: r.Fallback,
|
||||
Debug: r.Debug,
|
||||
WalkContext: r.WalkContext,
|
||||
Tree: r.Tree,
|
||||
Transform: r.Transform,
|
||||
Handle: r.Handle,
|
||||
Prefix: r.Prefix,
|
||||
IncludeFilter: r.IncludeFilter,
|
||||
IncludeRawFilter: r.IncludeRawFilter,
|
||||
LockType: r.LockType,
|
||||
NoShift: r.NoShift,
|
||||
Fallback: r.Fallback,
|
||||
Debug: r.Debug,
|
||||
WalkContext: r.WalkContext,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -458,13 +449,12 @@ func (r *NodeShiftTreeWalker[T]) SkipPrefix(prefix ...string) {
|
||||
r.skipPrefixes = append(r.skipPrefixes, prefix...)
|
||||
}
|
||||
|
||||
// ShouldSkip returns whether the given key should be skipped in the walk.
|
||||
func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string, v T) bool {
|
||||
for _, prefix := range r.skipPrefixes {
|
||||
if strings.HasPrefix(s, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
func (r *NodeShiftTreeWalker[T]) ShouldSkipKey(s string) bool {
|
||||
return hstrings.HasAnyPrefix(s, r.skipPrefixes...)
|
||||
}
|
||||
|
||||
// ShouldSkip returns whether the given value should be skipped in the walk.
|
||||
func (r *NodeShiftTreeWalker[T]) ShouldSkipValue(s string, v T) bool {
|
||||
if r.IncludeRawFilter != nil {
|
||||
if !r.IncludeRawFilter(s, v) {
|
||||
return true
|
||||
@@ -473,106 +463,148 @@ func (r *NodeShiftTreeWalker[T]) ShouldSkip(s string, v T) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
|
||||
if r.Tree == nil {
|
||||
panic("Tree is required")
|
||||
}
|
||||
type walkHandler[T any] struct {
|
||||
r *NodeShiftTreeWalker[T]
|
||||
handle func(s string, v T) (radix.WalkFlag, T, error)
|
||||
}
|
||||
|
||||
func (h *walkHandler[T]) Check(s string) (radix.WalkFlag, error) {
|
||||
if h.r == nil {
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
if h.r.ShouldSkipKey(s) {
|
||||
return radix.WalkSkip, nil
|
||||
}
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
func (h *walkHandler[T]) Handle(s string, v T) (radix.WalkFlag, T, error) {
|
||||
if h.handle == nil {
|
||||
return radix.WalkContinue, v, nil
|
||||
}
|
||||
return h.handle(s, v)
|
||||
}
|
||||
|
||||
func (r *NodeShiftTreeWalker[T]) lock() {
|
||||
switch r.LockType {
|
||||
case LockTypeRead:
|
||||
r.Tree.mu.RLock()
|
||||
case LockTypeWrite:
|
||||
r.Tree.mu.Lock()
|
||||
default:
|
||||
// No lock
|
||||
}
|
||||
}
|
||||
|
||||
func (r *NodeShiftTreeWalker[T]) unlock() {
|
||||
switch r.LockType {
|
||||
case LockTypeRead:
|
||||
r.Tree.mu.RUnlock()
|
||||
case LockTypeWrite:
|
||||
r.Tree.mu.Unlock()
|
||||
default:
|
||||
// No lock
|
||||
}
|
||||
}
|
||||
|
||||
func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
|
||||
var deletes []string
|
||||
|
||||
handleT := func(s string, t T) (ns NodeTransformState, err error) {
|
||||
if r.IncludeFilter != nil && !r.IncludeFilter(s, t) {
|
||||
return
|
||||
}
|
||||
if r.Transform != nil {
|
||||
if !r.NoShift {
|
||||
panic("Transform must be performed with NoShift=true")
|
||||
}
|
||||
if ns, err = func() (ns NodeTransformState, err error) {
|
||||
t, ns, err = r.Transform(s, t)
|
||||
if ns >= NodeTransformStateSkip || err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
switch ns {
|
||||
case NodeTransformStateReplaced:
|
||||
// Delay insert until after the walk to
|
||||
// avoid locking issues.
|
||||
if r.TransformDelayInsert {
|
||||
r.WalkContext.HooksPost1().Push(
|
||||
func() error {
|
||||
r.Tree.InsertRaw(s, t)
|
||||
return nil
|
||||
},
|
||||
)
|
||||
} else {
|
||||
r.Tree.InsertRaw(s, t)
|
||||
}
|
||||
case NodeTransformStateDeleted:
|
||||
// Delay delete until after the walk.
|
||||
deletes = append(deletes, s)
|
||||
ns = NodeTransformStateSkip
|
||||
}
|
||||
return
|
||||
}(); ns >= NodeTransformStateSkip || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if r.Handle != nil {
|
||||
var terminate bool
|
||||
terminate, err = r.Handle(s, t)
|
||||
if terminate || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
return func() error {
|
||||
if r.LockType > LockTypeNone {
|
||||
unlock := r.Tree.Lock(r.LockType == LockTypeWrite)
|
||||
defer unlock()
|
||||
}
|
||||
r.lock()
|
||||
defer r.unlock()
|
||||
|
||||
r.resetLocalState()
|
||||
|
||||
main := r.Tree
|
||||
|
||||
var err error
|
||||
|
||||
handleV := func(s string, v any) (terminate bool) {
|
||||
var handleV radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
var zero T
|
||||
// Context cancellation check.
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
err = ctx.Err()
|
||||
return true
|
||||
return radix.WalkStop, zero, ctx.Err()
|
||||
}
|
||||
if r.ShouldSkip(s, v.(T)) {
|
||||
return false
|
||||
if r.ShouldSkipValue(s, v) {
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
var t T
|
||||
if r.NoShift {
|
||||
t = v.(T)
|
||||
t = v
|
||||
} else {
|
||||
var ok bool
|
||||
t, ok = r.toT(r.Tree, v)
|
||||
if !ok {
|
||||
return false
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
}
|
||||
var ns NodeTransformState
|
||||
ns, err = handleT(s, t)
|
||||
if ns == NodeTransformStateTerminate || err != nil {
|
||||
return true
|
||||
var (
|
||||
ns NodeTransformState
|
||||
t2 T
|
||||
)
|
||||
if r.IncludeFilter != nil && !r.IncludeFilter(s, t) {
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
return false
|
||||
if r.Transform != nil {
|
||||
if !r.NoShift {
|
||||
panic("Transform must be performed with NoShift=true")
|
||||
}
|
||||
var err error
|
||||
ns, err = func() (ns NodeTransformState, err error) {
|
||||
t2, ns, err = r.Transform(s, t)
|
||||
if ns >= NodeTransformStateSkip || err != nil {
|
||||
return
|
||||
}
|
||||
switch ns {
|
||||
case NodeTransformStateReplaced:
|
||||
case NodeTransformStateDeleted:
|
||||
// Delay delete until after the walk.
|
||||
deletes = append(deletes, s)
|
||||
ns = NodeTransformStateSkip
|
||||
}
|
||||
return
|
||||
}()
|
||||
|
||||
if ns == NodeTransformStateTerminate || err != nil {
|
||||
return radix.WalkStop, zero, err
|
||||
}
|
||||
|
||||
if ns == NodeTransformStateSkip {
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if r.Handle != nil {
|
||||
f, err := r.Handle(s, t)
|
||||
if f.ShouldSet() {
|
||||
panic("Handle cannot replace nodes in the tree, use Transform for that")
|
||||
}
|
||||
if f.ShouldStop() || err != nil {
|
||||
return f, zero, err
|
||||
}
|
||||
}
|
||||
if ns == NodeTransformStateTerminate {
|
||||
return radix.WalkStop, zero, nil
|
||||
}
|
||||
if ns == NodeTransformStateReplaced {
|
||||
return radix.WalkSet | radix.WalkContinue, t2, nil
|
||||
}
|
||||
return radix.WalkContinue, zero, nil
|
||||
}
|
||||
|
||||
handle := &walkHandler[T]{
|
||||
r: r,
|
||||
handle: handleV,
|
||||
}
|
||||
|
||||
if r.Prefix != "" {
|
||||
main.tree.WalkPrefix(r.Prefix, handleV)
|
||||
if err := main.tree.WalkPrefix(r.Prefix, handle); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
main.tree.Walk(handleV)
|
||||
if err := main.tree.Walk(handle); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// This is currently only performed with no shift.
|
||||
@@ -580,7 +612,7 @@ func (r *NodeShiftTreeWalker[T]) Walk(ctx context.Context) error {
|
||||
main.tree.Delete(s)
|
||||
}
|
||||
|
||||
return err
|
||||
return nil
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -588,8 +620,8 @@ func (r *NodeShiftTreeWalker[T]) resetLocalState() {
|
||||
r.skipPrefixes = nil
|
||||
}
|
||||
|
||||
func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v any) (T, bool) {
|
||||
return tree.shift(v.(T), r.Fallback)
|
||||
func (r *NodeShiftTreeWalker[T]) toT(tree *NodeShiftTree[T], v T) (T, bool) {
|
||||
return tree.shift(v, r.Fallback)
|
||||
}
|
||||
|
||||
func (r *NodeShiftTree[T]) Has(s string) bool {
|
||||
@@ -613,7 +645,7 @@ func (r *NodeShiftTree[T]) get(s string) (T, bool) {
|
||||
var t T
|
||||
return t, false
|
||||
}
|
||||
if v, ok := r.shift(v.(T), false); ok {
|
||||
if v, ok := r.shift(v, false); ok {
|
||||
return v, true
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
"iter"
|
||||
"sync"
|
||||
|
||||
radix "github.com/armon/go-radix"
|
||||
radix "github.com/gohugoio/go-radix"
|
||||
)
|
||||
|
||||
// Tree is a non thread safe radix tree that holds T.
|
||||
@@ -43,28 +43,24 @@ type TreeCommon[T any] interface {
|
||||
}
|
||||
|
||||
func NewSimpleTree[T any]() *SimpleTree[T] {
|
||||
return &SimpleTree[T]{tree: radix.New()}
|
||||
return &SimpleTree[T]{tree: radix.New[T]()}
|
||||
}
|
||||
|
||||
// SimpleTree is a radix tree that holds T.
|
||||
// This tree is not thread safe.
|
||||
type SimpleTree[T any] struct {
|
||||
tree *radix.Tree
|
||||
tree *radix.Tree[T]
|
||||
zero T
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) Get(s string) T {
|
||||
if v, ok := tree.tree.Get(s); ok {
|
||||
return v.(T)
|
||||
}
|
||||
return tree.zero
|
||||
v, _ := tree.tree.Get(s)
|
||||
return v
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) LongestPrefix(s string) (string, T) {
|
||||
if s, v, ok := tree.tree.LongestPrefix(s); ok {
|
||||
return s, v.(T)
|
||||
}
|
||||
return "", tree.zero
|
||||
s, v, _ := tree.tree.LongestPrefix(s)
|
||||
return s, v
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) Insert(s string, v T) T {
|
||||
@@ -73,85 +69,79 @@ func (tree *SimpleTree[T]) Insert(s string, v T) T {
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) Walk(f func(s string, v T) (bool, error)) error {
|
||||
var err error
|
||||
tree.tree.Walk(func(s string, v any) bool {
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
var b bool
|
||||
b, err = f(s, v.(T))
|
||||
if err != nil {
|
||||
return true
|
||||
b, err := f(s, v)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, tree.zero, err
|
||||
}
|
||||
return b
|
||||
})
|
||||
return err
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
return tree.tree.Walk(walkFn)
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) WalkPrefix(s string, f func(s string, v T) (bool, error)) error {
|
||||
var err error
|
||||
tree.tree.WalkPrefix(s, func(s string, v any) bool {
|
||||
var b bool
|
||||
b, err = f(s, v.(T))
|
||||
if err != nil {
|
||||
return true
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
b, err := f(s, v)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, tree.zero, err
|
||||
}
|
||||
return b
|
||||
})
|
||||
|
||||
return err
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
return tree.tree.WalkPrefix(s, walkFn)
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) WalkPath(s string, f func(s string, v T) (bool, error)) error {
|
||||
var err error
|
||||
tree.tree.WalkPath(s, func(s string, v any) bool {
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
var b bool
|
||||
b, err = f(s, v.(T))
|
||||
if err != nil {
|
||||
return true
|
||||
b, err = f(s, v)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, tree.zero, err
|
||||
}
|
||||
return b
|
||||
})
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
tree.tree.WalkPath(s, walkFn)
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *SimpleTree[T]) All() iter.Seq2[string, T] {
|
||||
return func(yield func(s string, v T) bool) {
|
||||
tree.tree.Walk(func(s string, v any) bool {
|
||||
return !yield(s, v.(T))
|
||||
})
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
if !yield(s, v) {
|
||||
return radix.WalkStop, tree.zero, nil
|
||||
}
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
tree.tree.Walk(walkFn)
|
||||
}
|
||||
}
|
||||
|
||||
// NewSimpleThreadSafeTree creates a new SimpleTree.
|
||||
func NewSimpleThreadSafeTree[T any]() *SimpleThreadSafeTree[T] {
|
||||
return &SimpleThreadSafeTree[T]{tree: radix.New(), mu: new(sync.RWMutex)}
|
||||
return &SimpleThreadSafeTree[T]{tree: radix.New[T](), mu: new(sync.RWMutex)}
|
||||
}
|
||||
|
||||
// SimpleThreadSafeTree is a thread safe radix tree that holds T.
|
||||
type SimpleThreadSafeTree[T any] struct {
|
||||
mu *sync.RWMutex
|
||||
tree *radix.Tree
|
||||
tree *radix.Tree[T]
|
||||
zero T
|
||||
}
|
||||
|
||||
var noopFunc = func() {}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) Get(s string) T {
|
||||
tree.mu.RLock()
|
||||
defer tree.mu.RUnlock()
|
||||
|
||||
if v, ok := tree.tree.Get(s); ok {
|
||||
return v.(T)
|
||||
}
|
||||
return tree.zero
|
||||
v, _ := tree.tree.Get(s)
|
||||
return v
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) LongestPrefix(s string) (string, T) {
|
||||
tree.mu.RLock()
|
||||
defer tree.mu.RUnlock()
|
||||
|
||||
if s, v, ok := tree.tree.LongestPrefix(s); ok {
|
||||
return s, v.(T)
|
||||
}
|
||||
return "", tree.zero
|
||||
s, v, _ := tree.tree.LongestPrefix(s)
|
||||
return s, v
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) Insert(s string, v T) T {
|
||||
@@ -162,60 +152,65 @@ func (tree *SimpleThreadSafeTree[T]) Insert(s string, v T) T {
|
||||
return v
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) Lock(lockType LockType) func() {
|
||||
func (tree *SimpleThreadSafeTree[T]) Lock(lockType LockType) {
|
||||
switch lockType {
|
||||
case LockTypeNone:
|
||||
return noopFunc
|
||||
case LockTypeRead:
|
||||
tree.mu.RLock()
|
||||
return tree.mu.RUnlock
|
||||
case LockTypeWrite:
|
||||
tree.mu.Lock()
|
||||
return tree.mu.Unlock
|
||||
}
|
||||
return noopFunc
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) Unlock(lockType LockType) {
|
||||
switch lockType {
|
||||
case LockTypeRead:
|
||||
tree.mu.RUnlock()
|
||||
case LockTypeWrite:
|
||||
tree.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
|
||||
commit := tree.Lock(lockType)
|
||||
defer commit()
|
||||
var err error
|
||||
tree.tree.WalkPrefix(s, func(s string, v any) bool {
|
||||
tree.Lock(lockType)
|
||||
defer tree.Unlock(lockType)
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
var b bool
|
||||
b, err = f(s, v.(T))
|
||||
if err != nil {
|
||||
return true
|
||||
b, err := f(s, v)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, tree.zero, err
|
||||
}
|
||||
return b
|
||||
})
|
||||
|
||||
return err
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
return tree.tree.WalkPrefix(s, walkFn)
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) WalkPath(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
|
||||
commit := tree.Lock(lockType)
|
||||
defer commit()
|
||||
tree.Lock(lockType)
|
||||
defer tree.Unlock(lockType)
|
||||
var err error
|
||||
tree.tree.WalkPath(s, func(s string, v any) bool {
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
var b bool
|
||||
b, err = f(s, v.(T))
|
||||
if err != nil {
|
||||
return true
|
||||
b, err = f(s, v)
|
||||
if b || err != nil {
|
||||
return radix.WalkStop, tree.zero, err
|
||||
}
|
||||
return b
|
||||
})
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
tree.tree.WalkPath(s, walkFn)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (tree *SimpleThreadSafeTree[T]) All(lockType LockType) iter.Seq2[string, T] {
|
||||
commit := tree.Lock(lockType)
|
||||
defer commit()
|
||||
return func(yield func(s string, v T) bool) {
|
||||
tree.tree.Walk(func(s string, v any) bool {
|
||||
return !yield(s, v.(T))
|
||||
})
|
||||
tree.Lock(lockType)
|
||||
defer tree.Unlock(lockType)
|
||||
var walkFn radix.WalkFn[T] = func(s string, v T) (radix.WalkFlag, T, error) {
|
||||
if !yield(s, v) {
|
||||
return radix.WalkStop, tree.zero, nil
|
||||
}
|
||||
return radix.WalkContinue, tree.zero, nil
|
||||
}
|
||||
tree.tree.Walk(walkFn)
|
||||
}
|
||||
}
|
||||
|
||||
// iter.Seq[*TemplWithBaseApplied]
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
radix "github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/collections"
|
||||
"github.com/gohugoio/hugo/common/maps"
|
||||
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
|
||||
@@ -139,23 +140,27 @@ type MutableTree interface {
|
||||
DeleteRaw(key string)
|
||||
DeletePrefix(prefix string) int
|
||||
DeletePrefixRaw(prefix string) int
|
||||
Lock(writable bool) (commit func())
|
||||
Lock(writable bool)
|
||||
Unlock(writable bool)
|
||||
CanLock() bool // Used for troubleshooting only.
|
||||
}
|
||||
|
||||
// WalkableTree is a tree that can be walked.
|
||||
type WalkableTree[T any] interface {
|
||||
WalkPrefixRaw(prefix string, walker func(key string, value T) bool)
|
||||
WalkPrefixRaw(prefix string, walker radix.WalkFn[T]) error
|
||||
}
|
||||
|
||||
var _ WalkableTree[any] = (*WalkableTrees[any])(nil)
|
||||
|
||||
type WalkableTrees[T any] []WalkableTree[T]
|
||||
|
||||
func (t WalkableTrees[T]) WalkPrefixRaw(prefix string, walker func(key string, value T) bool) {
|
||||
func (t WalkableTrees[T]) WalkPrefixRaw(prefix string, walker radix.WalkFn[T]) error {
|
||||
for _, tree := range t {
|
||||
tree.WalkPrefixRaw(prefix, walker)
|
||||
if err := tree.WalkPrefixRaw(prefix, walker); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ MutableTree = MutableTrees(nil)
|
||||
@@ -184,15 +189,15 @@ func (t MutableTrees) DeletePrefixRaw(prefix string) int {
|
||||
return count
|
||||
}
|
||||
|
||||
func (t MutableTrees) Lock(writable bool) (commit func()) {
|
||||
commits := make([]func(), len(t))
|
||||
for i, tree := range t {
|
||||
commits[i] = tree.Lock(writable)
|
||||
func (t MutableTrees) Lock(writable bool) {
|
||||
for _, tree := range t {
|
||||
tree.Lock(writable)
|
||||
}
|
||||
return func() {
|
||||
for _, commit := range commits {
|
||||
commit()
|
||||
}
|
||||
}
|
||||
|
||||
func (t MutableTrees) Unlock(writable bool) {
|
||||
for _, tree := range t {
|
||||
tree.Unlock(writable)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,11 +223,8 @@ type WalkContext[T any] struct {
|
||||
eventMu sync.Mutex
|
||||
events []*Event[T]
|
||||
|
||||
hooksPost1Init sync.Once
|
||||
hooksPost1 *collections.Stack[func() error]
|
||||
|
||||
hooksPost2Init sync.Once
|
||||
hooksPost2 *collections.Stack[func() error]
|
||||
hooksPostInit sync.Once
|
||||
hooksPost *collections.Stack[func() error]
|
||||
}
|
||||
|
||||
type eventHandlers[T any] map[string][]func(*Event[T])
|
||||
@@ -259,32 +261,19 @@ func (ctx *WalkContext[T]) HandleEvents() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ctx *WalkContext[T]) HooksPost1() *collections.Stack[func() error] {
|
||||
ctx.hooksPost1Init.Do(func() {
|
||||
ctx.hooksPost1 = collections.NewStack[func() error]()
|
||||
func (ctx *WalkContext[T]) HooksPost() *collections.Stack[func() error] {
|
||||
ctx.hooksPostInit.Do(func() {
|
||||
ctx.hooksPost = collections.NewStack[func() error]()
|
||||
})
|
||||
return ctx.hooksPost1
|
||||
return ctx.hooksPost
|
||||
}
|
||||
|
||||
func (ctx *WalkContext[T]) HooksPost2() *collections.Stack[func() error] {
|
||||
ctx.hooksPost2Init.Do(func() {
|
||||
ctx.hooksPost2 = collections.NewStack[func() error]()
|
||||
})
|
||||
return ctx.hooksPost2
|
||||
}
|
||||
|
||||
func (ctx *WalkContext[T]) HandleHooks1AndEventsAndHooks2() error {
|
||||
for _, hook := range ctx.HooksPost1().All() {
|
||||
if err := hook(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (ctx *WalkContext[T]) HandleEventsAndHooks() error {
|
||||
if err := ctx.HandleEvents(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, hook := range ctx.HooksPost2().All() {
|
||||
for _, hook := range ctx.HooksPost().All() {
|
||||
if err := hook(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -93,8 +93,12 @@ func (t *TreeShiftTreeSlice[T]) Insert(s string, v T) T {
|
||||
return t.tree().Insert(s, v)
|
||||
}
|
||||
|
||||
func (t *TreeShiftTreeSlice[T]) Lock(lockType LockType) func() {
|
||||
return t.tree().Lock(lockType)
|
||||
func (t *TreeShiftTreeSlice[T]) Lock(lockType LockType) {
|
||||
t.tree().Lock(lockType)
|
||||
}
|
||||
|
||||
func (t *TreeShiftTreeSlice[T]) Unlock(lockType LockType) {
|
||||
t.tree().Unlock(lockType)
|
||||
}
|
||||
|
||||
func (t *TreeShiftTreeSlice[T]) WalkPrefix(lockType LockType, s string, f func(s string, v T) (bool, error)) error {
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/cache/dynacache"
|
||||
"github.com/gohugoio/hugo/config/allconfig"
|
||||
"github.com/gohugoio/hugo/hugofs/hglob"
|
||||
@@ -536,8 +537,11 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: s.pageMap.treePages,
|
||||
LockType: doctree.LockTypeRead,
|
||||
Handle: func(s string, n contentNode) (bool, error) {
|
||||
return fn(s, n.(*pageState)), nil
|
||||
Handle: func(s string, n contentNode) (radix.WalkFlag, error) {
|
||||
if fn(s, n.(*pageState)) {
|
||||
return radix.WalkStop, nil
|
||||
}
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
return w.Walk(context.Background())
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
"github.com/bep/debounce"
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/bufferpool"
|
||||
"github.com/gohugoio/hugo/deps"
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
@@ -261,9 +262,9 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error {
|
||||
return errors.New("rebuild called when not in watch mode")
|
||||
}
|
||||
|
||||
h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNode) bool {
|
||||
h.pageTrees.treePagesResources.WalkPrefixRaw("", func(key string, n contentNode) (radix.WalkFlag, contentNode, error) {
|
||||
cnh.resetBuildState(n)
|
||||
return false
|
||||
return radix.WalkContinue, nil, nil
|
||||
})
|
||||
|
||||
for _, s := range h.Sites {
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/paths"
|
||||
"github.com/gohugoio/hugo/common/types"
|
||||
"github.com/gohugoio/hugo/hugolib/doctree"
|
||||
@@ -167,9 +168,9 @@ func (pt pageTree) Sections() page.Pages {
|
||||
Tree: tree,
|
||||
Prefix: prefix,
|
||||
}
|
||||
w.Handle = func(ss string, n contentNode) (bool, error) {
|
||||
w.Handle = func(ss string, n contentNode) (radix.WalkFlag, error) {
|
||||
if !cnh.isBranchNode(n) {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
if currentBranchPrefix == "" || !strings.HasPrefix(ss, currentBranchPrefix) {
|
||||
if p, ok := n.(*pageState); ok && p.IsSection() && p.m.shouldList(false) && p.Parent() == pt.p {
|
||||
@@ -179,7 +180,7 @@ func (pt pageTree) Sections() page.Pages {
|
||||
}
|
||||
}
|
||||
currentBranchPrefix = ss + "/"
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
if err := w.Walk(context.Background()); err != nil {
|
||||
|
||||
@@ -30,6 +30,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/cache/dynacache"
|
||||
"github.com/gohugoio/hugo/common/hstore"
|
||||
"github.com/gohugoio/hugo/common/hsync"
|
||||
@@ -928,7 +929,7 @@ func (s *Site) initRenderFormats() {
|
||||
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: s.pageMap.treePages,
|
||||
Handle: func(key string, n contentNode) (bool, error) {
|
||||
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
if p, ok := n.(*pageState); ok {
|
||||
for _, f := range p.m.pageConfig.ConfiguredOutputFormats {
|
||||
if !formatSet[f.Name] {
|
||||
@@ -937,7 +938,7 @@ func (s *Site) initRenderFormats() {
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/go-radix"
|
||||
"github.com/gohugoio/hugo/common/herrors"
|
||||
"github.com/gohugoio/hugo/hugolib/doctree"
|
||||
"github.com/gohugoio/hugo/tpl/tplimpl"
|
||||
@@ -87,18 +88,18 @@ func (s *Site) renderPages(ctx *siteRenderContext) error {
|
||||
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: s.pageMap.treePages,
|
||||
Handle: func(key string, n contentNode) (bool, error) {
|
||||
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
if p, ok := n.(*pageState); ok {
|
||||
if cfg.shouldRender(ctx.infol, p) {
|
||||
select {
|
||||
case <-s.h.Done():
|
||||
return true, nil
|
||||
return radix.WalkStop, nil
|
||||
default:
|
||||
pages <- p
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
|
||||
@@ -270,16 +271,16 @@ func (s *Site) renderPaginator(p *pageState, templ *tplimpl.TemplInfo) error {
|
||||
func (s *Site) renderAliases() error {
|
||||
w := &doctree.NodeShiftTreeWalker[contentNode]{
|
||||
Tree: s.pageMap.treePages,
|
||||
Handle: func(key string, n contentNode) (bool, error) {
|
||||
Handle: func(key string, n contentNode) (radix.WalkFlag, error) {
|
||||
p := n.(*pageState)
|
||||
|
||||
// We cannot alias a page that's not rendered.
|
||||
if p.m.noLink() || p.skipRender() {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
if len(p.Aliases()) == 0 {
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
}
|
||||
|
||||
pathSeen := make(map[string]bool)
|
||||
@@ -324,11 +325,11 @@ func (s *Site) renderAliases() error {
|
||||
|
||||
err := s.writeDestAlias(a, plink, f, p)
|
||||
if err != nil {
|
||||
return true, err
|
||||
return radix.WalkStop, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
return radix.WalkContinue, nil
|
||||
},
|
||||
}
|
||||
return w.Walk(context.TODO())
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# TODO1 clean up when done.
|
||||
go generate ./gen
|
||||
javy compile js/greet.bundle.js -d -o wasm/greet.wasm
|
||||
javy compile js/renderkatex.bundle.js -d -o wasm/renderkatex.wasm
|
||||
|
||||
@@ -202,63 +202,6 @@ func (t *templateNamespace) templatesIn(in tpl.Template) iter.Seq[tpl.Template]
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
|
||||
func (t *templateHandler) applyBaseTemplate(overlay, base templateInfo) (tpl.Template, error) {
|
||||
if overlay.isText {
|
||||
var (
|
||||
templ = t.main.getPrototypeText(prototypeCloneIDBaseof).New(overlay.name)
|
||||
err error
|
||||
)
|
||||
|
||||
if !base.IsZero() {
|
||||
templ, err = templ.Parse(base.template)
|
||||
if err != nil {
|
||||
return nil, base.errWithFileContext("text: base: parse failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
templ, err = texttemplate.Must(templ.Clone()).Parse(overlay.template)
|
||||
if err != nil {
|
||||
return nil, overlay.errWithFileContext("text: overlay: parse failed", err)
|
||||
}
|
||||
|
||||
// The extra lookup is a workaround, see
|
||||
// * https://github.com/golang/go/issues/16101
|
||||
// * https://github.com/gohugoio/hugo/issues/2549
|
||||
// templ = templ.Lookup(templ.Name())
|
||||
|
||||
return templ, nil
|
||||
}
|
||||
|
||||
var (
|
||||
templ = t.main.getPrototypeHTML(prototypeCloneIDBaseof).New(overlay.name)
|
||||
err error
|
||||
)
|
||||
|
||||
if !base.IsZero() {
|
||||
templ, err = templ.Parse(base.template)
|
||||
if err != nil {
|
||||
return nil, base.errWithFileContext("html: base: parse failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
templ, err = htmltemplate.Must(templ.Clone()).Parse(overlay.template)
|
||||
if err != nil {
|
||||
return nil, overlay.errWithFileContext("html: overlay: parse failed", err)
|
||||
}
|
||||
|
||||
// The extra lookup is a workaround, see
|
||||
// * https://github.com/golang/go/issues/16101
|
||||
// * https://github.com/gohugoio/hugo/issues/2549
|
||||
templ = templ.Lookup(templ.Name())
|
||||
|
||||
return templ, err
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
var baseTemplateDefineRe = regexp.MustCompile(`^{{-?\s*define`)
|
||||
|
||||
// needsBaseTemplate returns true if the first non-comment template block is a
|
||||
|
||||
@@ -269,14 +269,13 @@ func (ti *TemplInfo) SubCategory() SubCategory {
|
||||
|
||||
func (ti *TemplInfo) BaseVariantsSeq() iter.Seq[*TemplWithBaseApplied] {
|
||||
return func(yield func(*TemplWithBaseApplied) bool) {
|
||||
ti.baseVariants.Walk(func(key string, v map[TemplateDescriptor]*TemplWithBaseApplied) (bool, error) {
|
||||
for _, v := range ti.baseVariants.All() {
|
||||
for _, vv := range v {
|
||||
if !yield(vv) {
|
||||
return true, nil
|
||||
return
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user