averello/hugo-mirror
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2025-12-13 20:36:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

tpl/openapi: Add support for OpenAPI external file references

Browse Source 
Fixes #8067
This commit is contained in:
Bjørn Erik Pedersen
2025-11-20 11:01:23 +01:00
parent 2b337cd212
commit 84950ed2b2
8 changed files with 404 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
deps/deps.go vendored
View File

@@ -417,6 +417,9 @@ type DepsCfg struct {
// i18n handling.
TranslationProvider ResourceProvider
// Build triggered by the IntegrationTest framework.
IsIntegrationTest bool
// ChangesFromBuild for changes passed back to the server/watch process.
ChangesFromBuild chan []identity.Identity
}

21
hugolib/integrationtest_builder.go
View File

@@ -35,6 +35,7 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugofs/hglob"
"github.com/gohugoio/hugo/hugolib/sitesmatrix"
"github.com/gohugoio/hugo/identity"
"github.com/spf13/afero"
"github.com/spf13/cast"
"golang.org/x/text/unicode/norm"
@@ -838,7 +839,11 @@ func (s *IntegrationTestBuilder) initBuilder() error {
s.Assert(err, qt.IsNil)
depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr()}
// changes received from Hugo in watch mode.
// In the full setup, this channel is created in the commands package.
changesFromBuild := make(chan []identity.Identity, 10)
depsCfg := deps.DepsCfg{Configs: res, Fs: fs, LogLevel: logger.Level(), StdErr: logger.StdErr(), ChangesFromBuild: changesFromBuild, IsIntegrationTest: true}
sites, err := NewHugoSites(depsCfg)
if err != nil {
initErr = err
@@ -849,6 +854,20 @@ func (s *IntegrationTestBuilder) initBuilder() error {
return
}
go func() {
for id := range changesFromBuild {
whatChanged := &WhatChanged{}
for _, v := range id {
whatChanged.Add(v)
}
bcfg := s.Cfg.BuildCfg
bcfg.WhatChanged = whatChanged
if err := s.build(bcfg); err != nil {
s.Fatalf("Build failed after change: %s", err)
}
}
}()
s.H = sites
s.fs = fs

2
hugolib/rebuild_test.go
View File

@@ -845,7 +845,7 @@ func TestRebuildEditSectionRemoveDate(t *testing.T) {
func TestRebuildVariations(t *testing.T) {
// t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4
// This leaktest seems to be a little bit shaky on Travis.
if !htesting.IsCI() {
if !htesting.IsRealCI() {
defer leaktest.CheckTimeout(t, 10*time.Second)()
}

8
hugolib/site.go
View File

@@ -265,6 +265,14 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
return nil, err
}
// Prevent leaking goroutines in tests.
if cfg.IsIntegrationTest && cfg.ChangesFromBuild != nil {
firstSiteDeps.BuildClosers.Add(types.CloserFunc(func() error {
close(cfg.ChangesFromBuild)
return nil
}))
}
batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps)
if err != nil {
return nil, err

13
resources/resource.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/internal"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/common/hashing"
"github.com/gohugoio/hugo/common/herrors"
@@ -703,6 +704,18 @@ func InternalResourceSourcePath(r resource.Resource) string {
return ""
}
// InternalResourceSourceContent is used internally to get the source content for a Resource.
func InternalResourceSourceContent(ctx context.Context, r resource.Resource) (string, error) {
if cp, ok := r.(resource.ContentProvider); ok {
c, err := cp.Content(ctx)
if err != nil {
return "", err
}
return cast.ToStringE(c)
}
return "", nil
}
// InternalResourceSourcePathBestEffort is used internally to get the source path for a Resource.
// Used for error messages etc.
// It will fall back to the target path if the source path is not available.

12
tpl/openapi/openapi3/init.go
View File

@@ -18,6 +18,7 @@ import (
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/tpl/internal"
resourcestpl "github.com/gohugoio/hugo/tpl/resources"
)
const name = "openapi3"
@@ -29,6 +30,17 @@ func init() {
ns := &internal.TemplateFuncsNamespace{
Name: name,
Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil },
OnCreated: func(m map[string]any) {
for _, v := range m {
switch v := v.(type) {
case *resourcestpl.Namespace:
ctx.resourcesNs = v
}
}
if ctx.resourcesNs == nil {
panic("resources namespace not found")
}
},
}
ns.AddMethodMapping(ctx.Unmarshal,

114
tpl/openapi/openapi3/openapi3.go
View File

@@ -15,16 +15,26 @@
package openapi3
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"path"
"path/filepath"
"strings"
kopenapi3 "github.com/getkin/kin-openapi/openapi3"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/common/hashing"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
resourcestpl "github.com/gohugoio/hugo/tpl/resources"
"github.com/mitchellh/mapstructure"
)
// New returns a new instance of the openapi3-namespaced template functions.
@@ -37,8 +47,9 @@ func New(deps *deps.Deps) *Namespace {
// Namespace provides template functions for the "openapi3".
type Namespace struct {
cache *dynacache.Partition[string, *OpenAPIDocument]
deps *deps.Deps
cache *dynacache.Partition[string, *OpenAPIDocument]
deps *deps.Deps
resourcesNs *resourcestpl.Namespace
}
// OpenAPIDocument represents an OpenAPI 3 document.
@@ -51,13 +62,35 @@ func (o *OpenAPIDocument) GetIdentityGroup() identity.Identity {
return o.identityGroup
}
type unmarshalOptions struct {
// Options passed to resources.GetRemote when resolving remote $ref.
GetRemote map[string]any
}
// Unmarshal unmarshals the given resource into an OpenAPI 3 document.
func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument, error) {
func (ns *Namespace) Unmarshal(ctx context.Context, args ...any) (*OpenAPIDocument, error) {
if len(args) < 1 || len(args) > 2 {
return nil, errors.New("must provide a Resource and optionally an options map")
}
r := args[0].(resource.UnmarshableResource)
key := r.Key()
if key == "" {
return nil, errors.New("no Key set in Resource")
}
var opts unmarshalOptions
if len(args) > 1 {
optsm, err := maps.ToStringMapE(args[1])
if err != nil {
return nil, err
}
if err := mapstructure.WeakDecode(optsm, &opts); err != nil {
return nil, err
}
key += "_" + hashing.HashString(optsm)
}
v, err := ns.cache.GetOrCreate(key, func(string) (*OpenAPIDocument, error) {
f := metadecoders.FormatFromStrings(r.MediaType().Suffixes()...)
if f == "" {
@@ -86,9 +119,37 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument
return nil, err
}
err = kopenapi3.NewLoader().ResolveRefsIn(s, nil)
var resourcePath string
if res, ok := r.(resource.Resource); ok {
resourcePath = resources.InternalResourceSourcePath(res)
}
var relDir string
if resourcePath != "" {
if rel, ok := ns.deps.Assets.MakePathRelative(filepath.FromSlash(resourcePath), true); ok {
relDir = filepath.Dir(rel)
}
}
return &OpenAPIDocument{T: s, identityGroup: identity.FirstIdentity(r)}, err
var idm identity.Manager = identity.NopManager
if v := identity.GetDependencyManager(r); v != nil {
idm = v
}
idg := identity.FirstIdentity(r)
resolver := &refResolver{
ctx: ctx,
idm: idm,
opts: opts,
relBase: filepath.ToSlash(relDir),
ns: ns,
}
loader := kopenapi3.NewLoader()
loader.IsExternalRefsAllowed = true
loader.ReadFromURIFunc = resolver.resolveExternalRef
err = loader.ResolveRefsIn(s, nil)
return &OpenAPIDocument{T: s, identityGroup: idg}, err
})
if err != nil {
return nil, err
@@ -96,3 +157,46 @@ func (ns *Namespace) Unmarshal(r resource.UnmarshableResource) (*OpenAPIDocument
return v, nil
}
type refResolver struct {
ctx context.Context
idm identity.Manager
opts unmarshalOptions
relBase string
ns *Namespace
}
// resolveExternalRef resolves external references in OpenAPI documents by either fetching
// remote URLs or loading local files from the assets directory, depending on the reference location.
func (r *refResolver) resolveExternalRef(loader *kopenapi3.Loader, loc *url.URL) ([]byte, error) {
if loc.Scheme != "" && loc.Host != "" {
res, err := r.ns.resourcesNs.GetRemote(loc.String(), r.opts.GetRemote)
if err != nil {
return nil, fmt.Errorf("failed to get remote ref %q: %w", loc.String(), err)
}
content, err := resources.InternalResourceSourceContent(r.ctx, res)
if err != nil {
return nil, fmt.Errorf("failed to read remote ref %q: %w", loc.String(), err)
}
r.idm.AddIdentity(identity.FirstIdentity(res))
return []byte(content), nil
}
var filename string
if strings.HasPrefix(loc.Path, "/") {
filename = loc.Path
} else {
filename = path.Join(r.relBase, loc.Path)
}
res := r.ns.resourcesNs.Get(filename)
if res == nil {
return nil, fmt.Errorf("local ref %q not found", loc.String())
}
content, err := resources.InternalResourceSourceContent(r.ctx, res)
if err != nil {
return nil, fmt.Errorf("failed to read local ref %q: %w", loc.String(), err)
}
r.idm.AddIdentity(identity.FirstIdentity(res))
return []byte(content), nil
}

247
tpl/openapi/openapi3/openapi3_integration_test.go
View File

@@ -14,8 +14,12 @@
package openapi3_test
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gohugoio/hugo/hugolib"
)
@@ -49,20 +53,15 @@ paths:
type: array
items:
type: string
-- config.toml --
-- hugo.toml --
baseURL = 'http://example.com/'
-- layouts/index.html --
disableLiveReload = true
-- layouts/home.html --
{{ $api := resources.Get "api/myapi.yaml" | openapi3.Unmarshal }}
API: {{ $api.Info.Title | safeHTML }}
`
b := hugolib.NewIntegrationTestBuilder(
hugolib.IntegrationTestConfig{
T: t,
Running: true,
TxtarString: files,
},
).Build()
b := hugolib.TestRunning(t, files)
b.AssertFileContent("public/index.html", `API: Sample API`)
@@ -72,3 +71,233 @@ API: {{ $api.Info.Title | safeHTML }}
b.AssertFileContent("public/index.html", `API: Hugo API`)
}
// Test data borrowed/adapted from https://github.com/getkin/kin-openapi/tree/master/openapi3/testdata/refInLocalRef
// The templates below would be simpler if kin-openapi's T would serialize to JSON better, see https://github.com/getkin/kin-openapi/issues/561
const refLocalTemplate = `
-- hugo.toml --
baseURL = 'http://example.com/'
disableKinds = ["page", "section", "taxonomy", "term", "sitemap", "robotsTXT", "404"]
disableLiveReload = true
[HTTPCache]
[HTTPCache.cache]
[HTTPCache.cache.for]
includes = ['**']
[[HTTPCache.polls]]
high = '100ms'
low = '50ms'
[HTTPCache.polls.for]
includes = ['**']
-- assets/api/myapi.json --
{
"openapi": "3.0.3",
"info": {
"title": "Reference in reference example",
"version": "1.0.0"
},
"paths": {
"/api/test/ref/in/ref": {
"post": {
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties" : {
"data": {
"$ref": "#/components/schemas/Request"
}
}
}
}
}
},
"responses": {
"200": {
"description": "Successful response",
"content": {
"application/json": {
"schema": {
"$ref": "messages/response.json"
}
}
}
}
}
}
}
},
"components": {
"schemas": {
"Request": {
"$ref": "messages/request.json"
}
}
}
}
-- assets/api/messages/request.json --
{
"type": "object",
"required": [
"definition_reference"
],
"properties": {
"definition_reference": {
"$ref": "./data.json"
}
}
}
-- assets/api/messages/response.json --
{
"type": "object",
"properties": {
"id": {
"type": "integer",
"format": "uint64"
}
}
}
-- assets/api/messages/data.json --
{
"type": "object",
"properties": {
"id": {
"type": "integer",
"format": "int32"
},
"ref_prop_part": {
"$ref": "DATAPART_REF"
}
}
}
-- assets/api/messages/dataPart.json --
{
"type": "object",
"properties": {
"idPart": {
"type": "integer",
"format": "int64"
}
}
}
-- layouts/home.html --
{{ $getremote := dict
"method" "post"
"key" time.Now.UnixNano
}}
{{ $opts := dict
"getremote" $getremote
}}
{{ with resources.Get "api/myapi.json" }}
{{ with openapi3.Unmarshal . $opts }}
Title: {{ .Info.Title | safeHTML }}
{{ range $k, $v := .Paths.Map }}
{{ $post := index $v.Post }}
{{ with index $post.RequestBody.Value.Content }}
{{ $mt := (index . "application/json")}}
{{ $data := index $mt.Schema.Value.Properties "data" }}
RequestBody: {{ template "printschema" $mt.Schema.Value }}$
{{ end }}
{{ $response := index (index $post.Responses.Map "200").Value.Content "application/json" }}
Response: {{ template "printschema" $response.Schema.Value }}
{{ end }}
{{ end }}
{{ end }}
{{ define "printschema" }}{{ with .Format}}Format: {{ . }}|{{ end }}{{ with .Properties }} Properties: {{ range $k, $v := . }}{{ $k}}: {{ template "printitems" . -}}{{ end }}{{ end -}}${{ end }}
{{ define "printitems" }}{{ if eq (printf "%T" .) "*openapi3.SchemaRef" }}{{ template "printschema" .Value }}{{ else }}{{ template "printitem" . -}}{{ end -}}{{ end }}
{{ define "printitem" }}{{ printf "%T: %s" . (.|debug.Dump) | safeHTML }}{{ end }}
`
func TestUnmarshalRefLocal(t *testing.T) {
files := strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", "./dataPart.json")
b := hugolib.Test(t, files)
b.AssertFileContent("public/index.html",
`Reference in reference example`,
"RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int64|$",
"Response: Properties: id: Format: uint64|$",
)
}
func TestUnmarshalRefLocalEdit(t *testing.T) {
files := strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", "./dataPart.json")
b := hugolib.TestRunning(t, files)
b.AssertFileContent("public/index.html",
"RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int64|$",
)
b.EditFileReplaceAll("assets/api/messages/dataPart.json", "int64", "int8").Build()
b.AssertFileContent("public/index.html",
"RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int8|$",
)
}
func TestUnmarshalRefRemote(t *testing.T) {
createFiles := func(t *testing.T) string {
counter := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" && r.URL.Path == "/api/messages/dataPart.json" {
n := 8
// This endpoint will also be called by the poller, so we distinguish
// between the first and subsequent calls.
if counter > 0 {
n = 16
}
counter++
w.Header().Set("Content-Type", "application/json")
response := fmt.Sprintf(`{
"type": "object",
"properties": {
"idPart": {
"type": "integer",
"format": "int%d"
}
}
}`, n)
w.Write([]byte(response))
return
}
http.Error(w, "Not found", http.StatusNotFound)
}))
t.Cleanup(func() {
ts.Close()
})
dataPartRef := ts.URL + "/api/messages/dataPart.json"
return strings.ReplaceAll(refLocalTemplate, "DATAPART_REF", dataPartRef)
}
t.Run("Build", func(t *testing.T) {
b := hugolib.Test(t, createFiles(t))
b.AssertFileContent("public/index.html",
`Reference in reference example`,
"RequestBody: Properties: data: Properties: definition_reference: Properties: id: Format: int32|$ref_prop_part: Properties: idPart: Format: int8|$",
"Response: Properties: id: Format: uint64|$",
)
})
t.Run("Rebuild", func(t *testing.T) {
b := hugolib.TestRunning(t, createFiles(t))
b.AssertFileContent("public/index.html",
"idPart: Format: int8|$",
)
// Rebuild triggered by remote polling.
time.Sleep(800 * time.Millisecond)
b.AssertFileContent("public/index.html",
"idPart: Format: int16|$",
)
})
}