phoenix/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-06-27 21:25:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a HTTP cache for remote resources.

Browse source 
Fixes #12502
Closes #11891
This commit is contained in:
Bjørn Erik Pedersen 2024-05-17 17:06:47 +02:00
parent c71e24af51
commit 447108fed2
No known key found for this signature in database
32 changed files with 1150 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

122
cache/filecache/filecache.go vendored
View file

@ -1,4 +1,4 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -23,6 +23,7 @@ import (
"sync"
"time"
"github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/hugofs"
@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string,
return
}
// NamedLock locks the given id. The lock is released when the returned function is called.
func (c *Cache) NamedLock(id string) func() {
id = cleanID(id)
c.nlocker.Lock(id)
return func() {
c.nlocker.Unlock(id)
}
}
// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
@ -218,7 +228,23 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
var buff bytes.Buffer
return info,
hugio.ToReadCloser(&buff),
afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
c.writeReader(id, io.TeeReader(r, &buff))
}
func (c *Cache) writeReader(id string, r io.Reader) error {
dir := filepath.Dir(id)
if dir != "" {
_ = c.Fs.MkdirAll(dir, 0o777)
}
f, err := c.Fs.Create(id)
if err != nil {
return err
}
defer f.Close()
_, _ = io.Copy(f, r)
return nil
}
// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
@ -253,9 +279,10 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
return info, b, nil
}
if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
if err := c.writeReader(id, bytes.NewReader(b)); err != nil {
return info, nil, err
}
return info, b, nil
}
@ -305,16 +332,8 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
return nil
}
if c.maxAge > 0 {
fi, err := c.Fs.Stat(id)
if err != nil {
return nil
}
if c.isExpired(fi.ModTime()) {
c.Fs.Remove(id)
return nil
}
if removed, err := c.removeIfExpired(id); err != nil || removed {
return nil
}
f, err := c.Fs.Open(id)
@ -325,6 +344,49 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
return f
}
func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
if c.maxAge == 0 {
// No caching.
return nil, false
}
f, err := c.Fs.Open(id)
if err != nil {
return nil, false
}
defer f.Close()
b, err := io.ReadAll(f)
if err != nil {
return nil, false
}
removed, err := c.removeIfExpired(id)
if err != nil {
return nil, false
}
return b, removed
}
func (c *Cache) removeIfExpired(id string) (bool, error) {
if c.maxAge <= 0 {
return false, nil
}
fi, err := c.Fs.Stat(id)
if err != nil {
return false, err
}
if c.isExpired(fi.ModTime()) {
c.Fs.Remove(id)
return true, nil
}
return false, nil
}
func (c *Cache) isExpired(modTime time.Time) bool {
if c.maxAge < 0 {
return false
@ -398,3 +460,37 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) {
func cleanID(name string) string {
return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator)
}
// AsHTTPCache returns an httpcache.Cache implementation for this file cache.
// Note that none of the methods are protected by named locks, so you need to make sure
// to do that in your own code.
func (c *Cache) AsHTTPCache() httpcache.Cache {
return &httpCache{c: c}
}
type httpCache struct {
c *Cache
}
func (h *httpCache) Get(id string) (resp []byte, ok bool) {
id = cleanID(id)
b, removed := h.c.getBytesAndRemoveIfExpired(id)
return b, !removed
}
func (h *httpCache) Set(id string, resp []byte) {
if h.c.maxAge == 0 {
return
}
id = cleanID(id)
if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil {
panic(err)
}
}
func (h *httpCache) Delete(key string) {
h.c.Fs.Remove(key)
}

208
cache/httpcache/httpcache.go vendored Normal file
View file

@ -0,0 +1,208 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httpcache
import (
"encoding/json"
"time"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/common/predicate"
"github.com/gohugoio/hugo/config"
"github.com/mitchellh/mapstructure"
)
// DefaultConfig holds the default configuration for the HTTP cache.
var DefaultConfig = Config{
Cache: Cache{
For: GlobMatcher{
Excludes: []string{"**"},
},
},
Polls: []PollConfig{
{
For: GlobMatcher{
Includes: []string{"**"},
},
Disable: true,
},
},
}
// Config holds the configuration for the HTTP cache.
type Config struct {
// Configures the HTTP cache behaviour (RFC 9111).
// When this is not enabled for a resource, Hugo will go straight to the file cache.
Cache Cache
// Polls holds a list of configurations for polling remote resources to detect changes in watch mode.
// This can be disabled for some resources, typically if they are known to not change.
Polls []PollConfig
}
type Cache struct {
// Enable HTTP cache behaviour (RFC 9111) for these rsources.
For GlobMatcher
}
func (c *Config) Compile() (ConfigCompiled, error) {
var cc ConfigCompiled
p, err := c.Cache.For.CompilePredicate()
if err != nil {
return cc, err
}
cc.For = p
for _, pc := range c.Polls {
p, err := pc.For.CompilePredicate()
if err != nil {
return cc, err
}
cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{
For: p,
Config: pc,
})
}
return cc, nil
}
// PollConfig holds the configuration for polling remote resources to detect changes in watch mode.
// TODO1 make sure this enabled only in watch mode.
type PollConfig struct {
// What remote resources to apply this configuration to.
For GlobMatcher
// Disable polling for this configuration.
Disable bool
// Low is the lower bound for the polling interval.
// This is the starting point when the resource has recently changed,
// if that resource stops changing, the polling interval will gradually increase towards High.
Low time.Duration
// High is the upper bound for the polling interval.
// This is the interval used when the resource is stable.
High time.Duration
}
func (c PollConfig) MarshalJSON() (b []byte, err error) {
// Marshal the durations as strings.
type Alias PollConfig
return json.Marshal(&struct {
Low string
High string
Alias
}{
Low: c.Low.String(),
High: c.High.String(),
Alias: (Alias)(c),
})
}
type GlobMatcher struct {
// Excludes holds a list of glob patterns that will be excluded.
Excludes []string
// Includes holds a list of glob patterns that will be included.
Includes []string
}
type ConfigCompiled struct {
For predicate.P[string]
PollConfigs []PollConfigCompiled
}
func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled {
for _, pc := range c.PollConfigs {
if pc.For(s) {
return pc
}
}
return PollConfigCompiled{}
}
func (c *ConfigCompiled) IsPollingDisabled() bool {
for _, pc := range c.PollConfigs {
if !pc.Config.Disable {
return false
}
}
return true
}
type PollConfigCompiled struct {
For predicate.P[string]
Config PollConfig
}
func (p PollConfigCompiled) IsZero() bool {
return p.For == nil
}
func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) {
var p predicate.P[string]
for _, include := range gm.Includes {
g, err := glob.Compile(include, '/')
if err != nil {
return nil, err
}
fn := func(s string) bool {
return g.Match(s)
}
p = p.Or(fn)
}
for _, exclude := range gm.Excludes {
g, err := glob.Compile(exclude, '/')
if err != nil {
return nil, err
}
fn := func(s string) bool {
return !g.Match(s)
}
p = p.And(fn)
}
return p, nil
}
func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) {
if len(m) == 0 {
return DefaultConfig, nil
}
var c Config
dc := &mapstructure.DecoderConfig{
Result: &c,
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
WeaklyTypedInput: true,
}
decoder, err := mapstructure.NewDecoder(dc)
if err != nil {
return c, err
}
if err := decoder.Decode(m); err != nil {
return c, err
}
return c, nil
}

64
cache/httpcache/httpcache_integration_test.go vendored Normal file
View file

@ -0,0 +1,64 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httpcache_test
import (
"testing"
"time"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/hugolib"
)
func TestConfigCustom(t *testing.T) {
files := `
-- hugo.toml --
[httpcache]
[httpcache.cache.for]
includes = ["**gohugo.io**"]
[[httpcache.polls]]
low = "5s"
high = "32s"
[httpcache.polls.for]
includes = ["**gohugo.io**"]
`
b := hugolib.Test(t, files)
httpcacheConf := b.H.Configs.Base.HTTPCache
compiled := b.H.Configs.Base.C.HTTPCache
b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"})
b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil)
pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg")
b.Assert(pc.Config.Low, qt.Equals, 5*time.Second)
b.Assert(pc.Config.High, qt.Equals, 32*time.Second)
b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue)
}
func TestConfigDefault(t *testing.T) {
files := `
-- hugo.toml --
`
b := hugolib.Test(t, files)
compiled := b.H.Configs.Base.C.HTTPCache
b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse)
b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse)
b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue)
}

42
cache/httpcache/httpcache_test.go vendored Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httpcache
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestGlobMatcher(t *testing.T) {
c := qt.New(t)
g := GlobMatcher{
Includes: []string{"**/*.jpg", "**.png", "**/bar/**"},
Excludes: []string{"**/foo.jpg", "**.css"},
}
p, err := g.CompilePredicate()
c.Assert(err, qt.IsNil)
c.Assert(p("foo.jpg"), qt.IsFalse)
c.Assert(p("foo.png"), qt.IsTrue)
c.Assert(p("foo/bar.jpg"), qt.IsTrue)
c.Assert(p("foo/bar.png"), qt.IsTrue)
c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse)
c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
c.Assert(p("foo.css"), qt.IsFalse)
c.Assert(p("foo/bar/foo.css"), qt.IsFalse)
c.Assert(p("foo/bar/foo.xml"), qt.IsTrue)
}

14
commands/commandeer.go
View file

@ -48,6 +48,7 @@ import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/kinds"
"github.com/spf13/afero"
"github.com/spf13/cobra"
@ -103,6 +104,9 @@ type rootCommand struct {
commonConfigs *lazycache.Cache[int32, *commonConfig]
hugoSites *lazycache.Cache[int32, *hugolib.HugoSites]
// changesFromBuild received from Hugo in watch mode.
changesFromBuild chan []identity.Identity
commands []simplecobra.Commander
// Flags
@ -304,7 +308,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
@ -316,12 +320,16 @@ func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) {
if err != nil {
return nil, err
}
depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()}
depsCfg := r.newDepsConfig(conf)
return hugolib.NewHugoSites(depsCfg)
})
return h, err
}
func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg {
return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild}
}
func (r *rootCommand) Name() string {
return "hugo"
}
@ -408,6 +416,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
return err
}
r.changesFromBuild = make(chan []identity.Identity, 10)
r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5})
// We don't want to keep stale HugoSites in memory longer than needed.
r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{

32
commands/hugobuilder.go
View file

@ -43,6 +43,7 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/livereload"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/watcher"
@ -343,6 +344,24 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
go func() {
for {
select {
case changes := <-c.r.changesFromBuild:
unlock, err := h.LockBuild()
if err != nil {
c.r.logger.Errorln("Failed to acquire a build lock: %s", err)
return
}
c.changeDetector.PrepareNew()
err = c.rebuildSitesForChanges(changes)
if err != nil {
c.r.logger.Errorln("Error while watching:", err)
}
if c.s != nil && c.s.doLiveReload {
if c.changeDetector == nil || len(c.changeDetector.changed()) > 0 {
livereload.ForceRefresh()
}
}
unlock()
case evs := <-watcher.Events:
unlock, err := h.LockBuild()
if err != nil {
@ -1019,6 +1038,19 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error {
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...)
}
func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error {
c.errState.setBuildErr(nil)
h, err := c.hugo()
if err != nil {
return err
}
whatChanged := &hugolib.WhatChanged{}
whatChanged.Add(ids...)
err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()})
c.errState.setBuildErr(err)
return err
}
func (c *hugoBuilder) reloadConfig() error {
c.r.Reset()
c.r.configVersionID.Add(1)

8
common/maps/maps.go
View file

@ -112,17 +112,17 @@ func ToSliceStringMap(in any) ([]map[string]any, error) {
}
// LookupEqualFold finds key in m with case insensitive equality checks.
func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) {
func LookupEqualFold[T any | string](m map[string]T, key string) (T, string, bool) {
if v, found := m[key]; found {
return v, true
return v, key, true
}
for k, v := range m {
if strings.EqualFold(k, key) {
return v, true
return v, k, true
}
}
var s T
return s, false
return s, "", false
}
// MergeShallow merges src into dst, but only if the key does not already exist in dst.

6
common/maps/maps_test.go
View file

@ -180,16 +180,18 @@ func TestLookupEqualFold(t *testing.T) {
"B": "bv",
}
v, found := LookupEqualFold(m1, "b")
v, k, found := LookupEqualFold(m1, "b")
c.Assert(found, qt.IsTrue)
c.Assert(v, qt.Equals, "bv")
c.Assert(k, qt.Equals, "B")
m2 := map[string]string{
"a": "av",
"B": "bv",
}
v, found = LookupEqualFold(m2, "b")
v, k, found = LookupEqualFold(m2, "b")
c.Assert(found, qt.IsTrue)
c.Assert(k, qt.Equals, "B")
c.Assert(v, qt.Equals, "bv")
}

6
common/predicate/predicate.go
View file

@ -24,6 +24,9 @@ func (p P[T]) And(ps ...P[T]) P[T] {
return false
}
}
if p == nil {
return true
}
return p(v)
}
}
@ -36,6 +39,9 @@ func (p P[T]) Or(ps ...P[T]) P[T] {
return true
}
}
if p == nil {
return false
}
return p(v)
}
}

153
common/tasks/tasks.go Normal file
View file

@ -0,0 +1,153 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"sync"
"time"
)
// RunEvery runs a function at intervals defined by the function itself.
// Functions can be added and removed while running.
type RunEvery struct {
// Any error returned from the function will be passed to this function.
HandleError func(string, error)
// If set, the function will be run immediately.
RunImmediately bool
// The named functions to run.
funcs map[string]*Func
mu sync.Mutex
started bool
closed bool
quit chan struct{}
}
type Func struct {
// The shortest interval between each run.
IntervalLow time.Duration
// The longest interval between each run.
IntervalHigh time.Duration
// The function to run.
F func(interval time.Duration) (time.Duration, error)
interval time.Duration
last time.Time
}
func (r *RunEvery) Start() error {
if r.started {
return nil
}
r.started = true
r.quit = make(chan struct{})
go func() {
if r.RunImmediately {
r.run()
}
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-r.quit:
return
case <-ticker.C:
r.run()
}
}
}()
return nil
}
// Close stops the RunEvery from running.
func (r *RunEvery) Close() error {
if r.closed {
return nil
}
r.closed = true
if r.quit != nil {
close(r.quit)
}
return nil
}
// Add adds a function to the RunEvery.
func (r *RunEvery) Add(name string, f Func) {
r.mu.Lock()
defer r.mu.Unlock()
if r.funcs == nil {
r.funcs = make(map[string]*Func)
}
if f.IntervalLow == 0 {
f.IntervalLow = 500 * time.Millisecond
}
if f.IntervalHigh <= f.IntervalLow {
f.IntervalHigh = 20 * time.Second
}
start := f.IntervalHigh / 3
if start < f.IntervalLow {
start = f.IntervalLow
}
f.interval = start
f.last = time.Now()
r.funcs[name] = &f
}
// Remove removes a function from the RunEvery.
func (r *RunEvery) Remove(name string) {
r.mu.Lock()
defer r.mu.Unlock()
delete(r.funcs, name)
}
// Has returns whether the RunEvery has a function with the given name.
func (r *RunEvery) Has(name string) bool {
r.mu.Lock()
defer r.mu.Unlock()
_, found := r.funcs[name]
return found
}
func (r *RunEvery) run() {
r.mu.Lock()
defer r.mu.Unlock()
for name, f := range r.funcs {
if time.Now().Before(f.last.Add(f.interval)) {
continue
}
f.last = time.Now()
interval, err := f.F(f.interval)
if err != nil && r.HandleError != nil {
r.HandleError(name, err)
}
if interval < f.IntervalLow {
interval = f.IntervalLow
}
if interval > f.IntervalHigh {
interval = f.IntervalHigh
}
f.interval = interval
}
}

47
common/types/closer.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import "sync"
type Closer interface {
Close() error
}
type CloseAdder interface {
Add(Closer)
}
type Closers struct {
mu sync.Mutex
cs []Closer
}
func (cs *Closers) Add(c Closer) {
cs.mu.Lock()
defer cs.mu.Unlock()
cs.cs = append(cs.cs, c)
}
func (cs *Closers) Close() error {
cs.mu.Lock()
defer cs.mu.Unlock()
for _, c := range cs.cs {
c.Close()
}
cs.cs = cs.cs[:0]
return nil
}

12
config/allconfig/allconfig.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
@ -119,6 +120,10 @@ type Config struct {
// <docsmeta>{"identifiers": ["caches"] }</docsmeta>
Caches filecache.Configs `mapstructure:"-"`
// The httpcache configuration section contains HTTP-cache-related configuration options.
// <docsmeta>{"identifiers": ["httpcache"] }</docsmeta>
HTTPCache httpcache.Config `mapstructure:"-"`
// The markup configuration section contains markup-related configuration options.
// <docsmeta>{"identifiers": ["markup"] }</docsmeta>
Markup markup_config.Config `mapstructure:"-"`
@ -359,6 +364,11 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
}
}
httpCache, err := c.HTTPCache.Compile()
if err != nil {
return err
}
c.C = &ConfigCompiled{
Timeout: timeout,
BaseURL: baseURL,
@ -374,6 +384,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...),
MainSections: c.MainSections,
Clock: clock,
HTTPCache: httpCache,
transientErr: transientErr,
}
@ -413,6 +424,7 @@ type ConfigCompiled struct {
SegmentFilter segments.SegmentFilter
MainSections []string
Clock time.Time
HTTPCache httpcache.ConfigCompiled
// This is set to the last transient error found during config compilation.
// With themes/modules we compute the configuration in multiple passes, and

14
config/allconfig/alldecoders.go
View file

@ -18,6 +18,8 @@ import (
"strings"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
@ -96,6 +98,18 @@ var allDecoderSetups = map[string]decodeWeight{
return err
},
},
"httpcache": {
key: "httpcache",
decode: func(d decodeWeight, p decodeConfig) error {
var err error
p.c.HTTPCache, err = httpcache.DecodeConfig(p.bcfg, p.p.GetStringMap(d.key))
if p.c.IgnoreCache {
p.c.HTTPCache.Cache.For.Excludes = []string{"**"}
p.c.HTTPCache.Cache.For.Includes = []string{}
}
return err
},
},
"build": {
key: "build",
decode: func(d decodeWeight, p decodeConfig) error {

2
config/allconfig/configlanguage.go
View file

@ -173,6 +173,8 @@ func (c ConfigLanguage) GetConfigSection(s string) any {
return c.m.Modules
case "deployment":
return c.config.Deployment
case "httpCacheCompiled":
return c.config.C.HTTPCache
default:
panic("not implemented: " + s)
}

46
deps/deps.go vendored
View file

@ -15,11 +15,13 @@ import (
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/allconfig"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/postpub"
@ -85,7 +87,7 @@ type Deps struct {
BuildEndListeners *Listeners
// Resources that gets closed when the build is done or the server shuts down.
BuildClosers *Closers
BuildClosers *types.Closers
// This is common/global for all sites.
BuildState *BuildState
@ -143,7 +145,7 @@ func (d *Deps) Init() error {
}
if d.BuildClosers == nil {
d.BuildClosers = &Closers{}
d.BuildClosers = &types.Closers{}
}
if d.Metrics == nil && d.Conf.TemplateMetrics() {
@ -208,7 +210,7 @@ func (d *Deps) Init() error {
return fmt.Errorf("failed to create file caches from configuration: %w", err)
}
resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper)
resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState)
if err != nil {
return fmt.Errorf("failed to create resource spec: %w", err)
}
@ -353,6 +355,9 @@ type DepsCfg struct {
// i18n handling.
TranslationProvider ResourceProvider
// ChangesFromBuild for changes passed back to the server/watch process.
ChangesFromBuild chan []identity.Identity
}
// BuildState are state used during a build.
@ -361,11 +366,19 @@ type BuildState struct {
mu sync.Mutex // protects state below.
OnSignalRebuild func(ids ...identity.Identity)
// A set of filenames in /public that
// contains a post-processing prefix.
filenamesWithPostPrefix map[string]bool
}
var _ identity.SignalRebuilder = (*BuildState)(nil)
func (b *BuildState) SignalRebuild(ids ...identity.Identity) {
b.OnSignalRebuild(ids...)
}
func (b *BuildState) AddFilenameWithPostPrefix(filename string) {
b.mu.Lock()
defer b.mu.Unlock()
@ -389,30 +402,3 @@ func (b *BuildState) GetFilenamesWithPostPrefix() []string {
func (b *BuildState) Incr() int {
return int(atomic.AddUint64(&b.counter, uint64(1)))
}
type Closer interface {
Close() error
}
type Closers struct {
mu sync.Mutex
cs []Closer
}
func (cs *Closers) Add(c Closer) {
cs.mu.Lock()
defer cs.mu.Unlock()
cs.cs = append(cs.cs, c)
}
func (cs *Closers) Close() error {
cs.mu.Lock()
defer cs.mu.Unlock()
for _, c := range cs.cs {
c.Close()
}
cs.cs = cs.cs[:0]
return nil
}

1
go.mod
View file

@ -35,6 +35,7 @@ require (
github.com/gobuffalo/flect v1.0.2
github.com/gobwas/glob v0.2.3
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e
github.com/gohugoio/httpcache v0.6.0
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0
github.com/gohugoio/locales v0.14.0

21
go.sum
View file

@ -53,6 +53,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
@ -67,9 +68,11 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU=
github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI=
github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.50.7 h1:odKb+uneeGgF2jgAerKjFzpljiyZxleV4SHB7oBK+YA=
@ -171,6 +174,7 @@ github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dU
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -195,8 +199,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8=
github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@ -207,12 +209,19 @@ github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1Rf
github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw=
github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI=
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA=
github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs=
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY=
github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ=
github.com/gohugoio/httpcache v0.5.0 h1:9xi4VuXd+KT3h0jOs8DlZxTMu5CtjDr0BvQMAuL/O5I=
github.com/gohugoio/httpcache v0.5.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/httpcache v0.6.0 h1:5pYJM43Yoc4uvIJ+/e770PS48srTumvuQZpuBfGFZV0=
github.com/gohugoio/httpcache v0.6.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ3Vs=
github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 h1:YhxZNU8y2vxV6Ibr7QJzzUlpr8oHHWX/l+Q1R/a5Zao=
github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0/go.mod h1:0cuvOnGKW7WeXA3i7qK6IS07FH1bgJ2XzOjQ7BMJYH4=
github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg=
@ -274,12 +283,15 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE=
github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk=
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
@ -315,6 +327,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@ -394,6 +407,7 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@ -439,7 +453,9 @@ github.com/tdewolff/parse/v2 v2.7.13 h1:iSiwOUkCYLNfapHoqdLcqZVgvQ0jrsao8YYKP/UJ
github.com/tdewolff/parse/v2 v2.7.13/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo=
github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -806,6 +822,7 @@ google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0=
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=

2
hugolib/content_map_page.go
View file

@ -975,7 +975,7 @@ type contentTreeReverseIndexMap struct {
type sitePagesAssembler struct {
*Site
assembleChanges *whatChanged
assembleChanges *WhatChanged
ctx context.Context
}

3
hugolib/hugo_sites.go
View file

@ -405,8 +405,9 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) {
type BuildCfg struct {
// Skip rendering. Useful for testing.
SkipRender bool
// Use this to indicate what changed (for rebuilds).
whatChanged *whatChanged
WhatChanged *WhatChanged
// This is a partial re-render of some selected pages.
PartialReRender bool

45
hugolib/hugo_sites_build.go
View file

@ -114,9 +114,9 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
// Need a pointer as this may be modified.
conf := &config
if conf.whatChanged == nil {
if conf.WhatChanged == nil {
// Assume everything has changed
conf.whatChanged = &whatChanged{needsPagesAssembly: true}
conf.WhatChanged = &WhatChanged{needsPagesAssembly: true}
}
var prepareErr error
@ -128,7 +128,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
s.Deps.BuildStartListeners.Notify()
}
if len(events) > 0 {
if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 {
// Rebuild
if err := h.initRebuild(conf); err != nil {
return fmt.Errorf("initRebuild: %w", err)
@ -224,7 +224,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error {
})
for _, s := range h.Sites {
s.resetBuildState(config.whatChanged.needsPagesAssembly)
s.resetBuildState(config.WhatChanged.needsPagesAssembly)
}
h.reset(config)
@ -245,7 +245,9 @@ func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *Bui
if len(events) > 0 {
// This is a rebuild
return h.processPartial(ctx, l, config, init, events)
return h.processPartialFileEvents(ctx, l, config, init, events)
} else if len(config.WhatChanged.Changes()) > 0 {
return h.processPartialRebuildChanges(ctx, l, config)
}
return h.processFull(ctx, l, config)
}
@ -256,8 +258,8 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
l = l.WithField("step", "assemble")
defer loggers.TimeTrackf(l, time.Now(), nil, "")
if !bcfg.whatChanged.needsPagesAssembly {
changes := bcfg.whatChanged.Drain()
if !bcfg.WhatChanged.needsPagesAssembly {
changes := bcfg.WhatChanged.Drain()
if len(changes) > 0 {
if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
return err
@ -273,7 +275,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
for i, s := range h.Sites {
assemblers[i] = &sitePagesAssembler{
Site: s,
assembleChanges: bcfg.whatChanged,
assembleChanges: bcfg.WhatChanged,
ctx: ctx,
}
}
@ -289,7 +291,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
return err
}
changes := bcfg.whatChanged.Drain()
changes := bcfg.WhatChanged.Drain()
// Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation
// of what needs to be re-built.
@ -612,8 +614,19 @@ func (p pathChange) isStructuralChange() bool {
return p.delete || p.isDir
}
// processPartial prepares the Sites' sources for a partial rebuild.
func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error {
if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil {
return err
}
if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
return err
}
return nil
}
// processPartialFileEvents prepares the Sites' sources for a partial rebuild.
func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error {
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("File events:\n")
@ -887,13 +900,13 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
resourceFiles := h.fileEventsContentPaths(addedOrChangedContent)
changed := &whatChanged{
changed := &WhatChanged{
needsPagesAssembly: needsPagesAssemble,
identitySet: make(identity.Identities),
}
changed.Add(changes...)
config.whatChanged = changed
config.WhatChanged = changed
if err := init(config); err != nil {
return err
@ -977,14 +990,14 @@ func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConf
}
if len(bi.ChangedIdentities) > 0 {
buildConfig.whatChanged.Add(bi.ChangedIdentities...)
buildConfig.whatChanged.needsPagesAssembly = true
buildConfig.WhatChanged.Add(bi.ChangedIdentities...)
buildConfig.WhatChanged.needsPagesAssembly = true
}
for _, p := range bi.DeletedPaths {
pp := path.Join(bi.Path.Base(), p)
if v, ok := s.pageMap.treePages.Delete(pp); ok {
buildConfig.whatChanged.Add(v.GetIdentity())
buildConfig.WhatChanged.Add(v.GetIdentity())
}
}
}

12
hugolib/site.go
View file

@ -371,14 +371,14 @@ func (s *Site) watching() bool {
return s.h != nil && s.h.Configs.Base.Internal.Watch
}
type whatChanged struct {
type WhatChanged struct {
mu sync.Mutex
needsPagesAssembly bool
identitySet identity.Identities
}
func (w *whatChanged) Add(ids ...identity.Identity) {
func (w *WhatChanged) Add(ids ...identity.Identity) {
w.mu.Lock()
defer w.mu.Unlock()
@ -391,24 +391,24 @@ func (w *whatChanged) Add(ids ...identity.Identity) {
}
}
func (w *whatChanged) Clear() {
func (w *WhatChanged) Clear() {
w.mu.Lock()
defer w.mu.Unlock()
w.clear()
}
func (w *whatChanged) clear() {
func (w *WhatChanged) clear() {
w.identitySet = identity.Identities{}
}
func (w *whatChanged) Changes() []identity.Identity {
func (w *WhatChanged) Changes() []identity.Identity {
if w == nil || w.identitySet == nil {
return nil
}
return w.identitySet.AsSlice()
}
func (w *whatChanged) Drain() []identity.Identity {
func (w *WhatChanged) Drain() []identity.Identity {
w.mu.Lock()
defer w.mu.Unlock()
ids := w.identitySet.AsSlice()

22
hugolib/site_new.go
View file

@ -141,10 +141,23 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
memCache := dynacache.New(dynacache.Options{Watching: conf.Watching(), Log: logger})
var h *HugoSites
onSignalRebuild := func(ids ...identity.Identity) {
// This channel is buffered, but make sure we do this in a non-blocking way.
if cfg.ChangesFromBuild != nil {
go func() {
cfg.ChangesFromBuild <- ids
}()
}
}
firstSiteDeps := &deps.Deps{
Fs: cfg.Fs,
Log: logger,
Conf: conf,
Fs: cfg.Fs,
Log: logger,
Conf: conf,
BuildState: &deps.BuildState{
OnSignalRebuild: onSignalRebuild,
},
MemCache: memCache,
TemplateProvider: tplimpl.DefaultTemplateProvider,
TranslationProvider: i18n.NewTranslationProvider(),
@ -261,7 +274,8 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
return li.Lang < lj.Lang
})
h, err := newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
var err error
h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
if err == nil && h == nil {
panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")
}

5
identity/identity.go
View file

@ -241,6 +241,11 @@ type IdentityProvider interface {
GetIdentity() Identity
}
// SignalRebuilder is an optional interface for types that can signal a rebuild.
type SignalRebuilder interface {
SignalRebuild(ids ...Identity)
}
// IncrementByOne implements Incrementer adding 1 every time Incr is called.
type IncrementByOne struct {
counter uint64

2
media/config.go
View file

@ -194,7 +194,7 @@ func DecodeTypes(in map[string]any) (*config.ConfigNamespace[map[string]MediaTyp
return nil, nil, err
}
mm := maps.ToStringMap(v)
suffixes, found := maps.LookupEqualFold(mm, "suffixes")
suffixes, _, found := maps.LookupEqualFold(mm, "suffixes")
if found {
mediaType.SuffixesCSV = strings.TrimSpace(strings.ToLower(strings.Join(cast.ToStringSlice(suffixes), ",")))
}

8
parser/lowercase_camel_json.go
View file

@ -46,6 +46,12 @@ type LowerCaseCamelJSONMarshaller struct {
Value any
}
var preserveUpperCaseKeyRe = regexp.MustCompile(`^"HTTP`)
func preserveUpperCaseKey(match []byte) bool {
return preserveUpperCaseKeyRe.Match(match)
}
func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
marshalled, err := json.Marshal(c.Value)
@ -59,7 +65,7 @@ func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) {
// Empty keys are valid JSON, only lowercase if we do not have an
// empty key.
if len(match) > 2 {
if len(match) > 2 && !preserveUpperCaseKey(match) {
// Decode first rune after the double quotes
r, width := utf8.DecodeRune(match[1:])
r = unicode.ToLower(r)

6
resources/resource_cache.go
View file

@ -36,6 +36,11 @@ func newResourceCache(rs *Spec, memCache *dynacache.Cache) *ResourceCache {
"/res1",
dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
),
CacheResourceRemote: dynacache.GetOrCreatePartition[string, resource.Resource](
memCache,
"/resr",
dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40},
),
cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](
memCache,
"/ress",
@ -53,6 +58,7 @@ type ResourceCache struct {
sync.RWMutex
cacheResource *dynacache.Partition[string, resource.Resource]
CacheResourceRemote *dynacache.Partition[string, resource.Resource]
cacheResources *dynacache.Partition[string, resource.Resources]
cacheResourceTransformation *dynacache.Partition[string, *resourceAdapterInner]

74
resources/resource_factories/create/create.go
View file

@ -23,6 +23,9 @@ import (
"strings"
"time"
"github.com/bep/logg"
"github.com/gohugoio/httpcache"
hhttpcache "github.com/gohugoio/hugo/cache/httpcache"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/identity"
@ -31,7 +34,9 @@ import (
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hcontext"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/tasks"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
)
@ -39,19 +44,76 @@ import (
// Client contains methods to create Resource objects.
// tasks to Resource objects.
type Client struct {
rs *resources.Spec
httpClient *http.Client
cacheGetResource *filecache.Cache
rs *resources.Spec
httpClient *http.Client
httpCacheConfig hhttpcache.ConfigCompiled
cacheGetResource *filecache.Cache
resourceIDDispatcher hcontext.ContextDispatcher[string]
// Set when watching.
remoteResourceChecker *tasks.RunEvery
remoteResourceLogger logg.LevelLogger
}
type contextKey string
// New creates a new Client with the given specification.
func New(rs *resources.Spec) *Client {
fileCache := rs.FileCaches.GetResourceCache()
resourceIDDispatcher := hcontext.NewContextDispatcher[string](contextKey("resourceID"))
httpCacheConfig := rs.Cfg.GetConfigSection("httpCacheCompiled").(hhttpcache.ConfigCompiled)
var remoteResourceChecker *tasks.RunEvery
if rs.Cfg.Watching() && !httpCacheConfig.IsPollingDisabled() {
remoteResourceChecker = &tasks.RunEvery{
HandleError: func(name string, err error) {
rs.Logger.Warnf("Failed to check remote resource: %s", err)
},
RunImmediately: false,
}
if err := remoteResourceChecker.Start(); err != nil {
panic(err)
}
rs.BuildClosers.Add(remoteResourceChecker)
}
httpTimeout := 2 * time.Minute // Need to cover retries.
if httpTimeout < (rs.Cfg.Timeout() + 30*time.Second) {
httpTimeout = rs.Cfg.Timeout() + 30*time.Second
}
return &Client{
rs: rs,
rs: rs,
httpCacheConfig: httpCacheConfig,
resourceIDDispatcher: resourceIDDispatcher,
remoteResourceChecker: remoteResourceChecker,
remoteResourceLogger: rs.Logger.InfoCommand("remote"),
httpClient: &http.Client{
Timeout: time.Minute,
Timeout: httpTimeout,
Transport: &httpcache.Transport{
Cache: fileCache.AsHTTPCache(),
CacheKey: func(req *http.Request) string {
return resourceIDDispatcher.Get(req.Context())
},
Around: func(req *http.Request, key string) func() {
return fileCache.NamedLock(key)
},
AlwaysUseCachedResponse: func(req *http.Request, key string) bool {
return !httpCacheConfig.For(req.URL.String())
},
ShouldCache: func(req *http.Request, resp *http.Response, key string) bool {
return shouldCache(resp.StatusCode)
},
MarkCachedResponses: true,
EnableETagPair: true,
Transport: &transport{
Cfg: rs.Cfg,
Logger: rs.Logger,
},
},
},
cacheGetResource: rs.FileCaches.GetResourceCache(),
cacheGetResource: fileCache,
}
}

3
resources/resource_factories/create/create_integration_test.go
View file

@ -134,8 +134,7 @@ mediaTypes = ['text/plain']
// This is hard to get stable on GitHub Actions, it sometimes succeeds due to timing issues.
if err != nil {
b.AssertLogContains("Got Err")
b.AssertLogContains("Retry timeout")
b.AssertLogContains("ContentLength:0")
b.AssertLogContains("retry timeout")
}
})
}

366
resources/resource_factories/create/remote.go
View file

@ -14,22 +14,27 @@
package create
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"math/rand"
"mime"
"net/http"
"net/http/httputil"
"net/url"
"path"
"strings"
"time"
gmaps "maps"
"github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/tasks"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources"
@ -92,6 +97,60 @@ var temporaryHTTPStatusCodes = map[int]bool{
504: true,
}
func (c *Client) configurePollingIfEnabled(uri, optionsKey string, getRes func() (*http.Response, error)) {
if c.remoteResourceChecker == nil {
return
}
// Set up polling for changes to this resource.
pollingConfig := c.httpCacheConfig.PollConfigFor(uri)
if pollingConfig.IsZero() || pollingConfig.Config.Disable {
return
}
if c.remoteResourceChecker.Has(optionsKey) {
return
}
var lastChange time.Time
c.remoteResourceChecker.Add(optionsKey,
tasks.Func{
IntervalLow: pollingConfig.Config.Low,
IntervalHigh: pollingConfig.Config.High,
F: func(interval time.Duration) (time.Duration, error) {
start := time.Now()
defer func() {
duration := time.Since(start)
c.rs.Logger.Debugf("Polled remote resource for changes in %13s. Interval: %4s (low: %4s high: %4s) resource: %q ", duration, interval, pollingConfig.Config.Low, pollingConfig.Config.High, uri)
}()
// TODO(bep) figure out a ways to remove unused tasks.
res, err := getRes()
if err != nil {
return pollingConfig.Config.High, err
}
// The caching is delayed until the body is read.
io.Copy(io.Discard, res.Body)
res.Body.Close()
x1, x2 := res.Header.Get(httpcache.XETag1), res.Header.Get(httpcache.XETag2)
if x1 != x2 {
lastChange = time.Now()
c.remoteResourceLogger.Logf("detected change in remote resource %q", uri)
c.rs.Rebuilder.SignalRebuild(identity.StringIdentity(optionsKey))
}
if time.Since(lastChange) < 10*time.Second {
// The user is typing, check more often.
return 0, nil
}
// Increase the interval to avoid hammering the server.
interval += 1 * time.Second
return interval, nil
},
})
}
// FromRemote expects one or n-parts of a URL to a resource
// If you provide multiple parts they will be joined together to the final URL.
func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) {
@ -101,168 +160,139 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
}
method := "GET"
if s, ok := maps.LookupEqualFold(optionsm, "method"); ok {
if s, _, ok := maps.LookupEqualFold(optionsm, "method"); ok {
method = strings.ToUpper(s.(string))
}
isHeadMethod := method == "HEAD"
resourceID := calculateResourceID(uri, optionsm)
optionsm = gmaps.Clone(optionsm)
userKey, optionsKey := remoteResourceKeys(uri, optionsm)
_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
// A common pattern is to use the key in the options map as
// a way to control cache eviction,
// so make sure we use any user provided kehy as the file cache key,
// but the auto generated and more stable key for everything else.
filecacheKey := userKey
return c.rs.ResourceCache.CacheResourceRemote.GetOrCreate(optionsKey, func(key string) (resource.Resource, error) {
options, err := decodeRemoteOptions(optionsm)
if err != nil {
return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err)
}
if err := c.validateFromRemoteArgs(uri, options); err != nil {
return nil, err
}
var (
start time.Time
nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
nextSleepLimit = time.Duration(5) * time.Second
)
getRes := func() (*http.Response, error) {
ctx := context.Background()
ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey)
for {
b, retry, err := func() ([]byte, bool, error) {
req, err := options.NewRequest(uri)
if err != nil {
return nil, false, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
}
res, err := c.httpClient.Do(req)
if err != nil {
return nil, false, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusNotFound {
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, temporaryHTTPStatusCodes[res.StatusCode], toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
}
}
b, err := httputil.DumpResponse(res, true)
if err != nil {
return nil, false, toHTTPError(err, res, !isHeadMethod)
}
return b, false, nil
}()
req, err := options.NewRequest(uri)
if err != nil {
if retry {
if start.IsZero() {
start = time.Now()
} else if d := time.Since(start) + nextSleep; d >= c.rs.Cfg.Timeout() {
c.rs.Logger.Errorf("Retry timeout (configured to %s) fetching remote resource.", c.rs.Cfg.Timeout())
return nil, err
}
time.Sleep(nextSleep)
if nextSleep < nextSleepLimit {
nextSleep *= 2
}
continue
}
return nil, err
return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
}
return hugio.ToReadCloser(bytes.NewReader(b)), nil
req = req.WithContext(ctx)
return c.httpClient.Do(req)
}
})
if err != nil {
return nil, err
}
defer httpResponse.Close()
res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
// Not found. This matches how looksup for local resources work.
return nil, nil
}
var (
body []byte
mediaType media.Type
)
// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
if !isHeadMethod && res.Body != nil {
body, err = io.ReadAll(res.Body)
res, err := getRes()
if err != nil {
return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
return nil, err
}
}
defer res.Body.Close()
filename := path.Base(rURL.Path)
if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
if _, ok := params["filename"]; ok {
filename = params["filename"]
c.configurePollingIfEnabled(uri, optionsKey, getRes)
if res.StatusCode == http.StatusNotFound {
// Not found. This matches how lookups for local resources work.
return nil, nil
}
}
contentType := res.Header.Get("Content-Type")
// For HEAD requests we have no body to work with, so we need to use the Content-Type header.
if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
var found bool
mediaType, found = c.rs.MediaTypes().GetByType(contentType)
if !found {
// A media type not configured in Hugo, just create one from the content type string.
mediaType, _ = media.FromString(contentType)
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
}
}
if mediaType.IsZero() {
var extensionHints []string
// mime.ExtensionsByType gives a long list of extensions for text/plain,
// just use ".txt".
if strings.HasPrefix(contentType, "text/plain") {
extensionHints = []string{".txt"}
} else {
exts, _ := mime.ExtensionsByType(contentType)
if exts != nil {
extensionHints = exts
var (
body []byte
mediaType media.Type
)
// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
if !isHeadMethod && res.Body != nil {
body, err = io.ReadAll(res.Body)
if err != nil {
return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
}
}
// Look for a file extension. If it's .txt, look for a more specific.
if extensionHints == nil || extensionHints[0] == ".txt" {
if ext := path.Ext(filename); ext != "" {
extensionHints = []string{ext}
filename := path.Base(rURL.Path)
if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
if _, ok := params["filename"]; ok {
filename = params["filename"]
}
}
// Now resolve the media type primarily using the content.
mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
contentType := res.Header.Get("Content-Type")
}
// For HEAD requests we have no body to work with, so we need to use the Content-Type header.
if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) {
var found bool
mediaType, found = c.rs.MediaTypes().GetByType(contentType)
if !found {
// A media type not configured in Hugo, just create one from the content type string.
mediaType, _ = media.FromString(contentType)
}
}
if mediaType.IsZero() {
return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
}
if mediaType.IsZero() {
resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
data := responseToData(res, false)
var extensionHints []string
return c.rs.NewResource(
resources.ResourceSourceDescriptor{
MediaType: mediaType,
Data: data,
GroupIdentity: identity.StringIdentity(resourceID),
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
},
TargetPath: resourceID,
})
// mime.ExtensionsByType gives a long list of extensions for text/plain,
// just use ".txt".
if strings.HasPrefix(contentType, "text/plain") {
extensionHints = []string{".txt"}
} else {
exts, _ := mime.ExtensionsByType(contentType)
if exts != nil {
extensionHints = exts
}
}
// Look for a file extension. If it's .txt, look for a more specific.
if extensionHints == nil || extensionHints[0] == ".txt" {
if ext := path.Ext(filename); ext != "" {
extensionHints = []string{ext}
}
}
// Now resolve the media type primarily using the content.
mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body)
}
if mediaType.IsZero() {
return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
}
userKey = filename[:len(filename)-len(path.Ext(filename))] + "_" + userKey + mediaType.FirstSuffix.FullSuffix
data := responseToData(res, false)
return c.rs.NewResource(
resources.ResourceSourceDescriptor{
MediaType: mediaType,
Data: data,
GroupIdentity: identity.StringIdentity(optionsKey),
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
},
TargetPath: userKey,
})
})
}
func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
@ -277,11 +307,17 @@ func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) e
return nil
}
func calculateResourceID(uri string, optionsm map[string]any) string {
if key, found := maps.LookupEqualFold(optionsm, "key"); found {
return identity.HashString(key)
func remoteResourceKeys(uri string, optionsm map[string]any) (string, string) {
var userKey string
if key, k, found := maps.LookupEqualFold(optionsm, "key"); found {
userKey = identity.HashString(key)
delete(optionsm, k)
}
return identity.HashString(uri, optionsm)
optionsKey := identity.HashString(uri, optionsm)
if userKey == "" {
userKey = optionsKey
}
return userKey, optionsKey
}
func addDefaultHeaders(req *http.Request) {
@ -350,3 +386,71 @@ func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) {
return options, nil
}
var _ http.RoundTripper = (*transport)(nil)
type transport struct {
Cfg config.AllProvider
Logger loggers.Logger
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
defer func() {
if resp != nil && resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusNotModified {
t.Logger.Debugf("Fetched remote resource: %s", req.URL.String())
}
}()
var (
start time.Time
nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond
nextSleepLimit = time.Duration(5) * time.Second
retry bool
)
for {
resp, retry, err = func() (*http.Response, bool, error) {
resp2, err := http.DefaultTransport.RoundTrip(req)
if err != nil {
return resp2, false, err
}
if resp2.StatusCode != http.StatusNotFound && resp2.StatusCode != http.StatusNotModified {
if resp2.StatusCode < 200 || resp2.StatusCode > 299 {
return resp2, temporaryHTTPStatusCodes[resp2.StatusCode], nil
}
}
return resp2, false, nil
}()
if retry {
if start.IsZero() {
start = time.Now()
} else if d := time.Since(start) + nextSleep; d >= t.Cfg.Timeout() {
msg := "<nil>"
if resp != nil {
msg = resp.Status
}
err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD")
return resp, err
}
time.Sleep(nextSleep)
if nextSleep < nextSleepLimit {
nextSleep *= 2
}
continue
}
return
}
}
// We need to send the redirect responses back to the HTTP client from RoundTrip,
// but we don't want to cache them.
func shouldCache(statusCode int) bool {
switch statusCode {
case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect:
return false
}
return true
}

18
resources/resource_factories/create/remote_test.go
View file

@ -115,15 +115,21 @@ func TestOptionsNewRequest(t *testing.T) {
c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"foo"})
}
func TestCalculateResourceID(t *testing.T) {
func TestRemoteResourceKeys(t *testing.T) {
t.Parallel()
c := qt.New(t)
c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675")
c.Assert(calculateResourceID("foo", map[string]any{"bar": "baz"}), qt.Equals, "7294498335241413323")
check := func(uri string, optionsm map[string]any, expect1, expect2 string) {
got1, got2 := remoteResourceKeys(uri, optionsm)
c.Assert(got1, qt.Equals, expect1)
c.Assert(got2, qt.Equals, expect2)
}
c.Assert(calculateResourceID("foo", map[string]any{"key": "1234", "bar": "baz"}), qt.Equals, "14904296279238663669")
c.Assert(calculateResourceID("asdf", map[string]any{"key": "1234", "bar": "asdf"}), qt.Equals, "14904296279238663669")
c.Assert(calculateResourceID("asdf", map[string]any{"key": "12345", "bar": "asdf"}), qt.Equals, "12191037851845371770")
check("foo", nil, "5917621528921068675", "5917621528921068675")
check("foo", map[string]any{"bar": "baz"}, "7294498335241413323", "7294498335241413323")
check("foo", map[string]any{"key": "1234", "bar": "baz"}, "14904296279238663669", "7294498335241413323")
check("foo", map[string]any{"key": "12345", "bar": "baz"}, "12191037851845371770", "7294498335241413323")
check("asdf", map[string]any{"key": "1234", "bar": "asdf"}, "14904296279238663669", "3787889110563790121")
check("asdf", map[string]any{"key": "12345", "bar": "asdf"}, "12191037851845371770", "3787889110563790121")
}

19
resources/resource_spec.go
View file

@ -29,6 +29,7 @@ import (
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/identity"
@ -53,6 +54,8 @@ func NewSpec(
logger loggers.Logger,
errorHandler herrors.ErrorSender,
execHelper *hexec.Exec,
buildClosers types.CloseAdder,
rebuilder identity.SignalRebuilder,
) (*Spec, error) {
conf := s.Cfg.GetConfig().(*allconfig.Config)
imgConfig := conf.Imaging
@ -87,10 +90,12 @@ func NewSpec(
}
rs := &Spec{
PathSpec: s,
Logger: logger,
ErrorSender: errorHandler,
imaging: imaging,
PathSpec: s,
Logger: logger,
ErrorSender: errorHandler,
BuildClosers: buildClosers,
Rebuilder: rebuilder,
imaging: imaging,
ImageCache: newImageCache(
fileCaches.ImageCache(),
memCache,
@ -111,8 +116,10 @@ func NewSpec(
type Spec struct {
*helpers.PathSpec
Logger loggers.Logger
ErrorSender herrors.ErrorSender
Logger loggers.Logger
ErrorSender herrors.ErrorSender
BuildClosers types.CloseAdder
Rebuilder identity.SignalRebuilder
TextTemplates tpl.TemplateParseFinder

3
tpl/resources/resources.go
View file

@ -369,7 +369,7 @@ func (ns *Namespace) ToCSS(args ...any) (resource.Resource, error) {
}
if m != nil {
if t, found := maps.LookupEqualFold(m, "transpiler"); found {
if t, _, found := maps.LookupEqualFold(m, "transpiler"); found {
switch t {
case transpilerDart, transpilerLibSass:
transpiler = cast.ToString(t)
@ -440,7 +440,6 @@ func (ns *Namespace) Babel(args ...any) (resource.Resource, error) {
var options babel.Options
if m != nil {
options, err = babel.DecodeOptions(m)
if err != nil {
return nil, err
}