From 447108fed2842e264897659856e9fd9cdc32ca23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 17 May 2024 17:06:47 +0200 Subject: [PATCH] Add a HTTP cache for remote resources. Fixes #12502 Closes #11891 --- cache/filecache/filecache.go | 122 +++++- cache/httpcache/httpcache.go | 208 ++++++++++ cache/httpcache/httpcache_integration_test.go | 64 +++ cache/httpcache/httpcache_test.go | 42 ++ commands/commandeer.go | 14 +- commands/hugobuilder.go | 32 ++ common/maps/maps.go | 8 +- common/maps/maps_test.go | 6 +- common/predicate/predicate.go | 6 + common/tasks/tasks.go | 153 ++++++++ common/types/closer.go | 47 +++ config/allconfig/allconfig.go | 12 + config/allconfig/alldecoders.go | 14 + config/allconfig/configlanguage.go | 2 + deps/deps.go | 46 +-- go.mod | 1 + go.sum | 21 +- hugolib/content_map_page.go | 2 +- hugolib/hugo_sites.go | 3 +- hugolib/hugo_sites_build.go | 45 ++- hugolib/site.go | 12 +- hugolib/site_new.go | 22 +- identity/identity.go | 5 + media/config.go | 2 +- parser/lowercase_camel_json.go | 8 +- resources/resource_cache.go | 6 + resources/resource_factories/create/create.go | 74 +++- .../create/create_integration_test.go | 3 +- resources/resource_factories/create/remote.go | 366 +++++++++++------- .../resource_factories/create/remote_test.go | 18 +- resources/resource_spec.go | 19 +- tpl/resources/resources.go | 3 +- 32 files changed, 1150 insertions(+), 236 deletions(-) create mode 100644 cache/httpcache/httpcache.go create mode 100644 cache/httpcache/httpcache_integration_test.go create mode 100644 cache/httpcache/httpcache_test.go create mode 100644 common/tasks/tasks.go create mode 100644 common/types/closer.go diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 093d2941c..01c466ca6 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2024 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import ( "sync" "time" + "github.com/gohugoio/httpcache" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/hugofs" @@ -182,6 +183,15 @@ func (c *Cache) ReadOrCreate(id string, return } +// NamedLock locks the given id. The lock is released when the returned function is called. +func (c *Cache) NamedLock(id string) func() { + id = cleanID(id) + c.nlocker.Lock(id) + return func() { + c.nlocker.Unlock(id) + } +} + // GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will // be invoked and the result cached. // This method is protected by a named lock using the given id as identifier. @@ -218,7 +228,23 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It var buff bytes.Buffer return info, hugio.ToReadCloser(&buff), - afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff)) + c.writeReader(id, io.TeeReader(r, &buff)) +} + +func (c *Cache) writeReader(id string, r io.Reader) error { + dir := filepath.Dir(id) + if dir != "" { + _ = c.Fs.MkdirAll(dir, 0o777) + } + f, err := c.Fs.Create(id) + if err != nil { + return err + } + defer f.Close() + + _, _ = io.Copy(f, r) + + return nil } // GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice. @@ -253,9 +279,10 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item return info, b, nil } - if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil { + if err := c.writeReader(id, bytes.NewReader(b)); err != nil { return info, nil, err } + return info, b, nil } @@ -305,16 +332,8 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser { return nil } - if c.maxAge > 0 { - fi, err := c.Fs.Stat(id) - if err != nil { - return nil - } - - if c.isExpired(fi.ModTime()) { - c.Fs.Remove(id) - return nil - } + if removed, err := c.removeIfExpired(id); err != nil || removed { + return nil } f, err := c.Fs.Open(id) @@ -325,6 +344,49 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser { return f } +func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) { + if c.maxAge == 0 { + // No caching. + return nil, false + } + + f, err := c.Fs.Open(id) + if err != nil { + return nil, false + } + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil, false + } + + removed, err := c.removeIfExpired(id) + if err != nil { + return nil, false + } + + return b, removed +} + +func (c *Cache) removeIfExpired(id string) (bool, error) { + if c.maxAge <= 0 { + return false, nil + } + + fi, err := c.Fs.Stat(id) + if err != nil { + return false, err + } + + if c.isExpired(fi.ModTime()) { + c.Fs.Remove(id) + return true, nil + } + + return false, nil +} + func (c *Cache) isExpired(modTime time.Time) bool { if c.maxAge < 0 { return false @@ -398,3 +460,37 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { func cleanID(name string) string { return strings.TrimPrefix(filepath.Clean(name), helpers.FilePathSeparator) } + +// AsHTTPCache returns an httpcache.Cache implementation for this file cache. +// Note that none of the methods are protected by named locks, so you need to make sure +// to do that in your own code. +func (c *Cache) AsHTTPCache() httpcache.Cache { + return &httpCache{c: c} +} + +type httpCache struct { + c *Cache +} + +func (h *httpCache) Get(id string) (resp []byte, ok bool) { + id = cleanID(id) + b, removed := h.c.getBytesAndRemoveIfExpired(id) + + return b, !removed +} + +func (h *httpCache) Set(id string, resp []byte) { + if h.c.maxAge == 0 { + return + } + + id = cleanID(id) + + if err := h.c.writeReader(id, bytes.NewReader(resp)); err != nil { + panic(err) + } +} + +func (h *httpCache) Delete(key string) { + h.c.Fs.Remove(key) +} diff --git a/cache/httpcache/httpcache.go b/cache/httpcache/httpcache.go new file mode 100644 index 000000000..ff360001f --- /dev/null +++ b/cache/httpcache/httpcache.go @@ -0,0 +1,208 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache + +import ( + "encoding/json" + "time" + + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/config" + "github.com/mitchellh/mapstructure" +) + +// DefaultConfig holds the default configuration for the HTTP cache. +var DefaultConfig = Config{ + Cache: Cache{ + For: GlobMatcher{ + Excludes: []string{"**"}, + }, + }, + Polls: []PollConfig{ + { + For: GlobMatcher{ + Includes: []string{"**"}, + }, + Disable: true, + }, + }, +} + +// Config holds the configuration for the HTTP cache. +type Config struct { + // Configures the HTTP cache behaviour (RFC 9111). + // When this is not enabled for a resource, Hugo will go straight to the file cache. + Cache Cache + + // Polls holds a list of configurations for polling remote resources to detect changes in watch mode. + // This can be disabled for some resources, typically if they are known to not change. + Polls []PollConfig +} + +type Cache struct { + // Enable HTTP cache behaviour (RFC 9111) for these rsources. + For GlobMatcher +} + +func (c *Config) Compile() (ConfigCompiled, error) { + var cc ConfigCompiled + + p, err := c.Cache.For.CompilePredicate() + if err != nil { + return cc, err + } + + cc.For = p + + for _, pc := range c.Polls { + + p, err := pc.For.CompilePredicate() + if err != nil { + return cc, err + } + + cc.PollConfigs = append(cc.PollConfigs, PollConfigCompiled{ + For: p, + Config: pc, + }) + } + + return cc, nil +} + +// PollConfig holds the configuration for polling remote resources to detect changes in watch mode. +// TODO1 make sure this enabled only in watch mode. +type PollConfig struct { + // What remote resources to apply this configuration to. + For GlobMatcher + + // Disable polling for this configuration. + Disable bool + + // Low is the lower bound for the polling interval. + // This is the starting point when the resource has recently changed, + // if that resource stops changing, the polling interval will gradually increase towards High. + Low time.Duration + + // High is the upper bound for the polling interval. + // This is the interval used when the resource is stable. + High time.Duration +} + +func (c PollConfig) MarshalJSON() (b []byte, err error) { + // Marshal the durations as strings. + type Alias PollConfig + return json.Marshal(&struct { + Low string + High string + Alias + }{ + Low: c.Low.String(), + High: c.High.String(), + Alias: (Alias)(c), + }) +} + +type GlobMatcher struct { + // Excludes holds a list of glob patterns that will be excluded. + Excludes []string + + // Includes holds a list of glob patterns that will be included. + Includes []string +} + +type ConfigCompiled struct { + For predicate.P[string] + PollConfigs []PollConfigCompiled +} + +func (c *ConfigCompiled) PollConfigFor(s string) PollConfigCompiled { + for _, pc := range c.PollConfigs { + if pc.For(s) { + return pc + } + } + return PollConfigCompiled{} +} + +func (c *ConfigCompiled) IsPollingDisabled() bool { + for _, pc := range c.PollConfigs { + if !pc.Config.Disable { + return false + } + } + return true +} + +type PollConfigCompiled struct { + For predicate.P[string] + Config PollConfig +} + +func (p PollConfigCompiled) IsZero() bool { + return p.For == nil +} + +func (gm *GlobMatcher) CompilePredicate() (func(string) bool, error) { + var p predicate.P[string] + for _, include := range gm.Includes { + g, err := glob.Compile(include, '/') + if err != nil { + return nil, err + } + fn := func(s string) bool { + return g.Match(s) + } + p = p.Or(fn) + } + + for _, exclude := range gm.Excludes { + g, err := glob.Compile(exclude, '/') + if err != nil { + return nil, err + } + fn := func(s string) bool { + return !g.Match(s) + } + p = p.And(fn) + } + + return p, nil +} + +func DecodeConfig(bcfg config.BaseConfig, m map[string]any) (Config, error) { + if len(m) == 0 { + return DefaultConfig, nil + } + + var c Config + + dc := &mapstructure.DecoderConfig{ + Result: &c, + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + } + + decoder, err := mapstructure.NewDecoder(dc) + if err != nil { + return c, err + } + + if err := decoder.Decode(m); err != nil { + return c, err + } + + return c, nil +} diff --git a/cache/httpcache/httpcache_integration_test.go b/cache/httpcache/httpcache_integration_test.go new file mode 100644 index 000000000..d3337c023 --- /dev/null +++ b/cache/httpcache/httpcache_integration_test.go @@ -0,0 +1,64 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache_test + +import ( + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" +) + +func TestConfigCustom(t *testing.T) { + files := ` +-- hugo.toml -- +[httpcache] +[httpcache.cache.for] +includes = ["**gohugo.io**"] +[[httpcache.polls]] +low = "5s" +high = "32s" +[httpcache.polls.for] +includes = ["**gohugo.io**"] + + +` + + b := hugolib.Test(t, files) + + httpcacheConf := b.H.Configs.Base.HTTPCache + compiled := b.H.Configs.Base.C.HTTPCache + + b.Assert(httpcacheConf.Cache.For.Includes, qt.DeepEquals, []string{"**gohugo.io**"}) + b.Assert(httpcacheConf.Cache.For.Excludes, qt.IsNil) + + pc := compiled.PollConfigFor("https://gohugo.io/foo.jpg") + b.Assert(pc.Config.Low, qt.Equals, 5*time.Second) + b.Assert(pc.Config.High, qt.Equals, 32*time.Second) + b.Assert(compiled.PollConfigFor("https://example.com/foo.jpg").IsZero(), qt.IsTrue) +} + +func TestConfigDefault(t *testing.T) { + files := ` +-- hugo.toml -- +` + b := hugolib.Test(t, files) + + compiled := b.H.Configs.Base.C.HTTPCache + + b.Assert(compiled.For("https://gohugo.io/posts.json"), qt.IsFalse) + b.Assert(compiled.For("https://gohugo.io/foo.jpg"), qt.IsFalse) + b.Assert(compiled.PollConfigFor("https://gohugo.io/foo.jpg").Config.Disable, qt.IsTrue) +} diff --git a/cache/httpcache/httpcache_test.go b/cache/httpcache/httpcache_test.go new file mode 100644 index 000000000..e3659f97b --- /dev/null +++ b/cache/httpcache/httpcache_test.go @@ -0,0 +1,42 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpcache + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestGlobMatcher(t *testing.T) { + c := qt.New(t) + + g := GlobMatcher{ + Includes: []string{"**/*.jpg", "**.png", "**/bar/**"}, + Excludes: []string{"**/foo.jpg", "**.css"}, + } + + p, err := g.CompilePredicate() + c.Assert(err, qt.IsNil) + + c.Assert(p("foo.jpg"), qt.IsFalse) + c.Assert(p("foo.png"), qt.IsTrue) + c.Assert(p("foo/bar.jpg"), qt.IsTrue) + c.Assert(p("foo/bar.png"), qt.IsTrue) + c.Assert(p("foo/bar/foo.jpg"), qt.IsFalse) + c.Assert(p("foo/bar/foo.css"), qt.IsFalse) + c.Assert(p("foo.css"), qt.IsFalse) + c.Assert(p("foo/bar/foo.css"), qt.IsFalse) + c.Assert(p("foo/bar/foo.xml"), qt.IsTrue) +} diff --git a/commands/commandeer.go b/commands/commandeer.go index 59fe32f74..f18a95bb9 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -48,6 +48,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/resources/kinds" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -103,6 +104,9 @@ type rootCommand struct { commonConfigs *lazycache.Cache[int32, *commonConfig] hugoSites *lazycache.Cache[int32, *hugolib.HugoSites] + // changesFromBuild received from Hugo in watch mode. + changesFromBuild chan []identity.Identity + commands []simplecobra.Commander // Flags @@ -304,7 +308,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) { h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { - depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()} + depsCfg := r.newDepsConfig(conf) return hugolib.NewHugoSites(depsCfg) }) return h, err @@ -316,12 +320,16 @@ func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) { if err != nil { return nil, err } - depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level()} + depsCfg := r.newDepsConfig(conf) return hugolib.NewHugoSites(depsCfg) }) return h, err } +func (r *rootCommand) newDepsConfig(conf *commonConfig) deps.DepsCfg { + return deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, LogOut: r.logger.Out(), LogLevel: r.logger.Level(), ChangesFromBuild: r.changesFromBuild} +} + func (r *rootCommand) Name() string { return "hugo" } @@ -408,6 +416,8 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error { return err } + r.changesFromBuild = make(chan []identity.Identity, 10) + r.commonConfigs = lazycache.New(lazycache.Options[int32, *commonConfig]{MaxEntries: 5}) // We don't want to keep stale HugoSites in memory longer than needed. r.hugoSites = lazycache.New(lazycache.Options[int32, *hugolib.HugoSites]{ diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index 32b7e1de8..99bd8a04a 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -43,6 +43,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/livereload" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/watcher" @@ -343,6 +344,24 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa go func() { for { select { + case changes := <-c.r.changesFromBuild: + unlock, err := h.LockBuild() + if err != nil { + c.r.logger.Errorln("Failed to acquire a build lock: %s", err) + return + } + c.changeDetector.PrepareNew() + err = c.rebuildSitesForChanges(changes) + if err != nil { + c.r.logger.Errorln("Error while watching:", err) + } + if c.s != nil && c.s.doLiveReload { + if c.changeDetector == nil || len(c.changeDetector.changed()) > 0 { + livereload.ForceRefresh() + } + } + unlock() + case evs := <-watcher.Events: unlock, err := h.LockBuild() if err != nil { @@ -1019,6 +1038,19 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error { return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}, events...) } +func (c *hugoBuilder) rebuildSitesForChanges(ids []identity.Identity) error { + c.errState.setBuildErr(nil) + h, err := c.hugo() + if err != nil { + return err + } + whatChanged := &hugolib.WhatChanged{} + whatChanged.Add(ids...) + err = h.Build(hugolib.BuildCfg{NoBuildLock: true, WhatChanged: whatChanged, RecentlyVisited: c.visitedURLs, ErrRecovery: c.errState.wasErr()}) + c.errState.setBuildErr(err) + return err +} + func (c *hugoBuilder) reloadConfig() error { c.r.Reset() c.r.configVersionID.Add(1) diff --git a/common/maps/maps.go b/common/maps/maps.go index 2686baad6..f9171ebf2 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -112,17 +112,17 @@ func ToSliceStringMap(in any) ([]map[string]any, error) { } // LookupEqualFold finds key in m with case insensitive equality checks. -func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) { +func LookupEqualFold[T any | string](m map[string]T, key string) (T, string, bool) { if v, found := m[key]; found { - return v, true + return v, key, true } for k, v := range m { if strings.EqualFold(k, key) { - return v, true + return v, k, true } } var s T - return s, false + return s, "", false } // MergeShallow merges src into dst, but only if the key does not already exist in dst. diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go index 098098388..b4f9c5a3d 100644 --- a/common/maps/maps_test.go +++ b/common/maps/maps_test.go @@ -180,16 +180,18 @@ func TestLookupEqualFold(t *testing.T) { "B": "bv", } - v, found := LookupEqualFold(m1, "b") + v, k, found := LookupEqualFold(m1, "b") c.Assert(found, qt.IsTrue) c.Assert(v, qt.Equals, "bv") + c.Assert(k, qt.Equals, "B") m2 := map[string]string{ "a": "av", "B": "bv", } - v, found = LookupEqualFold(m2, "b") + v, k, found = LookupEqualFold(m2, "b") c.Assert(found, qt.IsTrue) + c.Assert(k, qt.Equals, "B") c.Assert(v, qt.Equals, "bv") } diff --git a/common/predicate/predicate.go b/common/predicate/predicate.go index f9cb1bb2b..f71536474 100644 --- a/common/predicate/predicate.go +++ b/common/predicate/predicate.go @@ -24,6 +24,9 @@ func (p P[T]) And(ps ...P[T]) P[T] { return false } } + if p == nil { + return true + } return p(v) } } @@ -36,6 +39,9 @@ func (p P[T]) Or(ps ...P[T]) P[T] { return true } } + if p == nil { + return false + } return p(v) } } diff --git a/common/tasks/tasks.go b/common/tasks/tasks.go new file mode 100644 index 000000000..1f7e061f9 --- /dev/null +++ b/common/tasks/tasks.go @@ -0,0 +1,153 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "sync" + "time" +) + +// RunEvery runs a function at intervals defined by the function itself. +// Functions can be added and removed while running. +type RunEvery struct { + // Any error returned from the function will be passed to this function. + HandleError func(string, error) + + // If set, the function will be run immediately. + RunImmediately bool + + // The named functions to run. + funcs map[string]*Func + + mu sync.Mutex + started bool + closed bool + quit chan struct{} +} + +type Func struct { + // The shortest interval between each run. + IntervalLow time.Duration + + // The longest interval between each run. + IntervalHigh time.Duration + + // The function to run. + F func(interval time.Duration) (time.Duration, error) + + interval time.Duration + last time.Time +} + +func (r *RunEvery) Start() error { + if r.started { + return nil + } + + r.started = true + r.quit = make(chan struct{}) + + go func() { + if r.RunImmediately { + r.run() + } + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-r.quit: + return + case <-ticker.C: + r.run() + } + } + }() + + return nil +} + +// Close stops the RunEvery from running. +func (r *RunEvery) Close() error { + if r.closed { + return nil + } + r.closed = true + if r.quit != nil { + close(r.quit) + } + return nil +} + +// Add adds a function to the RunEvery. +func (r *RunEvery) Add(name string, f Func) { + r.mu.Lock() + defer r.mu.Unlock() + if r.funcs == nil { + r.funcs = make(map[string]*Func) + } + if f.IntervalLow == 0 { + f.IntervalLow = 500 * time.Millisecond + } + if f.IntervalHigh <= f.IntervalLow { + f.IntervalHigh = 20 * time.Second + } + + start := f.IntervalHigh / 3 + if start < f.IntervalLow { + start = f.IntervalLow + } + f.interval = start + f.last = time.Now() + + r.funcs[name] = &f +} + +// Remove removes a function from the RunEvery. +func (r *RunEvery) Remove(name string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.funcs, name) +} + +// Has returns whether the RunEvery has a function with the given name. +func (r *RunEvery) Has(name string) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, found := r.funcs[name] + return found +} + +func (r *RunEvery) run() { + r.mu.Lock() + defer r.mu.Unlock() + for name, f := range r.funcs { + if time.Now().Before(f.last.Add(f.interval)) { + continue + } + f.last = time.Now() + interval, err := f.F(f.interval) + if err != nil && r.HandleError != nil { + r.HandleError(name, err) + } + + if interval < f.IntervalLow { + interval = f.IntervalLow + } + + if interval > f.IntervalHigh { + interval = f.IntervalHigh + } + f.interval = interval + } +} diff --git a/common/types/closer.go b/common/types/closer.go new file mode 100644 index 000000000..2844b1986 --- /dev/null +++ b/common/types/closer.go @@ -0,0 +1,47 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "sync" + +type Closer interface { + Close() error +} + +type CloseAdder interface { + Add(Closer) +} + +type Closers struct { + mu sync.Mutex + cs []Closer +} + +func (cs *Closers) Add(c Closer) { + cs.mu.Lock() + defer cs.mu.Unlock() + cs.cs = append(cs.cs, c) +} + +func (cs *Closers) Close() error { + cs.mu.Lock() + defer cs.mu.Unlock() + for _, c := range cs.cs { + c.Close() + } + + cs.cs = cs.cs[:0] + + return nil +} diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 76153f5c0..5ff456d55 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -27,6 +27,7 @@ import ( "time" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/cache/httpcache" "github.com/gohugoio/hugo/common/hugo" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" @@ -119,6 +120,10 @@ type Config struct { // {"identifiers": ["caches"] } Caches filecache.Configs `mapstructure:"-"` + // The httpcache configuration section contains HTTP-cache-related configuration options. + // {"identifiers": ["httpcache"] } + HTTPCache httpcache.Config `mapstructure:"-"` + // The markup configuration section contains markup-related configuration options. // {"identifiers": ["markup"] } Markup markup_config.Config `mapstructure:"-"` @@ -359,6 +364,11 @@ func (c *Config) CompileConfig(logger loggers.Logger) error { } } + httpCache, err := c.HTTPCache.Compile() + if err != nil { + return err + } + c.C = &ConfigCompiled{ Timeout: timeout, BaseURL: baseURL, @@ -374,6 +384,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error { SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...), MainSections: c.MainSections, Clock: clock, + HTTPCache: httpCache, transientErr: transientErr, } @@ -413,6 +424,7 @@ type ConfigCompiled struct { SegmentFilter segments.SegmentFilter MainSections []string Clock time.Time + HTTPCache httpcache.ConfigCompiled // This is set to the last transient error found during config compilation. // With themes/modules we compute the configuration in multiple passes, and diff --git a/config/allconfig/alldecoders.go b/config/allconfig/alldecoders.go index 7d968e4ad..fc033821e 100644 --- a/config/allconfig/alldecoders.go +++ b/config/allconfig/alldecoders.go @@ -18,6 +18,8 @@ import ( "strings" "github.com/gohugoio/hugo/cache/filecache" + + "github.com/gohugoio/hugo/cache/httpcache" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" @@ -96,6 +98,18 @@ var allDecoderSetups = map[string]decodeWeight{ return err }, }, + "httpcache": { + key: "httpcache", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.HTTPCache, err = httpcache.DecodeConfig(p.bcfg, p.p.GetStringMap(d.key)) + if p.c.IgnoreCache { + p.c.HTTPCache.Cache.For.Excludes = []string{"**"} + p.c.HTTPCache.Cache.For.Includes = []string{} + } + return err + }, + }, "build": { key: "build", decode: func(d decodeWeight, p decodeConfig) error { diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index a215fb5e4..1d2cb5ce3 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -173,6 +173,8 @@ func (c ConfigLanguage) GetConfigSection(s string) any { return c.m.Modules case "deployment": return c.config.Deployment + case "httpCacheCompiled": + return c.config.C.HTTPCache default: panic("not implemented: " + s) } diff --git a/deps/deps.go b/deps/deps.go index 41a8ecb3e..678f8a2fc 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -15,11 +15,13 @@ import ( "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/postpub" @@ -85,7 +87,7 @@ type Deps struct { BuildEndListeners *Listeners // Resources that gets closed when the build is done or the server shuts down. - BuildClosers *Closers + BuildClosers *types.Closers // This is common/global for all sites. BuildState *BuildState @@ -143,7 +145,7 @@ func (d *Deps) Init() error { } if d.BuildClosers == nil { - d.BuildClosers = &Closers{} + d.BuildClosers = &types.Closers{} } if d.Metrics == nil && d.Conf.TemplateMetrics() { @@ -208,7 +210,7 @@ func (d *Deps) Init() error { return fmt.Errorf("failed to create file caches from configuration: %w", err) } - resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper) + resourceSpec, err := resources.NewSpec(d.PathSpec, common, fileCaches, d.MemCache, d.BuildState, d.Log, d, d.ExecHelper, d.BuildClosers, d.BuildState) if err != nil { return fmt.Errorf("failed to create resource spec: %w", err) } @@ -353,6 +355,9 @@ type DepsCfg struct { // i18n handling. TranslationProvider ResourceProvider + + // ChangesFromBuild for changes passed back to the server/watch process. + ChangesFromBuild chan []identity.Identity } // BuildState are state used during a build. @@ -361,11 +366,19 @@ type BuildState struct { mu sync.Mutex // protects state below. + OnSignalRebuild func(ids ...identity.Identity) + // A set of filenames in /public that // contains a post-processing prefix. filenamesWithPostPrefix map[string]bool } +var _ identity.SignalRebuilder = (*BuildState)(nil) + +func (b *BuildState) SignalRebuild(ids ...identity.Identity) { + b.OnSignalRebuild(ids...) +} + func (b *BuildState) AddFilenameWithPostPrefix(filename string) { b.mu.Lock() defer b.mu.Unlock() @@ -389,30 +402,3 @@ func (b *BuildState) GetFilenamesWithPostPrefix() []string { func (b *BuildState) Incr() int { return int(atomic.AddUint64(&b.counter, uint64(1))) } - -type Closer interface { - Close() error -} - -type Closers struct { - mu sync.Mutex - cs []Closer -} - -func (cs *Closers) Add(c Closer) { - cs.mu.Lock() - defer cs.mu.Unlock() - cs.cs = append(cs.cs, c) -} - -func (cs *Closers) Close() error { - cs.mu.Lock() - defer cs.mu.Unlock() - for _, c := range cs.cs { - c.Close() - } - - cs.cs = cs.cs[:0] - - return nil -} diff --git a/go.mod b/go.mod index e82180e0e..574bbdc4e 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/gobuffalo/flect v1.0.2 github.com/gobwas/glob v0.2.3 github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e + github.com/gohugoio/httpcache v0.6.0 github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 github.com/gohugoio/locales v0.14.0 diff --git a/go.sum b/go.sum index a59cbf6d6..923f6faea 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -67,9 +68,11 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/assert/v2 v2.6.0 h1:o3WJwILtexrEUk3cUVal3oiQY2tfgr/FHWiz/v2n4FU= +github.com/alecthomas/assert/v2 v2.6.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.13.0 h1:VP72+99Fb2zEcYM0MeaWJmV+xQvz5v5cxRHd+ooU1lI= github.com/alecthomas/chroma/v2 v2.13.0/go.mod h1:BUGjjsD+ndS6eX37YgTchSEG+Jg9Jv1GiZs9sqPqztk= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M= github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.50.7 h1:odKb+uneeGgF2jgAerKjFzpljiyZxleV4SHB7oBK+YA= @@ -171,6 +174,7 @@ github.com/disintegration/gift v1.2.1/go.mod h1:Jh2i7f7Q2BM7Ezno3PhfezbR1xpUg9dU github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -195,8 +199,6 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getkin/kin-openapi v0.123.0 h1:zIik0mRwFNLyvtXK274Q6ut+dPh6nlxBp0x7mNrPhs8= github.com/getkin/kin-openapi v0.123.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= -github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= -github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -207,12 +209,19 @@ github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1Rf github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e h1:QArsSubW7eDh8APMXkByjQWvuljwPGAGQpJEFn0F0wY= github.com/gohugoio/go-i18n/v2 v2.1.3-0.20230805085216-e63c13218d0e/go.mod h1:3Ltoo9Banwq0gOtcOwxuHG6omk+AwsQPADyw2vQYOJQ= +github.com/gohugoio/httpcache v0.5.0 h1:9xi4VuXd+KT3h0jOs8DlZxTMu5CtjDr0BvQMAuL/O5I= +github.com/gohugoio/httpcache v0.5.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI= +github.com/gohugoio/httpcache v0.6.0 h1:5pYJM43Yoc4uvIJ+/e770PS48srTumvuQZpuBfGFZV0= +github.com/gohugoio/httpcache v0.6.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI= +github.com/gohugoio/httpcache v0.7.0 h1:ukPnn04Rgvx48JIinZvZetBfHaWE7I01JR2Q2RrQ3Vs= +github.com/gohugoio/httpcache v0.7.0/go.mod h1:fMlPrdY/vVJhAriLZnrF5QpN3BNAcoBClgAyQd+lGFI= github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0 h1:YhxZNU8y2vxV6Ibr7QJzzUlpr8oHHWX/l+Q1R/a5Zao= github.com/gohugoio/hugo-goldmark-extensions/extras v0.1.0/go.mod h1:0cuvOnGKW7WeXA3i7qK6IS07FH1bgJ2XzOjQ7BMJYH4= github.com/gohugoio/hugo-goldmark-extensions/passthrough v0.2.0 h1:PCtO5l++psZf48yen2LxQ3JiOXxaRC6v0594NeHvGZg= @@ -274,12 +283,15 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= +github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= +github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -315,6 +327,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -394,6 +407,7 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -439,7 +453,9 @@ github.com/tdewolff/parse/v2 v2.7.13 h1:iSiwOUkCYLNfapHoqdLcqZVgvQ0jrsao8YYKP/UJ github.com/tdewolff/parse/v2 v2.7.13/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo= +github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -806,6 +822,7 @@ google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/neurosnap/sentences.v1 v1.0.6/go.mod h1:YlK+SN+fLQZj+kY3r8DkGDhDr91+S3JmTb5LSxFRQo0= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index a8f5b5fd7..0ce43ea68 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -975,7 +975,7 @@ type contentTreeReverseIndexMap struct { type sitePagesAssembler struct { *Site - assembleChanges *whatChanged + assembleChanges *WhatChanged ctx context.Context } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 61a07812d..25a79d65a 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -405,8 +405,9 @@ func (h *HugoSites) withPage(fn func(s string, p *pageState) bool) { type BuildCfg struct { // Skip rendering. Useful for testing. SkipRender bool + // Use this to indicate what changed (for rebuilds). - whatChanged *whatChanged + WhatChanged *WhatChanged // This is a partial re-render of some selected pages. PartialReRender bool diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 8a4966055..fe05f5174 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -114,9 +114,9 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { // Need a pointer as this may be modified. conf := &config - if conf.whatChanged == nil { + if conf.WhatChanged == nil { // Assume everything has changed - conf.whatChanged = &whatChanged{needsPagesAssembly: true} + conf.WhatChanged = &WhatChanged{needsPagesAssembly: true} } var prepareErr error @@ -128,7 +128,7 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { s.Deps.BuildStartListeners.Notify() } - if len(events) > 0 { + if len(events) > 0 || len(conf.WhatChanged.Changes()) > 0 { // Rebuild if err := h.initRebuild(conf); err != nil { return fmt.Errorf("initRebuild: %w", err) @@ -224,7 +224,7 @@ func (h *HugoSites) initRebuild(config *BuildCfg) error { }) for _, s := range h.Sites { - s.resetBuildState(config.whatChanged.needsPagesAssembly) + s.resetBuildState(config.WhatChanged.needsPagesAssembly) } h.reset(config) @@ -245,7 +245,9 @@ func (h *HugoSites) process(ctx context.Context, l logg.LevelLogger, config *Bui if len(events) > 0 { // This is a rebuild - return h.processPartial(ctx, l, config, init, events) + return h.processPartialFileEvents(ctx, l, config, init, events) + } else if len(config.WhatChanged.Changes()) > 0 { + return h.processPartialRebuildChanges(ctx, l, config) } return h.processFull(ctx, l, config) } @@ -256,8 +258,8 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil l = l.WithField("step", "assemble") defer loggers.TimeTrackf(l, time.Now(), nil, "") - if !bcfg.whatChanged.needsPagesAssembly { - changes := bcfg.whatChanged.Drain() + if !bcfg.WhatChanged.needsPagesAssembly { + changes := bcfg.WhatChanged.Drain() if len(changes) > 0 { if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil { return err @@ -273,7 +275,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil for i, s := range h.Sites { assemblers[i] = &sitePagesAssembler{ Site: s, - assembleChanges: bcfg.whatChanged, + assembleChanges: bcfg.WhatChanged, ctx: ctx, } } @@ -289,7 +291,7 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil return err } - changes := bcfg.whatChanged.Drain() + changes := bcfg.WhatChanged.Drain() // Changes from the assemble step (e.g. lastMod, cascade) needs a re-calculation // of what needs to be re-built. @@ -612,8 +614,19 @@ func (p pathChange) isStructuralChange() bool { return p.delete || p.isDir } -// processPartial prepares the Sites' sources for a partial rebuild. -func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error { +func (h *HugoSites) processPartialRebuildChanges(ctx context.Context, l logg.LevelLogger, config *BuildCfg) error { + if err := h.resolveAndClearStateForIdentities(ctx, l, nil, config.WhatChanged.Drain()); err != nil { + return err + } + + if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil { + return err + } + return nil +} + +// processPartialFileEvents prepares the Sites' sources for a partial rebuild. +func (h *HugoSites) processPartialFileEvents(ctx context.Context, l logg.LevelLogger, config *BuildCfg, init func(config *BuildCfg) error, events []fsnotify.Event) error { h.Log.Trace(logg.StringFunc(func() string { var sb strings.Builder sb.WriteString("File events:\n") @@ -887,13 +900,13 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf resourceFiles := h.fileEventsContentPaths(addedOrChangedContent) - changed := &whatChanged{ + changed := &WhatChanged{ needsPagesAssembly: needsPagesAssemble, identitySet: make(identity.Identities), } changed.Add(changes...) - config.whatChanged = changed + config.WhatChanged = changed if err := init(config); err != nil { return err @@ -977,14 +990,14 @@ func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, buildConf } if len(bi.ChangedIdentities) > 0 { - buildConfig.whatChanged.Add(bi.ChangedIdentities...) - buildConfig.whatChanged.needsPagesAssembly = true + buildConfig.WhatChanged.Add(bi.ChangedIdentities...) + buildConfig.WhatChanged.needsPagesAssembly = true } for _, p := range bi.DeletedPaths { pp := path.Join(bi.Path.Base(), p) if v, ok := s.pageMap.treePages.Delete(pp); ok { - buildConfig.whatChanged.Add(v.GetIdentity()) + buildConfig.WhatChanged.Add(v.GetIdentity()) } } } diff --git a/hugolib/site.go b/hugolib/site.go index d9103e737..b4b89975d 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -371,14 +371,14 @@ func (s *Site) watching() bool { return s.h != nil && s.h.Configs.Base.Internal.Watch } -type whatChanged struct { +type WhatChanged struct { mu sync.Mutex needsPagesAssembly bool identitySet identity.Identities } -func (w *whatChanged) Add(ids ...identity.Identity) { +func (w *WhatChanged) Add(ids ...identity.Identity) { w.mu.Lock() defer w.mu.Unlock() @@ -391,24 +391,24 @@ func (w *whatChanged) Add(ids ...identity.Identity) { } } -func (w *whatChanged) Clear() { +func (w *WhatChanged) Clear() { w.mu.Lock() defer w.mu.Unlock() w.clear() } -func (w *whatChanged) clear() { +func (w *WhatChanged) clear() { w.identitySet = identity.Identities{} } -func (w *whatChanged) Changes() []identity.Identity { +func (w *WhatChanged) Changes() []identity.Identity { if w == nil || w.identitySet == nil { return nil } return w.identitySet.AsSlice() } -func (w *whatChanged) Drain() []identity.Identity { +func (w *WhatChanged) Drain() []identity.Identity { w.mu.Lock() defer w.mu.Unlock() ids := w.identitySet.AsSlice() diff --git a/hugolib/site_new.go b/hugolib/site_new.go index 788b80a3f..2ba5ef2fb 100644 --- a/hugolib/site_new.go +++ b/hugolib/site_new.go @@ -141,10 +141,23 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { memCache := dynacache.New(dynacache.Options{Watching: conf.Watching(), Log: logger}) + var h *HugoSites + onSignalRebuild := func(ids ...identity.Identity) { + // This channel is buffered, but make sure we do this in a non-blocking way. + if cfg.ChangesFromBuild != nil { + go func() { + cfg.ChangesFromBuild <- ids + }() + } + } + firstSiteDeps := &deps.Deps{ - Fs: cfg.Fs, - Log: logger, - Conf: conf, + Fs: cfg.Fs, + Log: logger, + Conf: conf, + BuildState: &deps.BuildState{ + OnSignalRebuild: onSignalRebuild, + }, MemCache: memCache, TemplateProvider: tplimpl.DefaultTemplateProvider, TranslationProvider: i18n.NewTranslationProvider(), @@ -261,7 +274,8 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return li.Lang < lj.Lang }) - h, err := newHugoSites(cfg, firstSiteDeps, pageTrees, sites) + var err error + h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites) if err == nil && h == nil { panic("hugo: newHugoSitesNew returned nil error and nil HugoSites") } diff --git a/identity/identity.go b/identity/identity.go index f924f335c..d106eb1fc 100644 --- a/identity/identity.go +++ b/identity/identity.go @@ -241,6 +241,11 @@ type IdentityProvider interface { GetIdentity() Identity } +// SignalRebuilder is an optional interface for types that can signal a rebuild. +type SignalRebuilder interface { + SignalRebuild(ids ...Identity) +} + // IncrementByOne implements Incrementer adding 1 every time Incr is called. type IncrementByOne struct { counter uint64 diff --git a/media/config.go b/media/config.go index 18e983369..e00837e5e 100644 --- a/media/config.go +++ b/media/config.go @@ -194,7 +194,7 @@ func DecodeTypes(in map[string]any) (*config.ConfigNamespace[map[string]MediaTyp return nil, nil, err } mm := maps.ToStringMap(v) - suffixes, found := maps.LookupEqualFold(mm, "suffixes") + suffixes, _, found := maps.LookupEqualFold(mm, "suffixes") if found { mediaType.SuffixesCSV = strings.TrimSpace(strings.ToLower(strings.Join(cast.ToStringSlice(suffixes), ","))) } diff --git a/parser/lowercase_camel_json.go b/parser/lowercase_camel_json.go index af0891de6..c61a4078e 100644 --- a/parser/lowercase_camel_json.go +++ b/parser/lowercase_camel_json.go @@ -46,6 +46,12 @@ type LowerCaseCamelJSONMarshaller struct { Value any } +var preserveUpperCaseKeyRe = regexp.MustCompile(`^"HTTP`) + +func preserveUpperCaseKey(match []byte) bool { + return preserveUpperCaseKeyRe.Match(match) +} + func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) { marshalled, err := json.Marshal(c.Value) @@ -59,7 +65,7 @@ func (c LowerCaseCamelJSONMarshaller) MarshalJSON() ([]byte, error) { // Empty keys are valid JSON, only lowercase if we do not have an // empty key. - if len(match) > 2 { + if len(match) > 2 && !preserveUpperCaseKey(match) { // Decode first rune after the double quotes r, width := utf8.DecodeRune(match[1:]) r = unicode.ToLower(r) diff --git a/resources/resource_cache.go b/resources/resource_cache.go index bf930c71d..a3ba9aa26 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -36,6 +36,11 @@ func newResourceCache(rs *Spec, memCache *dynacache.Cache) *ResourceCache { "/res1", dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40}, ), + CacheResourceRemote: dynacache.GetOrCreatePartition[string, resource.Resource]( + memCache, + "/resr", + dynacache.OptionsPartition{ClearWhen: dynacache.ClearOnChange, Weight: 40}, + ), cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources]( memCache, "/ress", @@ -53,6 +58,7 @@ type ResourceCache struct { sync.RWMutex cacheResource *dynacache.Partition[string, resource.Resource] + CacheResourceRemote *dynacache.Partition[string, resource.Resource] cacheResources *dynacache.Partition[string, resource.Resources] cacheResourceTransformation *dynacache.Partition[string, *resourceAdapterInner] diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 4725cf390..35a1fb59d 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -23,6 +23,9 @@ import ( "strings" "time" + "github.com/bep/logg" + "github.com/gohugoio/httpcache" + hhttpcache "github.com/gohugoio/hugo/cache/httpcache" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/identity" @@ -31,7 +34,9 @@ import ( "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/hcontext" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/tasks" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" ) @@ -39,19 +44,76 @@ import ( // Client contains methods to create Resource objects. // tasks to Resource objects. type Client struct { - rs *resources.Spec - httpClient *http.Client - cacheGetResource *filecache.Cache + rs *resources.Spec + httpClient *http.Client + httpCacheConfig hhttpcache.ConfigCompiled + cacheGetResource *filecache.Cache + resourceIDDispatcher hcontext.ContextDispatcher[string] + + // Set when watching. + remoteResourceChecker *tasks.RunEvery + remoteResourceLogger logg.LevelLogger } +type contextKey string + // New creates a new Client with the given specification. func New(rs *resources.Spec) *Client { + fileCache := rs.FileCaches.GetResourceCache() + resourceIDDispatcher := hcontext.NewContextDispatcher[string](contextKey("resourceID")) + httpCacheConfig := rs.Cfg.GetConfigSection("httpCacheCompiled").(hhttpcache.ConfigCompiled) + var remoteResourceChecker *tasks.RunEvery + if rs.Cfg.Watching() && !httpCacheConfig.IsPollingDisabled() { + remoteResourceChecker = &tasks.RunEvery{ + HandleError: func(name string, err error) { + rs.Logger.Warnf("Failed to check remote resource: %s", err) + }, + RunImmediately: false, + } + + if err := remoteResourceChecker.Start(); err != nil { + panic(err) + } + + rs.BuildClosers.Add(remoteResourceChecker) + } + + httpTimeout := 2 * time.Minute // Need to cover retries. + if httpTimeout < (rs.Cfg.Timeout() + 30*time.Second) { + httpTimeout = rs.Cfg.Timeout() + 30*time.Second + } + return &Client{ - rs: rs, + rs: rs, + httpCacheConfig: httpCacheConfig, + resourceIDDispatcher: resourceIDDispatcher, + remoteResourceChecker: remoteResourceChecker, + remoteResourceLogger: rs.Logger.InfoCommand("remote"), httpClient: &http.Client{ - Timeout: time.Minute, + Timeout: httpTimeout, + Transport: &httpcache.Transport{ + Cache: fileCache.AsHTTPCache(), + CacheKey: func(req *http.Request) string { + return resourceIDDispatcher.Get(req.Context()) + }, + Around: func(req *http.Request, key string) func() { + return fileCache.NamedLock(key) + }, + AlwaysUseCachedResponse: func(req *http.Request, key string) bool { + return !httpCacheConfig.For(req.URL.String()) + }, + ShouldCache: func(req *http.Request, resp *http.Response, key string) bool { + return shouldCache(resp.StatusCode) + }, + MarkCachedResponses: true, + EnableETagPair: true, + Transport: &transport{ + Cfg: rs.Cfg, + Logger: rs.Logger, + }, + }, }, - cacheGetResource: rs.FileCaches.GetResourceCache(), + cacheGetResource: fileCache, } } diff --git a/resources/resource_factories/create/create_integration_test.go b/resources/resource_factories/create/create_integration_test.go index 61bc17adb..17084574d 100644 --- a/resources/resource_factories/create/create_integration_test.go +++ b/resources/resource_factories/create/create_integration_test.go @@ -134,8 +134,7 @@ mediaTypes = ['text/plain'] // This is hard to get stable on GitHub Actions, it sometimes succeeds due to timing issues. if err != nil { b.AssertLogContains("Got Err") - b.AssertLogContains("Retry timeout") - b.AssertLogContains("ContentLength:0") + b.AssertLogContains("retry timeout") } }) } diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go index c2d17e7a5..ef8078228 100644 --- a/resources/resource_factories/create/remote.go +++ b/resources/resource_factories/create/remote.go @@ -14,22 +14,27 @@ package create import ( - "bufio" "bytes" + "context" "fmt" "io" "math/rand" "mime" "net/http" - "net/http/httputil" "net/url" "path" "strings" "time" + gmaps "maps" + + "github.com/gohugoio/httpcache" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/tasks" "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -92,6 +97,60 @@ var temporaryHTTPStatusCodes = map[int]bool{ 504: true, } +func (c *Client) configurePollingIfEnabled(uri, optionsKey string, getRes func() (*http.Response, error)) { + if c.remoteResourceChecker == nil { + return + } + + // Set up polling for changes to this resource. + pollingConfig := c.httpCacheConfig.PollConfigFor(uri) + if pollingConfig.IsZero() || pollingConfig.Config.Disable { + return + } + + if c.remoteResourceChecker.Has(optionsKey) { + return + } + + var lastChange time.Time + c.remoteResourceChecker.Add(optionsKey, + tasks.Func{ + IntervalLow: pollingConfig.Config.Low, + IntervalHigh: pollingConfig.Config.High, + F: func(interval time.Duration) (time.Duration, error) { + start := time.Now() + defer func() { + duration := time.Since(start) + c.rs.Logger.Debugf("Polled remote resource for changes in %13s. Interval: %4s (low: %4s high: %4s) resource: %q ", duration, interval, pollingConfig.Config.Low, pollingConfig.Config.High, uri) + }() + // TODO(bep) figure out a ways to remove unused tasks. + res, err := getRes() + if err != nil { + return pollingConfig.Config.High, err + } + // The caching is delayed until the body is read. + io.Copy(io.Discard, res.Body) + res.Body.Close() + x1, x2 := res.Header.Get(httpcache.XETag1), res.Header.Get(httpcache.XETag2) + if x1 != x2 { + lastChange = time.Now() + c.remoteResourceLogger.Logf("detected change in remote resource %q", uri) + c.rs.Rebuilder.SignalRebuild(identity.StringIdentity(optionsKey)) + } + + if time.Since(lastChange) < 10*time.Second { + // The user is typing, check more often. + return 0, nil + } + + // Increase the interval to avoid hammering the server. + interval += 1 * time.Second + + return interval, nil + }, + }) +} + // FromRemote expects one or n-parts of a URL to a resource // If you provide multiple parts they will be joined together to the final URL. func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resource, error) { @@ -101,168 +160,139 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou } method := "GET" - if s, ok := maps.LookupEqualFold(optionsm, "method"); ok { + if s, _, ok := maps.LookupEqualFold(optionsm, "method"); ok { method = strings.ToUpper(s.(string)) } isHeadMethod := method == "HEAD" - resourceID := calculateResourceID(uri, optionsm) + optionsm = gmaps.Clone(optionsm) + userKey, optionsKey := remoteResourceKeys(uri, optionsm) - _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { + // A common pattern is to use the key in the options map as + // a way to control cache eviction, + // so make sure we use any user provided kehy as the file cache key, + // but the auto generated and more stable key for everything else. + filecacheKey := userKey + + return c.rs.ResourceCache.CacheResourceRemote.GetOrCreate(optionsKey, func(key string) (resource.Resource, error) { options, err := decodeRemoteOptions(optionsm) if err != nil { return nil, fmt.Errorf("failed to decode options for resource %s: %w", uri, err) } + if err := c.validateFromRemoteArgs(uri, options); err != nil { return nil, err } - var ( - start time.Time - nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond - nextSleepLimit = time.Duration(5) * time.Second - ) + getRes := func() (*http.Response, error) { + ctx := context.Background() + ctx = c.resourceIDDispatcher.Set(ctx, filecacheKey) - for { - b, retry, err := func() ([]byte, bool, error) { - req, err := options.NewRequest(uri) - if err != nil { - return nil, false, fmt.Errorf("failed to create request for resource %s: %w", uri, err) - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, false, err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusNotFound { - if res.StatusCode < 200 || res.StatusCode > 299 { - return nil, temporaryHTTPStatusCodes[res.StatusCode], toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod) - } - } - - b, err := httputil.DumpResponse(res, true) - if err != nil { - return nil, false, toHTTPError(err, res, !isHeadMethod) - } - - return b, false, nil - }() + req, err := options.NewRequest(uri) if err != nil { - if retry { - if start.IsZero() { - start = time.Now() - } else if d := time.Since(start) + nextSleep; d >= c.rs.Cfg.Timeout() { - c.rs.Logger.Errorf("Retry timeout (configured to %s) fetching remote resource.", c.rs.Cfg.Timeout()) - return nil, err - } - time.Sleep(nextSleep) - if nextSleep < nextSleepLimit { - nextSleep *= 2 - } - continue - } - return nil, err + return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err) } - return hugio.ToReadCloser(bytes.NewReader(b)), nil + req = req.WithContext(ctx) + return c.httpClient.Do(req) } - }) - if err != nil { - return nil, err - } - defer httpResponse.Close() - res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil) - if err != nil { - return nil, err - } - defer res.Body.Close() - - if res.StatusCode == http.StatusNotFound { - // Not found. This matches how looksup for local resources work. - return nil, nil - } - - var ( - body []byte - mediaType media.Type - ) - // A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored. - // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD - if !isHeadMethod && res.Body != nil { - body, err = io.ReadAll(res.Body) + res, err := getRes() if err != nil { - return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err) + return nil, err } - } + defer res.Body.Close() - filename := path.Base(rURL.Path) - if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { - if _, ok := params["filename"]; ok { - filename = params["filename"] + c.configurePollingIfEnabled(uri, optionsKey, getRes) + + if res.StatusCode == http.StatusNotFound { + // Not found. This matches how lookups for local resources work. + return nil, nil } - } - contentType := res.Header.Get("Content-Type") - - // For HEAD requests we have no body to work with, so we need to use the Content-Type header. - if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) { - var found bool - mediaType, found = c.rs.MediaTypes().GetByType(contentType) - if !found { - // A media type not configured in Hugo, just create one from the content type string. - mediaType, _ = media.FromString(contentType) + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod) } - } - if mediaType.IsZero() { - - var extensionHints []string - - // mime.ExtensionsByType gives a long list of extensions for text/plain, - // just use ".txt". - if strings.HasPrefix(contentType, "text/plain") { - extensionHints = []string{".txt"} - } else { - exts, _ := mime.ExtensionsByType(contentType) - if exts != nil { - extensionHints = exts + var ( + body []byte + mediaType media.Type + ) + // A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD + if !isHeadMethod && res.Body != nil { + body, err = io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err) } } - // Look for a file extension. If it's .txt, look for a more specific. - if extensionHints == nil || extensionHints[0] == ".txt" { - if ext := path.Ext(filename); ext != "" { - extensionHints = []string{ext} + filename := path.Base(rURL.Path) + if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { + if _, ok := params["filename"]; ok { + filename = params["filename"] } } - // Now resolve the media type primarily using the content. - mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body) + contentType := res.Header.Get("Content-Type") - } + // For HEAD requests we have no body to work with, so we need to use the Content-Type header. + if isHeadMethod || c.rs.ExecHelper.Sec().HTTP.MediaTypes.Accept(contentType) { + var found bool + mediaType, found = c.rs.MediaTypes().GetByType(contentType) + if !found { + // A media type not configured in Hugo, just create one from the content type string. + mediaType, _ = media.FromString(contentType) + } + } - if mediaType.IsZero() { - return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri) - } + if mediaType.IsZero() { - resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix - data := responseToData(res, false) + var extensionHints []string - return c.rs.NewResource( - resources.ResourceSourceDescriptor{ - MediaType: mediaType, - Data: data, - GroupIdentity: identity.StringIdentity(resourceID), - LazyPublish: true, - OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { - return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil - }, - TargetPath: resourceID, - }) + // mime.ExtensionsByType gives a long list of extensions for text/plain, + // just use ".txt". + if strings.HasPrefix(contentType, "text/plain") { + extensionHints = []string{".txt"} + } else { + exts, _ := mime.ExtensionsByType(contentType) + if exts != nil { + extensionHints = exts + } + } + + // Look for a file extension. If it's .txt, look for a more specific. + if extensionHints == nil || extensionHints[0] == ".txt" { + if ext := path.Ext(filename); ext != "" { + extensionHints = []string{ext} + } + } + + // Now resolve the media type primarily using the content. + mediaType = media.FromContent(c.rs.MediaTypes(), extensionHints, body) + + } + + if mediaType.IsZero() { + return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri) + } + + userKey = filename[:len(filename)-len(path.Ext(filename))] + "_" + userKey + mediaType.FirstSuffix.FullSuffix + data := responseToData(res, false) + + return c.rs.NewResource( + resources.ResourceSourceDescriptor{ + MediaType: mediaType, + Data: data, + GroupIdentity: identity.StringIdentity(optionsKey), + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil + }, + TargetPath: userKey, + }) + }) } func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error { @@ -277,11 +307,17 @@ func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) e return nil } -func calculateResourceID(uri string, optionsm map[string]any) string { - if key, found := maps.LookupEqualFold(optionsm, "key"); found { - return identity.HashString(key) +func remoteResourceKeys(uri string, optionsm map[string]any) (string, string) { + var userKey string + if key, k, found := maps.LookupEqualFold(optionsm, "key"); found { + userKey = identity.HashString(key) + delete(optionsm, k) } - return identity.HashString(uri, optionsm) + optionsKey := identity.HashString(uri, optionsm) + if userKey == "" { + userKey = optionsKey + } + return userKey, optionsKey } func addDefaultHeaders(req *http.Request) { @@ -350,3 +386,71 @@ func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) { return options, nil } + +var _ http.RoundTripper = (*transport)(nil) + +type transport struct { + Cfg config.AllProvider + Logger loggers.Logger +} + +func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + defer func() { + if resp != nil && resp.StatusCode != http.StatusNotFound && resp.StatusCode != http.StatusNotModified { + t.Logger.Debugf("Fetched remote resource: %s", req.URL.String()) + } + }() + + var ( + start time.Time + nextSleep = time.Duration((rand.Intn(1000) + 100)) * time.Millisecond + nextSleepLimit = time.Duration(5) * time.Second + retry bool + ) + + for { + resp, retry, err = func() (*http.Response, bool, error) { + resp2, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + return resp2, false, err + } + + if resp2.StatusCode != http.StatusNotFound && resp2.StatusCode != http.StatusNotModified { + if resp2.StatusCode < 200 || resp2.StatusCode > 299 { + return resp2, temporaryHTTPStatusCodes[resp2.StatusCode], nil + } + } + return resp2, false, nil + }() + + if retry { + if start.IsZero() { + start = time.Now() + } else if d := time.Since(start) + nextSleep; d >= t.Cfg.Timeout() { + msg := "" + if resp != nil { + msg = resp.Status + } + err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD") + return resp, err + } + time.Sleep(nextSleep) + if nextSleep < nextSleepLimit { + nextSleep *= 2 + } + continue + } + + return + } +} + +// We need to send the redirect responses back to the HTTP client from RoundTrip, +// but we don't want to cache them. +func shouldCache(statusCode int) bool { + switch statusCode { + case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, http.StatusTemporaryRedirect, http.StatusPermanentRedirect: + return false + } + return true +} diff --git a/resources/resource_factories/create/remote_test.go b/resources/resource_factories/create/remote_test.go index 21314ad34..49d0b1541 100644 --- a/resources/resource_factories/create/remote_test.go +++ b/resources/resource_factories/create/remote_test.go @@ -115,15 +115,21 @@ func TestOptionsNewRequest(t *testing.T) { c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"foo"}) } -func TestCalculateResourceID(t *testing.T) { +func TestRemoteResourceKeys(t *testing.T) { t.Parallel() c := qt.New(t) - c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675") - c.Assert(calculateResourceID("foo", map[string]any{"bar": "baz"}), qt.Equals, "7294498335241413323") + check := func(uri string, optionsm map[string]any, expect1, expect2 string) { + got1, got2 := remoteResourceKeys(uri, optionsm) + c.Assert(got1, qt.Equals, expect1) + c.Assert(got2, qt.Equals, expect2) + } - c.Assert(calculateResourceID("foo", map[string]any{"key": "1234", "bar": "baz"}), qt.Equals, "14904296279238663669") - c.Assert(calculateResourceID("asdf", map[string]any{"key": "1234", "bar": "asdf"}), qt.Equals, "14904296279238663669") - c.Assert(calculateResourceID("asdf", map[string]any{"key": "12345", "bar": "asdf"}), qt.Equals, "12191037851845371770") + check("foo", nil, "5917621528921068675", "5917621528921068675") + check("foo", map[string]any{"bar": "baz"}, "7294498335241413323", "7294498335241413323") + check("foo", map[string]any{"key": "1234", "bar": "baz"}, "14904296279238663669", "7294498335241413323") + check("foo", map[string]any{"key": "12345", "bar": "baz"}, "12191037851845371770", "7294498335241413323") + check("asdf", map[string]any{"key": "1234", "bar": "asdf"}, "14904296279238663669", "3787889110563790121") + check("asdf", map[string]any{"key": "12345", "bar": "asdf"}, "12191037851845371770", "3787889110563790121") } diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 644259e48..ef76daa1a 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -29,6 +29,7 @@ import ( "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/identity" @@ -53,6 +54,8 @@ func NewSpec( logger loggers.Logger, errorHandler herrors.ErrorSender, execHelper *hexec.Exec, + buildClosers types.CloseAdder, + rebuilder identity.SignalRebuilder, ) (*Spec, error) { conf := s.Cfg.GetConfig().(*allconfig.Config) imgConfig := conf.Imaging @@ -87,10 +90,12 @@ func NewSpec( } rs := &Spec{ - PathSpec: s, - Logger: logger, - ErrorSender: errorHandler, - imaging: imaging, + PathSpec: s, + Logger: logger, + ErrorSender: errorHandler, + BuildClosers: buildClosers, + Rebuilder: rebuilder, + imaging: imaging, ImageCache: newImageCache( fileCaches.ImageCache(), memCache, @@ -111,8 +116,10 @@ func NewSpec( type Spec struct { *helpers.PathSpec - Logger loggers.Logger - ErrorSender herrors.ErrorSender + Logger loggers.Logger + ErrorSender herrors.ErrorSender + BuildClosers types.CloseAdder + Rebuilder identity.SignalRebuilder TextTemplates tpl.TemplateParseFinder diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 04af756ef..34b4464be 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -369,7 +369,7 @@ func (ns *Namespace) ToCSS(args ...any) (resource.Resource, error) { } if m != nil { - if t, found := maps.LookupEqualFold(m, "transpiler"); found { + if t, _, found := maps.LookupEqualFold(m, "transpiler"); found { switch t { case transpilerDart, transpilerLibSass: transpiler = cast.ToString(t) @@ -440,7 +440,6 @@ func (ns *Namespace) Babel(args ...any) (resource.Resource, error) { var options babel.Options if m != nil { options, err = babel.DecodeOptions(m) - if err != nil { return nil, err }