Fix "context canceled" with partial

Make sure the context used for timeouts isn't created based on the incoming
context, as we have cases where this can cancel the context prematurely.

Fixes #10789

cache/filecache/filecache_pruner.go

@@ -66,15 +66,21 @@ func (c *Cache) Prune(force bool) (int, error) {
 
 		if info.IsDir() {
 			f, err := c.Fs.Open(name)
+
 			if err != nil {
 				// This cache dir may not exist.
 				return nil
 			}
-			defer f.Close()
 			_, err = f.Readdirnames(1)
+			f.Close()
 			if err == io.EOF {
 				// Empty dir.
-				err = c.Fs.Remove(name)
+				if name == "." {
+					// e.g. /_gen/images -- keep it even if empty.
+					err = nil
+				} else {
+					err = c.Fs.Remove(name)
+				}
 			}
 
 			if err != nil && !herrors.IsNotExist(err) {

cache/filecache/integration_test.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache_test
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+// See issue #10781. That issue wouldn't have been triggered if we kept
+// the empty root directories (e.g. _resources/gen/images).
+// It's still an upstream Go issue that we also need to handle, but
+// this is a test for the first part.
+func TestPruneShouldPreserveEmptyCacheRoots(t *testing.T) {
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+-- content/_index.md --
+---
+title: "Home"
+---
+
+`
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{T: t, TxtarString: files, RunGC: true, NeedsOsFS: true},
+	).Build()
+
+	_, err := b.H.BaseFs.ResourcesCache.Stat(filepath.Join("_gen", "images"))
+
+	b.Assert(err, qt.IsNil)
+
+}
+
+func TestPruneImages(t *testing.T) {
+	files := `
+-- hugo.toml --
+baseURL = "https://example.com"
+[caches]
+[caches.images]
+maxAge = "200ms"
+dir = ":resourceDir/_gen"
+-- content/_index.md --
+---
+title: "Home"
+---
+-- assets/a/pixel.png --
+iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==
+-- layouts/index.html --
+{{ $img := resources.GetMatch "**.png" }}
+{{ $img = $img.Resize "3x3" }}
+{{ $img.RelPermalink }}
+
+
+
+`
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{T: t, TxtarString: files, RunGC: true, NeedsOsFS: true},
+	).Build()
+
+	b.Assert(b.GCCount, qt.Equals, 0)
+
+	imagesCacheDir := filepath.Join("_gen", "images")
+	_, err := b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir)
+
+	b.Assert(err, qt.IsNil)
+
+	// TODO(bep) we need a way to test full rebuilds.
+	// For now, just sleep a little so the cache elements expires.
+	time.Sleep(300 * time.Millisecond)
+
+	b.RenameFile("assets/a/pixel.png", "assets/b/pixel2.png").Build()
+	b.Assert(b.GCCount, qt.Equals, 1)
+	// Build it again to GC the empty a dir.
+	b.Build()
+	_, err = b.H.BaseFs.ResourcesCache.Stat(filepath.Join(imagesCacheDir, "a"))
+	b.Assert(err, qt.Not(qt.IsNil))
+	_, err = b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir)
+	b.Assert(err, qt.IsNil)
+
+}

deps/deps.go

@@ -1,6 +1,7 @@
 package deps
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -71,7 +72,7 @@ type Deps struct {
 	FileCaches filecache.Caches
 
 	// The translation func to use
-	Translate func(translationID string, templateData any) string `json:"-"`
+	Translate func(ctx context.Context, translationID string, templateData any) string `json:"-"`
 
 	// The language in use. TODO(bep) consolidate with site
 	Language *langs.Language

hugolib/integrationtest_builder.go

@@ -84,6 +84,7 @@ type IntegrationTestBuilder struct {
 	renamedFiles []string
 
 	buildCount int
+	GCCount    int
 	counters   *testCounters
 	logBuff    lockingBuffer
 
@@ -193,6 +194,9 @@ func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder {
 	if s.Cfg.Verbose || err != nil {
 		fmt.Println(s.logBuff.String())
 	}
+	if s.Cfg.RunGC {
+		s.GCCount, err = s.H.GC()
+	}
 	s.Assert(err, qt.IsNil)
 	return s
 }
@@ -263,6 +267,7 @@ func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBui
 	absNewFilename := s.absFilename(new)
 	s.renamedFiles = append(s.renamedFiles, absOldFilename)
 	s.createdFiles = append(s.createdFiles, absNewFilename)
+	s.Assert(s.fs.Source.MkdirAll(filepath.Dir(absNewFilename), 0777), qt.IsNil)
 	s.Assert(s.fs.Source.Rename(absOldFilename, absNewFilename), qt.IsNil)
 	return s
 }
@@ -488,6 +493,9 @@ type IntegrationTestConfig struct {
 	// Whether it needs the real file system (e.g. for js.Build tests).
 	NeedsOsFS bool
 
+	// Whether to run GC after each build.
+	RunGC bool
+
 	// Do not remove the temp dir after the test.
 	PrintAndKeepTempDir bool
 

langs/i18n/i18n.go

@@ -14,6 +14,7 @@
 package i18n
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"strings"
@@ -24,11 +25,12 @@ import (
 	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/resources/page"
 
 	"github.com/gohugoio/go-i18n/v2/i18n"
 )
 
-type translateFunc func(translationID string, templateData any) string
+type translateFunc func(ctx context.Context, translationID string, templateData any) string
 
 var i18nWarningLogger = helpers.NewDistinctErrorLogger()
 
@@ -58,7 +60,7 @@ func (t Translator) Func(lang string) translateFunc {
 	}
 
 	t.logger.Infoln("i18n not initialized; if you need string translations, check that you have a bundle in /i18n that matches the site language or the default language.")
-	return func(translationID string, args any) string {
+	return func(ctx context.Context, translationID string, args any) string {
 		return ""
 	}
 }
@@ -71,7 +73,7 @@ func (t Translator) initFuncs(bndl *i18n.Bundle) {
 		// This may be pt-BR; make it case insensitive.
 		currentLangKey := strings.ToLower(strings.TrimPrefix(currentLangStr, artificialLangTagPrefix))
 		localizer := i18n.NewLocalizer(bndl, currentLangStr)
-		t.translateFuncs[currentLangKey] = func(translationID string, templateData any) string {
+		t.translateFuncs[currentLangKey] = func(ctx context.Context, translationID string, templateData any) string {
 			pluralCount := getPluralCount(templateData)
 
 			if templateData != nil {
@@ -81,6 +83,16 @@ func (t Translator) initFuncs(bndl *i18n.Bundle) {
 					// and we keep it like this to avoid breaking
 					// lots of sites in the wild.
 					templateData = intCount(cast.ToInt(templateData))
+				} else {
+					if p, ok := templateData.(page.Page); ok {
+						// See issue 10782.
+						// The i18n has its own template handling and does not know about
+						// the context.Context.
+						// A common pattern is to pass Page to i18n, and use .ReadingTime etc.
+						// We need to improve this, but that requires some upstream changes.
+						// For now, just creata a wrepper.
+						templateData = page.PageWithContext{Page: p, Ctx: ctx}
+					}
 				}
 			}
 

langs/i18n/i18n_test.go

@@ -14,6 +14,7 @@
 package i18n
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"testing"
@@ -408,9 +409,10 @@ other = "{{ . }} miesiąca"
 			c.Assert(d.LoadResources(), qt.IsNil)
 
 			f := tp.t.Func(test.lang)
+			ctx := context.Background()
 
 			for _, variant := range test.variants {
-				c.Assert(f(test.id, variant.Key), qt.Equals, variant.Value, qt.Commentf("input: %v", variant.Key))
+				c.Assert(f(ctx, test.id, variant.Key), qt.Equals, variant.Value, qt.Commentf("input: %v", variant.Key))
 				c.Assert(int(depsCfg.Logger.LogCounters().WarnCounter.Count()), qt.Equals, 0)
 			}
 
@@ -422,7 +424,7 @@ other = "{{ . }} miesiąca"
 func doTestI18nTranslate(t testing.TB, test i18nTest, cfg config.Provider) string {
 	tp := prepareTranslationProvider(t, test, cfg)
 	f := tp.t.Func(test.lang)
-	return f(test.id, test.args)
+	return f(context.Background(), test.id, test.args)
 }
 
 type countField struct {
@@ -542,7 +544,7 @@ func BenchmarkI18nTranslate(b *testing.B) {
 			b.ResetTimer()
 			for i := 0; i < b.N; i++ {
 				f := tp.t.Func(test.lang)
-				actual := f(test.id, test.args)
+				actual := f(context.Background(), test.id, test.args)
 				if actual != test.expected {
 					b.Fatalf("expected %v got %v", test.expected, actual)
 				}

langs/i18n/integration_test.go

@@ -55,3 +55,51 @@ l1: {{ i18n "l1"  }}|l2: {{ i18n "l2"  }}|l3: {{ i18n "l3"  }}
 l1: l1main|l2: l2main|l3: l3theme
 	`)
 }
+
+func TestPassPageToI18n(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- config.toml --
+-- content/_index.md --
+---
+title: "Home"
+---
+Duis quis irure id nisi sunt minim aliqua occaecat. Aliqua cillum labore consectetur quis culpa tempor quis non officia cupidatat in ad cillum. Velit irure pariatur nisi adipisicing officia reprehenderit commodo esse non.
+
+Ullamco cupidatat nostrud ut reprehenderit. Consequat nisi culpa magna amet tempor velit reprehenderit. Ad minim eiusmod tempor nostrud eu aliquip consectetur commodo ut in aliqua enim. Cupidatat voluptate laborum consequat qui nulla laborum laborum aute ea culpa nulla dolor cillum veniam. Commodo esse tempor qui labore aute aliqua sint nulla do.
+
+Ad deserunt esse nostrud labore. Amet reprehenderit fugiat nostrud eu reprehenderit sit reprehenderit minim deserunt esse id occaecat cillum. Ad qui Lorem cillum laboris ipsum anim in culpa ad dolor consectetur minim culpa.
+
+Lorem cupidatat officia aute in eu commodo anim nulla deserunt occaecat reprehenderit dolore. Eu cupidatat reprehenderit ipsum sit laboris proident. Duis quis nulla tempor adipisicing. Adipisicing amet ad reprehenderit non mollit. Cupidatat proident tempor laborum sit ipsum adipisicing sunt magna labore. Eu irure nostrud cillum exercitation tempor proident. Laborum magna nisi consequat do sint occaecat magna incididunt.
+
+Sit mollit amet esse dolore in labore aliquip eu duis officia incididunt. Esse veniam labore excepteur eiusmod occaecat ullamco magna sunt. Ipsum occaecat exercitation anim fugiat in amet excepteur excepteur aliquip laborum. Aliquip aliqua consequat officia sit sint amet aliqua ipsum eu veniam. Id enim quis ea in eu consequat exercitation occaecat veniam consequat anim nulla adipisicing minim. Ut duis cillum laboris duis non commodo eu aliquip tempor nisi aute do.
+
+Ipsum nulla esse excepteur ut aliqua esse incididunt deserunt veniam dolore est laborum nisi veniam. Magna eiusmod Lorem do tempor incididunt ut aute aliquip ipsum ea laboris culpa. Occaecat do officia velit fugiat culpa eu minim magna sint occaecat sunt. Duis magna proident incididunt est cupidatat proident esse proident ut ipsum non dolor Lorem eiusmod. Officia quis irure id eu aliquip.
+
+Duis anim elit in officia in in aliquip est. Aliquip nisi labore qui elit elit cupidatat ut labore incididunt eiusmod ipsum. Sit irure nulla non cupidatat exercitation sit culpa nisi ex dolore. Culpa nisi duis duis eiusmod commodo nulla.
+
+Et magna aliqua amet qui mollit. Eiusmod aute ut anim ea est fugiat non nisi in laborum ullamco. Proident mollit sunt nostrud irure esse sunt eiusmod deserunt dolor. Irure aute ad magna est consequat duis cupidatat consequat. Enim tempor aute cillum quis ea do enim proident incididunt aliquip cillum tempor minim. Nulla minim tempor proident in excepteur consectetur veniam.
+
+Exercitation tempor nulla incididunt deserunt laboris ad incididunt aliqua exercitation. Adipisicing laboris veniam aute eiusmod qui magna fugiat velit. Aute quis officia anim commodo id fugiat nostrud est. Quis ipsum amet velit adipisicing eu anim minim eu est in culpa aute. Esse in commodo irure enim proident reprehenderit ullamco in dolore aute cillum.
+
+Irure excepteur ex occaecat ipsum laboris fugiat exercitation. Exercitation adipisicing velit excepteur eu culpa consequat exercitation dolore. In laboris aute quis qui mollit minim culpa. Magna velit ea aliquip veniam fugiat mollit veniam.
+-- i18n/en.toml --
+[a]
+other = 'Reading time: {{ .ReadingTime }}'
+-- layouts/index.html --
+i18n: {{ i18n "a" . }}|
+
+`
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/index.html", `
+	i18n: Reading time: 3|
+	`)
+}

lazy/init.go

@@ -180,14 +180,15 @@ func (ini *Init) checkDone() {
 }
 
 func (ini *Init) withTimeout(ctx context.Context, timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) {
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+	// Create a new context with a timeout not connected to the incoming context.
+	waitCtx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 	c := make(chan verr, 1)
 
 	go func() {
 		v, err := f(ctx)
 		select {
-		case <-ctx.Done():
+		case <-waitCtx.Done():
 			return
 		default:
 			c <- verr{v: v, err: err}
@@ -195,7 +196,7 @@ func (ini *Init) withTimeout(ctx context.Context, timeout time.Duration, f func(
 	}()
 
 	select {
-	case <-ctx.Done():
+	case <-waitCtx.Done():
 		return nil, errors.New("timed out initializing value. You may have a circular loop in a shortcode, or your site may have resources that take longer to build than the `timeout` limit in your Hugo config file.")
 	case ve := <-c:
 		return ve.v, ve.err

lazy/init_test.go

@@ -126,12 +126,6 @@ func TestInitAddWithTimeoutTimeout(t *testing.T) {
 
 	init := New().AddWithTimeout(100*time.Millisecond, func(ctx context.Context) (any, error) {
 		time.Sleep(500 * time.Millisecond)
-		select {
-		case <-ctx.Done():
-			return nil, nil
-		default:
-		}
-		t.Fatal("slept")
 		return nil, nil
 	})
 

resources/page/page.go

@@ -471,3 +471,45 @@ type DeprecatedWarningPageMethods any // This was emptied in Hugo 0.93.0.
 // Move here to trigger ERROR instead of WARNING.
 // TODO(bep) create wrappers and put into the Page once it has some methods.
 type DeprecatedErrorPageMethods any
+
+// PageWithContext is a Page with a context.Context.
+type PageWithContext struct {
+	Page
+	Ctx context.Context
+}
+
+func (p PageWithContext) Content() (any, error) {
+	return p.Page.Content(p.Ctx)
+}
+
+func (p PageWithContext) Plain() string {
+	return p.Page.Plain(p.Ctx)
+}
+
+func (p PageWithContext) PlainWords() []string {
+	return p.Page.PlainWords(p.Ctx)
+}
+
+func (p PageWithContext) Summary() template.HTML {
+	return p.Page.Summary(p.Ctx)
+}
+
+func (p PageWithContext) Truncated() bool {
+	return p.Page.Truncated(p.Ctx)
+}
+
+func (p PageWithContext) FuzzyWordCount() int {
+	return p.Page.FuzzyWordCount(p.Ctx)
+}
+
+func (p PageWithContext) WordCount() int {
+	return p.Page.WordCount(p.Ctx)
+}
+
+func (p PageWithContext) ReadingTime() int {
+	return p.Page.ReadingTime(p.Ctx)
+}
+
+func (p PageWithContext) Len() int {
+	return p.Page.Len(p.Ctx)
+}

resources/transform.go

@@ -164,12 +164,12 @@ type resourceAdapter struct {
 	*resourceAdapterInner
 }
 
-func (r *resourceAdapter) Content(context.Context) (any, error) {
+func (r *resourceAdapter) Content(ctx context.Context) (any, error) {
 	r.init(false, true)
 	if r.transformationsErr != nil {
 		return nil, r.transformationsErr
 	}
-	return r.target.Content(context.Background())
+	return r.target.Content(ctx)
 }
 
 func (r *resourceAdapter) Err() resource.ResourceError {

tpl/lang/lang.go

@@ -15,6 +15,7 @@
 package lang
 
 import (
+	"context"
 	"fmt"
 	"math"
 	"strconv"
@@ -45,7 +46,7 @@ type Namespace struct {
 }
 
 // Translate returns a translated string for id.
-func (ns *Namespace) Translate(id any, args ...any) (string, error) {
+func (ns *Namespace) Translate(ctx context.Context, id any, args ...any) (string, error) {
 	var templateData any
 
 	if len(args) > 0 {
@@ -60,7 +61,7 @@ func (ns *Namespace) Translate(id any, args ...any) (string, error) {
 		return "", nil
 	}
 
-	return ns.deps.Translate(sid, templateData), nil
+	return ns.deps.Translate(ctx, sid, templateData), nil
 }
 
 // FormatNumber formats number with the given precision for the current language.

tpl/partials/integration_test.go

@@ -324,3 +324,31 @@ timeout = '200ms'
 	b.Assert(err.Error(), qt.Contains, "timed out")
 
 }
+
+// See Issue #10789
+func TestReturnExecuteFromTemplateInPartial(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- config.toml --
+baseURL = 'http://example.com/'
+-- layouts/index.html --
+{{ $r :=  partial "foo" }}
+FOO:{{ $r.Content }}
+-- layouts/partials/foo.html --
+{{ $r := §§{{ partial "bar" }}§§ | resources.FromString "bar.html" | resources.ExecuteAsTemplate "bar.html" . }}
+{{ return $r }}
+-- layouts/partials/bar.html --
+BAR
+  `
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/index.html", "OO:BAR")
+
+}

tpl/partials/partials.go

@@ -129,7 +129,10 @@ func (ns *Namespace) Include(ctx context.Context, name string, contextList ...an
 }
 
 func (ns *Namespace) includWithTimeout(ctx context.Context, name string, dataList ...any) includeResult {
-	ctx, cancel := context.WithTimeout(ctx, ns.deps.Timeout)
+	// There are situation where the ctx we pass on to the partial lives longer than
+	// the partial itself. For example, when the partial returns the result from reosurces.ExecuteAsTemplate.
+	// Because of that, create a completely new context here.
+	timeoutCtx, cancel := context.WithTimeout(context.Background(), ns.deps.Timeout)
 	defer cancel()
 
 	res := make(chan includeResult, 1)
@@ -141,8 +144,8 @@ func (ns *Namespace) includWithTimeout(ctx context.Context, name string, dataLis
 	select {
 	case r := <-res:
 		return r
-	case <-ctx.Done():
-		err := ctx.Err()
+	case <-timeoutCtx.Done():
+		err := timeoutCtx.Err()
 		if err == context.DeadlineExceeded {
 			err = fmt.Errorf("partial %q timed out after %s. This is most likely due to infinite recursion. If this is just a slow template, you can try to increase the 'timeout' config setting.", name, ns.deps.Timeout)
 		}