output: Rework the base template logic

Extract the logic to a testable function and add support for custom output types.

Fixes #2995

helpers/pathspec.go

@@ -94,3 +94,18 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) *PathSpec {
 func (p *PathSpec) PaginatePath() string {
 	return p.paginatePath
 }
+
+// WorkingDir returns the configured workingDir.
+func (p *PathSpec) WorkingDir() string {
+	return p.workingDir
+}
+
+// LayoutDir returns the relative layout dir in the currenct Hugo project.
+func (p *PathSpec) LayoutDir() string {
+	return p.layoutDir
+}
+
+// Theme returns the theme name if set.
+func (p *PathSpec) Theme() string {
+	return p.theme
+}

hugolib/page.go

@@ -923,10 +923,11 @@ func (p *Page) update(f interface{}) error {
 				p.s.Log.ERROR.Printf("Failed to parse lastmod '%v' in page %s", v, p.File.Path())
 			}
 		case "outputs":
-			outputs := cast.ToStringSlice(v)
-			if len(outputs) > 0 {
+			o := cast.ToStringSlice(v)
+			if len(o) > 0 {
 				// Output formats are exlicitly set in front matter, use those.
-				outFormats, err := output.GetTypes(outputs...)
+				outFormats, err := output.GetFormats(o...)
+
 				if err != nil {
 					p.s.Log.ERROR.Printf("Failed to resolve output formats: %s", err)
 				} else {

hugolib/site_render.go

@@ -63,7 +63,9 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
 	var mainPageOutput *PageOutput
 
 	for page := range pages {
+
 		for i, outFormat := range page.outputFormats {
+
 			pageOutput, err := newPageOutput(page, i > 0, outFormat)
 
 			if err != nil {

output/layout_base.go

@@ -0,0 +1,175 @@
+// Copyright 2017-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package output
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/hugo/helpers"
+)
+
+const baseFileBase = "baseof"
+
+var (
+	aceTemplateInnerMarkers = [][]byte{[]byte("= content")}
+	goTemplateInnerMarkers  = [][]byte{[]byte("{{define"), []byte("{{ define")}
+)
+
+type TemplateNames struct {
+	Name            string
+	OverlayFilename string
+	MasterFilename  string
+}
+
+// TODO(bep) output this is refactoring in progress.
+type TemplateLookupDescriptor struct {
+	// The full path to the site or theme root.
+	WorkingDir string
+
+	// Main project layout dir, defaults to "layouts"
+	LayoutDir string
+
+	// The path to the template relative the the base.
+	//  I.e. shortcodes/youtube.html
+	RelPath string
+
+	// The template name prefix to look for, i.e. "theme".
+	Prefix string
+
+	// The theme name if active.
+	Theme string
+
+	FileExists  func(filename string) (bool, error)
+	ContainsAny func(filename string, subslices [][]byte) (bool, error)
+}
+
+func CreateTemplateID(d TemplateLookupDescriptor) (TemplateNames, error) {
+
+	var id TemplateNames
+
+	name := filepath.FromSlash(d.RelPath)
+
+	if d.Prefix != "" {
+		name = strings.Trim(d.Prefix, "/") + "/" + name
+	}
+
+	baseLayoutDir := filepath.Join(d.WorkingDir, d.LayoutDir)
+	fullPath := filepath.Join(baseLayoutDir, d.RelPath)
+
+	// The filename will have a suffix with an optional type indicator.
+	// Examples:
+	// index.html
+	// index.amp.html
+	// index.json
+	filename := filepath.Base(d.RelPath)
+
+	var ext, outFormat string
+
+	parts := strings.Split(filename, ".")
+	if len(parts) > 2 {
+		outFormat = parts[1]
+		ext = parts[2]
+	} else if len(parts) > 1 {
+		ext = parts[1]
+	}
+
+	filenameNoSuffix := parts[0]
+
+	id.OverlayFilename = fullPath
+	id.Name = name
+
+	// Ace and Go templates may have both a base and inner template.
+	pathDir := filepath.Dir(fullPath)
+
+	if ext == "amber" || strings.HasSuffix(pathDir, "partials") || strings.HasSuffix(pathDir, "shortcodes") {
+		// No base template support
+		return id, nil
+	}
+
+	innerMarkers := goTemplateInnerMarkers
+
+	var baseFilename string
+
+	if outFormat != "" {
+		baseFilename = fmt.Sprintf("%s.%s.%s", baseFileBase, outFormat, ext)
+	} else {
+		baseFilename = fmt.Sprintf("%s.%s", baseFileBase, ext)
+	}
+
+	if ext == "ace" {
+		innerMarkers = aceTemplateInnerMarkers
+	}
+
+	// This may be a view that shouldn't have base template
+	// Have to look inside it to make sure
+	needsBase, err := d.ContainsAny(fullPath, innerMarkers)
+	if err != nil {
+		return id, err
+	}
+
+	if needsBase {
+		currBaseFilename := fmt.Sprintf("%s-%s", filenameNoSuffix, baseFilename)
+
+		templateDir := filepath.Dir(fullPath)
+		themeDir := filepath.Join(d.WorkingDir, d.Theme)
+
+		baseTemplatedDir := strings.TrimPrefix(templateDir, baseLayoutDir)
+		baseTemplatedDir = strings.TrimPrefix(baseTemplatedDir, helpers.FilePathSeparator)
+
+		// Look for base template in the follwing order:
+		//   1. <current-path>/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>.
+		//   2. <current-path>/baseof.<outputFormat>(optional).<suffix>
+		//   3. _default/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>.
+		//   4. _default/baseof.<outputFormat>(optional).<suffix>
+		// For each of the steps above, it will first look in the project, then, if theme is set,
+		// in the theme's layouts folder.
+		// Also note that the <current-path> may be both the project's layout folder and the theme's.
+		pairsToCheck := [][]string{
+			[]string{baseTemplatedDir, currBaseFilename},
+			[]string{baseTemplatedDir, baseFilename},
+			[]string{"_default", currBaseFilename},
+			[]string{"_default", baseFilename},
+		}
+
+	Loop:
+		for _, pair := range pairsToCheck {
+			pathsToCheck := basePathsToCheck(pair, baseLayoutDir, themeDir)
+
+			for _, pathToCheck := range pathsToCheck {
+				if ok, err := d.FileExists(pathToCheck); err == nil && ok {
+					id.MasterFilename = pathToCheck
+					break Loop
+				}
+			}
+		}
+	}
+
+	return id, nil
+
+}
+
+func basePathsToCheck(path []string, layoutDir, themeDir string) []string {
+	// Always look in the project.
+	pathsToCheck := []string{filepath.Join((append([]string{layoutDir}, path...))...)}
+
+	// May have a theme
+	if themeDir != "" {
+		pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{themeDir, "layouts"}, path...))...))
+	}
+
+	return pathsToCheck
+
+}

output/layout_base_test.go

@@ -0,0 +1,159 @@
+// Copyright 2017-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package output
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLayoutBase(t *testing.T) {
+
+	var (
+		workingDir     = "/sites/mysite/"
+		layoutBase1    = "layouts"
+		layoutPath1    = "_default/single.html"
+		layoutPathAmp  = "_default/single.amp.html"
+		layoutPathJSON = "_default/single.json"
+	)
+
+	for _, this := range []struct {
+		name                 string
+		d                    TemplateLookupDescriptor
+		needsBase            bool
+		basePathMatchStrings string
+		expect               TemplateNames
+	}{
+		{"No base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, false, "",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+			}},
+		{"Base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, true, "",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.html",
+			}},
+		{"Base in theme", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, Theme: "mytheme"}, true,
+			"mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/mytheme/layouts/_default/baseof.html",
+			}},
+		{"Template in theme, base in theme", TemplateLookupDescriptor{WorkingDir: filepath.Join(workingDir, "mytheme"), LayoutDir: layoutBase1, RelPath: layoutPath1, Theme: "mytheme"}, true,
+			"mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/mytheme/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/mytheme/layouts/_default/baseof.html",
+			}},
+		{"Template in theme, base in site", TemplateLookupDescriptor{WorkingDir: filepath.Join(workingDir, "mytheme"), LayoutDir: layoutBase1, RelPath: layoutPath1, Theme: "mytheme"}, true,
+			"mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/mytheme/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/mytheme/layouts/_default/baseof.html",
+			}},
+		{"Template in site, base in theme", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, Theme: "mytheme"}, true,
+			"/sites/mysite/mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "_default/single.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/mytheme/layouts/_default/baseof.html",
+			}},
+		{"With prefix, base in theme", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1,
+			Theme: "mytheme", Prefix: "someprefix"}, true,
+			"mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "someprefix/_default/single.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				MasterFilename:  "/sites/mysite/mytheme/layouts/_default/baseof.html",
+			}},
+		{"Partial", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "partials/menu.html"}, true,
+			"mytheme/layouts/_default/baseof.html",
+			TemplateNames{
+				Name:            "partials/menu.html",
+				OverlayFilename: "/sites/mysite/layouts/partials/menu.html",
+			}},
+		{"AMP, no base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, false, "",
+			TemplateNames{
+				Name:            "_default/single.amp.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+			}},
+		{"JSON, no base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, false, "",
+			TemplateNames{
+				Name:            "_default/single.json",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.json",
+			}},
+		{"AMP with base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html",
+			TemplateNames{
+				Name:            "_default/single.amp.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.amp.html",
+			}},
+		{"AMP with no match in base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html",
+			TemplateNames{
+				Name:            "_default/single.amp.html",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+				// There is a single-baseof.html, but that makes no sense.
+				MasterFilename: "",
+			}},
+
+		{"JSON with base", TemplateLookupDescriptor{WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, true, "single-baseof.json",
+			TemplateNames{
+				Name:            "_default/single.json",
+				OverlayFilename: "/sites/mysite/layouts/_default/single.json",
+				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.json",
+			}},
+	} {
+		t.Run(this.name, func(t *testing.T) {
+
+			fileExists := func(filename string) (bool, error) {
+				stringsToMatch := strings.Split(this.basePathMatchStrings, "|")
+				for _, s := range stringsToMatch {
+					if strings.Contains(filename, s) {
+						return true, nil
+					}
+
+				}
+				return false, nil
+			}
+
+			needsBase := func(filename string, subslices [][]byte) (bool, error) {
+				return this.needsBase, nil
+			}
+
+			this.d.WorkingDir = filepath.FromSlash(this.d.WorkingDir)
+			this.d.LayoutDir = filepath.FromSlash(this.d.LayoutDir)
+			this.d.RelPath = filepath.FromSlash(this.d.RelPath)
+			this.d.ContainsAny = needsBase
+			this.d.FileExists = fileExists
+
+			this.expect.MasterFilename = filepath.FromSlash(this.expect.MasterFilename)
+			this.expect.OverlayFilename = filepath.FromSlash(this.expect.OverlayFilename)
+
+			id, err := CreateTemplateID(this.d)
+
+			require.NoError(t, err)
+			require.Equal(t, this.expect, id, this.name)
+
+		})
+	}
+
+}

output/outputFormat.go

@@ -92,7 +92,7 @@ type Format struct {
 	NoUgly bool
 }
 
-func GetType(key string) (Format, bool) {
+func GetFormat(key string) (Format, bool) {
 	found, ok := builtInTypes[key]
 	if !ok {
 		found, ok = builtInTypes[strings.ToLower(key)]
@@ -101,11 +101,11 @@ func GetType(key string) (Format, bool) {
 }
 
 // TODO(bep) outputs rewamp on global config?
-func GetTypes(keys ...string) (Formats, error) {
+func GetFormats(keys ...string) (Formats, error) {
 	var types []Format
 
 	for _, key := range keys {
-		tpe, ok := GetType(key)
+		tpe, ok := GetFormat(key)
 		if !ok {
 			return types, fmt.Errorf("OutputFormat with key %q not found", key)
 		}

output/outputFormat_test.go

@@ -34,10 +34,10 @@ func TestDefaultTypes(t *testing.T) {
 }
 
 func TestGetType(t *testing.T) {
-	tp, _ := GetType("html")
+	tp, _ := GetFormat("html")
 	require.Equal(t, HTMLType, tp)
-	tp, _ = GetType("HTML")
+	tp, _ = GetFormat("HTML")
 	require.Equal(t, HTMLType, tp)
-	_, found := GetType("FOO")
+	_, found := GetFormat("FOO")
 	require.False(t, found)
 }

tpl/tplimpl/template.go

@@ -14,7 +14,6 @@
 package tplimpl
 
 import (
-	"fmt"
 	"html/template"
 	"io"
 	"os"
@@ -28,6 +27,7 @@ import (
 	bp "github.com/spf13/hugo/bufferpool"
 	"github.com/spf13/hugo/deps"
 	"github.com/spf13/hugo/helpers"
+	"github.com/spf13/hugo/output"
 	"github.com/yosssi/ace"
 )
 
@@ -478,80 +478,44 @@ func (t *GoHTMLTemplate) loadTemplates(absPath string, prefix string) {
 				return nil
 			}
 
-			tplName := t.GenerateTemplateNameFrom(absPath, path)
+			workingDir := t.PathSpec.WorkingDir()
+			themeDir := t.PathSpec.GetThemeDir()
 
-			if prefix != "" {
-				tplName = strings.Trim(prefix, "/") + "/" + tplName
+			if themeDir != "" && strings.HasPrefix(absPath, themeDir) {
+				workingDir = themeDir
 			}
 
-			var baseTemplatePath string
+			li := strings.LastIndex(path, t.PathSpec.LayoutDir()) + len(t.PathSpec.LayoutDir()) + 1
 
-			// Ace and Go templates may have both a base and inner template.
-			pathDir := filepath.Dir(path)
-			if filepath.Ext(path) != ".amber" && !strings.HasSuffix(pathDir, "partials") && !strings.HasSuffix(pathDir, "shortcodes") {
-
-				innerMarkers := goTemplateInnerMarkers
-				baseFileName := fmt.Sprintf("%s.html", baseFileBase)
-
-				if filepath.Ext(path) == ".ace" {
-					innerMarkers = aceTemplateInnerMarkers
-					baseFileName = fmt.Sprintf("%s.ace", baseFileBase)
-				}
-
-				// This may be a view that shouldn't have base template
-				// Have to look inside it to make sure
-				needsBase, err := helpers.FileContainsAny(path, innerMarkers, t.Fs.Source)
-				if err != nil {
-					return err
-				}
-				if needsBase {
-
-					layoutDir := t.PathSpec.GetLayoutDirPath()
-					currBaseFilename := fmt.Sprintf("%s-%s", helpers.Filename(path), baseFileName)
-					templateDir := filepath.Dir(path)
-					themeDir := filepath.Join(t.PathSpec.GetThemeDir())
-					relativeThemeLayoutsDir := filepath.Join(t.PathSpec.GetRelativeThemeDir(), "layouts")
-
-					var baseTemplatedDir string
-
-					if strings.HasPrefix(templateDir, relativeThemeLayoutsDir) {
-						baseTemplatedDir = strings.TrimPrefix(templateDir, relativeThemeLayoutsDir)
-					} else {
-						baseTemplatedDir = strings.TrimPrefix(templateDir, layoutDir)
-					}
-
-					baseTemplatedDir = strings.TrimPrefix(baseTemplatedDir, helpers.FilePathSeparator)
-
-					// Look for base template in the follwing order:
-					//   1. <current-path>/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
-					//   2. <current-path>/baseof.<suffix>
-					//   3. _default/<template-name>-baseof.<suffix>, e.g. list-baseof.<suffix>.
-					//   4. _default/baseof.<suffix>
-					// For each of the steps above, it will first look in the project, then, if theme is set,
-					// in the theme's layouts folder.
-
-					pairsToCheck := [][]string{
-						[]string{baseTemplatedDir, currBaseFilename},
-						[]string{baseTemplatedDir, baseFileName},
-						[]string{"_default", currBaseFilename},
-						[]string{"_default", baseFileName},
-					}
-
-				Loop:
-					for _, pair := range pairsToCheck {
-						pathsToCheck := basePathsToCheck(pair, layoutDir, themeDir)
-						for _, pathToCheck := range pathsToCheck {
-							if ok, err := helpers.Exists(pathToCheck, t.Fs.Source); err == nil && ok {
-								baseTemplatePath = pathToCheck
-								break Loop
-							}
-						}
-					}
-				}
+			if li < 0 {
+				// Possibly a theme
+				li = strings.LastIndex(path, "layouts") + 8
 			}
 
-			if err := t.AddTemplateFile(tplName, baseTemplatePath, path); err != nil {
-				t.Log.ERROR.Printf("Failed to add template %s in path %s: %s", tplName, path, err)
+			relPath := path[li:]
+
+			descriptor := output.TemplateLookupDescriptor{
+				WorkingDir: workingDir,
+				LayoutDir:  t.PathSpec.LayoutDir(),
+				RelPath:    relPath,
+				Prefix:     prefix,
+				Theme:      t.PathSpec.Theme(),
+				FileExists: func(filename string) (bool, error) {
+					return helpers.Exists(filename, t.Fs.Source)
+				},
+				ContainsAny: func(filename string, subslices [][]byte) (bool, error) {
+					return helpers.FileContainsAny(filename, subslices, t.Fs.Source)
+				},
+			}
+
+			tplID, err := output.CreateTemplateID(descriptor)
+			if err != nil {
+				t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err)
+				return nil
+			}
+
+			if err := t.AddTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil {
+				t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err)
 			}
 
 		}
@@ -562,19 +526,6 @@ func (t *GoHTMLTemplate) loadTemplates(absPath string, prefix string) {
 	}
 }
 
-func basePathsToCheck(path []string, layoutDir, themeDir string) []string {
-	// Always look in the project.
-	pathsToCheck := []string{filepath.Join((append([]string{layoutDir}, path...))...)}
-
-	// May have a theme
-	if themeDir != "" {
-		pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{themeDir, "layouts"}, path...))...))
-	}
-
-	return pathsToCheck
-
-}
-
 func (t *GoHTMLTemplate) LoadTemplatesWithPrefix(absPath string, prefix string) {
 	t.loadTemplates(absPath, prefix)
 }