hugo/tpl/tplimpl/template_ast_transformers.go
Bjørn Erik Pedersen c6d650c8c8
tpl/tplimpl: Rework template management to get rid of concurrency issues
This more or less completes the simplification of the template handling code in Hugo started in v0.62.

The main motivation was to fix a long lasting issue about a crash in HTML content files  without front matter.

But this commit also comes with a big functional improvement.

As we now have moved the base template evaluation to the build stage we now use the same lookup rules for `baseof` as for `list` etc. type of templates.

This means that in this simple example you can have a `baseof` template for the `blog` section without having to duplicate the others:

```
layouts
├── _default
│   ├── baseof.html
│   ├── list.html
│   └── single.html
└── blog
    └── baseof.html
```

Also, when simplifying code, you often get rid of some double work, as shown in the "site building" benchmarks below.

These benchmarks looks suspiciously good, but I have repeated the below with ca. the same result. Compared to master:

```
name                              old time/op    new time/op    delta
SiteNew/Bundle_with_image-16        13.1ms ± 1%    10.5ms ± 1%  -19.34%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16    13.0ms ± 0%    10.7ms ± 1%  -18.05%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16      46.4ms ± 2%    43.1ms ± 1%   -7.15%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16            52.2ms ± 2%    47.8ms ± 1%   -8.30%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16        77.9ms ± 1%    70.9ms ± 1%   -9.01%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16      43.0ms ± 0%    37.2ms ± 1%  -13.54%  (p=0.029 n=4+4)
SiteNew/Page_collections-16         58.2ms ± 1%    52.4ms ± 1%   -9.95%  (p=0.029 n=4+4)

name                              old alloc/op   new alloc/op   delta
SiteNew/Bundle_with_image-16        3.81MB ± 0%    2.22MB ± 0%  -41.70%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16    3.60MB ± 0%    2.01MB ± 0%  -44.20%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16      19.3MB ± 1%    14.1MB ± 0%  -26.91%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16            70.7MB ± 0%    69.0MB ± 0%   -2.40%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16        37.1MB ± 0%    31.2MB ± 0%  -15.94%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16      17.6MB ± 0%    10.6MB ± 0%  -39.92%  (p=0.029 n=4+4)
SiteNew/Page_collections-16         25.9MB ± 0%    21.2MB ± 0%  -17.99%  (p=0.029 n=4+4)

name                              old allocs/op  new allocs/op  delta
SiteNew/Bundle_with_image-16         52.3k ± 0%     26.1k ± 0%  -50.18%  (p=0.029 n=4+4)
SiteNew/Bundle_with_JSON_file-16     52.3k ± 0%     26.1k ± 0%  -50.16%  (p=0.029 n=4+4)
SiteNew/Tags_and_categories-16        336k ± 1%      269k ± 0%  -19.90%  (p=0.029 n=4+4)
SiteNew/Canonify_URLs-16              422k ± 0%      395k ± 0%   -6.43%  (p=0.029 n=4+4)
SiteNew/Deep_content_tree-16          401k ± 0%      313k ± 0%  -21.79%  (p=0.029 n=4+4)
SiteNew/Many_HTML_templates-16        247k ± 0%      143k ± 0%  -42.17%  (p=0.029 n=4+4)
SiteNew/Page_collections-16           282k ± 0%      207k ± 0%  -26.55%  (p=0.029 n=4+4)
```

Fixes #6716
Fixes #6760
Fixes #6768
Fixes #6778
2020-01-22 09:39:49 +01:00

350 lines
8.2 KiB
Go

// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tplimpl
import (
"regexp"
"strings"
htmltemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/tpl"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
)
type templateType int
const (
templateUndefined templateType = iota
templateShortcode
templatePartial
)
type templateContext struct {
visited map[string]bool
templateNotFound map[string]bool
identityNotFound map[string]bool
lookupFn func(name string) *templateState
// The last error encountered.
err error
// Set when we're done checking for config header.
configChecked bool
t *templateState
// Store away the return node in partials.
returnNode *parse.CommandNode
}
func (c templateContext) getIfNotVisited(name string) *templateState {
if c.visited[name] {
return nil
}
c.visited[name] = true
templ := c.lookupFn(name)
if templ == nil {
// This may be a inline template defined outside of this file
// and not yet parsed. Unusual, but it happens.
// Store the name to try again later.
c.templateNotFound[name] = true
}
return templ
}
func newTemplateContext(
t *templateState,
lookupFn func(name string) *templateState) *templateContext {
return &templateContext{
t: t,
lookupFn: lookupFn,
visited: make(map[string]bool),
templateNotFound: make(map[string]bool),
identityNotFound: make(map[string]bool),
}
}
func applyTemplateTransformers(
t *templateState,
lookupFn func(name string) *templateState) (*templateContext, error) {
if t == nil {
return nil, errors.New("expected template, but none provided")
}
c := newTemplateContext(t, lookupFn)
tree := getParseTree(t.Template)
_, err := c.applyTransformations(tree.Root)
if err == nil && c.returnNode != nil {
// This is a partial with a return statement.
c.t.parseInfo.HasReturn = true
tree.Root = c.wrapInPartialReturnWrapper(tree.Root)
}
return c, err
}
func getParseTree(templ tpl.Template) *parse.Tree {
templ = unwrap(templ)
if text, ok := templ.(*texttemplate.Template); ok {
return text.Tree
}
return templ.(*htmltemplate.Template).Tree
}
const (
partialReturnWrapperTempl = `{{ $_hugo_dot := $ }}{{ $ := .Arg }}{{ with .Arg }}{{ $_hugo_dot.Set ("PLACEHOLDER") }}{{ end }}`
)
var (
partialReturnWrapper *parse.ListNode
)
func init() {
templ, err := texttemplate.New("").Parse(partialReturnWrapperTempl)
if err != nil {
panic(err)
}
partialReturnWrapper = templ.Tree.Root
}
func (c *templateContext) wrapInPartialReturnWrapper(n *parse.ListNode) *parse.ListNode {
wrapper := partialReturnWrapper.CopyList()
withNode := wrapper.Nodes[2].(*parse.WithNode)
retn := withNode.List.Nodes[0]
setCmd := retn.(*parse.ActionNode).Pipe.Cmds[0]
setPipe := setCmd.Args[1].(*parse.PipeNode)
// Replace PLACEHOLDER with the real return value.
// Note that this is a PipeNode, so it will be wrapped in parens.
setPipe.Cmds = []*parse.CommandNode{c.returnNode}
withNode.List.Nodes = append(n.Nodes, retn)
return wrapper
}
// applyTransformations do 2 things:
// 1) Parses partial return statement.
// 2) Tracks template (partial) dependencies and some other info.
func (c *templateContext) applyTransformations(n parse.Node) (bool, error) {
switch x := n.(type) {
case *parse.ListNode:
if x != nil {
c.applyTransformationsToNodes(x.Nodes...)
}
case *parse.ActionNode:
c.applyTransformationsToNodes(x.Pipe)
case *parse.IfNode:
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
case *parse.WithNode:
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
case *parse.RangeNode:
c.applyTransformationsToNodes(x.Pipe, x.List, x.ElseList)
case *parse.TemplateNode:
subTempl := c.getIfNotVisited(x.Name)
if subTempl != nil {
c.applyTransformationsToNodes(getParseTree(subTempl.Template).Root)
}
case *parse.PipeNode:
c.collectConfig(x)
for i, cmd := range x.Cmds {
keep, _ := c.applyTransformations(cmd)
if !keep {
x.Cmds = append(x.Cmds[:i], x.Cmds[i+1:]...)
}
}
case *parse.CommandNode:
c.collectPartialInfo(x)
c.collectInner(x)
keep := c.collectReturnNode(x)
for _, elem := range x.Args {
switch an := elem.(type) {
case *parse.PipeNode:
c.applyTransformations(an)
}
}
return keep, c.err
}
return true, c.err
}
func (c *templateContext) applyTransformationsToNodes(nodes ...parse.Node) {
for _, node := range nodes {
c.applyTransformations(node)
}
}
func (c *templateContext) hasIdent(idents []string, ident string) bool {
for _, id := range idents {
if id == ident {
return true
}
}
return false
}
// collectConfig collects and parses any leading template config variable declaration.
// This will be the first PipeNode in the template, and will be a variable declaration
// on the form:
// {{ $_hugo_config:= `{ "version": 1 }` }}
func (c *templateContext) collectConfig(n *parse.PipeNode) {
if c.t.typ != templateShortcode {
return
}
if c.configChecked {
return
}
c.configChecked = true
if len(n.Decl) != 1 || len(n.Cmds) != 1 {
// This cannot be a config declaration
return
}
v := n.Decl[0]
if len(v.Ident) == 0 || v.Ident[0] != "$_hugo_config" {
return
}
cmd := n.Cmds[0]
if len(cmd.Args) == 0 {
return
}
if s, ok := cmd.Args[0].(*parse.StringNode); ok {
errMsg := "failed to decode $_hugo_config in template"
m, err := maps.ToStringMapE(s.Text)
if err != nil {
c.err = errors.Wrap(err, errMsg)
return
}
if err := mapstructure.WeakDecode(m, &c.t.parseInfo.Config); err != nil {
c.err = errors.Wrap(err, errMsg)
}
}
}
// collectInner determines if the given CommandNode represents a
// shortcode call to its .Inner.
func (c *templateContext) collectInner(n *parse.CommandNode) {
if c.t.typ != templateShortcode {
return
}
if c.t.parseInfo.IsInner || len(n.Args) == 0 {
return
}
for _, arg := range n.Args {
var idents []string
switch nt := arg.(type) {
case *parse.FieldNode:
idents = nt.Ident
case *parse.VariableNode:
idents = nt.Ident
}
if c.hasIdent(idents, "Inner") {
c.t.parseInfo.IsInner = true
break
}
}
}
var partialRe = regexp.MustCompile(`^partial(Cached)?$|^partials\.Include(Cached)?$`)
func (c *templateContext) collectPartialInfo(x *parse.CommandNode) {
if len(x.Args) < 2 {
return
}
first := x.Args[0]
var id string
switch v := first.(type) {
case *parse.IdentifierNode:
id = v.Ident
case *parse.ChainNode:
id = v.String()
}
if partialRe.MatchString(id) {
partialName := strings.Trim(x.Args[1].String(), "\"")
if !strings.Contains(partialName, ".") {
partialName += ".html"
}
partialName = "partials/" + partialName
info := c.lookupFn(partialName)
if info != nil {
c.t.Add(info)
} else {
// Delay for later
c.identityNotFound[partialName] = true
}
}
}
func (c *templateContext) collectReturnNode(n *parse.CommandNode) bool {
if c.t.typ != templatePartial || c.returnNode != nil {
return true
}
if len(n.Args) < 2 {
return true
}
ident, ok := n.Args[0].(*parse.IdentifierNode)
if !ok || ident.Ident != "return" {
return true
}
c.returnNode = n
// Remove the "return" identifiers
c.returnNode.Args = c.returnNode.Args[1:]
return false
}
func findTemplateIn(name string, in tpl.Template) (tpl.Template, bool) {
in = unwrap(in)
if text, ok := in.(*texttemplate.Template); ok {
if templ := text.Lookup(name); templ != nil {
return templ, true
}
return nil, false
}
if templ := in.(*htmltemplate.Template).Lookup(name); templ != nil {
return templ, true
}
return nil, false
}