From 55fcd2f30f4358a800932aa60160935c0c22f96d Mon Sep 17 00:00:00 2001 From: bep Date: Mon, 27 Oct 2014 21:48:30 +0100 Subject: [PATCH] Shortcode rewrite, take 2 This commit contains a restructuring and partial rewrite of the shortcode handling. Prior to this commit rendering of the page content was mingled with handling of the shortcodes. This led to several oddities. The new flow is: 1. Shortcodes are extracted from page and replaced with placeholders. 2. Shortcodes are processed and rendered 3. Page is processed 4. The placeholders are replaced with the rendered shortcodes The handling of summaries is also made simpler by this. This commit also introduces some other chenges: 1. distinction between shortcodes that need further processing and those who do not: * `{{< >}}`: Typically raw HTML. Will not be processed. * `{{% %}}`: Will be processed by the page's markup engine (Markdown or (infuture) Asciidoctor) The above also involves a new shortcode-parser, with lexical scanning inspired by Rob Pike's talk called "Lexical Scanning in Go", which should be easier to understand, give better error messages and perform better. 2. If you want to exclude a shortcode from being processed (for documentation etc.), the inner part of the shorcode must be commented out, i.e. `{{%/* movie 47238zzb */%}}`. See the updated shortcode section in the documentation for further examples. The new parser supports nested shortcodes. This isn't new, but has two related design choices worth mentioning: * The shortcodes will be rendered individually, so If both `{{< >}}` and `{{% %}}` are used in the nested hierarchy, one will be passed through the page's markdown processor, the other not. * To avoid potential costly overhead of always looking far ahead for a possible closing tag, this implementation looks at the template itself, and is branded as a container with inner content if it contains a reference to `.Inner` Fixes #565 Fixes #480 Fixes #461 And probably some others. --- docs/content/extras/shortcodes.md | 21 +- helpers/pygments.go | 13 +- hugolib/page.go | 82 ++-- hugolib/page_test.go | 10 +- hugolib/shortcode.go | 534 ++++++++++++++++---------- hugolib/shortcode_test.go | 271 +++++++++++++- hugolib/shortcodeparser.go | 598 ++++++++++++++++++++++++++++++ hugolib/shortcodeparser_test.go | 162 ++++++++ hugolib/template.go | 6 - 9 files changed, 1441 insertions(+), 256 deletions(-) create mode 100644 hugolib/shortcodeparser.go create mode 100644 hugolib/shortcodeparser_test.go diff --git a/docs/content/extras/shortcodes.md b/docs/content/extras/shortcodes.md index 5464b1a57..c0b6e0b04 100644 --- a/docs/content/extras/shortcodes.md +++ b/docs/content/extras/shortcodes.md @@ -29,8 +29,8 @@ want a [partial template](/templates/partial) instead. ## Using a shortcode -In your content files, a shortcode can be called by using '`{{% name parameters -%}}`' respectively. Shortcodes are space delimited (parameters with spaces +In your content files, a shortcode can be called by using '`{{%/* name parameters +*/%}}`' respectively. Shortcodes are space delimited (parameters with spaces can be quoted). The first word is always the name of the shortcode. Parameters follow the name. @@ -43,7 +43,7 @@ shortcodes match (name only), the closing being prepended with a slash. Example of a paired shortcode: - {{ % highlight go %}} A bunch of code here {{ % /highlight %}} + {{%/* highlight go */%}} A bunch of code here {{%/* /highlight */%}} ## Hugo Shortcodes @@ -60,9 +60,8 @@ HTML. Read more on [highlighting](/extras/highlighting). closing shortcode. #### Example -The example has an extra space between the “`{{`” and “`%`” characters to prevent rendering here. - {{ % highlight html %}} + {{%/* highlight html */%}}

{{ .Title }}

@@ -71,7 +70,7 @@ The example has an extra space between the “`{{`” and “`%`” characters t {{ end }}
- {{ % /highlight %}} + {{%/* /highlight */%}} #### Example Output @@ -104,7 +103,7 @@ The example has an extra space between the “`{{`” and “`%`” characters t #### Example *Example has an extra space so Hugo doesn’t actually render it*. - {{ % figure src="/media/spf13.jpg" title="Steve Francia" %}} + {{%/* figure src="/media/spf13.jpg" title="Steve Francia" */%}} #### Example output @@ -157,7 +156,7 @@ You can also use the variable `.Page` to access all the normal [Page Variables]( ## Single Positional Example: youtube - {{% youtube 09jf3ow9jfw %}} + {{%/* youtube 09jf3ow9jfw */%}} Would load the template /layouts/shortcodes/youtube.html @@ -179,7 +178,7 @@ This would be rendered as: ## Single Named Example: image with caption *Example has an extra space so Hugo doesn’t actually render it* - {{ % img src="/media/spf13.jpg" title="Steve Francia" %}} + {{%/* img src="/media/spf13.jpg" title="Steve Francia" */%}} Would load the template /layouts/shortcodes/img.html @@ -216,11 +215,11 @@ Would be rendered as: *Example has an extra space so Hugo doesn’t actually render it*. - {{ % highlight html %}} + {{%/* highlight html */%}} This HTML - {{ % /highlight %}} + {{%/* /highlight */%}} The template for this utilizes the following code (already include in Hugo) diff --git a/helpers/pygments.go b/helpers/pygments.go index 2ff500da3..bb7790533 100644 --- a/helpers/pygments.go +++ b/helpers/pygments.go @@ -1,4 +1,4 @@ -// Copyright © 2013 Steve Francia . +// Copyright © 2013-14 Steve Francia . // // Licensed under the Simple Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,11 +23,18 @@ import ( "github.com/spf13/viper" ) -func Highlight(code string, lexer string) string { - var pygmentsBin = "pygmentize" +const pygmentsBin = "pygmentize" +func HasPygments() bool { if _, err := exec.LookPath(pygmentsBin); err != nil { + return false + } + return true +} +func Highlight(code string, lexer string) string { + + if !HasPygments() { jww.WARN.Println("Highlighting requires Pygments to be installed and in the path") return code } diff --git a/hugolib/page.go b/hugolib/page.go index e30506c26..14a290c7e 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -17,6 +17,10 @@ import ( "bytes" "errors" "fmt" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/parser" + jww "github.com/spf13/jwalterweatherman" + "github.com/spf13/viper" "html/template" "io" "net/url" @@ -25,12 +29,8 @@ import ( "time" "github.com/spf13/cast" - "github.com/spf13/hugo/helpers" "github.com/spf13/hugo/hugofs" - "github.com/spf13/hugo/parser" "github.com/spf13/hugo/source" - jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/viper" ) type Page struct { @@ -47,14 +47,15 @@ type Page struct { Tmpl Template Markup string - extension string - contentType string - renderable bool - layout string - linkTitle string - frontmatter []byte - rawContent []byte - plain string // TODO should be []byte + extension string + contentType string + renderable bool + layout string + linkTitle string + frontmatter []byte + rawContent []byte + contentShortCodes map[string]string + plain string // TODO should be []byte PageMeta Source Position @@ -83,7 +84,7 @@ type Pages []*Page func (p *Page) Plain() string { if len(p.plain) == 0 { - p.plain = helpers.StripHTML(StripShortcodes(string(p.renderBytes(p.rawContent)))) + p.plain = helpers.StripHTML(string(p.Content)) } return p.plain } @@ -100,13 +101,33 @@ func (p *Page) UniqueId() string { return p.Source.UniqueId() } +// for logging +func (p *Page) lineNumRawContentStart() int { + return bytes.Count(p.frontmatter, []byte("\n")) + 1 +} + func (p *Page) setSummary() { + + // at this point, p.rawContent contains placeholders for the short codes, + // rendered and ready in p.contentShortcodes + if bytes.Contains(p.rawContent, helpers.SummaryDivider) { // If user defines split: - // Split then render + // Split, replace shortcode tokens, then render p.Truncated = true // by definition header := bytes.Split(p.rawContent, helpers.SummaryDivider)[0] - p.Summary = helpers.BytesToHTML(p.renderBytes(header)) + renderedHeader := p.renderBytes(header) + numShortcodesInHeader := bytes.Count(header, []byte(shortcodePlaceholderPrefix)) + if len(p.contentShortCodes) > 0 { + tmpContentWithTokensReplaced, err := + replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, numShortcodesInHeader, true, p.contentShortCodes) + if err != nil { + jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error()) + } else { + renderedHeader = tmpContentWithTokensReplaced + } + } + p.Summary = helpers.BytesToHTML(renderedHeader) } else { // If hugo defines split: // render, strip html, then split @@ -217,9 +238,6 @@ func (p *Page) ReadFrom(buf io.Reader) (err error) { return } - //analyze for raw stats - p.analyzePage() - return nil } @@ -550,7 +568,6 @@ func (page *Page) parse(reader io.Reader) error { } page.rawContent = psr.Content() - page.setSummary() return nil } @@ -613,15 +630,32 @@ func (page *Page) SaveSource() error { } func (p *Page) ProcessShortcodes(t Template) { - p.rawContent = []byte(ShortcodesHandle(string(p.rawContent), p, t)) - p.Summary = template.HTML(ShortcodesHandle(string(p.Summary), p, t)) + + // these short codes aren't used until after Page render, + // but processed here to avoid coupling + tmpContent, tmpContentShortCodes := extractAndRenderShortcodes(string(p.rawContent), p, t) + p.rawContent = []byte(tmpContent) + p.contentShortCodes = tmpContentShortCodes + } func (page *Page) Convert() error { markupType := page.guessMarkupType() switch markupType { case "markdown", "rst": + tmpContent, tmpTableOfContents := helpers.ExtractTOC(page.renderContent(helpers.RemoveSummaryDivider(page.rawContent))) + + if len(page.contentShortCodes) > 0 { + tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, -1, true, page.contentShortCodes) + + if err != nil { + jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error()) + } else { + tmpContent = tmpContentWithTokensReplaced + } + } + page.Content = helpers.BytesToHTML(tmpContent) page.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) case "html": @@ -629,6 +663,12 @@ func (page *Page) Convert() error { default: return fmt.Errorf("Error converting unsupported file type '%s' for page '%s'", markupType, page.Source.Path()) } + + // now we know enough to create a summary of the page and count some words + page.setSummary() + //analyze for raw stats + page.analyzePage() + return nil } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 1334b675a..3af1d1971 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -116,7 +116,7 @@ Some more text SIMPLE_PAGE_WITH_SHORTCODE_IN_SUMMARY = `--- title: Simple --- -Summary Next Line. {{% img src="/not/real" %}}. +Summary Next Line. {{
}}. More text here. Some more text @@ -335,14 +335,18 @@ func TestPageWithDelimiter(t *testing.T) { } func TestPageWithShortCodeInSummary(t *testing.T) { + s := new(Site) + s.prepTemplates() p, _ := NewPage("simple.md") err := p.ReadFrom(strings.NewReader(SIMPLE_PAGE_WITH_SHORTCODE_IN_SUMMARY)) - p.Convert() if err != nil { t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) } + p.ProcessShortcodes(s.Tmpl) + p.Convert() + checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "

Summary Next Line. {{% img src=“/not/real” %}}.\nMore text here.

\n\n

Some more text

\n") + checkPageContent(t, p, "

Summary Next Line. \n

\n \n \n \n \n
\n.\nMore text here.

\n\n

Some more text

\n") checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text") checkPageType(t, p, "page") checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index ef413bfb3..6dfc4ef02 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -1,4 +1,4 @@ -// Copyright © 2013 Steve Francia . +// Copyright © 2013-14 Steve Francia . // // Licensed under the Simple Public License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,13 +15,14 @@ package hugolib import ( "bytes" - "html/template" - "reflect" - "strings" - "unicode" - + "fmt" "github.com/spf13/hugo/helpers" jww "github.com/spf13/jwalterweatherman" + "html/template" + "reflect" + "regexp" + "strconv" + "strings" ) type ShortcodeFunc func([]string) string @@ -76,75 +77,357 @@ func (scp *ShortcodeWithPage) Get(key interface{}) interface{} { } -type Shortcodes map[string]ShortcodeFunc +// Note - this value must not contain any markup syntax +const shortcodePlaceholderPrefix = "HUGOSHORTCODE" + +type shortcode struct { + name string + inner []interface{} // string or nested shortcode + params interface{} // map or array + err error + doMarkup bool +} + +func (sc shortcode) String() string { + // for testing (mostly), so any change here will break tests! + return fmt.Sprintf("%s(%q, %t){%s}", sc.name, sc.params, sc.doMarkup, sc.inner) +} + +// all in one go: extract, render and replace +// only used for testing +func ShortcodesHandle(stringToParse string, page *Page, t Template) string { + + tmpContent, tmpShortcodes := extractAndRenderShortcodes(stringToParse, page, t) + + if len(tmpShortcodes) > 0 { + tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, -1, true, tmpShortcodes) + + if err != nil { + jww.ERROR.Printf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error()) + } else { + return string(tmpContentWithTokensReplaced) + } + } + + return string(tmpContent) +} + +var isInnerShortcodeCache = make(map[string]bool) + +// to avoid potential costly look-aheads for closing tags we look inside the template itself +// we could change the syntax to self-closing tags, but that would make users cry +// the value found is cached +func isInnerShortcode(t *template.Template) bool { + if m, ok := isInnerShortcodeCache[t.Name()]; ok { + return m + } + + match, _ := regexp.MatchString("{{.*?\\.Inner.*?}}", t.Tree.Root.String()) + isInnerShortcodeCache[t.Name()] = match + + return match +} + +func createShortcodePlaceholder(id int) string { + return fmt.Sprintf("
%s-%d
", shortcodePlaceholderPrefix, id) +} + +func renderShortcodes(sc shortcode, p *Page, t Template) string { + + tokenizedRenderedShortcodes := make(map[string](string)) + startCount := 0 + + shortcodes := renderShortcode(sc, tokenizedRenderedShortcodes, startCount, p, t) + + // placeholders will be numbered from 1.. and top down + for i := 1; i <= len(tokenizedRenderedShortcodes); i++ { + placeHolder := createShortcodePlaceholder(i) + shortcodes = strings.Replace(shortcodes, placeHolder, tokenizedRenderedShortcodes[placeHolder], 1) + } + return shortcodes +} + +func renderShortcode(sc shortcode, tokenizedShortcodes map[string](string), cnt int, p *Page, t Template) string { + var data = &ShortcodeWithPage{Params: sc.params, Page: p} + tmpl := GetTemplate(sc.name, t) + + if tmpl == nil { + jww.ERROR.Printf("Unable to locate template for shortcode '%s' in page %s", sc.name, p.BaseFileName()) + return "" + } + + if len(sc.inner) > 0 { + var inner string + for _, innerData := range sc.inner { + switch innerData.(type) { + case string: + inner += innerData.(string) + case shortcode: + // nested shortcodes will be rendered individually, replace them with temporary numbered tokens + cnt++ + placeHolder := createShortcodePlaceholder(cnt) + renderedContent := renderShortcode(innerData.(shortcode), tokenizedShortcodes, cnt, p, t) + tokenizedShortcodes[placeHolder] = renderedContent + inner += placeHolder + default: + jww.ERROR.Printf("Illegal state on shortcode rendering of '%s' in page %s. Illegal type in inner data: %s ", + sc.name, p.BaseFileName(), reflect.TypeOf(innerData)) + return "" + } + } + + if sc.doMarkup { + data.Inner = template.HTML(helpers.RenderBytes([]byte(inner), p.guessMarkupType(), p.UniqueId())) + } else { + data.Inner = template.HTML(inner) + } + + } + + return ShortcodeRender(tmpl, data) +} + +func extractAndRenderShortcodes(stringToParse string, p *Page, t Template) (string, map[string]string) { + + content, shortcodes, err := extractShortcodes(stringToParse, p, t) + renderedShortcodes := make(map[string]string) + + if err != nil { + // try to render what we have whilst logging the error + jww.ERROR.Println(err.Error()) + } + + for key, sc := range shortcodes { + if sc.err != nil { + // need to have something to replace with + renderedShortcodes[key] = "" + } else { + renderedShortcodes[key] = renderShortcodes(sc, p, t) + } + } + + return content, renderedShortcodes + +} + +// pageTokens state: +// - before: positioned just before the shortcode start +// - after: shortcode(s) consumed (plural when they are nested) +func extractShortcode(pt *pageTokens, p *Page, t Template) (shortcode, error) { + sc := shortcode{} + var isInner = false + + var currItem item + var cnt = 0 + +Loop: + for { + currItem = pt.next() + + switch currItem.typ { + case tLeftDelimScWithMarkup, tLeftDelimScNoMarkup: + next := pt.peek() + if next.typ == tScClose { + continue + } + + if cnt > 0 { + // nested shortcode; append it to inner content + pt.backup3(currItem, next) + nested, err := extractShortcode(pt, p, t) + if err == nil { + sc.inner = append(sc.inner, nested) + } else { + return sc, err + } + + } else { + sc.doMarkup = currItem.typ == tLeftDelimScWithMarkup + } + + cnt++ + + case tRightDelimScWithMarkup, tRightDelimScNoMarkup: + // we trust the template on this: + // if there's no inner, we're done + if !isInner { + return sc, nil + } + + case tScClose: + if !isInner { + next := pt.peek() + if next.typ == tError { + // return that error, more specific + continue + } + return sc, fmt.Errorf("Shortcode '%s' has no .Inner, yet a closing tag was provided", next.val) + } + pt.consume(2) + return sc, nil + case tText: + sc.inner = append(sc.inner, currItem.val) + case tScName: + sc.name = currItem.val + tmpl := GetTemplate(sc.name, t) -func ShortcodesHandle(stringToParse string, p *Page, t Template) string { - leadStart := strings.Index(stringToParse, `{{%`) - if leadStart >= 0 { - leadEnd := strings.Index(stringToParse[leadStart:], `%}}`) + leadStart - if leadEnd > leadStart { - name, par := SplitParams(stringToParse[leadStart+3 : leadEnd]) - tmpl := GetTemplate(name, t) if tmpl == nil { - return stringToParse + return sc, fmt.Errorf("Unable to locate template for shortcode '%s' in page %s", sc.name, p.BaseFileName()) } - params := Tokenize(par) - // Always look for closing tag. - endStart, endEnd := FindEnd(stringToParse[leadEnd:], name) - var data = &ShortcodeWithPage{Params: params, Page: p} - if endStart > 0 { - s := stringToParse[leadEnd+3 : leadEnd+endStart] - data.Inner = template.HTML(helpers.RenderBytes([]byte(CleanP(ShortcodesHandle(s, p, t))), p.guessMarkupType(), p.UniqueId())) - remainder := CleanP(stringToParse[leadEnd+endEnd:]) + isInner = isInnerShortcode(tmpl) - return CleanP(stringToParse[:leadStart]) + - ShortcodeRender(tmpl, data) + - CleanP(ShortcodesHandle(remainder, p, t)) + case tScParam: + if !pt.isValueNext() { + continue + } else if pt.peek().typ == tScParamVal { + // named params + if sc.params == nil { + params := make(map[string]string) + params[currItem.val] = pt.next().val + sc.params = params + } else { + params := sc.params.(map[string]string) + params[currItem.val] = pt.next().val + } + } else { + // positional params + if sc.params == nil { + var params []string + params = append(params, currItem.val) + sc.params = params + } else { + params := sc.params.([]string) + params = append(params, currItem.val) + sc.params = params + } } - return CleanP(stringToParse[:leadStart]) + - ShortcodeRender(tmpl, data) + - CleanP(ShortcodesHandle(stringToParse[leadEnd+3:], p, - t)) + + case tError, tEOF: + // handled by caller + pt.backup() + break Loop + } } - return stringToParse + return sc, nil } -// Clean up odd behavior when closing tag is on first line -// or opening tag is on the last line due to extra line in markdown file -func CleanP(str string) string { - if strings.HasSuffix(strings.TrimSpace(str), "

") { - idx := strings.LastIndex(str, "

") - str = str[:idx] +func extractShortcodes(stringToParse string, p *Page, t Template) (string, map[string]shortcode, error) { + + shortCodes := make(map[string]shortcode) + + startIdx := strings.Index(stringToParse, "{{") + + // short cut for docs with no shortcodes + if startIdx < 0 { + return stringToParse, shortCodes, nil } - if strings.HasPrefix(strings.TrimSpace(str), "

") { - str = str[strings.Index(str, "

")+5:] - } + // the parser takes a string; + // since this is an internal API, it could make sense to use the mutable []byte all the way, but + // it seems that the time isn't really spent in the byte copy operations, and the impl. gets a lot cleaner + pt := &pageTokens{lexer: newShortcodeLexer("parse-page", stringToParse, pos(startIdx))} - return str -} + id := 1 // incremented id, will be appended onto temp. shortcode placeholders + var result bytes.Buffer -func FindEnd(str string, name string) (int, int) { - var endPos int - var startPos int - var try []string + // the parser is guaranteed to return items in proper order or fail, so … + // … it's safe to keep some "global" state + var currItem item + var currShortcode shortcode + var err error - try = append(try, "{{% /"+name+" %}}") - try = append(try, "{{% /"+name+"%}}") - try = append(try, "{{%/"+name+"%}}") - try = append(try, "{{%/"+name+" %}}") +Loop: + for { + currItem = pt.next() - lowest := len(str) - for _, x := range try { - start := strings.Index(str, x) - if start < lowest && start > 0 { - startPos = start - endPos = startPos + len(x) + switch currItem.typ { + case tText: + result.WriteString(currItem.val) + case tLeftDelimScWithMarkup, tLeftDelimScNoMarkup: + // let extractShortcode handle left delim (will do so recursively) + pt.backup() + if currShortcode, err = extractShortcode(pt, p, t); err != nil { + return result.String(), shortCodes, err + } + + if currShortcode.params == nil { + currShortcode.params = make([]string, 0) + } + + // wrap it in a block level element to let it be left alone by the markup engine + placeHolder := createShortcodePlaceholder(id) + result.WriteString(placeHolder) + shortCodes[placeHolder] = currShortcode + id++ + case tEOF: + break Loop + case tError: + err := fmt.Errorf("%s:%d: %s", + p.BaseFileName(), (p.lineNumRawContentStart() + pt.lexer.lineNum() - 1), currItem) + currShortcode.err = err + return result.String(), shortCodes, err } } - return startPos, endPos + return result.String(), shortCodes, nil + +} + +// Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content. +// This assumes that all tokens exist in the input string and that they are in order. +// numReplacements = -1 will do len(replacements), and it will always start from the beginning (1) +// wrappendInDiv = true means that the token is wrapped in a
+func replaceShortcodeTokens(source []byte, prefix string, numReplacements int, wrappedInDiv bool, replacements map[string]string) ([]byte, error) { + + if numReplacements < 0 { + numReplacements = len(replacements) + } + + if numReplacements == 0 { + return source, nil + } + + newLen := len(source) + + for i := 1; i <= numReplacements; i++ { + key := prefix + "-" + strconv.Itoa(i) + + if wrappedInDiv { + key = "
" + key + "
" + } + val := []byte(replacements[key]) + + newLen += (len(val) - len(key)) + } + + buff := make([]byte, newLen) + + width := 0 + start := 0 + + for i := 0; i < numReplacements; i++ { + tokenNum := i + 1 + oldVal := prefix + "-" + strconv.Itoa(tokenNum) + if wrappedInDiv { + oldVal = "
" + oldVal + "
" + } + newVal := []byte(replacements[oldVal]) + j := start + + k := bytes.Index(source[start:], []byte(oldVal)) + if k < 0 { + // this should never happen, but let the caller decide to panic or not + return nil, fmt.Errorf("illegal state in content; shortcode token #%d is missing or out of order", tokenNum) + } + j += k + + width += copy(buff[width:], source[start:j]) + width += copy(buff[width:], newVal) + start = j + len(oldVal) + } + width += copy(buff[width:], source[start:]) + return buff[0:width], nil } func GetTemplate(name string, t Template) *template.Template { @@ -157,143 +440,6 @@ func GetTemplate(name string, t Template) *template.Template { return t.Lookup("_internal/shortcodes/" + name + ".html") } -func StripShortcodes(stringToParse string) string { - posStart := strings.Index(stringToParse, "{{%") - if posStart > 0 { - posEnd := strings.Index(stringToParse[posStart:], "%}}") + posStart - if posEnd > posStart { - newString := stringToParse[:posStart] + StripShortcodes(stringToParse[posEnd+3:]) - return newString - } - } - return stringToParse -} - -func CleanupSpacesAroundEquals(rawfirst []string) []string { - var first = make([]string, 0) - - for i := 0; i < len(rawfirst); i++ { - v := rawfirst[i] - index := strings.Index(v, "=") - - if index == len(v)-1 { - // Trailing '=' - if len(rawfirst) > i { - if v == "=" { - first[len(first)-1] = first[len(first)-1] + v + rawfirst[i+1] // concat prior with this and next - i++ // Skip next - } else { - // Trailing ' = ' - first = append(first, v+rawfirst[i+1]) // append this token and the next - i++ // Skip next - } - } else { - break - } - } else if index == 0 { - // Leading '=' - first[len(first)-1] = first[len(first)-1] + v // concat this token to the prior one - continue - } else { - first = append(first, v) - } - } - - return first -} - -func Tokenize(in string) interface{} { - var final = make([]string, 0) - - // if there isn't a space or an equal sign, no need to parse - if strings.Index(in, " ") < 0 && strings.Index(in, "=") < 0 { - return append(final, in) - } - - var keys = make([]string, 0) - inQuote := false - start := 0 - - first := CleanupSpacesAroundEquals(strings.Fields(in)) - - for i, v := range first { - index := strings.Index(v, "=") - if !inQuote { - if index > 1 { - keys = append(keys, v[:index]) - v = v[index+1:] - } - } - - // Adjusted to handle htmlencoded and non htmlencoded input - if !strings.HasPrefix(v, "“") && !strings.HasPrefix(v, "\"") && !inQuote { - final = append(final, v) - } else if inQuote && (strings.HasSuffix(v, "”") || - strings.HasSuffix(v, "\"")) && !strings.HasSuffix(v, "\\\"") { - if strings.HasSuffix(v, "\"") { - first[i] = v[:len(v)-1] - } else { - first[i] = v[:len(v)-7] - } - final = append(final, strings.Join(first[start:i+1], " ")) - inQuote = false - } else if (strings.HasPrefix(v, "“") || - strings.HasPrefix(v, "\"")) && !inQuote { - if strings.HasSuffix(v, "”") || strings.HasSuffix(v, - "\"") { - if strings.HasSuffix(v, "\"") { - if len(v) > 1 { - final = append(final, v[1:len(v)-1]) - } else { - final = append(final, "") - } - } else { - final = append(final, v[7:len(v)-7]) - } - } else { - start = i - if strings.HasPrefix(v, "\"") { - first[i] = v[1:] - } else { - first[i] = v[7:] - } - inQuote = true - } - } - - // No closing "... just make remainder the final token - if inQuote && i == len(first) { - final = append(final, first[start:]...) - } - } - - if len(keys) > 0 && (len(keys) != len(final)) { - // This will happen if the quotes aren't balanced - return final - } - - if len(keys) > 0 { - var m = make(map[string]string) - for i, k := range keys { - m[k] = final[i] - } - - return m - } - - return final -} - -func SplitParams(in string) (name string, par2 string) { - newIn := strings.TrimSpace(in) - i := strings.IndexFunc(newIn, unicode.IsSpace) - if i < 1 { - return strings.TrimSpace(in), "" - } - - return strings.TrimSpace(newIn[:i+1]), strings.TrimSpace(newIn[i+1:]) -} - func ShortcodeRender(tmpl *template.Template, data *ShortcodeWithPage) string { buffer := new(bytes.Buffer) err := tmpl.Execute(buffer, data) diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 6a5aadd79..91e297805 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -1,6 +1,12 @@ package hugolib import ( + "fmt" + "github.com/spf13/hugo/helpers" + "github.com/spf13/viper" + "reflect" + "regexp" + "sort" "strings" "testing" ) @@ -21,40 +27,40 @@ func CheckShortCodeMatch(t *testing.T, input, expected string, template Template func TestNonSC(t *testing.T) { tem := NewTemplate() - - CheckShortCodeMatch(t, "{{% movie 47238zzb %}}", "{{% movie 47238zzb %}}", tem) + // notice the syntax diff from 0.12, now comment delims must be added + CheckShortCodeMatch(t, "{{%/* movie 47238zzb */%}}", "{{% movie 47238zzb %}}", tem) } func TestPositionalParamSC(t *testing.T) { tem := NewTemplate() tem.AddInternalShortcode("video.html", `Playing Video {{ .Get 0 }}`) - CheckShortCodeMatch(t, "{{% video 47238zzb %}}", "Playing Video 47238zzb", tem) - CheckShortCodeMatch(t, "{{% video 47238zzb 132 %}}", "Playing Video 47238zzb", tem) - CheckShortCodeMatch(t, "{{%video 47238zzb%}}", "Playing Video 47238zzb", tem) - CheckShortCodeMatch(t, "{{%video 47238zzb %}}", "Playing Video 47238zzb", tem) - CheckShortCodeMatch(t, "{{% video 47238zzb %}}", "Playing Video 47238zzb", tem) + CheckShortCodeMatch(t, "{{< video 47238zzb >}}", "Playing Video 47238zzb", tem) + CheckShortCodeMatch(t, "{{< video 47238zzb 132 >}}", "Playing Video 47238zzb", tem) + CheckShortCodeMatch(t, "{{