Use a regular expression in replaceShortcodeTokens

This fixes a bug where a shortcode needs to be expanded multiple times,
which can arise in practice when using reference links.
This commit is contained in:
Jeffrey Tolar 2015-01-28 21:11:41 -06:00 committed by bep
parent ba53799fdb
commit 366c557251
4 changed files with 49 additions and 66 deletions

View file

@ -60,7 +60,7 @@ func (h markdownHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
if len(p.contentShortCodes) > 0 {
tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, -1, true, p.contentShortCodes)
tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, true, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
@ -113,7 +113,7 @@ func (h rstHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
if len(p.contentShortCodes) > 0 {
tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, -1, true, p.contentShortCodes)
tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, true, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())

View file

@ -162,10 +162,9 @@ func (p *Page) setSummary() {
p.Truncated = true // by definition
header := bytes.Split(p.rawContent, helpers.SummaryDivider)[0]
renderedHeader := p.renderBytes(header)
numShortcodesInHeader := bytes.Count(header, []byte(shortcodePlaceholderPrefix))
if len(p.contentShortCodes) > 0 {
tmpContentWithTokensReplaced, err :=
replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, numShortcodesInHeader, true, p.contentShortCodes)
replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, true, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error())
} else {

View file

@ -20,7 +20,6 @@ import (
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"sync"
@ -132,7 +131,7 @@ func ShortcodesHandle(stringToParse string, page *Page, t tpl.Template) string {
tmpContent, tmpShortcodes := extractAndRenderShortcodes(stringToParse, page, t)
if len(tmpShortcodes) > 0 {
tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, -1, true, tmpShortcodes)
tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, true, tmpShortcodes)
if err != nil {
jww.ERROR.Printf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error())
@ -428,60 +427,44 @@ Loop:
}
// Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content.
// This assumes that all tokens exist in the input string and that they are in order.
// numReplacements = -1 will do len(replacements), and it will always start from the beginning (1)
// wrapped = true means that the token has been wrapped in {@{@/@}@}
func replaceShortcodeTokens(source []byte, prefix string, numReplacements int, wrapped bool, replacements map[string]string) ([]byte, error) {
func replaceShortcodeTokens(source []byte, prefix string, wrapped bool, replacements map[string]string) (b []byte, err error) {
var re *regexp.Regexp
if numReplacements < 0 {
numReplacements = len(replacements)
}
if numReplacements == 0 {
return source, nil
}
newLen := len(source)
for i := 1; i <= numReplacements; i++ {
key := prefix + "-" + strconv.Itoa(i)
if wrapped {
key = "{@{@" + key + "@}@}"
if wrapped {
re, err = regexp.Compile(`\{@\{@` + regexp.QuoteMeta(prefix) + `-\d+@\}@\}`)
if err != nil {
return nil, err
}
} else {
re, err = regexp.Compile(regexp.QuoteMeta(prefix) + `-(\d+)`)
if err != nil {
return nil, err
}
val := []byte(replacements[key])
newLen += (len(val) - len(key))
}
buff := make([]byte, newLen)
width := 0
start := 0
for i := 0; i < numReplacements; i++ {
tokenNum := i + 1
oldVal := prefix + "-" + strconv.Itoa(tokenNum)
if wrapped {
oldVal = "{@{@" + oldVal + "@}@}"
// use panic/recover for reporting if an unknown
defer func() {
if r := recover(); r != nil {
var ok bool
b = nil
err, ok = r.(error)
if !ok {
err = fmt.Errorf("unexpected panic during replaceShortcodeTokens: %v", r)
}
}
newVal := []byte(replacements[oldVal])
j := start
}()
b = re.ReplaceAllFunc(source, func(m []byte) []byte {
key := string(m)
k := bytes.Index(source[start:], []byte(oldVal))
if k < 0 {
// this should never happen, but let the caller decide to panic or not
return nil, fmt.Errorf("illegal state in content; shortcode token #%d is missing or out of order (%q)", tokenNum, source)
if val, ok := replacements[key]; ok {
return []byte(val)
} else {
panic(fmt.Errorf("unknown shortcode token %q", key))
}
j += k
})
width += copy(buff[width:], source[start:j])
width += copy(buff[width:], newVal)
start = j + len(oldVal)
}
width += copy(buff[width:], source[start:])
return buff[0:width], nil
return b, err
}
func GetTemplate(name string, t tpl.Template) *template.Template {

View file

@ -281,23 +281,24 @@ func collectAndShortShortcodes(shortcodes map[string]shortcode) []string {
func TestReplaceShortcodeTokens(t *testing.T) {
for i, this := range []struct {
input []byte
prefix string
replacements map[string]string
numReplacements int
wrappedInDiv bool
expect interface{}
input []byte
prefix string
replacements map[string]string
wrappedInDiv bool
expect interface{}
}{
{[]byte("Hello PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "World"}, -1, false, []byte("Hello World.")},
{[]byte("A {@{@A-1@}@} asdf {@{@A-2@}@}."), "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, -1, true, []byte("A v1 asdf v2.")},
{[]byte("Hello PREFIX2-1. Go PREFIX2-2, Go, Go PREFIX2-3 Go Go!."), "PREFIX2", map[string]string{"PREFIX2-1": "Europe", "PREFIX2-2": "Jonny", "PREFIX2-3": "Johnny"}, -1, false, []byte("Hello Europe. Go Jonny, Go, Go Johnny Go Go!.")},
{[]byte("A PREFIX-2 PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, -1, false, false},
{[]byte("A PREFIX-1 PREFIX-2"), "PREFIX", map[string]string{"PREFIX-1": "A"}, -1, false, []byte("A A PREFIX-2")},
{[]byte("A PREFIX-1 but not the second."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, -1, false, false},
{[]byte("An PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, 1, false, []byte("An A.")},
{[]byte("An PREFIX-1 PREFIX-2."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, 1, false, []byte("An A PREFIX-2.")},
{[]byte("Hello PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "World"}, false, []byte("Hello World.")},
{[]byte("A {@{@A-1@}@} asdf {@{@A-2@}@}."), "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, true, []byte("A v1 asdf v2.")},
{[]byte("Hello PREFIX2-1. Go PREFIX2-2, Go, Go PREFIX2-3 Go Go!."), "PREFIX2", map[string]string{"PREFIX2-1": "Europe", "PREFIX2-2": "Jonny", "PREFIX2-3": "Johnny"}, false, []byte("Hello Europe. Go Jonny, Go, Go Johnny Go Go!.")},
{[]byte("A PREFIX-2 PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("A B A.")},
{[]byte("A PREFIX-1 PREFIX-2"), "PREFIX", map[string]string{"PREFIX-1": "A"}, false, false},
{[]byte("A PREFIX-1 but not the second."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("A A but not the second.")},
{[]byte("An PREFIX-1."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("An A.")},
{[]byte("An PREFIX-1 PREFIX-2."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B"}, false, []byte("An A B.")},
{[]byte("A PREFIX-1 PREFIX-2 PREFIX-3 PREFIX-1 PREFIX-3."), "PREFIX", map[string]string{"PREFIX-1": "A", "PREFIX-2": "B", "PREFIX-3": "C"}, false, []byte("A A B C A C.")},
{[]byte("A {@{@PREFIX-1@}@} {@{@PREFIX-2@}@} {@{@PREFIX-3@}@} {@{@PREFIX-1@}@} {@{@PREFIX-3@}@}."), "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "A", "{@{@PREFIX-2@}@}": "B", "{@{@PREFIX-3@}@}": "C"}, true, []byte("A A B C A C.")},
} {
results, err := replaceShortcodeTokens(this.input, this.prefix, this.numReplacements, this.wrappedInDiv, this.replacements)
results, err := replaceShortcodeTokens(this.input, this.prefix, this.wrappedInDiv, this.replacements)
if b, ok := this.expect.(bool); ok && !b {
if err == nil {