diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index a0fe5d158..e375b0eba 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -602,8 +602,8 @@ func (h *HugoSites) Pages() Pages { } func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) { - if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 { - p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName()) + if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 { + p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName()) err := p.shortcodeState.executeShortcodesForDelta(p) if err != nil { diff --git a/hugolib/orderedMap.go b/hugolib/orderedMap.go new file mode 100644 index 000000000..c8879baa7 --- /dev/null +++ b/hugolib/orderedMap.go @@ -0,0 +1,100 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "sync" +) + +type orderedMap struct { + sync.RWMutex + keys []interface{} + m map[interface{}]interface{} +} + +func newOrderedMap() *orderedMap { + return &orderedMap{m: make(map[interface{}]interface{})} +} + +func newOrderedMapFromStringMapString(m map[string]string) *orderedMap { + om := newOrderedMap() + for k, v := range m { + om.Add(k, v) + } + return om +} + +func (m *orderedMap) Add(k, v interface{}) { + m.Lock() + _, found := m.m[k] + if found { + panic(fmt.Sprintf("%v already added", v)) + } + m.m[k] = v + m.keys = append(m.keys, k) + m.Unlock() + +} + +func (m *orderedMap) Get(k interface{}) (interface{}, bool) { + m.RLock() + defer m.RUnlock() + v, found := m.m[k] + return v, found +} + +func (m *orderedMap) Contains(k interface{}) bool { + m.RLock() + defer m.RUnlock() + _, found := m.m[k] + return found +} + +func (m *orderedMap) Keys() []interface{} { + m.RLock() + defer m.RUnlock() + return m.keys +} + +func (m *orderedMap) Len() int { + m.RLock() + defer m.RUnlock() + return len(m.keys) +} + +// Some shortcuts for known types. +func (m *orderedMap) getShortcode(k interface{}) *shortcode { + v, found := m.Get(k) + if !found { + return nil + } + return v.(*shortcode) +} + +func (m *orderedMap) getShortcodeRenderer(k interface{}) func() (string, error) { + v, found := m.Get(k) + if !found { + return nil + } + return v.(func() (string, error)) +} + +func (m *orderedMap) getString(k interface{}) string { + v, found := m.Get(k) + if !found { + return "" + } + return v.(string) +} diff --git a/hugolib/orderedMap_test.go b/hugolib/orderedMap_test.go new file mode 100644 index 000000000..fc3d25080 --- /dev/null +++ b/hugolib/orderedMap_test.go @@ -0,0 +1,69 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOrderedMap(t *testing.T) { + t.Parallel() + assert := require.New(t) + + m := newOrderedMap() + m.Add("b", "vb") + m.Add("c", "vc") + m.Add("a", "va") + b, f1 := m.Get("b") + + assert.True(f1) + assert.Equal(b, "vb") + assert.True(m.Contains("b")) + assert.False(m.Contains("e")) + + assert.Equal([]interface{}{"b", "c", "a"}, m.Keys()) + +} + +func TestOrderedMapConcurrent(t *testing.T) { + t.Parallel() + assert := require.New(t) + + var wg sync.WaitGroup + + m := newOrderedMap() + + for i := 1; i < 20; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + key := fmt.Sprintf("key%d", id) + val := key + "val" + m.Add(key, val) + v, found := m.Get(key) + assert.True(found) + assert.Equal(v, val) + assert.True(m.Contains(key)) + assert.True(m.Len() > 0) + assert.True(len(m.Keys()) > 0) + }(i) + + } + + wg.Wait() +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index df4acba5f..933bbe44e 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -180,11 +180,11 @@ type shortcodeHandler struct { p *PageWithoutContent // This is all shortcode rendering funcs for all potential output formats. - contentShortcodes map[scKey]func() (string, error) + contentShortcodes *orderedMap // This map contains the new or changed set of shortcodes that need // to be rendered for the current output format. - contentShortcodesDelta map[scKey]func() (string, error) + contentShortcodesDelta *orderedMap // This maps the shorcode placeholders with the rendered content. // We will do (potential) partial re-rendering per output format, @@ -192,7 +192,7 @@ type shortcodeHandler struct { renderedShortcodes map[string]string // Maps the shortcodeplaceholder with the actual shortcode. - shortcodes map[string]shortcode + shortcodes *orderedMap // All the shortcode names in this set. nameSet map[string]bool @@ -216,8 +216,8 @@ func (s *shortcodeHandler) createShortcodePlaceholder() string { func newShortcodeHandler(p *Page) *shortcodeHandler { return &shortcodeHandler{ p: p.withoutContent(), - contentShortcodes: make(map[scKey]func() (string, error)), - shortcodes: make(map[string]shortcode), + contentShortcodes: newOrderedMap(), + shortcodes: newOrderedMap(), nameSet: make(map[string]bool), renderedShortcodes: make(map[string]string), } @@ -259,7 +259,7 @@ const innerNewlineRegexp = "\n" const innerCleanupRegexp = `\A

(.*)

\n\z` const innerCleanupExpand = "$1" -func prepareShortcodeForPage(placeholder string, sc shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) { +func prepareShortcodeForPage(placeholder string, sc *shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) { m := make(map[scKey]func() (string, error)) lang := p.Lang() @@ -277,7 +277,7 @@ func prepareShortcodeForPage(placeholder string, sc shortcode, parent *Shortcode func renderShortcode( tmplKey scKey, - sc shortcode, + sc *shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) string { @@ -298,8 +298,8 @@ func renderShortcode( switch innerData.(type) { case string: inner += innerData.(string) - case shortcode: - inner += renderShortcode(tmplKey, innerData.(shortcode), data, p) + case *shortcode: + inner += renderShortcode(tmplKey, innerData.(*shortcode), data, p) default: p.s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", sc.name, p.Path(), reflect.TypeOf(innerData)) @@ -363,48 +363,51 @@ func (s *shortcodeHandler) updateDelta() bool { contentShortcodes := s.contentShortcodesForOutputFormat(s.p.s.rc.Format) - if s.contentShortcodesDelta == nil || len(s.contentShortcodesDelta) == 0 { + if s.contentShortcodesDelta == nil || s.contentShortcodesDelta.Len() == 0 { s.contentShortcodesDelta = contentShortcodes return true } - delta := make(map[scKey]func() (string, error)) + delta := newOrderedMap() - for k, v := range contentShortcodes { - if _, found := s.contentShortcodesDelta[k]; !found { - delta[k] = v + for _, k := range contentShortcodes.Keys() { + if !s.contentShortcodesDelta.Contains(k) { + v, _ := contentShortcodes.Get(k) + delta.Add(k, v) } } s.contentShortcodesDelta = delta - return len(delta) > 0 + return delta.Len() > 0 } -func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map[scKey]func() (string, error) { - contentShortcodesForOuputFormat := make(map[scKey]func() (string, error)) +func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) *orderedMap { + contentShortcodesForOuputFormat := newOrderedMap() lang := s.p.Lang() - for shortcodePlaceholder := range s.shortcodes { + for _, key := range s.shortcodes.Keys() { + shortcodePlaceholder := key.(string) + // shortcodePlaceholder := s.shortcodes.getShortcode(key) key := newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder) - renderFn, found := s.contentShortcodes[key] + renderFn, found := s.contentShortcodes.Get(key) if !found { key.OutputFormat = "" - renderFn, found = s.contentShortcodes[key] + renderFn, found = s.contentShortcodes.Get(key) } // Fall back to HTML if !found && key.Suffix != "html" { key.Suffix = "html" - renderFn, found = s.contentShortcodes[key] + renderFn, found = s.contentShortcodes.Get(key) } if !found { panic(fmt.Sprintf("Shortcode %q could not be found", shortcodePlaceholder)) } - contentShortcodesForOuputFormat[newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder)] = renderFn + contentShortcodesForOuputFormat.Add(newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder), renderFn) } return contentShortcodesForOuputFormat @@ -412,27 +415,29 @@ func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) error { - for k, render := range s.contentShortcodesDelta { + for _, k := range s.contentShortcodesDelta.Keys() { + render := s.contentShortcodesDelta.getShortcodeRenderer(k) renderedShortcode, err := render() if err != nil { return fmt.Errorf("Failed to execute shortcode in page %q: %s", p.Path(), err) } - s.renderedShortcodes[k.ShortcodePlaceholder] = renderedShortcode + s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode } return nil } -func createShortcodeRenderers(shortcodes map[string]shortcode, p *PageWithoutContent) map[scKey]func() (string, error) { +func createShortcodeRenderers(shortcodes *orderedMap, p *PageWithoutContent) *orderedMap { - shortcodeRenderers := make(map[scKey]func() (string, error)) + shortcodeRenderers := newOrderedMap() - for k, v := range shortcodes { - prepared := prepareShortcodeForPage(k, v, nil, p) + for _, k := range shortcodes.Keys() { + v := shortcodes.getShortcode(k) + prepared := prepareShortcodeForPage(k.(string), v, nil, p) for kk, vv := range prepared { - shortcodeRenderers[kk] = vv + shortcodeRenderers.Add(kk, vv) } } @@ -444,8 +449,8 @@ var errShortCodeIllegalState = errors.New("Illegal shortcode state") // pageTokens state: // - before: positioned just before the shortcode start // - after: shortcode(s) consumed (plural when they are nested) -func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (shortcode, error) { - sc := shortcode{} +func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (*shortcode, error) { + sc := &shortcode{} var isInner = false var currItem item @@ -616,7 +621,7 @@ Loop: placeHolder := s.createShortcodePlaceholder() result.WriteString(placeHolder) - s.shortcodes[placeHolder] = currShortcode + s.shortcodes.Add(placeHolder, currShortcode) case tEOF: break Loop case tError: diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 564ffcd70..3e8a952e6 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -433,16 +433,17 @@ func TestExtractShortcodes(t *testing.T) { t.Fatalf("[%d] %s: Failed to compile regexp %q: %q", i, this.name, expected, err) } - if strings.Count(content, shortcodePlaceholderPrefix) != len(shortCodes) { - t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, len(shortCodes)) + if strings.Count(content, shortcodePlaceholderPrefix) != shortCodes.Len() { + t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, shortCodes.Len()) } if !r.MatchString(content) { t.Fatalf("[%d] %s: Shortcode extract didn't match. got %q but expected %q", i, this.name, content, expected) } - for placeHolder, sc := range shortCodes { - if !strings.Contains(content, placeHolder) { + for _, placeHolder := range shortCodes.Keys() { + sc := shortCodes.getShortcode(placeHolder) + if !strings.Contains(content, placeHolder.(string)) { t.Fatalf("[%d] %s: Output does not contain placeholder %q", i, this.name, placeHolder) } @@ -753,10 +754,11 @@ NotFound: {{< thisDoesNotExist >}} } -func collectAndSortShortcodes(shortcodes map[string]shortcode) []string { +func collectAndSortShortcodes(shortcodes *orderedMap) []string { var asArray []string - for key, sc := range shortcodes { + for _, key := range shortcodes.Keys() { + sc := shortcodes.getShortcode(key) asArray = append(asArray, fmt.Sprintf("%s:%s", key, sc)) } @@ -881,3 +883,48 @@ func TestScKey(t *testing.T) { newDefaultScKey("IJKL")) } + +func TestPreserveShortcodeOrder(t *testing.T) { + t.Parallel() + assert := require.New(t) + + contentTemplate := `--- +title: doc%d +weight: %d +--- +# doc + +{{< increment >}}{{< s1 >}}{{< increment >}}{{< s2 >}}{{< increment >}}{{< s3 >}}{{< increment >}}{{< s4 >}}{{< increment >}}{{< s5 >}} + + +` + + shortCodeTemplate := `v%d: {{ .Page.Scratch.Get "v" }}|` + + var shortcodes []string + var content []string + + shortcodes = append(shortcodes, []string{"shortcodes/increment.html", `{{ .Page.Scratch.Add "v" 1}}`}...) + + for i := 1; i <= 5; i++ { + shortcodes = append(shortcodes, []string{fmt.Sprintf("shortcodes/s%d.html", i), fmt.Sprintf(shortCodeTemplate, i)}...) + } + + for i := 1; i <= 3; i++ { + content = append(content, []string{fmt.Sprintf("p%d.md", i), fmt.Sprintf(contentTemplate, i, i)}...) + } + + builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig() + + builder.WithContent(content...).WithTemplatesAdded(shortcodes...).CreateSites().Build(BuildCfg{}) + + s := builder.H.Sites[0] + assert.Equal(3, len(s.RegularPages)) + + p1 := s.RegularPages[0] + + if !strings.Contains(string(p1.content()), `v1: 1|v2: 2|v3: 3|v4: 4|v5: 5`) { + t.Fatal(p1.content()) + } + +}