From fb33d8286d78a78a74deb44355b621852a1c4033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 25 Sep 2017 08:59:02 +0200 Subject: [PATCH] Use Chroma as new default syntax highlighter If you want to use Pygments, set `pygmentsUseClassic=true` in your site config. Fixes #3888 --- commands/genchromastyles.go | 70 ++++++++ commands/hugo.go | 1 + deps/deps.go | 13 +- helpers/content.go | 44 ++++- helpers/content_renderer.go | 13 +- helpers/content_renderer_test.go | 54 +++--- helpers/pygments.go | 246 +++++++++++++++++++++++----- helpers/pygments_test.go | 201 ++++++++++++++++++++++- helpers/testhelpers_test.go | 6 +- hugolib/config.go | 15 +- hugolib/embedded_shortcodes_test.go | 9 +- hugolib/shortcode_test.go | 4 +- hugolib/site.go | 8 +- tpl/collections/collections_test.go | 6 +- tpl/data/resources_test.go | 6 +- tpl/transform/transform.go | 3 +- tpl/transform/transform_test.go | 7 +- vendor/vendor.json | 54 +++++- 18 files changed, 652 insertions(+), 108 deletions(-) create mode 100644 commands/genchromastyles.go diff --git a/commands/genchromastyles.go b/commands/genchromastyles.go new file mode 100644 index 000000000..66a2b50a6 --- /dev/null +++ b/commands/genchromastyles.go @@ -0,0 +1,70 @@ +// Copyright 2017-present The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "os" + + "github.com/alecthomas/chroma" + "github.com/alecthomas/chroma/formatters/html" + "github.com/alecthomas/chroma/styles" + "github.com/spf13/cobra" +) + +type genChromaStyles struct { + style string + highlightStyle string + linesStyle string + cmd *cobra.Command +} + +// TODO(bep) highlight +func createGenChromaStyles() *genChromaStyles { + g := &genChromaStyles{ + cmd: &cobra.Command{ + Use: "chromastyles", + Short: "Generate CSS stylesheet for the Chroma code highlighter", + Long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if pygmentsUseClasses is enabled in config. + +See https://help.farbox.com/pygments.html for preview of available styles`, + }, + } + + g.cmd.RunE = func(cmd *cobra.Command, args []string) error { + return g.generate() + } + + g.cmd.PersistentFlags().StringVar(&g.style, "style", "friendly", "highlighter style (see https://help.farbox.com/pygments.html)") + g.cmd.PersistentFlags().StringVar(&g.highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)") + g.cmd.PersistentFlags().StringVar(&g.linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)") + + return g +} + +func (g *genChromaStyles) generate() error { + builder := styles.Get(g.style).Builder() + if g.highlightStyle != "" { + builder.Add(chroma.LineHighlight, g.highlightStyle) + } + if g.linesStyle != "" { + builder.Add(chroma.LineNumbers, g.linesStyle) + } + style, err := builder.Build() + if err != nil { + return err + } + formatter := html.New(html.WithClasses()) + formatter.WriteCSS(os.Stdout, style) + return nil +} diff --git a/commands/hugo.go b/commands/hugo.go index d8527a3aa..388f55db9 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -199,6 +199,7 @@ func AddCommands() { genCmd.AddCommand(gendocCmd) genCmd.AddCommand(genmanCmd) genCmd.AddCommand(createGenDocsHelper().cmd) + genCmd.AddCommand(createGenChromaStyles().cmd) } // initHugoBuilderFlags initializes all common flags, typically used by the diff --git a/deps/deps.go b/deps/deps.go index ed073c5d3..d8ba3313e 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -114,6 +114,11 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } + contentSpec, err := helpers.NewContentSpec(cfg.Language) + if err != nil { + return nil, err + } + d := &Deps{ Fs: fs, Log: logger, @@ -121,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) { translationProvider: cfg.TranslationProvider, WithTemplate: cfg.WithTemplate, PathSpec: ps, - ContentSpec: helpers.NewContentSpec(cfg.Language), + ContentSpec: contentSpec, Cfg: cfg.Language, Language: cfg.Language, } @@ -139,7 +144,11 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) { return nil, err } - d.ContentSpec = helpers.NewContentSpec(l) + d.ContentSpec, err = helpers.NewContentSpec(l) + if err != nil { + return nil, err + } + d.Cfg = l d.Language = l diff --git a/helpers/content.go b/helpers/content.go index 3c81fcd31..7f5975869 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -48,19 +48,49 @@ type ContentSpec struct { footnoteAnchorPrefix string footnoteReturnLinkContents string + Highlight func(code, lang, optsStr string) (string, error) + defatultPygmentsOpts map[string]string + cfg config.Provider } // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider) *ContentSpec { - return &ContentSpec{ +func NewContentSpec(cfg config.Provider) (*ContentSpec, error) { + spec := &ContentSpec{ blackfriday: cfg.GetStringMap("blackfriday"), footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"), footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"), cfg: cfg, } + + // Highlighting setup + options, err := parseDefaultPygmentsOpts(cfg) + if err != nil { + return nil, err + } + spec.defatultPygmentsOpts = options + + // Use the Pygmentize on path if present + useClassic := false + h := newHiglighters(spec) + + if cfg.GetBool("pygmentsUseClassic") { + if !hasPygments() { + jww.WARN.Println("Highlighting with pygmentsUseClassic set requires Pygments to be installed and in the path") + } else { + useClassic = true + } + } + + if useClassic { + spec.Highlight = h.pygmentsHighlight + } else { + spec.Highlight = h.chromaHighlight + } + + return spec, nil } // Blackfriday holds configuration values for Blackfriday rendering. @@ -198,7 +228,7 @@ func BytesToHTML(b []byte) template.HTML { } // getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration. -func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer { +func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer { renderParameters := blackfriday.HtmlRendererParameters{ FootnoteAnchorPrefix: c.footnoteAnchorPrefix, FootnoteReturnLinkContents: c.footnoteReturnLinkContents, @@ -248,6 +278,7 @@ func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) bl } return &HugoHTMLRenderer{ + cs: c, RenderingContext: ctx, Renderer: blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), } @@ -299,7 +330,7 @@ func (c ContentSpec) markdownRender(ctx *RenderingContext) []byte { } // getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration. -func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer { +func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer { renderParameters := mmark.HtmlRendererParameters{ FootnoteAnchorPrefix: c.footnoteAnchorPrefix, FootnoteReturnLinkContents: c.footnoteReturnLinkContents, @@ -320,8 +351,9 @@ func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContex htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS return &HugoMmarkHTMLRenderer{ - mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), - c.cfg, + cs: c, + Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters), + Cfg: c.cfg, } } diff --git a/helpers/content_renderer.go b/helpers/content_renderer.go index 63be58104..9026a683b 100644 --- a/helpers/content_renderer.go +++ b/helpers/content_renderer.go @@ -16,6 +16,7 @@ package helpers import ( "bytes" "html" + "strings" "github.com/gohugoio/hugo/config" "github.com/miekg/mmark" @@ -25,6 +26,7 @@ import ( // HugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html // Enabling Hugo to customise the rendering experience type HugoHTMLRenderer struct { + cs *ContentSpec *RenderingContext blackfriday.Renderer } @@ -34,8 +36,9 @@ type HugoHTMLRenderer struct { func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) { if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) { opts := r.Cfg.GetString("pygmentsOptions") - str := html.UnescapeString(string(text)) - out.WriteString(Highlight(r.RenderingContext.Cfg, str, lang, opts)) + str := strings.Trim(html.UnescapeString(string(text)), "\n\r") + highlighted, _ := r.cs.Highlight(str, lang, opts) + out.WriteString(highlighted) } else { r.Renderer.BlockCode(out, text, lang) } @@ -88,6 +91,7 @@ func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int) // HugoMmarkHTMLRenderer wraps a mmark.Renderer, typically a mmark.html, // enabling Hugo to customise the rendering experience. type HugoMmarkHTMLRenderer struct { + cs *ContentSpec mmark.Renderer Cfg config.Provider } @@ -96,8 +100,9 @@ type HugoMmarkHTMLRenderer struct { // Pygments is used if it is setup to handle code fences. func (r *HugoMmarkHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) { if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) { - str := html.UnescapeString(string(text)) - out.WriteString(Highlight(r.Cfg, str, lang, "")) + str := strings.Trim(html.UnescapeString(string(text)), "\n\r") + highlighted, _ := r.cs.Highlight(str, lang, "") + out.WriteString(highlighted) } else { r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts) } diff --git a/helpers/content_renderer_test.go b/helpers/content_renderer_test.go index 3bd038547..698e3a151 100644 --- a/helpers/content_renderer_test.go +++ b/helpers/content_renderer_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/spf13/viper" + "github.com/stretchr/testify/require" ) // Renders a codeblock using Blackfriday @@ -42,11 +43,7 @@ func (c ContentSpec) renderWithMmark(input string) string { } func TestCodeFence(t *testing.T) { - - if !HasPygments() { - t.Skip("Skipping Pygments test as Pygments is not installed or available.") - return - } + assert := require.New(t) type test struct { enabled bool @@ -55,36 +52,39 @@ func TestCodeFence(t *testing.T) { // Pygments 2.0 and 2.1 have slightly different outputs so only do partial matching data := []test{ - {true, "", `(?s)^
.*?
\n$`}, - {false, "", `(?s)^
.*?
\n$`}, + {true, "", `(?s)^
\n?.*?\n?
\n?$`}, + {false, "", `(?s)^.*?\n$`}, } - for i, d := range data { - v := viper.New() + for _, useClassic := range []bool{false, true} { + for i, d := range data { + v := viper.New() + v.Set("pygmentsStyle", "monokai") + v.Set("pygmentsUseClasses", true) + v.Set("pygmentsCodeFences", d.enabled) + v.Set("pygmentsUseClassic", useClassic) - v.Set("pygmentsStyle", "monokai") - v.Set("pygmentsUseClasses", true) - v.Set("pygmentsCodeFences", d.enabled) + c, err := NewContentSpec(v) + assert.NoError(err) - c := NewContentSpec(v) + result := c.render(d.input) - result := c.render(d.input) + expectedRe, err := regexp.Compile(d.expected) - expectedRe, err := regexp.Compile(d.expected) + if err != nil { + t.Fatal("Invalid regexp", err) + } + matched := expectedRe.MatchString(result) - if err != nil { - t.Fatal("Invalid regexp", err) - } - matched := expectedRe.MatchString(result) + if !matched { + t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result) + } - if !matched { - t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result) - } - - result = c.renderWithMmark(d.input) - matched = expectedRe.MatchString(result) - if !matched { - t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result) + result = c.renderWithMmark(d.input) + matched = expectedRe.MatchString(result) + if !matched { + t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result) + } } } } diff --git a/helpers/pygments.go b/helpers/pygments.go index 60f62a88f..9253445e7 100644 --- a/helpers/pygments.go +++ b/helpers/pygments.go @@ -21,9 +21,18 @@ import ( "io/ioutil" "os/exec" "path/filepath" + "regexp" "sort" + "strconv" "strings" + "github.com/alecthomas/chroma" + "github.com/alecthomas/chroma/formatters" + "github.com/alecthomas/chroma/formatters/html" + "github.com/alecthomas/chroma/lexers" + "github.com/alecthomas/chroma/styles" + bp "github.com/gohugoio/hugo/bufferpool" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" jww "github.com/spf13/jwalterweatherman" @@ -31,27 +40,62 @@ import ( const pygmentsBin = "pygmentize" -// HasPygments checks to see if Pygments is installed and available +// TODO(bep) document chroma -s perldoc --html --html-styles +// hasPygments checks to see if Pygments is installed and available // on the system. -func HasPygments() bool { +func hasPygments() bool { if _, err := exec.LookPath(pygmentsBin); err != nil { return false } return true } -// Highlight takes some code and returns highlighted code. -func Highlight(cfg config.Provider, code, lang, optsStr string) string { - if !HasPygments() { - jww.WARN.Println("Highlighting requires Pygments to be installed and in the path") - return code +type highlighters struct { + cs *ContentSpec + ignoreCache bool + cacheDir string +} + +func newHiglighters(cs *ContentSpec) highlighters { + return highlighters{cs: cs, ignoreCache: cs.cfg.GetBool("ignoreCache"), cacheDir: cs.cfg.GetString("cacheDir")} +} + +func (h highlighters) chromaHighlight(code, lang, optsStr string) (string, error) { + opts, err := h.cs.parsePygmentsOpts(optsStr) + if err != nil { + jww.ERROR.Print(err.Error()) + return code, err } - options, err := parsePygmentsOpts(cfg, optsStr) + style, found := opts["style"] + if !found || style == "" { + style = "friendly" + } + + f, err := h.cs.chromaFormatterFromOptions(opts) + if err != nil { + jww.ERROR.Print(err.Error()) + return code, err + } + + b := bp.GetBuffer() + defer bp.PutBuffer(b) + + err = chromaHighlight(b, code, lang, style, f) + if err != nil { + jww.ERROR.Print(err.Error()) + return code, err + } + + return h.injectCodeTag(`
`+b.String()+"
", lang), nil +} + +func (h highlighters) pygmentsHighlight(code, lang, optsStr string) (string, error) { + options, err := h.cs.createPygmentsOptionsString(optsStr) if err != nil { jww.ERROR.Print(err.Error()) - return code + return code, nil } // Try to read from cache first @@ -62,32 +106,30 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string { fs := hugofs.Os - ignoreCache := cfg.GetBool("ignoreCache") - cacheDir := cfg.GetString("cacheDir") var cachefile string - if !ignoreCache && cacheDir != "" { - cachefile = filepath.Join(cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil))) + if !h.ignoreCache && h.cacheDir != "" { + cachefile = filepath.Join(h.cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil))) exists, err := Exists(cachefile, fs) if err != nil { jww.ERROR.Print(err.Error()) - return code + return code, nil } if exists { f, err := fs.Open(cachefile) if err != nil { jww.ERROR.Print(err.Error()) - return code + return code, nil } s, err := ioutil.ReadAll(f) if err != nil { jww.ERROR.Print(err.Error()) - return code + return code, nil } - return string(s) + return string(s), nil } } @@ -109,26 +151,58 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string { if err := cmd.Run(); err != nil { jww.ERROR.Print(stderr.String()) - return code + return code, err } str := string(normalizeExternalHelperLineFeeds([]byte(out.String()))) - // inject code tag into Pygments output - if lang != "" && strings.Contains(str, "
") {
-		codeTag := fmt.Sprintf(`
`, lang, lang)
-		str = strings.Replace(str, "
", codeTag, 1)
-		str = strings.Replace(str, "
", "
", 1) - } + str = h.injectCodeTag(str, lang) - if !ignoreCache && cachefile != "" { + if !h.ignoreCache && cachefile != "" { // Write cache file if err := WriteToDisk(cachefile, strings.NewReader(str), fs); err != nil { jww.ERROR.Print(stderr.String()) } } - return str + return str, nil +} + +var preRe = regexp.MustCompile(`(?s)(.*?)(.*?)(
)`) + +func (h highlighters) injectCodeTag(code, lang string) string { + if lang == "" { + return code + } + codeTag := fmt.Sprintf(``, lang, lang) + return preRe.ReplaceAllString(code, fmt.Sprintf("$1%s$2$3", codeTag)) +} + +func chromaHighlight(w io.Writer, source, lexer, style string, f chroma.Formatter) error { + l := lexers.Get(lexer) + if l == nil { + l = lexers.Analyse(source) + } + if l == nil { + l = lexers.Fallback + } + l = chroma.Coalesce(l) + + if f == nil { + f = formatters.Fallback + } + + s := styles.Get(style) + if s == nil { + s = styles.Fallback + } + + it, err := l.Tokenise(nil, source) + if err != nil { + return err + } + + return f.Format(w, s, it) } var pygmentsKeywords = make(map[string]bool) @@ -158,23 +232,30 @@ func init() { pygmentsKeywords["startinline"] = true } -func parseOptions(options map[string]string, in string) error { +func parseOptions(defaults map[string]string, in string) (map[string]string, error) { in = strings.Trim(in, " ") + opts := make(map[string]string) + + if defaults != nil { + for k, v := range defaults { + opts[k] = v + } + } if in == "" { - return nil + return opts, nil } for _, v := range strings.Split(in, ",") { keyVal := strings.Split(v, "=") key := strings.ToLower(strings.Trim(keyVal[0], " ")) if len(keyVal) != 2 || !pygmentsKeywords[key] { - return fmt.Errorf("invalid Pygments option: %s", key) + return opts, fmt.Errorf("invalid Pygments option: %s", key) } - options[key] = keyVal[1] + opts[key] = keyVal[1] } - return nil + return opts, nil } func createOptionsString(options map[string]string) string { @@ -196,8 +277,7 @@ func createOptionsString(options map[string]string) string { } func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) { - options := make(map[string]string) - err := parseOptions(options, cfg.GetString("pygmentsOptions")) + options, err := parseOptions(nil, cfg.GetString("pygmentsOptions")) if err != nil { return nil, err } @@ -222,16 +302,100 @@ func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) { return options, nil } -func parsePygmentsOpts(cfg config.Provider, in string) (string, error) { - options, err := parseDefaultPygmentsOpts(cfg) - if err != nil { - return "", err +func (cs *ContentSpec) chromaFormatterFromOptions(pygmentsOpts map[string]string) (chroma.Formatter, error) { + var options = []html.Option{html.TabWidth(4)} + + if pygmentsOpts["noclasses"] == "false" { + options = append(options, html.WithClasses()) } - err = parseOptions(options, in) - if err != nil { - return "", err + if pygmentsOpts["linenos"] != "" { + options = append(options, html.WithLineNumbers()) } - return createOptionsString(options), nil + startLineStr := pygmentsOpts["linenostart"] + var startLine = 1 + if startLineStr != "" { + + line, err := strconv.Atoi(strings.TrimSpace(startLineStr)) + if err == nil { + startLine = line + options = append(options, html.BaseLineNumber(startLine)) + } + } + + hlLines := pygmentsOpts["hl_lines"] + + if hlLines != "" { + ranges, err := hlLinesToRanges(startLine, hlLines) + + if err == nil { + options = append(options, html.HighlightLines(ranges)) + } + } + + return html.New(options...), nil +} + +func (cs *ContentSpec) parsePygmentsOpts(in string) (map[string]string, error) { + opts, err := parseOptions(cs.defatultPygmentsOpts, in) + if err != nil { + return nil, err + } + return opts, nil + +} + +func (cs *ContentSpec) createPygmentsOptionsString(in string) (string, error) { + opts, err := cs.parsePygmentsOpts(in) + if err != nil { + return "", err + } + return createOptionsString(opts), nil +} + +// startLine compansates for https://github.com/alecthomas/chroma/issues/30 +func hlLinesToRanges(startLine int, s string) ([][2]int, error) { + var ranges [][2]int + s = strings.TrimSpace(s) + + if s == "" { + return ranges, nil + } + + // Variants: + // 1 2 3 4 + // 1-2 3-4 + // 1-2 3 + // 1 3-4 + // 1 3-4 + fields := strings.Split(s, " ") + for _, field := range fields { + field = strings.TrimSpace(field) + if field == "" { + continue + } + numbers := strings.Split(field, "-") + var r [2]int + first, err := strconv.Atoi(numbers[0]) + if err != nil { + return ranges, err + } + first = first + startLine - 1 + r[0] = first + if len(numbers) > 1 { + second, err := strconv.Atoi(numbers[1]) + if err != nil { + return ranges, err + } + second = second + startLine - 1 + r[1] = second + } else { + r[1] = first + } + + ranges = append(ranges, r) + } + return ranges, nil + } diff --git a/helpers/pygments_test.go b/helpers/pygments_test.go index 1fce17859..ee8076c71 100644 --- a/helpers/pygments_test.go +++ b/helpers/pygments_test.go @@ -14,12 +14,19 @@ package helpers import ( + "fmt" + "reflect" "testing" + "github.com/alecthomas/chroma/formatters/html" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" ) func TestParsePygmentsArgs(t *testing.T) { + assert := require.New(t) + for i, this := range []struct { in string pygmentsStyle string @@ -38,8 +45,10 @@ func TestParsePygmentsArgs(t *testing.T) { v := viper.New() v.Set("pygmentsStyle", this.pygmentsStyle) v.Set("pygmentsUseClasses", this.pygmentsUseClasses) + spec, err := NewContentSpec(v) + assert.NoError(err) - result1, err := parsePygmentsOpts(v, this.in) + result1, err := spec.createPygmentsOptionsString(this.in) if b, ok := this.expect1.(bool); ok && !b { if err == nil { t.Errorf("[%d] parsePygmentArgs didn't return an expected error", i) @@ -58,6 +67,8 @@ func TestParsePygmentsArgs(t *testing.T) { } func TestParseDefaultPygmentsArgs(t *testing.T) { + assert := require.New(t) + expect := "encoding=utf8,noclasses=false,style=foo" for i, this := range []struct { @@ -83,7 +94,10 @@ func TestParseDefaultPygmentsArgs(t *testing.T) { v.Set("pygmentsUseClasses", b) } - result, err := parsePygmentsOpts(v, this.in) + spec, err := NewContentSpec(v) + assert.NoError(err) + + result, err := spec.createPygmentsOptionsString(this.in) if err != nil { t.Errorf("[%d] parsePygmentArgs failed: %s", i, err) continue @@ -93,3 +107,186 @@ func TestParseDefaultPygmentsArgs(t *testing.T) { } } } + +type chromaInfo struct { + classes bool + lineNumbers bool + highlightRangesLen int + highlightRangesStr string + baseLineNumber int +} + +func formatterChromaInfo(f *html.Formatter) chromaInfo { + v := reflect.ValueOf(f).Elem() + c := chromaInfo{} + // Hack: + c.classes = v.FieldByName("classes").Bool() + c.lineNumbers = v.FieldByName("lineNumbers").Bool() + c.baseLineNumber = int(v.FieldByName("baseLineNumber").Int()) + vv := v.FieldByName("highlightRanges") + c.highlightRangesLen = vv.Len() + c.highlightRangesStr = fmt.Sprint(vv) + + return c +} + +func TestChromaHTMLHighlight(t *testing.T) { + assert := require.New(t) + + v := viper.New() + v.Set("pygmentsUseClasses", true) + spec, err := NewContentSpec(v) + assert.NoError(err) + + result, err := spec.Highlight(`echo "Hello"`, "bash", "") + assert.NoError(err) + + assert.Contains(result, `echo "Hello"`) + +} + +func TestChromaHTMLFormatterFromOptions(t *testing.T) { + assert := require.New(t) + + for i, this := range []struct { + in string + pygmentsStyle interface{} + pygmentsUseClasses interface{} + pygmentsOptions string + assert func(c chromaInfo) + }{ + {"", "monokai", true, "style=manni,noclasses=true", func(c chromaInfo) { + assert.True(c.classes) + assert.False(c.lineNumbers) + assert.Equal(0, c.highlightRangesLen) + + }}, + {"", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) { + assert.True(c.classes) + }}, + {"linenos=sure,hl_lines=1 2 3", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) { + assert.True(c.classes) + assert.True(c.lineNumbers) + assert.Equal(3, c.highlightRangesLen) + assert.Equal("[[1 1] [2 2] [3 3]]", c.highlightRangesStr) + assert.Equal(1, c.baseLineNumber) + }}, + {"linenos=sure,hl_lines=1,linenostart=4", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) { + assert.True(c.classes) + assert.True(c.lineNumbers) + assert.Equal(1, c.highlightRangesLen) + // This compansates for https://github.com/alecthomas/chroma/issues/30 + assert.Equal("[[4 4]]", c.highlightRangesStr) + assert.Equal(4, c.baseLineNumber) + }}, + {"style=monokai,noclasses=false", nil, nil, "style=manni,noclasses=true", func(c chromaInfo) { + assert.True(c.classes) + }}, + {"style=monokai,noclasses=true", "friendly", false, "style=manni,noclasses=false", func(c chromaInfo) { + assert.False(c.classes) + }}, + } { + v := viper.New() + + v.Set("pygmentsOptions", this.pygmentsOptions) + + if s, ok := this.pygmentsStyle.(string); ok { + v.Set("pygmentsStyle", s) + } + + if b, ok := this.pygmentsUseClasses.(bool); ok { + v.Set("pygmentsUseClasses", b) + } + + spec, err := NewContentSpec(v) + assert.NoError(err) + + opts, err := spec.parsePygmentsOpts(this.in) + if err != nil { + t.Fatalf("[%d] parsePygmentsOpts failed: %s", i, err) + } + + chromaFormatter, err := spec.chromaFormatterFromOptions(opts) + if err != nil { + t.Fatalf("[%d] chromaFormatterFromOptions failed: %s", i, err) + } + + this.assert(formatterChromaInfo(chromaFormatter.(*html.Formatter))) + } +} + +func TestHlLinesToRanges(t *testing.T) { + var zero [][2]int + + for _, this := range []struct { + in string + startLine int + expected interface{} + }{ + {"", 1, zero}, + {"1 4", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}}, + {"1 4", 2, [][2]int{[2]int{2, 2}, [2]int{5, 5}}}, + {"1-4 5-8", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}}, + {" 1 4 ", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}}, + {"1-4 5-8 ", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}}, + {"1-4 5", 1, [][2]int{[2]int{1, 4}, [2]int{5, 5}}}, + {"4 5-9", 1, [][2]int{[2]int{4, 4}, [2]int{5, 9}}}, + {" 1 -4 5 - 8 ", 1, true}, + {"a b", 1, true}, + } { + got, err := hlLinesToRanges(this.startLine, this.in) + + if expectErr, ok := this.expected.(bool); ok && expectErr { + if err == nil { + t.Fatal("No error") + } + } else if err != nil { + t.Fatalf("Got error: %s", err) + } else if !reflect.DeepEqual(this.expected, got) { + t.Fatalf("Expected\n%v but got\n%v", this.expected, got) + } + } +} + +func BenchmarkChromaHighlight(b *testing.B) { + assert := require.New(b) + v := viper.New() + + v.Set("pygmentsstyle", "trac") + v.Set("pygmentsuseclasses", false) + v.Set("pygmentsuseclassic", false) + + code := `// GetTitleFunc returns a func that can be used to transform a string to +// title case. +// +// The supported styles are +// +// - "Go" (strings.Title) +// - "AP" (see https://www.apstylebook.com/) +// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html) +// +// If an unknown or empty style is provided, AP style is what you get. +func GetTitleFunc(style string) func(s string) string { + switch strings.ToLower(style) { + case "go": + return strings.Title + case "chicago": + tc := transform.NewTitleConverter(transform.ChicagoStyle) + return tc.Title + default: + tc := transform.NewTitleConverter(transform.APStyle) + return tc.Title + } +} +` + + spec, err := NewContentSpec(v) + assert.NoError(err) + + for i := 0; i < b.N; i++ { + _, err := spec.Highlight(code, "go", "linenos=inline,hl_lines=8 15-17") + if err != nil { + b.Fatal(err) + } + } +} diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 86f141146..518a5bc23 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -34,5 +34,9 @@ func newTestCfg(fs *hugofs.Fs) *viper.Viper { func newTestContentSpec() *ContentSpec { v := viper.New() - return NewContentSpec(v) + spec, err := NewContentSpec(v) + if err != nil { + panic(err) + } + return spec } diff --git a/hugolib/config.go b/hugolib/config.go index dcc56486a..2406ba771 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -16,11 +16,12 @@ package hugolib import ( "fmt" + "io" + "strings" + "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" "github.com/spf13/viper" - "io" - "strings" ) // LoadConfig loads Hugo configuration into a new Viper and then adds @@ -84,9 +85,12 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper. return v, nil } -func loadDefaultSettingsFor(v *viper.Viper) { +func loadDefaultSettingsFor(v *viper.Viper) error { - c := helpers.NewContentSpec(v) + c, err := helpers.NewContentSpec(v) + if err != nil { + return err + } v.SetDefault("cleanDestinationDir", false) v.SetDefault("watch", false) @@ -120,6 +124,7 @@ func loadDefaultSettingsFor(v *viper.Viper) { v.SetDefault("pygmentsStyle", "monokai") v.SetDefault("pygmentsUseClasses", false) v.SetDefault("pygmentsCodeFences", false) + v.SetDefault("pygmentsUseClassic", false) v.SetDefault("pygmentsOptions", "") v.SetDefault("disableLiveReload", false) v.SetDefault("pluralizeListTitles", true) @@ -146,4 +151,6 @@ func loadDefaultSettingsFor(v *viper.Viper) { v.SetDefault("ignoreFiles", make([]string, 0)) v.SetDefault("disableAliases", false) v.SetDefault("debug", false) + + return nil } diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 1c861dc90..6167cded6 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -24,7 +24,6 @@ import ( "github.com/gohugoio/hugo/deps" - "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/tpl" "github.com/stretchr/testify/require" ) @@ -80,22 +79,18 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { func TestShortcodeHighlight(t *testing.T) { t.Parallel() - if !helpers.HasPygments() { - t.Skip("Skip test as Pygments is not installed") - } - for _, this := range []struct { in, expected string }{ {`{{< highlight java >}} void do(); {{< /highlight >}}`, - "(?s)
.*?void do().*?
\n", + `(?s)
}}
 void do();
 {{< /highlight >}}`,
-			"(?s)
.*?void.*?do.*?().*?
\n", + `(?s)
`,
 		},
 	} {
 
diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go
index 485ae4b69..5af4ad774 100644
--- a/hugolib/shortcode_test.go
+++ b/hugolib/shortcode_test.go
@@ -482,7 +482,7 @@ e`,
 		// #2223 pygments
 		{"sect/doc6.md", "\n```bash\nb: {{< b >}} c: {{% c %}}\n```\n",
 			filepath.FromSlash("public/sect/doc6/index.html"),
-			"b: b c: c\n
\n"}, + `b: b c: c`}, // #2249 {"sect/doc7.ad", `_Shortcodes:_ *b: {{< b >}} c: {{% c %}}*`, filepath.FromSlash("public/sect/doc7/index.html"), @@ -561,7 +561,7 @@ tags: } else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() { fmt.Println("Skip Rst test case as no rst2html present.") continue - } else if strings.Contains(test.expected, "code") && !helpers.HasPygments() { + } else if strings.Contains(test.expected, "code") { fmt.Println("Skip Pygments test case as no pygments present.") continue } diff --git a/hugolib/site.go b/hugolib/site.go index e8b2422b1..39908d810 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -291,7 +291,9 @@ func NewSite(cfg deps.DepsCfg) (*Site, error) { // Note: This is mainly used in single site tests. func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { v := viper.New() - loadDefaultSettingsFor(v) + if err := loadDefaultSettingsFor(v); err != nil { + return nil, err + } return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...) } @@ -300,7 +302,9 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) ( // Note: This is mainly used in single site tests. func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { v := viper.New() - loadDefaultSettingsFor(v) + if err := loadDefaultSettingsFor(v); err != nil { + return nil, err + } return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...) } diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index c82d3c3bb..f35e29459 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -776,10 +776,14 @@ type TstX struct { func newDeps(cfg config.Provider) *deps.Deps { l := helpers.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") + cs, err := helpers.NewContentSpec(l) + if err != nil { + panic(err) + } return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), - ContentSpec: helpers.NewContentSpec(l), + ContentSpec: cs, Log: jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime), } } diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index de83f771d..f0b027955 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -166,9 +166,13 @@ func TestScpGetRemoteParallel(t *testing.T) { func newDeps(cfg config.Provider) *deps.Deps { l := helpers.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") + cs, err := helpers.NewContentSpec(l) + if err != nil { + panic(err) + } return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), - ContentSpec: helpers.NewContentSpec(l), + ContentSpec: cs, } } diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index 8d404f5a7..f1ffa77ae 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -55,7 +55,8 @@ func (ns *Namespace) Highlight(s interface{}, lang, opts string) (template.HTML, return "", err } - return template.HTML(helpers.Highlight(ns.deps.Cfg, html.UnescapeString(ss), lang, opts)), nil + highlighted, _ := ns.deps.ContentSpec.Highlight(html.UnescapeString(ss), lang, opts) + return template.HTML(highlighted), nil } // HTMLEscape returns a copy of s with reserved HTML characters escaped. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 5fb80c236..429b206fd 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -226,9 +226,14 @@ func TestPlainify(t *testing.T) { func newDeps(cfg config.Provider) *deps.Deps { l := helpers.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") + cs, err := helpers.NewContentSpec(l) + if err != nil { + panic(err) + } + return &deps.Deps{ Cfg: cfg, Fs: hugofs.NewMem(l), - ContentSpec: helpers.NewContentSpec(l), + ContentSpec: cs, } } diff --git a/vendor/vendor.json b/vendor/vendor.json index daf61f623..f0834cd84 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -20,6 +20,36 @@ "revision": "bbf7a2afc14f93e1e0a5c06df524fbd75e5031e5", "revisionTime": "2017-03-24T14:02:28Z" }, + { + "checksumSHA1": "Aq9XVBGDFH92BXKVPK+rexqDkTo=", + "path": "github.com/alecthomas/chroma", + "revision": "b0295f66bdb7c61d54906003d7649185794e21b4", + "revisionTime": "2017-09-25T05:25:32Z" + }, + { + "checksumSHA1": "Q/9AbXGrFHtlZB6tyoYUq1ipvqU=", + "path": "github.com/alecthomas/chroma/formatters", + "revision": "b0295f66bdb7c61d54906003d7649185794e21b4", + "revisionTime": "2017-09-25T05:25:32Z" + }, + { + "checksumSHA1": "EbtkLGHGij3Q91njJQJeZRKD3OI=", + "path": "github.com/alecthomas/chroma/formatters/html", + "revision": "b0295f66bdb7c61d54906003d7649185794e21b4", + "revisionTime": "2017-09-25T05:25:32Z" + }, + { + "checksumSHA1": "ANyNTHVz5LdPPADsExM5WpBJe4c=", + "path": "github.com/alecthomas/chroma/lexers", + "revision": "1af7e1a0bc5c04ec39b8e6d25d70de8eafcf76ab", + "revisionTime": "2017-09-23T12:45:05Z" + }, + { + "checksumSHA1": "Nm8r5bmokRePD0D7WU+rXYxOO9A=", + "path": "github.com/alecthomas/chroma/styles", + "revision": "b0295f66bdb7c61d54906003d7649185794e21b4", + "revisionTime": "2017-09-25T05:25:32Z" + }, { "checksumSHA1": "7yrV1Gzr1ajco1xJ1gsyqRDTY2U=", "path": "github.com/bep/gitmap", @@ -38,6 +68,12 @@ "revision": "23709d0847197db6021a51fdb193e66e9222d4e7", "revisionTime": "2017-06-03T12:52:39Z" }, + { + "checksumSHA1": "d/czTNq3bacK85PFEKcHvW6aR80=", + "path": "github.com/danwakefield/fnmatch", + "revision": "cbb64ac3d964b81592e64f957ad53df015803288", + "revisionTime": "2016-04-03T17:12:40Z" + }, { "checksumSHA1": "OFu4xJEIjiI8Suu+j/gabfp+y6Q=", "origin": "github.com/stretchr/testify/vendor/github.com/davecgh/go-spew/spew", @@ -51,6 +87,18 @@ "revision": "fb8d9b44afdc258bfff6052d3667521babcb2239", "revisionTime": "2015-12-10T17:00:30Z" }, + { + "checksumSHA1": "6y/Ht8J58EotTDBEIuE3+s4AnL8=", + "path": "github.com/dlclark/regexp2", + "revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957", + "revisionTime": "2017-07-18T21:59:41Z" + }, + { + "checksumSHA1": "k0JXX65FspyueQ8/1i50DGRiCUk=", + "path": "github.com/dlclark/regexp2/syntax", + "revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957", + "revisionTime": "2017-07-18T21:59:41Z" + }, { "checksumSHA1": "248k9rTfZ4kAknuomoKsdBG9zCU=", "path": "github.com/eknkc/amber", @@ -219,12 +267,6 @@ "revision": "3e70a1a463008cea6726380c908b1a6a8bdf7b24", "revisionTime": "2017-05-12T15:20:54Z" }, - { - "checksumSHA1": "F1IYMLBLAZaTOWnmXsgaxTGvrWI=", - "path": "github.com/pelletier/go-buffruneio", - "revision": "c37440a7cf42ac63b919c752ca73a85067e05992", - "revisionTime": "2017-02-27T22:03:11Z" - }, { "checksumSHA1": "zZg0J0MqvnqXVYo644QDvnUinrc=", "path": "github.com/pelletier/go-toml",