phoenix/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-07-02 14:31:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use Chroma as new default syntax highlighter

Browse source 
If you want to use Pygments, set `pygmentsUseClassic=true` in your site config.

Fixes #3888
This commit is contained in:
Bjørn Erik Pedersen 2017-09-25 08:59:02 +02:00 committed by GitHub
parent 81ed564793
commit fb33d8286d
18 changed files with 652 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
commands/genchromastyles.go Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"os"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/styles"
"github.com/spf13/cobra"
)
type genChromaStyles struct {
style string
highlightStyle string
linesStyle string
cmd *cobra.Command
}
// TODO(bep) highlight
func createGenChromaStyles() *genChromaStyles {
g := &genChromaStyles{
cmd: &cobra.Command{
Use: "chromastyles",
Short: "Generate CSS stylesheet for the Chroma code highlighter",
Long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if pygmentsUseClasses is enabled in config.
See https://help.farbox.com/pygments.html for preview of available styles`,
},
}
g.cmd.RunE = func(cmd *cobra.Command, args []string) error {
return g.generate()
}
g.cmd.PersistentFlags().StringVar(&g.style, "style", "friendly", "highlighter style (see https://help.farbox.com/pygments.html)")
g.cmd.PersistentFlags().StringVar(&g.highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)")
g.cmd.PersistentFlags().StringVar(&g.linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)")
return g
}
func (g *genChromaStyles) generate() error {
builder := styles.Get(g.style).Builder()
if g.highlightStyle != "" {
builder.Add(chroma.LineHighlight, g.highlightStyle)
}
if g.linesStyle != "" {
builder.Add(chroma.LineNumbers, g.linesStyle)
}
style, err := builder.Build()
if err != nil {
return err
}
formatter := html.New(html.WithClasses())
formatter.WriteCSS(os.Stdout, style)
return nil
}

1
commands/hugo.go
View file

@ -199,6 +199,7 @@ func AddCommands() {
genCmd.AddCommand(gendocCmd)
genCmd.AddCommand(genmanCmd)
genCmd.AddCommand(createGenDocsHelper().cmd)
genCmd.AddCommand(createGenChromaStyles().cmd)
}
// initHugoBuilderFlags initializes all common flags, typically used by the

13
deps/deps.go vendored
View file

@ -114,6 +114,11 @@ func New(cfg DepsCfg) (*Deps, error) {
return nil, err
}
contentSpec, err := helpers.NewContentSpec(cfg.Language)
if err != nil {
return nil, err
}
d := &Deps{
Fs: fs,
Log: logger,
@ -121,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) {
translationProvider: cfg.TranslationProvider,
WithTemplate: cfg.WithTemplate,
PathSpec: ps,
ContentSpec: helpers.NewContentSpec(cfg.Language),
ContentSpec: contentSpec,
Cfg: cfg.Language,
Language: cfg.Language,
}
@ -139,7 +144,11 @@ func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
return nil, err
}
d.ContentSpec = helpers.NewContentSpec(l)
d.ContentSpec, err = helpers.NewContentSpec(l)
if err != nil {
return nil, err
}
d.Cfg = l
d.Language = l

44
helpers/content.go
View file

@ -48,19 +48,49 @@ type ContentSpec struct {
footnoteAnchorPrefix string
footnoteReturnLinkContents string
Highlight func(code, lang, optsStr string) (string, error)
defatultPygmentsOpts map[string]string
cfg config.Provider
}
// NewContentSpec returns a ContentSpec initialized
// with the appropriate fields from the given config.Provider.
func NewContentSpec(cfg config.Provider) *ContentSpec {
return &ContentSpec{
func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
spec := &ContentSpec{
blackfriday: cfg.GetStringMap("blackfriday"),
footnoteAnchorPrefix: cfg.GetString("footnoteAnchorPrefix"),
footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
cfg: cfg,
}
// Highlighting setup
options, err := parseDefaultPygmentsOpts(cfg)
if err != nil {
return nil, err
}
spec.defatultPygmentsOpts = options
// Use the Pygmentize on path if present
useClassic := false
h := newHiglighters(spec)
if cfg.GetBool("pygmentsUseClassic") {
if !hasPygments() {
jww.WARN.Println("Highlighting with pygmentsUseClassic set requires Pygments to be installed and in the path")
} else {
useClassic = true
}
}
if useClassic {
spec.Highlight = h.pygmentsHighlight
} else {
spec.Highlight = h.chromaHighlight
}
return spec, nil
}
// Blackfriday holds configuration values for Blackfriday rendering.
@ -198,7 +228,7 @@ func BytesToHTML(b []byte) template.HTML {
}
// getHTMLRenderer creates a new Blackfriday HTML Renderer with the given configuration.
func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
func (c *ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) blackfriday.Renderer {
renderParameters := blackfriday.HtmlRendererParameters{
FootnoteAnchorPrefix: c.footnoteAnchorPrefix,
FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
@ -248,6 +278,7 @@ func (c ContentSpec) getHTMLRenderer(defaultFlags int, ctx *RenderingContext) bl
}
return &HugoHTMLRenderer{
cs: c,
RenderingContext: ctx,
Renderer: blackfriday.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
}
@ -299,7 +330,7 @@ func (c ContentSpec) markdownRender(ctx *RenderingContext) []byte {
}
// getMmarkHTMLRenderer creates a new mmark HTML Renderer with the given configuration.
func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
func (c *ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContext) mmark.Renderer {
renderParameters := mmark.HtmlRendererParameters{
FootnoteAnchorPrefix: c.footnoteAnchorPrefix,
FootnoteReturnLinkContents: c.footnoteReturnLinkContents,
@ -320,8 +351,9 @@ func (c ContentSpec) getMmarkHTMLRenderer(defaultFlags int, ctx *RenderingContex
htmlFlags |= mmark.HTML_FOOTNOTE_RETURN_LINKS
return &HugoMmarkHTMLRenderer{
mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
c.cfg,
cs: c,
Renderer: mmark.HtmlRendererWithParameters(htmlFlags, "", "", renderParameters),
Cfg: c.cfg,
}
}

13
helpers/content_renderer.go
View file

@ -16,6 +16,7 @@ package helpers
import (
"bytes"
"html"
"strings"
"github.com/gohugoio/hugo/config"
"github.com/miekg/mmark"
@ -25,6 +26,7 @@ import (
// HugoHTMLRenderer wraps a blackfriday.Renderer, typically a blackfriday.Html
// Enabling Hugo to customise the rendering experience
type HugoHTMLRenderer struct {
cs *ContentSpec
*RenderingContext
blackfriday.Renderer
}
@ -34,8 +36,9 @@ type HugoHTMLRenderer struct {
func (r *HugoHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
opts := r.Cfg.GetString("pygmentsOptions")
str := html.UnescapeString(string(text))
out.WriteString(Highlight(r.RenderingContext.Cfg, str, lang, opts))
str := strings.Trim(html.UnescapeString(string(text)), "\n\r")
highlighted, _ := r.cs.Highlight(str, lang, opts)
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang)
}
@ -88,6 +91,7 @@ func (r *HugoHTMLRenderer) List(out *bytes.Buffer, text func() bool, flags int)
// HugoMmarkHTMLRenderer wraps a mmark.Renderer, typically a mmark.html,
// enabling Hugo to customise the rendering experience.
type HugoMmarkHTMLRenderer struct {
cs *ContentSpec
mmark.Renderer
Cfg config.Provider
}
@ -96,8 +100,9 @@ type HugoMmarkHTMLRenderer struct {
// Pygments is used if it is setup to handle code fences.
func (r *HugoMmarkHTMLRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string, caption []byte, subfigure bool, callouts bool) {
if r.Cfg.GetBool("pygmentsCodeFences") && (lang != "" || r.Cfg.GetBool("pygmentsCodeFencesGuessSyntax")) {
str := html.UnescapeString(string(text))
out.WriteString(Highlight(r.Cfg, str, lang, ""))
str := strings.Trim(html.UnescapeString(string(text)), "\n\r")
highlighted, _ := r.cs.Highlight(str, lang, "")
out.WriteString(highlighted)
} else {
r.Renderer.BlockCode(out, text, lang, caption, subfigure, callouts)
}

54
helpers/content_renderer_test.go
View file

@ -19,6 +19,7 @@ import (
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
// Renders a codeblock using Blackfriday
@ -42,11 +43,7 @@ func (c ContentSpec) renderWithMmark(input string) string {
}
func TestCodeFence(t *testing.T) {
if !HasPygments() {
t.Skip("Skipping Pygments test as Pygments is not installed or available.")
return
}
assert := require.New(t)
type test struct {
enabled bool
@ -55,36 +52,39 @@ func TestCodeFence(t *testing.T) {
// Pygments 2.0 and 2.1 have slightly different outputs so only do partial matching
data := []test{
{true, "<html></html>", `(?s)^<div class="highlight"><pre><code class="language-html" data-lang="html">.*?</code></pre></div>\n$`},
{false, "<html></html>", `(?s)^<pre><code class="language-html">.*?</code></pre>\n$`},
{true, "<html></html>", `(?s)^<div class="highlight">\n?<pre.*><code class="language-html" data-lang="html">.*?</code></pre>\n?</div>\n?$`},
{false, "<html></html>", `(?s)^<pre.*><code class="language-html">.*?</code></pre>\n$`},
}
for i, d := range data {
v := viper.New()
for _, useClassic := range []bool{false, true} {
for i, d := range data {
v := viper.New()
v.Set("pygmentsStyle", "monokai")
v.Set("pygmentsUseClasses", true)
v.Set("pygmentsCodeFences", d.enabled)
v.Set("pygmentsUseClassic", useClassic)
v.Set("pygmentsStyle", "monokai")
v.Set("pygmentsUseClasses", true)
v.Set("pygmentsCodeFences", d.enabled)
c, err := NewContentSpec(v)
assert.NoError(err)
c := NewContentSpec(v)
result := c.render(d.input)
result := c.render(d.input)
expectedRe, err := regexp.Compile(d.expected)
expectedRe, err := regexp.Compile(d.expected)
if err != nil {
t.Fatal("Invalid regexp", err)
}
matched := expectedRe.MatchString(result)
if err != nil {
t.Fatal("Invalid regexp", err)
}
matched := expectedRe.MatchString(result)
if !matched {
t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
}
if !matched {
t.Errorf("Test %d failed. BlackFriday enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
}
result = c.renderWithMmark(d.input)
matched = expectedRe.MatchString(result)
if !matched {
t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
result = c.renderWithMmark(d.input)
matched = expectedRe.MatchString(result)
if !matched {
t.Errorf("Test %d failed. Mmark enabled:%t, Expected:\n%q got:\n%q", i, d.enabled, d.expected, result)
}
}
}
}

246
helpers/pygments.go
View file

@ -21,9 +21,18 @@ import (
"io/ioutil"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/hugofs"
jww "github.com/spf13/jwalterweatherman"
@ -31,27 +40,62 @@ import (
const pygmentsBin = "pygmentize"
// HasPygments checks to see if Pygments is installed and available
// TODO(bep) document chroma -s perldoc --html --html-styles
// hasPygments checks to see if Pygments is installed and available
// on the system.
func HasPygments() bool {
func hasPygments() bool {
if _, err := exec.LookPath(pygmentsBin); err != nil {
return false
}
return true
}
// Highlight takes some code and returns highlighted code.
func Highlight(cfg config.Provider, code, lang, optsStr string) string {
if !HasPygments() {
jww.WARN.Println("Highlighting requires Pygments to be installed and in the path")
return code
type highlighters struct {
cs *ContentSpec
ignoreCache bool
cacheDir string
}
func newHiglighters(cs *ContentSpec) highlighters {
return highlighters{cs: cs, ignoreCache: cs.cfg.GetBool("ignoreCache"), cacheDir: cs.cfg.GetString("cacheDir")}
}
func (h highlighters) chromaHighlight(code, lang, optsStr string) (string, error) {
opts, err := h.cs.parsePygmentsOpts(optsStr)
if err != nil {
jww.ERROR.Print(err.Error())
return code, err
}
options, err := parsePygmentsOpts(cfg, optsStr)
style, found := opts["style"]
if !found || style == "" {
style = "friendly"
}
f, err := h.cs.chromaFormatterFromOptions(opts)
if err != nil {
jww.ERROR.Print(err.Error())
return code, err
}
b := bp.GetBuffer()
defer bp.PutBuffer(b)
err = chromaHighlight(b, code, lang, style, f)
if err != nil {
jww.ERROR.Print(err.Error())
return code, err
}
return h.injectCodeTag(`<div class="highlight">`+b.String()+"</div>", lang), nil
}
func (h highlighters) pygmentsHighlight(code, lang, optsStr string) (string, error) {
options, err := h.cs.createPygmentsOptionsString(optsStr)
if err != nil {
jww.ERROR.Print(err.Error())
return code
return code, nil
}
// Try to read from cache first
@ -62,32 +106,30 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string {
fs := hugofs.Os
ignoreCache := cfg.GetBool("ignoreCache")
cacheDir := cfg.GetString("cacheDir")
var cachefile string
if !ignoreCache && cacheDir != "" {
cachefile = filepath.Join(cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
if !h.ignoreCache && h.cacheDir != "" {
cachefile = filepath.Join(h.cacheDir, fmt.Sprintf("pygments-%x", hash.Sum(nil)))
exists, err := Exists(cachefile, fs)
if err != nil {
jww.ERROR.Print(err.Error())
return code
return code, nil
}
if exists {
f, err := fs.Open(cachefile)
if err != nil {
jww.ERROR.Print(err.Error())
return code
return code, nil
}
s, err := ioutil.ReadAll(f)
if err != nil {
jww.ERROR.Print(err.Error())
return code
return code, nil
}
return string(s)
return string(s), nil
}
}
@ -109,26 +151,58 @@ func Highlight(cfg config.Provider, code, lang, optsStr string) string {
if err := cmd.Run(); err != nil {
jww.ERROR.Print(stderr.String())
return code
return code, err
}
str := string(normalizeExternalHelperLineFeeds([]byte(out.String())))
// inject code tag into Pygments output
if lang != "" && strings.Contains(str, "<pre>") {
codeTag := fmt.Sprintf(`<pre><code class="language-%s" data-lang="%s">`, lang, lang)
str = strings.Replace(str, "<pre>", codeTag, 1)
str = strings.Replace(str, "</pre>", "</code></pre>", 1)
}
str = h.injectCodeTag(str, lang)
if !ignoreCache && cachefile != "" {
if !h.ignoreCache && cachefile != "" {
// Write cache file
if err := WriteToDisk(cachefile, strings.NewReader(str), fs); err != nil {
jww.ERROR.Print(stderr.String())
}
}
return str
return str, nil
}
var preRe = regexp.MustCompile(`(?s)(.*?<pre.*?>)(.*?)(</pre>)`)
func (h highlighters) injectCodeTag(code, lang string) string {
if lang == "" {
return code
}
codeTag := fmt.Sprintf(`<code class="language-%s" data-lang="%s">`, lang, lang)
return preRe.ReplaceAllString(code, fmt.Sprintf("$1%s$2</code>$3", codeTag))
}
func chromaHighlight(w io.Writer, source, lexer, style string, f chroma.Formatter) error {
l := lexers.Get(lexer)
if l == nil {
l = lexers.Analyse(source)
}
if l == nil {
l = lexers.Fallback
}
l = chroma.Coalesce(l)
if f == nil {
f = formatters.Fallback
}
s := styles.Get(style)
if s == nil {
s = styles.Fallback
}
it, err := l.Tokenise(nil, source)
if err != nil {
return err
}
return f.Format(w, s, it)
}
var pygmentsKeywords = make(map[string]bool)
@ -158,23 +232,30 @@ func init() {
pygmentsKeywords["startinline"] = true
}
func parseOptions(options map[string]string, in string) error {
func parseOptions(defaults map[string]string, in string) (map[string]string, error) {
in = strings.Trim(in, " ")
opts := make(map[string]string)
if defaults != nil {
for k, v := range defaults {
opts[k] = v
}
}
if in == "" {
return nil
return opts, nil
}
for _, v := range strings.Split(in, ",") {
keyVal := strings.Split(v, "=")
key := strings.ToLower(strings.Trim(keyVal[0], " "))
if len(keyVal) != 2 || !pygmentsKeywords[key] {
return fmt.Errorf("invalid Pygments option: %s", key)
return opts, fmt.Errorf("invalid Pygments option: %s", key)
}
options[key] = keyVal[1]
opts[key] = keyVal[1]
}
return nil
return opts, nil
}
func createOptionsString(options map[string]string) string {
@ -196,8 +277,7 @@ func createOptionsString(options map[string]string) string {
}
func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
options := make(map[string]string)
err := parseOptions(options, cfg.GetString("pygmentsOptions"))
options, err := parseOptions(nil, cfg.GetString("pygmentsOptions"))
if err != nil {
return nil, err
}
@ -222,16 +302,100 @@ func parseDefaultPygmentsOpts(cfg config.Provider) (map[string]string, error) {
return options, nil
}
func parsePygmentsOpts(cfg config.Provider, in string) (string, error) {
options, err := parseDefaultPygmentsOpts(cfg)
if err != nil {
return "", err
func (cs *ContentSpec) chromaFormatterFromOptions(pygmentsOpts map[string]string) (chroma.Formatter, error) {
var options = []html.Option{html.TabWidth(4)}
if pygmentsOpts["noclasses"] == "false" {
options = append(options, html.WithClasses())
}
err = parseOptions(options, in)
if err != nil {
return "", err
if pygmentsOpts["linenos"] != "" {
options = append(options, html.WithLineNumbers())
}
return createOptionsString(options), nil
startLineStr := pygmentsOpts["linenostart"]
var startLine = 1
if startLineStr != "" {
line, err := strconv.Atoi(strings.TrimSpace(startLineStr))
if err == nil {
startLine = line
options = append(options, html.BaseLineNumber(startLine))
}
}
hlLines := pygmentsOpts["hl_lines"]
if hlLines != "" {
ranges, err := hlLinesToRanges(startLine, hlLines)
if err == nil {
options = append(options, html.HighlightLines(ranges))
}
}
return html.New(options...), nil
}
func (cs *ContentSpec) parsePygmentsOpts(in string) (map[string]string, error) {
opts, err := parseOptions(cs.defatultPygmentsOpts, in)
if err != nil {
return nil, err
}
return opts, nil
}
func (cs *ContentSpec) createPygmentsOptionsString(in string) (string, error) {
opts, err := cs.parsePygmentsOpts(in)
if err != nil {
return "", err
}
return createOptionsString(opts), nil
}
// startLine compansates for https://github.com/alecthomas/chroma/issues/30
func hlLinesToRanges(startLine int, s string) ([][2]int, error) {
var ranges [][2]int
s = strings.TrimSpace(s)
if s == "" {
return ranges, nil
}
// Variants:
// 1 2 3 4
// 1-2 3-4
// 1-2 3
// 1 3-4
// 1 3-4
fields := strings.Split(s, " ")
for _, field := range fields {
field = strings.TrimSpace(field)
if field == "" {
continue
}
numbers := strings.Split(field, "-")
var r [2]int
first, err := strconv.Atoi(numbers[0])
if err != nil {
return ranges, err
}
first = first + startLine - 1
r[0] = first
if len(numbers) > 1 {
second, err := strconv.Atoi(numbers[1])
if err != nil {
return ranges, err
}
second = second + startLine - 1
r[1] = second
} else {
r[1] = first
}
ranges = append(ranges, r)
}
return ranges, nil
}

201
helpers/pygments_test.go
View file

@ -14,12 +14,19 @@
package helpers
import (
"fmt"
"reflect"
"testing"
"github.com/alecthomas/chroma/formatters/html"
"github.com/spf13/viper"
"github.com/stretchr/testify/require"
)
func TestParsePygmentsArgs(t *testing.T) {
assert := require.New(t)
for i, this := range []struct {
in string
pygmentsStyle string
@ -38,8 +45,10 @@ func TestParsePygmentsArgs(t *testing.T) {
v := viper.New()
v.Set("pygmentsStyle", this.pygmentsStyle)
v.Set("pygmentsUseClasses", this.pygmentsUseClasses)
spec, err := NewContentSpec(v)
assert.NoError(err)
result1, err := parsePygmentsOpts(v, this.in)
result1, err := spec.createPygmentsOptionsString(this.in)
if b, ok := this.expect1.(bool); ok && !b {
if err == nil {
t.Errorf("[%d] parsePygmentArgs didn't return an expected error", i)
@ -58,6 +67,8 @@ func TestParsePygmentsArgs(t *testing.T) {
}
func TestParseDefaultPygmentsArgs(t *testing.T) {
assert := require.New(t)
expect := "encoding=utf8,noclasses=false,style=foo"
for i, this := range []struct {
@ -83,7 +94,10 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
v.Set("pygmentsUseClasses", b)
}
result, err := parsePygmentsOpts(v, this.in)
spec, err := NewContentSpec(v)
assert.NoError(err)
result, err := spec.createPygmentsOptionsString(this.in)
if err != nil {
t.Errorf("[%d] parsePygmentArgs failed: %s", i, err)
continue
@ -93,3 +107,186 @@ func TestParseDefaultPygmentsArgs(t *testing.T) {
}
}
}
type chromaInfo struct {
classes bool
lineNumbers bool
highlightRangesLen int
highlightRangesStr string
baseLineNumber int
}
func formatterChromaInfo(f *html.Formatter) chromaInfo {
v := reflect.ValueOf(f).Elem()
c := chromaInfo{}
// Hack:
c.classes = v.FieldByName("classes").Bool()
c.lineNumbers = v.FieldByName("lineNumbers").Bool()
c.baseLineNumber = int(v.FieldByName("baseLineNumber").Int())
vv := v.FieldByName("highlightRanges")
c.highlightRangesLen = vv.Len()
c.highlightRangesStr = fmt.Sprint(vv)
return c
}
func TestChromaHTMLHighlight(t *testing.T) {
assert := require.New(t)
v := viper.New()
v.Set("pygmentsUseClasses", true)
spec, err := NewContentSpec(v)
assert.NoError(err)
result, err := spec.Highlight(`echo "Hello"`, "bash", "")
assert.NoError(err)
assert.Contains(result, `<code class="language-bash" data-lang="bash"><span class="s7d2">echo</span> <span class="sc1c">&#34;Hello&#34;</span></code>`)
}
func TestChromaHTMLFormatterFromOptions(t *testing.T) {
assert := require.New(t)
for i, this := range []struct {
in string
pygmentsStyle interface{}
pygmentsUseClasses interface{}
pygmentsOptions string
assert func(c chromaInfo)
}{
{"", "monokai", true, "style=manni,noclasses=true", func(c chromaInfo) {
assert.True(c.classes)
assert.False(c.lineNumbers)
assert.Equal(0, c.highlightRangesLen)
}},
{"", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
assert.True(c.classes)
}},
{"linenos=sure,hl_lines=1 2 3", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
assert.True(c.classes)
assert.True(c.lineNumbers)
assert.Equal(3, c.highlightRangesLen)
assert.Equal("[[1 1] [2 2] [3 3]]", c.highlightRangesStr)
assert.Equal(1, c.baseLineNumber)
}},
{"linenos=sure,hl_lines=1,linenostart=4", nil, nil, "style=monokai,noclasses=false", func(c chromaInfo) {
assert.True(c.classes)
assert.True(c.lineNumbers)
assert.Equal(1, c.highlightRangesLen)
// This compansates for https://github.com/alecthomas/chroma/issues/30
assert.Equal("[[4 4]]", c.highlightRangesStr)
assert.Equal(4, c.baseLineNumber)
}},
{"style=monokai,noclasses=false", nil, nil, "style=manni,noclasses=true", func(c chromaInfo) {
assert.True(c.classes)
}},
{"style=monokai,noclasses=true", "friendly", false, "style=manni,noclasses=false", func(c chromaInfo) {
assert.False(c.classes)
}},
} {
v := viper.New()
v.Set("pygmentsOptions", this.pygmentsOptions)
if s, ok := this.pygmentsStyle.(string); ok {
v.Set("pygmentsStyle", s)
}
if b, ok := this.pygmentsUseClasses.(bool); ok {
v.Set("pygmentsUseClasses", b)
}
spec, err := NewContentSpec(v)
assert.NoError(err)
opts, err := spec.parsePygmentsOpts(this.in)
if err != nil {
t.Fatalf("[%d] parsePygmentsOpts failed: %s", i, err)
}
chromaFormatter, err := spec.chromaFormatterFromOptions(opts)
if err != nil {
t.Fatalf("[%d] chromaFormatterFromOptions failed: %s", i, err)
}
this.assert(formatterChromaInfo(chromaFormatter.(*html.Formatter)))
}
}
func TestHlLinesToRanges(t *testing.T) {
var zero [][2]int
for _, this := range []struct {
in string
startLine int
expected interface{}
}{
{"", 1, zero},
{"1 4", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}},
{"1 4", 2, [][2]int{[2]int{2, 2}, [2]int{5, 5}}},
{"1-4 5-8", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}},
{" 1 4 ", 1, [][2]int{[2]int{1, 1}, [2]int{4, 4}}},
{"1-4 5-8 ", 1, [][2]int{[2]int{1, 4}, [2]int{5, 8}}},
{"1-4 5", 1, [][2]int{[2]int{1, 4}, [2]int{5, 5}}},
{"4 5-9", 1, [][2]int{[2]int{4, 4}, [2]int{5, 9}}},
{" 1 -4 5 - 8 ", 1, true},
{"a b", 1, true},
} {
got, err := hlLinesToRanges(this.startLine, this.in)
if expectErr, ok := this.expected.(bool); ok && expectErr {
if err == nil {
t.Fatal("No error")
}
} else if err != nil {
t.Fatalf("Got error: %s", err)
} else if !reflect.DeepEqual(this.expected, got) {
t.Fatalf("Expected\n%v but got\n%v", this.expected, got)
}
}
}
func BenchmarkChromaHighlight(b *testing.B) {
assert := require.New(b)
v := viper.New()
v.Set("pygmentsstyle", "trac")
v.Set("pygmentsuseclasses", false)
v.Set("pygmentsuseclassic", false)
code := `// GetTitleFunc returns a func that can be used to transform a string to
// title case.
//
// The supported styles are
//
// - "Go" (strings.Title)
// - "AP" (see https://www.apstylebook.com/)
// - "Chicago" (see http://www.chicagomanualofstyle.org/home.html)
//
// If an unknown or empty style is provided, AP style is what you get.
func GetTitleFunc(style string) func(s string) string {
switch strings.ToLower(style) {
case "go":
return strings.Title
case "chicago":
tc := transform.NewTitleConverter(transform.ChicagoStyle)
return tc.Title
default:
tc := transform.NewTitleConverter(transform.APStyle)
return tc.Title
}
}
`
spec, err := NewContentSpec(v)
assert.NoError(err)
for i := 0; i < b.N; i++ {
_, err := spec.Highlight(code, "go", "linenos=inline,hl_lines=8 15-17")
if err != nil {
b.Fatal(err)
}
}
}

6
helpers/testhelpers_test.go
View file

@ -34,5 +34,9 @@ func newTestCfg(fs *hugofs.Fs) *viper.Viper {
func newTestContentSpec() *ContentSpec {
v := viper.New()
return NewContentSpec(v)
spec, err := NewContentSpec(v)
if err != nil {
panic(err)
}
return spec
}

15
hugolib/config.go
View file

@ -16,11 +16,12 @@ package hugolib
import (
"fmt"
"io"
"strings"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero"
"github.com/spf13/viper"
"io"
"strings"
)
// LoadConfig loads Hugo configuration into a new Viper and then adds
@ -84,9 +85,12 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
return v, nil
}
func loadDefaultSettingsFor(v *viper.Viper) {
func loadDefaultSettingsFor(v *viper.Viper) error {
c := helpers.NewContentSpec(v)
c, err := helpers.NewContentSpec(v)
if err != nil {
return err
}
v.SetDefault("cleanDestinationDir", false)
v.SetDefault("watch", false)
@ -120,6 +124,7 @@ func loadDefaultSettingsFor(v *viper.Viper) {
v.SetDefault("pygmentsStyle", "monokai")
v.SetDefault("pygmentsUseClasses", false)
v.SetDefault("pygmentsCodeFences", false)
v.SetDefault("pygmentsUseClassic", false)
v.SetDefault("pygmentsOptions", "")
v.SetDefault("disableLiveReload", false)
v.SetDefault("pluralizeListTitles", true)
@ -146,4 +151,6 @@ func loadDefaultSettingsFor(v *viper.Viper) {
v.SetDefault("ignoreFiles", make([]string, 0))
v.SetDefault("disableAliases", false)
v.SetDefault("debug", false)
return nil
}

9
hugolib/embedded_shortcodes_test.go
View file

@ -24,7 +24,6 @@ import (
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/tpl"
"github.com/stretchr/testify/require"
)
@ -80,22 +79,18 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
func TestShortcodeHighlight(t *testing.T) {
t.Parallel()
if !helpers.HasPygments() {
t.Skip("Skip test as Pygments is not installed")
}
for _, this := range []struct {
in, expected string
}{
{`{{< highlight java >}}
void do();
{{< /highlight >}}`,
"(?s)<div class=\"highlight\" style=\"background: #ffffff\"><pre style=\"line-height: 125%\">.*?void</span> do().*?</pre></div>\n",
`(?s)<div class="highlight"><pre style="background-color:#fff;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java"`,
},
{`{{< highlight java "style=friendly" >}}
void do();
{{< /highlight >}}`,
"(?s)<div class=\"highlight\" style=\"background: #f0f0f0\"><pre style=\"line-height: 125%\">.*?void</span>.*?do</span>.*?().*?</pre></div>\n",
`(?s)<div class="highlight"><pre style="background-color:#f0f0f0;-moz-tab-size:4;-o-tab-size:4;tab-size:4"><code class="language-java" data-lang="java">`,
},
} {

4
hugolib/shortcode_test.go
View file

@ -482,7 +482,7 @@ e`,
// #2223 pygments
{"sect/doc6.md", "\n```bash\nb: {{< b >}} c: {{% c %}}\n```\n",
filepath.FromSlash("public/sect/doc6/index.html"),
"b: b c: c\n</code></pre></div>\n"},
`<span class="s1f40">b: b c: c`},
// #2249
{"sect/doc7.ad", `_Shortcodes:_ *b: {{< b >}} c: {{% c %}}*`,
filepath.FromSlash("public/sect/doc7/index.html"),
@ -561,7 +561,7 @@ tags:
} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {
fmt.Println("Skip Rst test case as no rst2html present.")
continue
} else if strings.Contains(test.expected, "code") && !helpers.HasPygments() {
} else if strings.Contains(test.expected, "code") {
fmt.Println("Skip Pygments test case as no pygments present.")
continue
}

8
hugolib/site.go
View file

@ -291,7 +291,9 @@ func NewSite(cfg deps.DepsCfg) (*Site, error) {
// Note: This is mainly used in single site tests.
func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
v := viper.New()
loadDefaultSettingsFor(v)
if err := loadDefaultSettingsFor(v); err != nil {
return nil, err
}
return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...)
}
@ -300,7 +302,9 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (
// Note: This is mainly used in single site tests.
func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
v := viper.New()
loadDefaultSettingsFor(v)
if err := loadDefaultSettingsFor(v); err != nil {
return nil, err
}
return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...)
}

6
tpl/collections/collections_test.go
View file

@ -776,10 +776,14 @@ type TstX struct {
func newDeps(cfg config.Provider) *deps.Deps {
l := helpers.NewLanguage("en", cfg)
l.Set("i18nDir", "i18n")
cs, err := helpers.NewContentSpec(l)
if err != nil {
panic(err)
}
return &deps.Deps{
Cfg: cfg,
Fs: hugofs.NewMem(l),
ContentSpec: helpers.NewContentSpec(l),
ContentSpec: cs,
Log: jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime),
}
}

6
tpl/data/resources_test.go
View file

@ -166,9 +166,13 @@ func TestScpGetRemoteParallel(t *testing.T) {
func newDeps(cfg config.Provider) *deps.Deps {
l := helpers.NewLanguage("en", cfg)
l.Set("i18nDir", "i18n")
cs, err := helpers.NewContentSpec(l)
if err != nil {
panic(err)
}
return &deps.Deps{
Cfg: cfg,
Fs: hugofs.NewMem(l),
ContentSpec: helpers.NewContentSpec(l),
ContentSpec: cs,
}
}

3
tpl/transform/transform.go
View file

@ -55,7 +55,8 @@ func (ns *Namespace) Highlight(s interface{}, lang, opts string) (template.HTML,
return "", err
}
return template.HTML(helpers.Highlight(ns.deps.Cfg, html.UnescapeString(ss), lang, opts)), nil
highlighted, _ := ns.deps.ContentSpec.Highlight(html.UnescapeString(ss), lang, opts)
return template.HTML(highlighted), nil
}
// HTMLEscape returns a copy of s with reserved HTML characters escaped.

7
tpl/transform/transform_test.go
View file

@ -226,9 +226,14 @@ func TestPlainify(t *testing.T) {
func newDeps(cfg config.Provider) *deps.Deps {
l := helpers.NewLanguage("en", cfg)
l.Set("i18nDir", "i18n")
cs, err := helpers.NewContentSpec(l)
if err != nil {
panic(err)
}
return &deps.Deps{
Cfg: cfg,
Fs: hugofs.NewMem(l),
ContentSpec: helpers.NewContentSpec(l),
ContentSpec: cs,
}
}

54
vendor/vendor.json vendored
View file

@ -20,6 +20,36 @@
"revision": "bbf7a2afc14f93e1e0a5c06df524fbd75e5031e5",
"revisionTime": "2017-03-24T14:02:28Z"
},
{
"checksumSHA1": "Aq9XVBGDFH92BXKVPK+rexqDkTo=",
"path": "github.com/alecthomas/chroma",
"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
"revisionTime": "2017-09-25T05:25:32Z"
},
{
"checksumSHA1": "Q/9AbXGrFHtlZB6tyoYUq1ipvqU=",
"path": "github.com/alecthomas/chroma/formatters",
"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
"revisionTime": "2017-09-25T05:25:32Z"
},
{
"checksumSHA1": "EbtkLGHGij3Q91njJQJeZRKD3OI=",
"path": "github.com/alecthomas/chroma/formatters/html",
"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
"revisionTime": "2017-09-25T05:25:32Z"
},
{
"checksumSHA1": "ANyNTHVz5LdPPADsExM5WpBJe4c=",
"path": "github.com/alecthomas/chroma/lexers",
"revision": "1af7e1a0bc5c04ec39b8e6d25d70de8eafcf76ab",
"revisionTime": "2017-09-23T12:45:05Z"
},
{
"checksumSHA1": "Nm8r5bmokRePD0D7WU+rXYxOO9A=",
"path": "github.com/alecthomas/chroma/styles",
"revision": "b0295f66bdb7c61d54906003d7649185794e21b4",
"revisionTime": "2017-09-25T05:25:32Z"
},
{
"checksumSHA1": "7yrV1Gzr1ajco1xJ1gsyqRDTY2U=",
"path": "github.com/bep/gitmap",
@ -38,6 +68,12 @@
"revision": "23709d0847197db6021a51fdb193e66e9222d4e7",
"revisionTime": "2017-06-03T12:52:39Z"
},
{
"checksumSHA1": "d/czTNq3bacK85PFEKcHvW6aR80=",
"path": "github.com/danwakefield/fnmatch",
"revision": "cbb64ac3d964b81592e64f957ad53df015803288",
"revisionTime": "2016-04-03T17:12:40Z"
},
{
"checksumSHA1": "OFu4xJEIjiI8Suu+j/gabfp+y6Q=",
"origin": "github.com/stretchr/testify/vendor/github.com/davecgh/go-spew/spew",
@ -51,6 +87,18 @@
"revision": "fb8d9b44afdc258bfff6052d3667521babcb2239",
"revisionTime": "2015-12-10T17:00:30Z"
},
{
"checksumSHA1": "6y/Ht8J58EotTDBEIuE3+s4AnL8=",
"path": "github.com/dlclark/regexp2",
"revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957",
"revisionTime": "2017-07-18T21:59:41Z"
},
{
"checksumSHA1": "k0JXX65FspyueQ8/1i50DGRiCUk=",
"path": "github.com/dlclark/regexp2/syntax",
"revision": "487489b64fb796de2e55f4e8a4ad1e145f80e957",
"revisionTime": "2017-07-18T21:59:41Z"
},
{
"checksumSHA1": "248k9rTfZ4kAknuomoKsdBG9zCU=",
"path": "github.com/eknkc/amber",
@ -219,12 +267,6 @@
"revision": "3e70a1a463008cea6726380c908b1a6a8bdf7b24",
"revisionTime": "2017-05-12T15:20:54Z"
},
{
"checksumSHA1": "F1IYMLBLAZaTOWnmXsgaxTGvrWI=",
"path": "github.com/pelletier/go-buffruneio",
"revision": "c37440a7cf42ac63b919c752ca73a85067e05992",
"revisionTime": "2017-02-27T22:03:11Z"
},
{
"checksumSHA1": "zZg0J0MqvnqXVYo644QDvnUinrc=",
"path": "github.com/pelletier/go-toml",