From c9e679075fcc08d6a3aa612018e3bdbda71adebb Mon Sep 17 00:00:00 2001 From: Asanka Herath Date: Sat, 18 Jul 2020 15:21:51 -0400 Subject: [PATCH 1/2] Add configuration options for pandoc Current options are: markup: pandoc: filters: - list - of - filters extensions: - list - of - extensions extraArgs: - --extra-arguments - --one-per-line Generalize some Pandoc options. Support configuring a bibliography in markup config Anonymous Update [pandoc] Allow page parameters to override site parameters. This allows specifying things like this in the page frontmatter: --- title: Something bibliography: source: mybibliography.bib pandoc: filter: - make-diagrams.lua ... These options are local to the page. Specifying the same under `markup` in the site configuration applies those settings to all pages. Paths (filters, bibliography, citation style) are resolved relative to the page, site, and the `static` folder. [pandoc] Support metadata Support specifying Pandoc metadata in the site configuration and page configuration using the following syntax: Site (in `config.yaml`): ```yaml markup: pandoc: metadata: link-citations: true ``` Or in frontmatter: ```yaml --- pandoc: metadata: link-citations: true ... ``` [pandoc] Simplify path management. No need for any fancy path lookup gymnastics. `pandoc`'s `--resource-path` option does the legwork of locating resources on multiple directories. [pandoc] Don't use x != "" to denote failure. --- markup/bibliography/config.go | 25 ++++ markup/markup_config/config.go | 7 ++ markup/pandoc/convert.go | 75 +++++++++--- markup/pandoc/pandoc_config/pandoc.go | 159 ++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 markup/bibliography/config.go create mode 100644 markup/pandoc/pandoc_config/pandoc.go diff --git a/markup/bibliography/config.go b/markup/bibliography/config.go new file mode 100644 index 000000000..74357f165 --- /dev/null +++ b/markup/bibliography/config.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bibliography + +type Config struct { + // File containing bibliography. E.g. 'my-doc.bibtex'. By default assumed + // to be in BibTex format. + Source string + + // Path to .csl file describing citation file. + CitationStyle string +} + +var Default Config diff --git a/markup/markup_config/config.go b/markup/markup_config/config.go index 0350a78a2..be17c7b99 100644 --- a/markup/markup_config/config.go +++ b/markup/markup_config/config.go @@ -17,8 +17,10 @@ import ( "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config" + "github.com/gohugoio/hugo/markup/bibliography" "github.com/gohugoio/hugo/markup/goldmark/goldmark_config" "github.com/gohugoio/hugo/markup/highlight" + "github.com/gohugoio/hugo/markup/pandoc/pandoc_config" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/mitchellh/mapstructure" ) @@ -33,12 +35,16 @@ type Config struct { // Table of contents configuration TableOfContents tableofcontents.Config + Bibliography bibliography.Config // Configuration for the Goldmark markdown engine. Goldmark goldmark_config.Config // Configuration for the Asciidoc external markdown engine. AsciidocExt asciidocext_config.Config + + // Configuration for Pandoc external markdown engine. + Pandoc pandoc_config.Config } func Decode(cfg config.Provider) (conf Config, err error) { @@ -102,6 +108,7 @@ var Default = Config{ TableOfContents: tableofcontents.DefaultConfig, Highlight: highlight.DefaultConfig, + Bibliography: bibliography.Default, Goldmark: goldmark_config.Default, AsciidocExt: asciidocext_config.Default, diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index eaa9bfb6a..a4caea57f 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -15,14 +15,34 @@ package pandoc import ( + "strings" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" + "github.com/mitchellh/mapstructure" + "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/markup/bibliography" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/internal" + "github.com/gohugoio/hugo/markup/pandoc/pandoc_config" + + "path" ) +type paramer interface { + Param(interface{}) (interface{}, error) +} + +type searchPaths struct { + Paths []string +} + +func (s *searchPaths) AsResourcePath() string { + return strings.Join(s.Paths, ":") +} + // Provider is the package entry point. var Provider converter.ProviderProvider = provider{} @@ -32,15 +52,15 @@ type provider struct { func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) { return converter.NewProvider("pandoc", func(ctx converter.DocumentContext) (converter.Converter, error) { return &pandocConverter{ - ctx: ctx, - cfg: cfg, + docCtx: ctx, + cfg: cfg, }, nil }), nil } type pandocConverter struct { - ctx converter.DocumentContext - cfg converter.ProviderConfig + docCtx converter.DocumentContext + cfg converter.ProviderConfig } func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.ResultRender, error) { @@ -56,30 +76,53 @@ func (c *pandocConverter) Supports(feature identity.Identity) bool { } // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. -func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { +func (c *pandocConverter) getPandocContent(src []byte) []byte { logger := c.cfg.Logger - binaryName := getPandocBinaryName() - if binaryName == "" { + pandocPath, pandocFound := getPandocBinaryName() + if !pandocFound { logger.Println("pandoc not found in $PATH: Please install.\n", " Leaving pandoc content unrendered.") - return src, nil + return src } - args := []string{"--mathjax"} - return internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args) + + var pandocConfig pandoc_config.Config = c.cfg.MarkupConfig.Pandoc + var bibConfig bibliography.Config = c.cfg.MarkupConfig.Bibliography + + if pageParameters, ok := c.docCtx.Document.(paramer); ok { + if bibParam, err := pageParameters.Param("bibliography"); err == nil { + mapstructure.WeakDecode(bibParam, &bibConfig) + } + + if pandocParam, err := pageParameters.Param("pandoc"); err == nil { + mapstructure.WeakDecode(pandocParam, &pandocConfig) + } + } + + arguments := pandocConfig.AsPandocArguments() + + if bibConfig.Source != "" { + arguments = append(arguments, "--citeproc", "--bibliography", bibConfig.Source) + if bibConfig.CitationStyle != "" { + arguments = append(arguments, "--csl", bibConfig.CitationStyle) + } + } + + resourcePath := strings.Join([]string{path.Dir(c.docCtx.Filename), "static", "."}, ":") + arguments = append(arguments, "--resource-path", resourcePath) + + renderedContent, _ := internal.ExternallyRenderContent(c.cfg, c.docCtx, src, pandocPath, arguments) + return renderedContent } const pandocBinary = "pandoc" -func getPandocBinaryName() string { - if hexec.InPath(pandocBinary) { - return pandocBinary - } - return "" +func getPandocBinaryName() (string, bool) { + return pandocBinary, hexec.InPath(pandocBinary) } // Supports returns whether Pandoc is installed on this computer. func Supports() bool { - hasBin := getPandocBinaryName() != "" + _, hasBin := getPandocBinaryName() if htesting.SupportsAll() { if !hasBin { panic("pandoc not installed") diff --git a/markup/pandoc/pandoc_config/pandoc.go b/markup/pandoc/pandoc_config/pandoc.go new file mode 100644 index 000000000..4c586b754 --- /dev/null +++ b/markup/pandoc/pandoc_config/pandoc.go @@ -0,0 +1,159 @@ +package pandoc_config + +import ( + "fmt" + "strings" +) + +// Config contains configuration settings for Pandoc. +type Config struct { + // Input format. Use the 'Extensions' field to specify extensions thereof. + // Only specify the bare format here. Defaults to 'markdown' if empty. Invoke + // "pandoc --list-input-formats" to see the list of supported input formats + // including various Markdown dialects. + InputFormat string + + // If true, the output format is HTML (i.e. "--to=html"). Otherwise the output + // format is HTML5 (i.e. "--to=html5"). + UseLegacyHtml bool + + // Equivalent to specifying "--mathjax". For compatibility, this option is + // always true if none of the other math options are used. + // See https://pandoc.org/MANUAL.html#math-rendering-in-html + UseMathjax bool + + // Equivalent to specifying "--mathml". + // See https://pandoc.org/MANUAL.html#math-rendering-in-html + UseMathml bool + + // Equivalent to specifying "--webtex". + // See https://pandoc.org/MANUAL.html#math-rendering-in-html. Uses the default + // Webtex rendering URL. + UseWebtex bool + + // Equivalent to specifying "--katex". + // See https://pandoc.org/MANUAL.html#math-rendering-in-html + UseKatex bool + + // List of filters to use. These translate to '--filter=' or '--lua-filter' + // arguments to the pandoc invocation. The order of elements in `Filters` + // is preserved when constructing the `pandoc` commandline. + // + // Use the prefix 'lua:' or the suffix '.lua' to indicate Lua filters. + Filters []string + + // List of Pandoc Markdown extensions to use. No need to include default + // extensions. Specifying ["foo", "bar"] is equivalent to specifying + // --from=markdown+foo+bar on the pandoc commandline. + Extensions []string + + // List of input format extensions to use. Specifying ["foo", "bar"] is + // equivalent to specifying --from=markdown+foo+bar on the pandoc commandline + // assuming InputFormat is "markdown". + InputExtensions []string + + // List of output format extensions to use. Specifying ["foo", "bar"] is + // equivalent to specifying --to=html5+foo+bar on the pandoc commandline, + // assuming UseLegacyHTML is false. Invoke "pandoc --list-extensions=html5" to + // or "pandoc --list-extensions=html5" to see the list of extensions that can + // be specified here. + OutputExtensions []string + + // Metadata. The dictionary keys and values are handled in the obvious way. + Metadata map[string]interface{} + + // Extra commandline options passed to the pandoc invocation. These options + // are appended to the commandline after the format and filter options. + // Arguments are passed in literally. Hence must have the "--" or "-" prefix + // where applicable. + ExtraArgs []string +} + +func (c *Config) getInputArg() string { + var b strings.Builder + b.WriteString("--from=") + if len(c.InputFormat) > 0 { + b.WriteString(c.InputFormat) + } else { + b.WriteString("markdown") + } + + for _, extension := range c.InputExtensions { + b.WriteString("+") + b.WriteString(extension) + } + return b.String() +} + +func (c *Config) getOutputArg() string { + var b strings.Builder + b.WriteString("--to=") + if c.UseLegacyHtml { + b.WriteString("html") + } else { + b.WriteString("html5") + } + + for _, extension := range c.OutputExtensions { + b.WriteString("+") + b.WriteString(extension) + } + return b.String() +} + +func (c *Config) getMathRenderingArg() string { + switch { + case c.UseMathml: + return "--mathml" + case c.UseWebtex: + return "--webtex" + case c.UseKatex: + return "--katex" + default: + return "--mathjax" + } +} + +func (c *Config) getMetadataArgs() []string { + var args []string + for k, iv := range c.Metadata { + var v string + if sv, ok := iv.(string); ok { + v = sv + } else if sv, ok := iv.(fmt.Stringer); ok { + v = sv.String() + } else { + v = fmt.Sprintf("%v", iv) + } + args = append(args, fmt.Sprintf("-M%s=%s", k, v)) + } + return args +} + +func (c *Config) getFilterArgs() []string { + var args []string + for _, filterPath := range c.Filters { + if strings.HasPrefix(filterPath, "lua:") || strings.HasSuffix(filterPath, ".lua") { + args = append(args, fmt.Sprintf("--lua-filter=%s", strings.TrimPrefix(filterPath, "lua:"))) + } else { + args = append(args, fmt.Sprintf("--filter=%s", filterPath)) + } + } + return args +} + +// AsPandocArguments returns a list of strings that can be used as arguments to +// a "pandoc" invocation. All the settings contained in Config are represented +// in the returned list of arguments. +func (c *Config) AsPandocArguments() []string { + args := []string{ + c.getInputArg(), + c.getOutputArg(), + c.getMathRenderingArg()} + + args = append(args, c.getMetadataArgs()...) + args = append(args, c.getFilterArgs()...) + args = append(args, c.ExtraArgs...) + + return args +} From b5c5a8bd761acd9d7295b3cb2c05df18d73f6bba Mon Sep 17 00:00:00 2001 From: Asanka Herath Date: Sat, 18 Jul 2020 15:21:51 -0400 Subject: [PATCH 2/2] Add configuration options for pandoc Current options are: markup: pandoc: filters: - list - of - filters extensions: - list - of - extensions extraArgs: - --extra-arguments - --one-per-line Generalize some Pandoc options. Support configuring a bibliography in markup config Anonymous Update [pandoc] Allow page parameters to override site parameters. This allows specifying things like this in the page frontmatter: --- title: Something bibliography: source: mybibliography.bib pandoc: filter: - make-diagrams.lua ... These options are local to the page. Specifying the same under `markup` in the site configuration applies those settings to all pages. Paths (filters, bibliography, citation style) are resolved relative to the page, site, and the `static` folder. [pandoc] Support metadata Support specifying Pandoc metadata in the site configuration and page configuration using the following syntax: Site (in `config.yaml`): ```yaml markup: pandoc: metadata: link-citations: true ``` Or in frontmatter: ```yaml --- pandoc: metadata: link-citations: true ... ``` [pandoc] Simplify path management. No need for any fancy path lookup gymnastics. `pandoc`'s `--resource-path` option does the legwork of locating resources on multiple directories. [pandoc] Don't use x != "" to denote failure. --- markup/markup_config/config.go | 1 + markup/pandoc/convert.go | 17 +++++++---------- markup/pandoc/pandoc_config/pandoc.go | 6 ++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/markup/markup_config/config.go b/markup/markup_config/config.go index be17c7b99..2dccbd6f8 100644 --- a/markup/markup_config/config.go +++ b/markup/markup_config/config.go @@ -111,5 +111,6 @@ var Default = Config{ Bibliography: bibliography.Default, Goldmark: goldmark_config.Default, + Pandoc: pandoc_config.Default, AsciidocExt: asciidocext_config.Default, } diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index a4caea57f..90a4fcc54 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -15,6 +15,7 @@ package pandoc import ( + "errors" "strings" "github.com/gohugoio/hugo/common/hexec" @@ -22,7 +23,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/identity" - "github.com/gohugoio/hugo/markup/bibliography" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/internal" @@ -64,7 +64,7 @@ type pandocConverter struct { } func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.ResultRender, error) { - b, err := c.getPandocContent(ctx.Src, c.ctx) + b, err := c.getPandocContent(ctx.Src) if err != nil { return nil, err } @@ -76,17 +76,14 @@ func (c *pandocConverter) Supports(feature identity.Identity) bool { } // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. -func (c *pandocConverter) getPandocContent(src []byte) []byte { - logger := c.cfg.Logger +func (c *pandocConverter) getPandocContent(src []byte) ([]byte, error) { pandocPath, pandocFound := getPandocBinaryName() if !pandocFound { - logger.Println("pandoc not found in $PATH: Please install.\n", - " Leaving pandoc content unrendered.") - return src + return nil, errors.New("pandoc not found in $PATH: Please install.") } - var pandocConfig pandoc_config.Config = c.cfg.MarkupConfig.Pandoc - var bibConfig bibliography.Config = c.cfg.MarkupConfig.Bibliography + var pandocConfig pandoc_config.Config = c.cfg.MarkupConfig().Pandoc + var bibConfig bibliography.Config = c.cfg.MarkupConfig().Bibliography if pageParameters, ok := c.docCtx.Document.(paramer); ok { if bibParam, err := pageParameters.Param("bibliography"); err == nil { @@ -111,7 +108,7 @@ func (c *pandocConverter) getPandocContent(src []byte) []byte { arguments = append(arguments, "--resource-path", resourcePath) renderedContent, _ := internal.ExternallyRenderContent(c.cfg, c.docCtx, src, pandocPath, arguments) - return renderedContent + return renderedContent, nil } const pandocBinary = "pandoc" diff --git a/markup/pandoc/pandoc_config/pandoc.go b/markup/pandoc/pandoc_config/pandoc.go index 4c586b754..45d7cc6ad 100644 --- a/markup/pandoc/pandoc_config/pandoc.go +++ b/markup/pandoc/pandoc_config/pandoc.go @@ -69,6 +69,12 @@ type Config struct { ExtraArgs []string } +var Default = Config{ + InputFormat: "markdown", + UseLegacyHtml: false, + UseMathjax: true, +} + func (c *Config) getInputArg() string { var b strings.Builder b.WriteString("--from=")