Add emoji support

This uses the Emoji map from https://github.com/kyokomi/emoji -- but with a custom replacement implementation.

The built-in are fine for most use cases, but in Hugo we do care about pure speed.

The benchmarks below are skewed in Hugo's direction as the source and result is a byte slice,
Kyokomi's implementation works best with strings.

Curious: The easy-to-use `strings.Replacer` is also plenty fast.

```
BenchmarkEmojiKyokomiFprint-4  	   20000	     86038 ns/op	   33960 B/op	     117 allocs/op
BenchmarkEmojiKyokomiSprint-4  	   20000	     83252 ns/op	   38232 B/op	     122 allocs/op
BenchmarkEmojiStringsReplacer-4	  100000	     21092 ns/op	   17248 B/op	      25 allocs/op
BenchmarkHugoEmoji-4           	  500000	      5728 ns/op	     624 B/op	      13 allocs/op
```

Fixes #1891
This commit is contained in:
Bjørn Erik Pedersen 2016-02-25 00:52:11 +01:00 committed by Cameron Moore
parent 5926c6c8d5
commit cafb784799
7 changed files with 256 additions and 3 deletions

View file

@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved. // Copyright 2016 The Hugo Authors. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -301,6 +301,7 @@ func LoadDefaultSettings() {
viper.SetDefault("SectionPagesMenu", "") viper.SetDefault("SectionPagesMenu", "")
viper.SetDefault("DisablePathToLower", false) viper.SetDefault("DisablePathToLower", false)
viper.SetDefault("HasCJKLanguage", false) viper.SetDefault("HasCJKLanguage", false)
viper.SetDefault("EnableEmoji", false)
} }
// InitializeConfig initializes a config file with sensible default configuration flags. // InitializeConfig initializes a config file with sensible default configuration flags.

View file

@ -99,6 +99,9 @@ Following is a list of Hugo-defined variables that you can configure and their c
disableRobotsTXT: false disableRobotsTXT: false
# edit new content with this editor, if provided # edit new content with this editor, if provided
editor: "" editor: ""
# Enable Emoji emoticons support for page content.
# See www.emoji-cheat-sheet.com
enableEmoji: false
footnoteAnchorPrefix: "" footnoteAnchorPrefix: ""
footnoteReturnLinkContents: "" footnoteReturnLinkContents: ""
# google analytics tracking id # google analytics tracking id

View file

@ -413,6 +413,14 @@ These are formatted with the layout string.
e.g. `{{ dateFormat "Monday, Jan 2, 2006" "2015-01-21" }}` → "Wednesday, Jan 21, 2015" e.g. `{{ dateFormat "Monday, Jan 2, 2006" "2015-01-21" }}` → "Wednesday, Jan 21, 2015"
### emojify
Runs the string through the Emoji emoticons processor. The result will be declared as "safe" so Go templates will not filter it.
See the [Emoji cheat sheet](http://www.emoji-cheat-sheet.com/) for available emoticons.
e.g. `{{ "I :heart: Hugo" | emojify }}`
### highlight ### highlight
Takes a string of code and a language, uses Pygments to return the syntax highlighted code in HTML. Takes a string of code and a language, uses Pygments to return the syntax highlighted code in HTML.
Used in the [highlight shortcode](/extras/highlighting/). Used in the [highlight shortcode](/extras/highlighting/).

94
helpers/emoji.go Normal file
View file

@ -0,0 +1,94 @@
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"bytes"
"github.com/kyokomi/emoji"
"sync"
)
var (
emojiInit sync.Once
emojis = make(map[string][]byte)
emojiDelim = []byte(":")
emojiWordDelim = []byte(" ")
emojiMaxSize int
)
// Emojify "emojifies" the input source.
// Note that the input byte slice will be modified if needed.
// See http://www.emoji-cheat-sheet.com/
func Emojify(source []byte) []byte {
emojiInit.Do(initEmoji)
start := 0
k := bytes.Index(source[start:], emojiDelim)
for k != -1 {
j := start + k
upper := j + emojiMaxSize
if upper > len(source) {
upper = len(source)
}
endEmoji := bytes.Index(source[j+1:upper], emojiDelim)
if endEmoji < 0 {
break
}
nextWordDelim := bytes.Index(source[j:upper], emojiWordDelim)
if endEmoji == 0 || (nextWordDelim != -1 && nextWordDelim < endEmoji) {
start += endEmoji + 1
} else {
endKey := endEmoji + j + 2
emojiKey := source[j:endKey]
if emoji, ok := emojis[string(emojiKey)]; ok {
source = append(source[:j], append(emoji, source[endKey:]...)...)
}
start += endEmoji
}
if start >= len(source) {
break
}
k = bytes.Index(source[start:], emojiDelim)
}
return source
}
func initEmoji() {
emojiMap := emoji.CodeMap()
for k, v := range emojiMap {
emojis[k] = []byte(v + emoji.ReplacePadding)
if len(k) > emojiMaxSize {
emojiMaxSize = len(k)
}
}
}

128
helpers/emoji_test.go Normal file
View file

@ -0,0 +1,128 @@
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package helpers
import (
"github.com/kyokomi/emoji"
"github.com/spf13/hugo/bufferpool"
"reflect"
"strings"
"testing"
)
func TestEmojiCustom(t *testing.T) {
for i, this := range []struct {
input string
expect []byte
}{
{"A :smile: a day", []byte(emoji.Sprint("A :smile: a day"))},
{"A few :smile:s a day", []byte(emoji.Sprint("A few :smile:s a day"))},
{"A :smile: and a :beer: makes the day for sure.", []byte(emoji.Sprint("A :smile: and a :beer: makes the day for sure."))},
{"A :smile: and: a :beer:", []byte(emoji.Sprint("A :smile: and: a :beer:"))},
{"A :diamond_shape_with_a_dot_inside: and then some.", []byte(emoji.Sprint("A :diamond_shape_with_a_dot_inside: and then some."))},
{":smile:", []byte(emoji.Sprint(":smile:"))},
{":smi", []byte(":smi")},
{"A :smile:", []byte(emoji.Sprint("A :smile:"))},
{":beer:!", []byte(emoji.Sprint(":beer:!"))},
{"::smile:", []byte(emoji.Sprint("::smile:"))},
{":beer::", []byte(emoji.Sprint(":beer::"))},
{" :beer: :", []byte(emoji.Sprint(" :beer: :"))},
{":beer: and :smile: and another :beer:!", []byte(emoji.Sprint(":beer: and :smile: and another :beer:!"))},
{" :beer: : ", []byte(emoji.Sprint(" :beer: : "))},
{"No smilies for you!", []byte("No smilies for you!")},
{" The motto: no smiles! ", []byte(" The motto: no smiles! ")},
{":hugo_is_the_best_static_gen:", []byte(":hugo_is_the_best_static_gen:")},
{"은행 :smile: 은행", []byte(emoji.Sprint("은행 :smile: 은행"))},
} {
result := Emojify([]byte(this.input))
if !reflect.DeepEqual(result, this.expect) {
t.Errorf("[%d] got '%q' but expected %q", i, result, this.expect)
}
}
}
// The Emoji benchmarks below are heavily skewed in Hugo's direction:
//
// Hugo have a byte slice, wants a byte slice and doesn't mind if the original is modified.
func BenchmarkEmojiKyokomiFprint(b *testing.B) {
f := func(in []byte) []byte {
buff := bufferpool.GetBuffer()
defer bufferpool.PutBuffer(buff)
emoji.Fprint(buff, string(in))
bc := make([]byte, buff.Len(), buff.Len())
copy(bc, buff.Bytes())
return bc
}
doBenchmarkEmoji(b, f)
}
func BenchmarkEmojiKyokomiSprint(b *testing.B) {
f := func(in []byte) []byte {
return []byte(emoji.Sprint(string(in)))
}
doBenchmarkEmoji(b, f)
}
func BenchmarkHugoEmoji(b *testing.B) {
doBenchmarkEmoji(b, Emojify)
}
func doBenchmarkEmoji(b *testing.B, f func(in []byte) []byte) {
type input struct {
in []byte
expect []byte
}
data := []struct {
input string
expect string
}{
{"A :smile: a day", emoji.Sprint("A :smile: a day")},
{"A :smile: and a :beer: day keeps the doctor away", emoji.Sprint("A :smile: and a :beer: day keeps the doctor away")},
{"A :smile: a day and 10 " + strings.Repeat(":beer: ", 10), emoji.Sprint("A :smile: a day and 10 " + strings.Repeat(":beer: ", 10))},
{"No smiles today.", "No smiles today."},
{"No smiles for you or " + strings.Repeat("you ", 1000), "No smiles for you or " + strings.Repeat("you ", 1000)},
}
var in []input = make([]input, b.N*len(data))
var cnt = 0
for i := 0; i < b.N; i++ {
for _, this := range data {
in[cnt] = input{[]byte(this.input), []byte(this.expect)}
cnt++
}
}
b.ResetTimer()
cnt = 0
for i := 0; i < b.N; i++ {
for j := range data {
currIn := in[cnt]
cnt++
result := f(currIn.in)
if len(result) != len(currIn.expect) {
b.Fatalf("[%d] emoji std, got \n%q but expected \n%q", j, result, currIn.expect)
}
}
}
}

View file

@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved. // Copyright 2016 The Hugo Authors. All rights reserved.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -18,6 +18,7 @@ import (
"github.com/spf13/hugo/source" "github.com/spf13/hugo/source"
"github.com/spf13/hugo/tpl" "github.com/spf13/hugo/tpl"
jww "github.com/spf13/jwalterweatherman" jww "github.com/spf13/jwalterweatherman"
"github.com/spf13/viper"
) )
func init() { func init() {
@ -114,6 +115,10 @@ func commonConvert(p *Page, t tpl.Template) HandledResult {
var err error var err error
if viper.GetBool("EnableEmoji") {
p.rawContent = helpers.Emojify(p.rawContent)
}
renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)) renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
if len(p.contentShortCodes) > 0 { if len(p.contentShortCodes) > 0 {

View file

@ -1,4 +1,4 @@
// Copyright 2015 The Hugo Authors. All rights reserved. // Copyright 2016 The Hugo Authors. All rights reserved.
// //
// Portions Copyright The Go Authors. // Portions Copyright The Go Authors.
@ -1156,6 +1156,19 @@ func jsonify(v interface{}) (template.HTML, error) {
return "", err return "", err
} }
return template.HTML(b), nil return template.HTML(b), nil
}
// emojify "emojifies" the given string.
//
// See http://www.emoji-cheat-sheet.com/
func emojify(in interface{}) (template.HTML, error) {
str, err := cast.ToStringE(in)
if err != nil {
return "", err
}
return template.HTML(helpers.Emojify([]byte(str))), nil
} }
func refPage(page interface{}, ref, methodName string) template.HTML { func refPage(page interface{}, ref, methodName string) template.HTML {
@ -1715,6 +1728,7 @@ func init() {
"dict": dictionary, "dict": dictionary,
"div": func(a, b interface{}) (interface{}, error) { return doArithmetic(a, b, '/') }, "div": func(a, b interface{}) (interface{}, error) { return doArithmetic(a, b, '/') },
"echoParam": returnWhenSet, "echoParam": returnWhenSet,
"emojify": emojify,
"eq": eq, "eq": eq,
"first": first, "first": first,
"ge": ge, "ge": ge,