Hugo import from jekyll

usage: hugo import jekyll jekyll_root_path target_path

Implemented:
 * Create new hugo site
 * Create config.yaml
 * Convert all markdown contents.
 * Copy all other files and folders to static

Fixes #101
This commit is contained in:
coderzh 2015-10-01 09:04:30 +08:00 committed by Bjørn Erik Pedersen
parent 08d41c3a48
commit 9a2f6c62a6
3 changed files with 588 additions and 0 deletions

View file

@ -83,6 +83,7 @@ func AddCommands() {
HugoCmd.AddCommand(undraftCmd)
HugoCmd.AddCommand(genautocompleteCmd)
HugoCmd.AddCommand(gendocCmd)
HugoCmd.AddCommand(importCmd)
}
//Initializes flags

483
commands/import.go Normal file
View file

@ -0,0 +1,483 @@
// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/spf13/cast"
"github.com/spf13/cobra"
"github.com/spf13/hugo/helpers"
"github.com/spf13/hugo/hugofs"
"github.com/spf13/hugo/hugolib"
"github.com/spf13/hugo/parser"
jww "github.com/spf13/jwalterweatherman"
)
func init() {
importCmd.AddCommand(importJekyllCmd)
}
var importCmd = &cobra.Command{
Use: "import",
Short: "import from others",
Long: `import from others like jekyll.
Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
Run: nil,
}
var importJekyllCmd = &cobra.Command{
Use: "jekyll",
Short: "hugo import from jekyll",
Long: `hugo import from jekyll.
Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.",
Run: importFromJekyll,
}
func importFromJekyll(cmd *cobra.Command, args []string) {
jww.SetLogThreshold(jww.LevelTrace)
jww.SetStdoutThreshold(jww.LevelWarn)
if len(args) < 2 {
jww.ERROR.Println(`Import jekyll requires two path, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.")
return
}
jekyllRoot, err := filepath.Abs(filepath.Clean(args[0]))
if err != nil {
jww.ERROR.Println("Path error:", args[0])
return
}
targetDir, err := filepath.Abs(filepath.Clean(args[1]))
if err != nil {
jww.ERROR.Println("Path error:", args[1])
return
}
createSiteFromJekyll(jekyllRoot, targetDir)
jww.INFO.Println("Import jekyll from:", jekyllRoot, "to:", targetDir)
fmt.Println("Importing...")
fileCount := 0
callback := func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if fi.IsDir() {
return nil
}
relPath, err := filepath.Rel(jekyllRoot, path)
if err != nil {
jww.ERROR.Println("Get rel path error:", path)
return err
}
relPath = filepath.ToSlash(relPath)
var draft bool = false
switch {
case strings.HasPrefix(relPath, "_posts/"):
relPath = "content/post" + relPath[len("_posts"):]
case strings.HasPrefix(relPath, "_drafts/"):
relPath = "content/draft" + relPath[len("_drafts"):]
draft = true
default:
return nil
}
fileCount++
return convertJekyllPost(path, relPath, targetDir, draft)
}
err = filepath.Walk(jekyllRoot, callback)
if err != nil {
fmt.Println(err)
} else {
fmt.Println("Congratulations!", fileCount, "posts imported!")
fmt.Println("Now, start hugo by yourself: \n" +
"$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove")
fmt.Println("$ cd " + args[1] + "\n$ hugo server -w --theme=herring-cove")
}
}
func createSiteFromJekyll(jekyllRoot, targetDir string) {
mkdir(targetDir, "layouts")
mkdir(targetDir, "content")
mkdir(targetDir, "archetypes")
mkdir(targetDir, "static")
mkdir(targetDir, "data")
mkdir(targetDir, "themes")
jekyllConfig := loadJekyllConfig(jekyllRoot)
createConfigFromJekyll(targetDir, "yaml", jekyllConfig)
copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"))
}
func loadJekyllConfig(jekyllRoot string) map[string]interface{} {
fs := hugofs.SourceFs
path := filepath.Join(jekyllRoot, "_config.yml")
exists, err := helpers.Exists(path, fs)
if err != nil || !exists {
return nil
}
f, err := fs.Open(path)
if err != nil {
return nil
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
return nil
}
c, err := parser.HandleYAMLMetaData(b)
if err != nil {
return nil
}
return c.(map[string]interface{})
}
func createConfigFromJekyll(inpath string, kind string, jekyllConfig map[string]interface{}) (err error) {
title := "My New Hugo Site"
baseurl := "http://replace-this-with-your-hugo-site.com/"
for key, value := range jekyllConfig {
lowerKey := strings.ToLower(key)
switch lowerKey {
case "title":
if str, ok := value.(string); ok {
title = str
}
case "url":
if str, ok := value.(string); ok {
baseurl = str
}
}
}
in := map[string]interface{}{
"baseurl": baseurl,
"title": title,
"languageCode": "en-us",
"disablePathToLower": true,
}
kind = parser.FormatSanitize(kind)
by, err := parser.InterfaceToConfig(in, parser.FormatToLeadRune(kind))
if err != nil {
return err
}
err = helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), bytes.NewReader(by), hugofs.SourceFs)
if err != nil {
return
}
return nil
}
func copyFile(source string, dest string) (err error) {
sf, err := os.Open(source)
if err != nil {
return err
}
defer sf.Close()
df, err := os.Create(dest)
if err != nil {
return err
}
defer df.Close()
_, err = io.Copy(df, sf)
if err == nil {
si, err := os.Stat(source)
if err != nil {
err = os.Chmod(dest, si.Mode())
}
}
return
}
func copyDir(source string, dest string) (err error) {
fi, err := os.Stat(source)
if err != nil {
return err
}
if !fi.IsDir() {
return errors.New(source + " is not a directory")
}
err = os.MkdirAll(dest, fi.Mode())
if err != nil {
return err
}
entries, err := ioutil.ReadDir(source)
for _, entry := range entries {
sfp := filepath.Join(source, entry.Name())
dfp := filepath.Join(dest, entry.Name())
if entry.IsDir() {
err = copyDir(sfp, dfp)
if err != nil {
jww.ERROR.Println(err)
}
} else {
err = copyFile(sfp, dfp)
if err != nil {
jww.ERROR.Println(err)
}
}
}
return nil
}
func copyJekyllFilesAndFolders(jekyllRoot string, dest string) (err error) {
fi, err := os.Stat(jekyllRoot)
if err != nil {
return err
}
if !fi.IsDir() {
return errors.New(jekyllRoot + " is not a directory")
}
err = os.MkdirAll(dest, fi.Mode())
if err != nil {
return err
}
entries, err := ioutil.ReadDir(jekyllRoot)
for _, entry := range entries {
sfp := filepath.Join(jekyllRoot, entry.Name())
dfp := filepath.Join(dest, entry.Name())
if entry.IsDir() {
if entry.Name()[0] != '_' && entry.Name()[0] != '.' {
err = copyDir(sfp, dfp)
if err != nil {
jww.ERROR.Println(err)
}
}
} else {
lowerEntryName := strings.ToLower(entry.Name())
exceptSuffix := []string{".md", ".markdown", ".html", ".htm",
".xml", ".textile", "rakefile", "gemfile", ".lock"}
isExcept := false
for _, suffix := range exceptSuffix {
if strings.HasSuffix(lowerEntryName, suffix) {
isExcept = true
break
}
}
if !isExcept && entry.Name()[0] != '.' && entry.Name()[0] != '_' {
err = copyFile(sfp, dfp)
if err != nil {
jww.ERROR.Println(err)
}
}
}
}
return nil
}
func parseJekyllFilename(filename string) (time.Time, string, error) {
re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`)
r := re.FindAllStringSubmatch(filename, -1)
if len(r) == 0 {
return time.Now(), "", errors.New("filename not match")
}
postDate, err := time.Parse("2006-01-02", r[0][1])
if err != nil {
return time.Now(), "", err
}
postName := r[0][2]
return postDate, postName, nil
}
func convertJekyllPost(path, relPath, targetDir string, draft bool) error {
jww.TRACE.Println("Converting", path)
filename := filepath.Base(path)
postDate, postName, err := parseJekyllFilename(filename)
if err != nil {
jww.ERROR.Println("Parse filename error:", filename)
return err
}
jww.TRACE.Println(filename, postDate, postName)
targetFile := filepath.Join(targetDir, relPath)
targetParentDir := filepath.Dir(targetFile)
os.MkdirAll(targetParentDir, 0777)
contentBytes, err := ioutil.ReadFile(path)
if err != nil {
jww.ERROR.Println("Read file error:", path)
return err
}
psr, err := parser.ReadFrom(bytes.NewReader(contentBytes))
if err != nil {
jww.ERROR.Println("Parse file error:", path)
return err
}
metadata, err := psr.Metadata()
if err != nil {
jww.ERROR.Println("Processing file error:", path)
return err
}
newmetadata, err := convertJekyllMetaData(metadata, postName, postDate, draft)
if err != nil {
jww.ERROR.Println("Convert metadata error:", path)
return err
}
jww.TRACE.Println(newmetadata)
content := convertJekyllContent(newmetadata, string(psr.Content()))
page, err := hugolib.NewPage(filename)
if err != nil {
jww.ERROR.Println("New page error", filename)
return err
}
page.SetDir(targetParentDir)
page.SetSourceContent([]byte(content))
page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
page.SaveSourceAs(targetFile)
jww.TRACE.Println("Target file:", targetFile)
return nil
}
func convertJekyllMetaData(m interface{}, postName string, postDate time.Time, draft bool) (interface{}, error) {
url := postDate.Format("/2006/01/02/") + postName + "/"
metadata, err := cast.ToStringMapE(m)
if err != nil {
return nil, err
}
if draft {
metadata["draft"] = true
}
for key, value := range metadata {
lowerKey := strings.ToLower(key)
switch lowerKey {
case "layout":
delete(metadata, key)
case "permalink":
if str, ok := value.(string); ok {
url = str
}
delete(metadata, key)
case "category":
if str, ok := value.(string); ok {
metadata["categories"] = []string{str}
}
delete(metadata, key)
case "excerpt_separator":
if key != lowerKey {
delete(metadata, key)
metadata[lowerKey] = value
}
case "date":
if str, ok := value.(string); ok {
re := regexp.MustCompile(`(\d+):(\d+):(\d+)`)
r := re.FindAllStringSubmatch(str, -1)
if len(r) > 0 {
hour, _ := strconv.Atoi(r[0][1])
minute, _ := strconv.Atoi(r[0][2])
second, _ := strconv.Atoi(r[0][3])
postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC)
}
}
delete(metadata, key)
}
}
metadata["url"] = url
metadata["date"] = postDate.Format(time.RFC3339)
return metadata, nil
}
func convertJekyllContent(m interface{}, content string) string {
metadata, _ := cast.ToStringMapE(m)
lines := strings.Split(content, "\n")
var resultLines []string
for _, line := range lines {
resultLines = append(resultLines, strings.Trim(line, "\r\n"))
}
content = strings.Join(resultLines, "\n")
excerptSep := "<!--more-->"
if value, ok := metadata["excerpt_separator"]; ok {
if str, strOk := value.(string); strOk {
content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1)
}
}
replaceList := []struct {
re *regexp.Regexp
replace string
}{
{regexp.MustCompile("<!-- more -->"), "<!--more-->"},
{regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"},
{regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), "{{< highlight $1 >}}"},
{regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"},
}
for _, replace := range replaceList {
content = replace.re.ReplaceAllString(content, replace.replace)
}
return content
}

104
commands/import_test.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"encoding/json"
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestParseJekyllFilename(t *testing.T) {
filenameArray := []string{
"2015-01-02-test.md",
"2012-03-15-中文.markup",
}
expectResult := []struct {
postDate time.Time
postName string
}{
{time.Date(2015, time.January, 2, 0, 0, 0, 0, time.UTC), "test"},
{time.Date(2012, time.March, 15, 0, 0, 0, 0, time.UTC), "中文"},
}
for i, filename := range filenameArray {
postDate, postName, err := parseJekyllFilename(filename)
assert.Equal(t, err, nil)
assert.Equal(t, expectResult[i].postDate.Format("2006-01-02"), postDate.Format("2006-01-02"))
assert.Equal(t, expectResult[i].postName, postName)
}
}
func TestConvertJekyllMetadata(t *testing.T) {
testDataList := []struct {
metadata interface{}
postName string
postDate time.Time
draft bool
expect string
}{
{map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},
{map[interface{}]interface{}{}, "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), true,
`{"date":"2015-10-01T00:00:00Z","draft":true,"url":"/2015/10/01/testPost/"}`},
{map[interface{}]interface{}{"Permalink": "/permalink.html", "layout": "post"},
"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},
{map[interface{}]interface{}{"permalink": "/permalink.html"},
"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`},
{map[interface{}]interface{}{"category": nil, "permalink": 123},
"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"date":"2015-10-01T00:00:00Z","url":"/2015/10/01/testPost/"}`},
{map[interface{}]interface{}{"Excerpt_Separator": "sep"},
"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep","url":"/2015/10/01/testPost/"}`},
{map[interface{}]interface{}{"category": "book", "layout": "post", "Others": "Goods", "Date": "2015-10-01 12:13:11"},
"testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false,
`{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z","url":"/2015/10/01/testPost/"}`},
}
for _, data := range testDataList {
result, err := convertJekyllMetaData(data.metadata, data.postName, data.postDate, data.draft)
assert.Equal(t, nil, err)
jsonResult, err := json.Marshal(result)
assert.Equal(t, nil, err)
assert.Equal(t, data.expect, string(jsonResult))
}
}
func TestConvertJekyllContent(t *testing.T) {
testDataList := []struct {
metadata interface{}
content string
expect string
}{
{map[interface{}]interface{}{},
`Test content\n<!-- more -->\npart2 content`, `Test content\n<!--more-->\npart2 content`},
{map[interface{}]interface{}{"excerpt_separator": "<!--sep-->"},
`Test content\n<!--sep-->\npart2 content`, `Test content\n<!--more-->\npart2 content`},
{map[interface{}]interface{}{}, "{% raw %}text{% endraw %}", "text"},
{map[interface{}]interface{}{}, "{%raw%} text2 {%endraw %}", "text2"},
{map[interface{}]interface{}{},
"{% highlight go %}\nvar s int\n{% endhighlight %}",
"{{< highlight go >}}\nvar s int\n{{< / highlight >}}"},
}
for _, data := range testDataList {
result := convertJekyllContent(data.metadata, data.content)
assert.Equal(t, data.expect, result)
}
}