DataDir: Minor polish and add missing tests

Also, now logs an ERROR on duplicate keys, instead of returning an error and make sure sub-folders take presedence in data dir.
This commit is contained in:
bep 2015-02-08 21:09:14 +01:00
parent 773812de6f
commit 1c50f775b5
2 changed files with 73 additions and 20 deletions

View file

@ -269,12 +269,12 @@ func (s *Site) addTemplate(name, data string) error {
func (s *Site) loadData(fs source.Input) (err error) { func (s *Site) loadData(fs source.Input) (err error) {
s.Data = make(map[string]interface{}) s.Data = make(map[string]interface{})
var current map[string]interface{}
for _, r := range fs.Files() { for _, r := range fs.Files() {
// Crawl in data tree to insert data // Crawl in data tree to insert data
var current map[string]interface{}
current = s.Data current = s.Data
for _, key := range strings.Split(r.Dir(), string(os.PathSeparator)) { for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) {
if key != "" { if key != "" {
if _, ok := current[key]; !ok { if _, ok := current[key]; !ok {
current[key] = make(map[string]interface{}) current[key] = make(map[string]interface{})
@ -283,8 +283,7 @@ func (s *Site) loadData(fs source.Input) (err error) {
} }
} }
// Read data file data, err := readData(r)
data, err := readFile(r)
if err != nil { if err != nil {
return err return err
} }
@ -295,10 +294,10 @@ func (s *Site) loadData(fs source.Input) (err error) {
for key, value := range current[r.BaseFileName()].(map[string]interface{}) { for key, value := range current[r.BaseFileName()].(map[string]interface{}) {
if _, override := data[key]; override { if _, override := data[key]; override {
return errors.New("Data in " + r.Path() + " is overrided in subfolder.") // filepath.Walk walks the files in lexical order, '/' comes before '.'
} else { jww.ERROR.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path())
data[key] = value
} }
data[key] = value
} }
} }
@ -309,7 +308,7 @@ func (s *Site) loadData(fs source.Input) (err error) {
return return
} }
func readFile(f *source.File) (interface{}, error) { func readData(f *source.File) (interface{}, error) {
switch f.Extension() { switch f.Extension() {
case "yaml", "yml": case "yaml", "yml":
return parser.HandleYamlMetaData(f.Bytes()) return parser.HandleYamlMetaData(f.Bytes())
@ -318,7 +317,7 @@ func readFile(f *source.File) (interface{}, error) {
case "toml": case "toml":
return parser.HandleTomlMetaData(f.Bytes()) return parser.HandleTomlMetaData(f.Bytes())
default: default:
return nil, errors.New("Not supported for data: " + f.Extension()) return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())
} }
} }
@ -328,11 +327,13 @@ func (s *Site) Process() (err error) {
} }
s.prepTemplates() s.prepTemplates()
s.Tmpl.PrintErrors() s.Tmpl.PrintErrors()
s.timerStep("initialize & template prep")
if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil { if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil {
return return
} }
s.timerStep("load data") s.timerStep("load data")
s.timerStep("initialize & template prep")
if err = s.CreatePages(); err != nil { if err = s.CreatePages(); err != nil {
return return
} }

View file

@ -3,9 +3,9 @@ package hugolib
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/spf13/hugo/parser"
"html/template" "html/template"
"io" "io"
"os"
"path/filepath" "path/filepath"
"strings" "strings"
"testing" "testing"
@ -17,6 +17,7 @@ import (
"github.com/spf13/hugo/target" "github.com/spf13/hugo/target"
"github.com/spf13/hugo/tpl" "github.com/spf13/hugo/tpl"
"github.com/spf13/viper" "github.com/spf13/viper"
"reflect"
) )
const ( const (
@ -747,17 +748,68 @@ func TestWeightedTaxonomies(t *testing.T) {
} }
} }
func TestDataDir(t *testing.T) { func TestDataDirJson(t *testing.T) {
sources := []source.ByteSource{ sources := []source.ByteSource{
{filepath.FromSlash("test" + string(os.PathSeparator) + "foo.yaml"), []byte("bar: foofoo")}, {filepath.FromSlash("test/foo.json"), []byte(`{ "bar": "foofoo" }`)},
{filepath.FromSlash("test.yaml"), []byte("hello:\n- world: foo")}, {filepath.FromSlash("test.json"), []byte(`{ "hello": [ { "world": "foo" } ] }`)},
} }
s := &Site{} expected, err := parser.HandleJsonMetaData([]byte(`{ "test": { "hello": [{ "world": "foo" }] , "foo": { "bar":"foofoo" } } }`))
s.loadData(&source.InMemorySource{ByteSource: sources})
expected := "map[test:map[hello:[map[world:foo]] foo:map[bar:foofoo]]]" if err != nil {
if fmt.Sprint(s.Data) != expected { t.Fatalf("Error %s", err)
t.Errorf("Expected structure '%s', got '%s'", expected, s.Data) }
doTestDataDir(t, expected, sources)
}
func TestDataDirToml(t *testing.T) {
sources := []source.ByteSource{
{filepath.FromSlash("test/kung.toml"), []byte("[foo]\nbar = 1")},
}
expected, err := parser.HandleTomlMetaData([]byte("[test]\n[test.kung]\n[test.kung.foo]\nbar = 1"))
if err != nil {
t.Fatalf("Error %s", err)
}
doTestDataDir(t, expected, sources)
}
func TestDataDirYamlWithOverridenValue(t *testing.T) {
sources := []source.ByteSource{
// filepath.Walk walks the files in lexical order, '/' comes before '.'. Simulate this:
{filepath.FromSlash("a.yaml"), []byte("a: 1")},
{filepath.FromSlash("test/v1.yaml"), []byte("v1-2: 2")},
{filepath.FromSlash("test/v2.yaml"), []byte("v2:\n- 2\n- 3")},
{filepath.FromSlash("test.yaml"), []byte("v1: 1")},
}
expected := map[string]interface{}{"a": map[string]interface{}{"a": 1},
"test": map[string]interface{}{"v1": map[string]interface{}{"v1-2": 2}, "v2": map[string]interface{}{"v2": []interface{}{2, 3}}}}
doTestDataDir(t, expected, sources)
}
func TestDataDirUnknownFormat(t *testing.T) {
sources := []source.ByteSource{
{filepath.FromSlash("test.roml"), []byte("boo")},
}
s := &Site{}
err := s.loadData(&source.InMemorySource{ByteSource: sources})
if err == nil {
t.Fatalf("Should return an error")
}
}
func doTestDataDir(t *testing.T, expected interface{}, sources []source.ByteSource) {
s := &Site{}
err := s.loadData(&source.InMemorySource{ByteSource: sources})
if err != nil {
t.Fatalf("Error loading data: %s", err)
}
if !reflect.DeepEqual(expected, s.Data) {
t.Errorf("Expected structure\n%#v got\n%#v", expected, s.Data)
} }
} }