Reads data files inside data/ and makes data available in .Site.Data

Fixes #476.

Conflicts:
	hugolib/site.go

commands/hugo.go

@@ -119,6 +119,7 @@ func InitializeConfig() {
 	viper.SetDefault("StaticDir", "static")
 	viper.SetDefault("ArchetypeDir", "archetypes")
 	viper.SetDefault("PublishDir", "public")
+	viper.SetDefault("DataDir", "data")
 	viper.SetDefault("DefaultLayout", "post")
 	viper.SetDefault("BuildDrafts", false)
 	viper.SetDefault("BuildFuture", false)
@@ -287,6 +288,7 @@ func getDirList() []string {
 		return nil
 	}
 
+	filepath.Walk(helpers.AbsPathify(viper.GetString("DataDir")), walker)
 	filepath.Walk(helpers.AbsPathify(viper.GetString("ContentDir")), walker)
 	filepath.Walk(helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
 	filepath.Walk(helpers.AbsPathify(viper.GetString("StaticDir")), walker)

commands/new.go

@@ -130,6 +130,7 @@ func NewSite(cmd *cobra.Command, args []string) {
 	mkdir(createpath, "content")
 	mkdir(createpath, "archetypes")
 	mkdir(createpath, "static")
+	mkdir(createpath, "data")
 
 	createConfig(createpath, configFormat)
 }

hugolib/site.go

@@ -34,6 +34,7 @@ import (
 	bp "github.com/spf13/hugo/bufferpool"
 	"github.com/spf13/hugo/helpers"
 	"github.com/spf13/hugo/hugofs"
+	"github.com/spf13/hugo/parser"
 	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/target"
 	"github.com/spf13/hugo/tpl"
@@ -81,6 +82,7 @@ type Site struct {
 	params      map[string]interface{}
 	draftCount  int
 	futureCount int
+	Data        map[string]interface{}
 }
 
 type targetList struct {
@@ -112,6 +114,7 @@ type SiteInfo struct {
 	BuildDrafts         bool
 	canonifyUrls        bool
 	paginationPageCount uint64
+	Data            *map[string]interface{}
 }
 
 // SiteSocial is a place to put social details on a site level. These are the
@@ -264,12 +267,71 @@ func (s *Site) addTemplate(name, data string) error {
 	return s.Tmpl.AddTemplate(name, data)
 }
 
+func (s *Site) loadData(fs source.Input) (err error) {
+	s.Data = make(map[string]interface{})
+
+	for _, r := range fs.Files() {
+		// Crawl in data tree to insert data
+		var current map[string]interface{}
+		current = s.Data
+		for _, key := range strings.Split(r.Dir(), string(os.PathSeparator)) {
+			if key != "" {
+				if _, ok := current[key]; !ok {
+					current[key] = make(map[string]interface{})
+				}
+				current = current[key].(map[string]interface{})
+			}
+		}
+
+		// Read data file
+		data, err := readFile(r)
+		if err != nil {
+			return err
+		}
+
+		// Copy content from current to data when needed
+		if _, ok := current[r.BaseFileName()]; ok {
+			data := data.(map[string]interface{})
+
+			for key, value := range current[r.BaseFileName()].(map[string]interface{}) {
+				if _, override := data[key]; override {
+					return errors.New("Data in " + r.Path() + " is overrided in subfolder.")
+				} else {
+					data[key] = value
+				}
+			}
+		}
+
+		// Insert data
+		current[r.BaseFileName()] = data
+	}
+
+	return
+}
+
+func readFile(f *source.File) (interface{}, error) {
+	switch f.Extension() {
+	case "yaml", "yml":
+		return parser.HandleYamlMetaData(f.Bytes())
+	case "json":
+		return parser.HandleJsonMetaData(f.Bytes())
+	case "toml":
+		return parser.HandleTomlMetaData(f.Bytes())
+	default:
+		return nil, errors.New("Not supported for data: " + f.Extension())
+	}
+}
+
 func (s *Site) Process() (err error) {
 	if err = s.initialize(); err != nil {
 		return
 	}
 	s.prepTemplates()
 	s.Tmpl.PrintErrors()
+	if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil {
+		return
+	}
+	s.timerStep("load data")
 	s.timerStep("initialize & template prep")
 	if err = s.CreatePages(); err != nil {
 		return
@@ -379,6 +441,7 @@ func (s *Site) initializeSiteInfo() {
 		Menus:           &s.Menus,
 		Params:          params,
 		Permalinks:      permalinks,
+		Data:            &s.Data,
 	}
 }
 
@@ -386,6 +449,10 @@ func (s *Site) hasTheme() bool {
 	return viper.GetString("theme") != ""
 }
 
+func (s *Site) absDataDir() string {
+	return helpers.AbsPathify(viper.GetString("DataDir"))
+}
+
 func (s *Site) absThemeDir() string {
 	return helpers.AbsPathify("themes/" + viper.GetString("theme"))
 }

hugolib/site_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"html/template"
 	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -745,3 +746,18 @@ func TestWeightedTaxonomies(t *testing.T) {
 		t.Errorf("Pages in unexpected order, 'bza' expected first, got '%v'", s.Taxonomies["categories"]["e"][0].Page.Title)
 	}
 }
+
+func TestDataDir(t *testing.T) {
+	sources := []source.ByteSource{
+		{filepath.FromSlash("test" + string(os.PathSeparator) + "foo.yaml"), []byte("bar: foofoo")},
+		{filepath.FromSlash("test.yaml"), []byte("hello:\n- world: foo")},
+	}
+
+	s := &Site{}
+	s.loadData(&source.InMemorySource{ByteSource: sources})
+
+	expected := "map[test:map[hello:[map[world:foo]] foo:map[bar:foofoo]]]"
+	if fmt.Sprint(s.Data) != expected {
+		t.Errorf("Expected structure '%s', got '%s'", expected, s.Data)
+	}
+}