Add data files support in themes

If duplicate keys, the main data dir wins.

Fixes #892

commands/hugo.go

@@ -252,13 +252,13 @@ func copyStatic() error {
 	syncer.SrcFs = hugofs.SourceFs
 	syncer.DestFs = hugofs.DestinationFS
 
-	if themeSet() {
-		themeDir := helpers.AbsPathify("themes/"+viper.GetString("theme")) + "/static/"
-		if _, err := os.Stat(themeDir); os.IsNotExist(err) {
-			jww.ERROR.Println("Unable to find static directory for theme:", viper.GetString("theme"), "in", themeDir)
-			return nil
-		}
+	themeDir, err := helpers.GetThemeStaticDirPath()
+	if err != nil {
+		jww.ERROR.Println(err)
+		return nil
+	}
 
+	if themeDir != "" {
 		// Copy Static to Destination
 		jww.INFO.Println("syncing from", themeDir, "to", publishDir)
 		utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir))
@@ -292,17 +292,13 @@ func getDirList() []string {
 	filepath.Walk(helpers.AbsPathify(viper.GetString("ContentDir")), walker)
 	filepath.Walk(helpers.AbsPathify(viper.GetString("LayoutDir")), walker)
 	filepath.Walk(helpers.AbsPathify(viper.GetString("StaticDir")), walker)
-	if themeSet() {
+	if helpers.ThemeSet() {
 		filepath.Walk(helpers.AbsPathify("themes/"+viper.GetString("theme")), walker)
 	}
 
 	return a
 }
 
-func themeSet() bool {
-	return viper.GetString("theme") != ""
-}
-
 func buildSite(watching ...bool) (err error) {
 	startTime := time.Now()
 	site := &hugolib.Site{}

helpers/general.go

@@ -19,13 +19,13 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	bp "github.com/spf13/hugo/bufferpool"
+	"github.com/spf13/viper"
 	"io"
 	"net"
 	"path/filepath"
 	"reflect"
 	"strings"
-
-	bp "github.com/spf13/hugo/bufferpool"
 )
 
 // Filepath separator defined by os.Separator.
@@ -100,6 +100,10 @@ func BytesToReader(in []byte) io.Reader {
 	return bytes.NewReader(in)
 }
 
+func ThemeSet() bool {
+	return viper.GetString("theme") != ""
+}
+
 // SliceToLower goes through the source slice and lowers all values.
 func SliceToLower(s []string) []string {
 	if s == nil {

helpers/path.go

@@ -178,6 +178,29 @@ func GetStaticDirPath() string {
 	return AbsPathify(viper.GetString("StaticDir"))
 }
 
+// GetThemeStaticDirPath returns the theme's static dir path if theme is set.
+// If theme is set and the static dir doesn't exist, an error is returned.
+func GetThemeStaticDirPath() (string, error) {
+	return getThemeDirPath("static")
+}
+
+// GetThemeStaticDirPath returns the theme's data dir path if theme is set.
+// If theme is set and the data dir doesn't exist, an error is returned.
+func GetThemeDataDirPath() (string, error) {
+	return getThemeDirPath("data")
+}
+
+func getThemeDirPath(path string) (string, error) {
+	var themeDir string
+	if ThemeSet() {
+		themeDir = AbsPathify("themes/"+viper.GetString("theme")) + FilePathSeparator + path
+		if _, err := os.Stat(themeDir); os.IsNotExist(err) {
+			return "", fmt.Errorf("Unable to find %s directory for theme %s in %s", path, viper.GetString("theme"), themeDir)
+		}
+	}
+	return themeDir, nil
+}
+
 func GetThemesDirPath() string {
 	return AbsPathify(filepath.Join("themes", viper.GetString("theme"), "static"))
 }

hugolib/site.go

@@ -267,42 +267,46 @@ func (s *Site) addTemplate(name, data string) error {
 	return s.Tmpl.AddTemplate(name, data)
 }
 
-func (s *Site) loadData(fs source.Input) (err error) {
+func (s *Site) loadData(sources []source.Input) (err error) {
 	s.Data = make(map[string]interface{})
 	var current map[string]interface{}
-
-	for _, r := range fs.Files() {
-		// Crawl in data tree to insert data
-		current = s.Data
-		for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) {
-			if key != "" {
-				if _, ok := current[key]; !ok {
-					current[key] = make(map[string]interface{})
+	for _, currentSource := range sources {
+		for _, r := range currentSource.Files() {
+			// Crawl in data tree to insert data
+			current = s.Data
+			for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) {
+				if key != "" {
+					if _, ok := current[key]; !ok {
+						current[key] = make(map[string]interface{})
+					}
+					current = current[key].(map[string]interface{})
 				}
-				current = current[key].(map[string]interface{})
 			}
-		}
 
-		data, err := readData(r)
-		if err != nil {
-			return fmt.Errorf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err)
-		}
+			data, err := readData(r)
+			if err != nil {
+				return fmt.Errorf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err)
+			}
 
-		// Copy content from current to data when needed
-		if _, ok := current[r.BaseFileName()]; ok {
-			data := data.(map[string]interface{})
+			// Copy content from current to data when needed
+			if _, ok := current[r.BaseFileName()]; ok {
+				data := data.(map[string]interface{})
 
-			for key, value := range current[r.BaseFileName()].(map[string]interface{}) {
-				if _, override := data[key]; override {
-					// filepath.Walk walks the files in lexical order, '/' comes before '.'
-					jww.ERROR.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path())
+				for key, value := range current[r.BaseFileName()].(map[string]interface{}) {
+					if _, override := data[key]; override {
+						// filepath.Walk walks the files in lexical order, '/' comes before '.'
+						// this warning could happen if
+						// 1. A theme uses the same key; the main data folder wins
+						// 2. A sub folder uses the same key: the sub folder wins
+						jww.WARN.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path())
+					}
+					data[key] = value
 				}
-				data[key] = value
 			}
-		}
 
-		// Insert data
-		current[r.BaseFileName()] = data
+			// Insert data
+			current[r.BaseFileName()] = data
+		}
 	}
 
 	return
@@ -329,7 +333,17 @@ func (s *Site) Process() (err error) {
 	s.Tmpl.PrintErrors()
 	s.timerStep("initialize & template prep")
 
-	if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil {
+	dataSources := make([]source.Input, 0, 2)
+
+	dataSources = append(dataSources, &source.Filesystem{Base: s.absDataDir()})
+
+	// have to be last - duplicate keys in earlier entries will win
+	themeStaticDir, err := helpers.GetThemeDataDirPath()
+	if err == nil {
+		dataSources = append(dataSources, &source.Filesystem{Base: themeStaticDir})
+	}
+
+	if err = s.loadData(dataSources); err != nil {
 		return
 	}
 	s.timerStep("load data")

hugolib/site_test.go

@@ -760,7 +760,7 @@ func TestDataDirJson(t *testing.T) {
 		t.Fatalf("Error %s", err)
 	}
 
-	doTestDataDir(t, expected, sources)
+	doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}})
 }
 
 func TestDataDirToml(t *testing.T) {
@@ -774,7 +774,7 @@ func TestDataDirToml(t *testing.T) {
 		t.Fatalf("Error %s", err)
 	}
 
-	doTestDataDir(t, expected, sources)
+	doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}})
 }
 
 func TestDataDirYamlWithOverridenValue(t *testing.T) {
@@ -789,7 +789,24 @@ func TestDataDirYamlWithOverridenValue(t *testing.T) {
 	expected := map[string]interface{}{"a": map[string]interface{}{"a": 1},
 		"test": map[string]interface{}{"v1": map[string]interface{}{"v1-2": 2}, "v2": map[string]interface{}{"v2": []interface{}{2, 3}}}}
 
-	doTestDataDir(t, expected, sources)
+	doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}})
+}
+
+// issue 892
+func TestDataDirMultipleSources(t *testing.T) {
+	s1 := []source.ByteSource{
+		{filepath.FromSlash("test/first.toml"), []byte("[foo]\nbar = 1")},
+	}
+
+	s2 := []source.ByteSource{
+		{filepath.FromSlash("test/first.toml"), []byte("[foo]\nbar = 2")},
+		{filepath.FromSlash("test/second.toml"), []byte("[foo]\ntender = 2")},
+	}
+
+	expected := map[string]interface{}{"a": map[string]interface{}{"a": 1}}
+
+	doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: s1}, &source.InMemorySource{ByteSource: s2}})
+
 }
 
 func TestDataDirUnknownFormat(t *testing.T) {
@@ -797,15 +814,15 @@ func TestDataDirUnknownFormat(t *testing.T) {
 		{filepath.FromSlash("test.roml"), []byte("boo")},
 	}
 	s := &Site{}
-	err := s.loadData(&source.InMemorySource{ByteSource: sources})
+	err := s.loadData([]source.Input{&source.InMemorySource{ByteSource: sources}})
 	if err == nil {
 		t.Fatalf("Should return an error")
 	}
 }
 
-func doTestDataDir(t *testing.T, expected interface{}, sources []source.ByteSource) {
+func doTestDataDir(t *testing.T, expected interface{}, sources []source.Input) {
 	s := &Site{}
-	err := s.loadData(&source.InMemorySource{ByteSource: sources})
+	err := s.loadData(sources)
 	if err != nil {
 		t.Fatalf("Error loading data: %s", err)
 	}