From 5e2d3d2e10a4cf1935c2226b023bde2f3ce9e7c0 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sat, 27 Dec 2014 14:40:10 +1100 Subject: [PATCH] Merge to stay up to date with current dev Feature: GetJson and GetJson in short codes or other layout files. For more details please see: http://cyrillschumacher.com/2014/12/21/dynamic-pages-with-gohugo.io/ --- commands/hugo.go | 12 +- helpers/path.go | 25 ++++ helpers/path_test.go | 28 +++++ tpl/template_resources.go | 220 +++++++++++++++++++++++++++++++++ tpl/template_resources_test.go | 181 +++++++++++++++++++++++++++ 5 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 tpl/template_resources.go create mode 100644 tpl/template_resources_test.go diff --git a/commands/hugo.go b/commands/hugo.go index 2a3f99b1e..4d7c36105 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -56,7 +56,7 @@ var hugoCmdV *cobra.Command //Flags that are to be added to commands. var BuildWatch, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool -var Source, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string +var Source, CacheDir, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string //Execute adds all child commands to the root command HugoCmd and sets flags appropriately. func Execute() { @@ -83,6 +83,7 @@ func init() { HugoCmd.PersistentFlags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files") HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file") HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from") + HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "$TMPDIR/hugo_cache/", "filesystem path to cache directory") HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to") HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)") HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output") @@ -203,6 +204,15 @@ func InitializeConfig() { viper.Set("WorkingDir", dir) } + if CacheDir != "" { + if helpers.FilePathSeparator != CacheDir[len(CacheDir)-1:] { + CacheDir = CacheDir + helpers.FilePathSeparator + } + viper.Set("CacheDir", CacheDir) + } else { + viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs)) + } + if VerboseLog || Logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") { if viper.IsSet("LogFile") && viper.GetString("LogFile") != "" { jww.SetLogFile(viper.GetString("LogFile")) diff --git a/helpers/path.go b/helpers/path.go index 9c3c2ba15..f6c77f9c3 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -437,3 +437,28 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { _, err = io.Copy(file, r) return } + +// GetTempDir returns the OS default temp directory with trailing slash +// if subPath is not empty then it will be created recursively +func GetTempDir(subPath string, fs afero.Fs) string { + dir := os.TempDir() + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + if subPath != "" { + dir = dir + MakePath(subPath) + + if exists, _ := Exists(dir, fs); exists { + return dir + } + + err := fs.MkdirAll(dir, 0777) // rwx, rw, r + if err != nil { + panic(err) + } + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + } + return dir +} diff --git a/helpers/path_test.go b/helpers/path_test.go index bc0a54790..9be1c9fc4 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -647,3 +647,31 @@ func TestWriteToDisk(t *testing.T) { reader.Seek(0, 0) } } + +func TestGetTempDir(t *testing.T) { + dir := os.TempDir() + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + testDir := "hugoTestFolder" + FilePathSeparator + tests := []struct { + input string + expected string + }{ + {"", dir}, + {testDir + " Foo bar ", dir + testDir + "--Foo-bar" + FilePathSeparator}, + {testDir + "Foo.Bar/foo_Bar-Foo", dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator}, + {testDir + "fOO,bar:foo%bAR", dir + testDir + "fOObarfoobAR" + FilePathSeparator}, + {testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator}, + {testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator}, + {testDir + "은행", dir + testDir + "은행" + FilePathSeparator}, + {testDir + "Банковский кассир", dir + testDir + "Банковский-кассир" + FilePathSeparator}, + } + + for _, test := range tests { + output := GetTempDir(test.input, new(afero.MemMapFs)) + if output != test.expected { + t.Errorf("Expected %#v, got %#v\n", test.expected, output) + } + } +} diff --git a/tpl/template_resources.go b/tpl/template_resources.go new file mode 100644 index 000000000..d4e105fcb --- /dev/null +++ b/tpl/template_resources.go @@ -0,0 +1,220 @@ +// Copyright © 2013-14 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tpl + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/hugofs" + jww "github.com/spf13/jwalterweatherman" + "github.com/spf13/viper" +) + +var remoteUrlLock = &remoteLock{m: make(map[string]*sync.Mutex)} + +type remoteLock struct { + sync.RWMutex + m map[string]*sync.Mutex +} + +// resLock locks an URL during download +func (l *remoteLock) UrlLock(url string) { + l.Lock() + if _, ok := l.m[url]; !ok { + l.m[url] = &sync.Mutex{} + } + l.Unlock() // call this Unlock before the next lock will be called. NFI why but defer doesn't work. + l.m[url].Lock() +} + +// resUnlock unlocks an URL when the download has been finished. Use only in defer calls. +func (l *remoteLock) UrlUnlock(url string) { + l.RLock() + defer l.RUnlock() + if um, ok := l.m[url]; ok { + um.Unlock() + } +} + +// getFileID returns the cache ID for a string +func getCacheFileID(id string) string { + return viper.GetString("CacheDir") + url.QueryEscape(id) +} + +// resGetCache returns the content for an ID from the file cache or an error +// if the file is not found returns nil,nil +func resGetCache(id string, fs afero.Fs) ([]byte, error) { + fID := getCacheFileID(id) + isExists, err := helpers.Exists(fID, fs) + if err != nil { + return nil, err + } + if !isExists { + return nil, nil + } + + f, err := fs.Open(fID) + if err != nil { + return nil, err + } + + return ioutil.ReadAll(f) +} + +// resWriteCache writes bytes to an ID into the file cache +func resWriteCache(id string, c []byte, fs afero.Fs) error { + fID := getCacheFileID(id) + f, err := fs.Create(fID) + if err != nil { + return err + } + n, err := f.Write(c) + if n == 0 { + return errors.New("No bytes written to file: " + fID) + } + return err +} + +// resGetRemote loads the content of a remote file. This method is thread safe. +func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) { + + c, err := resGetCache(url, fs) + if c != nil && err == nil { + return c, nil + } + if err != nil { + return nil, err + } + + // avoid race condition with locks, block other goroutines if the current url is processing + remoteUrlLock.UrlLock(url) + defer func() { remoteUrlLock.UrlUnlock(url) }() + + // avoid multiple locks due to calling resGetCache twice + c, err = resGetCache(url, fs) + if c != nil && err == nil { + return c, nil + } + if err != nil { + return nil, err + } + + jww.INFO.Printf("Downloading: %s ...", url) + res, err := hc.Get(url) + if err != nil { + return nil, err + } + c, err = ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, err + } + err = resWriteCache(url, c, fs) + if err != nil { + return nil, err + } + jww.INFO.Printf("... and cached to: %s", getCacheFileID(url)) + return c, nil +} + +// resGetLocal loads the content of a local file +func resGetLocal(url string, fs afero.Fs) ([]byte, error) { + p := "" + if viper.GetString("WorkingDir") != "" { + p = viper.GetString("WorkingDir") + if helpers.FilePathSeparator != p[len(p)-1:] { + p = p + helpers.FilePathSeparator + } + } + jFile := p + url + if e, err := helpers.Exists(jFile, fs); !e { + return nil, err + } + + f, err := fs.Open(jFile) + if err != nil { + return nil, err + } + return ioutil.ReadAll(f) +} + +// resGetResource loads the content of a local or remote file +func resGetResource(url string) ([]byte, error) { + if url == "" { + return nil, nil + } + if strings.Contains(url, "://") { + return resGetRemote(url, hugofs.SourceFs, http.DefaultClient) + } + return resGetLocal(url, hugofs.SourceFs) +} + +// GetJson expects the url to a resource which can either be a local or a remote one. +// GetJson returns nil or parsed JSON to use in a short code. +func GetJson(url string) interface{} { + c, err := resGetResource(url) + if err != nil { + jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err) + return nil + } + + var v interface{} + err = json.Unmarshal(c, &v) + if err != nil { + jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err) + return nil + } + return v +} + +// parseCsv parses bytes of csv data into a slice slice string or an error +func parseCsv(c []byte, sep string) ([][]string, error) { + if len(sep) != 1 { + return nil, errors.New("Incorrect length of csv separator: " + sep) + } + b := bytes.NewReader(c) + r := csv.NewReader(b) + rSep := []rune(sep) + r.Comma = rSep[0] + r.FieldsPerRecord = 0 + return r.ReadAll() +} + +// GetCsv expects the url to a resource which can either be a local or a remote one and the type +// of the data separator which can be comma, semi-colon, pipe, but only one character. +// GetCsv returns nil or a slice slice to use in a short code. +func GetCsv(url string, sep string) [][]string { + + c, err := resGetResource(url) + if err != nil { + jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err) + return nil + } + d, err := parseCsv(c, sep) + if err != nil { + jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err) + return nil + } + return d +} diff --git a/tpl/template_resources_test.go b/tpl/template_resources_test.go new file mode 100644 index 000000000..40ea1fc43 --- /dev/null +++ b/tpl/template_resources_test.go @@ -0,0 +1,181 @@ +// Copyright © 2013-14 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tpl + +import ( + "bytes" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" +) + +func TestScpCache(t *testing.T) { + + tests := []struct { + path string + content []byte + }{ + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, + {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`)}, + {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`)}, + {"трям/трям", []byte(`T€st трям/трям Content 123`)}, + {"은행", []byte(`T€st C은행ontent 123`)}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)}, + } + + fs := new(afero.MemMapFs) + + for _, test := range tests { + c, err := resGetCache(test.path, fs) + if err != nil { + t.Errorf("Error getting cache: %s", err) + } + if c != nil { + t.Errorf("There is content where there should not be anything: %s", string(c)) + } + + err = resWriteCache(test.path, test.content, fs) + if err != nil { + t.Errorf("Error writing cache: %s", err) + } + + c, err = resGetCache(test.path, fs) + if err != nil { + t.Errorf("Error getting cache after writing: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + } + } +} + +func TestScpGetLocal(t *testing.T) { + fs := new(afero.MemMapFs) + ps := helpers.FilePathSeparator + tests := []struct { + path string + content []byte + }{ + {"testpath" + ps + "test.txt", []byte(`T€st Content 123 fOO,bar:foo%bAR`)}, + {"FOo" + ps + "BaR.html", []byte(`FOo/BaR.html T€st Content 123`)}, + {"трям" + ps + "трям", []byte(`T€st трям/трям Content 123`)}, + {"은행", []byte(`T€st C은행ontent 123`)}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)}, + } + + for _, test := range tests { + r := bytes.NewReader(test.content) + err := helpers.WriteToDisk(test.path, r, fs) + if err != nil { + t.Error(err) + } + + c, err := resGetLocal(test.path, fs) + if err != nil { + t.Errorf("Error getting resource content: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + } + } + +} + +func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, *http.Client) { + testServer := httptest.NewServer(http.HandlerFunc(handler)) + client := &http.Client{ + Transport: &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return url.Parse(testServer.URL) }}, + } + return testServer, client +} + +func TestScpGetRemote(t *testing.T) { + fs := new(afero.MemMapFs) + + tests := []struct { + path string + content []byte + }{ + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, + {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)}, + {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)}, + } + + for _, test := range tests { + + srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) { + w.Write(test.content) + }) + defer func() { srv.Close() }() + + c, err := resGetRemote(test.path, fs, cl) + if err != nil { + t.Errorf("Error getting resource content: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c)) + } + cc, cErr := resGetCache(test.path, fs) + if cErr != nil { + t.Error(cErr) + } + if bytes.Compare(cc, test.content) != 0 { + t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(c)) + } + } +} + +func TestParseCsv(t *testing.T) { + + tests := []struct { + csv []byte + sep string + exp string + err bool + }{ + {[]byte("a,b,c\nd,e,f\n"), "", "", true}, + {[]byte("a,b,c\nd,e,f\n"), "~/", "", true}, + {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false}, + {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false}, + {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true}, + {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false}, + } + for _, test := range tests { + csv, err := parseCsv(test.csv, test.sep) + if test.err && err == nil { + t.Error("Expecting an error") + } + if test.err { + continue + } + if !test.err && err != nil { + t.Error(err) + } + + act := "" + for _, v := range csv { + act = act + strings.Join(v, "") + } + + if act != test.exp { + t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv) + } + + } +}