From fcd63de3a54fadcd30972654d8eb86dc4d889784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 5 Jun 2021 12:44:45 +0200 Subject: [PATCH] tpl/data: Misc header improvements, tests, allow multiple headers of same key Closes #5617 --- common/types/convert.go | 41 ++- common/types/convert_test.go | 2 + docs/content/en/templates/data-templates.md | 34 +-- tpl/data/data.go | 99 +++++-- tpl/data/data_test.go | 307 ++++++++++++-------- tpl/data/resources.go | 9 +- 6 files changed, 304 insertions(+), 188 deletions(-) diff --git a/common/types/convert.go b/common/types/convert.go index 137029a0e..7beb3404e 100644 --- a/common/types/convert.go +++ b/common/types/convert.go @@ -15,21 +15,52 @@ package types import ( "encoding/json" + "fmt" "html/template" + "reflect" "github.com/spf13/cast" ) -// ToStringSlicePreserveString converts v to a string slice. -// If v is a string, it will be wrapped in a string slice. +// ToStringSlicePreserveString is the same as ToStringSlicePreserveStringE, +// but it never fails. func ToStringSlicePreserveString(v interface{}) []string { + vv, _ := ToStringSlicePreserveStringE(v) + return vv +} + +// ToStringSlicePreserveStringE converts v to a string slice. +// If v is a string, it will be wrapped in a string slice. +func ToStringSlicePreserveStringE(v interface{}) ([]string, error) { if v == nil { - return nil + return nil, nil } if sds, ok := v.(string); ok { - return []string{sds} + return []string{sds}, nil } - return cast.ToStringSlice(v) + result, err := cast.ToStringSliceE(v) + if err == nil { + return result, nil + } + + // Probably []int or similar. Fall back to reflect. + vv := reflect.ValueOf(v) + + switch vv.Kind() { + case reflect.Slice, reflect.Array: + result = make([]string, vv.Len()) + for i := 0; i < vv.Len(); i++ { + s, err := cast.ToStringE(vv.Index(i).Interface()) + if err != nil { + return nil, err + } + result[i] = s + } + return result, nil + default: + return nil, fmt.Errorf("failed to convert %T to a string slice", v) + } + } // TypeToString converts v to a string if it's a valid string type. diff --git a/common/types/convert_test.go b/common/types/convert_test.go index d053ede60..364228f41 100644 --- a/common/types/convert_test.go +++ b/common/types/convert_test.go @@ -24,7 +24,9 @@ func TestToStringSlicePreserveString(t *testing.T) { c := qt.New(t) c.Assert(ToStringSlicePreserveString("Hugo"), qt.DeepEquals, []string{"Hugo"}) + c.Assert(ToStringSlicePreserveString(qt.Commentf("Hugo")), qt.DeepEquals, []string{"Hugo"}) c.Assert(ToStringSlicePreserveString([]interface{}{"A", "B"}), qt.DeepEquals, []string{"A", "B"}) + c.Assert(ToStringSlicePreserveString([]int{1, 3}), qt.DeepEquals, []string{"1", "3"}) c.Assert(ToStringSlicePreserveString(nil), qt.IsNil) } diff --git a/docs/content/en/templates/data-templates.md b/docs/content/en/templates/data-templates.md index 661c0bdfa..b3edf8f44 100644 --- a/docs/content/en/templates/data-templates.md +++ b/docs/content/en/templates/data-templates.md @@ -114,19 +114,10 @@ You can use the following code to render the `Short Description` in your layout: Note the use of the [`markdownify` template function][markdownify]. This will send the description through the Blackfriday Markdown rendering engine. - -## Data-Driven Content +## Get Remote Data -In addition to the [data files](/extras/datafiles/) feature, Hugo also has a "data-driven content" feature, which lets you load any [JSON](https://www.json.org/) or [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) file from nearly any resource. - -Data-driven content currently consists of two functions, `getJSON` and `getCSV`, which are available in all template files. - -## Implementation details - -### Call the Functions with a URL - -In your template, call the functions like this: +Use `getJSON` or `getCSV` to get remote data: ``` {{ $dataJ := getJSON "url" }} @@ -155,19 +146,18 @@ This will resolve internally to the following: {{ $gistJ := getJSON "https://api.github.com/users/GITHUB_USERNAME/gists" }} ``` -Finally, you can range over an array. This example will output the -first 5 gists for a GitHub user: +### Add HTTP headers + +{{< new-in "0.84.0" >}} Both `getJSON` and `getCSV` takes an optional map as the last argument, e.g.: ``` - +{{ $data := getJSON "https://example.org/api" (dict "Authorization" "Bearer abcd") }} +``` + +If you need multiple values for the same header key, use a slice: + +``` +{{ $data := getJSON "https://example.org/api" (dict "X-List" (slice "a" "b" "c")) }} ``` ### Example for CSV files diff --git a/tpl/data/data.go b/tpl/data/data.go index 4cb8b5e78..e993ed140 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -23,6 +23,10 @@ import ( "net/http" "strings" + "github.com/gohugoio/hugo/common/maps" + + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/common/constants" "github.com/gohugoio/hugo/common/loggers" @@ -59,14 +63,10 @@ type Namespace struct { // If you provide multiple parts for the URL they will be joined together to the final URL. // GetCSV returns nil or a slice slice to use in a short code. func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err error) { - url := joinURL(args) + url, headers := toURLAndHeaders(args) cache := ns.cacheGetCSV unmarshal := func(b []byte) (bool, error) { - if !bytes.Contains(b, []byte(sep)) { - return false, _errors.Errorf("cannot find separator %s in CSV for %s", sep, url) - } - if d, err = parseCSV(b, sep); err != nil { err = _errors.Wrapf(err, "failed to parse CSV file %s", url) @@ -82,17 +82,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url) } - req.Header.Add("Accept", "text/csv") - req.Header.Add("Accept", "text/plain") - - // Add custom user headers to the get request - finalArg := args[len(args)-1] - - if userHeaders, ok := finalArg.(map[string]interface{}); ok { - for key, val := range userHeaders { - req.Header.Add(key, val.(string)) - } - } + // Add custom user headers. + addUserProvidedHeaders(headers, req) + addDefaultHeaders(req, "text/csv", "text/plain") err = ns.getResource(cache, unmarshal, req) if err != nil { @@ -108,7 +100,7 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err // GetJSON returns nil or parsed JSON to use in a short code. func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { var v interface{} - url := joinURL(args) + url, headers := toURLAndHeaders(args) cache := ns.cacheGetJSON req, err := http.NewRequest("GET", url, nil) @@ -124,17 +116,8 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { return false, nil } - req.Header.Add("Accept", "application/json") - req.Header.Add("User-Agent", "Hugo Static Site Generator") - - // Add custom user headers to the get request - finalArg := args[len(args)-1] - - if userHeaders, ok := finalArg.(map[string]interface{}); ok { - for key, val := range userHeaders { - req.Header.Add(key, val.(string)) - } - } + addUserProvidedHeaders(headers, req) + addDefaultHeaders(req, "application/json") err = ns.getResource(cache, unmarshal, req) if err != nil { @@ -145,8 +128,64 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { return v, nil } -func joinURL(urlParts []interface{}) string { - return strings.Join(cast.ToStringSlice(urlParts), "") +func addDefaultHeaders(req *http.Request, accepts ...string) { + for _, accept := range accepts { + if !hasHeaderValue(req.Header, "Accept", accept) { + req.Header.Add("Accept", accept) + } + } + if !hasHeaderKey(req.Header, "User-Agent") { + req.Header.Add("User-Agent", "Hugo Static Site Generator") + } +} + +func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { + if headers == nil { + return + } + for key, val := range headers { + vals := types.ToStringSlicePreserveString(val) + for _, s := range vals { + req.Header.Add(key, s) + } + } +} + +func hasHeaderValue(m http.Header, key, value string) bool { + var s []string + var ok bool + + if s, ok = m[key]; !ok { + return false + } + + for _, v := range s { + if v == value { + return true + } + } + return false +} + +func hasHeaderKey(m http.Header, key string) bool { + _, ok := m[key] + return ok +} + +func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) { + if len(urlParts) == 0 { + return "", nil + } + + // The last argument may be a map. + headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1]) + if err == nil { + urlParts = urlParts[:len(urlParts)-1] + } else { + headers = nil + } + + return strings.Join(cast.ToStringSlice(urlParts), ""), headers } // parseCSV parses bytes of CSV data into a slice slice string or an error diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go index 6b62a2b0d..8a18a19e4 100644 --- a/tpl/data/data_test.go +++ b/tpl/data/data_test.go @@ -14,12 +14,16 @@ package data import ( + "bytes" + "html/template" "net/http" "net/http/httptest" "path/filepath" "strings" "testing" + "github.com/gohugoio/hugo/common/maps" + qt "github.com/frankban/quicktest" ) @@ -46,12 +50,6 @@ func TestGetCSV(t *testing.T) { "gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n", false, }, - { - ",", - `http://error.no.sep/`, - "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n", - false, - }, { ",", `http://nofound/404`, @@ -73,66 +71,54 @@ func TestGetCSV(t *testing.T) { false, }, } { - msg := qt.Commentf("Test %d", i) - ns := newTestNs() + c.Run(test.url, func(c *qt.C) { + msg := qt.Commentf("Test %d", i) - // Setup HTTP test server - var srv *httptest.Server - srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { - if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") { - http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + ns := newTestNs() + + // Setup HTTP test server + var srv *httptest.Server + srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { + if !hasHeaderValue(r.Header, "Accept", "text/csv") && !hasHeaderValue(r.Header, "Accept", "text/plain") { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + if r.URL.Path == "/404" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + w.Header().Add("Content-type", "text/csv") + + w.Write([]byte(test.content)) + }) + defer func() { srv.Close() }() + + // Setup local test file for schema-less URLs + if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { + f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) + c.Assert(err, qt.IsNil, msg) + f.WriteString(test.content) + f.Close() + } + + // Get on with it + got, err := ns.GetCSV(test.sep, test.url) + + if _, ok := test.expect.(bool); ok { + c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) + c.Assert(got, qt.IsNil) return } - if r.URL.Path == "/404" { - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - - w.Header().Add("Content-type", "text/csv") - - w.Write([]byte(test.content)) - }) - defer func() { srv.Close() }() - - // Setup local test file for schema-less URLs - if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { - f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) c.Assert(err, qt.IsNil, msg) - f.WriteString(test.content) - f.Close() - } + c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0) + c.Assert(got, qt.Not(qt.IsNil), msg) + c.Assert(got, qt.DeepEquals, test.expect, msg) + }) - // Get on with it - got, err := ns.GetCSV(test.sep, test.url) - - if _, ok := test.expect.(bool); ok { - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) - // c.Assert(err, msg, qt.Not(qt.IsNil)) - c.Assert(got, qt.IsNil) - continue - } - - c.Assert(err, qt.IsNil, msg) - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0) - c.Assert(got, qt.Not(qt.IsNil), msg) - c.Assert(got, qt.DeepEquals, test.expect, msg) - - // Test user-defined headers as well - gotHeader, _ := ns.GetCSV(test.sep, test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"}) - - if _, ok := test.expect.(bool); ok { - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) - // c.Assert(err, msg, qt.Not(qt.IsNil)) - c.Assert(got, qt.IsNil) - continue - } - - c.Assert(err, qt.IsNil, msg) - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0) - c.Assert(gotHeader, qt.Not(qt.IsNil), msg) - c.Assert(gotHeader, qt.DeepEquals, test.expect, msg) } } @@ -178,68 +164,153 @@ func TestGetJSON(t *testing.T) { }, } { - msg := qt.Commentf("Test %d", i) - ns := newTestNs() + c.Run(test.url, func(c *qt.C) { - // Setup HTTP test server - var srv *httptest.Server - srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { - if !haveHeader(r.Header, "Accept", "application/json") { - http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + msg := qt.Commentf("Test %d", i) + ns := newTestNs() + + // Setup HTTP test server + var srv *httptest.Server + srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { + if !hasHeaderValue(r.Header, "Accept", "application/json") { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + if r.URL.Path == "/404" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + w.Header().Add("Content-type", "application/json") + + w.Write([]byte(test.content)) + }) + defer func() { srv.Close() }() + + // Setup local test file for schema-less URLs + if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { + f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) + c.Assert(err, qt.IsNil, msg) + f.WriteString(test.content) + f.Close() + } + + // Get on with it + got, _ := ns.GetJSON(test.url) + + if _, ok := test.expect.(bool); ok { + c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) return } - if r.URL.Path == "/404" { - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } + c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg) + c.Assert(got, qt.Not(qt.IsNil), msg) + c.Assert(got, qt.DeepEquals, test.expect) - w.Header().Add("Content-type", "application/json") - - w.Write([]byte(test.content)) }) - defer func() { srv.Close() }() - - // Setup local test file for schema-less URLs - if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { - f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) - c.Assert(err, qt.IsNil, msg) - f.WriteString(test.content) - f.Close() - } - - // Get on with it - got, _ := ns.GetJSON(test.url) - - if _, ok := test.expect.(bool); ok { - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) - // c.Assert(err, msg, qt.Not(qt.IsNil)) - continue - } - - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg) - c.Assert(got, qt.Not(qt.IsNil), msg) - c.Assert(got, qt.DeepEquals, test.expect) - - // Test user-defined headers as well - gotHeader, _ := ns.GetJSON(test.url, map[string]interface{}{"Accept-Charset": "utf-8", "Max-Forwards": "10"}) - - if _, ok := test.expect.(bool); ok { - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 1) - // c.Assert(err, msg, qt.Not(qt.IsNil)) - continue - } - - c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0, msg) - c.Assert(gotHeader, qt.Not(qt.IsNil), msg) - c.Assert(gotHeader, qt.DeepEquals, test.expect) } } -func TestJoinURL(t *testing.T) { +func TestHeaders(t *testing.T) { t.Parallel() c := qt.New(t) - c.Assert(joinURL([]interface{}{"https://foo?id=", 32}), qt.Equals, "https://foo?id=32") + + for _, test := range []struct { + name string + headers interface{} + assert func(c *qt.C, headers string) + }{ + { + `Misc header variants`, + map[string]interface{}{ + "Accept-Charset": "utf-8", + "Max-forwards": "10", + "X-Int": 32, + "X-Templ": template.HTML("a"), + "X-Multiple": []string{"a", "b"}, + "X-MultipleInt": []int{3, 4}, + }, + func(c *qt.C, headers string) { + c.Assert(headers, qt.Contains, "Accept-Charset: utf-8") + c.Assert(headers, qt.Contains, "Max-Forwards: 10") + c.Assert(headers, qt.Contains, "X-Int: 32") + c.Assert(headers, qt.Contains, "X-Templ: a") + c.Assert(headers, qt.Contains, "X-Multiple: a") + c.Assert(headers, qt.Contains, "X-Multiple: b") + c.Assert(headers, qt.Contains, "X-Multipleint: 3") + c.Assert(headers, qt.Contains, "X-Multipleint: 4") + c.Assert(headers, qt.Contains, "User-Agent: Hugo Static Site Generator") + }, + }, + { + `Params`, + maps.Params{ + "Accept-Charset": "utf-8", + }, + func(c *qt.C, headers string) { + c.Assert(headers, qt.Contains, "Accept-Charset: utf-8") + }, + }, + { + `Override User-Agent`, + map[string]interface{}{ + "User-Agent": "007", + }, + func(c *qt.C, headers string) { + c.Assert(headers, qt.Contains, "User-Agent: 007") + }, + }, + } { + + c.Run(test.name, func(c *qt.C) { + + ns := newTestNs() + + // Setup HTTP test server + var srv *httptest.Server + var headers bytes.Buffer + srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { + c.Assert(r.URL.String(), qt.Equals, "http://gohugo.io/api?foo") + w.Write([]byte("{}")) + r.Header.Write(&headers) + + }) + defer func() { srv.Close() }() + + testFunc := func(fn func(args ...interface{}) error) { + defer headers.Reset() + err := fn("http://example.org/api", "?foo", test.headers) + + c.Assert(err, qt.IsNil) + c.Assert(int(ns.deps.Log.LogCounters().ErrorCounter.Count()), qt.Equals, 0) + test.assert(c, headers.String()) + } + + testFunc(func(args ...interface{}) error { + _, err := ns.GetJSON(args...) + return err + }) + testFunc(func(args ...interface{}) error { + _, err := ns.GetCSV(",", args...) + return err + }) + + }) + + } +} + +func TestToURLAndHeaders(t *testing.T) { + t.Parallel() + c := qt.New(t) + url, headers := toURLAndHeaders([]interface{}{"https://foo?id=", 32}) + c.Assert(url, qt.Equals, "https://foo?id=32") + c.Assert(headers, qt.IsNil) + + url, headers = toURLAndHeaders([]interface{}{"https://foo?id=", 32, map[string]interface{}{"a": "b"}}) + c.Assert(url, qt.Equals, "https://foo?id=32") + c.Assert(headers, qt.DeepEquals, map[string]interface{}{"a": "b"}) } func TestParseCSV(t *testing.T) { @@ -276,19 +347,3 @@ func TestParseCSV(t *testing.T) { c.Assert(act, qt.Equals, test.exp, msg) } } - -func haveHeader(m http.Header, key, needle string) bool { - var s []string - var ok bool - - if s, ok = m[key]; !ok { - return false - } - - for _, v := range s { - if v == needle { - return true - } - } - return false -} diff --git a/tpl/data/resources.go b/tpl/data/resources.go index ba98f12b4..68f18c48e 100644 --- a/tpl/data/resources.go +++ b/tpl/data/resources.go @@ -14,6 +14,7 @@ package data import ( + "bytes" "io/ioutil" "net/http" "net/url" @@ -37,7 +38,9 @@ var ( // getRemote loads the content of a remote file. This method is thread safe. func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error { url := req.URL.String() - id := helpers.MD5String(url) + var headers bytes.Buffer + req.Header.Write(&headers) + id := helpers.MD5String(url + headers.String()) var handled bool var retry bool @@ -94,10 +97,6 @@ func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (b // getLocal loads the content of a local file func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) { filename := filepath.Join(cfg.GetString("workingDir"), url) - if e, err := helpers.Exists(filename, fs); !e { - return nil, err - } - return afero.ReadFile(fs, filename) }