From 08c0de5cc37cd4e512268b8f72ec5a6c68cd5754 Mon Sep 17 00:00:00 2001 From: Cameron Moore Date: Mon, 1 May 2017 22:41:08 -0500 Subject: [PATCH] tpl/data: Clean up data namespace - Move the main GetCSV and GetJSON into data.go. - Add error returns to GetCSV and GetJSON. - Add http client to Namespace for test mocking. - Send accept headers on remote requests. Fixes #3395 - Return an error on non-2XX HTTP response codes and don't retry. - Move cache tests to cache_test.go. --- tpl/data/cache_test.go | 63 ++++++++++ tpl/data/data.go | 114 ++++++++++++++++- tpl/data/data_test.go | 251 +++++++++++++++++++++++++++++++++++++ tpl/data/resources.go | 124 ++++-------------- tpl/data/resources_test.go | 234 ++++------------------------------ 5 files changed, 476 insertions(+), 310 deletions(-) create mode 100644 tpl/data/cache_test.go create mode 100644 tpl/data/data_test.go diff --git a/tpl/data/cache_test.go b/tpl/data/cache_test.go new file mode 100644 index 000000000..6057f0321 --- /dev/null +++ b/tpl/data/cache_test.go @@ -0,0 +1,63 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "fmt" + "testing" + + "github.com/spf13/afero" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestCache(t *testing.T) { + t.Parallel() + + fs := new(afero.MemMapFs) + + for i, test := range []struct { + path string + content []byte + ignore bool + }{ + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, + {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false}, + {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false}, + {"трям/трям", []byte(`T€st трям/трям Content 123`), false}, + {"은행", []byte(`T€st C은행ontent 123`), false}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true}, + } { + msg := fmt.Sprintf("Test #%d: %v", i, test) + + cfg := viper.New() + + c, err := getCache(test.path, fs, cfg, test.ignore) + assert.NoError(t, err, msg) + assert.Nil(t, c, msg) + + err = writeCache(test.path, test.content, fs, cfg, test.ignore) + assert.NoError(t, err, msg) + + c, err = getCache(test.path, fs, cfg, test.ignore) + assert.NoError(t, err, msg) + + if test.ignore { + assert.Nil(t, c, msg) + } else { + assert.Equal(t, string(test.content), string(c)) + } + } +} diff --git a/tpl/data/data.go b/tpl/data/data.go index a5bba32c3..ae0c7c216 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -13,16 +13,126 @@ package data -import "github.com/spf13/hugo/deps" +import ( + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "net/http" + "strings" + "time" + + "github.com/spf13/hugo/deps" + jww "github.com/spf13/jwalterweatherman" +) // New returns a new instance of the data-namespaced template functions. func New(deps *deps.Deps) *Namespace { return &Namespace{ - deps: deps, + deps: deps, + client: http.DefaultClient, } } // Namespace provides template functions for the "data" namespace. type Namespace struct { deps *deps.Deps + + client *http.Client +} + +// GetCSV expects a data separator and one or n-parts of a URL to a resource which +// can either be a local or a remote one. +// The data separator can be a comma, semi-colon, pipe, etc, but only one character. +// If you provide multiple parts for the URL they will be joined together to the final URL. +// GetCSV returns nil or a slice slice to use in a short code. +func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err error) { + url := strings.Join(urlParts, "") + + var clearCacheSleep = func(i int, u string) { + jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) + time.Sleep(resSleep) + deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) + } + + for i := 0; i <= resRetries; i++ { + var req *http.Request + req, err = http.NewRequest("GET", url, nil) + if err != nil { + jww.ERROR.Printf("Failed to create request for getJSON: %s", err) + return nil, err + } + + req.Header.Add("Accept", "text/csv") + req.Header.Add("Accept", "text/plain") + + var c []byte + c, err = ns.getResource(req) + if err != nil { + jww.ERROR.Printf("Failed to read csv resource %q with error message %s", url, err) + return nil, err + } + + if !bytes.Contains(c, []byte(sep)) { + err = errors.New("Cannot find separator " + sep + " in CSV.") + return + } + + if d, err = parseCSV(c, sep); err != nil { + jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err) + clearCacheSleep(i, url) + continue + } + break + } + return +} + +// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one. +// If you provide multiple parts they will be joined together to the final URL. +// GetJSON returns nil or parsed JSON to use in a short code. +func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) { + url := strings.Join(urlParts, "") + + for i := 0; i <= resRetries; i++ { + var req *http.Request + req, err = http.NewRequest("GET", url, nil) + if err != nil { + jww.ERROR.Printf("Failed to create request for getJSON: %s", err) + return nil, err + } + + req.Header.Add("Accept", "application/json") + + var c []byte + c, err = ns.getResource(req) + if err != nil { + jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err) + return nil, err + } + + err = json.Unmarshal(c, &v) + if err != nil { + jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err) + jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) + time.Sleep(resSleep) + deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) + continue + } + break + } + return +} + +// parseCSV parses bytes of CSV data into a slice slice string or an error +func parseCSV(c []byte, sep string) ([][]string, error) { + if len(sep) != 1 { + return nil, errors.New("Incorrect length of csv separator: " + sep) + } + b := bytes.NewReader(c) + r := csv.NewReader(b) + rSep := []rune(sep) + r.Comma = rSep[0] + r.FieldsPerRecord = 0 + return r.ReadAll() } diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go new file mode 100644 index 000000000..bcdddc9f4 --- /dev/null +++ b/tpl/data/data_test.go @@ -0,0 +1,251 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package data + +import ( + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetCSV(t *testing.T) { + t.Parallel() + + ns := New(newDeps(viper.New())) + + for i, test := range []struct { + sep string + url string + content string + expect interface{} + }{ + // Remotes + { + ",", + `http://success/`, + "gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm\n", + [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}, + }, + { + ",", + `http://error.extra.field/`, + "gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n", + false, + }, + { + ",", + `http://error.no.sep/`, + "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n", + false, + }, + { + ",", + `http://nofound/404`, + ``, + false, + }, + + // Locals + { + ";", + "pass/semi", + "gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n", + [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}, + }, + { + ";", + "fail/no-file", + "", + false, + }, + } { + msg := fmt.Sprintf("Test %d", i) + + // Setup HTTP test server + var srv *httptest.Server + srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { + if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + if r.URL.Path == "/404" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + w.Header().Add("Content-type", "text/csv") + + w.Write([]byte(test.content)) + }) + defer func() { srv.Close() }() + + // Setup local test file for schema-less URLs + if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { + f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) + require.NoError(t, err, msg) + f.WriteString(test.content) + f.Close() + } + + // Get on with it + got, err := ns.GetCSV(test.sep, test.url) + + if _, ok := test.expect.(bool); ok { + assert.Error(t, err, msg) + continue + } + require.NoError(t, err, msg) + require.NotNil(t, got, msg) + + assert.EqualValues(t, test.expect, got, msg) + } +} + +func TestGetJSON(t *testing.T) { + t.Parallel() + + ns := New(newDeps(viper.New())) + + for i, test := range []struct { + url string + content string + expect interface{} + }{ + { + `http://success/`, + `{"gomeetup":["Sydney","San Francisco","Stockholm"]}`, + map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}, + }, + { + `http://malformed/`, + `{gomeetup:["Sydney","San Francisco","Stockholm"]}`, + false, + }, + { + `http://nofound/404`, + ``, + false, + }, + // Locals + { + "pass/semi", + `{"gomeetup":["Sydney","San Francisco","Stockholm"]}`, + map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}, + }, + { + "fail/no-file", + "", + false, + }, + } { + msg := fmt.Sprintf("Test %d", i) + + // Setup HTTP test server + var srv *httptest.Server + srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) { + if !haveHeader(r.Header, "Accept", "application/json") { + http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) + return + } + + if r.URL.Path == "/404" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + w.Header().Add("Content-type", "application/json") + + w.Write([]byte(test.content)) + }) + defer func() { srv.Close() }() + + // Setup local test file for schema-less URLs + if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") { + f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url)) + require.NoError(t, err, msg) + f.WriteString(test.content) + f.Close() + } + + // Get on with it + got, err := ns.GetJSON(test.url) + + if _, ok := test.expect.(bool); ok { + assert.Error(t, err, msg) + continue + } + require.NoError(t, err, msg) + require.NotNil(t, got, msg) + + assert.EqualValues(t, test.expect, got, msg) + } +} + +func TestParseCSV(t *testing.T) { + t.Parallel() + + for i, test := range []struct { + csv []byte + sep string + exp string + err bool + }{ + {[]byte("a,b,c\nd,e,f\n"), "", "", true}, + {[]byte("a,b,c\nd,e,f\n"), "~/", "", true}, + {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false}, + {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false}, + {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true}, + {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false}, + } { + msg := fmt.Sprintf("Test %d: %v", i, test) + + csv, err := parseCSV(test.csv, test.sep) + if test.err { + assert.Error(t, err, msg) + continue + } + require.NoError(t, err, msg) + + act := "" + for _, v := range csv { + act = act + strings.Join(v, "") + } + + assert.Equal(t, test.exp, act, msg) + } +} + +func haveHeader(m http.Header, key, needle string) bool { + var s []string + var ok bool + + if s, ok = m[key]; !ok { + return false + } + + for _, v := range s { + if v == needle { + return true + } + } + return false +} diff --git a/tpl/data/resources.go b/tpl/data/resources.go index 56714e1a1..4a82cccab 100644 --- a/tpl/data/resources.go +++ b/tpl/data/resources.go @@ -14,14 +14,10 @@ package data import ( - "bytes" - "encoding/csv" - "encoding/json" - "errors" + "fmt" "io/ioutil" "net/http" "path/filepath" - "strings" "sync" "time" @@ -67,14 +63,16 @@ func (l *remoteLock) URLUnlock(url string) { } // getRemote loads the content of a remote file. This method is thread safe. -func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) { +func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) { + url := req.URL.String() + c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) - if c != nil && err == nil { - return c, nil - } if err != nil { return nil, err } + if c != nil { + return c, nil + } // avoid race condition with locks, block other goroutines if the current url is processing remoteURLLock.URLLock(url) @@ -82,27 +80,34 @@ func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([ // avoid multiple locks due to calling getCache twice c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) - if c != nil && err == nil { + if err != nil { + return nil, err + } + if c != nil { return c, nil } + + jww.INFO.Printf("Downloading: %s ...", url) + res, err := hc.Do(req) if err != nil { return nil, err } - jww.INFO.Printf("Downloading: %s ...", url) - res, err := hc.Get(url) - if err != nil { - return nil, err + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode)) } + c, err = ioutil.ReadAll(res.Body) res.Body.Close() if err != nil { return nil, err } + err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache")) if err != nil { return nil, err } + jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url)) return c, nil } @@ -119,90 +124,11 @@ func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) { } // getResource loads the content of a local or remote file -func (ns *Namespace) getResource(url string) ([]byte, error) { - if url == "" { - return nil, nil +func (ns *Namespace) getResource(req *http.Request) ([]byte, error) { + switch req.URL.Scheme { + case "": + return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg) + default: + return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client) } - if strings.Contains(url, "://") { - return getRemote(url, ns.deps.Fs.Source, ns.deps.Cfg, http.DefaultClient) - } - return getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg) -} - -// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one. -// If you provide multiple parts they will be joined together to the final URL. -// GetJSON returns nil or parsed JSON to use in a short code. -func (ns *Namespace) GetJSON(urlParts ...string) interface{} { - var v interface{} - url := strings.Join(urlParts, "") - - for i := 0; i <= resRetries; i++ { - c, err := ns.getResource(url) - if err != nil { - jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err) - return nil - } - - err = json.Unmarshal(c, &v) - if err != nil { - jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err) - jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) - time.Sleep(resSleep) - deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) - continue - } - break - } - return v -} - -// parseCSV parses bytes of CSV data into a slice slice string or an error -func parseCSV(c []byte, sep string) ([][]string, error) { - if len(sep) != 1 { - return nil, errors.New("Incorrect length of csv separator: " + sep) - } - b := bytes.NewReader(c) - r := csv.NewReader(b) - rSep := []rune(sep) - r.Comma = rSep[0] - r.FieldsPerRecord = 0 - return r.ReadAll() -} - -// GetCSV expects a data separator and one or n-parts of a URL to a resource which -// can either be a local or a remote one. -// The data separator can be a comma, semi-colon, pipe, etc, but only one character. -// If you provide multiple parts for the URL they will be joined together to the final URL. -// GetCSV returns nil or a slice slice to use in a short code. -func (ns *Namespace) GetCSV(sep string, urlParts ...string) [][]string { - var d [][]string - url := strings.Join(urlParts, "") - - var clearCacheSleep = func(i int, u string) { - jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep) - time.Sleep(resSleep) - deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg) - } - - for i := 0; i <= resRetries; i++ { - c, err := ns.getResource(url) - - if err == nil && !bytes.Contains(c, []byte(sep)) { - err = errors.New("Cannot find separator " + sep + " in CSV.") - } - - if err != nil { - jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err) - clearCacheSleep(i, url) - continue - } - - if d, err = parseCSV(c, sep); err != nil { - jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err) - clearCacheSleep(i, url) - continue - } - break - } - return d } diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 9f5fb6ef4..42d719184 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -19,7 +19,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "strings" "sync" "testing" "time" @@ -31,58 +30,9 @@ import ( "github.com/spf13/hugo/hugofs" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestScpCache(t *testing.T) { - t.Parallel() - - tests := []struct { - path string - content []byte - ignore bool - }{ - {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, - {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false}, - {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false}, - {"трям/трям", []byte(`T€st трям/трям Content 123`), false}, - {"은행", []byte(`T€st C은행ontent 123`), false}, - {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false}, - {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true}, - } - - fs := new(afero.MemMapFs) - - for _, test := range tests { - cfg := viper.New() - c, err := getCache(test.path, fs, cfg, test.ignore) - if err != nil { - t.Errorf("Error getting cache: %s", err) - } - if c != nil { - t.Errorf("There is content where there should not be anything: %s", string(c)) - } - - err = writeCache(test.path, test.content, fs, cfg, test.ignore) - if err != nil { - t.Errorf("Error writing cache: %s", err) - } - - c, err = getCache(test.path, fs, cfg, test.ignore) - if err != nil { - t.Errorf("Error getting cache after writing: %s", err) - } - if test.ignore { - if c != nil { - t.Errorf("Cache ignored but content is not nil: %s", string(c)) - } - } else { - if !bytes.Equal(c, test.content) { - t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) - } - } - } -} - func TestScpGetLocal(t *testing.T) { t.Parallel() v := viper.New() @@ -146,6 +96,10 @@ func TestScpGetRemote(t *testing.T) { } for _, test := range tests { + msg := fmt.Sprintf("%v", test) + + req, err := http.NewRequest("GET", test.path, nil) + require.NoError(t, err, msg) srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) { w.Write(test.content) @@ -154,41 +108,38 @@ func TestScpGetRemote(t *testing.T) { cfg := viper.New() - c, err := getRemote(test.path, fs, cfg, cl) - if err != nil { - t.Errorf("Error getting resource content: %s", err) - } - if !bytes.Equal(c, test.content) { - t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c)) - } - cc, cErr := getCache(test.path, fs, cfg, test.ignore) - if cErr != nil { - t.Error(cErr) - } + c, err := getRemote(req, fs, cfg, cl) + require.NoError(t, err, msg) + assert.Equal(t, string(test.content), string(c)) + + c, err = getCache(req.URL.String(), fs, cfg, test.ignore) + require.NoError(t, err, msg) + if test.ignore { - if cc != nil { - t.Errorf("Cache ignored but content is not nil: %s", string(cc)) - } + assert.Empty(t, c, msg) } else { - if !bytes.Equal(cc, test.content) { - t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(cc)) - } + assert.Equal(t, string(test.content), string(c)) + } } } func TestScpGetRemoteParallel(t *testing.T) { t.Parallel() - fs := new(afero.MemMapFs) + + ns := New(newDeps(viper.New())) + content := []byte(`T€st Content 123`) - url := "http://Foo.Bar/foo_Bar-Foo" srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) { w.Write(content) }) defer func() { srv.Close() }() - for _, ignoreCache := range []bool{false, true} { + url := "http://Foo.Bar/foo_Bar-Foo" + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + for _, ignoreCache := range []bool{false, true} { cfg := viper.New() cfg.Set("ignoreCache", ignoreCache) @@ -199,13 +150,9 @@ func TestScpGetRemoteParallel(t *testing.T) { go func(gor int) { defer wg.Done() for j := 0; j < 10; j++ { - c, err := getRemote(url, fs, cfg, cl) - if err != nil { - t.Errorf("Error getting resource content: %s", err) - } - if !bytes.Equal(c, content) { - t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(content), string(c)) - } + c, err := getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, cl) + assert.NoError(t, err) + assert.Equal(t, string(content), string(c)) time.Sleep(23 * time.Millisecond) } @@ -214,137 +161,6 @@ func TestScpGetRemoteParallel(t *testing.T) { wg.Wait() } - - t.Log("Done!") -} - -func TestParseCSV(t *testing.T) { - t.Parallel() - - tests := []struct { - csv []byte - sep string - exp string - err bool - }{ - {[]byte("a,b,c\nd,e,f\n"), "", "", true}, - {[]byte("a,b,c\nd,e,f\n"), "~/", "", true}, - {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false}, - {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false}, - {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true}, - {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false}, - } - for _, test := range tests { - csv, err := parseCSV(test.csv, test.sep) - if test.err && err == nil { - t.Error("Expecting an error") - } - if test.err { - continue - } - if !test.err && err != nil { - t.Error(err) - } - - act := "" - for _, v := range csv { - act = act + strings.Join(v, "") - } - - if act != test.exp { - t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv) - } - - } -} - -func TestGetJSONFailParse(t *testing.T) { - t.Parallel() - - ns := New(newDeps(viper.New())) - - reqCount := 0 - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if reqCount > 0 { - w.Header().Add("Content-type", "application/json") - fmt.Fprintln(w, `{"gomeetup":["Sydney", "San Francisco", "Stockholm"]}`) - } else { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintln(w, `ERROR 500`) - } - reqCount++ - })) - defer ts.Close() - url := ts.URL + "/test.json" - - want := map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}} - have := ns.GetJSON(url) - assert.NotNil(t, have) - if have != nil { - assert.EqualValues(t, want, have) - } -} - -func TestGetCSVFailParseSep(t *testing.T) { - t.Parallel() - - ns := New(newDeps(viper.New())) - - reqCount := 0 - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if reqCount > 0 { - w.Header().Add("Content-type", "application/json") - fmt.Fprintln(w, `gomeetup,city`) - fmt.Fprintln(w, `yes,Sydney`) - fmt.Fprintln(w, `yes,San Francisco`) - fmt.Fprintln(w, `yes,Stockholm`) - } else { - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintln(w, `ERROR 500`) - } - reqCount++ - })) - defer ts.Close() - url := ts.URL + "/test.csv" - - want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}} - have := ns.GetCSV(",", url) - assert.NotNil(t, have) - if have != nil { - assert.EqualValues(t, want, have) - } -} - -func TestGetCSVFailParse(t *testing.T) { - t.Parallel() - - ns := New(newDeps(viper.New())) - - reqCount := 0 - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Content-type", "application/json") - if reqCount > 0 { - fmt.Fprintln(w, `gomeetup,city`) - fmt.Fprintln(w, `yes,Sydney`) - fmt.Fprintln(w, `yes,San Francisco`) - fmt.Fprintln(w, `yes,Stockholm`) - } else { - fmt.Fprintln(w, `gomeetup,city`) - fmt.Fprintln(w, `yes,Sydney,Bondi,`) // wrong number of fields in line - fmt.Fprintln(w, `yes,San Francisco`) - fmt.Fprintln(w, `yes,Stockholm`) - } - reqCount++ - })) - defer ts.Close() - url := ts.URL + "/test.csv" - - want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}} - have := ns.GetCSV(",", url) - assert.NotNil(t, have) - if have != nil { - assert.EqualValues(t, want, have) - } } func newDeps(cfg config.Provider) *deps.Deps {