diff --git a/hugofs/glob.go b/hugofs/glob.go new file mode 100644 index 000000000..e4115ea7c --- /dev/null +++ b/hugofs/glob.go @@ -0,0 +1,85 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/hugofs/glob" + + "github.com/spf13/afero" +) + +// Glob walks the fs and passes all matches to the handle func. +// The handle func can return true to signal a stop. +func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error { + pattern = glob.NormalizePath(pattern) + if pattern == "" { + return nil + } + + g, err := glob.GetGlob(pattern) + if err != nil { + return nil + } + + hasSuperAsterisk := strings.Contains(pattern, "**") + levels := strings.Count(pattern, "/") + root := glob.ResolveRootDir(pattern) + + // Signals that we're done. + done := errors.New("done") + + wfn := func(p string, info FileMetaInfo, err error) error { + p = glob.NormalizePath(p) + if info.IsDir() { + if !hasSuperAsterisk { + // Avoid walking to the bottom if we can avoid it. + if p != "" && strings.Count(p, "/") >= levels { + return filepath.SkipDir + } + } + return nil + } + + if g.Match(p) { + d, err := handle(info) + if err != nil { + return err + } + if d { + return done + } + } + + return nil + } + + w := NewWalkway(WalkwayConfig{ + Root: root, + Fs: fs, + WalkFn: wfn, + }) + + err = w.Walk() + + if err != done { + return err + } + + return nil + +} diff --git a/resources/internal/glob.go b/hugofs/glob/glob.go similarity index 60% rename from resources/internal/glob.go rename to hugofs/glob/glob.go index a87a23f13..18d8d44cb 100644 --- a/resources/internal/glob.go +++ b/hugofs/glob/glob.go @@ -11,13 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package glob import ( + "path" + "path/filepath" "strings" "sync" "github.com/gobwas/glob" + "github.com/gobwas/glob/syntax" ) var ( @@ -46,3 +49,33 @@ func GetGlob(pattern string) (glob.Glob, error) { return g, nil } + +func NormalizePath(p string) string { + return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.") +} + +// ResolveRootDir takes a normalized path on the form "assets/**.json" and +// determines any root dir, i.e. any start path without any wildcards. +func ResolveRootDir(p string) string { + parts := strings.Split(path.Dir(p), "/") + var roots []string + for _, part := range parts { + isSpecial := false + for i := 0; i < len(part); i++ { + if syntax.Special(part[i]) { + isSpecial = true + break + } + } + if isSpecial { + break + } + roots = append(roots, part) + } + + if len(roots) == 0 { + return "" + } + + return strings.Join(roots, "/") +} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go new file mode 100644 index 000000000..2b1c74107 --- /dev/null +++ b/hugofs/glob/glob_test.go @@ -0,0 +1,63 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glob + +import ( + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestResolveRootDir(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + in string + expect string + }{ + {"data/foo.json", "data"}, + {"a/b/**/foo.json", "a/b"}, + {"dat?a/foo.json", ""}, + {"a/b[a-c]/foo.json", "a"}, + } { + + c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect) + } +} + +func TestNormalizePath(t *testing.T) { + c := qt.New(t) + + for _, test := range []struct { + in string + expect string + }{ + {filepath.FromSlash("data/FOO.json"), "data/foo.json"}, + {filepath.FromSlash("/data/FOO.json"), "data/foo.json"}, + {filepath.FromSlash("./FOO.json"), "foo.json"}, + {"//", ""}, + } { + + c.Assert(NormalizePath(test.in), qt.Equals, test.expect) + } +} + +func TestGetGlob(t *testing.T) { + c := qt.New(t) + g, err := GetGlob("**.JSON") + c.Assert(err, qt.IsNil) + c.Assert(g.Match("data/my.json"), qt.Equals, true) + +} diff --git a/hugofs/glob_test.go b/hugofs/glob_test.go new file mode 100644 index 000000000..3c7780685 --- /dev/null +++ b/hugofs/glob_test.go @@ -0,0 +1,61 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + "testing" + + "github.com/spf13/afero" + + qt "github.com/frankban/quicktest" +) + +func TestGlob(t *testing.T) { + c := qt.New(t) + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + create := func(filename string) { + err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777) + c.Assert(err, qt.IsNil) + } + + collect := func(pattern string) []string { + var paths []string + h := func(fi FileMetaInfo) (bool, error) { + paths = append(paths, fi.Meta().Path()) + return false, nil + } + err := Glob(fs, pattern, h) + c.Assert(err, qt.IsNil) + return paths + } + + create("root.json") + create("jsonfiles/d1.json") + create("jsonfiles/d2.json") + create("jsonfiles/sub/d3.json") + create("jsonfiles/d1.xml") + create("a/b/c/e/f.json") + + c.Assert(collect("**.json"), qt.HasLen, 5) + c.Assert(collect("**"), qt.HasLen, 6) + c.Assert(collect(""), qt.HasLen, 0) + c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2) + c.Assert(collect("*.json"), qt.HasLen, 1) + c.Assert(collect("**.xml"), qt.HasLen, 1) + c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2) + +} diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 75f5595e1..84c871e4d 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -502,3 +502,33 @@ func TestMultiSiteResource(t *testing.T) { b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes") } + +func TestResourcesMatch(t *testing.T) { + t.Parallel() + + b := newTestSitesBuilder(t) + + b.WithContent("page.md", "") + + b.WithSourceFile( + "assets/jsons/data1.json", "json1 content", + "assets/jsons/data2.json", "json2 content", + "assets/jsons/data3.xml", "xml content", + ) + + b.WithTemplates("index.html", ` +{{ $jsons := (resources.Match "jsons/*.json") }} +{{ $json := (resources.GetMatch "jsons/*.json") }} +{{ printf "JSONS: %d" (len $jsons) }} +JSON: {{ $json.RelPermalink }}: {{ $json.Content }} +{{ range $jsons }} +{{- .RelPermalink }}: {{ .Content }} +{{ end }} +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/index.html", + "JSON: /jsons/data1.json: json1 content", + "JSONS: 2", "/jsons/data1.json: json1 content") +} diff --git a/resources/resource/resources.go b/resources/resource/resources.go index 5c661c24e..ac5dd0b2b 100644 --- a/resources/resource/resources.go +++ b/resources/resource/resources.go @@ -17,7 +17,7 @@ import ( "fmt" "strings" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/hugofs/glob" ) // Resources represents a slice of resources, which can be a mix of different types. @@ -44,7 +44,7 @@ func (r Resources) ByType(tp string) Resources { // GetMatch finds the first Resource matching the given pattern, or nil if none found. // See Match for a more complete explanation about the rules used. func (r Resources) GetMatch(pattern string) Resource { - g, err := internal.GetGlob(pattern) + g, err := glob.GetGlob(pattern) if err != nil { return nil } @@ -68,7 +68,7 @@ func (r Resources) GetMatch(pattern string) Resource { // path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png". // See https://github.com/gobwas/glob for the full rules set. func (r Resources) Match(pattern string) Resources { - g, err := internal.GetGlob(pattern) + g, err := glob.GetGlob(pattern) if err != nil { return nil } diff --git a/resources/resource_cache.go b/resources/resource_cache.go index 8ff63beb0..656d4f826 100644 --- a/resources/resource_cache.go +++ b/resources/resource_cache.go @@ -37,7 +37,9 @@ type ResourceCache struct { rs *Spec sync.RWMutex - cache map[string]resource.Resource + + // Either resource.Resource or resource.Resources. + cache map[string]interface{} fileCache *filecache.Cache @@ -61,7 +63,7 @@ func newResourceCache(rs *Spec) *ResourceCache { return &ResourceCache{ rs: rs, fileCache: rs.FileCaches.AssetsCache(), - cache: make(map[string]resource.Resource), + cache: make(map[string]interface{}), nlocker: locker.NewLocker(), } } @@ -70,7 +72,7 @@ func (c *ResourceCache) clear() { c.Lock() defer c.Unlock() - c.cache = make(map[string]resource.Resource) + c.cache = make(map[string]interface{}) c.nlocker = locker.NewLocker() } @@ -84,7 +86,7 @@ func (c *ResourceCache) cleanKey(key string) string { return strings.TrimPrefix(path.Clean(key), "/") } -func (c *ResourceCache) get(key string) (resource.Resource, bool) { +func (c *ResourceCache) get(key string) (interface{}, bool) { c.RLock() defer c.RUnlock() r, found := c.cache[key] @@ -92,6 +94,22 @@ func (c *ResourceCache) get(key string) (resource.Resource, bool) { } func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) { + r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() }) + if r == nil || err != nil { + return nil, err + } + return r.(resource.Resource), nil +} + +func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) { + r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() }) + if r == nil || err != nil { + return nil, err + } + return r.(resource.Resources), nil +} + +func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) { key = c.cleanKey(path.Join(partition, key)) // First check in-memory cache. r, found := c.get(key) @@ -174,7 +192,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata) } -func (c *ResourceCache) set(key string, r resource.Resource) { +func (c *ResourceCache) set(key string, r interface{}) { c.Lock() defer c.Unlock() c.cache[key] = r diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 36a29e733..e42843c75 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -16,9 +16,12 @@ package create import ( + "path" "path/filepath" - "github.com/spf13/afero" + "github.com/gohugoio/hugo/hugofs/glob" + + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/resources" @@ -36,18 +39,75 @@ func New(rs *resources.Spec) *Client { return &Client{rs: rs} } -// Get creates a new Resource by opening the given filename in the given filesystem. -func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) { +// Get creates a new Resource by opening the given filename in the assets filesystem. +func (c *Client) Get(filename string) (resource.Resource, error) { filename = filepath.Clean(filename) return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{ - Fs: fs, + Fs: c.rs.BaseFs.Assets.Fs, LazyPublish: true, SourceFilename: filename}) }) } +// Match gets the resources matching the given pattern from the assets filesystem. +func (c *Client) Match(pattern string) (resource.Resources, error) { + return c.match(pattern, false) +} + +// GetMatch gets first resource matching the given pattern from the assets filesystem. +func (c *Client) GetMatch(pattern string) (resource.Resource, error) { + res, err := c.match(pattern, true) + if err != nil || len(res) == 0 { + return nil, err + } + return res[0], err +} + +func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) { + var partition string + if firstOnly { + partition = "__get-match" + } else { + partition = "__match" + } + + // TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199 + partition = path.Join(resources.CACHE_OTHER, partition) + key := glob.NormalizePath(pattern) + + return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) { + var res resource.Resources + + handle := func(info hugofs.FileMetaInfo) (bool, error) { + meta := info.Meta() + r, err := c.rs.New(resources.ResourceSourceDescriptor{ + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return meta.Open() + }, + RelTargetFilename: meta.Path()}) + + if err != nil { + return true, err + } + + res = append(res, r) + + return firstOnly, nil + + } + + if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil { + return nil, err + } + + return res, nil + + }) +} + // FromString creates a new Resource from a string with the given relative target path. func (c *Client) FromString(targetPath, content string) (resource.Resource, error) { return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) { diff --git a/resources/resource_metadata.go b/resources/resource_metadata.go index e019133d7..adb9d6867 100644 --- a/resources/resource_metadata.go +++ b/resources/resource_metadata.go @@ -17,7 +17,7 @@ import ( "fmt" "strconv" - "github.com/gohugoio/hugo/resources/internal" + "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/resources/resource" "github.com/pkg/errors" @@ -70,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res srcKey := strings.ToLower(cast.ToString(src)) - glob, err := internal.GetGlob(srcKey) + glob, err := glob.GetGlob(srcKey) if err != nil { return errors.Wrap(err, "failed to match resource with metadata") } diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index d32e12a05..3d688e21c 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -68,7 +68,7 @@ type Namespace struct { templatesClient *templates.Client } -// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order) +// Get locates the filename given in Hugo's assets filesystem // and creates a Resource object that can be used for further transformations. func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) { filenamestr, err := cast.ToStringE(filename) @@ -78,12 +78,50 @@ func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) { filenamestr = filepath.Clean(filenamestr) - // Resource Get'ing is currently limited to /assets to make it simpler - // to control the behaviour of publishing and partial rebuilding. - return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr) + return ns.createClient.Get(filenamestr) } +// GetMatch finds the first Resource matching the given pattern, or nil if none found. +// +// It looks for files in the assets file system. +// +// See Match for a more complete explanation about the rules used. +func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) { + patternStr, err := cast.ToStringE(pattern) + if err != nil { + return nil, err + } + + return ns.createClient.GetMatch(patternStr) + +} + +// Match gets all resources matching the given base path prefix, e.g +// "*.png" will match all png files. The "*" does not match path delimiters (/), +// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.: +// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and +// to match all PNG images below the images folder, use "images/**.jpg". +// +// The matching is case insensitive. +// +// Match matches by using the files name with path relative to the file system root +// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png". +// +// See https://github.com/gobwas/glob for the full rules set. +// +// It looks for files in the assets file system. +// +// See Match for a more complete explanation about the rules used. +func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) { + patternStr, err := cast.ToStringE(pattern) + if err != nil { + return nil, err + } + + return ns.createClient.Match(patternStr) +} + // Concat concatenates a slice of Resource objects. These resources must // (currently) be of the same Media Type. func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.Resource, error) {