diff --git a/create/content.go b/create/content.go index 7f444f9df..914c16bec 100644 --- a/create/content.go +++ b/create/content.go @@ -137,8 +137,9 @@ func (b *contentBuilder) buildDir() error { if !b.dirMap.siteUsed { // We don't need to build everything. contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + filename = strings.TrimPrefix(filename, string(os.PathSeparator)) for _, cn := range contentTargetFilenames { - if strings.HasPrefix(cn, filename) { + if strings.Contains(cn, filename) { return true } } @@ -205,7 +206,8 @@ func (b *contentBuilder) buildFile() error { if !usesSite { // We don't need to build everything. contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { - return strings.HasPrefix(contentPlaceholderAbsFilename, filename) + filename = strings.TrimPrefix(filename, string(os.PathSeparator)) + return strings.Contains(contentPlaceholderAbsFilename, filename) }) } diff --git a/docs/content/en/hugo-modules/configuration.md b/docs/content/en/hugo-modules/configuration.md index 1a3a285c7..9b94061e5 100644 --- a/docs/content/en/hugo-modules/configuration.md +++ b/docs/content/en/hugo-modules/configuration.md @@ -155,3 +155,15 @@ target lang : The language code, e.g. "en". Only relevant for `content` mounts, and `static` mounts when in multihost mode. +includeFiles (string or slice) +: One or more [glob](https://github.com/gobwas/glob) patterns matching files or directories to include. If `excludeFiles` is not set, the files matching `includeFiles` will be the files mounted. + +The glob patterns are matched to the filenames starting from the `source` root, they should have Unix styled slashes even on Windows, `/` matches the mount root and `**` can be used as a super-asterisk to match recursively down all directories, e.g `/posts/**.jpg`. + +The search is case-insensitive. + +{{< new-in "0.89.0" >}} + +excludeFiles (string or slice) +: One or more glob patterns matching files to exclude. + diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index fcf35d956..ea6ac4fd3 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugofs/files" "golang.org/x/text/unicode/norm" @@ -76,6 +78,9 @@ type FileMeta struct { Fs afero.Fs OpenFunc func() (afero.File, error) JoinStatFunc func(name string) (FileMetaInfo, error) + + // Include only files or directories that match. + InclusionFilter *glob.FilenameFilter } func (m *FileMeta) Copy() *FileMeta { @@ -95,10 +100,17 @@ func (m *FileMeta) Merge(from *FileMeta) { for i := 0; i < dstv.NumField(); i++ { v := dstv.Field(i) + if !v.CanSet() { + continue + } if !hreflect.IsTruthfulValue(v) { v.Set(srcv.Field(i)) } } + + if m.InclusionFilter == nil { + m.InclusionFilter = from.InclusionFilter + } } func (f *FileMeta) Open() (afero.File, error) { diff --git a/hugofs/filename_filter_fs.go b/hugofs/filename_filter_fs.go new file mode 100644 index 000000000..2a11335a3 --- /dev/null +++ b/hugofs/filename_filter_fs.go @@ -0,0 +1,170 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "os" + "strings" + "syscall" + "time" + + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/spf13/afero" +) + +func newFilenameFilterFs(fs afero.Fs, base string, filter *glob.FilenameFilter) afero.Fs { + return &filenameFilterFs{ + fs: fs, + base: base, + filter: filter, + } +} + +// filenameFilterFs is a filesystem that filters by filename. +type filenameFilterFs struct { + base string + fs afero.Fs + + filter *glob.FilenameFilter +} + +func (fs *filenameFilterFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, b, err := fs.fs.(afero.Lstater).LstatIfPossible(name) + if err != nil { + return nil, false, err + } + if !fs.filter.Match(name, fi.IsDir()) { + return nil, false, os.ErrNotExist + } + return fi, b, nil +} + +func (fs *filenameFilterFs) Open(name string) (afero.File, error) { + fi, err := fs.fs.Stat(name) + if err != nil { + return nil, err + } + + if !fs.filter.Match(name, fi.IsDir()) { + return nil, os.ErrNotExist + } + + f, err := fs.fs.Open(name) + if err != nil { + return nil, err + } + + if !fi.IsDir() { + return f, nil + } + + return &filenameFilterDir{ + File: f, + base: fs.base, + filter: fs.filter, + }, nil +} + +func (fs *filenameFilterFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + return fs.Open(name) +} + +func (fs *filenameFilterFs) Stat(name string) (os.FileInfo, error) { + fi, _, err := fs.LstatIfPossible(name) + return fi, err +} + +func (fs *filenameFilterFs) getOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { + return fs.Open(name) + } +} + +type filenameFilterDir struct { + afero.File + base string + filter *glob.FilenameFilter +} + +func (f *filenameFilterDir) Readdir(count int) ([]os.FileInfo, error) { + fis, err := f.File.Readdir(-1) + if err != nil { + return nil, err + } + + var result []os.FileInfo + for _, fi := range fis { + fim := fi.(FileMetaInfo) + if f.filter.Match(strings.TrimPrefix(fim.Meta().Filename, f.base), fim.IsDir()) { + result = append(result, fi) + } + } + + return result, nil +} + +func (f *filenameFilterDir) Readdirnames(count int) ([]string, error) { + dirsi, err := f.Readdir(count) + if err != nil { + return nil, err + } + + dirs := make([]string, len(dirsi)) + for i, d := range dirsi { + dirs[i] = d.Name() + } + return dirs, nil +} + +func (fs *filenameFilterFs) Chmod(n string, m os.FileMode) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) Chtimes(n string, a, m time.Time) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) Chown(n string, uid, gid int) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) ReadDir(name string) ([]os.FileInfo, error) { + panic("not implemented") +} + +func (fs *filenameFilterFs) Remove(n string) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) RemoveAll(p string) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) Rename(o, n string) error { + return syscall.EPERM +} +func (fs *filenameFilterFs) Create(n string) (afero.File, error) { + return nil, syscall.EPERM +} +func (fs *filenameFilterFs) Name() string { + return "FinameFilterFS" +} + +func (fs *filenameFilterFs) Mkdir(n string, p os.FileMode) error { + return syscall.EPERM +} + +func (fs *filenameFilterFs) MkdirAll(n string, p os.FileMode) error { + return syscall.EPERM +} diff --git a/hugofs/filename_filter_fs_test.go b/hugofs/filename_filter_fs_test.go new file mode 100644 index 000000000..b3e97a6a6 --- /dev/null +++ b/hugofs/filename_filter_fs_test.go @@ -0,0 +1,83 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/hugofs/glob" + + "github.com/spf13/afero" + + qt "github.com/frankban/quicktest" +) + +func TestFilenameFilterFs(t *testing.T) { + c := qt.New(t) + + base := filepath.FromSlash("/mybase") + + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + for _, letter := range []string{"a", "b", "c"} { + for i := 1; i <= 3; i++ { + c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.txt", i)), []byte("some text file for"+letter), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join(base, letter, fmt.Sprintf("my%d.json", i)), []byte("some json file for"+letter), 0755), qt.IsNil) + } + } + + fs = afero.NewBasePathFs(fs, base) + + filter, err := glob.NewFilenameFilter(nil, []string{"/b/**.txt"}) + c.Assert(err, qt.IsNil) + + fs = newFilenameFilterFs(fs, base, filter) + + assertExists := func(filename string, shouldExist bool) { + filename = filepath.Clean(filename) + _, err1 := fs.Stat(filename) + f, err2 := fs.Open(filename) + if shouldExist { + c.Assert(err1, qt.IsNil) + c.Assert(err2, qt.IsNil) + defer f.Close() + + } else { + for _, err := range []error{err1, err2} { + c.Assert(err, qt.Not(qt.IsNil)) + c.Assert(errors.Is(err, os.ErrNotExist), qt.IsTrue) + } + } + } + + assertExists("/a/my1.txt", true) + assertExists("/b/my1.txt", false) + + dirB, err := fs.Open("/b") + defer dirB.Close() + c.Assert(err, qt.IsNil) + dirBEntries, err := dirB.Readdirnames(-1) + c.Assert(dirBEntries, qt.DeepEquals, []string{"my1.json", "my2.json", "my3.json"}) + + dirC, err := fs.Open("/c") + defer dirC.Close() + c.Assert(err, qt.IsNil) + dirCEntries, err := dirC.Readdirnames(-1) + c.Assert(dirCEntries, qt.DeepEquals, []string{"my1.json", "my1.txt", "my2.json", "my2.txt", "my3.json", "my3.txt"}) + +} diff --git a/hugofs/glob/filename_filter.go b/hugofs/glob/filename_filter.go new file mode 100644 index 000000000..c4b582bd5 --- /dev/null +++ b/hugofs/glob/filename_filter.go @@ -0,0 +1,159 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glob + +import ( + "path" + "path/filepath" + "strings" + + "github.com/gobwas/glob" +) + +type FilenameFilter struct { + shouldInclude func(filename string) bool + inclusions []glob.Glob + dirInclusions []glob.Glob + exclusions []glob.Glob + isWindows bool +} + +func normalizeFilenameGlobPattern(s string) string { + // Use Unix separators even on Windows. + s = filepath.ToSlash(s) + if !strings.HasPrefix(s, "/") { + s = "/" + s + } + return s +} + +// NewFilenameFilter creates a new Glob where the Match method will +// return true if the file should be included. +// Note that the inclusions will be checked first. +func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { + if inclusions == nil && exclusions == nil { + return nil, nil + } + filter := &FilenameFilter{isWindows: isWindows} + + for _, include := range inclusions { + include = normalizeFilenameGlobPattern(include) + g, err := filenamesGlobCache.GetGlob(include) + if err != nil { + return nil, err + } + filter.inclusions = append(filter.inclusions, g) + + // For mounts that do directory walking (e.g. content) we + // must make sure that all directories up to this inclusion also + // gets included. + dir := path.Dir(include) + parts := strings.Split(dir, "/") + for i, _ := range parts { + pattern := "/" + filepath.Join(parts[:i+1]...) + g, err := filenamesGlobCache.GetGlob(pattern) + if err != nil { + return nil, err + } + filter.dirInclusions = append(filter.dirInclusions, g) + } + } + + for _, exclude := range exclusions { + exclude = normalizeFilenameGlobPattern(exclude) + g, err := filenamesGlobCache.GetGlob(exclude) + if err != nil { + return nil, err + } + filter.exclusions = append(filter.exclusions, g) + } + + return filter, nil +} + +// MustNewFilenameFilter invokes NewFilenameFilter and panics on error. +func MustNewFilenameFilter(inclusions, exclusions []string) *FilenameFilter { + filter, err := NewFilenameFilter(inclusions, exclusions) + if err != nil { + panic(err) + } + return filter +} + +// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. +func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { + return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} +} + +// Match returns whether filename should be included. +func (f *FilenameFilter) Match(filename string, isDir bool) bool { + if f == nil { + return true + } + return f.doMatch(filename, isDir) + /*if f.shouldInclude == nil { + fmt.Printf("Match: %q (%t) => %t\n", filename, isDir, isMatch) + } + return isMatch*/ +} + +func (f *FilenameFilter) doMatch(filename string, isDir bool) bool { + if f == nil { + return true + } + + if !strings.HasPrefix(filename, filepathSeparator) { + filename = filepathSeparator + filename + } + + if f.shouldInclude != nil { + if f.shouldInclude(filename) { + return true + } + if f.isWindows { + // The Glob matchers below handles this by themselves, + // for the shouldInclude we need to take some extra steps + // to make this robust. + winFilename := filepath.FromSlash(filename) + if filename != winFilename { + if f.shouldInclude(winFilename) { + return true + } + } + } + + } + + for _, inclusion := range f.inclusions { + if inclusion.Match(filename) { + return true + } + } + + if isDir && f.inclusions != nil { + for _, inclusion := range f.dirInclusions { + if inclusion.Match(filename) { + return true + } + } + } + + for _, exclusion := range f.exclusions { + if exclusion.Match(filename) { + return false + } + } + + return f.inclusions == nil && f.shouldInclude == nil +} diff --git a/hugofs/glob/filename_filter_test.go b/hugofs/glob/filename_filter_test.go new file mode 100644 index 000000000..1fce5b135 --- /dev/null +++ b/hugofs/glob/filename_filter_test.go @@ -0,0 +1,70 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package glob + +import ( + "path/filepath" + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestFilenameFilter(t *testing.T) { + c := qt.New(t) + + excludeAlmostAllJSON, err := NewFilenameFilter([]string{"/a/b/c/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.json"), false), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c/foo.bar"), false), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a/b"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/a"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match("", true), qt.Equals, true) + + excludeAllButFooJSON, err := NewFilenameFilter([]string{"/a/**/foo.json"}, []string{"**.json"}) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/data/my.json"), false), qt.Equals, false) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c/d/e/foo.json"), false), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/c"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/a/b/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/"), true), qt.Equals, true) + c.Assert(excludeAllButFooJSON.Match(filepath.FromSlash("/b"), true), qt.Equals, false) + c.Assert(err, qt.IsNil) + + nopFilter, err := NewFilenameFilter(nil, nil) + c.Assert(err, qt.IsNil) + c.Assert(nopFilter.Match("ab.txt", false), qt.Equals, true) + + includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) + c.Assert(err, qt.IsNil) + c.Assert(includeOnlyFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.jpg", false), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.gif", false), qt.Equals, false) + + exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) + c.Assert(err, qt.IsNil) + c.Assert(exlcudeOnlyFilter.Match("ab.json", false), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.jpg", false), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.gif", false), qt.Equals, true) + + var nilFilter *FilenameFilter + c.Assert(nilFilter.Match("ab.gif", false), qt.Equals, true) + + funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) + c.Assert(funcFilter.Match("ab.json", false), qt.Equals, true) + c.Assert(funcFilter.Match("ab.bson", false), qt.Equals, false) + +} diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index 6dd0df5ed..9e928ec32 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2021 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ package glob import ( + "os" "path" "path/filepath" "runtime" @@ -24,6 +25,8 @@ import ( "github.com/gobwas/glob/syntax" ) +const filepathSeparator = string(os.PathSeparator) + var ( isWindows = runtime.GOOS == "windows" defaultGlobCache = &globCache{ @@ -33,7 +36,7 @@ var ( } filenamesGlobCache = &globCache{ - isCaseSensitive: true, // TODO(bep) bench + isCaseSensitive: false, // As long as the search strings are all lower case, this does not allocate. isWindows: isWindows, cache: make(map[string]globErr), } @@ -161,78 +164,3 @@ func HasGlobChar(s string) bool { } return false } - -type FilenameFilter struct { - shouldInclude func(filename string) bool - inclusions []glob.Glob - exclusions []glob.Glob - isWindows bool -} - -// NewFilenameFilter creates a new Glob where the Match method will -// return true if the file should be exluded. -// Note that the inclusions will be checked first. -func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { - filter := &FilenameFilter{isWindows: isWindows} - - for _, include := range inclusions { - g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include)) - if err != nil { - return nil, err - } - filter.inclusions = append(filter.inclusions, g) - } - for _, exclude := range exclusions { - g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude)) - if err != nil { - return nil, err - } - filter.exclusions = append(filter.exclusions, g) - } - - return filter, nil -} - -// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. -func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { - return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} -} - -// Match returns whether filename should be included. -func (f *FilenameFilter) Match(filename string) bool { - if f == nil { - return true - } - - if f.shouldInclude != nil { - if f.shouldInclude(filename) { - return true - } - if f.isWindows { - // The Glob matchers below handles this by themselves, - // for the shouldInclude we need to take some extra steps - // to make this robust. - winFilename := filepath.FromSlash(filename) - if filename != winFilename { - if f.shouldInclude(winFilename) { - return true - } - } - } - - } - - for _, inclusion := range f.inclusions { - if inclusion.Match(filename) { - return true - } - } - - for _, exclusion := range f.exclusions { - if exclusion.Match(filename) { - return false - } - } - - return f.inclusions == nil && f.shouldInclude == nil -} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go index 7ef3fbbed..66efe9e53 100644 --- a/hugofs/glob/glob_test.go +++ b/hugofs/glob/glob_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2021 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,7 +15,6 @@ package glob import ( "path/filepath" - "strings" "testing" qt "github.com/frankban/quicktest" @@ -67,51 +66,38 @@ func TestNormalizePath(t *testing.T) { } func TestGetGlob(t *testing.T) { - c := qt.New(t) - g, err := GetGlob("**.JSON") - c.Assert(err, qt.IsNil) - c.Assert(g.Match("data/my.json"), qt.Equals, true) -} - -func TestFilenameFilter(t *testing.T) { - c := qt.New(t) - - excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"}) - c.Assert(err, qt.IsNil) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true) - c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false) - - nopFilter, err := NewFilenameFilter(nil, nil) - c.Assert(err, qt.IsNil) - c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true) - - includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) - c.Assert(err, qt.IsNil) - c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true) - c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false) - - exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) - c.Assert(err, qt.IsNil) - c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false) - c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false) - c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true) - - var nilFilter *FilenameFilter - c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true) - - funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) - c.Assert(funcFilter.Match("ab.json"), qt.Equals, true) - c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false) - + for _, cache := range []*globCache{defaultGlobCache, filenamesGlobCache} { + c := qt.New(t) + g, err := cache.GetGlob("**.JSON") + c.Assert(err, qt.IsNil) + c.Assert(g.Match("data/my.jSon"), qt.Equals, true) + } } func BenchmarkGetGlob(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := GetGlob("**/foo") - if err != nil { - b.Fatal(err) - } + + runBench := func(name string, cache *globCache, search string) { + b.Run(name, func(b *testing.B) { + g, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } + for i := 0; i < b.N; i++ { + _ = g.Match(search) + } + }) } + + runBench("Default cache", defaultGlobCache, "abcde") + runBench("Filenames cache, lowercase searchs", filenamesGlobCache, "abcde") + runBench("Filenames cache, mixed case searchs", filenamesGlobCache, "abCDe") + + b.Run("GetGlob", func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := GetGlob("**/foo") + if err != nil { + b.Fatal(err) + } + } + }) } diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 6441693ad..bd10144ff 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -142,6 +142,13 @@ func (r RootMapping) filename(name string) string { return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) } +func (r RootMapping) trimFrom(name string) string { + if name == "" { + return "" + } + return strings.TrimPrefix(name, r.From) +} + // A RootMappingFs maps several roots into one. Note that the root of this filesystem // is directories only, and they will be returned in Readdir and Readdirnames // in the order given. @@ -170,7 +177,12 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { p = strings.TrimLeft(p, filepathSeparator) return p }) - fs := decorateDirs(bfs, r.Meta) + + fs := bfs + if r.Meta.InclusionFilter != nil { + fs = newFilenameFilterFs(fs, r.To, r.Meta.InclusionFilter) + } + fs = decorateDirs(fs, r.Meta) fi, err := fs.Stat("") if err != nil { return nil, errors.Wrap(err, "RootMappingFs.Dirs") @@ -368,6 +380,10 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) for _, fi := range direntries { meta := fi.(FileMetaInfo).Meta() meta.Merge(rm.Meta) + if !rm.Meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fi.IsDir()) { + continue + } + if fi.IsDir() { name := fi.Name() if seen[name] { @@ -508,7 +524,14 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } fileCount := 0 + var wasFiltered bool for _, root := range roots { + meta := root.fi.Meta() + if !meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), root.fi.IsDir()) { + wasFiltered = true + continue + } + if !root.fi.IsDir() { fileCount++ } @@ -518,6 +541,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } if fileCount == 0 { + if wasFiltered { + return nil, os.ErrNotExist + } // Dir only. return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil } @@ -531,6 +557,9 @@ func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { } func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) { + if !root.Meta.InclusionFilter.Match(root.trimFrom(name), root.fi.IsDir()) { + return nil, false, os.ErrNotExist + } filename := root.filename(name) fi, b, err := lstatIfPossible(fs.Fs, filename) @@ -586,16 +615,23 @@ func (f *rootMappingFile) Name() string { func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { if f.File != nil { + fis, err := f.File.Readdir(count) if err != nil { return nil, err } - for i, fi := range fis { - fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + var result []os.FileInfo + for _, fi := range fis { + fim := decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + meta := fim.Meta() + if f.meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fim.IsDir()) { + result = append(result, fim) + } } - return fis, nil + return result, nil } + return f.fs.collectDirEntries(f.name) } diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index e83a46a87..c650e8f11 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -20,6 +20,8 @@ import ( "sort" "testing" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/config" qt "github.com/frankban/quicktest" @@ -483,3 +485,70 @@ func TestRootMappingFsOsBase(t *testing.T) { c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"}) } + +func TestRootMappingFileFilter(t *testing.T) { + c := qt.New(t) + fs := NewBaseFileDecorator(afero.NewMemMapFs()) + + for _, lang := range []string{"no", "en", "fr"} { + for i := 1; i <= 3; i++ { + c.Assert(afero.WriteFile(fs, filepath.Join(lang, fmt.Sprintf("my%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil) + } + } + + for _, lang := range []string{"no", "en", "fr"} { + for i := 1; i <= 3; i++ { + c.Assert(afero.WriteFile(fs, filepath.Join(lang, "sub", fmt.Sprintf("mysub%s%d.txt", lang, i)), []byte("some text file for"+lang), 0755), qt.IsNil) + } + } + + rm := []RootMapping{ + { + From: "content", + To: "no", + Meta: &FileMeta{Lang: "no", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + }, + { + From: "content", + To: "en", + Meta: &FileMeta{Lang: "en"}, + }, + { + From: "content", + To: "fr", + Meta: &FileMeta{Lang: "fr", InclusionFilter: glob.MustNewFilenameFilter(nil, []string{"**.txt"})}, + }, + } + + rfs, err := NewRootMappingFs(fs, rm...) + c.Assert(err, qt.IsNil) + + assertExists := func(filename string, shouldExist bool) { + c.Helper() + filename = filepath.Clean(filename) + _, err1 := rfs.Stat(filename) + f, err2 := rfs.Open(filename) + if shouldExist { + c.Assert(err1, qt.IsNil) + c.Assert(err2, qt.IsNil) + c.Assert(f.Close(), qt.IsNil) + } else { + c.Assert(err1, qt.Not(qt.IsNil)) + c.Assert(err2, qt.Not(qt.IsNil)) + } + } + + assertExists("content/myno1.txt", false) + assertExists("content/myen1.txt", true) + assertExists("content/myfr1.txt", false) + + dirEntriesSub, err := afero.ReadDir(rfs, filepath.Join("content", "sub")) + c.Assert(err, qt.IsNil) + c.Assert(len(dirEntriesSub), qt.Equals, 3) + + dirEntries, err := afero.ReadDir(rfs, "content") + + c.Assert(err, qt.IsNil) + c.Assert(len(dirEntries), qt.Equals, 4) + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 0a2c31240..cfbd295ba 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -25,6 +25,9 @@ import ( "sync" "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/hugofs/glob" + + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/common/loggers" "github.com/rogpeppe/go-internal/lockedfile" @@ -127,7 +130,8 @@ func (b *BaseFs) RelContentDir(filename string) string { return filename } -// AbsProjectContentDir tries to create a TODO1 +// AbsProjectContentDir tries to construct a filename below the most +// relevant content directory. func (b *BaseFs) AbsProjectContentDir(filename string) (string, string) { isAbs := filepath.IsAbs(filename) for _, dir := range b.SourceFilesystems.Content.Dirs { @@ -623,6 +627,14 @@ func (b *sourceFilesystemsBuilder) createModFs( mountWeight++ } + inclusionFilter, err := glob.NewFilenameFilter( + types.ToStringSlicePreserveString(mount.IncludeFiles), + types.ToStringSlicePreserveString(mount.ExcludeFiles), + ) + if err != nil { + return err + } + base, filename := absPathify(mount.Source) rm := hugofs.RootMapping{ @@ -631,9 +643,10 @@ func (b *sourceFilesystemsBuilder) createModFs( ToBasedir: base, Module: md.Module.Path(), Meta: &hugofs.FileMeta{ - Watch: md.Watch(), - Weight: mountWeight, - Classifier: files.ContentClassContent, + Watch: md.Watch(), + Weight: mountWeight, + Classifier: files.ContentClassContent, + InclusionFilter: inclusionFilter, }, } diff --git a/hugolib/mount_filters_test.go b/hugolib/mount_filters_test.go new file mode 100644 index 000000000..5e409fcaa --- /dev/null +++ b/hugolib/mount_filters_test.go @@ -0,0 +1,119 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/gohugoio/hugo/common/loggers" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/gohugoio/hugo/htesting" + "github.com/gohugoio/hugo/hugofs" + + qt "github.com/frankban/quicktest" +) + +func TestMountFilters(t *testing.T) { + t.Parallel() + b := newTestSitesBuilder(t) + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-test-mountfilters") + b.Assert(err, qt.IsNil) + defer clean() + + for _, component := range files.ComponentFolders { + b.Assert(os.MkdirAll(filepath.Join(workingDir, component), 0777), qt.IsNil) + } + b.WithWorkingDir(workingDir).WithLogger(loggers.NewInfoLogger()) + b.WithConfigFile("toml", fmt.Sprintf(` +workingDir = %q + +[module] +[[module.mounts]] +source = 'content' +target = 'content' +excludeFiles = "/a/c/**" +[[module.mounts]] +source = 'static' +target = 'static' +[[module.mounts]] +source = 'layouts' +target = 'layouts' +excludeFiles = "/**/foo.html" +[[module.mounts]] +source = 'data' +target = 'data' +includeFiles = "/mydata/**" +[[module.mounts]] +source = 'assets' +target = 'assets' +excludeFiles = "/**exclude.*" +[[module.mounts]] +source = 'i18n' +target = 'i18n' +[[module.mounts]] +source = 'archetypes' +target = 'archetypes' + + +`, workingDir)) + + b.WithContent("/a/b/p1.md", "---\ntitle: Include\n---") + b.WithContent("/a/c/p2.md", "---\ntitle: Exclude\n---") + + b.WithSourceFile( + "data/mydata/b.toml", `b1='bval'`, + "data/nodata/c.toml", `c1='bval'`, + "layouts/partials/foo.html", `foo`, + "assets/exclude.txt", `foo`, + "assets/js/exclude.js", `foo`, + "assets/js/include.js", `foo`, + "assets/js/exclude.js", `foo`, + ) + + b.WithTemplatesAdded("index.html", ` + +Data: {{ site.Data }}:END + +Template: {{ templates.Exists "partials/foo.html" }}:END +Resource1: {{ resources.Get "js/include.js" }}:END +Resource2: {{ resources.Get "js/exclude.js" }}:END +Resource3: {{ resources.Get "exclude.txt" }}:END +Resources: {{ resources.Match "**.js" }} +`) + + b.Build(BuildCfg{}) + + assertExists := func(name string, shouldExist bool) { + b.Helper() + b.Assert(b.CheckExists(filepath.Join(workingDir, name)), qt.Equals, shouldExist) + } + + assertExists("public/a/b/p1/index.html", true) + assertExists("public/a/c/p2/index.html", false) + + b.AssertFileContent(filepath.Join(workingDir, "public", "index.html"), ` +Data: map[mydata:map[b:map[b1:bval]]]:END +Template: false +Resource1: js/include.js:END +Resource2: :END +Resource3: :END +Resources: [js/include.js] +`) + +} diff --git a/modules/config.go b/modules/config.go index 45a2f22ee..f80a456cf 100644 --- a/modules/config.go +++ b/modules/config.go @@ -379,6 +379,11 @@ type Mount struct { Lang string // any language code associated with this mount. + // Include only files matching the given Glob patterns (string or slice). + IncludeFiles interface{} + + // Exclude all files matching the given Glob patterns (string or slice). + ExcludeFiles interface{} } func (m Mount) Component() string { diff --git a/source/sourceSpec.go b/source/sourceSpec.go index 3640c83d5..46a55bf64 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -74,7 +74,7 @@ func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *glob.FilenameFilter, f } } shouldInclude := func(filename string) bool { - if !inclusionFilter.Match(filename) { + if !inclusionFilter.Match(filename, false) { return false } for _, r := range regexps {