hugofs: Fix mount with hole regression

Fixes #6854
This commit is contained in:
Bjørn Erik Pedersen 2020-02-09 17:58:55 +01:00
parent 18888e09bb
commit b78576fd38
4 changed files with 129 additions and 10 deletions

View file

@ -81,12 +81,28 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
// NewBaseFileDecorator decorates the given Fs to provide the real filename // NewBaseFileDecorator decorates the given Fs to provide the real filename
// and an Opener func. // and an Opener func.
func NewBaseFileDecorator(fs afero.Fs) afero.Fs { func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
ffs := &baseFileDecoratorFs{Fs: fs} ffs := &baseFileDecoratorFs{Fs: fs}
decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) { decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
// Store away the original in case it's a symlink. // Store away the original in case it's a symlink.
meta := FileMeta{metaKeyName: fi.Name()} meta := FileMeta{metaKeyName: fi.Name()}
if fi.IsDir() {
meta[metaKeyJoinStat] = func(name string) (FileMetaInfo, error) {
joinedFilename := filepath.Join(filename, name)
fi, _, err := lstatIfPossible(fs, joinedFilename)
if err != nil {
return nil, err
}
fi, err = ffs.decorate(fi, joinedFilename)
if err != nil {
return nil, err
}
return fi.(FileMetaInfo), nil
}
}
isSymlink := isSymlink(fi) isSymlink := isSymlink(fi)
if isSymlink { if isSymlink {
meta[metaKeyOriginalFilename] = filename meta[metaKeyOriginalFilename] = filename

View file

@ -50,6 +50,7 @@ const (
metaKeyOpener = "opener" metaKeyOpener = "opener"
metaKeyIsOrdered = "isOrdered" metaKeyIsOrdered = "isOrdered"
metaKeyIsSymlink = "isSymlink" metaKeyIsSymlink = "isSymlink"
metaKeyJoinStat = "joinStat"
metaKeySkipDir = "skipDir" metaKeySkipDir = "skipDir"
metaKeyClassifier = "classifier" metaKeyClassifier = "classifier"
metaKeyTranslationBaseName = "translationBaseName" metaKeyTranslationBaseName = "translationBaseName"
@ -177,6 +178,14 @@ func (f FileMeta) Open() (afero.File, error) {
return v.(func() (afero.File, error))() return v.(func() (afero.File, error))()
} }
func (f FileMeta) JoinStat(name string) (FileMetaInfo, error) {
v, found := f[metaKeyJoinStat]
if !found {
return nil, os.ErrNotExist
}
return v.(func(name string) (FileMetaInfo, error))(name)
}
func (f FileMeta) stringV(key string) string { func (f FileMeta) stringV(key string) string {
if v, found := f[key]; found { if v, found := f[key]; found {
return v.(string) return v.(string)

View file

@ -128,6 +128,11 @@ type RootMapping struct {
} }
type keyRootMappings struct {
key string
roots []RootMapping
}
func (rm *RootMapping) clean() { func (rm *RootMapping) clean() {
rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator) rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator)
rm.To = filepath.Clean(rm.To) rm.To = filepath.Clean(rm.To)
@ -281,6 +286,21 @@ func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
return roots return roots
} }
func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
var roots []keyRootMappings
fs.rootMapToReal.WalkPath(prefix, func(s string, v interface{}) bool {
if strings.HasPrefix(prefix, s+filepathSeparator) {
roots = append(roots, keyRootMappings{
key: s,
roots: v.([]RootMapping),
})
}
return false
})
return roots
}
func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
meta := fis[0].Meta() meta := fis[0].Meta()
f, err := meta.Open() f, err := meta.Open()
@ -342,17 +362,15 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
seen := make(map[string]bool) // Prevent duplicate directories seen := make(map[string]bool) // Prevent duplicate directories
level := strings.Count(prefix, filepathSeparator) level := strings.Count(prefix, filepathSeparator)
// First add any real files/directories. collectDir := func(rm RootMapping, fi FileMetaInfo) error {
rms := fs.getRoot(prefix) f, err := fi.Meta().Open()
for _, rm := range rms {
f, err := rm.fi.Meta().Open()
if err != nil { if err != nil {
return nil, err return err
} }
direntries, err := f.Readdir(-1) direntries, err := f.Readdir(-1)
if err != nil { if err != nil {
f.Close() f.Close()
return nil, err return err
} }
for _, fi := range direntries { for _, fi := range direntries {
@ -374,6 +392,16 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
} }
f.Close() f.Close()
return nil
}
// First add any real files/directories.
rms := fs.getRoot(prefix)
for _, rm := range rms {
if err := collectDir(rm, rm.fi); err != nil {
return nil, err
}
} }
// Next add any file mounts inside the given directory. // Next add any file mounts inside the given directory.
@ -428,6 +456,22 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
return false return false
}) })
// Finally add any ancestor dirs with files in this directory.
ancestors := fs.getAncestors(prefix)
for _, root := range ancestors {
subdir := strings.TrimPrefix(prefix, root.key)
for _, rm := range root.roots {
if rm.fi.IsDir() {
fi, err := rm.fi.Meta().JoinStat(subdir)
if err == nil {
if err := collectDir(rm, fi); err != nil {
return nil, err
}
}
}
}
}
return fis, nil return fis, nil
} }

View file

@ -365,12 +365,18 @@ func TestRootMappingFsOs(t *testing.T) {
c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)
// https://github.com/gohugoio/hugo/issues/6854
mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)
rfs, err := newRootMappingFsFromFromTo( rfs, err := newRootMappingFsFromFromTo(
d, d,
fs, fs,
"static/bf1", filepath.Join(d, "f1t"), "static/bf1", filepath.Join(d, "f1t"),
"static/cf2", filepath.Join(d, "f2t"), "static/cf2", filepath.Join(d, "f2t"),
"static/af3", filepath.Join(d, "f3t"), "static/af3", filepath.Join(d, "f3t"),
"static", filepath.Join(d, "mystatic"),
"static/a/b/c", filepath.Join(d, "d1", "d2", "d3"), "static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
"layouts", filepath.Join(d, "d1"), "layouts", filepath.Join(d, "d1"),
) )
@ -400,13 +406,13 @@ func TestRootMappingFsOs(t *testing.T) {
} }
c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"}) c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})
c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"}) c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"}) c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})
all, err := collectFilenames(rfs, "static", "static") all, err := collectFilenames(rfs, "static", "static")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"}) c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "a/b/c/ms-1.txt", "cf2/myfile.txt"})
fis, err := collectFileinfos(rfs, "static", "static") fis, err := collectFileinfos(rfs, "static", "static")
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
@ -423,7 +429,7 @@ func TestRootMappingFsOs(t *testing.T) {
sortFileInfos(fileInfos) sortFileInfos(fileInfos)
i := 0 i := 0
for _, fi := range fileInfos { for _, fi := range fileInfos {
if fi.IsDir() { if fi.IsDir() || fi.Name() == "ms-1.txt" {
continue continue
} }
i++ i++
@ -437,3 +443,47 @@ func TestRootMappingFsOs(t *testing.T) {
_, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3")) _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))
c.Assert(err, qt.IsNil) c.Assert(err, qt.IsNil)
} }
func TestRootMappingFsOsBase(t *testing.T) {
c := qt.New(t)
fs := NewBaseFileDecorator(afero.NewOsFs())
d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os-base")
c.Assert(err, qt.IsNil)
defer clean()
// Deep structure
deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
for i := 1; i <= 3; i++ {
c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)
}
mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)
bfs := afero.NewBasePathFs(fs, d)
rfs, err := newRootMappingFsFromFromTo(
"",
bfs,
"static", "mystatic",
"static/a/b/c", filepath.Join("d1", "d2", "d3"),
)
getDirnames := func(dirname string) []string {
dirname = filepath.FromSlash(dirname)
f, err := rfs.Open(dirname)
c.Assert(err, qt.IsNil)
defer f.Close()
dirnames, err := f.Readdirnames(-1)
c.Assert(err, qt.IsNil)
sort.Strings(dirnames)
return dirnames
}
c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
}