hugo/hugofs/fileinfo.go

292 lines
6 KiB
Go
Raw Normal View History

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package hugofs provides the file systems used by Hugo.
package hugofs
import (
"os"
"path/filepath"
"reflect"
"runtime"
"sort"
"strings"
"time"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/hugofs/files"
"golang.org/x/text/unicode/norm"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/hreflect"
"github.com/spf13/afero"
)
func NewFileMeta() *FileMeta {
return &FileMeta{}
}
// PathFile returns the relative file path for the file source.
func (f *FileMeta) PathFile() string {
if f.BaseDir == "" {
return ""
}
return strings.TrimPrefix(strings.TrimPrefix(f.Filename, f.BaseDir), filepathSeparator)
}
type FileMeta struct {
Name string
Filename string
Path string
PathWalk string
OriginalFilename string
BaseDir string
SourceRoot string
MountRoot string
Module string
Weight int
Ordinal int
IsOrdered bool
IsSymlink bool
IsRootFile bool
Watch bool
Introduce a tree map for all content This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2019-09-10 09:26:34 +00:00
Classifier files.ContentClass
SkipDir bool
Lang string
TranslationBaseName string
TranslationBaseNameWithExt string
Translations []string
Fs afero.Fs
OpenFunc func() (afero.File, error)
JoinStatFunc func(name string) (FileMetaInfo, error)
// Include only files or directories that match.
InclusionFilter *glob.FilenameFilter
}
func (m *FileMeta) Copy() *FileMeta {
if m == nil {
return NewFileMeta()
}
c := *m
return &c
}
func (m *FileMeta) Merge(from *FileMeta) {
if m == nil || from == nil {
return
}
dstv := reflect.Indirect(reflect.ValueOf(m))
srcv := reflect.Indirect(reflect.ValueOf(from))
for i := 0; i < dstv.NumField(); i++ {
v := dstv.Field(i)
if !v.CanSet() {
continue
}
if !hreflect.IsTruthfulValue(v) {
v.Set(srcv.Field(i))
}
}
if m.InclusionFilter == nil {
m.InclusionFilter = from.InclusionFilter
}
}
func (f *FileMeta) Open() (afero.File, error) {
if f.OpenFunc == nil {
return nil, errors.New("OpenFunc not set")
}
return f.OpenFunc()
}
func (f *FileMeta) JoinStat(name string) (FileMetaInfo, error) {
if f.JoinStatFunc == nil {
return nil, os.ErrNotExist
}
return f.JoinStatFunc(name)
}
type FileMetaInfo interface {
os.FileInfo
Meta() *FileMeta
}
type fileInfoMeta struct {
os.FileInfo
m *FileMeta
}
// Name returns the file's name. Note that we follow symlinks,
// if supported by the file system, and the Name given here will be the
// name of the symlink, which is what Hugo needs in all situations.
func (fi *fileInfoMeta) Name() string {
if name := fi.m.Name; name != "" {
return name
}
return fi.FileInfo.Name()
}
func (fi *fileInfoMeta) Meta() *FileMeta {
return fi.m
}
func NewFileMetaInfo(fi os.FileInfo, m *FileMeta) FileMetaInfo {
if m == nil {
panic("FileMeta must be set")
}
if fim, ok := fi.(FileMetaInfo); ok {
m.Merge(fim.Meta())
}
return &fileInfoMeta{FileInfo: fi, m: m}
}
type dirNameOnlyFileInfo struct {
name string
modTime time.Time
}
func (fi *dirNameOnlyFileInfo) Name() string {
return fi.name
}
func (fi *dirNameOnlyFileInfo) Size() int64 {
panic("not implemented")
}
func (fi *dirNameOnlyFileInfo) Mode() os.FileMode {
return os.ModeDir
}
func (fi *dirNameOnlyFileInfo) ModTime() time.Time {
return fi.modTime
}
func (fi *dirNameOnlyFileInfo) IsDir() bool {
return true
}
func (fi *dirNameOnlyFileInfo) Sys() interface{} {
return nil
}
func newDirNameOnlyFileInfo(name string, meta *FileMeta, fileOpener func() (afero.File, error)) FileMetaInfo {
name = normalizeFilename(name)
_, base := filepath.Split(name)
m := meta.Copy()
if m.Filename == "" {
m.Filename = name
}
m.OpenFunc = fileOpener
m.IsOrdered = false
return NewFileMetaInfo(
&dirNameOnlyFileInfo{name: base, modTime: time.Now()},
m,
)
}
func decorateFileInfo(
fi os.FileInfo,
fs afero.Fs, opener func() (afero.File, error),
filename, filepath string, inMeta *FileMeta) FileMetaInfo {
var meta *FileMeta
var fim FileMetaInfo
filepath = strings.TrimPrefix(filepath, filepathSeparator)
var ok bool
if fim, ok = fi.(FileMetaInfo); ok {
meta = fim.Meta()
} else {
meta = NewFileMeta()
fim = NewFileMetaInfo(fi, meta)
}
if opener != nil {
meta.OpenFunc = opener
}
if fs != nil {
meta.Fs = fs
}
nfilepath := normalizeFilename(filepath)
nfilename := normalizeFilename(filename)
if nfilepath != "" {
meta.Path = nfilepath
}
if nfilename != "" {
meta.Filename = nfilename
}
meta.Merge(inMeta)
return fim
}
func isSymlink(fi os.FileInfo) bool {
return fi != nil && fi.Mode()&os.ModeSymlink == os.ModeSymlink
}
func fileInfosToFileMetaInfos(fis []os.FileInfo) []FileMetaInfo {
fims := make([]FileMetaInfo, len(fis))
for i, v := range fis {
fims[i] = v.(FileMetaInfo)
}
return fims
}
func normalizeFilename(filename string) string {
if filename == "" {
return ""
}
if runtime.GOOS == "darwin" {
// When a file system is HFS+, its filepath is in NFD form.
return norm.NFC.String(filename)
}
return filename
}
func fileInfosToNames(fis []os.FileInfo) []string {
names := make([]string, len(fis))
for i, d := range fis {
names[i] = d.Name()
}
return names
}
func fromSlash(filenames []string) []string {
for i, name := range filenames {
filenames[i] = filepath.FromSlash(name)
}
return filenames
}
func sortFileInfos(fis []os.FileInfo) {
sort.Slice(fis, func(i, j int) bool {
fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo)
return fimi.Meta().Filename < fimj.Meta().Filename
})
}