diff --git a/Gopkg.lock b/Gopkg.lock index 29f233299..6b96deb67 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -293,8 +293,8 @@ ".", "mem" ] - revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" - version = "v1.0.2" + revision = "63644898a8da0bc22138abf860edaf5277b6102e" + version = "v1.1.0" [[projects]] name = "github.com/spf13/cast" @@ -424,6 +424,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "13ab39f8bfafadc12c05726e565ee3f3d94bf7d6c0e8adf04056de0691bf2dd6" + inputs-digest = "edb250b53926de21df1740c379c76351b7e9b110c96a77078a10ba69bf31a2d4" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index fc1af824b..e14d6dc00 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -78,7 +78,7 @@ [[constraint]] name = "github.com/spf13/afero" - version = "^1.0.1" + version = "^1.1.0" [[constraint]] name = "github.com/spf13/cast" diff --git a/commands/hugo.go b/commands/hugo.go index a5b2c8895..ba8c0aef4 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -705,7 +705,7 @@ func (c *commandeer) getDirList() ([]string, error) { c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err) return nil } - linkfi, err := helpers.LstatIfOs(c.Fs.Source, link) + linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link) if err != nil { c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err) return nil @@ -743,9 +743,13 @@ func (c *commandeer) getDirList() ([]string, error) { // SymbolicWalk will log anny ERRORs _ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker) - _ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), symLinkWalker) _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker) _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker) + + for _, contentDir := range c.PathSpec().ContentDirs() { + _ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker) + } + for _, staticDir := range staticDirs { _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker) } diff --git a/common/types/types.go b/common/types/types.go index 291bf6cf3..a5805d07a 100644 --- a/common/types/types.go +++ b/common/types/types.go @@ -20,6 +20,12 @@ import ( "github.com/spf13/cast" ) +// KeyValueStr is a string tuple. +type KeyValueStr struct { + Key string + Value string +} + // KeyValues holds an key and a slice of values. type KeyValues struct { Key interface{} diff --git a/create/content.go b/create/content.go index 8af417294..29fe47394 100644 --- a/create/content.go +++ b/create/content.go @@ -63,7 +63,22 @@ func NewContent( return err } - contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath)) + // The site may have multiple content dirs, and we currently do not know which contentDir the + // user wants to create this content in. We should improve on this, but we start by testing if the + // provided path points to an existing dir. If so, use it as is. + var contentPath string + var exists bool + targetDir := filepath.Dir(targetPath) + + if targetDir != "" && targetDir != "." { + exists, _ = helpers.Exists(targetDir, ps.Fs.Source) + } + + if exists { + contentPath = targetPath + } else { + contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath)) + } if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil { return err diff --git a/create/content_template_handler.go b/create/content_template_handler.go index 705efbd20..e9e7cb62b 100644 --- a/create/content_template_handler.go +++ b/create/content_template_handler.go @@ -88,10 +88,15 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile err error ) - sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs) + ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg) + sp := source.NewSourceSpec(ps, ps.Fs.Source) + if err != nil { + return nil, err + } f := sp.NewFileInfo("", targetPath, false, nil) name := f.TranslationBaseName() + if name == "index" || name == "_index" { // Page bundles; the directory name will hopefully have a better name. dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator) diff --git a/create/content_test.go b/create/content_test.go index 914759164..62d5ed1da 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -75,7 +75,7 @@ func TestNewContent(t *testing.T) { for i, v := range c.expected { found := strings.Contains(content, v) if !found { - t.Errorf("[%d] %q missing from output:\n%q", i, v, content) + t.Fatalf("[%d] %q missing from output:\n%q", i, v, content) } } } diff --git a/deps/deps.go b/deps/deps.go index ac89d6cd6..fd9635444 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -126,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } - sp := source.NewSourceSpec(cfg.Language, fs) + sp := source.NewSourceSpec(ps, fs.Source) d := &Deps{ Fs: fs, diff --git a/helpers/language.go b/helpers/language.go index 49a25ccf7..731e9b088 100644 --- a/helpers/language.go +++ b/helpers/language.go @@ -41,6 +41,14 @@ type Language struct { Title string Weight int + Disabled bool + + // If set per language, this tells Hugo that all content files without any + // language indicator (e.g. my-page.en.md) is in this language. + // This is usually a path relative to the working dir, but it can be an + // absolute directory referenece. It is what we get. + ContentDir string + Cfg config.Provider // These are params declared in the [params] section of the language merged with the @@ -66,7 +74,13 @@ func NewLanguage(lang string, cfg config.Provider) *Language { params[k] = v } ToLowerMap(params) - l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})} + + defaultContentDir := cfg.GetString("contentDir") + if defaultContentDir == "" { + panic("contentDir not set") + } + + l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})} return l } diff --git a/helpers/language_test.go b/helpers/language_test.go index 68ee3506d..4c4670321 100644 --- a/helpers/language_test.go +++ b/helpers/language_test.go @@ -22,11 +22,12 @@ import ( func TestGetGlobalOnlySetting(t *testing.T) { v := viper.New() + v.Set("defaultContentLanguageInSubdir", true) + v.Set("contentDir", "content") + v.Set("paginatePath", "page") lang := NewDefaultLanguage(v) lang.Set("defaultContentLanguageInSubdir", false) lang.Set("paginatePath", "side") - v.Set("defaultContentLanguageInSubdir", true) - v.Set("paginatePath", "page") require.True(t, lang.GetBool("defaultContentLanguageInSubdir")) require.Equal(t, "side", lang.GetString("paginatePath")) @@ -37,6 +38,7 @@ func TestLanguageParams(t *testing.T) { v := viper.New() v.Set("p1", "p1cfg") + v.Set("contentDir", "content") lang := NewDefaultLanguage(v) lang.SetParam("p1", "p1p") diff --git a/helpers/path.go b/helpers/path.go index 0a8544357..7ac9208bf 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -33,7 +33,7 @@ var ( ErrThemeUndefined = errors.New("no theme set") // ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters. - ErrWalkRootTooShort = errors.New("Path too short. Stop walking.") + ErrPathTooShort = errors.New("file path is too short") ) // filepathPathBridge is a bridge for common functionality in filepath vs path @@ -446,7 +446,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { // Sanity check if len(root) < 4 { - return ErrWalkRootTooShort + return ErrPathTooShort } // Handle the root first @@ -481,7 +481,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error { } func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { - fileInfo, err := LstatIfOs(fs, path) + fileInfo, err := LstatIfPossible(fs, path) realPath := path if err != nil { @@ -493,7 +493,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) { if err != nil { return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err) } - fileInfo, err = LstatIfOs(fs, link) + fileInfo, err = LstatIfPossible(fs, link) if err != nil { return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err) } @@ -514,16 +514,14 @@ func GetRealPath(fs afero.Fs, path string) (string, error) { return realPath, nil } -// Code copied from Afero's path.go -// if the filesystem is OsFs use Lstat, else use fs.Stat -func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) { - _, ok := fs.(*afero.OsFs) - if ok { - info, err = os.Lstat(path) - } else { - info, err = fs.Stat(path) +// LstatIfPossible can be used to call Lstat if possible, else Stat. +func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { + if lstater, ok := fs.(afero.Lstater); ok { + fi, _, err := lstater.LstatIfPossible(path) + return fi, err } - return + + return fs.Stat(path) } // SafeWriteToDisk is the same as WriteToDisk diff --git a/helpers/path_test.go b/helpers/path_test.go index d2c577dae..c2ac19675 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -57,8 +57,10 @@ func TestMakePath(t *testing.T) { for _, test := range tests { v := viper.New() - l := NewDefaultLanguage(v) + v.Set("contentDir", "content") v.Set("removePathAccents", test.removeAccents) + + l := NewDefaultLanguage(v) p, err := NewPathSpec(hugofs.NewMem(v), l) require.NoError(t, err) @@ -71,6 +73,8 @@ func TestMakePath(t *testing.T) { func TestMakePathSanitized(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") + l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -98,6 +102,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) { v := viper.New() v.Set("disablePathToLower", true) + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) diff --git a/helpers/pathspec.go b/helpers/pathspec.go index d35538b85..b18408590 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -17,6 +17,9 @@ import ( "fmt" "strings" + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugofs" "github.com/spf13/cast" @@ -44,11 +47,13 @@ type PathSpec struct { theme string // Directories - contentDir string - themesDir string - layoutDir string - workingDir string - staticDirs []string + contentDir string + themesDir string + layoutDir string + workingDir string + staticDirs []string + absContentDirs []types.KeyValueStr + PublishDir string // The PathSpec looks up its config settings in both the current language @@ -65,6 +70,9 @@ type PathSpec struct { // The file systems to use Fs *hugofs.Fs + // The fine grained filesystems in play (resources, content etc.). + BaseFs *hugofs.BaseFs + // The config provider to use Cfg config.Provider } @@ -105,8 +113,65 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { languages = l } + defaultContentLanguage := cfg.GetString("defaultContentLanguage") + + // We will eventually pull out this badly placed path logic. + contentDir := cfg.GetString("contentDir") + workingDir := cfg.GetString("workingDir") + resourceDir := cfg.GetString("resourceDir") + publishDir := cfg.GetString("publishDir") + + if len(languages) == 0 { + // We have some old tests that does not test the entire chain, hence + // they have no languages. So create one so we get the proper filesystem. + languages = Languages{&Language{Lang: "en", ContentDir: contentDir}} + } + + absPuslishDir := AbsPathify(workingDir, publishDir) + if !strings.HasSuffix(absPuslishDir, FilePathSeparator) { + absPuslishDir += FilePathSeparator + } + // If root, remove the second '/' + if absPuslishDir == "//" { + absPuslishDir = FilePathSeparator + } + absResourcesDir := AbsPathify(workingDir, resourceDir) + if !strings.HasSuffix(absResourcesDir, FilePathSeparator) { + absResourcesDir += FilePathSeparator + } + if absResourcesDir == "//" { + absResourcesDir = FilePathSeparator + } + + contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages) + if err != nil { + return nil, err + } + + // Make sure we don't have any overlapping content dirs. That will never work. + for i, d1 := range absContentDirs { + for j, d2 := range absContentDirs { + if i == j { + continue + } + if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { + return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) + } + } + } + + resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir) + publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir) + + baseFs := &hugofs.BaseFs{ + ContentFs: contentFs, + ResourcesFs: resourcesFs, + PublishFs: publishFs, + } + ps := &PathSpec{ Fs: fs, + BaseFs: baseFs, Cfg: cfg, disablePathToLower: cfg.GetBool("disablePathToLower"), removePathAccents: cfg.GetBool("removePathAccents"), @@ -116,14 +181,15 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { Language: language, Languages: languages, defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"), - defaultContentLanguage: cfg.GetString("defaultContentLanguage"), + defaultContentLanguage: defaultContentLanguage, paginatePath: cfg.GetString("paginatePath"), BaseURL: baseURL, - contentDir: cfg.GetString("contentDir"), + contentDir: contentDir, themesDir: cfg.GetString("themesDir"), layoutDir: cfg.GetString("layoutDir"), - workingDir: cfg.GetString("workingDir"), + workingDir: workingDir, staticDirs: staticDirs, + absContentDirs: absContentDirs, theme: cfg.GetString("theme"), ProcessingStats: NewProcessingStats(lang), } @@ -135,13 +201,8 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { } } - publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator - // If root, remove the second '/' - if publishDir == "//" { - publishDir = FilePathSeparator - } - - ps.PublishDir = publishDir + // TODO(bep) remove this, eventually + ps.PublishDir = absPuslishDir return ps, nil } @@ -165,6 +226,107 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { return out } +func createContentFs(fs afero.Fs, + workingDir, + defaultContentLanguage string, + languages Languages) (afero.Fs, []types.KeyValueStr, error) { + + var contentLanguages Languages + var contentDirSeen = make(map[string]bool) + languageSet := make(map[string]bool) + + // The default content language needs to be first. + for _, language := range languages { + if language.Lang == defaultContentLanguage { + contentLanguages = append(contentLanguages, language) + contentDirSeen[language.ContentDir] = true + } + languageSet[language.Lang] = true + } + + for _, language := range languages { + if contentDirSeen[language.ContentDir] { + continue + } + if language.ContentDir == "" { + language.ContentDir = defaultContentLanguage + } + contentDirSeen[language.ContentDir] = true + contentLanguages = append(contentLanguages, language) + + } + + var absContentDirs []types.KeyValueStr + + fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) + return fs, absContentDirs, err + +} + +func createContentOverlayFs(source afero.Fs, + workingDir string, + languages Languages, + languageSet map[string]bool, + absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { + if len(languages) == 0 { + return source, nil + } + + language := languages[0] + + contentDir := language.ContentDir + if contentDir == "" { + panic("missing contentDir") + } + + absContentDir := AbsPathify(workingDir, language.ContentDir) + if !strings.HasSuffix(absContentDir, FilePathSeparator) { + absContentDir += FilePathSeparator + } + + // If root, remove the second '/' + if absContentDir == "//" { + absContentDir = FilePathSeparator + } + + if len(absContentDir) < 6 { + return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort) + } + + *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) + + overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) + if len(languages) == 1 { + return overlay, nil + } + + base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) + if err != nil { + return nil, err + } + + return hugofs.NewLanguageCompositeFs(base, overlay), nil + +} + +// RelContentDir tries to create a path relative to the content root from +// the given filename. The return value is the path and language code. +func (p *PathSpec) RelContentDir(filename string) (string, string) { + for _, dir := range p.absContentDirs { + if strings.HasPrefix(filename, dir.Value) { + rel := strings.TrimPrefix(filename, dir.Value) + return strings.TrimPrefix(rel, FilePathSeparator), dir.Key + } + } + // Either not a content dir or already relative. + return filename, "" +} + +// ContentDirs returns all the content dirs (absolute paths). +func (p *PathSpec) ContentDirs() []types.KeyValueStr { + return p.absContentDirs +} + // PaginatePath returns the configured root path used for paginator pages. func (p *PathSpec) PaginatePath() string { return p.paginatePath diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index e10ccc639..dc2079e06 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -24,6 +24,7 @@ import ( func TestNewPathSpecFromConfig(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") l := NewLanguage("no", v) v.Set("disablePathToLower", true) v.Set("removePathAccents", true) diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 518a5bc23..215ae9188 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -25,6 +25,7 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec { func newTestCfg(fs *hugofs.Fs) *viper.Viper { v := viper.New() + v.Set("contentDir", "content") v.SetFs(fs.Source) diff --git a/helpers/url_test.go b/helpers/url_test.go index 9572547c7..0ca3c8df2 100644 --- a/helpers/url_test.go +++ b/helpers/url_test.go @@ -27,6 +27,7 @@ import ( func TestURLize(t *testing.T) { v := viper.New() + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -88,6 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for _, test := range tests { v.Set("baseURL", test.baseURL) + v.Set("contentDir", "content") l := NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -166,6 +168,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, for i, test := range tests { v.Set("baseURL", test.baseURL) v.Set("canonifyURLs", test.canonify) + v.Set("contentDir", "content") l := NewLanguage(lang, v) p, _ := NewPathSpec(hugofs.NewMem(v), l) @@ -254,6 +257,7 @@ func TestURLPrep(t *testing.T) { for i, d := range data { v := viper.New() v.Set("uglyURLs", d.ugly) + v.Set("contentDir", "content") l := NewDefaultLanguage(v) p, _ := NewPathSpec(hugofs.NewMem(v), l) diff --git a/hugofs/base_fs.go b/hugofs/base_fs.go new file mode 100644 index 000000000..77af66dfe --- /dev/null +++ b/hugofs/base_fs.go @@ -0,0 +1,35 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "github.com/spf13/afero" +) + +// BaseFs contains the core base filesystems used by Hugo. The name "base" is used +// to underline that even if they can be composites, they all have a base path set to a specific +// resource folder, e.g "/my-project/content". So, no absolute filenames needed. +type BaseFs struct { + // The filesystem used to capture content. This can be a composite and + // language aware file system. + ContentFs afero.Fs + + // The filesystem used to store resources (processed images etc.). + // This usually maps to /my-project/resources. + ResourcesFs afero.Fs + + // The filesystem used to publish the rendered site. + // This usually maps to /my-project/public. + PublishFs afero.Fs +} diff --git a/hugofs/language_composite_fs.go b/hugofs/language_composite_fs.go new file mode 100644 index 000000000..2889f8a00 --- /dev/null +++ b/hugofs/language_composite_fs.go @@ -0,0 +1,51 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "github.com/spf13/afero" +) + +var ( + _ afero.Fs = (*languageCompositeFs)(nil) + _ afero.Lstater = (*languageCompositeFs)(nil) +) + +type languageCompositeFs struct { + *afero.CopyOnWriteFs +} + +// NewLanguageCompositeFs creates a composite and language aware filesystem. +// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename +// to the target filesystem. This information is available in Readdir, Stat etc. via the +// special LanguageFileInfo FileInfo implementation. +func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs { + return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}) +} + +// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged +// using the language as a weight. +func (fs *languageCompositeFs) Open(name string) (afero.File, error) { + f, err := fs.CopyOnWriteFs.Open(name) + if err != nil { + return nil, err + } + + fu, ok := f.(*afero.UnionFile) + if ok { + // This is a directory: Merge it. + fu.Merger = LanguageDirsMerger + } + return f, nil +} diff --git a/hugofs/language_composite_fs_test.go b/hugofs/language_composite_fs_test.go new file mode 100644 index 000000000..bb4ddf701 --- /dev/null +++ b/hugofs/language_composite_fs_test.go @@ -0,0 +1,106 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + + "strings" + + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestCompositeLanguagFsTest(t *testing.T) { + assert := require.New(t) + + languages := map[string]bool{ + "sv": true, + "en": true, + "nn": true, + } + msv := afero.NewMemMapFs() + baseSv := "/content/sv" + lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv)) + mnn := afero.NewMemMapFs() + baseNn := "/content/nn" + lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn)) + men := afero.NewMemMapFs() + baseEn := "/content/en" + lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn)) + + // The order will be sv, en, nn + composite := NewLanguageCompositeFs(lfsnn, lfsen) + composite = NewLanguageCompositeFs(composite, lfssv) + + afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755) + afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755) + afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755) + + // Swedish is the top layer. + assertLangFile(t, composite, "f1.txt", "sv") + + afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755) + afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755) + afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755) + + // English is in the middle, but the most specific language match wins. + //assertLangFile(t, composite, "f2.en.txt", "en") + + // Fetch some specific language versions + assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn") + assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en") + assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv") + + // Read the root + f, err := composite.Open("/") + assert.NoError(err) + defer f.Close() + files, err := f.Readdir(-1) + assert.Equal(4, len(files)) + expected := map[string]bool{ + filepath.FromSlash("/content/en/f1.txt"): true, + filepath.FromSlash("/content/nn/f1.txt"): true, + filepath.FromSlash("/content/sv/f1.txt"): true, + filepath.FromSlash("/content/en/f2.en.txt"): true, + } + got := make(map[string]bool) + + for _, fi := range files { + fil, ok := fi.(*LanguageFileInfo) + assert.True(ok) + got[fil.Filename()] = true + } + assert.Equal(expected, got) +} + +func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) { + f, err := fs.Open(filename) + if err != nil { + t.Fatal(err) + } + defer f.Close() + b, err := afero.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + s := string(b) + if !strings.Contains(s, match) { + t.Fatalf("got %q expected it to contain %q", s, match) + + } +} diff --git a/hugofs/language_fs.go b/hugofs/language_fs.go new file mode 100644 index 000000000..95ec0831e --- /dev/null +++ b/hugofs/language_fs.go @@ -0,0 +1,328 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/afero" +) + +const hugoFsMarker = "__hugofs" + +var ( + _ LanguageAnnouncer = (*LanguageFileInfo)(nil) + _ FilePather = (*LanguageFileInfo)(nil) + _ afero.Lstater = (*LanguageFs)(nil) +) + +// LanguageAnnouncer is aware of its language. +type LanguageAnnouncer interface { + Lang() string + TranslationBaseName() string +} + +// FilePather is aware of its file's location. +type FilePather interface { + // Filename gets the full path and filename to the file. + Filename() string + + // Path gets the content relative path including file name and extension. + // The directory is relative to the content root where "content" is a broad term. + Path() string + + // RealName is FileInfo.Name in its original form. + RealName() string + + BaseDir() string +} + +var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + m := make(map[string]*LanguageFileInfo) + + for _, fi := range lofi { + fil, ok := fi.(*LanguageFileInfo) + if !ok { + return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) + } + m[fil.virtualName] = fil + } + + for _, fi := range bofi { + fil, ok := fi.(*LanguageFileInfo) + if !ok { + return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi) + } + existing, found := m[fil.virtualName] + + if !found || existing.weight < fil.weight { + m[fil.virtualName] = fil + } + } + + merged := make([]os.FileInfo, len(m)) + i := 0 + for _, v := range m { + merged[i] = v + i++ + } + + return merged, nil +} + +type LanguageFileInfo struct { + os.FileInfo + lang string + baseDir string + realFilename string + relFilename string + name string + realName string + virtualName string + translationBaseName string + + // We add some weight to the files in their own language's content directory. + weight int +} + +func (fi *LanguageFileInfo) Filename() string { + return fi.realFilename +} + +func (fi *LanguageFileInfo) Path() string { + return fi.relFilename +} + +func (fi *LanguageFileInfo) RealName() string { + return fi.realName +} + +func (fi *LanguageFileInfo) BaseDir() string { + return fi.baseDir +} + +func (fi *LanguageFileInfo) Lang() string { + return fi.lang +} + +// TranslationBaseName returns the base filename without any extension or language +// identificator. +func (fi *LanguageFileInfo) TranslationBaseName() string { + return fi.translationBaseName +} + +// Name is the name of the file within this filesystem without any path info. +// It will be marked with language information so we can identify it as ours. +func (fi *LanguageFileInfo) Name() string { + return fi.name +} + +type languageFile struct { + afero.File + fs *LanguageFs +} + +// Readdir creates FileInfo entries by calling Lstat if possible. +func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) { + names, err := l.File.Readdirnames(c) + if err != nil { + return nil, err + } + + fis := make([]os.FileInfo, len(names)) + + for i, name := range names { + fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name)) + + if err != nil { + return nil, err + } + fis[i] = fi + } + + return fis, err +} + +type LanguageFs struct { + // This Fs is usually created with a BasePathFs + basePath string + lang string + nameMarker string + languages map[string]bool + afero.Fs +} + +func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs { + if lang == "" { + panic("no lang set for the language fs") + } + var basePath string + + if bfs, ok := fs.(*afero.BasePathFs); ok { + basePath, _ = bfs.RealPath("") + } + + marker := hugoFsMarker + "_" + lang + "_" + + return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker} +} + +func (fs *LanguageFs) Lang() string { + return fs.lang +} + +func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) { + name, err := fs.realName(name) + if err != nil { + return nil, err + } + + fi, err := fs.Fs.Stat(name) + if err != nil { + return nil, err + } + + return fs.newLanguageFileInfo(name, fi) +} + +func (fs *LanguageFs) Open(name string) (afero.File, error) { + name, err := fs.realName(name) + if err != nil { + return nil, err + } + f, err := fs.Fs.Open(name) + + if err != nil { + return nil, err + } + return &languageFile{File: f, fs: fs}, nil +} + +func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := fs.realName(name) + if err != nil { + return nil, false, err + } + + var fi os.FileInfo + var b bool + + if lif, ok := fs.Fs.(afero.Lstater); ok { + fi, b, err = lif.LstatIfPossible(name) + } else { + fi, err = fs.Fs.Stat(name) + } + + if err != nil { + return nil, b, err + } + + lfi, err := fs.newLanguageFileInfo(name, fi) + + return lfi, b, err +} + +func (fs *LanguageFs) realPath(name string) (string, error) { + if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok { + return baseFs.RealPath(name) + } + return name, nil +} + +func (fs *LanguageFs) realName(name string) (string, error) { + if strings.Contains(name, hugoFsMarker) { + if !strings.Contains(name, fs.nameMarker) { + return "", os.ErrNotExist + } + return strings.Replace(name, fs.nameMarker, "", 1), nil + } + + if fs.basePath == "" { + return name, nil + } + + return strings.TrimPrefix(name, fs.basePath), nil +} + +func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) { + filename = filepath.Clean(filename) + _, name := filepath.Split(filename) + + realName := name + virtualName := name + + realPath, err := fs.realPath(filename) + if err != nil { + return nil, err + } + + lang := fs.Lang() + + baseNameNoExt := "" + + if !fi.IsDir() { + + // Try to extract the language from the file name. + // Any valid language identificator in the name will win over the + // language set on the file system, e.g. "mypost.en.md". + baseName := filepath.Base(name) + ext := filepath.Ext(baseName) + baseNameNoExt = baseName + + if ext != "" { + baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext) + } + + fileLangExt := filepath.Ext(baseNameNoExt) + fileLang := strings.TrimPrefix(fileLangExt, ".") + + if fs.languages[fileLang] { + lang = fileLang + } + + baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt) + + // This connects the filename to the filesystem, not the language. + virtualName = baseNameNoExt + "." + lang + ext + + name = fs.nameMarker + name + } + + weight := 1 + // If this file's language belongs in this directory, add some weight to it + // to make it more important. + if lang == fs.Lang() { + weight = 2 + } + + if fi.IsDir() { + // For directories we always want to start from the union view. + realPath = strings.TrimPrefix(realPath, fs.basePath) + } + + return &LanguageFileInfo{ + lang: lang, + weight: weight, + realFilename: realPath, + realName: realName, + relFilename: strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)), + name: name, + virtualName: virtualName, + translationBaseName: baseNameNoExt, + baseDir: fs.basePath, + FileInfo: fi}, nil +} diff --git a/hugofs/language_fs_test.go b/hugofs/language_fs_test.go new file mode 100644 index 000000000..ac17a1930 --- /dev/null +++ b/hugofs/language_fs_test.go @@ -0,0 +1,54 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugofs + +import ( + "path/filepath" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestLanguagFs(t *testing.T) { + languages := map[string]bool{ + "sv": true, + } + base := filepath.FromSlash("/my/base") + assert := require.New(t) + m := afero.NewMemMapFs() + bfs := afero.NewBasePathFs(m, base) + lfs := NewLanguageFs("sv", languages, bfs) + assert.NotNil(lfs) + assert.Equal("sv", lfs.Lang()) + err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777) + assert.NoError(err) + fi, err := lfs.Stat(filepath.FromSlash("sect/page.md")) + assert.NoError(err) + assert.Equal("__hugofs_sv_page.md", fi.Name()) + + languager, ok := fi.(LanguageAnnouncer) + assert.True(ok) + + assert.Equal("sv", languager.Lang()) + + lfi, ok := fi.(*LanguageFileInfo) + assert.True(ok) + assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename()) + assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path()) + assert.Equal("page.sv.md", lfi.virtualName) + assert.Equal("__hugofs_sv_page.md", lfi.Name()) + assert.Equal("page.md", lfi.RealName()) + +} diff --git a/hugolib/config.go b/hugolib/config.go index 6eca1a969..9f206bc77 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -130,21 +130,17 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error } else { languages = make(map[string]interface{}) for k, v := range languagesFromConfig { - isDisabled := false for _, disabled := range disableLanguages { if disabled == defaultLang { return fmt.Errorf("cannot disable default language %q", defaultLang) } if strings.EqualFold(k, disabled) { - isDisabled = true + v.(map[string]interface{})["disabled"] = true break } } - if !isDisabled { - languages[k] = v - } - + languages[k] = v } } diff --git a/hugolib/disableKinds_test.go b/hugolib/disableKinds_test.go index d689f9dcc..edada1419 100644 --- a/hugolib/disableKinds_test.go +++ b/hugolib/disableKinds_test.go @@ -104,8 +104,8 @@ categories: writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1")) - writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10) + writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) + writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10) h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) diff --git a/hugolib/fileInfo.go b/hugolib/fileInfo.go index f819f6bfc..90cf91377 100644 --- a/hugolib/fileInfo.go +++ b/hugolib/fileInfo.go @@ -14,7 +14,6 @@ package hugolib import ( - "os" "strings" "github.com/gohugoio/hugo/helpers" @@ -25,11 +24,22 @@ import ( var ( _ source.File = (*fileInfo)(nil) _ source.ReadableFile = (*fileInfo)(nil) + _ pathLangFile = (*fileInfo)(nil) ) +// A partial interface to prevent ambigous compiler error. +type basePather interface { + Filename() string + RealName() string + BaseDir() string +} + type fileInfo struct { bundleTp bundleDirType + source.ReadableFile + basePather + overriddenLang string // Set if the content language for this file is disabled. @@ -43,6 +53,10 @@ func (fi *fileInfo) Lang() string { return fi.ReadableFile.Lang() } +func (fi *fileInfo) Filename() string { + return fi.basePather.Filename() +} + func (fi *fileInfo) isOwner() bool { return fi.bundleTp > bundleNot } @@ -55,12 +69,13 @@ func (fi *fileInfo) isContentFile() bool { return contentFileExtensionsSet[fi.Ext()] } -func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi os.FileInfo, tp bundleDirType) *fileInfo { +func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo { baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi) f := &fileInfo{ bundleTp: tp, ReadableFile: baseFi, + basePather: fi, } lang := f.Lang() diff --git a/hugolib/fileInfo_test.go b/hugolib/fileInfo_test.go deleted file mode 100644 index 18579c078..000000000 --- a/hugolib/fileInfo_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package hugolib - -import ( - "testing" - - "path/filepath" - - "github.com/gohugoio/hugo/source" - "github.com/stretchr/testify/require" -) - -func TestBundleFileInfo(t *testing.T) { - t.Parallel() - - assert := require.New(t) - cfg, fs := newTestBundleSourcesMultilingual(t) - sourceSpec := source.NewSourceSpec(cfg, fs) - - for _, this := range []struct { - filename string - check func(f *fileInfo) - }{ - {"/path/to/file.md", func(fi *fileInfo) { - assert.Equal("md", fi.Ext()) - assert.Equal("en", fi.Lang()) - assert.False(fi.isOwner()) - assert.True(fi.isContentFile()) - }}, - {"/path/to/file.JPG", func(fi *fileInfo) { - assert.Equal("jpg", fi.Ext()) - assert.False(fi.isContentFile()) - }}, - {"/path/to/file.nn.png", func(fi *fileInfo) { - assert.Equal("png", fi.Ext()) - assert.Equal("nn", fi.Lang()) - assert.Equal("file", fi.TranslationBaseName()) - assert.False(fi.isContentFile()) - }}, - } { - fi := newFileInfo( - sourceSpec, - filepath.FromSlash("/work/base"), - filepath.FromSlash(this.filename), - nil, bundleNot) - this.check(fi) - } - -} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 4e802270d..f0eb21dc1 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -75,19 +75,8 @@ func (h *HugoSites) langSite() map[string]*Site { // GetContentPage finds a Page with content given the absolute filename. // Returns nil if none found. func (h *HugoSites) GetContentPage(filename string) *Page { - s := h.Sites[0] - contendDir := filepath.Join(s.PathSpec.AbsPathify(s.Cfg.GetString("contentDir"))) - if !strings.HasPrefix(filename, contendDir) { - return nil - } - - rel := strings.TrimPrefix(filename, contendDir) - rel = strings.TrimPrefix(rel, helpers.FilePathSeparator) - for _, s := range h.Sites { - - pos := s.rawAllPages.findPagePosByFilePath(rel) - + pos := s.rawAllPages.findPagePosByFilename(filename) if pos == -1 { continue } @@ -95,19 +84,16 @@ func (h *HugoSites) GetContentPage(filename string) *Page { } // If not found already, this may be bundled in another content file. - rel = filepath.Dir(rel) + dir := filepath.Dir(filename) + for _, s := range h.Sites { - - pos := s.rawAllPages.findFirstPagePosByFilePathPrefix(rel) - + pos := s.rawAllPages.findPagePosByFilnamePrefix(dir) if pos == -1 { continue } return s.rawAllPages[pos] } - return nil - } // NewHugoSites creates a new collection of sites given the input sites, building @@ -126,18 +112,11 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { var contentChangeTracker *contentChangeMap - // Only needed in server mode. - // TODO(bep) clean up the running vs watching terms - if cfg.Running { - contentChangeTracker = &contentChangeMap{symContent: make(map[string]map[string]bool)} - } - h := &HugoSites{ - running: cfg.Running, - multilingual: langConfig, - multihost: cfg.Cfg.GetBool("multihost"), - ContentChanges: contentChangeTracker, - Sites: sites} + running: cfg.Running, + multilingual: langConfig, + multihost: cfg.Cfg.GetBool("multihost"), + Sites: sites} for _, s := range sites { s.owner = h @@ -149,6 +128,13 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h.Deps = sites[0].Deps + // Only needed in server mode. + // TODO(bep) clean up the running vs watching terms + if cfg.Running { + contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)} + h.ContentChanges = contentChangeTracker + } + if err := h.initGitInfo(); err != nil { return nil, err } @@ -212,6 +198,7 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { d.OutputFormatsConfig = s.outputFormatsConfig s.Deps = d } + s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig) if err != nil { return err @@ -260,6 +247,9 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { languages := getLanguages(cfg.Cfg) for _, lang := range languages { + if lang.Disabled { + continue + } var s *Site var err error cfg.Language = lang @@ -517,9 +507,9 @@ func (h *HugoSites) createMissingPages() error { return nil } -func (h *HugoSites) removePageByPath(path string) { +func (h *HugoSites) removePageByFilename(filename string) { for _, s := range h.Sites { - s.removePageByPath(path) + s.removePageFilename(filename) } } @@ -671,6 +661,8 @@ type contentChangeMap struct { branches []string leafs []string + pathSpec *helpers.PathSpec + // Hugo supports symlinked content (both directories and files). This // can lead to situations where the same file can be referenced from several // locations in /content -- which is really cool, but also means we have to @@ -683,7 +675,7 @@ type contentChangeMap struct { func (m *contentChangeMap) add(filename string, tp bundleDirType) { m.mu.Lock() - dir := filepath.Dir(filename) + dir := filepath.Dir(filename) + helpers.FilePathSeparator switch tp { case bundleBranch: m.branches = append(m.branches, dir) @@ -698,7 +690,7 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) { // Track the addition of bundle dirs. func (m *contentChangeMap) handleBundles(b *bundleDirs) { for _, bd := range b.bundles { - m.add(bd.fi.Filename(), bd.tp) + m.add(bd.fi.Path(), bd.tp) } } @@ -709,21 +701,21 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu m.mu.RLock() defer m.mu.RUnlock() - dir, name := filepath.Split(filename) - dir = strings.TrimSuffix(dir, helpers.FilePathSeparator) - fileTp, isContent := classifyBundledFile(name) - - // If the file itself is a bundle, no need to look further: - if fileTp > bundleNot { - return dir, dir, fileTp + // Bundles share resources, so we need to start from the virtual root. + relPath, _ := m.pathSpec.RelContentDir(filename) + dir, name := filepath.Split(relPath) + if !strings.HasSuffix(dir, helpers.FilePathSeparator) { + dir += helpers.FilePathSeparator } + fileTp, _ := classifyBundledFile(name) + // This may be a member of a bundle. Start with branch bundles, the most specific. - if !isContent { + if fileTp != bundleLeaf { for i, b := range m.branches { if b == dir { m.branches = append(m.branches[:i], m.branches[i+1:]...) - return dir, dir, bundleBranch + return dir, b, bundleBranch } } } @@ -732,7 +724,7 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu for i, l := range m.leafs { if strings.HasPrefix(dir, l) { m.leafs = append(m.leafs[:i], m.leafs[i+1:]...) - return dir, dir, bundleLeaf + return dir, l, bundleLeaf } } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 5e4f171da..1626fadcf 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -3,6 +3,7 @@ package hugolib import ( "bytes" "fmt" + "io" "strings" "testing" @@ -433,7 +434,7 @@ func TestMultiSitesRebuild(t *testing.T) { // t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4 // This leaktest seems to be a little bit shaky on Travis. if !isCI() { - defer leaktest.CheckTimeout(t, 30*time.Second)() + defer leaktest.CheckTimeout(t, 10*time.Second)() } assert := require.New(t) @@ -459,6 +460,8 @@ func TestMultiSitesRebuild(t *testing.T) { b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour") b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello") + contentFs := b.H.BaseFs.ContentFs + for i, this := range []struct { preFunc func(t *testing.T) events []fsnotify.Event @@ -490,9 +493,9 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - writeNewContentFile(t, fs, "new_en_1", "2016-07-31", "content/new1.en.md", -5) - writeNewContentFile(t, fs, "new_en_2", "1989-07-30", "content/new2.en.md", -10) - writeNewContentFile(t, fs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10) + writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5) + writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10) + writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10) }, []fsnotify.Event{ {Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create}, @@ -513,10 +516,10 @@ func TestMultiSitesRebuild(t *testing.T) { }, { func(t *testing.T) { - p := "content/sect/doc1.en.md" - doc1 := readSource(t, fs, p) + p := "sect/doc1.en.md" + doc1 := readFileFromFs(t, contentFs, p) doc1 += "CHANGED" - writeSource(t, fs, p, doc1) + writeToFs(t, contentFs, p, doc1) }, []fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc1.en.md"), Op: fsnotify.Write}}, func(t *testing.T) { @@ -529,7 +532,7 @@ func TestMultiSitesRebuild(t *testing.T) { // Rename a file { func(t *testing.T) { - if err := fs.Source.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil { + if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil { t.Fatalf("Rename failed: %s", err) } }, @@ -650,7 +653,7 @@ weight = 15 title = "Svenska" ` - writeNewContentFile(t, fs, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10) + writeNewContentFile(t, fs.Source, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10) // replace the config b.WithNewConfig(newConfig) @@ -1038,18 +1041,31 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { if err != nil { // Print some debug info root := strings.Split(filename, helpers.FilePathSeparator)[0] - afero.Walk(fs, root, func(path string, info os.FileInfo, err error) error { - if info != nil && !info.IsDir() { - fmt.Println(" ", path) - } - - return nil - }) + printFs(fs, root, os.Stdout) Fatalf(t, "Failed to read file: %s", err) } return string(b) } +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\tLANG: " + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} + const testPageTemplate = `--- title: "%s" publishdate: "%s" @@ -1062,9 +1078,9 @@ func newTestPage(title, date string, weight int) string { return fmt.Sprintf(testPageTemplate, title, date, weight, title) } -func writeNewContentFile(t *testing.T, fs *hugofs.Fs, title, date, filename string, weight int) { +func writeNewContentFile(t *testing.T, fs afero.Fs, title, date, filename string, weight int) { content := newTestPage(title, date, weight) - writeSource(t, fs, filename, content) + writeToFs(t, fs, filename, content) } type multiSiteTestBuilder struct { diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go new file mode 100644 index 000000000..7195e8e7b --- /dev/null +++ b/hugolib/language_content_dir_test.go @@ -0,0 +1,253 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +/* + +/en/p1.md +/nn/p1.md + +.Readdir + +- Name() => p1.en.md, p1.nn.md + +.Stat(name) + +.Open() --- real file name + + +*/ + +func TestLanguageContentRoot(t *testing.T) { + t.Parallel() + assert := require.New(t) + + config := ` +baseURL = "https://example.org/" + +defaultContentLanguage = "en" +defaultContentLanguageInSubdir = true + +contentDir = "content/main" +workingDir = "/my/project" + +[Languages] +[Languages.en] +weight = 10 +title = "In English" +languageName = "English" + +[Languages.nn] +weight = 20 +title = "På Norsk" +languageName = "Norsk" +# This tells Hugo that all content in this directory is in the Norwegian language. +# It does not have to have the "my-page.nn.md" format. It can, but that is optional. +contentDir = "content/norsk" + +[Languages.sv] +weight = 30 +title = "På Svenska" +languageName = "Svensk" +contentDir = "content/svensk" +` + + pageTemplate := ` +--- +title: %s +slug: %s +weight: %d +--- + +Content. + +` + + pageBundleTemplate := ` +--- +title: %s +weight: %d +--- + +Content. + +` + var contentFiles []string + section := "sect" + + var contentRoot = func(lang string) string { + contentRoot := "content/main" + + switch lang { + case "nn": + contentRoot = "content/norsk" + case "sv": + contentRoot = "content/svensk" + } + return contentRoot + "/" + section + } + + for _, lang := range []string{"en", "nn", "sv"} { + for j := 1; j <= 10; j++ { + if (lang == "nn" || lang == "en") && j%4 == 0 { + // Skip 4 and 8 for nn + // We also skip it for en, but that is added to the Swedish directory below. + continue + } + + if lang == "sv" && j%5 == 0 { + // Skip 5 and 10 for sv + continue + } + + base := fmt.Sprintf("p-%s-%d", lang, j) + slug := fmt.Sprintf("%s", base) + langID := "" + + if lang == "sv" && j%4 == 0 { + // Put an English page in the Swedish content dir. + langID = ".en" + } + + if lang == "en" && j == 8 { + // This should win over the sv variant above. + langID = ".en" + } + + slug += langID + + contentRoot := contentRoot(lang) + + filename := filepath.Join(contentRoot, fmt.Sprintf("page%d%s.md", j, langID)) + contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, j)) + } + } + + // Put common translations in all of them + for i, lang := range []string{"en", "nn", "sv"} { + contentRoot := contentRoot(lang) + + slug := fmt.Sprintf("common_%s", lang) + + filename := filepath.Join(contentRoot, "common.md") + contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, 100+i)) + + for j, lang2 := range []string{"en", "nn", "sv"} { + filename := filepath.Join(contentRoot, fmt.Sprintf("translated_all.%s.md", lang2)) + langSlug := slug + "_translated_all_" + lang2 + contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 200+i+j)) + } + + for j, lang2 := range []string{"sv", "nn"} { + if lang == "en" { + continue + } + filename := filepath.Join(contentRoot, fmt.Sprintf("translated_some.%s.md", lang2)) + langSlug := slug + "_translated_some_" + lang2 + contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 300+i+j)) + } + } + + // Add a bundle with some images + for i, lang := range []string{"en", "nn", "sv"} { + contentRoot := contentRoot(lang) + slug := fmt.Sprintf("bundle_%s", lang) + filename := filepath.Join(contentRoot, "mybundle", "index.md") + contentFiles = append(contentFiles, filename, fmt.Sprintf(pageBundleTemplate, slug, 400+i)) + if lang == "en" { + imageFilename := filepath.Join(contentRoot, "mybundle", "logo.png") + contentFiles = append(contentFiles, imageFilename, "PNG Data") + } + imageFilename := filepath.Join(contentRoot, "mybundle", "featured.png") + contentFiles = append(contentFiles, imageFilename, fmt.Sprintf("PNG Data for %s", lang)) + + // Add some bundled pages + contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 401+i)) + contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "sub", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 402+i)) + + } + + b := newTestSitesBuilder(t) + b.WithWorkingDir("/my/project").WithConfigFile("toml", config).WithContent(contentFiles...).CreateSites() + + _ = os.Stdout + //printFs(b.H.BaseFs.ContentFs, "/", os.Stdout) + + b.Build(BuildCfg{}) + + assert.Equal(3, len(b.H.Sites)) + + enSite := b.H.Sites[0] + nnSite := b.H.Sites[1] + svSite := b.H.Sites[2] + + //dumpPages(nnSite.RegularPages...) + assert.Equal(12, len(nnSite.RegularPages)) + assert.Equal(13, len(enSite.RegularPages)) + + assert.Equal(10, len(svSite.RegularPages)) + + for i, p := range enSite.RegularPages { + j := i + 1 + msg := fmt.Sprintf("Test %d", j) + assert.Equal("en", p.Lang(), msg) + assert.Equal("sect", p.Section()) + if j < 9 { + if j%4 == 0 { + assert.Contains(p.Title(), fmt.Sprintf("p-sv-%d.en", i+1), msg) + } else { + assert.Contains(p.Title(), "p-en", msg) + } + } + } + + // Check bundles + bundleEn := enSite.RegularPages[len(enSite.RegularPages)-1] + bundleNn := nnSite.RegularPages[len(nnSite.RegularPages)-1] + bundleSv := svSite.RegularPages[len(svSite.RegularPages)-1] + + assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink()) + assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink()) + + assert.Equal(4, len(bundleEn.Resources)) + assert.Equal(4, len(bundleNn.Resources)) + assert.Equal(4, len(bundleSv.Resources)) + + assert.Equal("/en/sect/mybundle/logo.png", bundleEn.Resources.GetMatch("logo*").RelPermalink()) + assert.Equal("/nn/sect/mybundle/logo.png", bundleNn.Resources.GetMatch("logo*").RelPermalink()) + assert.Equal("/sv/sect/mybundle/logo.png", bundleSv.Resources.GetMatch("logo*").RelPermalink()) + + b.AssertFileContent("/my/project/public/sv/sect/mybundle/featured.png", "PNG Data for sv") + b.AssertFileContent("/my/project/public/nn/sect/mybundle/featured.png", "PNG Data for nn") + b.AssertFileContent("/my/project/public/en/sect/mybundle/featured.png", "PNG Data for en") + b.AssertFileContent("/my/project/public/en/sect/mybundle/logo.png", "PNG Data") + b.AssertFileContent("/my/project/public/sv/sect/mybundle/logo.png", "PNG Data") + b.AssertFileContent("/my/project/public/nn/sect/mybundle/logo.png", "PNG Data") + + nnSect := nnSite.getPage(KindSection, "sect") + assert.NotNil(nnSect) + assert.Equal(12, len(nnSect.Pages)) + nnHome, _ := nnSite.Info.Home() + assert.Equal("/nn/", nnHome.RelPermalink()) + +} diff --git a/hugolib/menu_test.go b/hugolib/menu_test.go index 357499526..6a8c89b95 100644 --- a/hugolib/menu_test.go +++ b/hugolib/menu_test.go @@ -74,8 +74,8 @@ Menu Main: {{ partial "menu.html" (dict "page" . "menu" "main") }}`, writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10)) writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5)) - writeNewContentFile(t, fs, "Section One", "2017-01-01", "content/sect1/_index.md", 100) - writeNewContentFile(t, fs, "Section Five", "2017-01-01", "content/sect5/_index.md", 10) + writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100) + writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10) err := h.Build(BuildCfg{}) diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index 101de7ace..a3f3828ef 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -111,6 +111,10 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L language.LanguageName = cast.ToString(v) case "weight": language.Weight = cast.ToInt(v) + case "contentdir": + language.ContentDir = cast.ToString(v) + case "disabled": + language.Disabled = cast.ToBool(v) case "params": m := cast.ToStringMap(v) // Needed for case insensitive fetching of params values diff --git a/hugolib/page.go b/hugolib/page.go index e0e002e59..2b3ca37ba 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -388,9 +388,9 @@ func (ps Pages) String() string { return fmt.Sprintf("Pages(%d)", len(ps)) } -func (ps Pages) findPagePosByFilePath(inPath string) int { +func (ps Pages) findPagePosByFilename(filename string) int { for i, x := range ps { - if x.Source.Path() == inPath { + if x.Source.Filename() == filename { return i } } @@ -412,16 +412,26 @@ func (ps Pages) removeFirstIfFound(p *Page) Pages { return ps } -func (ps Pages) findFirstPagePosByFilePathPrefix(prefix string) int { +func (ps Pages) findPagePosByFilnamePrefix(prefix string) int { if prefix == "" { return -1 } + + lenDiff := -1 + currPos := -1 + prefixLen := len(prefix) + + // Find the closest match for i, x := range ps { - if strings.HasPrefix(x.Source.Path(), prefix) { - return i + if strings.HasPrefix(x.Source.Filename(), prefix) { + diff := len(x.Source.Filename()) - prefixLen + if lenDiff == -1 || diff < lenDiff { + lenDiff = diff + currPos = i + } } } - return -1 + return currPos } // findPagePos Given a page, it will find the position in Pages diff --git a/hugolib/page_bundler.go b/hugolib/page_bundler.go index bedfd58f0..e55e0a92b 100644 --- a/hugolib/page_bundler.go +++ b/hugolib/page_bundler.go @@ -17,7 +17,6 @@ import ( "fmt" "math" "runtime" - "strings" // Use this until errgroup gets ported to context // See https://github.com/golang/go/issues/19781 @@ -26,8 +25,6 @@ import ( ) type siteContentProcessor struct { - baseDir string - site *Site handleContent contentHandler @@ -41,7 +38,7 @@ type siteContentProcessor struct { fileSinglesChan chan *fileInfo // These assets should be just copied to destination. - fileAssetsChan chan []string + fileAssetsChan chan []pathLangFile numWorkers int @@ -67,14 +64,14 @@ func (s *siteContentProcessor) processSingle(fi *fileInfo) { } } -func (s *siteContentProcessor) processAssets(assets []string) { +func (s *siteContentProcessor) processAssets(assets []pathLangFile) { select { case s.fileAssetsChan <- assets: case <-s.ctx.Done(): } } -func newSiteContentProcessor(ctx context.Context, baseDir string, partialBuild bool, s *Site) *siteContentProcessor { +func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { numWorkers := 12 if n := runtime.NumCPU() * 3; n > numWorkers { numWorkers = n @@ -85,12 +82,11 @@ func newSiteContentProcessor(ctx context.Context, baseDir string, partialBuild b return &siteContentProcessor{ ctx: ctx, partialBuild: partialBuild, - baseDir: baseDir, site: s, handleContent: newHandlerChain(s), fileBundlesChan: make(chan *bundleDir, numWorkers), fileSinglesChan: make(chan *fileInfo, numWorkers), - fileAssetsChan: make(chan []string, numWorkers), + fileAssetsChan: make(chan []pathLangFile, numWorkers), numWorkers: numWorkers, pagesChan: make(chan *Page, numWorkers), } @@ -143,18 +139,16 @@ func (s *siteContentProcessor) process(ctx context.Context) error { g2.Go(func() error { for { select { - case filenames, ok := <-s.fileAssetsChan: + case files, ok := <-s.fileAssetsChan: if !ok { return nil } - for _, filename := range filenames { - name := strings.TrimPrefix(filename, s.baseDir) - f, err := s.site.Fs.Source.Open(filename) + for _, file := range files { + f, err := s.site.BaseFs.ContentFs.Open(file.Filename()) if err != nil { - return err + return fmt.Errorf("failed to open assets file: %s", err) } - - err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, name, f) + err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) f.Close() if err != nil { return err @@ -204,11 +198,11 @@ func (s *siteContentProcessor) process(ctx context.Context) error { } func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { - ctx := &handlerContext{source: file, baseDir: s.baseDir, pages: s.pagesChan} + ctx := &handlerContext{source: file, pages: s.pagesChan} return s.handleContent(ctx).err } func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { - ctx := &handlerContext{bundle: bundle, baseDir: s.baseDir, pages: s.pagesChan} + ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} return s.handleContent(ctx).err } diff --git a/hugolib/page_bundler_capture.go b/hugolib/page_bundler_capture.go index 4d8f39fb7..255a8efda 100644 --- a/hugolib/page_bundler_capture.go +++ b/hugolib/page_bundler_capture.go @@ -17,17 +17,21 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "runtime" "strings" "sync" + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/helpers" "golang.org/x/sync/errgroup" "github.com/gohugoio/hugo/source" - "github.com/spf13/afero" jww "github.com/spf13/jwalterweatherman" ) @@ -44,8 +48,6 @@ type capturer struct { fs afero.Fs logger *jww.Notepad - baseDir string - // Filenames limits the content to process to a list of filenames/directories. // This is used for partial building in server mode. filenames []string @@ -62,7 +64,7 @@ func newCapturer( sourceSpec *source.SourceSpec, handler captureResultHandler, contentChanges *contentChangeMap, - baseDir string, filenames ...string) *capturer { + filenames ...string) *capturer { numWorkers := 4 if n := runtime.NumCPU(); n > numWorkers { @@ -73,10 +75,11 @@ func newCapturer( sem: make(chan bool, numWorkers), handler: handler, sourceSpec: sourceSpec, + fs: sourceSpec.Fs, logger: logger, contentChanges: contentChanges, - fs: sourceSpec.Fs.Source, baseDir: baseDir, seen: make(map[string]bool), - filenames: filenames} + seen: make(map[string]bool), + filenames: filenames} return c } @@ -85,7 +88,7 @@ func newCapturer( // these channels. type captureResultHandler interface { handleSingles(fis ...*fileInfo) - handleCopyFiles(filenames ...string) + handleCopyFiles(fis ...pathLangFile) captureBundlesHandler } @@ -110,10 +113,10 @@ func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) { } } -func (c *captureResultHandlerChain) handleCopyFiles(filenames ...string) { +func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) { for _, h := range c.handlers { if hh, ok := h.(captureResultHandler); ok { - hh.handleCopyFiles(filenames...) + hh.handleCopyFiles(files...) } } } @@ -139,7 +142,7 @@ func (c *capturer) capturePartial(filenames ...string) error { return err } default: - fi, _, err := c.getRealFileInfo(resolvedFilename) + fi, err := c.resolveRealPath(resolvedFilename) if os.IsNotExist(err) { // File has been deleted. continue @@ -147,9 +150,9 @@ func (c *capturer) capturePartial(filenames ...string) error { // Just in case the owning dir is a new symlink -- this will // create the proper mapping for it. - c.getRealFileInfo(dir) + c.resolveRealPath(dir) - f, active := c.newFileInfo(resolvedFilename, fi, tp) + f, active := c.newFileInfo(fi, tp) if active { c.copyOrHandleSingle(f) } @@ -164,7 +167,7 @@ func (c *capturer) capture() error { return c.capturePartial(c.filenames...) } - err := c.handleDir(c.baseDir) + err := c.handleDir(helpers.FilePathSeparator) if err != nil { return err } @@ -196,6 +199,7 @@ func (c *capturer) handleNestedDir(dirname string) error { func (c *capturer) handleBranchDir(dirname string) error { files, err := c.readDir(dirname) if err != nil { + return err } @@ -205,7 +209,7 @@ func (c *capturer) handleBranchDir(dirname string) error { for _, fi := range files { if !fi.IsDir() { - tp, _ := classifyBundledFile(fi.Name()) + tp, _ := classifyBundledFile(fi.RealName()) if dirType == bundleNot { dirType = tp } @@ -222,25 +226,35 @@ func (c *capturer) handleBranchDir(dirname string) error { dirs := newBundleDirs(bundleBranch, c) - for _, fi := range files { + var secondPass []*fileInfo + // Handle potential bundle headers first. + for _, fi := range files { if fi.IsDir() { continue } - tp, isContent := classifyBundledFile(fi.Name()) + tp, isContent := classifyBundledFile(fi.RealName()) + + f, active := c.newFileInfo(fi, tp) - f, active := c.newFileInfo(fi.filename, fi.FileInfo, tp) if !active { continue } - if f.isOwner() { - dirs.addBundleHeader(f) - } else if !isContent { - // This is a partial update -- we only care about the files that - // is in this bundle. - dirs.addBundleFiles(f) + + if !f.isOwner() { + if !isContent { + // This is a partial update -- we only care about the files that + // is in this bundle. + secondPass = append(secondPass, f) + } + continue } + dirs.addBundleHeader(f) + } + + for _, f := range secondPass { + dirs.addBundleFiles(f) } c.handler.handleBundles(dirs) @@ -250,6 +264,7 @@ func (c *capturer) handleBranchDir(dirname string) error { } func (c *capturer) handleDir(dirname string) error { + files, err := c.readDir(dirname) if err != nil { return err @@ -290,7 +305,8 @@ func (c *capturer) handleDir(dirname string) error { for i, fi := range files { if !fi.IsDir() { - tp, isContent := classifyBundledFile(fi.Name()) + tp, isContent := classifyBundledFile(fi.RealName()) + fileBundleTypes[i] = tp if !isBranch { isBranch = tp == bundleBranch @@ -317,6 +333,7 @@ func (c *capturer) handleDir(dirname string) error { var fileInfos = make([]*fileInfo, 0, len(files)) for i, fi := range files { + currentType := bundleNot if !fi.IsDir() { @@ -329,7 +346,9 @@ func (c *capturer) handleDir(dirname string) error { if bundleType == bundleNot && currentType != bundleNot { bundleType = currentType } - f, active := c.newFileInfo(fi.filename, fi.FileInfo, currentType) + + f, active := c.newFileInfo(fi, currentType) + if !active { continue } @@ -343,8 +362,7 @@ func (c *capturer) handleDir(dirname string) error { for _, fi := range fileInfos { if fi.FileInfo().IsDir() { // Handle potential nested bundles. - filename := fi.Filename() - if err := c.handleNestedDir(filename); err != nil { + if err := c.handleNestedDir(fi.Path()); err != nil { return err } } else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) { @@ -376,23 +394,23 @@ func (c *capturer) handleDir(dirname string) error { func (c *capturer) handleNonBundle( dirname string, - fileInfos []fileInfoName, + fileInfos pathLangFileFis, singlesOnly bool) error { for _, fi := range fileInfos { if fi.IsDir() { - if err := c.handleNestedDir(fi.filename); err != nil { + if err := c.handleNestedDir(fi.Filename()); err != nil { return err } } else { if singlesOnly { - f, active := c.newFileInfo(fi.filename, fi, bundleNot) + f, active := c.newFileInfo(fi, bundleNot) if !active { continue } c.handler.handleSingles(f) } else { - c.handler.handleCopyFiles(fi.filename) + c.handler.handleCopyFiles(fi) } } } @@ -405,7 +423,7 @@ func (c *capturer) copyOrHandleSingle(fi *fileInfo) { c.handler.handleSingles(fi) } else { // These do not currently need any further processing. - c.handler.handleCopyFiles(fi.Filename()) + c.handler.handleCopyFiles(fi) } } @@ -430,7 +448,7 @@ func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirT fileInfos = append(fileInfos, fis...) } } - err := c.collectFiles(fi.Filename(), collector) + err := c.collectFiles(fi.Path(), collector) if err != nil { return nil, err } @@ -462,6 +480,7 @@ func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirT } func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error { + filesInDir, err := c.readDir(dirname) if err != nil { return err @@ -469,12 +488,12 @@ func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInf for _, fi := range filesInDir { if fi.IsDir() { - err := c.collectFiles(fi.filename, handleFiles) + err := c.collectFiles(fi.Filename(), handleFiles) if err != nil { return err } } else { - f, active := c.newFileInfo(fi.filename, fi.FileInfo, bundleNot) + f, active := c.newFileInfo(fi, bundleNot) if active { handleFiles(f) } @@ -484,27 +503,29 @@ func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInf return nil } -func (c *capturer) readDir(dirname string) ([]fileInfoName, error) { +func (c *capturer) readDir(dirname string) (pathLangFileFis, error) { if c.sourceSpec.IgnoreFile(dirname) { return nil, nil } dir, err := c.fs.Open(dirname) if err != nil { - return nil, err + return nil, fmt.Errorf("readDir: %s", err) } defer dir.Close() - names, err := dir.Readdirnames(-1) + fis, err := dir.Readdir(-1) if err != nil { return nil, err } - fis := make([]fileInfoName, 0, len(names)) + pfis := make(pathLangFileFis, 0, len(fis)) - for _, name := range names { - filename := filepath.Join(dirname, name) - if !c.sourceSpec.IgnoreFile(filename) { - fi, _, err := c.getRealFileInfo(filename) + for _, fi := range fis { + fip := fi.(pathLangFileFi) + + if !c.sourceSpec.IgnoreFile(fip.Filename()) { + + err := c.resolveRealPathIn(fip) if err != nil { // It may have been deleted in the meantime. @@ -514,23 +535,30 @@ func (c *capturer) readDir(dirname string) ([]fileInfoName, error) { return nil, err } - fis = append(fis, fileInfoName{filename: filename, FileInfo: fi}) + pfis = append(pfis, fip) } } - return fis, nil + return pfis, nil } -func (c *capturer) newFileInfo(filename string, fi os.FileInfo, tp bundleDirType) (*fileInfo, bool) { - f := newFileInfo(c.sourceSpec, c.baseDir, filename, fi, tp) +func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) { + f := newFileInfo(c.sourceSpec, "", "", fi, tp) return f, !f.disabled } -type fileInfoName struct { - os.FileInfo - filename string +type pathLangFile interface { + hugofs.LanguageAnnouncer + hugofs.FilePather } +type pathLangFileFi interface { + os.FileInfo + pathLangFile +} + +type pathLangFileFis []pathLangFileFi + type bundleDirs struct { tp bundleDirType // Maps languages to bundles. @@ -589,16 +617,17 @@ func (b *bundleDirs) addBundleContentFile(fi *fileInfo) { b.bundles[fi.Lang()] = dir } - dir.resources[fi.Filename()] = fi + dir.resources[fi.Path()] = fi } func (b *bundleDirs) addBundleFiles(fi *fileInfo) { dir := filepath.ToSlash(fi.Dir()) p := dir + fi.TranslationBaseName() + "." + fi.Ext() for lang, bdir := range b.bundles { - key := lang + p + key := path.Join(lang, p) + // Given mypage.de.md (German translation) and mypage.md we pick the most - // the specific for that language. + // specific for that language. if fi.Lang() == lang || !b.langOverrides[key] { bdir.resources[key] = fi } @@ -623,40 +652,53 @@ func (c *capturer) isSeen(dirname string) bool { return false } -func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) { - fileInfo, err := c.lstatIfOs(path) - realPath := path - +func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) { + fileInfo, err := c.lstatIfPossible(path) if err != nil { - return nil, "", err + return nil, err } + return fileInfo, c.resolveRealPathIn(fileInfo) +} + +func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error { + + basePath := fileInfo.BaseDir() + path := fileInfo.Filename() + + realPath := path if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(path) if err != nil { - return nil, "", fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err) + return fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err) } - fileInfo, err = c.lstatIfOs(link) + // This is a file on the outside of any base fs, so we have to use the os package. + sfi, err := os.Stat(link) if err != nil { - return nil, "", fmt.Errorf("Cannot stat %q, error was: %s", link, err) + return fmt.Errorf("Cannot stat %q, error was: %s", link, err) + } + + // TODO(bep) improve all of this. + if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok { + a.FileInfo = sfi } realPath = link - if realPath != path && fileInfo.IsDir() && c.isSeen(realPath) { + if realPath != path && sfi.IsDir() && c.isSeen(realPath) { // Avoid cyclic symlinks. // Note that this may prevent some uses that isn't cyclic and also // potential useful, but this implementation is both robust and simple: // We stop at the first directory that we have seen before, e.g. // /content/blog will only be processed once. - return nil, realPath, errSkipCyclicDir + return errSkipCyclicDir } if c.contentChanges != nil { // Keep track of symbolic links in watch mode. var from, to string - if fileInfo.IsDir() { + if sfi.IsDir() { from = realPath to = path @@ -667,12 +709,11 @@ func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) { from = from + helpers.FilePathSeparator } - baseDir := c.baseDir - if !strings.HasSuffix(baseDir, helpers.FilePathSeparator) { - baseDir = baseDir + helpers.FilePathSeparator + if !strings.HasSuffix(basePath, helpers.FilePathSeparator) { + basePath = basePath + helpers.FilePathSeparator } - if strings.HasPrefix(from, baseDir) { + if strings.HasPrefix(from, basePath) { // With symbolic links inside /content we need to keep // a reference to both. This may be confusing with --navigateToChanged // but the user has chosen this him or herself. @@ -688,9 +729,13 @@ func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) { } } - return fileInfo, realPath, nil + return nil } -func (c *capturer) lstatIfOs(path string) (os.FileInfo, error) { - return helpers.LstatIfOs(c.fs, path) +func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) { + fi, err := helpers.LstatIfPossible(c.fs, path) + if err != nil { + return nil, err + } + return fi.(pathLangFileFi), nil } diff --git a/hugolib/page_bundler_capture_test.go b/hugolib/page_bundler_capture_test.go index 25d9be5a6..6ef396d29 100644 --- a/hugolib/page_bundler_capture_test.go +++ b/hugolib/page_bundler_capture_test.go @@ -15,6 +15,7 @@ package hugolib import ( "fmt" + "os" "path" "path/filepath" "sort" @@ -62,13 +63,12 @@ func (s *storeFilenames) handleBundles(d *bundleDirs) { s.dirKeys = append(s.dirKeys, keys...) } -func (s *storeFilenames) handleCopyFiles(names ...string) { +func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) { s.Lock() defer s.Unlock() - for _, name := range names { - s.copyNames = append(s.copyNames, filepath.ToSlash(name)) + for _, file := range files { + s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename())) } - } func (s *storeFilenames) sortedStr() string { @@ -83,13 +83,12 @@ func (s *storeFilenames) sortedStr() string { func TestPageBundlerCaptureSymlinks(t *testing.T) { assert := require.New(t) - cfg, fs, workDir := newTestBundleSymbolicSources(t) - contentDir := "base" - sourceSpec := source.NewSourceSpec(cfg, fs) + ps, workDir := newTestBundleSymbolicSources(t) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} logger := newErrorLogger() - c := newCapturer(logger, sourceSpec, fileStore, nil, filepath.Join(workDir, contentDir)) + c := newCapturer(logger, sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -110,6 +109,7 @@ C: /base/symbolic3/s1.png /base/symbolic3/s2.png ` + got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1) got = strings.Replace(got, "//", "/", -1) @@ -120,20 +120,26 @@ C: } } -func TestPageBundlerCapture(t *testing.T) { +func TestPageBundlerCaptureBasic(t *testing.T) { t.Parallel() assert := require.New(t) - cfg, fs := newTestBundleSources(t) + fs, cfg := newTestBundleSources(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + ps, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) - sourceSpec := source.NewSourceSpec(cfg, fs) + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base")) + c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) + printFs(fs.Source, "", os.Stdout) + expected := ` F: /work/base/_1.md @@ -165,10 +171,16 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) { t.Parallel() assert := require.New(t) - cfg, fs := newTestBundleSourcesMultilingual(t) - sourceSpec := source.NewSourceSpec(cfg, fs) + fs, cfg := newTestBundleSourcesMultilingual(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + + ps, err := helpers.NewPathSpec(fs, cfg) + assert.NoError(err) + + sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs) fileStore := &storeFilenames{} - c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base")) + c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil) assert.NoError(c.capture()) @@ -204,23 +216,24 @@ C: if expected != got { diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got)) t.Log(got) - t.Fatalf("Failed:\n%s", diff) + t.Fatalf("Failed:\n%s", strings.Join(diff, "\n")) } } type noOpFileStore int -func (noOpFileStore) handleSingles(fis ...*fileInfo) {} -func (noOpFileStore) handleBundles(b *bundleDirs) {} -func (noOpFileStore) handleCopyFiles(names ...string) {} +func (noOpFileStore) handleSingles(fis ...*fileInfo) {} +func (noOpFileStore) handleBundles(b *bundleDirs) {} +func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {} func BenchmarkPageBundlerCapture(b *testing.B) { capturers := make([]*capturer, b.N) for i := 0; i < b.N; i++ { cfg, fs := newTestCfg() - sourceSpec := source.NewSourceSpec(cfg, fs) + ps, _ := helpers.NewPathSpec(fs, cfg) + sourceSpec := source.NewSourceSpec(ps, fs.Source) base := fmt.Sprintf("base%d", i) for j := 1; j <= 5; j++ { diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go index 19e48ea7d..477f336fc 100644 --- a/hugolib/page_bundler_handlers.go +++ b/hugolib/page_bundler_handlers.go @@ -101,9 +101,6 @@ type handlerContext struct { bundle *bundleDir - // The source baseDir, e.g. "/myproject/content/" - baseDir string - source *fileInfo // Relative path to the target. @@ -130,7 +127,7 @@ func (c *handlerContext) targetPath() string { return c.target } - return strings.TrimPrefix(c.source.Filename(), c.baseDir) + return c.source.Filename() } func (c *handlerContext) file() *fileInfo { @@ -326,7 +323,6 @@ func (c *contentHandlers) createResource() contentHandler { resource, err := c.s.resourceSpec.NewResourceFromFilename( ctx.parentPage.subResourceTargetPathFactory, - c.s.absPublishDir(), ctx.source.Filename(), ctx.target) return handlerResult{err: err, handled: true, resource: resource} @@ -335,8 +331,9 @@ func (c *contentHandlers) createResource() contentHandler { func (c *contentHandlers) copyFile() contentHandler { return func(ctx *handlerContext) handlerResult { - f, err := c.s.Fs.Source.Open(ctx.source.Filename()) + f, err := c.s.BaseFs.ContentFs.Open(ctx.source.Filename()) if err != nil { + err := fmt.Errorf("failed to open file in copyFile: %s", err) return handlerResult{err: err} } diff --git a/hugolib/page_bundler_test.go b/hugolib/page_bundler_test.go index 572d84bcd..52b5b830d 100644 --- a/hugolib/page_bundler_test.go +++ b/hugolib/page_bundler_test.go @@ -20,6 +20,8 @@ import ( "strings" "testing" + "github.com/gohugoio/hugo/helpers" + "io" "github.com/spf13/afero" @@ -38,7 +40,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestPageBundlerSite(t *testing.T) { +func TestPageBundlerSiteRegular(t *testing.T) { t.Parallel() for _, ugly := range []bool{false, true} { @@ -46,7 +48,9 @@ func TestPageBundlerSite(t *testing.T) { func(t *testing.T) { assert := require.New(t) - cfg, fs := newTestBundleSources(t) + fs, cfg := newTestBundleSources(t) + assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) cfg.Set("permalinks", map[string]string{ "a": ":sections/:filename", @@ -141,6 +145,8 @@ func TestPageBundlerSite(t *testing.T) { assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename()) assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink()) + + printFs(th.Fs.Destination, "", os.Stdout) th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content") th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content") @@ -195,8 +201,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) { func(t *testing.T) { assert := require.New(t) - cfg, fs := newTestBundleSourcesMultilingual(t) - + fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("uglyURLs", ugly) assert.NoError(loadDefaultSettingsFor(cfg)) @@ -260,7 +265,7 @@ func TestMultilingualDisableDefaultLanguage(t *testing.T) { t.Parallel() assert := require.New(t) - cfg, _ := newTestBundleSourcesMultilingual(t) + _, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("disableLanguages", []string{"en"}) @@ -275,10 +280,12 @@ func TestMultilingualDisableLanguage(t *testing.T) { t.Parallel() assert := require.New(t) - cfg, fs := newTestBundleSourcesMultilingual(t) + fs, cfg := newTestBundleSourcesMultilingual(t) cfg.Set("disableLanguages", []string{"nn"}) assert.NoError(loadDefaultSettingsFor(cfg)) + assert.NoError(loadLanguageSettings(cfg, nil)) + sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg}) assert.NoError(err) assert.Equal(1, len(sites.Sites)) @@ -302,7 +309,9 @@ func TestMultilingualDisableLanguage(t *testing.T) { func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) { assert := require.New(t) - cfg, fs, workDir := newTestBundleSymbolicSources(t) + ps, workDir := newTestBundleSymbolicSources(t) + cfg := ps.Cfg + fs := ps.Fs s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{}) @@ -401,7 +410,7 @@ HEADLESS {{< myShort >}} } -func newTestBundleSources(t *testing.T) (*viper.Viper, *hugofs.Fs) { +func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) { cfg, fs := newTestCfg() assert := require.New(t) @@ -543,10 +552,11 @@ Content for 은행. src.Close() assert.NoError(err) - return cfg, fs + return fs, cfg + } -func newTestBundleSourcesMultilingual(t *testing.T) (*viper.Viper, *hugofs.Fs) { +func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) { cfg, fs := newTestCfg() workDir := "/work" @@ -626,10 +636,10 @@ TheContent. writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.nn.md"), pageContent) writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "page.md"), pageContent) - return cfg, fs + return fs, cfg } -func newTestBundleSymbolicSources(t *testing.T) (*viper.Viper, *hugofs.Fs, string) { +func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, string) { assert := require.New(t) // We need to use the OS fs for this. cfg := viper.New() @@ -650,6 +660,10 @@ func newTestBundleSymbolicSources(t *testing.T) (*viper.Viper, *hugofs.Fs, strin cfg.Set("contentDir", contentDir) cfg.Set("baseURL", "https://example.com") + if err := loadLanguageSettings(cfg, nil); err != nil { + t.Fatal(err) + } + layout := `{{ .Title }}|{{ .Content }}` pageContent := `--- slug: %s @@ -709,5 +723,7 @@ TheContent. os.Chdir(workDir) assert.NoError(err) - return cfg, fs, workDir + ps, _ := helpers.NewPathSpec(fs, cfg) + + return ps, workDir } diff --git a/hugolib/page_collections.go b/hugolib/page_collections.go index 157d4e68d..74f7d608c 100644 --- a/hugolib/page_collections.go +++ b/hugolib/page_collections.go @@ -179,8 +179,8 @@ func (c *PageCollections) addPage(page *Page) { c.rawAllPages = append(c.rawAllPages, page) } -func (c *PageCollections) removePageByPath(path string) { - if i := c.rawAllPages.findPagePosByFilePath(path); i >= 0 { +func (c *PageCollections) removePageFilename(filename string) { + if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 { c.clearResourceCacheForPage(c.rawAllPages[i]) c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...) } @@ -218,6 +218,7 @@ func (c *PageCollections) clearResourceCacheForPage(page *Page) { if len(page.Resources) > 0 { first := page.Resources[0] dir := path.Dir(first.RelPermalink()) + dir = strings.TrimPrefix(dir, page.LanguagePrefix()) // This is done to keep the memory usage in check when doing live reloads. page.s.resourceSpec.DeleteCacheByPrefix(dir) } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index d862dc8e9..2b679c842 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -1109,60 +1109,6 @@ func TestCreatePage(t *testing.T) { } } -func TestPageKind(t *testing.T) { - t.Parallel() - const sep = helpers.FilePathSeparator - var tests = []struct { - file string - kind string - }{ - {"_index.md", KindHome}, - {"about.md", KindPage}, - {"sectionA" + sep + "_index.md", KindSection}, - {"sectionA" + sep + "about.md", KindPage}, - {"categories" + sep + "_index.md", KindTaxonomyTerm}, - {"categories" + sep + "categoryA" + sep + "_index.md", KindTaxonomy}, - {"tags" + sep + "_index.md", KindTaxonomyTerm}, - {"tags" + sep + "tagA" + sep + "_index.md", KindTaxonomy}, - - // nn is configured as a language - {"_index.nn.md", KindHome}, - {"about.nn.md", KindPage}, - {"sectionA" + sep + "_index.nn.md", KindSection}, - {"sectionA" + sep + "about.nn.md", KindPage}, - - // should NOT be categorized as KindHome - {"_indexNOT.md", KindPage}, - - // To be consistent with FileInfo.TranslationBaseName(), - // language codes not explicitly configured for the site - // are not treated as such. "fr" is not configured as - // a language in the test site, so ALL of the - // following should be KindPage - {"_index.fr.md", KindPage}, //not KindHome - {"about.fr.md", KindPage}, - {"sectionA" + sep + "_index.fr.md", KindPage}, // KindSection - {"sectionA" + sep + "about.fr.md", KindPage}, - } - - for _, test := range tests { - s := newTestSite(t, "languages.nn.languageName", "Nynorsk") - taxonomies := make(map[string]string) - taxonomies["tag"] = "tags" - taxonomies["category"] = "categories" - s.Taxonomies = make(TaxonomyList) - for _, plural := range taxonomies { - s.Taxonomies[plural] = make(Taxonomy) - } - - p, _ := s.NewPage(test.file) - p.setValuesForKind(s) - if p.Kind != test.kind { - t.Errorf("for %s expected p.Kind == %s, got %s", test.file, test.kind, p.Kind) - } - } -} - func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) { t.Parallel() var tests = []struct { diff --git a/hugolib/prune_resources.go b/hugolib/prune_resources.go index 8eddafb53..e9d2bf96e 100644 --- a/hugolib/prune_resources.go +++ b/hugolib/prune_resources.go @@ -25,7 +25,9 @@ import ( // GC requires a build first. func (h *HugoSites) GC() (int, error) { s := h.Sites[0] - imageCacheDir := s.resourceSpec.AbsGenImagePath + fs := h.PathSpec.BaseFs.ResourcesFs + + imageCacheDir := s.resourceSpec.GenImagePath if len(imageCacheDir) < 10 { panic("invalid image cache") } @@ -43,7 +45,7 @@ func (h *HugoSites) GC() (int, error) { counter := 0 - err := afero.Walk(s.Fs.Source, imageCacheDir, func(path string, info os.FileInfo, err error) error { + err := afero.Walk(fs, imageCacheDir, func(path string, info os.FileInfo, err error) error { if info == nil { return nil } @@ -53,7 +55,7 @@ func (h *HugoSites) GC() (int, error) { } if info.IsDir() { - f, err := s.Fs.Source.Open(path) + f, err := fs.Open(path) if err != nil { return nil } @@ -69,7 +71,7 @@ func (h *HugoSites) GC() (int, error) { inUse := isInUse(path) if !inUse { - err := s.Fs.Source.Remove(path) + err := fs.Remove(path) if err != nil && !os.IsNotExist(err) { s.Log.ERROR.Printf("Failed to remove %q: %s", path, err) } else { diff --git a/hugolib/site.go b/hugolib/site.go index 7d67cf218..4027ce075 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -746,9 +746,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { } } if removed && isContentFile(ev.Name) { - path, _ := helpers.GetRelativePath(ev.Name, s.getContentDir(ev.Name)) - - h.removePageByPath(path) + h.removePageByFilename(ev.Name) } sourceReallyChanged = append(sourceReallyChanged, ev) @@ -890,7 +888,7 @@ func (s *Site) handleDataFile(r source.ReadableFile) error { func (s *Site) readData(f source.ReadableFile) (interface{}, error) { file, err := f.Open() if err != nil { - return nil, err + return nil, fmt.Errorf("readData: failed to open data file: %s", err) } defer file.Close() content := helpers.ReaderToBytes(file) @@ -1295,9 +1293,9 @@ func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { } } -func (c *contentCaptureResultHandler) handleCopyFiles(filenames ...string) { +func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) { for _, proc := range c.contentProcessors { - proc.processAssets(filenames) + proc.processAssets(files) } } @@ -1305,15 +1303,16 @@ func (s *Site) readAndProcessContent(filenames ...string) error { ctx := context.Background() g, ctx := errgroup.WithContext(ctx) - sourceSpec := source.NewSourceSpec(s.owner.Cfg, s.Fs) - baseDir := s.absContentDir() defaultContentLanguage := s.SourceSpec.DefaultContentLanguage contentProcessors := make(map[string]*siteContentProcessor) var defaultContentProcessor *siteContentProcessor sites := s.owner.langSite() for k, v := range sites { - proc := newSiteContentProcessor(ctx, baseDir, len(filenames) > 0, v) + if v.Language.Disabled { + continue + } + proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) contentProcessors[k] = proc if k == defaultContentLanguage { defaultContentProcessor = proc @@ -1330,6 +1329,8 @@ func (s *Site) readAndProcessContent(filenames ...string) error { mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} + sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs) + if s.running() { // Need to track changes. bundleMap = s.owner.ContentChanges @@ -1339,7 +1340,7 @@ func (s *Site) readAndProcessContent(filenames ...string) error { handler = mainHandler } - c := newCapturer(s.Log, sourceSpec, handler, bundleMap, baseDir, filenames...) + c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...) err1 := c.capture() diff --git a/hugolib/site_url_test.go b/hugolib/site_url_test.go index 671695b4d..2be615963 100644 --- a/hugolib/site_url_test.go +++ b/hugolib/site_url_test.go @@ -119,10 +119,12 @@ Do not go gentle into that good night. notUgly := s.getPage(KindPage, "sect1/p1.md") assert.NotNil(notUgly) + assert.Equal("sect1", notUgly.Section()) assert.Equal("/sect1/p1/", notUgly.RelPermalink()) ugly := s.getPage(KindPage, "sect2/p2.md") assert.NotNil(ugly) + assert.Equal("sect2", ugly.Section()) assert.Equal("/sect2/p2.html", ugly.RelPermalink()) } diff --git a/hugolib/taxonomy_test.go b/hugolib/taxonomy_test.go index 7ec4bbf91..0445de58f 100644 --- a/hugolib/taxonomy_test.go +++ b/hugolib/taxonomy_test.go @@ -115,8 +115,8 @@ permalinkeds: writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- pl1")) writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- pl1")) - writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) - writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10) + writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10) + writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10) err := h.Build(BuildCfg{}) diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index 6f513b3bf..5300eee22 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -51,6 +51,11 @@ type sitesBuilder struct { // Default toml configFormat string + // Default is empty. + // TODO(bep) revisit this and consider always setting it to something. + // Consider this in relation to using the BaseFs.PublishFs to all publishing. + workingDir string + // Base data/content contentFilePairs []string templateFilePairs []string @@ -83,6 +88,11 @@ func (s *sitesBuilder) Running() *sitesBuilder { return s } +func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder { + s.workingDir = dir + return s +} + func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder { if format == "" { format = "toml" @@ -233,7 +243,17 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) * } for i := 0; i < len(filenameContent); i += 2 { filename, content := filenameContent[i], filenameContent[i+1] - writeSource(s.T, s.Fs, filepath.Join(folder, filename), content) + target := folder + // TODO(bep) clean up this magic. + if strings.HasPrefix(filename, folder) { + target = "" + } + + if s.workingDir != "" { + target = filepath.Join(s.workingDir, target) + } + + writeSource(s.T, s.Fs, filepath.Join(target, filename), content) } return s } @@ -458,6 +478,7 @@ func newTestDefaultPathSpec() *helpers.PathSpec { v := viper.New() // Easier to reason about in tests. v.Set("disablePathToLower", true) + v.Set("contentDir", "content") fs := hugofs.NewDefault(v) ps, _ := helpers.NewPathSpec(fs, v) return ps diff --git a/i18n/i18n_test.go b/i18n/i18n_test.go index 057fc35e9..4f5b3fbac 100644 --- a/i18n/i18n_test.go +++ b/i18n/i18n_test.go @@ -200,6 +200,7 @@ func TestI18nTranslate(t *testing.T) { var actual, expected string v := viper.New() v.SetDefault("defaultContentLanguage", "en") + v.Set("contentDir", "content") // Test without and with placeholders for _, enablePlaceholders := range []bool{false, true} { diff --git a/i18n/translationProvider.go b/i18n/translationProvider.go index 52aada8bd..fa5664210 100644 --- a/i18n/translationProvider.go +++ b/i18n/translationProvider.go @@ -39,7 +39,7 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir")) - sp := source.NewSourceSpec(d.Cfg, d.Fs) + sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source) sources := []source.Input{sp.NewFilesystem(dir)} themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath() diff --git a/resource/image.go b/resource/image.go index a1e0aac70..84dc56fa7 100644 --- a/resource/image.go +++ b/resource/image.go @@ -419,7 +419,7 @@ func (i *Image) initConfig() error { config image.Config ) - f, err = i.spec.Fs.Source.Open(i.AbsSourceFilename()) + f, err = i.sourceFs().Open(i.AbsSourceFilename()) if err != nil { return } @@ -432,13 +432,17 @@ func (i *Image) initConfig() error { i.config = config }) - return err + if err != nil { + return fmt.Errorf("failed to load image config: %s", err) + } + + return nil } func (i *Image) decodeSource() (image.Image, error) { - file, err := i.spec.Fs.Source.Open(i.AbsSourceFilename()) + file, err := i.sourceFs().Open(i.AbsSourceFilename()) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open image for decode: %s", err) } defer file.Close() img, _, err := image.Decode(file) @@ -448,32 +452,32 @@ func (i *Image) decodeSource() (image.Image, error) { func (i *Image) copyToDestination(src string) error { var res error i.copyToDestinationInit.Do(func() { - target := filepath.Join(i.absPublishDir, i.target()) + target := i.target() // Fast path: // This is a processed version of the original. // If it exists on destination with the same filename and file size, it is // the same file, so no need to transfer it again. - if fi, err := i.spec.Fs.Destination.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() { + if fi, err := i.spec.BaseFs.PublishFs.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() { return } - in, err := i.spec.Fs.Source.Open(src) + in, err := i.sourceFs().Open(src) if err != nil { res = err return } defer in.Close() - out, err := i.spec.Fs.Destination.Create(target) + out, err := i.spec.BaseFs.PublishFs.Create(target) if err != nil && os.IsNotExist(err) { // When called from shortcodes, the target directory may not exist yet. // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { + if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { res = err return } - out, err = i.spec.Fs.Destination.Create(target) + out, err = i.spec.BaseFs.PublishFs.Create(target) if err != nil { res = err return @@ -491,20 +495,23 @@ func (i *Image) copyToDestination(src string) error { } }) - return res + if res != nil { + return fmt.Errorf("failed to copy image to destination: %s", res) + } + return nil } func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error { - target := filepath.Join(i.absPublishDir, filename) + target := filepath.Clean(filename) - file1, err := i.spec.Fs.Destination.Create(target) + file1, err := i.spec.BaseFs.PublishFs.Create(target) if err != nil && os.IsNotExist(err) { // When called from shortcodes, the target directory may not exist yet. // See https://github.com/gohugoio/hugo/issues/4202 - if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { + if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil { return err } - file1, err = i.spec.Fs.Destination.Create(target) + file1, err = i.spec.BaseFs.PublishFs.Create(target) if err != nil { return err } @@ -518,11 +525,11 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource if resourceCacheFilename != "" { // Also save it to the image resource cache for later reuse. - if err = i.spec.Fs.Source.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil { + if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil { return err } - file2, err := i.spec.Fs.Source.Create(resourceCacheFilename) + file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename) if err != nil { return err } diff --git a/resource/image_cache.go b/resource/image_cache.go index e63989f24..5985797d6 100644 --- a/resource/image_cache.go +++ b/resource/image_cache.go @@ -24,10 +24,9 @@ import ( ) type imageCache struct { - absPublishDir string - absCacheDir string - pathSpec *helpers.PathSpec - mu sync.RWMutex + cacheDir string + pathSpec *helpers.PathSpec + mu sync.RWMutex store map[string]*Image } @@ -82,14 +81,14 @@ func (c *imageCache) getOrCreate( parent.createMu.Lock() defer parent.createMu.Unlock() - cacheFilename := filepath.Join(c.absCacheDir, key) + cacheFilename := filepath.Join(c.cacheDir, key) // The definition of this counter is not that we have processed that amount // (e.g. resized etc.), it can be fetched from file cache, // but the count of processed image variations for this site. c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages) - exists, err := helpers.Exists(cacheFilename, c.pathSpec.Fs.Source) + exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs) if err != nil { return nil, err } @@ -97,7 +96,9 @@ func (c *imageCache) getOrCreate( if exists { img = parent.clone() img.relTargetPath.file = relTarget.file - img.absSourceFilename = cacheFilename + img.sourceFilename = cacheFilename + // We have to look resources file system for this. + img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs } else { img, err = create(cacheFilename) if err != nil { @@ -124,8 +125,8 @@ func (c *imageCache) getOrCreate( } -func newImageCache(ps *helpers.PathSpec, absCacheDir, absPublishDir string) *imageCache { - return &imageCache{pathSpec: ps, store: make(map[string]*Image), absCacheDir: absCacheDir, absPublishDir: absPublishDir} +func newImageCache(ps *helpers.PathSpec, cacheDir string) *imageCache { + return &imageCache{pathSpec: ps, store: make(map[string]*Image), cacheDir: cacheDir} } func timeTrack(start time.Time, name string) { diff --git a/resource/image_test.go b/resource/image_test.go index 0111d0850..39e538d33 100644 --- a/resource/image_test.go +++ b/resource/image_test.go @@ -16,6 +16,7 @@ package resource import ( "fmt" "math/rand" + "os" "path/filepath" "strconv" "testing" @@ -57,12 +58,14 @@ func TestParseImageConfig(t *testing.T) { } } -func TestImageTransform(t *testing.T) { +func TestImageTransformBasic(t *testing.T) { assert := require.New(t) image := fetchSunset(assert) + printFs(image.sourceFs(), "", os.Stdout) + assert.Equal("/a/sunset.jpg", image.RelPermalink()) assert.Equal("image", image.ResourceType()) @@ -75,19 +78,19 @@ func TestImageTransform(t *testing.T) { assert.NoError(err) assert.Equal(320, resized0x.Width()) assert.Equal(200, resized0x.Height()) - assertFileCache(assert, image.spec.Fs, resized0x.RelPermalink(), 320, 200) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200) resizedx0, err := image.Resize("200x") assert.NoError(err) assert.Equal(200, resizedx0.Width()) assert.Equal(125, resizedx0.Height()) - assertFileCache(assert, image.spec.Fs, resizedx0.RelPermalink(), 200, 125) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125) resizedAndRotated, err := image.Resize("x200 r90") assert.NoError(err) assert.Equal(125, resizedAndRotated.Width()) assert.Equal(200, resizedAndRotated.Height()) - assertFileCache(assert, image.spec.Fs, resizedAndRotated.RelPermalink(), 125, 200) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200) assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink()) assert.Equal(300, resized.Width()) @@ -112,20 +115,20 @@ func TestImageTransform(t *testing.T) { assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink()) assert.Equal(200, filled.Width()) assert.Equal(100, filled.Height()) - assertFileCache(assert, image.spec.Fs, filled.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100) smart, err := image.Fill("200x100 smart") assert.NoError(err) assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink()) assert.Equal(200, smart.Width()) assert.Equal(100, smart.Height()) - assertFileCache(assert, image.spec.Fs, smart.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100) // Check cache filledAgain, err := image.Fill("200x100 bottomLeft") assert.NoError(err) assert.True(filled == filledAgain) - assertFileCache(assert, image.spec.Fs, filledAgain.RelPermalink(), 200, 100) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100) } @@ -295,10 +298,10 @@ func TestImageResizeInSubPath(t *testing.T) { assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink()) assert.Equal(101, resized.Width()) - assertFileCache(assert, image.spec.Fs, resized.RelPermalink(), 101, 101) - publishedImageFilename := filepath.Join("/public", resized.RelPermalink()) - assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101) - assert.NoError(image.spec.Fs.Destination.Remove(publishedImageFilename)) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101) + publishedImageFilename := filepath.Clean(resized.RelPermalink()) + assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) + assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename)) // Cleare mem cache to simulate reading from the file cache. resized.spec.imageCache.clear() @@ -307,8 +310,8 @@ func TestImageResizeInSubPath(t *testing.T) { assert.NoError(err) assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink()) assert.Equal(101, resizedAgain.Width()) - assertFileCache(assert, image.spec.Fs, resizedAgain.RelPermalink(), 101, 101) - assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101) + assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101) + assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101) } diff --git a/resource/resource.go b/resource/resource.go index 2732f8b37..7fe3b4ff9 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/spf13/afero" + "github.com/spf13/cast" "github.com/gobwas/glob" @@ -214,6 +216,7 @@ func getGlob(pattern string) (glob.Glob, error) { type Spec struct { *helpers.PathSpec + mimeTypes media.Types // Holds default filter settings etc. @@ -221,7 +224,7 @@ type Spec struct { imageCache *imageCache - AbsGenImagePath string + GenImagePath string } func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { @@ -232,41 +235,44 @@ func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) { } s.GetLayoutDirPath() - genImagePath := s.AbsPathify(filepath.Join(s.Cfg.GetString("resourceDir"), "_gen", "images")) + genImagePath := filepath.FromSlash("_gen/images") - return &Spec{AbsGenImagePath: genImagePath, PathSpec: s, imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( - s, - // We're going to write a cache pruning routine later, so make it extremely - // unlikely that the user shoots him or herself in the foot - // and this is set to a value that represents data he/she - // cares about. This should be set in stone once released. - genImagePath, - s.AbsPathify(s.Cfg.GetString("publishDir")))}, nil + return &Spec{PathSpec: s, + GenImagePath: genImagePath, + imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache( + s, + // We're going to write a cache pruning routine later, so make it extremely + // unlikely that the user shoots him or herself in the foot + // and this is set to a value that represents data he/she + // cares about. This should be set in stone once released. + genImagePath, + )}, nil } func (r *Spec) NewResourceFromFile( targetPathBuilder func(base string) string, - absPublishDir string, file source.File, relTargetFilename string) (Resource, error) { - return r.newResource(targetPathBuilder, absPublishDir, file.Filename(), file.FileInfo(), relTargetFilename) + return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename) } func (r *Spec) NewResourceFromFilename( targetPathBuilder func(base string) string, - absPublishDir, absSourceFilename, relTargetFilename string) (Resource, error) { - fi, err := r.Fs.Source.Stat(absSourceFilename) + fi, err := r.sourceFs().Stat(absSourceFilename) if err != nil { return nil, err } - return r.newResource(targetPathBuilder, absPublishDir, absSourceFilename, fi, relTargetFilename) + return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename) +} + +func (r *Spec) sourceFs() afero.Fs { + return r.PathSpec.BaseFs.ContentFs } func (r *Spec) newResource( targetPathBuilder func(base string) string, - absPublishDir, absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) { var mimeType string @@ -283,7 +289,7 @@ func (r *Spec) newResource( } } - gr := r.newGenericResource(targetPathBuilder, fi, absPublishDir, absSourceFilename, relTargetFilename, mimeType) + gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType) if mimeType == "image" { ext := strings.ToLower(helpers.Ext(absSourceFilename)) @@ -295,9 +301,9 @@ func (r *Spec) newResource( return gr, nil } - f, err := r.Fs.Source.Open(absSourceFilename) + f, err := gr.sourceFs().Open(absSourceFilename) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to open image source file: %s", err) } defer f.Close() @@ -369,15 +375,30 @@ type genericResource struct { params map[string]interface{} // Absolute filename to the source, including any content folder path. - absSourceFilename string - absPublishDir string - resourceType string - osFileInfo os.FileInfo + // Note that this is absolute in relation to the filesystem it is stored in. + // It can be a base path filesystem, and then this filename will not match + // the path to the file on the real filesystem. + sourceFilename string + + // This may be set to tell us to look in another filesystem for this resource. + // We, by default, use the sourceFs filesystem in the spec below. + overriddenSourceFs afero.Fs + + spec *Spec + + resourceType string + osFileInfo os.FileInfo - spec *Spec targetPathBuilder func(rel string) string } +func (l *genericResource) sourceFs() afero.Fs { + if l.overriddenSourceFs != nil { + return l.overriddenSourceFs + } + return l.spec.sourceFs() +} + func (l *genericResource) Permalink() string { return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String()) } @@ -455,19 +476,16 @@ func (l *genericResource) ResourceType() string { } func (l *genericResource) AbsSourceFilename() string { - return l.absSourceFilename + return l.sourceFilename } func (l *genericResource) Publish() error { - f, err := l.spec.Fs.Source.Open(l.AbsSourceFilename()) + f, err := l.sourceFs().Open(l.AbsSourceFilename()) if err != nil { return err } defer f.Close() - - target := filepath.Join(l.absPublishDir, l.target()) - - return helpers.WriteToDisk(target, f, l.spec.Fs.Destination) + return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs) } const counterPlaceHolder = ":counter" @@ -574,8 +592,7 @@ func (l *genericResource) target() string { func (r *Spec) newGenericResource( targetPathBuilder func(base string) string, osFileInfo os.FileInfo, - absPublishDir, - absSourceFilename, + sourceFilename, baseFilename, resourceType string) *genericResource { @@ -587,8 +604,7 @@ func (r *Spec) newGenericResource( return &genericResource{ targetPathBuilder: targetPathBuilder, osFileInfo: osFileInfo, - absPublishDir: absPublishDir, - absSourceFilename: absSourceFilename, + sourceFilename: sourceFilename, relTargetPath: dirFile{dir: fpath, file: fname}, resourceType: resourceType, spec: r, diff --git a/resource/resource_test.go b/resource/resource_test.go index 396b40446..40061e5c4 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -29,7 +29,7 @@ func TestGenericResource(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) - r := spec.newGenericResource(nil, nil, "/public", "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css") assert.Equal("https://example.com/foo.css", r.Permalink()) assert.Equal("/foo.css", r.RelPermalink()) @@ -44,7 +44,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { factory := func(s string) string { return path.Join("/foo", s) } - r := spec.newGenericResource(factory, nil, "/public", "/a/foo.css", "foo.css", "css") + r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css") assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink()) @@ -55,11 +55,11 @@ func TestNewResourceFromFilename(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) - writeSource(t, spec.Fs, "/project/a/b/logo.png", "image") - writeSource(t, spec.Fs, "/root/a/b/data.json", "json") + writeSource(t, spec.Fs, "content/a/b/logo.png", "image") + writeSource(t, spec.Fs, "content/a/b/data.json", "json") - r, err := spec.NewResourceFromFilename(nil, "/public", - filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.NewResourceFromFilename(nil, + filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) assert.NoError(err) assert.NotNil(r) @@ -67,7 +67,7 @@ func TestNewResourceFromFilename(t *testing.T) { assert.Equal("/a/b/logo.png", r.RelPermalink()) assert.Equal("https://example.com/a/b/logo.png", r.Permalink()) - r, err = spec.NewResourceFromFilename(nil, "/public", "/root/a/b/data.json", "a/b/data.json") + r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json") assert.NoError(err) assert.NotNil(r) @@ -82,10 +82,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) { assert := require.New(t) spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs") - writeSource(t, spec.Fs, "/project/a/b/logo.png", "image") + writeSource(t, spec.Fs, "content/a/b/logo.png", "image") - r, err := spec.NewResourceFromFilename(nil, "/public", - filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) + r, err := spec.NewResourceFromFilename(nil, + filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png")) assert.NoError(err) assert.NotNil(r) @@ -101,10 +101,10 @@ func TestResourcesByType(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/a/logo.png", "logo.css", "image"), - spec.newGenericResource(nil, nil, "/public", "/a/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/a/foo3.css", "foo3.css", "css")} + spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), + spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"), + spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"), + spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")} assert.Len(resources.ByType("css"), 3) assert.Len(resources.ByType("image"), 1) @@ -115,11 +115,11 @@ func TestResourcesGetByPrefix(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css")} + spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), + spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), + spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), + spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), + spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")} assert.Nil(resources.GetByPrefix("asdf")) assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink()) @@ -144,14 +144,14 @@ func TestResourcesGetMatch(t *testing.T) { assert := require.New(t) spec := newTestResourceSpec(assert) resources := Resources{ - spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"), - spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"), - spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/b/c/foo4.css", "c/foo4.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/b/c/foo5.css", "c/foo5.css", "css"), - spec.newGenericResource(nil, nil, "/public", "/b/c/d/foo6.css", "c/d/foo6.css", "css"), + spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"), + spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"), + spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"), + spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"), + spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"), + spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"), + spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"), + spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"), } assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink()) @@ -373,12 +373,12 @@ func TestAssignMetadata(t *testing.T) { }}, } { - foo2 = spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css") - logo2 = spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image") - foo1 = spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css") - logo1 = spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image") - foo3 = spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css") - logo3 = spec.newGenericResource(nil, nil, "/public", "/b/logo3.png", "logo3.png", "image") + foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css") + logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image") + foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css") + logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image") + foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css") + logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image") resources = Resources{ foo2, @@ -428,7 +428,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) { a100 := strings.Repeat("a", 100) pattern := "a*a*a*a*a*a*a*a*b" - resources := Resources{spec.newGenericResource(nil, nil, "/public", "/a/"+a100, a100, "css")} + resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")} b.ResetTimer() for i := 0; i < b.N; i++ { @@ -444,17 +444,17 @@ func benchResources(b *testing.B) Resources { for i := 0; i < 30; i++ { name := fmt.Sprintf("abcde%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) } for i := 0; i < 30; i++ { name := fmt.Sprintf("efghi%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) } for i := 0; i < 30; i++ { name := fmt.Sprintf("jklmn%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/b/sub/"+name, "sub/"+name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css")) } return resources @@ -482,7 +482,7 @@ func BenchmarkAssignMetadata(b *testing.B) { } for i := 0; i < 20; i++ { name := fmt.Sprintf("foo%d_%d.css", i%5, i) - resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css")) + resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css")) } b.StartTimer() diff --git a/resource/testhelpers_test.go b/resource/testhelpers_test.go index a9015ab2c..9b50633bd 100644 --- a/resource/testhelpers_test.go +++ b/resource/testhelpers_test.go @@ -4,6 +4,7 @@ import ( "path/filepath" "testing" + "fmt" "image" "io" "io/ioutil" @@ -27,7 +28,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec { func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec { cfg := viper.New() cfg.Set("baseURL", baseURL) - cfg.Set("resourceDir", "/res") + cfg.Set("resourceDir", "resources") + cfg.Set("contentDir", "content") imagingCfg := map[string]interface{}{ "resampleFilter": "linear", @@ -60,9 +62,8 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec { workDir = "/private" + workDir } - contentDir := "base" cfg.Set("workingDir", workDir) - cfg.Set("contentDir", contentDir) + cfg.Set("contentDir", filepath.Join(workDir, "content")) cfg.Set("resourceDir", filepath.Join(workDir, "res")) fs := hugofs.NewFrom(hugofs.Os, cfg) @@ -97,10 +98,8 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R src, err := os.Open(filepath.FromSlash("testdata/" + name)) assert.NoError(err) - workingDir := spec.Cfg.GetString("workingDir") - f := filepath.Join(workingDir, name) - - out, err := spec.Fs.Source.Create(f) + assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755)) + out, err := spec.BaseFs.ContentFs.Create(name) assert.NoError(err) _, err = io.Copy(out, src) out.Close() @@ -111,14 +110,17 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R return path.Join("/a", s) } - r, err := spec.NewResourceFromFilename(factory, "/public", f, name) + r, err := spec.NewResourceFromFilename(factory, name, name) assert.NoError(err) return r } -func assertImageFile(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) { - f, err := fs.Source.Open(filename) +func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { + f, err := fs.Open(filename) + if err != nil { + printFs(fs, "", os.Stdout) + } assert.NoError(err) defer f.Close() @@ -129,8 +131,8 @@ func assertImageFile(assert *require.Assertions, fs *hugofs.Fs, filename string, assert.Equal(height, config.Height) } -func assertFileCache(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) { - assertImageFile(assert, fs, filepath.Join("/res/_gen/images", filename), width, height) +func assertFileCache(assert *require.Assertions, fs afero.Fs, filename string, width, height int) { + assertImageFile(assert, fs, filepath.Join("_gen/images", filename), width, height) } func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) { @@ -142,3 +144,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { t.Fatalf("Failed to write file: %s", err) } } + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + if info != nil && !info.IsDir() { + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\t" + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) +} diff --git a/source/content_directory_test.go b/source/content_directory_test.go index 9874acec2..ed00af625 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -17,11 +17,15 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" + "github.com/stretchr/testify/require" ) func TestIgnoreDotFilesAndDirectories(t *testing.T) { + assert := require.New(t) tests := []struct { path string @@ -35,7 +39,6 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { {"foobar/.barfoo.md", true, nil}, {".barfoo.md", true, nil}, {".md", true, nil}, - {"", true, nil}, {"foobar/barfoo.md~", true, nil}, {".foobar/barfoo.md~", true, nil}, {"foobar~/barfoo.md", false, nil}, @@ -51,9 +54,13 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { for i, test := range tests { v := viper.New() + v.Set("contentDir", "content") v.Set("ignoreFiles", test.ignoreFilesRegexpes) + fs := hugofs.NewMem(v) + ps, err := helpers.NewPathSpec(fs, v) + assert.NoError(err) - s := NewSourceSpec(v, hugofs.NewMem(v)) + s := NewSourceSpec(ps, fs.Source) if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored { t.Errorf("[%d] File not ignored", i) diff --git a/source/dirs_test.go b/source/dirs_test.go index f4650669c..46236120e 100644 --- a/source/dirs_test.go +++ b/source/dirs_test.go @@ -101,6 +101,8 @@ func TestStaticDirs(t *testing.T) { for i, test := range tests { msg := fmt.Sprintf("Test %d", i) v := viper.New() + v.Set("contentDir", "content") + fs := hugofs.NewMem(v) cfg := test.setup(v, fs) cfg.Set("workingDir", filepath.FromSlash("/work")) @@ -134,6 +136,7 @@ func TestStaticDirsFs(t *testing.T) { v.Set("workingDir", filepath.FromSlash("/work")) v.Set("theme", "mytheme") v.Set("themesDir", "themes") + v.Set("contentDir", "content") v.Set("staticDir", []string{"s1", "s2"}) v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)}) diff --git a/source/fileInfo.go b/source/fileInfo.go index 1cad1c4ee..882ef22a7 100644 --- a/source/fileInfo.go +++ b/source/fileInfo.go @@ -14,12 +14,17 @@ package source import ( + "fmt" "io" "os" "path/filepath" "strings" "sync" + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/helpers" ) @@ -86,7 +91,10 @@ type FileInfo struct { // Absolute filename to the file on disk. filename string - fi os.FileInfo + + sp *SourceSpec + + fi os.FileInfo // Derived from filename ext string // Extension without any "." @@ -104,8 +112,6 @@ type FileInfo struct { uniqueID string - sp *SourceSpec - lazyInit sync.Once } @@ -146,7 +152,6 @@ func (fi *FileInfo) init() { fi.lazyInit.Do(func() { relDir := strings.Trim(fi.relDir, helpers.FilePathSeparator) parts := strings.Split(relDir, helpers.FilePathSeparator) - var section string if (!fi.isLeafBundle && len(parts) == 1) || len(parts) > 1 { section = parts[0] @@ -161,6 +166,19 @@ func (fi *FileInfo) init() { func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo { + var lang, translationBaseName, relPath string + + if fp, ok := fi.(hugofs.FilePather); ok { + filename = fp.Filename() + baseDir = fp.BaseDir() + relPath = fp.Path() + } + + if fl, ok := fi.(hugofs.LanguageAnnouncer); ok { + lang = fl.Lang() + translationBaseName = fl.TranslationBaseName() + } + dir, name := filepath.Split(filename) if !strings.HasSuffix(dir, helpers.FilePathSeparator) { dir = dir + helpers.FilePathSeparator @@ -175,19 +193,20 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator) - relPath := filepath.Join(relDir, name) + if relPath == "" { + relPath = filepath.Join(relDir, name) + } ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), ".")) baseName := helpers.Filename(name) - lang := strings.TrimPrefix(filepath.Ext(baseName), ".") - var translationBaseName string - - if _, ok := sp.Languages[lang]; lang == "" || !ok { - lang = sp.DefaultContentLanguage - translationBaseName = baseName - } else { - translationBaseName = helpers.Filename(baseName) + if translationBaseName == "" { + // This is usyally provided by the filesystem. But this FileInfo is also + // created in a standalone context when doing "hugo new". This is + // an approximate implementation, which is "good enough" in that case. + translationBaseName = strings.TrimSuffix(baseName, ext) + fileLangExt := filepath.Ext(translationBaseName) + translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt) } f := &FileInfo{ @@ -211,5 +230,27 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f // Open implements ReadableFile. func (fi *FileInfo) Open() (io.ReadCloser, error) { - return fi.sp.Fs.Source.Open(fi.Filename()) + f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename()) + return f, err +} + +func printFs(fs afero.Fs, path string, w io.Writer) { + if fs == nil { + return + } + afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { + + if info != nil && !info.IsDir() { + + s := path + if lang, ok := info.(hugofs.LanguageAnnouncer); ok { + s = s + "\t" + lang.Lang() + } + if fp, ok := info.(hugofs.FilePather); ok { + s = s + "\t" + fp.Filename() + } + fmt.Fprintln(w, " ", s) + } + return nil + }) } diff --git a/source/fileInfo_test.go b/source/fileInfo_test.go index 1b9b130e4..770478e60 100644 --- a/source/fileInfo_test.go +++ b/source/fileInfo_test.go @@ -17,6 +17,12 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/helpers" + + "github.com/spf13/viper" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" "github.com/stretchr/testify/require" ) @@ -35,6 +41,8 @@ func TestFileInfo(t *testing.T) { assert.Equal(filepath.FromSlash("b/"), f.Dir()) assert.Equal(filepath.FromSlash("b/page.md"), f.Path()) assert.Equal("b", f.Section()) + assert.Equal(filepath.FromSlash("page"), f.TranslationBaseName()) + assert.Equal(filepath.FromSlash("page"), f.BaseFileName()) }}, {filepath.FromSlash("/a/"), filepath.FromSlash("/a/b/c/d/page.md"), func(f *FileInfo) { @@ -47,3 +55,39 @@ func TestFileInfo(t *testing.T) { } } + +func TestFileInfoLanguage(t *testing.T) { + assert := require.New(t) + langs := map[string]bool{ + "sv": true, + "en": true, + } + + m := afero.NewMemMapFs() + lfs := hugofs.NewLanguageFs("sv", langs, m) + v := viper.New() + v.Set("contentDir", "content") + + fs := hugofs.NewFrom(m, v) + + ps, err := helpers.NewPathSpec(fs, v) + assert.NoError(err) + s := SourceSpec{Fs: lfs, PathSpec: ps} + s.Languages = map[string]interface{}{ + "en": true, + } + + err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777) + assert.NoError(err) + err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777) + assert.NoError(err) + + sv, _ := lfs.Stat("page.md") + en, _ := lfs.Stat("page.en.md") + + fiSv := s.NewFileInfo("", "page.md", false, sv) + fiEn := s.NewFileInfo("", "page.en.md", false, en) + + assert.Equal("sv", fiSv.Lang()) + assert.Equal("en", fiEn.Lang()) +} diff --git a/source/filesystem.go b/source/filesystem.go index db004d3a1..50075e3c4 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -82,11 +82,11 @@ func (f *Filesystem) captureFiles() { if f.Fs == nil { panic("Must have a fs") } - err := helpers.SymbolicWalk(f.Fs.Source, f.Base, walker) + err := helpers.SymbolicWalk(f.Fs, f.Base, walker) if err != nil { jww.ERROR.Println(err) - if err == helpers.ErrWalkRootTooShort { + if err == helpers.ErrPathTooShort { panic("The root path is too short. If this is a test, make sure to init the content paths.") } } @@ -100,7 +100,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) { jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err) return false, nil } - linkfi, err := f.Fs.Source.Stat(link) + linkfi, err := f.Fs.Stat(link) if err != nil { jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) return false, nil diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 25ce0268f..82f02d404 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -18,6 +18,8 @@ import ( "runtime" "testing" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/spf13/viper" @@ -69,5 +71,7 @@ func TestUnicodeNorm(t *testing.T) { func newTestSourceSpec() SourceSpec { v := viper.New() - return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v} + v.Set("contentDir", "content") + ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v) + return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps} } diff --git a/source/sourceSpec.go b/source/sourceSpec.go index e40010162..634306e5f 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -18,17 +18,18 @@ import ( "path/filepath" "regexp" - "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" + "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" "github.com/spf13/cast" ) // SourceSpec abstracts language-specific file creation. // TODO(bep) rename to Spec type SourceSpec struct { - Cfg config.Provider - Fs *hugofs.Fs + *helpers.PathSpec + + Fs afero.Fs // This is set if the ignoreFiles config is set. ignoreFilesRe []*regexp.Regexp @@ -38,8 +39,9 @@ type SourceSpec struct { DisabledLanguages map[string]bool } -// NewSourceSpec initializes SourceSpec using languages from a given configuration. -func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec { +// NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec. +func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { + cfg := ps.Cfg defaultLang := cfg.GetString("defaultContentLanguage") languages := cfg.GetStringMap("languages") @@ -69,10 +71,17 @@ func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec { } } - return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} } func (s *SourceSpec) IgnoreFile(filename string) bool { + if filename == "" { + if _, ok := s.Fs.(*afero.OsFs); ok { + return true + } + return false + } + base := filepath.Base(filename) if len(base) > 0 { @@ -99,7 +108,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { - fi, err := helpers.LstatIfOs(s.Fs.Source, filename) + fi, err := helpers.LstatIfPossible(s.Fs, filename) if err != nil { return false, err } @@ -110,7 +119,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) { if fi.Mode()&os.ModeSymlink == os.ModeSymlink { link, err := filepath.EvalSymlinks(filename) - fi, err = helpers.LstatIfOs(s.Fs.Source, link) + fi, err = helpers.LstatIfPossible(s.Fs, link) if err != nil { return false, err } diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index f35e29459..68e7c59d6 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -369,7 +369,7 @@ func TestIntersect(t *testing.T) { func TestIsSet(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + ns := newTestNs() for i, test := range []struct { a interface{} @@ -787,3 +787,9 @@ func newDeps(cfg config.Provider) *deps.Deps { Log: jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime), } } + +func newTestNs() *Namespace { + v := viper.New() + v.Set("contentDir", "content") + return New(newDeps(v)) +} diff --git a/tpl/data/data_test.go b/tpl/data/data_test.go index bcdddc9f4..9b21dc8aa 100644 --- a/tpl/data/data_test.go +++ b/tpl/data/data_test.go @@ -21,7 +21,6 @@ import ( "strings" "testing" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -29,7 +28,7 @@ import ( func TestGetCSV(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + ns := newTestNs() for i, test := range []struct { sep string @@ -123,7 +122,7 @@ func TestGetCSV(t *testing.T) { func TestGetJSON(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + ns := newTestNs() for i, test := range []struct { url string diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index f0b027955..79e9b3907 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -127,7 +127,7 @@ func TestScpGetRemote(t *testing.T) { func TestScpGetRemoteParallel(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + ns := newTestNs() content := []byte(`T€st Content 123`) srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) { @@ -176,3 +176,9 @@ func newDeps(cfg config.Provider) *deps.Deps { ContentSpec: cs, } } + +func newTestNs() *Namespace { + v := viper.New() + v.Set("contentDir", "content") + return New(newDeps(v)) +} diff --git a/tpl/os/os.go b/tpl/os/os.go index 02faa2809..f005bd4a9 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -25,14 +25,29 @@ import ( // New returns a new instance of the os-namespaced template functions. func New(deps *deps.Deps) *Namespace { + + // Since Hugo 0.38 we can have multiple content dirs. This can make it hard to + // reason about where the file is placed relative to the project root. + // To make the {{ readFile .Filename }} variant just work, we create a composite + // filesystem that first checks the work dir fs and then the content fs. + var rfs afero.Fs + if deps.Fs != nil { + rfs = deps.Fs.WorkingDir + if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil { + rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.ContentFs, deps.Fs.WorkingDir)) + } + } + return &Namespace{ - deps: deps, + readFileFs: rfs, + deps: deps, } } // Namespace provides template functions for the "os" namespace. type Namespace struct { - deps *deps.Deps + readFileFs afero.Fs + deps *deps.Deps } // Getenv retrieves the value of the environment variable named by the key. @@ -46,10 +61,10 @@ func (ns *Namespace) Getenv(key interface{}) (string, error) { return _os.Getenv(skey), nil } -// readFile reads the file named by filename relative to the given basepath +// readFile reads the file named by filename in the given filesystem // and returns the contents as a string. // There is a upper size limit set at 1 megabytes. -func readFile(fs *afero.BasePathFs, filename string) (string, error) { +func readFile(fs afero.Fs, filename string) (string, error) { if filename == "" { return "", errors.New("readFile needs a filename") } @@ -79,7 +94,7 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) { return "", err } - return readFile(ns.deps.Fs.WorkingDir, s) + return readFile(ns.readFileFs, s) } // ReadDir lists the directory contents relative to the configured WorkingDir. diff --git a/tpl/tplimpl/template_funcs_test.go b/tpl/tplimpl/template_funcs_test.go index b8390c5b9..72842d308 100644 --- a/tpl/tplimpl/template_funcs_test.go +++ b/tpl/tplimpl/template_funcs_test.go @@ -65,6 +65,7 @@ func TestTemplateFuncsExamples(t *testing.T) { v.Set("workingDir", workingDir) v.Set("multilingual", true) + v.Set("contentDir", "content") v.Set("baseURL", "http://mysite.com/hugo/") v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v)) @@ -125,7 +126,10 @@ func TestPartialCached(t *testing.T) { var data struct { } - config := newDepsConfig(viper.New()) + v := viper.New() + v.Set("contentDir", "content") + + config := newDepsConfig(v) config.WithTemplate = func(templ tpl.TemplateHandler) error { err := templ.AddTemplate("partials/"+name, partial) diff --git a/tpl/tplimpl/template_test.go b/tpl/tplimpl/template_test.go index b94848394..78682e9ab 100644 --- a/tpl/tplimpl/template_test.go +++ b/tpl/tplimpl/template_test.go @@ -35,6 +35,7 @@ func TestHTMLEscape(t *testing.T) { "other": "

Hi!

", } v := viper.New() + v.Set("contentDir", "content") fs := hugofs.NewMem(v) //afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755) diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 5e3ed8b34..07c51c3b0 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -25,7 +25,9 @@ import ( func TestRemarshal(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) assert := require.New(t) tomlExample := `title = "Test Metadata" @@ -111,7 +113,10 @@ title: Test Metadata func TestRemarshalComments(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) + assert := require.New(t) input := ` @@ -153,7 +158,9 @@ Hugo = "Rules" func TestTestRemarshalError(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) assert := require.New(t) _, err := ns.Remarshal("asdf", "asdf") diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 195a0f15c..ab3beb804 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -32,7 +32,9 @@ type tstNoStringer struct{} func TestEmojify(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{} @@ -60,7 +62,9 @@ func TestEmojify(t *testing.T) { func TestHighlight(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{} @@ -90,7 +94,9 @@ func TestHighlight(t *testing.T) { func TestHTMLEscape(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{} @@ -118,7 +124,9 @@ func TestHTMLEscape(t *testing.T) { func TestHTMLUnescape(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{} @@ -146,7 +154,9 @@ func TestHTMLUnescape(t *testing.T) { func TestMarkdownify(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{} @@ -176,7 +186,9 @@ func TestMarkdownifyBlocksOfText(t *testing.T) { assert := require.New(t) - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) text := ` #First @@ -201,7 +213,9 @@ And then some. func TestPlainify(t *testing.T) { t.Parallel() - ns := New(newDeps(viper.New())) + v := viper.New() + v.Set("contentDir", "content") + ns := New(newDeps(v)) for i, test := range []struct { s interface{}