From 241b21b0fd34d91fccb2ce69874110dceae6f926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 4 Jan 2023 18:24:36 +0100 Subject: [PATCH] Create a struct with all of Hugo's config options Primary motivation is documentation, but it will also hopefully simplify the code. Also, * Lower case the default output format names; this is in line with the custom ones (map keys) and how it's treated all the places. This avoids doing `stringds.EqualFold` everywhere. Closes #10896 Closes #10620 --- .gitignore | 3 +- cache/docs.go | 2 + cache/filecache/filecache.go | 28 +- cache/filecache/filecache_config.go | 103 +- cache/filecache/filecache_config_test.go | 88 +- cache/filecache/filecache_pruner.go | 2 +- cache/filecache/filecache_pruner_test.go | 13 +- cache/filecache/filecache_test.go | 88 +- cache/filecache/integration_test.go | 9 +- commands/commandeer.go | 990 +++++++------ commands/commands.go | 343 +---- commands/commands_test.go | 411 ------ commands/config.go | 185 +-- commands/convert.go | 222 +-- commands/deploy.go | 84 +- commands/deploy_off.go | 48 + commands/env.go | 83 +- commands/gen.go | 207 ++- commands/genchromastyles.go | 72 - commands/gendoc.go | 98 -- commands/gendocshelper.go | 71 - commands/genman.go | 77 - commands/helpers.go | 143 +- commands/hugo_test.go | 206 --- commands/hugo_windows.go | 2 +- commands/{hugo.go => hugobuilder.go} | 1260 +++++++--------- commands/{import_jekyll.go => import.go} | 794 +++++----- commands/import_jekyll_test.go | 177 --- commands/limit_darwin.go | 84 -- commands/limit_others.go | 21 - commands/list.go | 295 ++-- commands/list_test.go | 68 - commands/mod.go | 447 +++--- commands/mod_npm.go | 56 - commands/new.go | 389 ++++- commands/new_content_test.go | 29 - commands/new_site.go | 167 --- commands/new_theme.go | 176 --- commands/nodeploy.go | 51 - commands/release.go | 73 +- commands/release_noop.go | 21 - commands/server.go | 1287 ++++++++++------- commands/server_errors.go | 31 - commands/server_test.go | 429 ------ commands/static_syncer.go | 129 -- commands/version.go | 44 - commands/xcommand_template.go | 78 + common/hstrings/strings.go | 57 + .../hstrings/strings_test.go | 28 +- common/htime/time.go | 9 + common/hugo/hugo.go | 27 +- common/loggers/ignorableLogger.go | 10 +- common/maps/maps.go | 35 +- common/maps/maps_test.go | 8 +- common/maps/params.go | 98 +- common/maps/params_test.go | 16 +- {hugolib/paths => common/urls}/baseURL.go | 57 +- .../paths => common/urls}/baseURL_test.go | 22 +- config/allconfig/allconfig.go | 813 +++++++++++ config/allconfig/alldecoders.go | 325 +++++ config/allconfig/configlanguage.go | 216 +++ config/allconfig/integration_test.go | 71 + config/allconfig/load.go | 559 +++++++ config/allconfig/load_test.go | 67 + config/commonConfig.go | 129 +- config/commonConfig_test.go | 5 +- config/compositeConfig.go | 117 -- config/configLoader.go | 8 + config/configProvider.go | 67 +- config/defaultConfigProvider.go | 97 +- config/namespace.go | 76 + config/namespace_test.go | 68 + config/security/securityConfig.go | 6 +- config/services/servicesConfig_test.go | 2 +- config/testconfig/testconfig.go | 84 ++ create/content.go | 2 +- create/content_test.go | 22 +- deploy/deploy.go | 92 +- deploy/deployConfig.go | 62 +- deploy/deployConfig_test.go | 12 +- deploy/deploy_test.go | 58 +- deps/deps.go | 450 +++--- deps/deps_test.go | 5 +- go.mod | 9 +- go.sum | 12 + helpers/content.go | 37 +- helpers/content_test.go | 71 +- helpers/general.go | 14 - helpers/general_test.go | 79 +- helpers/path.go | 17 +- helpers/path_test.go | 85 +- helpers/pathspec.go | 11 +- helpers/pathspec_test.go | 62 - helpers/testhelpers_test.go | 60 +- helpers/url.go | 33 +- helpers/url_test.go | 148 +- hugofs/fs.go | 43 +- hugofs/fs_test.go | 22 +- hugofs/noop_fs.go | 10 +- hugofs/rootmapping_fs_test.go | 2 +- hugolib/alias.go | 2 +- hugolib/breaking_changes_test.go | 118 +- hugolib/cascade_test.go | 54 +- hugolib/codeowners.go | 5 +- hugolib/config.go | 662 ++------- hugolib/config_test.go | 575 +++----- hugolib/configdir_test.go | 153 +- hugolib/content_map.go | 2 +- hugolib/content_map_page.go | 12 +- hugolib/datafiles_test.go | 448 +----- hugolib/dates_test.go | 2 +- hugolib/embedded_shortcodes_test.go | 420 +----- hugolib/filesystems/basefs.go | 24 +- hugolib/filesystems/basefs_test.go | 231 ++- hugolib/gitinfo.go | 4 +- hugolib/hugo_modules_test.go | 35 +- hugolib/hugo_sites.go | 343 +---- hugolib/hugo_sites_build.go | 100 +- hugolib/hugo_sites_build_errors_test.go | 1 + hugolib/hugo_sites_build_test.go | 19 +- hugolib/hugo_sites_multihost_test.go | 2 + hugolib/hugo_smoke_test.go | 26 +- hugolib/integrationtest_builder.go | 67 +- hugolib/language_content_dir_test.go | 2 +- hugolib/menu_test.go | 33 + hugolib/minify_publisher_test.go | 2 +- hugolib/multilingual.go | 82 -- hugolib/page.go | 17 +- hugolib/page__common.go | 6 +- hugolib/page__meta.go | 56 +- hugolib/page__new.go | 5 +- hugolib/page__paginator.go | 7 +- hugolib/page__paths.go | 4 +- hugolib/page__per_output.go | 2 +- hugolib/page_kinds.go | 4 +- hugolib/page_permalink_test.go | 35 +- hugolib/page_test.go | 168 +-- hugolib/pagebundler_test.go | 51 +- hugolib/pagecollections_test.go | 21 +- hugolib/pages_capture.go | 5 +- hugolib/pages_capture_test.go | 27 +- hugolib/pages_process.go | 3 +- hugolib/paths/paths.go | 173 +-- hugolib/paths/paths_test.go | 50 - hugolib/prune_resources.go | 2 +- hugolib/robotstxt_test.go | 2 +- hugolib/rss_test.go | 12 +- hugolib/shortcode_test.go | 5 +- hugolib/site.go | 847 +---------- hugolib/site_benchmark_new_test.go | 8 +- hugolib/site_new.go | 458 ++++++ hugolib/site_output_test.go | 36 +- hugolib/site_render.go | 27 +- hugolib/site_sections.go | 4 +- hugolib/site_sections_test.go | 7 +- hugolib/site_test.go | 204 ++- hugolib/site_url_test.go | 45 +- hugolib/sitemap_test.go | 31 +- hugolib/taxonomy_test.go | 7 +- hugolib/template_test.go | 15 +- hugolib/testhelpers_test.go | 123 +- langs/config.go | 227 +-- langs/i18n/i18n.go | 10 +- langs/i18n/i18n_test.go | 66 +- langs/i18n/translationProvider.go | 18 +- langs/language.go | 231 +-- langs/language_test.go | 29 - livereload/livereload.go | 2 +- main.go | 14 +- main_test.go | 382 +++++ markup/asciidocext/convert.go | 283 +--- markup/asciidocext/convert_test.go | 202 +-- markup/asciidocext/internal/converter.go | 274 ++++ markup/converter/converter.go | 8 +- markup/converter/hooks/hooks.go | 2 + markup/goldmark/convert.go | 8 +- markup/goldmark/convert_test.go | 213 ++- markup/goldmark/toc_test.go | 34 +- markup/highlight/config.go | 2 +- markup/highlight/highlight.go | 4 +- markup/markup.go | 12 +- markup/markup_config/config.go | 14 +- markup/markup_test.go | 11 +- markup/org/convert_test.go | 13 +- markup/pandoc/convert.go | 2 +- markup/rst/convert.go | 2 +- markup/tableofcontents/tableofcontents.go | 1 + media/builtin.go | 163 +++ media/config.go | 139 ++ media/config_test.go | 150 ++ media/mediaType.go | 305 +--- media/mediaType_test.go | 174 +-- minifiers/config.go | 23 +- minifiers/config_test.go | 16 +- minifiers/minifiers.go | 18 +- minifiers/minifiers_test.go | 73 +- modules/client.go | 4 +- modules/collect.go | 34 +- modules/config.go | 270 ++-- navigation/menu.go | 159 +- navigation/menu_cache_test.go | 4 +- navigation/pagemenus.go | 12 +- output/config.go | 147 ++ output/config_test.go | 98 ++ output/docshelper.go | 58 +- output/{ => layouts}/layout.go | 62 +- output/{ => layouts}/layout_test.go | 179 +-- output/outputFormat.go | 157 +- output/outputFormat_test.go | 145 +- parser/lowercase_camel_json.go | 57 + parser/lowercase_camel_json_test.go | 33 + parser/metadecoders/format.go | 24 +- parser/metadecoders/format_test.go | 19 - publisher/htmlElementsCollector_test.go | 7 +- publisher/publisher.go | 2 +- related/inverted_index.go | 36 +- related/inverted_index_test.go | 8 +- resources/assets/sunset.jpg | Bin 0 -> 90587 bytes resources/image.go | 6 +- resources/image_extended_test.go | 13 +- resources/image_test.go | 54 +- resources/images/config.go | 195 ++- resources/images/config_test.go | 26 +- resources/images/image.go | 28 +- resources/images/image_resource.go | 2 +- resources/page/page.go | 4 +- resources/page/page_marshaljson.autogen.go | 2 +- resources/page/page_matcher.go | 109 +- resources/page/page_matcher_test.go | 89 +- resources/page/page_nop.go | 6 +- resources/page/page_paths.go | 11 +- resources/page/page_paths_test.go | 141 +- resources/page/pagemeta/page_frontmatter.go | 69 +- .../page/pagemeta/page_frontmatter_test.go | 111 +- resources/page/pagemeta/pagemeta_test.go | 44 + resources/page/pages_language_merge.go | 1 + resources/page/pagination.go | 6 +- resources/page/pagination_test.go | 55 - resources/page/permalinks.go | 24 +- resources/page/permalinks_test.go | 38 +- resources/page/site.go | 214 ++- resources/page/testhelpers_page_test.go | 38 + resources/page/testhelpers_test.go | 178 ++- resources/postpub/fields_test.go | 4 +- resources/resource.go | 13 +- resources/resource/resources.go | 1 + resources/resource_cache.go | 17 +- .../resource_factories/bundler/bundler.go | 6 +- resources/resource_factories/create/create.go | 1 + resources/resource_factories/create/remote.go | 2 +- resources/resource_metadata_test.go | 221 --- resources/resource_spec.go | 100 +- resources/resource_test.go | 236 +-- .../resource_transformers/babel/babel.go | 2 +- .../htesting/testhelpers.go | 20 +- resources/resource_transformers/js/build.go | 6 +- resources/resource_transformers/js/options.go | 10 +- .../resource_transformers/js/options_test.go | 13 +- .../resource_transformers/minifier/minify.go | 2 +- .../resource_transformers/postcss/postcss.go | 11 +- .../tocss/dartsass/transform.go | 4 +- .../resource_transformers/tocss/scss/tocss.go | 10 +- resources/testhelpers_test.go | 98 +- resources/transform.go | 10 +- resources/transform_test.go | 92 +- source/content_directory_test.go | 37 +- source/fileInfo.go | 2 + source/fileInfo_test.go | 11 +- source/filesystem_test.go | 42 +- source/sourceSpec.go | 44 +- testscripts/commands/commands_errors.txt | 7 + testscripts/commands/completion.txt | 4 + testscripts/commands/config.txt | 19 + testscripts/commands/convert.txt | 42 + testscripts/commands/deploy.txt | 24 + testscripts/commands/env.txt | 5 + testscripts/commands/gen.txt | 19 + testscripts/commands/hugo.txt | 19 + testscripts/commands/hugo__errors.txt | 18 + testscripts/commands/hugo__flags.txt | 27 + testscripts/commands/hugo__watch.txt | 28 + testscripts/commands/import_jekyll.txt | 19 + testscripts/commands/list.txt | 34 + testscripts/commands/mod.txt | 44 + testscripts/commands/mod_npm.txt | 23 + testscripts/commands/mod_tidy.txt | 21 + testscripts/commands/new.txt | 27 + testscripts/commands/server.txt | 30 + testscripts/commands/server__edit_config.txt | 43 + testscripts/commands/server__edit_content.txt | 55 + testscripts/commands/server__multihost.txt | 32 + .../commands/server_render_static_to_disk.txt | 25 + .../commands/server_render_to_memory.txt | 25 + testscripts/commands/version.txt | 7 + testscripts/unfinished/noop.txt | 0 tpl/cast/docshelper.go | 14 +- tpl/collections/append_test.go | 6 +- tpl/collections/apply_test.go | 13 +- tpl/collections/collections.go | 6 +- tpl/collections/collections_test.go | 71 +- tpl/collections/complement_test.go | 6 +- tpl/collections/index.go | 2 +- tpl/collections/index_test.go | 5 +- tpl/collections/merge_test.go | 7 +- tpl/collections/sort.go | 6 +- tpl/collections/sort_test.go | 8 +- tpl/collections/symdiff_test.go | 6 +- tpl/collections/where.go | 4 +- tpl/collections/where_test.go | 8 +- tpl/compare/init.go | 5 +- tpl/crypto/crypto.go | 1 + tpl/data/data.go | 4 +- tpl/data/data_test.go | 4 +- tpl/data/resources.go | 7 +- tpl/data/resources_test.go | 71 +- tpl/fmt/fmt.go | 2 +- tpl/hugo/init.go | 3 + tpl/images/images_test.go | 11 +- tpl/lang/init.go | 2 +- tpl/math/math.go | 1 + tpl/openapi/openapi3/openapi3.go | 2 +- tpl/partials/partials.go | 9 +- tpl/path/path_test.go | 14 +- tpl/site/init.go | 9 +- tpl/strings/strings.go | 11 +- tpl/strings/strings_test.go | 6 +- tpl/template.go | 9 +- tpl/time/init.go | 4 +- tpl/tplimpl/template.go | 44 +- tpl/tplimpl/templateProvider.go | 22 +- tpl/tplimpl/template_funcs.go | 21 +- tpl/transform/transform_test.go | 25 - tpl/transform/unmarshal.go | 2 +- tpl/transform/unmarshal_test.go | 20 +- tpl/urls/urls.go | 2 +- tpl/urls/urls_test.go | 9 +- watchtestscripts.sh | 7 + 337 files changed, 13377 insertions(+), 14898 deletions(-) create mode 100644 cache/docs.go delete mode 100644 commands/commands_test.go create mode 100644 commands/deploy_off.go delete mode 100644 commands/genchromastyles.go delete mode 100644 commands/gendoc.go delete mode 100644 commands/gendocshelper.go delete mode 100644 commands/genman.go delete mode 100644 commands/hugo_test.go rename commands/{hugo.go => hugobuilder.go} (52%) rename commands/{import_jekyll.go => import.go} (69%) delete mode 100644 commands/import_jekyll_test.go delete mode 100644 commands/limit_darwin.go delete mode 100644 commands/limit_others.go delete mode 100644 commands/list_test.go delete mode 100644 commands/mod_npm.go delete mode 100644 commands/new_content_test.go delete mode 100644 commands/new_site.go delete mode 100644 commands/new_theme.go delete mode 100644 commands/nodeploy.go delete mode 100644 commands/release_noop.go delete mode 100644 commands/server_errors.go delete mode 100644 commands/server_test.go delete mode 100644 commands/static_syncer.go delete mode 100644 commands/version.go create mode 100644 commands/xcommand_template.go create mode 100644 common/hstrings/strings.go rename config/compositeConfig_test.go => common/hstrings/strings_test.go (53%) rename {hugolib/paths => common/urls}/baseURL.go (62%) rename {hugolib/paths => common/urls}/baseURL_test.go (74%) create mode 100644 config/allconfig/allconfig.go create mode 100644 config/allconfig/alldecoders.go create mode 100644 config/allconfig/configlanguage.go create mode 100644 config/allconfig/integration_test.go create mode 100644 config/allconfig/load.go create mode 100644 config/allconfig/load_test.go delete mode 100644 config/compositeConfig.go create mode 100644 config/namespace.go create mode 100644 config/namespace_test.go create mode 100644 config/testconfig/testconfig.go delete mode 100644 helpers/pathspec_test.go delete mode 100644 hugolib/multilingual.go delete mode 100644 hugolib/paths/paths_test.go create mode 100644 hugolib/site_new.go create mode 100644 main_test.go create mode 100644 markup/asciidocext/internal/converter.go create mode 100644 media/builtin.go create mode 100644 media/config.go create mode 100644 media/config_test.go create mode 100644 output/config.go create mode 100644 output/config_test.go rename output/{ => layouts}/layout.go (85%) rename output/{ => layouts}/layout_test.go (88%) create mode 100644 parser/lowercase_camel_json_test.go create mode 100644 resources/assets/sunset.jpg create mode 100644 resources/page/testhelpers_page_test.go delete mode 100644 resources/resource_metadata_test.go create mode 100644 testscripts/commands/commands_errors.txt create mode 100644 testscripts/commands/completion.txt create mode 100644 testscripts/commands/config.txt create mode 100644 testscripts/commands/convert.txt create mode 100644 testscripts/commands/deploy.txt create mode 100644 testscripts/commands/env.txt create mode 100644 testscripts/commands/gen.txt create mode 100644 testscripts/commands/hugo.txt create mode 100644 testscripts/commands/hugo__errors.txt create mode 100644 testscripts/commands/hugo__flags.txt create mode 100644 testscripts/commands/hugo__watch.txt create mode 100644 testscripts/commands/import_jekyll.txt create mode 100644 testscripts/commands/list.txt create mode 100644 testscripts/commands/mod.txt create mode 100644 testscripts/commands/mod_npm.txt create mode 100644 testscripts/commands/mod_tidy.txt create mode 100644 testscripts/commands/new.txt create mode 100644 testscripts/commands/server.txt create mode 100644 testscripts/commands/server__edit_config.txt create mode 100644 testscripts/commands/server__edit_content.txt create mode 100644 testscripts/commands/server__multihost.txt create mode 100644 testscripts/commands/server_render_static_to_disk.txt create mode 100644 testscripts/commands/server_render_to_memory.txt create mode 100644 testscripts/commands/version.txt create mode 100644 testscripts/unfinished/noop.txt create mode 100755 watchtestscripts.sh diff --git a/.gitignore b/.gitignore index 00b5b2e80..b170fe204 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -*.test \ No newline at end of file +*.test +imports.* \ No newline at end of file diff --git a/cache/docs.go b/cache/docs.go new file mode 100644 index 000000000..babecec22 --- /dev/null +++ b/cache/docs.go @@ -0,0 +1,2 @@ +// Package cache contains the differenct cache implementations. +package cache diff --git a/cache/filecache/filecache.go b/cache/filecache/filecache.go index 88a466218..05d9379b4 100644 --- a/cache/filecache/filecache.go +++ b/cache/filecache/filecache.go @@ -35,7 +35,7 @@ import ( var ErrFatal = errors.New("fatal filecache error") const ( - filecacheRootDirname = "filecache" + FilecacheRootDirname = "filecache" ) // Cache caches a set of files in a directory. This is usually a file on @@ -301,7 +301,7 @@ func (c *Cache) isExpired(modTime time.Time) bool { } // For testing -func (c *Cache) getString(id string) string { +func (c *Cache) GetString(id string) string { id = cleanID(id) c.nlocker.Lock(id) @@ -328,38 +328,24 @@ func (f Caches) Get(name string) *Cache { // NewCaches creates a new set of file caches from the given // configuration. func NewCaches(p *helpers.PathSpec) (Caches, error) { - var dcfg Configs - if c, ok := p.Cfg.Get("filecacheConfigs").(Configs); ok { - dcfg = c - } else { - var err error - dcfg, err = DecodeConfig(p.Fs.Source, p.Cfg) - if err != nil { - return nil, err - } - } - + dcfg := p.Cfg.GetConfigSection("caches").(Configs) fs := p.Fs.Source m := make(Caches) for k, v := range dcfg { var cfs afero.Fs - if v.isResourceDir { + if v.IsResourceDir { cfs = p.BaseFs.ResourcesCache } else { cfs = fs } if cfs == nil { - // TODO(bep) we still have some places that do not initialize the - // full dependencies of a site, e.g. the import Jekyll command. - // That command does not need these caches, so let us just continue - // for now. - continue + panic("nil fs") } - baseDir := v.Dir + baseDir := v.DirCompiled if err := cfs.MkdirAll(baseDir, 0777); err != nil && !os.IsExist(err) { return nil, err @@ -368,7 +354,7 @@ func NewCaches(p *helpers.PathSpec) (Caches, error) { bfs := afero.NewBasePathFs(cfs, baseDir) var pruneAllRootDir string - if k == cacheKeyModules { + if k == CacheKeyModules { pruneAllRootDir = "pkg" } diff --git a/cache/filecache/filecache_config.go b/cache/filecache/filecache_config.go index a82133ab7..e8019578a 100644 --- a/cache/filecache/filecache_config.go +++ b/cache/filecache/filecache_config.go @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package filecache provides a file based cache for Hugo. package filecache import ( @@ -21,11 +22,8 @@ import ( "time" "github.com/gohugoio/hugo/common/maps" - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "errors" "github.com/mitchellh/mapstructure" @@ -33,98 +31,102 @@ import ( ) const ( - cachesConfigKey = "caches" - resourcesGenDir = ":resourceDir/_gen" cacheDirProject = ":cacheDir/:project" ) -var defaultCacheConfig = Config{ +var defaultCacheConfig = FileCacheConfig{ MaxAge: -1, // Never expire Dir: cacheDirProject, } const ( - cacheKeyGetJSON = "getjson" - cacheKeyGetCSV = "getcsv" - cacheKeyImages = "images" - cacheKeyAssets = "assets" - cacheKeyModules = "modules" - cacheKeyGetResource = "getresource" + CacheKeyGetJSON = "getjson" + CacheKeyGetCSV = "getcsv" + CacheKeyImages = "images" + CacheKeyAssets = "assets" + CacheKeyModules = "modules" + CacheKeyGetResource = "getresource" ) -type Configs map[string]Config +type Configs map[string]FileCacheConfig +// For internal use. func (c Configs) CacheDirModules() string { - return c[cacheKeyModules].Dir + return c[CacheKeyModules].DirCompiled } var defaultCacheConfigs = Configs{ - cacheKeyModules: { + CacheKeyModules: { MaxAge: -1, Dir: ":cacheDir/modules", }, - cacheKeyGetJSON: defaultCacheConfig, - cacheKeyGetCSV: defaultCacheConfig, - cacheKeyImages: { + CacheKeyGetJSON: defaultCacheConfig, + CacheKeyGetCSV: defaultCacheConfig, + CacheKeyImages: { MaxAge: -1, Dir: resourcesGenDir, }, - cacheKeyAssets: { + CacheKeyAssets: { MaxAge: -1, Dir: resourcesGenDir, }, - cacheKeyGetResource: Config{ + CacheKeyGetResource: FileCacheConfig{ MaxAge: -1, // Never expire Dir: cacheDirProject, }, } -type Config struct { +type FileCacheConfig struct { // Max age of cache entries in this cache. Any items older than this will // be removed and not returned from the cache. - // a negative value means forever, 0 means cache is disabled. + // A negative value means forever, 0 means cache is disabled. + // Hugo is leninent with what types it accepts here, but we recommend using + // a duration string, a sequence of decimal numbers, each with optional fraction and a unit suffix, + // such as "300ms", "1.5h" or "2h45m". + // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". MaxAge time.Duration // The directory where files are stored. - Dir string + Dir string + DirCompiled string `json:"-"` // Will resources/_gen will get its own composite filesystem that // also checks any theme. - isResourceDir bool + IsResourceDir bool } // GetJSONCache gets the file cache for getJSON. func (f Caches) GetJSONCache() *Cache { - return f[cacheKeyGetJSON] + return f[CacheKeyGetJSON] } // GetCSVCache gets the file cache for getCSV. func (f Caches) GetCSVCache() *Cache { - return f[cacheKeyGetCSV] + return f[CacheKeyGetCSV] } // ImageCache gets the file cache for processed images. func (f Caches) ImageCache() *Cache { - return f[cacheKeyImages] + return f[CacheKeyImages] } // ModulesCache gets the file cache for Hugo Modules. func (f Caches) ModulesCache() *Cache { - return f[cacheKeyModules] + return f[CacheKeyModules] } // AssetsCache gets the file cache for assets (processed resources, SCSS etc.). func (f Caches) AssetsCache() *Cache { - return f[cacheKeyAssets] + return f[CacheKeyAssets] } // GetResourceCache gets the file cache for remote resources. func (f Caches) GetResourceCache() *Cache { - return f[cacheKeyGetResource] + return f[CacheKeyGetResource] } -func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { +func DecodeConfig(fs afero.Fs, bcfg config.BaseConfig, m map[string]any) (Configs, error) { c := make(Configs) valid := make(map[string]bool) // Add defaults @@ -133,8 +135,6 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { valid[k] = true } - m := cfg.GetStringMap(cachesConfigKey) - _, isOsFs := fs.(*afero.OsFs) for k, v := range m { @@ -170,9 +170,6 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { c[name] = cc } - // This is a very old flag in Hugo, but we need to respect it. - disabled := cfg.GetBool("ignoreCache") - for k, v := range c { dir := filepath.ToSlash(filepath.Clean(v.Dir)) hadSlash := strings.HasPrefix(dir, "/") @@ -180,12 +177,12 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { for i, part := range parts { if strings.HasPrefix(part, ":") { - resolved, isResource, err := resolveDirPlaceholder(fs, cfg, part) + resolved, isResource, err := resolveDirPlaceholder(fs, bcfg, part) if err != nil { return c, err } if isResource { - v.isResourceDir = true + v.IsResourceDir = true } parts[i] = resolved } @@ -195,33 +192,29 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { if hadSlash { dir = "/" + dir } - v.Dir = filepath.Clean(filepath.FromSlash(dir)) + v.DirCompiled = filepath.Clean(filepath.FromSlash(dir)) - if !v.isResourceDir { - if isOsFs && !filepath.IsAbs(v.Dir) { - return c, fmt.Errorf("%q must resolve to an absolute directory", v.Dir) + if !v.IsResourceDir { + if isOsFs && !filepath.IsAbs(v.DirCompiled) { + return c, fmt.Errorf("%q must resolve to an absolute directory", v.DirCompiled) } // Avoid cache in root, e.g. / (Unix) or c:\ (Windows) - if len(strings.TrimPrefix(v.Dir, filepath.VolumeName(v.Dir))) == 1 { - return c, fmt.Errorf("%q is a root folder and not allowed as cache dir", v.Dir) + if len(strings.TrimPrefix(v.DirCompiled, filepath.VolumeName(v.DirCompiled))) == 1 { + return c, fmt.Errorf("%q is a root folder and not allowed as cache dir", v.DirCompiled) } } - if !strings.HasPrefix(v.Dir, "_gen") { + if !strings.HasPrefix(v.DirCompiled, "_gen") { // We do cache eviction (file removes) and since the user can set // his/hers own cache directory, we really want to make sure // we do not delete any files that do not belong to this cache. // We do add the cache name as the root, but this is an extra safe // guard. We skip the files inside /resources/_gen/ because // that would be breaking. - v.Dir = filepath.Join(v.Dir, filecacheRootDirname, k) + v.DirCompiled = filepath.Join(v.DirCompiled, FilecacheRootDirname, k) } else { - v.Dir = filepath.Join(v.Dir, k) - } - - if disabled { - v.MaxAge = 0 + v.DirCompiled = filepath.Join(v.DirCompiled, k) } c[k] = v @@ -231,17 +224,15 @@ func DecodeConfig(fs afero.Fs, cfg config.Provider) (Configs, error) { } // Resolves :resourceDir => /myproject/resources etc., :cacheDir => ... -func resolveDirPlaceholder(fs afero.Fs, cfg config.Provider, placeholder string) (cacheDir string, isResource bool, err error) { - workingDir := cfg.GetString("workingDir") +func resolveDirPlaceholder(fs afero.Fs, bcfg config.BaseConfig, placeholder string) (cacheDir string, isResource bool, err error) { switch strings.ToLower(placeholder) { case ":resourcedir": return "", true, nil case ":cachedir": - d, err := helpers.GetCacheDir(fs, cfg) - return d, false, err + return bcfg.CacheDir, false, nil case ":project": - return filepath.Base(workingDir), false, nil + return filepath.Base(bcfg.WorkingDir), false, nil } return "", false, fmt.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder) diff --git a/cache/filecache/filecache_config_test.go b/cache/filecache/filecache_config_test.go index 1ed020ef1..f93c7060e 100644 --- a/cache/filecache/filecache_config_test.go +++ b/cache/filecache/filecache_config_test.go @@ -11,18 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package filecache +package filecache_test import ( "path/filepath" "runtime" - "strings" "testing" "time" "github.com/spf13/afero" + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/testconfig" qt "github.com/frankban/quicktest" ) @@ -57,22 +58,20 @@ dir = "/path/to/c4" cfg, err := config.FromConfigString(configStr, "toml") c.Assert(err, qt.IsNil) fs := afero.NewMemMapFs() - decoded, err := DecodeConfig(fs, cfg) - c.Assert(err, qt.IsNil) - + decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches c.Assert(len(decoded), qt.Equals, 6) c2 := decoded["getcsv"] c.Assert(c2.MaxAge.String(), qt.Equals, "11h0m0s") - c.Assert(c2.Dir, qt.Equals, filepath.FromSlash("/path/to/c2/filecache/getcsv")) + c.Assert(c2.DirCompiled, qt.Equals, filepath.FromSlash("/path/to/c2/filecache/getcsv")) c3 := decoded["images"] c.Assert(c3.MaxAge, qt.Equals, time.Duration(-1)) - c.Assert(c3.Dir, qt.Equals, filepath.FromSlash("/path/to/c3/filecache/images")) + c.Assert(c3.DirCompiled, qt.Equals, filepath.FromSlash("/path/to/c3/filecache/images")) c4 := decoded["getresource"] c.Assert(c4.MaxAge, qt.Equals, time.Duration(-1)) - c.Assert(c4.Dir, qt.Equals, filepath.FromSlash("/path/to/c4/filecache/getresource")) + c.Assert(c4.DirCompiled, qt.Equals, filepath.FromSlash("/path/to/c4/filecache/getresource")) } func TestDecodeConfigIgnoreCache(t *testing.T) { @@ -106,9 +105,7 @@ dir = "/path/to/c4" cfg, err := config.FromConfigString(configStr, "toml") c.Assert(err, qt.IsNil) fs := afero.NewMemMapFs() - decoded, err := DecodeConfig(fs, cfg) - c.Assert(err, qt.IsNil) - + decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches c.Assert(len(decoded), qt.Equals, 6) for _, v := range decoded { @@ -118,7 +115,7 @@ dir = "/path/to/c4" func TestDecodeConfigDefault(t *testing.T) { c := qt.New(t) - cfg := newTestConfig() + cfg := config.New() if runtime.GOOS == "windows" { cfg.Set("resourceDir", "c:\\cache\\resources") @@ -128,71 +125,22 @@ func TestDecodeConfigDefault(t *testing.T) { cfg.Set("resourceDir", "/cache/resources") cfg.Set("cacheDir", "/cache/thecache") } + cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject")) fs := afero.NewMemMapFs() - - decoded, err := DecodeConfig(fs, cfg) - - c.Assert(err, qt.IsNil) - + decoded := testconfig.GetTestConfigs(fs, cfg).Base.Caches c.Assert(len(decoded), qt.Equals, 6) - imgConfig := decoded[cacheKeyImages] - jsonConfig := decoded[cacheKeyGetJSON] + imgConfig := decoded[filecache.CacheKeyImages] + jsonConfig := decoded[filecache.CacheKeyGetJSON] if runtime.GOOS == "windows" { - c.Assert(imgConfig.Dir, qt.Equals, filepath.FromSlash("_gen/images")) + c.Assert(imgConfig.DirCompiled, qt.Equals, filepath.FromSlash("_gen/images")) } else { - c.Assert(imgConfig.Dir, qt.Equals, "_gen/images") - c.Assert(jsonConfig.Dir, qt.Equals, "/cache/thecache/hugoproject/filecache/getjson") + c.Assert(imgConfig.DirCompiled, qt.Equals, "_gen/images") + c.Assert(jsonConfig.DirCompiled, qt.Equals, "/cache/thecache/hugoproject/filecache/getjson") } - c.Assert(imgConfig.isResourceDir, qt.Equals, true) - c.Assert(jsonConfig.isResourceDir, qt.Equals, false) -} - -func TestDecodeConfigInvalidDir(t *testing.T) { - t.Parallel() - - c := qt.New(t) - - configStr := ` -resourceDir = "myresources" -contentDir = "content" -dataDir = "data" -i18nDir = "i18n" -layoutDir = "layouts" -assetDir = "assets" -archeTypedir = "archetypes" - -[caches] -[caches.getJSON] -maxAge = "10m" -dir = "/" - -` - if runtime.GOOS == "windows" { - configStr = strings.Replace(configStr, "/", "c:\\\\", 1) - } - - cfg, err := config.FromConfigString(configStr, "toml") - c.Assert(err, qt.IsNil) - fs := afero.NewMemMapFs() - - _, err = DecodeConfig(fs, cfg) - c.Assert(err, qt.Not(qt.IsNil)) -} - -func newTestConfig() config.Provider { - cfg := config.NewWithTestDefaults() - cfg.Set("workingDir", filepath.FromSlash("/my/cool/hugoproject")) - cfg.Set("contentDir", "content") - cfg.Set("dataDir", "data") - cfg.Set("resourceDir", "resources") - cfg.Set("i18nDir", "i18n") - cfg.Set("layoutDir", "layouts") - cfg.Set("archetypeDir", "archetypes") - cfg.Set("assetDir", "assets") - - return cfg + c.Assert(imgConfig.IsResourceDir, qt.Equals, true) + c.Assert(jsonConfig.IsResourceDir, qt.Equals, false) } diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go index b8aa76c15..e1b7f1947 100644 --- a/cache/filecache/filecache_pruner.go +++ b/cache/filecache/filecache_pruner.go @@ -31,7 +31,6 @@ import ( func (c Caches) Prune() (int, error) { counter := 0 for k, cache := range c { - count, err := cache.Prune(false) counter += count @@ -58,6 +57,7 @@ func (c *Cache) Prune(force bool) (int, error) { counter := 0 err := afero.Walk(c.Fs, "", func(name string, info os.FileInfo, err error) error { + if info == nil { return nil } diff --git a/cache/filecache/filecache_pruner_test.go b/cache/filecache/filecache_pruner_test.go index 46e1317ce..f0cecfe9f 100644 --- a/cache/filecache/filecache_pruner_test.go +++ b/cache/filecache/filecache_pruner_test.go @@ -11,13 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package filecache +package filecache_test import ( "fmt" "testing" "time" + "github.com/gohugoio/hugo/cache/filecache" "github.com/spf13/afero" qt "github.com/frankban/quicktest" @@ -52,10 +53,10 @@ maxAge = "200ms" dir = ":resourceDir/_gen" ` - for _, name := range []string{cacheKeyGetCSV, cacheKeyGetJSON, cacheKeyAssets, cacheKeyImages} { + for _, name := range []string{filecache.CacheKeyGetCSV, filecache.CacheKeyGetJSON, filecache.CacheKeyAssets, filecache.CacheKeyImages} { msg := qt.Commentf("cache: %s", name) p := newPathsSpec(t, afero.NewMemMapFs(), configStr) - caches, err := NewCaches(p) + caches, err := filecache.NewCaches(p) c.Assert(err, qt.IsNil) cache := caches[name] for i := 0; i < 10; i++ { @@ -75,7 +76,7 @@ dir = ":resourceDir/_gen" for i := 0; i < 10; i++ { id := fmt.Sprintf("i%d", i) - v := cache.getString(id) + v := cache.GetString(id) if i < 5 { c.Assert(v, qt.Equals, "") } else { @@ -83,7 +84,7 @@ dir = ":resourceDir/_gen" } } - caches, err = NewCaches(p) + caches, err = filecache.NewCaches(p) c.Assert(err, qt.IsNil) cache = caches[name] // Touch one and then prune. @@ -98,7 +99,7 @@ dir = ":resourceDir/_gen" // Now only the i5 should be left. for i := 0; i < 10; i++ { id := fmt.Sprintf("i%d", i) - v := cache.getString(id) + v := cache.GetString(id) if i != 5 { c.Assert(v, qt.Equals, "") } else { diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index 6b96a8601..61f9eda64 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package filecache +package filecache_test import ( "errors" @@ -23,13 +23,10 @@ import ( "testing" "time" - "github.com/gobwas/glob" - - "github.com/gohugoio/hugo/langs" - "github.com/gohugoio/hugo/modules" - + "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/testconfig" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" @@ -83,27 +80,19 @@ dir = ":cacheDir/c" p := newPathsSpec(t, osfs, configStr) - caches, err := NewCaches(p) + caches, err := filecache.NewCaches(p) c.Assert(err, qt.IsNil) cache := caches.Get("GetJSON") c.Assert(cache, qt.Not(qt.IsNil)) - c.Assert(cache.maxAge.String(), qt.Equals, "10h0m0s") bfs, ok := cache.Fs.(*afero.BasePathFs) c.Assert(ok, qt.Equals, true) filename, err := bfs.RealPath("key") c.Assert(err, qt.IsNil) - if test.cacheDir != "" { - c.Assert(filename, qt.Equals, filepath.Join(test.cacheDir, "c/"+filecacheRootDirname+"/getjson/key")) - } else { - // Temp dir. - c.Assert(filename, qt.Matches, ".*hugo_cache.*"+filecacheRootDirname+".*key") - } cache = caches.Get("Images") c.Assert(cache, qt.Not(qt.IsNil)) - c.Assert(cache.maxAge, qt.Equals, time.Duration(-1)) bfs, ok = cache.Fs.(*afero.BasePathFs) c.Assert(ok, qt.Equals, true) filename, _ = bfs.RealPath("key") @@ -125,7 +114,7 @@ dir = ":cacheDir/c" return []byte("bcd"), nil } - for _, ca := range []*Cache{caches.ImageCache(), caches.AssetsCache(), caches.GetJSONCache(), caches.GetCSVCache()} { + for _, ca := range []*filecache.Cache{caches.ImageCache(), caches.AssetsCache(), caches.GetJSONCache(), caches.GetCSVCache()} { for i := 0; i < 2; i++ { info, r, err := ca.GetOrCreate("a", rf("abc")) c.Assert(err, qt.IsNil) @@ -160,7 +149,7 @@ dir = ":cacheDir/c" c.Assert(info.Name, qt.Equals, "mykey") io.WriteString(w, "Hugo is great!") w.Close() - c.Assert(caches.ImageCache().getString("mykey"), qt.Equals, "Hugo is great!") + c.Assert(caches.ImageCache().GetString("mykey"), qt.Equals, "Hugo is great!") info, r, err := caches.ImageCache().Get("mykey") c.Assert(err, qt.IsNil) @@ -201,7 +190,7 @@ dir = "/cache/c" p := newPathsSpec(t, afero.NewMemMapFs(), configStr) - caches, err := NewCaches(p) + caches, err := filecache.NewCaches(p) c.Assert(err, qt.IsNil) const cacheName = "getjson" @@ -244,11 +233,11 @@ func TestFileCacheReadOrCreateErrorInRead(t *testing.T) { var result string - rf := func(failLevel int) func(info ItemInfo, r io.ReadSeeker) error { - return func(info ItemInfo, r io.ReadSeeker) error { + rf := func(failLevel int) func(info filecache.ItemInfo, r io.ReadSeeker) error { + return func(info filecache.ItemInfo, r io.ReadSeeker) error { if failLevel > 0 { if failLevel > 1 { - return ErrFatal + return filecache.ErrFatal } return errors.New("fail") } @@ -260,8 +249,8 @@ func TestFileCacheReadOrCreateErrorInRead(t *testing.T) { } } - bf := func(s string) func(info ItemInfo, w io.WriteCloser) error { - return func(info ItemInfo, w io.WriteCloser) error { + bf := func(s string) func(info filecache.ItemInfo, w io.WriteCloser) error { + return func(info filecache.ItemInfo, w io.WriteCloser) error { defer w.Close() result = s _, err := w.Write([]byte(s)) @@ -269,7 +258,7 @@ func TestFileCacheReadOrCreateErrorInRead(t *testing.T) { } } - cache := NewCache(afero.NewMemMapFs(), 100*time.Hour, "") + cache := filecache.NewCache(afero.NewMemMapFs(), 100*time.Hour, "") const id = "a32" @@ -283,60 +272,15 @@ func TestFileCacheReadOrCreateErrorInRead(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(result, qt.Equals, "v3") _, err = cache.ReadOrCreate(id, rf(2), bf("v3")) - c.Assert(err, qt.Equals, ErrFatal) -} - -func TestCleanID(t *testing.T) { - c := qt.New(t) - c.Assert(cleanID(filepath.FromSlash("/a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt")) - c.Assert(cleanID(filepath.FromSlash("a/b//c.txt")), qt.Equals, filepath.FromSlash("a/b/c.txt")) -} - -func initConfig(fs afero.Fs, cfg config.Provider) error { - if _, err := langs.LoadLanguageSettings(cfg, nil); err != nil { - return err - } - - modConfig, err := modules.DecodeConfig(cfg) - if err != nil { - return err - } - - workingDir := cfg.GetString("workingDir") - themesDir := cfg.GetString("themesDir") - if !filepath.IsAbs(themesDir) { - themesDir = filepath.Join(workingDir, themesDir) - } - globAll := glob.MustCompile("**", '/') - modulesClient := modules.NewClient(modules.ClientConfig{ - Fs: fs, - WorkingDir: workingDir, - ThemesDir: themesDir, - ModuleConfig: modConfig, - IgnoreVendor: globAll, - }) - - moduleConfig, err := modulesClient.Collect() - if err != nil { - return err - } - - if err := modules.ApplyProjectConfigDefaults(cfg, moduleConfig.ActiveModules[len(moduleConfig.ActiveModules)-1]); err != nil { - return err - } - - cfg.Set("allModules", moduleConfig.ActiveModules) - - return nil + c.Assert(err, qt.Equals, filecache.ErrFatal) } func newPathsSpec(t *testing.T, fs afero.Fs, configStr string) *helpers.PathSpec { c := qt.New(t) cfg, err := config.FromConfigString(configStr, "toml") c.Assert(err, qt.IsNil) - initConfig(fs, cfg) - config.SetBaseTestDefaults(cfg) - p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, nil) + acfg := testconfig.GetTestConfig(fs, cfg) + p, err := helpers.NewPathSpec(hugofs.NewFrom(fs, acfg.BaseConfig()), acfg, nil) c.Assert(err, qt.IsNil) return p } diff --git a/cache/filecache/integration_test.go b/cache/filecache/integration_test.go index 26653fc35..909895ec5 100644 --- a/cache/filecache/integration_test.go +++ b/cache/filecache/integration_test.go @@ -15,6 +15,9 @@ package filecache_test import ( "path/filepath" + + jww "github.com/spf13/jwalterweatherman" + "testing" "time" @@ -62,6 +65,7 @@ title: "Home" -- assets/a/pixel.png -- iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== -- layouts/index.html -- +{{ warnf "HOME!" }} {{ $img := resources.GetMatch "**.png" }} {{ $img = $img.Resize "3x3" }} {{ $img.RelPermalink }} @@ -71,10 +75,11 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA ` b := hugolib.NewIntegrationTestBuilder( - hugolib.IntegrationTestConfig{T: t, TxtarString: files, RunGC: true, NeedsOsFS: true}, + hugolib.IntegrationTestConfig{T: t, TxtarString: files, Running: true, RunGC: true, NeedsOsFS: true, LogLevel: jww.LevelInfo}, ).Build() b.Assert(b.GCCount, qt.Equals, 0) + b.Assert(b.H, qt.IsNotNil) imagesCacheDir := filepath.Join("_gen", "images") _, err := b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir) @@ -86,9 +91,11 @@ iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAA time.Sleep(300 * time.Millisecond) b.RenameFile("assets/a/pixel.png", "assets/b/pixel2.png").Build() + b.Assert(b.GCCount, qt.Equals, 1) // Build it again to GC the empty a dir. b.Build() + _, err = b.H.BaseFs.ResourcesCache.Stat(filepath.Join(imagesCacheDir, "a")) b.Assert(err, qt.Not(qt.IsNil)) _, err = b.H.BaseFs.ResourcesCache.Stat(imagesCacheDir) diff --git a/commands/commandeer.go b/commands/commandeer.go index 45385d509..ed578e9bf 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,513 +14,593 @@ package commands import ( + "context" "errors" "fmt" "io" - "net" "os" + "os/signal" "path/filepath" - "regexp" "sync" + "sync/atomic" + "syscall" "time" - hconfig "github.com/gohugoio/hugo/config" - - "golang.org/x/sync/semaphore" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/common/paths" - - "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" - "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/config" - - "github.com/spf13/cobra" - - "github.com/gohugoio/hugo/hugolib" - "github.com/spf13/afero" - "github.com/bep/clock" - "github.com/bep/debounce" + "github.com/bep/lazycache" "github.com/bep/overlayfs" - "github.com/gohugoio/hugo/common/types" + "github.com/bep/simplecobra" + + "github.com/gohugoio/hugo/common/hstrings" + "github.com/gohugoio/hugo/common/htime" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/hugolib" + "github.com/spf13/afero" + "github.com/spf13/cobra" ) -type commandeerHugoState struct { - *deps.DepsCfg - hugoSites *hugolib.HugoSites - fsCreate sync.Once - created chan struct{} -} +var ( + errHelp = errors.New("help requested") +) -type commandeer struct { - *commandeerHugoState - - logger loggers.Logger - serverConfig *config.Server - - buildLock func() (unlock func(), err error) - - // Loading state - mustHaveConfigFile bool - failOnInitErr bool - running bool - - // Currently only set when in "fast render mode". But it seems to - // be fast enough that we could maybe just add it for all server modes. - changeDetector *fileChangeDetector - - // We need to reuse these on server rebuilds. - publishDirFs afero.Fs - publishDirStaticFs afero.Fs - publishDirServerFs afero.Fs - - h *hugoBuilderCommon - ftch flagsToConfigHandler - - visitedURLs *types.EvictingStringQueue - - cfgInit func(c *commandeer) error - - // We watch these for changes. - configFiles []string - - // Used in cases where we get flooded with events in server mode. - debounce func(f func()) - - serverPorts []serverPortListener - - languages langs.Languages - doLiveReload bool - renderStaticToDisk bool - fastRenderMode bool - showErrorInBrowser bool - wasError bool - - configured bool - paused bool - - fullRebuildSem *semaphore.Weighted - - // Any error from the last build. - buildErr error -} - -type serverPortListener struct { - p int - ln net.Listener -} - -func newCommandeerHugoState() *commandeerHugoState { - return &commandeerHugoState{ - created: make(chan struct{}), - } -} - -func (c *commandeerHugoState) hugo() *hugolib.HugoSites { - <-c.created - return c.hugoSites -} - -func (c *commandeerHugoState) hugoTry() *hugolib.HugoSites { - select { - case <-c.created: - return c.hugoSites - case <-time.After(time.Millisecond * 100): - return nil - } -} - -func (c *commandeer) errCount() int { - return int(c.logger.LogCounters().ErrorCounter.Count()) -} - -func (c *commandeer) getErrorWithContext() any { - errCount := c.errCount() - - if errCount == 0 { - return nil - } - - m := make(map[string]any) - - //xwm["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors()))) - m["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors()))) - m["Version"] = hugo.BuildVersionString() - ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr) - m["Files"] = ferrors - - return m -} - -func (c *commandeer) Set(key string, value any) { - if c.configured { - panic("commandeer cannot be changed") - } - c.Cfg.Set(key, value) -} - -func (c *commandeer) initFs(fs *hugofs.Fs) error { - c.publishDirFs = fs.PublishDir - c.publishDirStaticFs = fs.PublishDirStatic - c.publishDirServerFs = fs.PublishDirServer - c.DepsCfg.Fs = fs - - return nil -} - -func (c *commandeer) initClock(loc *time.Location) error { - bt := c.Cfg.GetString("clock") - if bt == "" { - return nil - } - - t, err := cast.StringToDateInDefaultLocation(bt, loc) +// Execute executes a command. +func Execute(args []string) error { + x, err := newExec() if err != nil { - return fmt.Errorf(`failed to parse "clock" flag: %s`, err) + return err } - - htime.Clock = clock.Start(t) - return nil -} - -func newCommandeer(mustHaveConfigFile, failOnInitErr, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) { - var rebuildDebouncer func(f func()) - if running { - // The time value used is tested with mass content replacements in a fairly big Hugo site. - // It is better to wait for some seconds in those cases rather than get flooded - // with rebuilds. - rebuildDebouncer = debounce.New(4 * time.Second) - } - - out := io.Discard - if !h.quiet { - out = os.Stdout - } - - c := &commandeer{ - h: h, - ftch: f, - commandeerHugoState: newCommandeerHugoState(), - cfgInit: cfgInit, - visitedURLs: types.NewEvictingStringQueue(10), - debounce: rebuildDebouncer, - fullRebuildSem: semaphore.NewWeighted(1), - - // Init state - mustHaveConfigFile: mustHaveConfigFile, - failOnInitErr: failOnInitErr, - running: running, - - // This will be replaced later, but we need something to log to before the configuration is read. - logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, io.Discard, running), - } - - return c, c.loadConfig() -} - -type fileChangeDetector struct { - sync.Mutex - current map[string]string - prev map[string]string - - irrelevantRe *regexp.Regexp -} - -func (f *fileChangeDetector) OnFileClose(name, md5sum string) { - f.Lock() - defer f.Unlock() - f.current[name] = md5sum -} - -func (f *fileChangeDetector) changed() []string { - if f == nil { - return nil - } - f.Lock() - defer f.Unlock() - var c []string - for k, v := range f.current { - vv, found := f.prev[k] - if !found || v != vv { - c = append(c, k) - } - } - - return f.filterIrrelevant(c) -} - -func (f *fileChangeDetector) filterIrrelevant(in []string) []string { - var filtered []string - for _, v := range in { - if !f.irrelevantRe.MatchString(v) { - filtered = append(filtered, v) - } - } - return filtered -} - -func (f *fileChangeDetector) PrepareNew() { - if f == nil { - return - } - - f.Lock() - defer f.Unlock() - - if f.current == nil { - f.current = make(map[string]string) - f.prev = make(map[string]string) - return - } - - f.prev = make(map[string]string) - for k, v := range f.current { - f.prev[k] = v - } - f.current = make(map[string]string) -} - -func (c *commandeer) loadConfig() error { - if c.DepsCfg == nil { - c.DepsCfg = &deps.DepsCfg{} - } - - if c.logger != nil { - // Truncate the error log if this is a reload. - c.logger.Reset() - } - - cfg := c.DepsCfg - c.configured = false - cfg.Running = c.running - loggers.PanicOnWarning.Store(c.h.panicOnWarning) - - var dir string - if c.h.source != "" { - dir, _ = filepath.Abs(c.h.source) - } else { - dir, _ = os.Getwd() - } - - var sourceFs afero.Fs = hugofs.Os - if c.DepsCfg.Fs != nil { - sourceFs = c.DepsCfg.Fs.Source - } - - environment := c.h.getEnvironment(c.running) - - doWithConfig := func(cfg config.Provider) error { - if c.ftch != nil { - c.ftch.flagsToConfig(cfg) - } - - cfg.Set("workingDir", dir) - cfg.Set("environment", environment) - return nil - } - - cfgSetAndInit := func(cfg config.Provider) error { - c.Cfg = cfg - if c.cfgInit == nil { + args = mapLegacyArgs(args) + cd, err := x.Execute(context.Background(), args) + if err != nil { + if err == errHelp { + cd.CobraCommand.Help() + fmt.Println() return nil } - err := c.cfgInit(c) - return err - } - - configPath := c.h.source - if configPath == "" { - configPath = dir - } - config, configFiles, err := hugolib.LoadConfig( - hugolib.ConfigSourceDescriptor{ - Fs: sourceFs, - Logger: c.logger, - Path: configPath, - WorkingDir: dir, - Filename: c.h.cfgFile, - AbsConfigDir: c.h.getConfigDir(dir), - Environment: environment, - }, - cfgSetAndInit, - doWithConfig) - - if err != nil { - // We should improve the error handling here, - // but with hugo mod init and similar there is a chicken and egg situation - // with modules already configured in config.toml, so ignore those errors. - if c.mustHaveConfigFile || (c.failOnInitErr && !moduleNotFoundRe.MatchString(err.Error())) { - return err - } else { - // Just make it a warning. - c.logger.Warnln(err) - } - } else if c.mustHaveConfigFile && len(configFiles) == 0 { - return hugolib.ErrNoConfigFile - } - - c.configFiles = configFiles - - var ok bool - loc := time.Local - c.languages, ok = c.Cfg.Get("languagesSorted").(langs.Languages) - if ok { - loc = langs.GetLocation(c.languages[0]) - } - - err = c.initClock(loc) - if err != nil { - return err - } - - // Set some commonly used flags - c.doLiveReload = c.running && !c.Cfg.GetBool("disableLiveReload") - c.fastRenderMode = c.running && !c.Cfg.GetBool("disableFastRender") - c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError") - - // This is potentially double work, but we need to do this one more time now - // that all the languages have been configured. - if c.cfgInit != nil { - if err := c.cfgInit(c); err != nil { - return err + if simplecobra.IsCommandError(err) { + // Print the help, but also return the error to fail the command. + cd.CobraCommand.Help() + fmt.Println() } } + return err +} - logger, err := c.createLogger(config) +type commonConfig struct { + mu sync.Mutex + configs *allconfig.Configs + cfg config.Provider + fs *hugofs.Fs +} + +func (c *commonConfig) getFs() *hugofs.Fs { + c.mu.Lock() + defer c.mu.Unlock() + return c.fs +} + +// This is the root command. +type rootCommand struct { + Printf func(format string, v ...interface{}) + Println func(a ...interface{}) + Out io.Writer + + logger loggers.Logger + + // The main cache busting key for the caches below. + configVersionID atomic.Int32 + + // Some, but not all commands need access to these. + // Some needs more than one, so keep them in a small cache. + commonConfigs *lazycache.Cache[int32, *commonConfig] + hugoSites *lazycache.Cache[int32, *hugolib.HugoSites] + + commands []simplecobra.Commander + + // Flags + source string + baseURL string + buildWatch bool + forceSyncStatic bool + panicOnWarning bool + environment string + poll string + gc bool + + // Profile flags (for debugging of performance problems) + cpuprofile string + memprofile string + mutexprofile string + traceprofile string + printm bool + + // TODO(bep) var vs string + logging bool + verbose bool + verboseLog bool + debug bool + quiet bool + renderToMemory bool + + cfgFile string + cfgDir string + logFile string +} + +func (r *rootCommand) Build(cd *simplecobra.Commandeer, bcfg hugolib.BuildCfg, cfg config.Provider) (*hugolib.HugoSites, error) { + h, err := r.Hugo(cfg) if err != nil { - return err + return nil, err + } + if err := h.Build(bcfg); err != nil { + return nil, err } - cfg.Logger = logger - c.logger = logger - c.serverConfig, err = hconfig.DecodeServer(cfg.Cfg) - if err != nil { - return err + return h, nil +} + +func (r *rootCommand) Commands() []simplecobra.Commander { + return r.commands +} + +func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commonConfig, error) { + cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) { + fs := oldConf.fs + configs, err := allconfig.LoadConfig( + allconfig.ConfigSourceDescriptor{ + Flags: oldConf.cfg, + Fs: fs.Source, + Filename: r.cfgFile, + ConfigDir: r.cfgDir, + Environment: r.environment, + }, + ) + if err != nil { + return nil, err + } + + if !configs.Base.C.Clock.IsZero() { + // TODO(bep) find a better place for this. + htime.Clock = clock.Start(configs.Base.C.Clock) + } + + return &commonConfig{ + configs: configs, + cfg: oldConf.cfg, + fs: fs, + }, nil + + }) + + return cc, err + +} + +func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commonConfig, error) { + if cfg == nil { + panic("cfg must be set") } - - createMemFs := config.GetBool("renderToMemory") - c.renderStaticToDisk = config.GetBool("renderStaticToDisk") - // TODO(bep) we/I really need to look at the config set up, but to prevent changing too much - // we store away the original. - config.Set("publishDirOrig", config.GetString("publishDir")) - - if createMemFs { - // Rendering to memoryFS, publish to Root regardless of publishDir. - config.Set("publishDir", "/") - config.Set("publishDirStatic", "/") - } else if c.renderStaticToDisk { - // Hybrid, render dynamic content to Root. - config.Set("publishDirStatic", config.Get("publishDir")) - config.Set("publishDir", "/") - - } - - c.fsCreate.Do(func() { - // Assume both source and destination are using same filesystem. - fs := hugofs.NewFromSourceAndDestination(sourceFs, sourceFs, config) - - if c.publishDirFs != nil { - // Need to reuse the destination on server rebuilds. - fs.PublishDir = c.publishDirFs - fs.PublishDirStatic = c.publishDirStaticFs - fs.PublishDirServer = c.publishDirServerFs + cc, _, err := r.commonConfigs.GetOrCreate(key, func(key int32) (*commonConfig, error) { + var dir string + if r.source != "" { + dir, _ = filepath.Abs(r.source) } else { - if c.renderStaticToDisk { - publishDirStatic := config.GetString("publishDirStatic") - workingDir := config.GetString("workingDir") - absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic) + dir, _ = os.Getwd() + } - fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config) - // Writes the dynamic output to memory, - // while serve others directly from /public on disk. - dynamicFs := fs.PublishDir - staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic) + if cfg == nil { + cfg = config.New() + } + if !cfg.IsSet("publishDir") { + cfg.Set("publishDir", "public") + } + if !cfg.IsSet("renderToDisk") { + cfg.Set("renderToDisk", true) + } + if !cfg.IsSet("workingDir") { + cfg.Set("workingDir", dir) + } + cfg.Set("publishDirStatic", cfg.Get("publishDir")) + cfg.Set("publishDirDynamic", cfg.Get("publishDir")) - // Serve from both the static and dynamic fs, - // the first will take priority. - // THis is a read-only filesystem, - // we do all the writes to - // fs.Destination and fs.DestinationStatic. - fs.PublishDirServer = overlayfs.New( - overlayfs.Options{ - Fss: []afero.Fs{ - dynamicFs, - staticFs, - }, + renderStaticToDisk := cfg.GetBool("renderStaticToDisk") + + sourceFs := hugofs.Os + var desinationFs afero.Fs + if cfg.GetBool("renderToDisk") { + desinationFs = hugofs.Os + } else { + desinationFs = afero.NewMemMapFs() + if renderStaticToDisk { + // Hybrid, render dynamic content to Root. + cfg.Set("publishDirDynamic", "/") + } else { + // Rendering to memoryFS, publish to Root regardless of publishDir. + cfg.Set("publishDirDynamic", "/") + cfg.Set("publishDirStatic", "/") + } + } + + fs := hugofs.NewFromSourceAndDestination(sourceFs, desinationFs, cfg) + + if renderStaticToDisk { + dynamicFs := fs.PublishDir + publishDirStatic := cfg.GetString("publishDirStatic") + workingDir := cfg.GetString("workingDir") + absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic) + staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic) + + // Serve from both the static and dynamic fs, + // the first will take priority. + // THis is a read-only filesystem, + // we do all the writes to + // fs.Destination and fs.DestinationStatic. + fs.PublishDirServer = overlayfs.New( + overlayfs.Options{ + Fss: []afero.Fs{ + dynamicFs, + staticFs, }, - ) - fs.PublishDirStatic = staticFs - } else if createMemFs { - // Hugo writes the output to memory instead of the disk. - fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config) - } + }, + ) + fs.PublishDirStatic = staticFs + } - if c.fastRenderMode { - // For now, fast render mode only. It should, however, be fast enough - // for the full variant, too. - changeDetector := &fileChangeDetector{ - // We use this detector to decide to do a Hot reload of a single path or not. - // We need to filter out source maps and possibly some other to be able - // to make that decision. - irrelevantRe: regexp.MustCompile(`\.map$`), - } - - changeDetector.PrepareNew() - fs.PublishDir = hugofs.NewHashingFs(fs.PublishDir, changeDetector) - fs.PublishDirStatic = hugofs.NewHashingFs(fs.PublishDirStatic, changeDetector) - c.changeDetector = changeDetector + configs, err := allconfig.LoadConfig( + allconfig.ConfigSourceDescriptor{ + Flags: cfg, + Fs: fs.Source, + Filename: r.cfgFile, + ConfigDir: r.cfgDir, + Environment: r.environment, + }, + ) + if err != nil { + return nil, err } - if c.Cfg.GetBool("logPathWarnings") { + base := configs.Base + + if !base.C.Clock.IsZero() { + // TODO(bep) find a better place for this. + htime.Clock = clock.Start(configs.Base.C.Clock) + } + + if base.LogPathWarnings { // Note that we only care about the "dynamic creates" here, // so skip the static fs. fs.PublishDir = hugofs.NewCreateCountingFs(fs.PublishDir) } - // To debug hard-to-find path issues. - // fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`) - - err = c.initFs(fs) - if err != nil { - close(c.created) - return + commonConfig := &commonConfig{ + configs: configs, + cfg: cfg, + fs: fs, } - var h *hugolib.HugoSites - - var createErr error - h, createErr = hugolib.NewHugoSites(*c.DepsCfg) - if h == nil || c.failOnInitErr { - err = createErr - } - - c.hugoSites = h - // TODO(bep) improve. - if c.buildLock == nil && h != nil { - c.buildLock = h.LockBuild - } - close(c.created) + return commonConfig, nil }) + return cc, err + +} + +func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) { + h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { + conf.mu.Lock() + defer conf.mu.Unlock() + depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger} + return hugolib.NewHugoSites(depsCfg) + }) + return h, err +} + +func (r *rootCommand) Hugo(cfg config.Provider) (*hugolib.HugoSites, error) { + h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) { + conf, err := r.ConfigFromProvider(key, cfg) + if err != nil { + return nil, err + } + depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger} + return hugolib.NewHugoSites(depsCfg) + }) + return h, err +} + +func (r *rootCommand) Name() string { + return "hugo" +} + +func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + if !r.buildWatch { + defer r.timeTrack(time.Now(), "Total") + } + + b := newHugoBuilder(r, nil) + + if err := b.loadConfig(cd, true); err != nil { + return err + } + + err := func() error { + if r.buildWatch { + defer r.timeTrack(time.Now(), "Built") + } + err := b.build() + if err != nil { + r.Println("Error:", err.Error()) + } + return err + }() + if err != nil { return err } - cacheDir, err := helpers.GetCacheDir(sourceFs, config) + if !r.buildWatch { + // Done. + return nil + } + + watchDirs, err := b.getDirList() if err != nil { return err } - config.Set("cacheDir", cacheDir) + + watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs) + + for _, group := range watchGroups { + r.Printf("Watching for changes in %s\n", group) + } + watcher, err := b.newWatcher(r.poll, watchDirs...) + if err != nil { + return err + } + + defer watcher.Close() + + r.Println("Press Ctrl+C to stop") + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + <-sigs return nil } + +func (r *rootCommand) Init(cd, runner *simplecobra.Commandeer) error { + r.Out = os.Stdout + if r.quiet { + r.Out = io.Discard + } + r.Printf = func(format string, v ...interface{}) { + if !r.quiet { + fmt.Fprintf(r.Out, format, v...) + } + } + r.Println = func(a ...interface{}) { + if !r.quiet { + fmt.Fprintln(r.Out, a...) + } + } + _, running := runner.Command.(*serverCommand) + var err error + r.logger, err = r.createLogger(running) + if err != nil { + return err + } + + loggers.PanicOnWarning.Store(r.panicOnWarning) + r.commonConfigs = lazycache.New[int32, *commonConfig](lazycache.Options{MaxEntries: 5}) + r.hugoSites = lazycache.New[int32, *hugolib.HugoSites](lazycache.Options{MaxEntries: 5}) + + return nil +} + +func (r *rootCommand) createLogger(running bool) (loggers.Logger, error) { + var ( + logHandle = io.Discard + logThreshold = jww.LevelWarn + outHandle = r.Out + stdoutThreshold = jww.LevelWarn + ) + + if r.verboseLog || r.logging || (r.logFile != "") { + var err error + if r.logFile != "" { + logHandle, err = os.OpenFile(r.logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + return nil, fmt.Errorf("Failed to open log file %q: %s", r.logFile, err) + } + } else { + logHandle, err = os.CreateTemp("", "hugo") + if err != nil { + return nil, err + } + } + } else if r.verbose { + stdoutThreshold = jww.LevelInfo + } + + if r.debug { + stdoutThreshold = jww.LevelDebug + } + + if r.verboseLog { + logThreshold = jww.LevelInfo + if r.debug { + logThreshold = jww.LevelDebug + } + } + + loggers.InitGlobalLogger(stdoutThreshold, logThreshold, outHandle, logHandle) + helpers.InitLoggers() + return loggers.NewLogger(stdoutThreshold, logThreshold, outHandle, logHandle, running), nil +} + +func (r *rootCommand) Reset() { + r.logger.Reset() +} + +// IsTestRun reports whether the command is running as a test. +func (r *rootCommand) IsTestRun() bool { + return os.Getenv("HUGO_TESTRUN") != "" +} + +func (r *rootCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Use = "hugo [flags]" + cmd.Short = "hugo builds your site" + cmd.Long = `hugo is the main command, used to build your Hugo site. + +Hugo is a Fast and Flexible Static Site Generator +built with love by spf13 and friends in Go. + +Complete documentation is available at https://gohugo.io/.` + + // Configure persistent flags + cmd.PersistentFlags().StringVarP(&r.source, "source", "s", "", "filesystem path to read files relative from") + cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) + cmd.PersistentFlags().StringVarP(&r.environment, "environment", "e", "", "build environment") + cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory") + cmd.PersistentFlags().StringP("ignoreVendorPaths", "", "", "ignores any _vendor for module paths matching the given Glob pattern") + cmd.PersistentFlags().String("clock", "", "set the clock used by Hugo, e.g. --clock 2021-11-06T22:30:00.00+09:00") + + cmd.PersistentFlags().StringVar(&r.cfgFile, "config", "", "config file (default is hugo.yaml|json|toml)") + cmd.PersistentFlags().StringVar(&r.cfgDir, "configDir", "config", "config dir") + cmd.PersistentFlags().BoolVar(&r.quiet, "quiet", false, "build in quiet mode") + + // Set bash-completion + _ = cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, config.ValidConfigFileExtensions) + + cmd.PersistentFlags().BoolVarP(&r.verbose, "verbose", "v", false, "verbose output") + cmd.PersistentFlags().BoolVarP(&r.debug, "debug", "", false, "debug output") + cmd.PersistentFlags().BoolVar(&r.logging, "log", false, "enable Logging") + cmd.PersistentFlags().StringVar(&r.logFile, "logFile", "", "log File path (if set, logging enabled automatically)") + cmd.PersistentFlags().BoolVar(&r.verboseLog, "verboseLog", false, "verbose logging") + cmd.Flags().BoolVarP(&r.buildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed") + cmd.Flags().BoolVar(&r.renderToMemory, "renderToMemory", false, "render to memory (only useful for benchmark testing)") + + // Set bash-completion + _ = cmd.PersistentFlags().SetAnnotation("logFile", cobra.BashCompFilenameExt, []string{}) + + // Configure local flags + cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories") + cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft") + cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future") + cmd.Flags().BoolP("buildExpired", "E", false, "include expired content") + cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory") + cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory") + cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") + cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory") + cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to") + cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)") + cmd.Flags().StringVarP(&r.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. https://spf13.com/") + cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date, author, and CODEOWNERS info to the pages") + cmd.Flags().BoolVar(&r.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") + cmd.Flags().StringVar(&r.poll, "poll", "", "set this to a poll interval, e.g --poll 700ms, to use a poll based approach to watch for file system changes") + cmd.Flags().BoolVar(&r.panicOnWarning, "panicOnWarning", false, "panic on first WARNING log") + cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions") + cmd.Flags().Bool("templateMetricsHints", false, "calculate some improvement hints when combined with --templateMetrics") + cmd.Flags().BoolVar(&r.forceSyncStatic, "forceSyncStatic", false, "copy all files when static is changed.") + cmd.Flags().BoolP("noTimes", "", false, "don't sync modification time of files") + cmd.Flags().BoolP("noChmod", "", false, "don't sync permission mode of files") + cmd.Flags().BoolP("noBuildLock", "", false, "don't create .hugo_build.lock file") + cmd.Flags().BoolP("printI18nWarnings", "", false, "print missing translations") + cmd.Flags().BoolP("printPathWarnings", "", false, "print warnings on duplicate target paths etc.") + cmd.Flags().BoolP("printUnusedTemplates", "", false, "print warnings on unused templates.") + cmd.Flags().StringVarP(&r.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`") + cmd.Flags().StringVarP(&r.memprofile, "profile-mem", "", "", "write memory profile to `file`") + cmd.Flags().BoolVarP(&r.printm, "printMemoryUsage", "", false, "print memory usage to screen at intervals") + cmd.Flags().StringVarP(&r.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`") + cmd.Flags().StringVarP(&r.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)") + + // Hide these for now. + cmd.Flags().MarkHidden("profile-cpu") + cmd.Flags().MarkHidden("profile-mem") + cmd.Flags().MarkHidden("profile-mutex") + + cmd.Flags().StringSlice("disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)") + + cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)") + + // Set bash-completion. + // Each flag must first be defined before using the SetAnnotation() call. + _ = cmd.Flags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) + _ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{}) + _ = cmd.Flags().SetAnnotation("destination", cobra.BashCompSubdirsInDir, []string{}) + _ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"}) + + return nil +} + +func (r *rootCommand) timeTrack(start time.Time, name string) { + elapsed := time.Since(start) + r.Printf("%s in %v ms\n", name, int(1000*elapsed.Seconds())) +} + +type simpleCommand struct { + use string + name string + short string + long string + run func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *rootCommand, args []string) error + withc func(cmd *cobra.Command) + initc func(cd *simplecobra.Commandeer) error + + commands []simplecobra.Commander + + rootCmd *rootCommand +} + +func (c *simpleCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *simpleCommand) Name() string { + return c.name +} + +func (c *simpleCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + if c.run == nil { + return nil + } + return c.run(ctx, cd, c.rootCmd, args) +} + +func (c *simpleCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = c.short + cmd.Long = c.long + if c.use != "" { + cmd.Use = c.use + } + if c.withc != nil { + c.withc(cmd) + } + return nil +} + +func (c *simpleCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.rootCmd = cd.Root.Command.(*rootCommand) + if c.initc != nil { + return c.initc(cd) + } + return nil +} + +func mapLegacyArgs(args []string) []string { + if len(args) > 1 && args[0] == "new" && !hstrings.EqualAny(args[1], "site", "theme", "content") { + // Insert "content" as the second argument + args = append(args[:1], append([]string{"content"}, args[1:]...)...) + } + return args +} diff --git a/commands/commands.go b/commands/commands.go index 5b47ad82e..9d707b841 100644 --- a/commands/commands.go +++ b/commands/commands.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,331 +14,28 @@ package commands import ( - "fmt" - "os" - "time" - - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/common/loggers" - hpaths "github.com/gohugoio/hugo/common/paths" - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/spf13/cobra" + "github.com/bep/simplecobra" ) -type commandsBuilder struct { - hugoBuilderCommon - - commands []cmder -} - -func newCommandsBuilder() *commandsBuilder { - return &commandsBuilder{} -} - -func (b *commandsBuilder) addCommands(commands ...cmder) *commandsBuilder { - b.commands = append(b.commands, commands...) - return b -} - -func (b *commandsBuilder) addAll() *commandsBuilder { - b.addCommands( - b.newServerCmd(), - newVersionCmd(), - newEnvCmd(), - b.newConfigCmd(), - b.newDeployCmd(), - b.newConvertCmd(), - b.newNewCmd(), - b.newListCmd(), - newImportCmd(), - newGenCmd(), - createReleaser(), - b.newModCmd(), - ) - - return b -} - -func (b *commandsBuilder) build() *hugoCmd { - h := b.newHugoCmd() - addCommands(h.getCommand(), b.commands...) - return h -} - -func addCommands(root *cobra.Command, commands ...cmder) { - for _, command := range commands { - cmd := command.getCommand() - if cmd == nil { - continue - } - root.AddCommand(cmd) - } -} - -type baseCmd struct { - cmd *cobra.Command -} - -var _ commandsBuilderGetter = (*baseBuilderCmd)(nil) - -// Used in tests. -type commandsBuilderGetter interface { - getCommandsBuilder() *commandsBuilder -} - -type baseBuilderCmd struct { - *baseCmd - *commandsBuilder -} - -func (b *baseBuilderCmd) getCommandsBuilder() *commandsBuilder { - return b.commandsBuilder -} - -func (c *baseCmd) getCommand() *cobra.Command { - return c.cmd -} - -func newBaseCmd(cmd *cobra.Command) *baseCmd { - return &baseCmd{cmd: cmd} -} - -func (b *commandsBuilder) newBuilderCmd(cmd *cobra.Command) *baseBuilderCmd { - bcmd := &baseBuilderCmd{commandsBuilder: b, baseCmd: &baseCmd{cmd: cmd}} - bcmd.hugoBuilderCommon.handleFlags(cmd) - return bcmd -} - -func (b *commandsBuilder) newBuilderBasicCmd(cmd *cobra.Command) *baseBuilderCmd { - bcmd := &baseBuilderCmd{commandsBuilder: b, baseCmd: &baseCmd{cmd: cmd}} - bcmd.hugoBuilderCommon.handleCommonBuilderFlags(cmd) - return bcmd -} - -func (c *baseCmd) flagsToConfig(cfg config.Provider) { - initializeFlags(c.cmd, cfg) -} - -type hugoCmd struct { - *baseBuilderCmd - - // Need to get the sites once built. - c *commandeer -} - -var _ cmder = (*nilCommand)(nil) - -type nilCommand struct{} - -func (c *nilCommand) getCommand() *cobra.Command { - return nil -} - -func (c *nilCommand) flagsToConfig(cfg config.Provider) { -} - -func (b *commandsBuilder) newHugoCmd() *hugoCmd { - cc := &hugoCmd{} - - cc.baseBuilderCmd = b.newBuilderCmd(&cobra.Command{ - Use: "hugo", - Short: "hugo builds your site", - Long: `hugo is the main command, used to build your Hugo site. - -Hugo is a Fast and Flexible Static Site Generator -built with love by spf13 and friends in Go. - -Complete documentation is available at https://gohugo.io/.`, - RunE: func(cmd *cobra.Command, args []string) error { - defer cc.timeTrack(time.Now(), "Total") - cfgInit := func(c *commandeer) error { - if cc.buildWatch { - c.Set("disableLiveReload", true) - } - return nil - } - - // prevent cobra printing error so it can be handled here (before the timeTrack prints) - cmd.SilenceErrors = true - - c, err := initializeConfig(true, true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit) - if err != nil { - cmd.PrintErrln("Error:", err.Error()) - return err - } - cc.c = c - - err = c.build() - if err != nil { - cmd.PrintErrln("Error:", err.Error()) - } - return err +// newExec wires up all of Hugo's CLI. +func newExec() (*simplecobra.Exec, error) { + rootCmd := &rootCommand{ + commands: []simplecobra.Commander{ + newVersionCmd(), + newEnvCommand(), + newServerCommand(), + newDeployCommand(), + newConfigCommand(), + newNewCommand(), + newConvertCommand(), + newImportCommand(), + newListCommand(), + newModCommands(), + newGenCommand(), + newReleaseCommand(), }, - }) + } - cc.cmd.PersistentFlags().StringVar(&cc.cfgFile, "config", "", "config file (default is hugo.yaml|json|toml)") - cc.cmd.PersistentFlags().StringVar(&cc.cfgDir, "configDir", "config", "config dir") - cc.cmd.PersistentFlags().BoolVar(&cc.quiet, "quiet", false, "build in quiet mode") + return simplecobra.New(rootCmd) - // Set bash-completion - _ = cc.cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, config.ValidConfigFileExtensions) - - cc.cmd.PersistentFlags().BoolVarP(&cc.verbose, "verbose", "v", false, "verbose output") - cc.cmd.PersistentFlags().BoolVarP(&cc.debug, "debug", "", false, "debug output") - cc.cmd.PersistentFlags().BoolVar(&cc.logging, "log", false, "enable Logging") - cc.cmd.PersistentFlags().StringVar(&cc.logFile, "logFile", "", "log File path (if set, logging enabled automatically)") - cc.cmd.PersistentFlags().BoolVar(&cc.verboseLog, "verboseLog", false, "verbose logging") - - cc.cmd.Flags().BoolVarP(&cc.buildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed") - - cc.cmd.Flags().Bool("renderToMemory", false, "render to memory (only useful for benchmark testing)") - - // Set bash-completion - _ = cc.cmd.PersistentFlags().SetAnnotation("logFile", cobra.BashCompFilenameExt, []string{}) - - cc.cmd.SetGlobalNormalizationFunc(helpers.NormalizeHugoFlags) - cc.cmd.SilenceUsage = true - - return cc -} - -type hugoBuilderCommon struct { - source string - baseURL string - environment string - - buildWatch bool - panicOnWarning bool - poll string - clock string - - gc bool - - // Profile flags (for debugging of performance problems) - cpuprofile string - memprofile string - mutexprofile string - traceprofile string - printm bool - - // TODO(bep) var vs string - logging bool - verbose bool - verboseLog bool - debug bool - quiet bool - - cfgFile string - cfgDir string - logFile string -} - -func (cc *hugoBuilderCommon) timeTrack(start time.Time, name string) { - if cc.quiet { - return - } - elapsed := time.Since(start) - fmt.Printf("%s in %v ms\n", name, int(1000*elapsed.Seconds())) -} - -func (cc *hugoBuilderCommon) getConfigDir(baseDir string) string { - if cc.cfgDir != "" { - return hpaths.AbsPathify(baseDir, cc.cfgDir) - } - - if v, found := os.LookupEnv("HUGO_CONFIGDIR"); found { - return hpaths.AbsPathify(baseDir, v) - } - - return hpaths.AbsPathify(baseDir, "config") -} - -func (cc *hugoBuilderCommon) getEnvironment(isServer bool) string { - if cc.environment != "" { - return cc.environment - } - - if v, found := os.LookupEnv("HUGO_ENVIRONMENT"); found { - return v - } - - // Used by Netlify and Forestry - if v, found := os.LookupEnv("HUGO_ENV"); found { - return v - } - - if isServer { - return hugo.EnvironmentDevelopment - } - - return hugo.EnvironmentProduction -} - -func (cc *hugoBuilderCommon) handleCommonBuilderFlags(cmd *cobra.Command) { - cmd.PersistentFlags().StringVarP(&cc.source, "source", "s", "", "filesystem path to read files relative from") - cmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) - cmd.PersistentFlags().StringVarP(&cc.environment, "environment", "e", "", "build environment") - cmd.PersistentFlags().StringP("themesDir", "", "", "filesystem path to themes directory") - cmd.PersistentFlags().StringP("ignoreVendorPaths", "", "", "ignores any _vendor for module paths matching the given Glob pattern") - cmd.PersistentFlags().StringVar(&cc.clock, "clock", "", "set the clock used by Hugo, e.g. --clock 2021-11-06T22:30:00.00+09:00") -} - -func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) { - cc.handleCommonBuilderFlags(cmd) - cmd.Flags().Bool("cleanDestinationDir", false, "remove files from destination not found in static directories") - cmd.Flags().BoolP("buildDrafts", "D", false, "include content marked as draft") - cmd.Flags().BoolP("buildFuture", "F", false, "include content with publishdate in the future") - cmd.Flags().BoolP("buildExpired", "E", false, "include expired content") - cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory") - cmd.Flags().StringP("layoutDir", "l", "", "filesystem path to layout directory") - cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") - cmd.Flags().BoolP("ignoreCache", "", false, "ignores the cache directory") - cmd.Flags().StringP("destination", "d", "", "filesystem path to write files to") - cmd.Flags().StringSliceP("theme", "t", []string{}, "themes to use (located in /themes/THEMENAME/)") - cmd.Flags().StringVarP(&cc.baseURL, "baseURL", "b", "", "hostname (and path) to the root, e.g. https://spf13.com/") - cmd.Flags().Bool("enableGitInfo", false, "add Git revision, date, author, and CODEOWNERS info to the pages") - cmd.Flags().BoolVar(&cc.gc, "gc", false, "enable to run some cleanup tasks (remove unused cache files) after the build") - cmd.Flags().StringVar(&cc.poll, "poll", "", "set this to a poll interval, e.g --poll 700ms, to use a poll based approach to watch for file system changes") - cmd.Flags().BoolVar(&cc.panicOnWarning, "panicOnWarning", false, "panic on first WARNING log") - cmd.Flags().Bool("templateMetrics", false, "display metrics about template executions") - cmd.Flags().Bool("templateMetricsHints", false, "calculate some improvement hints when combined with --templateMetrics") - cmd.Flags().BoolP("forceSyncStatic", "", false, "copy all files when static is changed.") - cmd.Flags().BoolP("noTimes", "", false, "don't sync modification time of files") - cmd.Flags().BoolP("noChmod", "", false, "don't sync permission mode of files") - cmd.Flags().BoolP("noBuildLock", "", false, "don't create .hugo_build.lock file") - cmd.Flags().BoolP("printI18nWarnings", "", false, "print missing translations") - cmd.Flags().BoolP("printPathWarnings", "", false, "print warnings on duplicate target paths etc.") - cmd.Flags().BoolP("printUnusedTemplates", "", false, "print warnings on unused templates.") - cmd.Flags().StringVarP(&cc.cpuprofile, "profile-cpu", "", "", "write cpu profile to `file`") - cmd.Flags().StringVarP(&cc.memprofile, "profile-mem", "", "", "write memory profile to `file`") - cmd.Flags().BoolVarP(&cc.printm, "printMemoryUsage", "", false, "print memory usage to screen at intervals") - cmd.Flags().StringVarP(&cc.mutexprofile, "profile-mutex", "", "", "write Mutex profile to `file`") - cmd.Flags().StringVarP(&cc.traceprofile, "trace", "", "", "write trace to `file` (not useful in general)") - - // Hide these for now. - cmd.Flags().MarkHidden("profile-cpu") - cmd.Flags().MarkHidden("profile-mem") - cmd.Flags().MarkHidden("profile-mutex") - - cmd.Flags().StringSlice("disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)") - - cmd.Flags().Bool("minify", false, "minify any supported output format (HTML, XML etc.)") - - // Set bash-completion. - // Each flag must first be defined before using the SetAnnotation() call. - _ = cmd.Flags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{}) - _ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{}) - _ = cmd.Flags().SetAnnotation("destination", cobra.BashCompSubdirsInDir, []string{}) - _ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"}) -} - -func checkErr(logger loggers.Logger, err error, s ...string) { - if err == nil { - return - } - for _, message := range s { - logger.Errorln(message) - } - logger.Errorln(err) } diff --git a/commands/commands_test.go b/commands/commands_test.go deleted file mode 100644 index 35621854f..000000000 --- a/commands/commands_test.go +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/gohugoio/hugo/config" - - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/common/types" - - "github.com/spf13/cobra" - - qt "github.com/frankban/quicktest" -) - -func TestExecute(t *testing.T) { - c := qt.New(t) - - createSite := func(c *qt.C) string { - dir := createSimpleTestSite(t, testSiteConfig{}) - return dir - } - - c.Run("hugo", func(c *qt.C) { - dir := createSite(c) - resp := Execute([]string{"-s=" + dir}) - c.Assert(resp.Err, qt.IsNil) - result := resp.Result - c.Assert(len(result.Sites) == 1, qt.Equals, true) - c.Assert(len(result.Sites[0].RegularPages()) == 2, qt.Equals, true) - c.Assert(result.Sites[0].Info.Params()["myparam"], qt.Equals, "paramproduction") - }) - - c.Run("hugo, set environment", func(c *qt.C) { - dir := createSite(c) - resp := Execute([]string{"-s=" + dir, "-e=staging"}) - c.Assert(resp.Err, qt.IsNil) - result := resp.Result - c.Assert(result.Sites[0].Info.Params()["myparam"], qt.Equals, "paramstaging") - }) - - c.Run("convert toJSON", func(c *qt.C) { - dir := createSite(c) - output := filepath.Join(dir, "myjson") - resp := Execute([]string{"convert", "toJSON", "-s=" + dir, "-e=staging", "-o=" + output}) - c.Assert(resp.Err, qt.IsNil) - converted := readFileFrom(c, filepath.Join(output, "content", "p1.md")) - c.Assert(converted, qt.Equals, "{\n \"title\": \"P1\",\n \"weight\": 1\n}\n\nContent\n\n", qt.Commentf(converted)) - }) - - c.Run("config, set environment", func(c *qt.C) { - dir := createSite(c) - out, err := captureStdout(func() error { - resp := Execute([]string{"config", "-s=" + dir, "-e=staging"}) - return resp.Err - }) - c.Assert(err, qt.IsNil) - c.Assert(out, qt.Contains, "params = map[myparam:paramstaging]", qt.Commentf(out)) - }) - - c.Run("deploy, environment set", func(c *qt.C) { - dir := createSite(c) - resp := Execute([]string{"deploy", "-s=" + dir, "-e=staging", "--target=mydeployment", "--dryRun"}) - c.Assert(resp.Err, qt.Not(qt.IsNil)) - c.Assert(resp.Err.Error(), qt.Contains, `no driver registered for "hugocloud"`) - }) - - c.Run("list", func(c *qt.C) { - dir := createSite(c) - out, err := captureStdout(func() error { - resp := Execute([]string{"list", "all", "-s=" + dir, "-e=staging"}) - return resp.Err - }) - c.Assert(err, qt.IsNil) - c.Assert(out, qt.Contains, "p1.md") - }) - - c.Run("new theme", func(c *qt.C) { - dir := createSite(c) - themesDir := filepath.Join(dir, "mythemes") - resp := Execute([]string{"new", "theme", "mytheme", "-s=" + dir, "-e=staging", "--themesDir=" + themesDir}) - c.Assert(resp.Err, qt.IsNil) - themeTOML := readFileFrom(c, filepath.Join(themesDir, "mytheme", "theme.toml")) - c.Assert(themeTOML, qt.Contains, "name = \"Mytheme\"") - }) - - c.Run("new site", func(c *qt.C) { - dir := createSite(c) - siteDir := filepath.Join(dir, "mysite") - resp := Execute([]string{"new", "site", siteDir, "-e=staging"}) - c.Assert(resp.Err, qt.IsNil) - config := readFileFrom(c, filepath.Join(siteDir, "config.toml")) - c.Assert(config, qt.Contains, "baseURL = 'http://example.org/'") - checkNewSiteInited(c, siteDir) - }) -} - -func checkNewSiteInited(c *qt.C, basepath string) { - paths := []string{ - filepath.Join(basepath, "archetypes"), - filepath.Join(basepath, "assets"), - filepath.Join(basepath, "content"), - filepath.Join(basepath, "data"), - filepath.Join(basepath, "layouts"), - filepath.Join(basepath, "static"), - filepath.Join(basepath, "themes"), - filepath.Join(basepath, "config.toml"), - } - - for _, path := range paths { - _, err := os.Stat(path) - c.Assert(err, qt.IsNil) - } -} - -func readFileFrom(c *qt.C, filename string) string { - c.Helper() - filename = filepath.Clean(filename) - b, err := afero.ReadFile(hugofs.Os, filename) - c.Assert(err, qt.IsNil) - return string(b) -} - -func TestFlags(t *testing.T) { - c := qt.New(t) - - noOpRunE := func(cmd *cobra.Command, args []string) error { - return nil - } - - tests := []struct { - name string - args []string - check func(c *qt.C, cmd *serverCmd) - }{ - { - // https://github.com/gohugoio/hugo/issues/7642 - name: "ignoreVendorPaths", - args: []string{"server", "--ignoreVendorPaths=github.com/**"}, - check: func(c *qt.C, cmd *serverCmd) { - cfg := config.NewWithTestDefaults() - cmd.flagsToConfig(cfg) - c.Assert(cfg.Get("ignoreVendorPaths"), qt.Equals, "github.com/**") - }, - }, - { - name: "Persistent flags", - args: []string{ - "server", - "--config=myconfig.toml", - "--configDir=myconfigdir", - "--contentDir=mycontent", - "--disableKinds=page,home", - "--environment=testing", - "--configDir=myconfigdir", - "--layoutDir=mylayouts", - "--theme=mytheme", - "--gc", - "--themesDir=mythemes", - "--cleanDestinationDir", - "--navigateToChanged", - "--disableLiveReload", - "--noHTTPCache", - "--printI18nWarnings", - "--destination=/tmp/mydestination", - "-b=https://example.com/b/", - "--port=1366", - "--renderToDisk", - "--source=mysource", - "--printPathWarnings", - "--printUnusedTemplates", - }, - check: func(c *qt.C, sc *serverCmd) { - c.Assert(sc, qt.Not(qt.IsNil)) - c.Assert(sc.navigateToChanged, qt.Equals, true) - c.Assert(sc.disableLiveReload, qt.Equals, true) - c.Assert(sc.noHTTPCache, qt.Equals, true) - c.Assert(sc.renderToDisk, qt.Equals, true) - c.Assert(sc.serverPort, qt.Equals, 1366) - c.Assert(sc.environment, qt.Equals, "testing") - - cfg := config.NewWithTestDefaults() - sc.flagsToConfig(cfg) - c.Assert(cfg.GetString("publishDir"), qt.Equals, "/tmp/mydestination") - c.Assert(cfg.GetString("contentDir"), qt.Equals, "mycontent") - c.Assert(cfg.GetString("layoutDir"), qt.Equals, "mylayouts") - c.Assert(cfg.GetStringSlice("theme"), qt.DeepEquals, []string{"mytheme"}) - c.Assert(cfg.GetString("themesDir"), qt.Equals, "mythemes") - c.Assert(cfg.GetString("baseURL"), qt.Equals, "https://example.com/b/") - - c.Assert(cfg.Get("disableKinds"), qt.DeepEquals, []string{"page", "home"}) - - c.Assert(cfg.GetBool("gc"), qt.Equals, true) - - // The flag is named printPathWarnings - c.Assert(cfg.GetBool("logPathWarnings"), qt.Equals, true) - - // The flag is named printI18nWarnings - c.Assert(cfg.GetBool("logI18nWarnings"), qt.Equals, true) - }, - }, - } - - for _, test := range tests { - c.Run(test.name, func(c *qt.C) { - b := newCommandsBuilder() - root := b.addAll().build() - - for _, cmd := range b.commands { - if cmd.getCommand() == nil { - continue - } - // We are only interested in the flag handling here. - cmd.getCommand().RunE = noOpRunE - } - rootCmd := root.getCommand() - rootCmd.SetArgs(test.args) - c.Assert(rootCmd.Execute(), qt.IsNil) - test.check(c, b.commands[0].(*serverCmd)) - }) - } -} - -func TestCommandsExecute(t *testing.T) { - c := qt.New(t) - - dir := createSimpleTestSite(t, testSiteConfig{}) - dirOut := t.TempDir() - - sourceFlag := fmt.Sprintf("-s=%s", dir) - - tests := []struct { - commands []string - flags []string - expectErrToContain string - }{ - // TODO(bep) permission issue on my OSX? "operation not permitted" {[]string{"check", "ulimit"}, nil, false}, - {[]string{"env"}, nil, ""}, - {[]string{"version"}, nil, ""}, - // no args = hugo build - {nil, []string{sourceFlag}, ""}, - {nil, []string{sourceFlag, "--renderToMemory"}, ""}, - {[]string{"completion", "bash"}, nil, ""}, - {[]string{"completion", "fish"}, nil, ""}, - {[]string{"completion", "powershell"}, nil, ""}, - {[]string{"completion", "zsh"}, nil, ""}, - {[]string{"config"}, []string{sourceFlag}, ""}, - {[]string{"convert", "toTOML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "toml")}, ""}, - {[]string{"convert", "toYAML"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "yaml")}, ""}, - {[]string{"convert", "toJSON"}, []string{sourceFlag, "-o=" + filepath.Join(dirOut, "json")}, ""}, - {[]string{"gen", "chromastyles"}, []string{"--style=manni"}, ""}, - {[]string{"gen", "doc"}, []string{"--dir=" + filepath.Join(dirOut, "doc")}, ""}, - {[]string{"gen", "man"}, []string{"--dir=" + filepath.Join(dirOut, "man")}, ""}, - {[]string{"list", "drafts"}, []string{sourceFlag}, ""}, - {[]string{"list", "expired"}, []string{sourceFlag}, ""}, - {[]string{"list", "future"}, []string{sourceFlag}, ""}, - {[]string{"new", "new-page.md"}, []string{sourceFlag}, ""}, - {[]string{"new", "site", filepath.Join(dirOut, "new-site")}, nil, ""}, - {[]string{"unknowncommand"}, nil, "unknown command"}, - // TODO(bep) cli refactor fix https://github.com/gohugoio/hugo/issues/4450 - //{[]string{"new", "theme", filepath.Join(dirOut, "new-theme")}, nil,false}, - } - - for _, test := range tests { - name := "hugo" - if len(test.commands) > 0 { - name = test.commands[0] - } - c.Run(name, func(c *qt.C) { - b := newCommandsBuilder().addAll().build() - hugoCmd := b.getCommand() - test.flags = append(test.flags, "--quiet") - hugoCmd.SetArgs(append(test.commands, test.flags...)) - - // TODO(bep) capture output and add some simple asserts - // TODO(bep) misspelled subcommands does not return an error. We should investigate this - // but before that, check for "Error: unknown command". - - _, err := hugoCmd.ExecuteC() - if test.expectErrToContain != "" { - c.Assert(err, qt.Not(qt.IsNil)) - c.Assert(err.Error(), qt.Contains, test.expectErrToContain) - } else { - c.Assert(err, qt.IsNil) - } - - // Assert that we have not left any development debug artifacts in - // the code. - if b.c != nil { - _, ok := b.c.publishDirFs.(types.DevMarker) - c.Assert(ok, qt.Equals, false) - } - }) - - } -} - -type testSiteConfig struct { - configTOML string - contentDir string -} - -func createSimpleTestSite(t testing.TB, cfg testSiteConfig) string { - dir := t.TempDir() - - cfgStr := ` - -baseURL = "https://example.org" -title = "Hugo Commands" - - -` - - contentDir := "content" - - if cfg.configTOML != "" { - cfgStr = cfg.configTOML - } - if cfg.contentDir != "" { - contentDir = cfg.contentDir - } - - os.MkdirAll(filepath.Join(dir, "public"), 0777) - - // Just the basic. These are for CLI tests, not site testing. - writeFile(t, filepath.Join(dir, "config.toml"), cfgStr) - writeFile(t, filepath.Join(dir, "config", "staging", "params.toml"), `myparam="paramstaging"`) - writeFile(t, filepath.Join(dir, "config", "staging", "deployment.toml"), ` -[[targets]] -name = "mydeployment" -URL = "hugocloud://hugotestbucket" -`) - - writeFile(t, filepath.Join(dir, "config", "testing", "params.toml"), `myparam="paramtesting"`) - writeFile(t, filepath.Join(dir, "config", "production", "params.toml"), `myparam="paramproduction"`) - - writeFile(t, filepath.Join(dir, "static", "myfile.txt"), `Hello World!`) - - writeFile(t, filepath.Join(dir, contentDir, "p1.md"), ` ---- -title: "P1" -weight: 1 ---- - -Content - -`) - - writeFile(t, filepath.Join(dir, contentDir, "hügö.md"), ` ---- -weight: 2 ---- - -This is hügö. - -`) - - writeFile(t, filepath.Join(dir, "layouts", "_default", "single.html"), ` - -Single: {{ .Title }}|{{ .Content }} - -`) - - writeFile(t, filepath.Join(dir, "layouts", "404.html"), ` -404: {{ .Title }}|Not Found. - -`) - - writeFile(t, filepath.Join(dir, "layouts", "_default", "list.html"), ` - -List: {{ .Title }} -Environment: {{ hugo.Environment }} - -For issue 9788: -{{ $foo :="abc" | resources.FromString "foo.css" | minify | resources.PostProcess }} -PostProcess: {{ $foo.RelPermalink }} - -`) - - return dir -} - -func writeFile(t testing.TB, filename, content string) { - must(t, os.MkdirAll(filepath.Dir(filename), os.FileMode(0755))) - must(t, os.WriteFile(filename, []byte(content), os.FileMode(0755))) -} - -func must(t testing.TB, err error) { - if err != nil { - t.Fatal(err) - } -} diff --git a/commands/config.go b/commands/config.go index a5d8aab22..6f0a29b35 100644 --- a/commands/config.go +++ b/commands/config.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -9,129 +9,93 @@ // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and -// limitations under the License.Print the version number of Hug +// limitations under the License. package commands import ( + "context" "encoding/json" - "fmt" "os" - "reflect" - "regexp" - "sort" - "strings" "time" - "github.com/gohugoio/hugo/common/maps" - + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" - - "github.com/gohugoio/hugo/modules" - "github.com/spf13/cobra" ) -var _ cmder = (*configCmd)(nil) - -type configCmd struct { - *baseBuilderCmd -} - -func (b *commandsBuilder) newConfigCmd() *configCmd { - cc := &configCmd{} - cmd := &cobra.Command{ - Use: "config", - Short: "Print the site configuration", - Long: `Print the site configuration, both default and custom settings.`, - RunE: cc.printConfig, +// newConfigCommand creates a new config command and its subcommands. +func newConfigCommand() *configCommand { + return &configCommand{ + commands: []simplecobra.Commander{ + &configMountsCommand{}, + }, } - printMountsCmd := &cobra.Command{ - Use: "mounts", - Short: "Print the configured file mounts", - RunE: cc.printMounts, - } - - cmd.AddCommand(printMountsCmd) - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc } -func (c *configCmd) printMounts(cmd *cobra.Command, args []string) error { - cfg, err := initializeConfig(true, false, false, &c.hugoBuilderCommon, c, nil) +type configCommand struct { + r *rootCommand + + commands []simplecobra.Commander +} + +func (c *configCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *configCommand) Name() string { + return "config" +} + +func (c *configCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, nil)) if err != nil { return err } + config := conf.configs.Base - allModules := cfg.Cfg.Get("allmodules").(modules.Modules) + // Print it as JSON. + dec := json.NewEncoder(os.Stdout) + dec.SetIndent("", " ") + dec.SetEscapeHTML(false) - for _, m := range allModules { - if err := parser.InterfaceToConfig(&modMounts{m: m, verbose: c.verbose}, metadecoders.JSON, os.Stdout); err != nil { - return err - } + if err := dec.Encode(parser.ReplacingJSONMarshaller{Value: config, KeysToLower: true, OmitEmpty: true}); err != nil { + return err } return nil } -func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error { - cfg, err := initializeConfig(true, false, false, &c.hugoBuilderCommon, c, nil) - if err != nil { - return err - } - - allSettings := cfg.Cfg.Get("").(maps.Params) - - // We need to clean up this, but we store objects in the config that - // isn't really interesting to the end user, so filter these. - ignoreKeysRe := regexp.MustCompile("client|sorted|filecacheconfigs|allmodules|multilingual") - - separator := ": " - - if len(cfg.configFiles) > 0 && strings.HasSuffix(cfg.configFiles[0], ".toml") { - separator = " = " - } - - var keys []string - for k := range allSettings { - if ignoreKeysRe.MatchString(k) { - continue - } - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - kv := reflect.ValueOf(allSettings[k]) - if kv.Kind() == reflect.String { - fmt.Printf("%s%s\"%+v\"\n", k, separator, allSettings[k]) - } else { - fmt.Printf("%s%s%+v\n", k, separator, allSettings[k]) - } - } - +func (c *configCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Print the site configuration" + cmd.Long = `Print the site configuration, both default and custom settings.` return nil } -type modMounts struct { - verbose bool - m modules.Module +func (c *configCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + return nil } -type modMount struct { +type configModMount struct { Source string `json:"source"` Target string `json:"target"` Lang string `json:"lang,omitempty"` } +type configModMounts struct { + verbose bool + m modules.Module +} + // MarshalJSON is for internal use only. -func (m *modMounts) MarshalJSON() ([]byte, error) { - var mounts []modMount +func (m *configModMounts) MarshalJSON() ([]byte, error) { + var mounts []configModMount for _, mount := range m.m.Mounts() { - mounts = append(mounts, modMount{ + mounts = append(mounts, configModMount{ Source: mount.Source, Target: mount.Target, Lang: mount.Lang, @@ -154,7 +118,7 @@ func (m *modMounts) MarshalJSON() ([]byte, error) { Meta map[string]any `json:"meta"` HugoVersion modules.HugoVersion `json:"hugoVersion"` - Mounts []modMount `json:"mounts"` + Mounts []configModMount `json:"mounts"` }{ Path: m.m.Path(), Version: m.m.Version(), @@ -168,12 +132,12 @@ func (m *modMounts) MarshalJSON() ([]byte, error) { } return json.Marshal(&struct { - Path string `json:"path"` - Version string `json:"version"` - Time time.Time `json:"time"` - Owner string `json:"owner"` - Dir string `json:"dir"` - Mounts []modMount `json:"mounts"` + Path string `json:"path"` + Version string `json:"version"` + Time time.Time `json:"time"` + Owner string `json:"owner"` + Dir string `json:"dir"` + Mounts []configModMount `json:"mounts"` }{ Path: m.m.Path(), Version: m.m.Version(), @@ -184,3 +148,40 @@ func (m *modMounts) MarshalJSON() ([]byte, error) { }) } + +type configMountsCommand struct { + configCmd *configCommand +} + +func (c *configMountsCommand) Commands() []simplecobra.Commander { + return nil +} + +func (c *configMountsCommand) Name() string { + return "mounts" +} + +func (c *configMountsCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + r := c.configCmd.r + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, nil)) + if err != nil { + return err + } + + for _, m := range conf.configs.Modules { + if err := parser.InterfaceToConfig(&configModMounts{m: m, verbose: r.verbose}, metadecoders.JSON, os.Stdout); err != nil { + return err + } + } + return nil +} + +func (c *configMountsCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Print the configured file mounts" + return nil +} + +func (c *configMountsCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.configCmd = cd.Parent.Command.(*configCommand) + return nil +} diff --git a/commands/convert.go b/commands/convert.go index 1ec965a0b..0cae5ad7e 100644 --- a/commands/convert.go +++ b/commands/convert.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,122 +15,119 @@ package commands import ( "bytes" + "context" "fmt" "path/filepath" "strings" "time" - "github.com/gohugoio/hugo/parser/pageparser" - - "github.com/gohugoio/hugo/resources/page" - - "github.com/gohugoio/hugo/hugofs" - + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" - + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugolib" "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" - - "github.com/gohugoio/hugo/hugolib" - + "github.com/gohugoio/hugo/parser/pageparser" + "github.com/gohugoio/hugo/resources/page" "github.com/spf13/cobra" ) -var _ cmder = (*convertCmd)(nil) +func newConvertCommand() *convertCommand { + var c *convertCommand + c = &convertCommand{ + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "toJSON", + short: "Convert front matter to JSON", + long: `toJSON converts all front matter in the content directory +to use JSON for the front matter.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + return c.convertContents(metadecoders.JSON) + }, + withc: func(cmd *cobra.Command) { + }, + }, + &simpleCommand{ + name: "toTOML", + short: "Convert front matter to TOML", + long: `toTOML converts all front matter in the content directory +to use TOML for the front matter.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + return c.convertContents(metadecoders.TOML) + }, + withc: func(cmd *cobra.Command) { + }, + }, + &simpleCommand{ + name: "toYAML", + short: "Convert front matter to YAML", + long: `toYAML converts all front matter in the content directory +to use YAML for the front matter.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + return c.convertContents(metadecoders.YAML) + }, + withc: func(cmd *cobra.Command) { + }, + }, + }, + } + return c +} -type convertCmd struct { +type convertCommand struct { + // Flags. outputDir string unsafe bool - *baseBuilderCmd + // Deps. + r *rootCommand + h *hugolib.HugoSites + + // Commmands. + commands []simplecobra.Commander } -func (b *commandsBuilder) newConvertCmd() *convertCmd { - cc := &convertCmd{} - - cmd := &cobra.Command{ - Use: "convert", - Short: "Convert your content to different formats", - Long: `Convert your content (e.g. front matter) to different formats. - -See convert's subcommands toJSON, toTOML and toYAML for more information.`, - RunE: nil, - } - - cmd.AddCommand( - &cobra.Command{ - Use: "toJSON", - Short: "Convert front matter to JSON", - Long: `toJSON converts all front matter in the content directory -to use JSON for the front matter.`, - RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(metadecoders.JSON) - }, - }, - &cobra.Command{ - Use: "toTOML", - Short: "Convert front matter to TOML", - Long: `toTOML converts all front matter in the content directory -to use TOML for the front matter.`, - RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(metadecoders.TOML) - }, - }, - &cobra.Command{ - Use: "toYAML", - Short: "Convert front matter to YAML", - Long: `toYAML converts all front matter in the content directory -to use YAML for the front matter.`, - RunE: func(cmd *cobra.Command, args []string) error { - return cc.convertContents(metadecoders.YAML) - }, - }, - ) - - cmd.PersistentFlags().StringVarP(&cc.outputDir, "output", "o", "", "filesystem path to write files to") - cmd.PersistentFlags().BoolVar(&cc.unsafe, "unsafe", false, "enable less safe operations, please backup first") - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc +func (c *convertCommand) Commands() []simplecobra.Commander { + return c.commands } -func (cc *convertCmd) convertContents(format metadecoders.Format) error { - if cc.outputDir == "" && !cc.unsafe { - return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") - } +func (c *convertCommand) Name() string { + return "convert" +} - c, err := initializeConfig(true, false, false, &cc.hugoBuilderCommon, cc, nil) - if err != nil { - return err - } - - c.Cfg.Set("buildDrafts", true) - - h, err := hugolib.NewHugoSites(*c.DepsCfg) - if err != nil { - return err - } - - if err := h.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return err - } - - site := h.Sites[0] - - site.Log.Println("processing", len(site.AllPages()), "content files") - for _, p := range site.AllPages() { - if err := cc.convertAndSavePage(p, site, format); err != nil { - return err - } - } +func (c *convertCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { return nil } -func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, targetFormat metadecoders.Format) error { +func (c *convertCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Convert your content to different formats" + cmd.Long = `Convert your content (e.g. front matter) to different formats. + +See convert's subcommands toJSON, toTOML and toYAML for more information.` + + cmd.PersistentFlags().StringVarP(&c.outputDir, "output", "o", "", "filesystem path to write files to") + cmd.PersistentFlags().BoolVar(&c.unsafe, "unsafe", false, "enable less safe operations, please backup first") + + return nil +} + +func (c *convertCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + cfg := config.New() + cfg.Set("buildDrafts", true) + h, err := c.r.Hugo(flagsToCfg(cd, cfg)) + if err != nil { + return err + } + c.h = h + return nil +} + +func (c *convertCommand) convertAndSavePage(p page.Page, site *hugolib.Site, targetFormat metadecoders.Format) error { // The resources are not in .Site.AllPages. for _, r := range p.Resources().ByType("page") { - if err := cc.convertAndSavePage(r.(page.Page), site, targetFormat); err != nil { + if err := c.convertAndSavePage(r.(page.Page), site, targetFormat); err != nil { return err } } @@ -140,9 +137,9 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target return nil } - errMsg := fmt.Errorf("Error processing file %q", p.File().Path()) + errMsg := fmt.Errorf("error processing file %q", p.File().Path()) - site.Log.Infoln("Attempting to convert", p.File().Filename()) + site.Log.Infoln("ttempting to convert", p.File().Filename()) f := p.File() file, err := f.FileInfo().Meta().Open() @@ -182,26 +179,45 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target newFilename := p.File().Filename() - if cc.outputDir != "" { + if c.outputDir != "" { contentDir := strings.TrimSuffix(newFilename, p.File().Path()) contentDir = filepath.Base(contentDir) - newFilename = filepath.Join(cc.outputDir, contentDir, p.File().Path()) + newFilename = filepath.Join(c.outputDir, contentDir, p.File().Path()) } fs := hugofs.Os if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil { - return fmt.Errorf("Failed to save file %q:: %w", newFilename, err) + return fmt.Errorf("failed to save file %q:: %w", newFilename, err) } return nil } -type parsedFile struct { - frontMatterFormat metadecoders.Format - frontMatterSource []byte - frontMatter map[string]any +func (c *convertCommand) convertContents(format metadecoders.Format) error { + if c.outputDir == "" && !c.unsafe { + return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path") + } - // Everything after Front Matter - content []byte + if err := c.h.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { + return err + } + + site := c.h.Sites[0] + + var pagesBackedByFile page.Pages + for _, p := range site.AllPages() { + if p.File().IsZero() { + continue + } + pagesBackedByFile = append(pagesBackedByFile, p) + } + + site.Log.Println("processing", len(pagesBackedByFile), "content files") + for _, p := range site.AllPages() { + if err := c.convertAndSavePage(p, site, format); err != nil { + return err + } + } + return nil } diff --git a/commands/deploy.go b/commands/deploy.go index 295940c2e..0340ea3c4 100644 --- a/commands/deploy.go +++ b/commands/deploy.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,76 +14,58 @@ //go:build !nodeploy // +build !nodeploy +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package commands import ( "context" + "github.com/bep/simplecobra" "github.com/gohugoio/hugo/deploy" "github.com/spf13/cobra" ) -var _ cmder = (*deployCmd)(nil) +func newDeployCommand() simplecobra.Commander { -// deployCmd supports deploying sites to Cloud providers. -type deployCmd struct { - *baseBuilderCmd - - invalidateCDN bool - maxDeletes int - workers int -} - -// TODO: In addition to the "deploy" command, consider adding a "--deploy" -// flag for the default command; this would build the site and then deploy it. -// It's not obvious how to do this; would all of the deploy-specific flags -// have to exist at the top level as well? - -// TODO: The output files change every time "hugo" is executed, it looks -// like because of map order randomization. This means that you can -// run "hugo && hugo deploy" again and again and upload new stuff every time. Is -// this intended? - -func (b *commandsBuilder) newDeployCmd() *deployCmd { - cc := &deployCmd{} - - cmd := &cobra.Command{ - Use: "deploy", - Short: "Deploy your site to a Cloud provider.", - Long: `Deploy your site to a Cloud provider. + return &simpleCommand{ + name: "deploy", + short: "Deploy your site to a Cloud provider.", + long: `Deploy your site to a Cloud provider. See https://gohugo.io/hosting-and-deployment/hugo-deploy/ for detailed documentation. `, - - RunE: func(cmd *cobra.Command, args []string) error { - cfgInit := func(c *commandeer) error { - c.Set("invalidateCDN", cc.invalidateCDN) - c.Set("maxDeletes", cc.maxDeletes) - c.Set("workers", cc.workers) - return nil - } - comm, err := initializeConfig(true, true, false, &cc.hugoBuilderCommon, cc, cfgInit) + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfgWithAdditionalConfigBase(cd, nil, "deployment")) if err != nil { return err } - deployer, err := deploy.New(comm.Cfg, comm.hugo().PathSpec.PublishFs) + deployer, err := deploy.New(h.Configs.GetFirstLanguageConfig(), h.PathSpec.PublishFs) if err != nil { return err } - return deployer.Deploy(context.Background()) + return deployer.Deploy(ctx) + }, + withc: func(cmd *cobra.Command) { + cmd.Flags().String("target", "", "target deployment from deployments section in config file; defaults to the first one") + cmd.Flags().Bool("confirm", false, "ask for confirmation before making changes to the target") + cmd.Flags().Bool("dryRun", false, "dry run") + cmd.Flags().Bool("force", false, "force upload of all files") + cmd.Flags().Bool("invalidateCDN", true, "invalidate the CDN cache listed in the deployment target") + cmd.Flags().Int("maxDeletes", 256, "maximum # of files to delete, or -1 to disable") + cmd.Flags().Int("workers", 10, "number of workers to transfer files. defaults to 10") }, } - - cmd.Flags().String("target", "", "target deployment from deployments section in config file; defaults to the first one") - cmd.Flags().Bool("confirm", false, "ask for confirmation before making changes to the target") - cmd.Flags().Bool("dryRun", false, "dry run") - cmd.Flags().Bool("force", false, "force upload of all files") - cmd.Flags().BoolVar(&cc.invalidateCDN, "invalidateCDN", true, "invalidate the CDN cache listed in the deployment target") - cmd.Flags().IntVar(&cc.maxDeletes, "maxDeletes", 256, "maximum # of files to delete, or -1 to disable") - cmd.Flags().IntVar(&cc.workers, "workers", 10, "number of workers to transfer files. defaults to 10") - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc } diff --git a/commands/deploy_off.go b/commands/deploy_off.go new file mode 100644 index 000000000..5e9b91f16 --- /dev/null +++ b/commands/deploy_off.go @@ -0,0 +1,48 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build nodeploy +// +build nodeploy + +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "errors" + + "github.com/spf13/cobra" +) + +func newDeployCommand() simplecobra.Commander { + return &simpleCommand{ + name: "deploy", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + return nil + }, + withc: func(cmd *cobra.Command) { + cmd.Hidden = true + }, + } +} diff --git a/commands/env.go b/commands/env.go index 0fc509d6d..a6db551e9 100644 --- a/commands/env.go +++ b/commands/env.go @@ -1,4 +1,4 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,55 +14,50 @@ package commands import ( + "context" "runtime" + "github.com/bep/simplecobra" "github.com/gohugoio/hugo/common/hugo" - - "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" ) -var _ cmder = (*envCmd)(nil) - -type envCmd struct { - *baseCmd -} - -func newEnvCmd() *envCmd { - return &envCmd{ - baseCmd: newBaseCmd(&cobra.Command{ - Use: "env", - Short: "Print Hugo version and environment info", - Long: `Print Hugo version and environment info. This is useful in Hugo bug reports. - -If you add the -v flag, you will get a full dependency list. -`, - RunE: func(cmd *cobra.Command, args []string) error { - printHugoVersion() - jww.FEEDBACK.Printf("GOOS=%q\n", runtime.GOOS) - jww.FEEDBACK.Printf("GOARCH=%q\n", runtime.GOARCH) - jww.FEEDBACK.Printf("GOVERSION=%q\n", runtime.Version()) - - isVerbose, _ := cmd.Flags().GetBool("verbose") - - if isVerbose { - deps := hugo.GetDependencyList() - for _, dep := range deps { - jww.FEEDBACK.Printf("%s\n", dep) - } - } else { - // These are also included in the GetDependencyList above; - // always print these as these are most likely the most useful to know about. - deps := hugo.GetDependencyListNonGo() - for _, dep := range deps { - jww.FEEDBACK.Printf("%s\n", dep) - } +func newEnvCommand() simplecobra.Commander { + return &simpleCommand{ + name: "env", + short: "Print Hugo version and environment info", + long: "Print Hugo version and environment info. This is useful in Hugo bug reports", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + r.Printf("%s\n", hugo.BuildVersionString()) + r.Printf("GOOS=%q\n", runtime.GOOS) + r.Printf("GOARCH=%q\n", runtime.GOARCH) + r.Printf("GOVERSION=%q\n", runtime.Version()) + if r.verbose { + deps := hugo.GetDependencyList() + for _, dep := range deps { + r.Printf("%s\n", dep) } - - return nil - }, - }), + } else { + // These are also included in the GetDependencyList above; + // always print these as these are most likely the most useful to know about. + deps := hugo.GetDependencyListNonGo() + for _, dep := range deps { + r.Printf("%s\n", dep) + } + } + return nil + }, + } +} + +func newVersionCmd() simplecobra.Commander { + return &simpleCommand{ + name: "version", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + r.Println(hugo.BuildVersionString()) + return nil + }, + short: "Print Hugo version and environment info", + long: "Print Hugo version and environment info. This is useful in Hugo bug reports.", } - } diff --git a/commands/gen.go b/commands/gen.go index c44eba36c..7ff75372a 100644 --- a/commands/gen.go +++ b/commands/gen.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,27 +14,200 @@ package commands import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/alecthomas/chroma/v2/styles" + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/spf13/cobra" + "github.com/spf13/cobra/doc" ) -var _ cmder = (*genCmd)(nil) +func newGenCommand() *genCommand { + var ( + // Flags. + gendocdir string + genmandir string + + // Chroma flags. + style string + highlightStyle string + linesStyle string + ) + + newChromaStyles := func() simplecobra.Commander { + return &simpleCommand{ + name: "chromastyles", + short: "Generate CSS stylesheet for the Chroma code highlighter", + long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if markup.highlight.noClasses is disabled in config. + +See https://xyproto.github.io/splash/docs/all.html for a preview of the available styles`, + + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + builder := styles.Get(style).Builder() + if highlightStyle != "" { + builder.Add(chroma.LineHighlight, highlightStyle) + } + if linesStyle != "" { + builder.Add(chroma.LineNumbers, linesStyle) + } + style, err := builder.Build() + if err != nil { + return err + } + formatter := html.New(html.WithAllClasses(true)) + formatter.WriteCSS(os.Stdout, style) + return nil + }, + withc: func(cmd *cobra.Command) { + cmd.PersistentFlags().StringVar(&style, "style", "friendly", "highlighter style (see https://xyproto.github.io/splash/docs/)") + cmd.PersistentFlags().StringVar(&highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)") + cmd.PersistentFlags().StringVar(&linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)") + }, + } + } + + newMan := func() simplecobra.Commander { + return &simpleCommand{ + name: "man", + short: "Generate man pages for the Hugo CLI", + long: `This command automatically generates up-to-date man pages of Hugo's + command-line interface. By default, it creates the man page files + in the "man" directory under the current directory.`, + + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + header := &doc.GenManHeader{ + Section: "1", + Manual: "Hugo Manual", + Source: fmt.Sprintf("Hugo %s", hugo.CurrentVersion), + } + if !strings.HasSuffix(genmandir, helpers.FilePathSeparator) { + genmandir += helpers.FilePathSeparator + } + if found, _ := helpers.Exists(genmandir, hugofs.Os); !found { + r.Println("Directory", genmandir, "does not exist, creating...") + if err := hugofs.Os.MkdirAll(genmandir, 0777); err != nil { + return err + } + } + cd.CobraCommand.Root().DisableAutoGenTag = true + + r.Println("Generating Hugo man pages in", genmandir, "...") + doc.GenManTree(cd.CobraCommand.Root(), header, genmandir) + + r.Println("Done.") + + return nil + }, + withc: func(cmd *cobra.Command) { + cmd.PersistentFlags().StringVar(&genmandir, "dir", "man/", "the directory to write the man pages.") + // For bash-completion + cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{}) + }, + } + } + + newGen := func() simplecobra.Commander { + const gendocFrontmatterTemplate = `--- +title: "%s" +slug: %s +url: %s +--- +` + + return &simpleCommand{ + name: "doc", + short: "Generate Markdown documentation for the Hugo CLI.", + long: `Generate Markdown documentation for the Hugo CLI. + This command is, mostly, used to create up-to-date documentation + of Hugo's command-line interface for https://gohugo.io/. + + It creates one Markdown file per command with front matter suitable + for rendering in Hugo.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + cd.CobraCommand.VisitParents(func(c *cobra.Command) { + // Disable the "Auto generated by spf13/cobra on DATE" + // as it creates a lot of diffs. + c.DisableAutoGenTag = true + }) + if !strings.HasSuffix(gendocdir, helpers.FilePathSeparator) { + gendocdir += helpers.FilePathSeparator + } + if found, _ := helpers.Exists(gendocdir, hugofs.Os); !found { + r.Println("Directory", gendocdir, "does not exist, creating...") + if err := hugofs.Os.MkdirAll(gendocdir, 0777); err != nil { + return err + } + } + prepender := func(filename string) string { + name := filepath.Base(filename) + base := strings.TrimSuffix(name, path.Ext(name)) + url := "/commands/" + strings.ToLower(base) + "/" + return fmt.Sprintf(gendocFrontmatterTemplate, strings.Replace(base, "_", " ", -1), base, url) + } + + linkHandler := func(name string) string { + base := strings.TrimSuffix(name, path.Ext(name)) + return "/commands/" + strings.ToLower(base) + "/" + } + r.Println("Generating Hugo command-line documentation in", gendocdir, "...") + doc.GenMarkdownTreeCustom(cd.CobraCommand.Root(), gendocdir, prepender, linkHandler) + r.Println("Done.") + + return nil + }, + withc: func(cmd *cobra.Command) { + cmd.PersistentFlags().StringVar(&gendocdir, "dir", "/tmp/hugodoc/", "the directory to write the doc.") + // For bash-completion + cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{}) + }, + } + + } + + return &genCommand{ + commands: []simplecobra.Commander{ + newChromaStyles(), + newGen(), + newMan(), + }, + } -type genCmd struct { - *baseCmd } -func newGenCmd() *genCmd { - cc := &genCmd{} - cc.baseCmd = newBaseCmd(&cobra.Command{ - Use: "gen", - Short: "A collection of several useful generators.", - }) +type genCommand struct { + rootCmd *rootCommand - cc.cmd.AddCommand( - newGenDocCmd().getCommand(), - newGenManCmd().getCommand(), - createGenDocsHelper().getCommand(), - createGenChromaStyles().getCommand()) - - return cc + commands []simplecobra.Commander +} + +func (c *genCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *genCommand) Name() string { + return "gen" +} + +func (c *genCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + return nil +} + +func (c *genCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "A collection of several useful generators." + return nil +} + +func (c *genCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.rootCmd = cd.Root.Command.(*rootCommand) + return nil } diff --git a/commands/genchromastyles.go b/commands/genchromastyles.go deleted file mode 100644 index 4dfa77d2e..000000000 --- a/commands/genchromastyles.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "os" - - "github.com/alecthomas/chroma/v2" - "github.com/alecthomas/chroma/v2/formatters/html" - "github.com/alecthomas/chroma/v2/styles" - "github.com/spf13/cobra" -) - -var _ cmder = (*genChromaStyles)(nil) - -type genChromaStyles struct { - style string - highlightStyle string - linesStyle string - *baseCmd -} - -// TODO(bep) highlight -func createGenChromaStyles() *genChromaStyles { - g := &genChromaStyles{ - baseCmd: newBaseCmd(&cobra.Command{ - Use: "chromastyles", - Short: "Generate CSS stylesheet for the Chroma code highlighter", - Long: `Generate CSS stylesheet for the Chroma code highlighter for a given style. This stylesheet is needed if markup.highlight.noClasses is disabled in config. - -See https://xyproto.github.io/splash/docs/all.html for a preview of the available styles`, - }), - } - - g.cmd.RunE = func(cmd *cobra.Command, args []string) error { - return g.generate() - } - - g.cmd.PersistentFlags().StringVar(&g.style, "style", "friendly", "highlighter style (see https://xyproto.github.io/splash/docs/)") - g.cmd.PersistentFlags().StringVar(&g.highlightStyle, "highlightStyle", "bg:#ffffcc", "style used for highlighting lines (see https://github.com/alecthomas/chroma)") - g.cmd.PersistentFlags().StringVar(&g.linesStyle, "linesStyle", "", "style used for line numbers (see https://github.com/alecthomas/chroma)") - - return g -} - -func (g *genChromaStyles) generate() error { - builder := styles.Get(g.style).Builder() - if g.highlightStyle != "" { - builder.Add(chroma.LineHighlight, g.highlightStyle) - } - if g.linesStyle != "" { - builder.Add(chroma.LineNumbers, g.linesStyle) - } - style, err := builder.Build() - if err != nil { - return err - } - formatter := html.New(html.WithAllClasses(true)) - formatter.WriteCSS(os.Stdout, style) - return nil -} diff --git a/commands/gendoc.go b/commands/gendoc.go deleted file mode 100644 index 8ecb0ec0d..000000000 --- a/commands/gendoc.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "fmt" - "path" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/cobra" - "github.com/spf13/cobra/doc" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*genDocCmd)(nil) - -type genDocCmd struct { - gendocdir string - *baseCmd -} - -func newGenDocCmd() *genDocCmd { - const gendocFrontmatterTemplate = `--- -title: "%s" -slug: %s -url: %s ---- -` - - cc := &genDocCmd{} - - cc.baseCmd = newBaseCmd(&cobra.Command{ - Use: "doc", - Short: "Generate Markdown documentation for the Hugo CLI.", - Long: `Generate Markdown documentation for the Hugo CLI. - -This command is, mostly, used to create up-to-date documentation -of Hugo's command-line interface for https://gohugo.io/. - -It creates one Markdown file per command with front matter suitable -for rendering in Hugo.`, - - RunE: func(cmd *cobra.Command, args []string) error { - cmd.VisitParents(func(c *cobra.Command) { - // Disable the "Auto generated by spf13/cobra on DATE" - // as it creates a lot of diffs. - c.DisableAutoGenTag = true - }) - - if !strings.HasSuffix(cc.gendocdir, helpers.FilePathSeparator) { - cc.gendocdir += helpers.FilePathSeparator - } - if found, _ := helpers.Exists(cc.gendocdir, hugofs.Os); !found { - jww.FEEDBACK.Println("Directory", cc.gendocdir, "does not exist, creating...") - if err := hugofs.Os.MkdirAll(cc.gendocdir, 0777); err != nil { - return err - } - } - prepender := func(filename string) string { - name := filepath.Base(filename) - base := strings.TrimSuffix(name, path.Ext(name)) - url := "/commands/" + strings.ToLower(base) + "/" - return fmt.Sprintf(gendocFrontmatterTemplate, strings.Replace(base, "_", " ", -1), base, url) - } - - linkHandler := func(name string) string { - base := strings.TrimSuffix(name, path.Ext(name)) - return "/commands/" + strings.ToLower(base) + "/" - } - jww.FEEDBACK.Println("Generating Hugo command-line documentation in", cc.gendocdir, "...") - doc.GenMarkdownTreeCustom(cmd.Root(), cc.gendocdir, prepender, linkHandler) - jww.FEEDBACK.Println("Done.") - - return nil - }, - }) - - cc.cmd.PersistentFlags().StringVar(&cc.gendocdir, "dir", "/tmp/hugodoc/", "the directory to write the doc.") - - // For bash-completion - cc.cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{}) - - return cc -} diff --git a/commands/gendocshelper.go b/commands/gendocshelper.go deleted file mode 100644 index 34d45154f..000000000 --- a/commands/gendocshelper.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017-present The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/gohugoio/hugo/docshelper" - "github.com/spf13/cobra" -) - -var _ cmder = (*genDocsHelper)(nil) - -type genDocsHelper struct { - target string - *baseCmd -} - -func createGenDocsHelper() *genDocsHelper { - g := &genDocsHelper{ - baseCmd: newBaseCmd(&cobra.Command{ - Use: "docshelper", - Short: "Generate some data files for the Hugo docs.", - Hidden: true, - }), - } - - g.cmd.RunE = func(cmd *cobra.Command, args []string) error { - return g.generate() - } - - g.cmd.PersistentFlags().StringVarP(&g.target, "dir", "", "docs/data", "data dir") - - return g -} - -func (g *genDocsHelper) generate() error { - fmt.Println("Generate docs data to", g.target) - - targetFile := filepath.Join(g.target, "docs.json") - - f, err := os.Create(targetFile) - if err != nil { - return err - } - defer f.Close() - - enc := json.NewEncoder(f) - enc.SetIndent("", " ") - - if err := enc.Encode(docshelper.GetDocProvider()); err != nil { - return err - } - - fmt.Println("Done!") - return nil -} diff --git a/commands/genman.go b/commands/genman.go deleted file mode 100644 index 720046289..000000000 --- a/commands/genman.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2016 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "fmt" - "strings" - - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/cobra" - "github.com/spf13/cobra/doc" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*genManCmd)(nil) - -type genManCmd struct { - genmandir string - *baseCmd -} - -func newGenManCmd() *genManCmd { - cc := &genManCmd{} - - cc.baseCmd = newBaseCmd(&cobra.Command{ - Use: "man", - Short: "Generate man pages for the Hugo CLI", - Long: `This command automatically generates up-to-date man pages of Hugo's -command-line interface. By default, it creates the man page files -in the "man" directory under the current directory.`, - - RunE: func(cmd *cobra.Command, args []string) error { - header := &doc.GenManHeader{ - Section: "1", - Manual: "Hugo Manual", - Source: fmt.Sprintf("Hugo %s", hugo.CurrentVersion), - } - if !strings.HasSuffix(cc.genmandir, helpers.FilePathSeparator) { - cc.genmandir += helpers.FilePathSeparator - } - if found, _ := helpers.Exists(cc.genmandir, hugofs.Os); !found { - jww.FEEDBACK.Println("Directory", cc.genmandir, "does not exist, creating...") - if err := hugofs.Os.MkdirAll(cc.genmandir, 0777); err != nil { - return err - } - } - cmd.Root().DisableAutoGenTag = true - - jww.FEEDBACK.Println("Generating Hugo man pages in", cc.genmandir, "...") - doc.GenManTree(cmd.Root(), header, cc.genmandir) - - jww.FEEDBACK.Println("Done.") - - return nil - }, - }) - - cc.cmd.PersistentFlags().StringVar(&cc.genmandir, "dir", "man/", "the directory to write the man pages.") - - // For bash-completion - cc.cmd.PersistentFlags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{}) - - return cc -} diff --git a/commands/helpers.go b/commands/helpers.go index 71f686953..c342ce2c7 100644 --- a/commands/helpers.go +++ b/commands/helpers.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,16 +11,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package commands defines and implements command-line commands and flags -// used by Hugo. Commands and flags are implemented using Cobra. package commands import ( + "bytes" + "errors" "fmt" - "regexp" + "log" + "os" + "path/filepath" + "strings" + "github.com/bep/simplecobra" "github.com/gohugoio/hugo/config" - "github.com/spf13/cobra" + "github.com/gohugoio/hugo/helpers" + "github.com/spf13/afero" + "github.com/spf13/pflag" ) const ( @@ -30,50 +36,101 @@ const ( showCursor = ansiEsc + "[?25h" ) -type flagsToConfigHandler interface { - flagsToConfig(cfg config.Provider) +func newUserError(a ...any) *simplecobra.CommandError { + return &simplecobra.CommandError{Err: errors.New(fmt.Sprint(a...))} } -type cmder interface { - flagsToConfigHandler - getCommand() *cobra.Command +func setValueFromFlag(flags *pflag.FlagSet, key string, cfg config.Provider, targetKey string, force bool) { + key = strings.TrimSpace(key) + if (force && flags.Lookup(key) != nil) || flags.Changed(key) { + f := flags.Lookup(key) + configKey := key + if targetKey != "" { + configKey = targetKey + } + // Gotta love this API. + switch f.Value.Type() { + case "bool": + bv, _ := flags.GetBool(key) + cfg.Set(configKey, bv) + case "string": + cfg.Set(configKey, f.Value.String()) + case "stringSlice": + bv, _ := flags.GetStringSlice(key) + cfg.Set(configKey, bv) + case "int": + iv, _ := flags.GetInt(key) + cfg.Set(configKey, iv) + default: + panic(fmt.Sprintf("update switch with %s", f.Value.Type())) + } + + } } -// commandError is an error used to signal different error situations in command handling. -type commandError struct { - s string - userError bool +func flagsToCfg(cd *simplecobra.Commandeer, cfg config.Provider) config.Provider { + return flagsToCfgWithAdditionalConfigBase(cd, cfg, "") } -func (c commandError) Error() string { - return c.s -} - -func (c commandError) isUserError() bool { - return c.userError -} - -func newUserError(a ...any) commandError { - return commandError{s: fmt.Sprintln(a...), userError: true} -} - -func newSystemError(a ...any) commandError { - return commandError{s: fmt.Sprintln(a...), userError: false} -} - -func newSystemErrorF(format string, a ...any) commandError { - return commandError{s: fmt.Sprintf(format, a...), userError: false} -} - -// Catch some of the obvious user errors from Cobra. -// We don't want to show the usage message for every error. -// The below may be to generic. Time will show. -var userErrorRegexp = regexp.MustCompile("unknown flag") - -func isUserError(err error) bool { - if cErr, ok := err.(commandError); ok && cErr.isUserError() { - return true +func flagsToCfgWithAdditionalConfigBase(cd *simplecobra.Commandeer, cfg config.Provider, additionalConfigBase string) config.Provider { + if cfg == nil { + cfg = config.New() } - return userErrorRegexp.MatchString(err.Error()) + // Flags with a different name in the config. + keyMap := map[string]string{ + "minify": "minifyOutput", + "destination": "publishDir", + "printI18nWarnings": "logI18nWarnings", + "printPathWarnings": "logPathWarnings", + "editor": "newContentEditor", + } + + // Flags that we for some reason don't want to expose in the site config. + internalKeySet := map[string]bool{ + "quiet": true, + "verbose": true, + "watch": true, + "disableLiveReload": true, + "liveReloadPort": true, + "renderToMemory": true, + "clock": true, + } + + cmd := cd.CobraCommand + flags := cmd.Flags() + + flags.VisitAll(func(f *pflag.Flag) { + if f.Changed { + targetKey := f.Name + if internalKeySet[targetKey] { + targetKey = "internal." + targetKey + } else if mapped, ok := keyMap[targetKey]; ok { + targetKey = mapped + } + setValueFromFlag(flags, f.Name, cfg, targetKey, false) + if additionalConfigBase != "" { + setValueFromFlag(flags, f.Name, cfg, additionalConfigBase+"."+targetKey, true) + } + } + }) + + return cfg + +} + +func mkdir(x ...string) { + p := filepath.Join(x...) + err := os.MkdirAll(p, 0777) // before umask + if err != nil { + log.Fatal(err) + } +} + +func touchFile(fs afero.Fs, filename string) { + mkdir(filepath.Dir(filename)) + err := helpers.WriteToDisk(filename, bytes.NewReader([]byte{}), fs) + if err != nil { + log.Fatal(err) + } } diff --git a/commands/hugo_test.go b/commands/hugo_test.go deleted file mode 100644 index 1e1326642..000000000 --- a/commands/hugo_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "bytes" - "fmt" - "math/rand" - "path/filepath" - "strings" - "testing" - - "github.com/bep/clock" - qt "github.com/frankban/quicktest" - "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/afero" - "golang.org/x/tools/txtar" -) - -// Issue #5662 -func TestHugoWithContentDirOverride(t *testing.T) { - t.Parallel() - c := qt.New(t) - - files := ` --- config.toml -- -baseURL = "https://example.org" -title = "Hugo Commands" --- mycontent/p1.md -- ---- -title: "P1" ---- --- layouts/_default/single.html -- -Page: {{ .Title }}| - -` - s := newTestHugoCmdBuilder(c, files, []string{"-c", "mycontent"}).Build() - s.AssertFileContent("public/p1/index.html", `Page: P1|`) - -} - -// Issue #9794 -func TestHugoStaticFilesMultipleStaticAndManyFolders(t *testing.T) { - t.Parallel() - c := qt.New(t) - - files := ` --- config.toml -- -baseURL = "https://example.org" -theme = "mytheme" --- layouts/index.html -- -Home. - -` - const ( - numDirs = 33 - numFilesMax = 12 - ) - - r := rand.New(rand.NewSource(32)) - - for i := 0; i < numDirs; i++ { - for j := 0; j < r.Intn(numFilesMax); j++ { - if j%3 == 0 { - files += fmt.Sprintf("-- themes/mytheme/static/d%d/f%d.txt --\nHellot%d-%d\n", i, j, i, j) - files += fmt.Sprintf("-- themes/mytheme/static/d%d/ft%d.txt --\nHellot%d-%d\n", i, j, i, j) - } - files += fmt.Sprintf("-- static/d%d/f%d.txt --\nHello%d-%d\n", i, j, i, j) - } - } - - r = rand.New(rand.NewSource(32)) - - s := newTestHugoCmdBuilder(c, files, []string{"-c", "mycontent"}).Build() - for i := 0; i < numDirs; i++ { - for j := 0; j < r.Intn(numFilesMax); j++ { - if j%3 == 0 { - if j%3 == 0 { - s.AssertFileContent(fmt.Sprintf("public/d%d/ft%d.txt", i, j), fmt.Sprintf("Hellot%d-%d", i, j)) - } - s.AssertFileContent(fmt.Sprintf("public/d%d/f%d.txt", i, j), fmt.Sprintf("Hello%d-%d", i, j)) - } - } - } - -} - -// Issue #8787 -func TestHugoListCommandsWithClockFlag(t *testing.T) { - t.Cleanup(func() { htime.Clock = clock.System() }) - - c := qt.New(t) - - files := ` --- config.toml -- -baseURL = "https://example.org" -title = "Hugo Commands" -timeZone = "UTC" --- content/past.md -- ---- -title: "Past" -date: 2000-11-06 ---- --- content/future.md -- ---- -title: "Future" -date: 2200-11-06 ---- --- layouts/_default/single.html -- -Page: {{ .Title }}| - -` - s := newTestHugoCmdBuilder(c, files, []string{"list", "future"}) - s.captureOut = true - s.Build() - p := filepath.Join("content", "future.md") - s.AssertStdout(p + ",2200-11-06T00:00:00Z") - - s = newTestHugoCmdBuilder(c, files, []string{"list", "future", "--clock", "2300-11-06"}).Build() - s.AssertStdout("") -} - -type testHugoCmdBuilder struct { - *qt.C - - fs afero.Fs - dir string - files string - args []string - - captureOut bool - out string -} - -func newTestHugoCmdBuilder(c *qt.C, files string, args []string) *testHugoCmdBuilder { - s := &testHugoCmdBuilder{C: c, files: files, args: args} - s.dir = s.TempDir() - s.fs = afero.NewBasePathFs(hugofs.Os, s.dir) - - return s -} - -func (s *testHugoCmdBuilder) Build() *testHugoCmdBuilder { - data := txtar.Parse([]byte(s.files)) - - for _, f := range data.Files { - filename := filepath.Clean(f.Name) - data := bytes.TrimSuffix(f.Data, []byte("\n")) - s.Assert(s.fs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil) - s.Assert(afero.WriteFile(s.fs, filename, data, 0666), qt.IsNil) - } - - hugoCmd := newCommandsBuilder().addAll().build() - cmd := hugoCmd.getCommand() - args := append(s.args, "-s="+s.dir, "--quiet") - cmd.SetArgs(args) - - if s.captureOut { - out, err := captureStdout(func() error { - _, err := cmd.ExecuteC() - return err - }) - s.Assert(err, qt.IsNil) - s.out = out - } else { - _, err := cmd.ExecuteC() - s.Assert(err, qt.IsNil) - } - - return s -} - -func (s *testHugoCmdBuilder) AssertFileContent(filename string, matches ...string) { - s.Helper() - data, err := afero.ReadFile(s.fs, filename) - s.Assert(err, qt.IsNil) - content := strings.TrimSpace(string(data)) - for _, m := range matches { - lines := strings.Split(m, "\n") - for _, match := range lines { - match = strings.TrimSpace(match) - if match == "" || strings.HasPrefix(match, "#") { - continue - } - s.Assert(content, qt.Contains, match, qt.Commentf(m)) - } - } -} - -func (s *testHugoCmdBuilder) AssertStdout(match string) { - s.Helper() - content := strings.TrimSpace(s.out) - s.Assert(content, qt.Contains, strings.TrimSpace(match)) -} diff --git a/commands/hugo_windows.go b/commands/hugo_windows.go index 1724f12cd..e1fd98132 100644 --- a/commands/hugo_windows.go +++ b/commands/hugo_windows.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/commands/hugo.go b/commands/hugobuilder.go similarity index 52% rename from commands/hugo.go rename to commands/hugobuilder.go index 1a35d1626..7c6dbee35 100644 --- a/commands/hugo.go +++ b/commands/hugobuilder.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,353 +11,172 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package commands defines and implements command-line commands and flags -// used by Hugo. Commands and flags are implemented using Cobra. package commands import ( "context" + "errors" "fmt" - "io" "os" - "os/signal" "path/filepath" "runtime" "runtime/pprof" "runtime/trace" "strings" - "sync/atomic" - "syscall" + "sync" "time" - "github.com/gohugoio/hugo/hugofs/files" - "github.com/gohugoio/hugo/tpl" - + "github.com/bep/simplecobra" + "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/common/types" - - "github.com/gohugoio/hugo/hugofs" - - "github.com/gohugoio/hugo/resources/page" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/terminal" - - "github.com/gohugoio/hugo/hugolib/filesystems" - - "golang.org/x/sync/errgroup" - + "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" - - flag "github.com/spf13/pflag" - - "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/hugolib/filesystems" "github.com/gohugoio/hugo/livereload" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/watcher" - "github.com/spf13/afero" - "github.com/spf13/cobra" "github.com/spf13/fsync" - jww "github.com/spf13/jwalterweatherman" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" ) -// The Response value from Execute. -type Response struct { - // The build Result will only be set in the hugo build command. - Result *hugolib.HugoSites +type hugoBuilder struct { + r *rootCommand - // Err is set when the command failed to execute. - Err error + cunfMu sync.Mutex + conf_ *commonConfig - // The command that was executed. - Cmd *cobra.Command + // May be nil. + s *serverCommand + + // Currently only set when in "fast render mode". + changeDetector *fileChangeDetector + visitedURLs *types.EvictingStringQueue + + fullRebuildSem *semaphore.Weighted + debounce func(f func()) + + onConfigLoaded func(reloaded bool) error + + fastRenderMode bool + buildWatch bool + showErrorInBrowser bool + + errState hugoBuilderErrState } -// IsUserError returns true is the Response error is a user error rather than a -// system error. -func (r Response) IsUserError() bool { - return r.Err != nil && isUserError(r.Err) +func (c *hugoBuilder) conf() *commonConfig { + c.cunfMu.Lock() + defer c.cunfMu.Unlock() + return c.conf_ } -// Execute adds all child commands to the root command HugoCmd and sets flags appropriately. -// The args are usually filled with os.Args[1:]. -func Execute(args []string) Response { - hugoCmd := newCommandsBuilder().addAll().build() - cmd := hugoCmd.getCommand() - cmd.SetArgs(args) - - c, err := cmd.ExecuteC() - - var resp Response - - if c == cmd && hugoCmd.c != nil { - // Root command executed - resp.Result = hugoCmd.c.hugo() - } - - if err == nil { - errCount := int(loggers.GlobalErrorCounter.Count()) - if errCount > 0 { - err = fmt.Errorf("logged %d errors", errCount) - } else if resp.Result != nil { - errCount = resp.Result.NumLogErrors() - if errCount > 0 { - err = fmt.Errorf("logged %d errors", errCount) - } - } - - } - - resp.Err = err - resp.Cmd = c - - return resp +func (c *hugoBuilder) setConf(conf *commonConfig) { + c.cunfMu.Lock() + defer c.cunfMu.Unlock() + c.conf_ = conf } -// InitializeConfig initializes a config file with sensible default configuration flags. -func initializeConfig(mustHaveConfigFile, failOnInitErr, running bool, - h *hugoBuilderCommon, - f flagsToConfigHandler, - cfgInit func(c *commandeer) error) (*commandeer, error) { - c, err := newCommandeer(mustHaveConfigFile, failOnInitErr, running, h, f, cfgInit) - if err != nil { - return nil, err - } - - if h := c.hugoTry(); h != nil { - for _, s := range h.Sites { - s.RegisterMediaTypes() - } - } - - return c, nil +type hugoBuilderErrState struct { + mu sync.Mutex + paused bool + builderr error + waserr bool } -func (c *commandeer) createLogger(cfg config.Provider) (loggers.Logger, error) { - var ( - logHandle = io.Discard - logThreshold = jww.LevelWarn - logFile = cfg.GetString("logFile") - outHandle = io.Discard - stdoutThreshold = jww.LevelWarn - ) - - if !c.h.quiet { - outHandle = os.Stdout - } - - if c.h.verboseLog || c.h.logging || (c.h.logFile != "") { - var err error - if logFile != "" { - logHandle, err = os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) - if err != nil { - return nil, newSystemError("Failed to open log file:", logFile, err) - } - } else { - logHandle, err = os.CreateTemp("", "hugo") - if err != nil { - return nil, newSystemError(err) - } - } - } else if !c.h.quiet && cfg.GetBool("verbose") { - stdoutThreshold = jww.LevelInfo - } - - if cfg.GetBool("debug") { - stdoutThreshold = jww.LevelDebug - } - - if c.h.verboseLog { - logThreshold = jww.LevelInfo - if cfg.GetBool("debug") { - logThreshold = jww.LevelDebug - } - } - - loggers.InitGlobalLogger(stdoutThreshold, logThreshold, outHandle, logHandle) - helpers.InitLoggers() - - return loggers.NewLogger(stdoutThreshold, logThreshold, outHandle, logHandle, c.running), nil +func (e *hugoBuilderErrState) setPaused(p bool) { + e.mu.Lock() + defer e.mu.Unlock() + e.paused = p } -func initializeFlags(cmd *cobra.Command, cfg config.Provider) { - persFlagKeys := []string{ - "debug", - "verbose", - "logFile", - // Moved from vars - } - flagKeys := []string{ - "cleanDestinationDir", - "buildDrafts", - "buildFuture", - "buildExpired", - "clock", - "uglyURLs", - "canonifyURLs", - "enableRobotsTXT", - "enableGitInfo", - "pluralizeListTitles", - "preserveTaxonomyNames", - "ignoreCache", - "forceSyncStatic", - "noTimes", - "noChmod", - "noBuildLock", - "ignoreVendorPaths", - "templateMetrics", - "templateMetricsHints", - - // Moved from vars. - "baseURL", - "buildWatch", - "cacheDir", - "cfgFile", - "confirm", - "contentDir", - "debug", - "destination", - "disableKinds", - "dryRun", - "force", - "gc", - "printI18nWarnings", - "printUnusedTemplates", - "invalidateCDN", - "layoutDir", - "logFile", - "maxDeletes", - "quiet", - "renderToMemory", - "source", - "target", - "theme", - "themesDir", - "verbose", - "verboseLog", - "workers", - "duplicateTargetPaths", - } - - for _, key := range persFlagKeys { - setValueFromFlag(cmd.PersistentFlags(), key, cfg, "", false) - } - for _, key := range flagKeys { - setValueFromFlag(cmd.Flags(), key, cfg, "", false) - } - - setValueFromFlag(cmd.Flags(), "minify", cfg, "minifyOutput", true) - - // Set some "config aliases" - setValueFromFlag(cmd.Flags(), "destination", cfg, "publishDir", false) - setValueFromFlag(cmd.Flags(), "printI18nWarnings", cfg, "logI18nWarnings", false) - setValueFromFlag(cmd.Flags(), "printPathWarnings", cfg, "logPathWarnings", false) - +func (e *hugoBuilderErrState) isPaused() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.paused } -func setValueFromFlag(flags *flag.FlagSet, key string, cfg config.Provider, targetKey string, force bool) { - key = strings.TrimSpace(key) - if (force && flags.Lookup(key) != nil) || flags.Changed(key) { - f := flags.Lookup(key) - configKey := key - if targetKey != "" { - configKey = targetKey - } - // Gotta love this API. - switch f.Value.Type() { - case "bool": - bv, _ := flags.GetBool(key) - cfg.Set(configKey, bv) - case "string": - cfg.Set(configKey, f.Value.String()) - case "stringSlice": - bv, _ := flags.GetStringSlice(key) - cfg.Set(configKey, bv) - case "int": - iv, _ := flags.GetInt(key) - cfg.Set(configKey, iv) - default: - panic(fmt.Sprintf("update switch with %s", f.Value.Type())) - } - - } +func (e *hugoBuilderErrState) setBuildErr(err error) { + e.mu.Lock() + defer e.mu.Unlock() + e.builderr = err } -func (c *commandeer) fullBuild(noBuildLock bool) error { - var ( - g errgroup.Group - langCount map[string]uint64 - ) +func (e *hugoBuilderErrState) buildErr() error { + e.mu.Lock() + defer e.mu.Unlock() + return e.builderr +} - if !c.h.quiet { - fmt.Println("Start building sites … ") - fmt.Println(hugo.BuildVersionString()) - if terminal.IsTerminal(os.Stdout) { - defer func() { - fmt.Print(showCursor + clearLine) - }() - } - } +func (e *hugoBuilderErrState) setWasErr(w bool) { + e.mu.Lock() + defer e.mu.Unlock() + e.waserr = w +} - copyStaticFunc := func() error { - cnt, err := c.copyStatic() +func (e *hugoBuilderErrState) wasErr() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.waserr +} + +func (c *hugoBuilder) errCount() int { + return int(c.r.logger.LogCounters().ErrorCounter.Count()) +} + +// getDirList provides NewWatcher() with a list of directories to watch for changes. +func (c *hugoBuilder) getDirList() ([]string, error) { + var filenames []string + + walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { if err != nil { - return fmt.Errorf("Error copying static files: %w", err) + c.r.logger.Errorln("walker: ", err) + return nil } - langCount = cnt + + if fi.IsDir() { + if fi.Name() == ".git" || + fi.Name() == "node_modules" || fi.Name() == "bower_components" { + return filepath.SkipDir + } + + filenames = append(filenames, fi.Meta().Filename) + } + return nil } - buildSitesFunc := func() error { - if err := c.buildSites(noBuildLock); err != nil { - return fmt.Errorf("Error building site: %w", err) + + watchFiles := c.hugo().PathSpec.BaseFs.WatchDirs() + for _, fi := range watchFiles { + if !fi.IsDir() { + filenames = append(filenames, fi.Meta().Filename) + continue } - return nil - } - // Do not copy static files and build sites in parallel if cleanDestinationDir is enabled. - // This flag deletes all static resources in /public folder that are missing in /static, - // and it does so at the end of copyStatic() call. - if c.Cfg.GetBool("cleanDestinationDir") { - if err := copyStaticFunc(); err != nil { - return err - } - if err := buildSitesFunc(); err != nil { - return err - } - } else { - g.Go(copyStaticFunc) - g.Go(buildSitesFunc) - if err := g.Wait(); err != nil { - return err + + w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.r.logger, Info: fi, WalkFn: walkFn}) + if err := w.Walk(); err != nil { + c.r.logger.Errorln("walker: ", err) } } - for _, s := range c.hugo().Sites { - s.ProcessingStats.Static = langCount[s.Language().Lang] - } + filenames = helpers.UniqueStringsSorted(filenames) - if c.h.gc { - count, err := c.hugo().GC() - if err != nil { - return err - } - for _, s := range c.hugo().Sites { - // We have no way of knowing what site the garbage belonged to. - s.ProcessingStats.Cleaned = uint64(count) - } - } - - return nil + return filenames, nil } -func (c *commandeer) initCPUProfile() (func(), error) { - if c.h.cpuprofile == "" { +func (c *hugoBuilder) initCPUProfile() (func(), error) { + if c.r.cpuprofile == "" { return nil, nil } - f, err := os.Create(c.h.cpuprofile) + f, err := os.Create(c.r.cpuprofile) if err != nil { return nil, fmt.Errorf("failed to create CPU profile: %w", err) } @@ -370,61 +189,23 @@ func (c *commandeer) initCPUProfile() (func(), error) { }, nil } -func (c *commandeer) initMemProfile() { - if c.h.memprofile == "" { +func (c *hugoBuilder) initMemProfile() { + if c.r.memprofile == "" { return } - f, err := os.Create(c.h.memprofile) + f, err := os.Create(c.r.memprofile) if err != nil { - c.logger.Errorf("could not create memory profile: ", err) + c.r.logger.Errorf("could not create memory profile: ", err) } defer f.Close() runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { - c.logger.Errorf("could not write memory profile: ", err) + c.r.logger.Errorf("could not write memory profile: ", err) } } -func (c *commandeer) initTraceProfile() (func(), error) { - if c.h.traceprofile == "" { - return nil, nil - } - - f, err := os.Create(c.h.traceprofile) - if err != nil { - return nil, fmt.Errorf("failed to create trace file: %w", err) - } - - if err := trace.Start(f); err != nil { - return nil, fmt.Errorf("failed to start trace: %w", err) - } - - return func() { - trace.Stop() - f.Close() - }, nil -} - -func (c *commandeer) initMutexProfile() (func(), error) { - if c.h.mutexprofile == "" { - return nil, nil - } - - f, err := os.Create(c.h.mutexprofile) - if err != nil { - return nil, err - } - - runtime.SetMutexProfileFraction(1) - - return func() { - pprof.Lookup("mutex").WriteTo(f, 0) - f.Close() - }, nil -} - -func (c *commandeer) initMemTicker() func() { +func (c *hugoBuilder) initMemTicker() func() { memticker := time.NewTicker(5 * time.Second) quit := make(chan struct{}) printMem := func() { @@ -451,7 +232,25 @@ func (c *commandeer) initMemTicker() func() { } } -func (c *commandeer) initProfiling() (func(), error) { +func (c *hugoBuilder) initMutexProfile() (func(), error) { + if c.r.mutexprofile == "" { + return nil, nil + } + + f, err := os.Create(c.r.mutexprofile) + if err != nil { + return nil, err + } + + runtime.SetMutexProfileFraction(1) + + return func() { + pprof.Lookup("mutex").WriteTo(f, 0) + f.Close() + }, nil +} + +func (c *hugoBuilder) initProfiling() (func(), error) { stopCPUProf, err := c.initCPUProfile() if err != nil { return nil, err @@ -468,7 +267,7 @@ func (c *commandeer) initProfiling() (func(), error) { } var stopMemTicker func() - if c.h.printm { + if c.r.printm { stopMemTicker = c.initMemTicker() } @@ -492,368 +291,38 @@ func (c *commandeer) initProfiling() (func(), error) { }, nil } -func (c *commandeer) build() error { - stopProfiling, err := c.initProfiling() +func (c *hugoBuilder) initTraceProfile() (func(), error) { + if c.r.traceprofile == "" { + return nil, nil + } + + f, err := os.Create(c.r.traceprofile) if err != nil { - return err + return nil, fmt.Errorf("failed to create trace file: %w", err) } - defer func() { - if stopProfiling != nil { - stopProfiling() - } - }() - - if err := c.fullBuild(false); err != nil { - return err + if err := trace.Start(f); err != nil { + return nil, fmt.Errorf("failed to start trace: %w", err) } - if !c.h.quiet { - fmt.Println() - c.hugo().PrintProcessingStats(os.Stdout) - fmt.Println() - - hugofs.WalkFilesystems(c.publishDirFs, func(fs afero.Fs) bool { - if dfs, ok := fs.(hugofs.DuplicatesReporter); ok { - dupes := dfs.ReportDuplicates() - if dupes != "" { - c.logger.Warnln("Duplicate target paths:", dupes) - } - } - return false - }) - - unusedTemplates := c.hugo().Tmpl().(tpl.UnusedTemplatesProvider).UnusedTemplates() - for _, unusedTemplate := range unusedTemplates { - c.logger.Warnf("Template %s is unused, source file %s", unusedTemplate.Name(), unusedTemplate.Filename()) - } - } - - if c.h.buildWatch { - watchDirs, err := c.getDirList() - if err != nil { - return err - } - - baseWatchDir := c.Cfg.GetString("workingDir") - rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs) - - c.logger.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) - c.logger.Println("Press Ctrl+C to stop") - watcher, err := c.newWatcher(c.h.poll, watchDirs...) - checkErr(c.Logger, err) - defer watcher.Close() - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - <-sigs - } - - return nil -} - -func (c *commandeer) serverBuild() error { - stopProfiling, err := c.initProfiling() - if err != nil { - return err - } - - defer func() { - if stopProfiling != nil { - stopProfiling() - } - }() - - if err := c.fullBuild(false); err != nil { - return err - } - - // TODO(bep) Feedback? - if !c.h.quiet { - fmt.Println() - c.hugo().PrintProcessingStats(os.Stdout) - fmt.Println() - } - - return nil -} - -func (c *commandeer) copyStatic() (map[string]uint64, error) { - m, err := c.doWithPublishDirs(c.copyStaticTo) - if err == nil || herrors.IsNotExist(err) { - return m, nil - } - return m, err -} - -func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { - langCount := make(map[string]uint64) - - staticFilesystems := c.hugo().BaseFs.SourceFilesystems.Static - - if len(staticFilesystems) == 0 { - c.logger.Infoln("No static directories found to sync") - return langCount, nil - } - - for lang, fs := range staticFilesystems { - cnt, err := f(fs) - if err != nil { - return langCount, err - } - - if lang == "" { - // Not multihost - for _, l := range c.languages { - langCount[l.Lang] = cnt - } - } else { - langCount[lang] = cnt - } - } - - return langCount, nil -} - -type countingStatFs struct { - afero.Fs - statCounter uint64 -} - -func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { - f, err := fs.Fs.Stat(name) - if err == nil { - if !f.IsDir() { - atomic.AddUint64(&fs.statCounter, 1) - } - } - return f, err -} - -func chmodFilter(dst, src os.FileInfo) bool { - // Hugo publishes data from multiple sources, potentially - // with overlapping directory structures. We cannot sync permissions - // for directories as that would mean that we might end up with write-protected - // directories inside /public. - // One example of this would be syncing from the Go Module cache, - // which have 0555 directories. - return src.IsDir() -} - -func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) { - publishDir := helpers.FilePathSeparator - - if sourceFs.PublishFolder != "" { - publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) - } - - fs := &countingStatFs{Fs: sourceFs.Fs} - - syncer := fsync.NewSyncer() - syncer.NoTimes = c.Cfg.GetBool("noTimes") - syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.ChmodFilter = chmodFilter - syncer.SrcFs = fs - syncer.DestFs = c.Fs.PublishDirStatic - // Now that we are using a unionFs for the static directories - // We can effectively clean the publishDir on initial sync - syncer.Delete = c.Cfg.GetBool("cleanDestinationDir") - - if syncer.Delete { - c.logger.Infoln("removing all files from destination that don't exist in static dirs") - - syncer.DeleteFilter = func(f os.FileInfo) bool { - return f.IsDir() && strings.HasPrefix(f.Name(), ".") - } - } - c.logger.Infoln("syncing static files to", publishDir) - - // because we are using a baseFs (to get the union right). - // set sync src to root - err := syncer.Sync(publishDir, helpers.FilePathSeparator) - if err != nil { - return 0, err - } - - // Sync runs Stat 3 times for every source file (which sounds much) - numFiles := fs.statCounter / 3 - - return numFiles, err -} - -func (c *commandeer) firstPathSpec() *helpers.PathSpec { - return c.hugo().Sites[0].PathSpec -} - -func (c *commandeer) timeTrack(start time.Time, name string) { - // Note the use of time.Since here and time.Now in the callers. - // We have a htime.Sinnce, but that may be adjusted to the future, - // and that does not make sense here, esp. when used before the - // global Clock is initialized. - elapsed := time.Since(start) - c.logger.Printf("%s in %v ms", name, int(1000*elapsed.Seconds())) -} - -// getDirList provides NewWatcher() with a list of directories to watch for changes. -func (c *commandeer) getDirList() ([]string, error) { - var filenames []string - - walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { - if err != nil { - c.logger.Errorln("walker: ", err) - return nil - } - - if fi.IsDir() { - if fi.Name() == ".git" || - fi.Name() == "node_modules" || fi.Name() == "bower_components" { - return filepath.SkipDir - } - - filenames = append(filenames, fi.Meta().Filename) - } - - return nil - } - - watchFiles := c.hugo().PathSpec.BaseFs.WatchDirs() - for _, fi := range watchFiles { - if !fi.IsDir() { - filenames = append(filenames, fi.Meta().Filename) - continue - } - - w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: fi, WalkFn: walkFn}) - if err := w.Walk(); err != nil { - c.logger.Errorln("walker: ", err) - } - } - - filenames = helpers.UniqueStringsSorted(filenames) - - return filenames, nil -} - -func (c *commandeer) buildSites(noBuildLock bool) (err error) { - return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: noBuildLock}) -} - -func (c *commandeer) handleBuildErr(err error, msg string) { - c.buildErr = err - c.logger.Errorln(msg + ": " + cleanErrorLog(err.Error())) -} - -func (c *commandeer) rebuildSites(events []fsnotify.Event) error { - if c.buildErr != nil { - ferrs := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr) - for _, err := range ferrs { - events = append(events, fsnotify.Event{Name: err.Position().Filename, Op: fsnotify.Write}) - } - } - c.buildErr = nil - visited := c.visitedURLs.PeekAllSet() - if c.fastRenderMode { - // Make sure we always render the home pages - for _, l := range c.languages { - langPath := c.hugo().PathSpec.GetLangSubDir(l.Lang) - if langPath != "" { - langPath = langPath + "/" - } - home := c.hugo().PathSpec.PrependBasePath("/"+langPath, false) - visited[home] = true - } - } - return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: visited, ErrRecovery: c.wasError}, events...) -} - -func (c *commandeer) partialReRender(urls ...string) error { - defer func() { - c.wasError = false - }() - c.buildErr = nil - visited := make(map[string]bool) - for _, url := range urls { - visited[url] = true - } - - // Note: We do not set NoBuildLock as the file lock is not acquired at this stage. - return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyVisited: visited, PartialReRender: true, ErrRecovery: c.wasError}) -} - -func (c *commandeer) fullRebuild(changeType string) { - if changeType == configChangeGoMod { - // go.mod may be changed during the build itself, and - // we really want to prevent superfluous builds. - if !c.fullRebuildSem.TryAcquire(1) { - return - } - c.fullRebuildSem.Release(1) - } - - c.fullRebuildSem.Acquire(context.Background(), 1) - - go func() { - defer c.fullRebuildSem.Release(1) - - c.printChangeDetected(changeType) - - defer func() { - // Allow any file system events to arrive back. - // This will block any rebuild on config changes for the - // duration of the sleep. - time.Sleep(2 * time.Second) - }() - - defer c.timeTrack(time.Now(), "Rebuilt") - - c.commandeerHugoState = newCommandeerHugoState() - err := c.loadConfig() - if err != nil { - // Set the processing on pause until the state is recovered. - c.paused = true - c.handleBuildErr(err, "Failed to reload config") - - } else { - c.paused = false - } - - if !c.paused { - _, err := c.copyStatic() - if err != nil { - c.logger.Errorln(err) - return - } - - err = c.buildSites(true) - if err != nil { - c.logger.Errorln(err) - } else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { - livereload.ForceRefresh() - } - } - }() + return func() { + trace.Stop() + f.Close() + }, nil } // newWatcher creates a new watcher to watch filesystem events. -func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*watcher.Batcher, error) { - if runtime.GOOS == "darwin" { - tweakLimit() - } - - staticSyncer, err := newStaticSyncer(c) - if err != nil { - return nil, err - } +func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*watcher.Batcher, error) { + staticSyncer := &staticSyncer{c: c} var pollInterval time.Duration poll := pollIntervalStr != "" if poll { - pollInterval, err = types.ToDurationE(pollIntervalStr) + pollInterval, err := types.ToDurationE(pollIntervalStr) if err != nil { return nil, fmt.Errorf("invalid value for flag poll: %s", err) } - c.logger.Printf("Use watcher with poll interval %v", pollInterval) + c.r.logger.Printf("Use watcher with poll interval %v", pollInterval) } if pollInterval == 0 { @@ -878,9 +347,10 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat // Identifies changes to config (config.toml) files. configSet := make(map[string]bool) + configFiles := c.conf().configs.LoadingInfo.ConfigFiles - c.logger.Println("Watching for config changes in", strings.Join(c.configFiles, ", ")) - for _, configFile := range c.configFiles { + c.r.logger.Println("Watching for config changes in", strings.Join(configFiles, ", ")) + for _, configFile := range configFiles { watcher.Add(configFile) configSet[configFile] = true } @@ -889,9 +359,9 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat for { select { case evs := <-watcher.Events: - unlock, err := c.buildLock() + unlock, err := c.hugo().LockBuild() if err != nil { - c.logger.Errorln("Failed to acquire a build lock: %s", err) + c.r.logger.Errorln("Failed to acquire a build lock: %s", err) return } c.handleEvents(watcher, staticSyncer, evs, configSet) @@ -902,7 +372,7 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat unlock() case err := <-watcher.Errors(): if err != nil && !herrors.IsNotExist(err) { - c.logger.Errorln("Error while watching:", err) + c.r.logger.Errorln("Error while watching:", err) } } } @@ -911,30 +381,240 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat return watcher, nil } -func (c *commandeer) printChangeDetected(typ string) { - msg := "\nChange" - if typ != "" { - msg += " of " + typ +func (c *hugoBuilder) build() error { + stopProfiling, err := c.initProfiling() + if err != nil { + return err } - msg += " detected, rebuilding site." - c.logger.Println(msg) - const layout = "2006-01-02 15:04:05.000 -0700" - c.logger.Println(htime.Now().Format(layout)) + defer func() { + if stopProfiling != nil { + stopProfiling() + } + }() + + if err := c.fullBuild(false); err != nil { + return err + } + + if !c.r.quiet { + c.r.Println() + c.hugo().PrintProcessingStats(os.Stdout) + c.r.Println() + } + + return nil } -const ( - configChangeConfig = "config file" - configChangeGoMod = "go.mod file" - configChangeGoWork = "go work file" -) +func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) { + return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: noBuildLock}) +} -func (c *commandeer) handleEvents(watcher *watcher.Batcher, +func (c *hugoBuilder) copyStatic() (map[string]uint64, error) { + m, err := c.doWithPublishDirs(c.copyStaticTo) + if err == nil || herrors.IsNotExist(err) { + return m, nil + } + return m, err +} + +func (c *hugoBuilder) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := helpers.FilePathSeparator + + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) + } + + fs := &countingStatFs{Fs: sourceFs.Fs} + + syncer := fsync.NewSyncer() + syncer.NoTimes = c.conf().configs.Base.NoTimes + syncer.NoChmod = c.conf().configs.Base.NoChmod + syncer.ChmodFilter = chmodFilter + syncer.SrcFs = fs + syncer.DestFs = c.conf().fs.PublishDirStatic + // Now that we are using a unionFs for the static directories + // We can effectively clean the publishDir on initial sync + syncer.Delete = c.conf().configs.Base.CleanDestinationDir + + if syncer.Delete { + c.r.logger.Infoln("removing all files from destination that don't exist in static dirs") + + syncer.DeleteFilter = func(f os.FileInfo) bool { + return f.IsDir() && strings.HasPrefix(f.Name(), ".") + } + } + c.r.logger.Infoln("syncing static files to", publishDir) + + // because we are using a baseFs (to get the union right). + // set sync src to root + err := syncer.Sync(publishDir, helpers.FilePathSeparator) + if err != nil { + return 0, err + } + + // Sync runs Stat 3 times for every source file (which sounds much) + numFiles := fs.statCounter / 3 + + return numFiles, err +} + +func (c *hugoBuilder) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) { + langCount := make(map[string]uint64) + + staticFilesystems := c.hugo().BaseFs.SourceFilesystems.Static + + if len(staticFilesystems) == 0 { + c.r.logger.Infoln("No static directories found to sync") + return langCount, nil + } + + for lang, fs := range staticFilesystems { + cnt, err := f(fs) + if err != nil { + return langCount, err + } + if lang == "" { + // Not multihost + for _, l := range c.conf().configs.Languages { + langCount[l.Lang] = cnt + } + } else { + langCount[lang] = cnt + } + } + + return langCount, nil +} + +func (c *hugoBuilder) fullBuild(noBuildLock bool) error { + var ( + g errgroup.Group + langCount map[string]uint64 + ) + + if !c.r.quiet { + fmt.Println("Start building sites … ") + fmt.Println(hugo.BuildVersionString()) + if terminal.IsTerminal(os.Stdout) { + defer func() { + fmt.Print(showCursor + clearLine) + }() + } + } + + copyStaticFunc := func() error { + cnt, err := c.copyStatic() + if err != nil { + return fmt.Errorf("error copying static files: %w", err) + } + langCount = cnt + return nil + } + buildSitesFunc := func() error { + if err := c.buildSites(noBuildLock); err != nil { + return fmt.Errorf("error building site: %w", err) + } + return nil + } + // Do not copy static files and build sites in parallel if cleanDestinationDir is enabled. + // This flag deletes all static resources in /public folder that are missing in /static, + // and it does so at the end of copyStatic() call. + if c.conf().configs.Base.CleanDestinationDir { + if err := copyStaticFunc(); err != nil { + return err + } + if err := buildSitesFunc(); err != nil { + return err + } + } else { + g.Go(copyStaticFunc) + g.Go(buildSitesFunc) + if err := g.Wait(); err != nil { + return err + } + } + + for _, s := range c.hugo().Sites { + s.ProcessingStats.Static = langCount[s.Language().Lang] + } + + if c.r.gc { + count, err := c.hugo().GC() + if err != nil { + return err + } + for _, s := range c.hugo().Sites { + // We have no way of knowing what site the garbage belonged to. + s.ProcessingStats.Cleaned = uint64(count) + } + } + + return nil +} + +func (c *hugoBuilder) fullRebuild(changeType string) { + if changeType == configChangeGoMod { + // go.mod may be changed during the build itself, and + // we really want to prevent superfluous builds. + if !c.fullRebuildSem.TryAcquire(1) { + return + } + c.fullRebuildSem.Release(1) + } + + c.fullRebuildSem.Acquire(context.Background(), 1) + + go func() { + defer c.fullRebuildSem.Release(1) + + c.printChangeDetected(changeType) + + defer func() { + // Allow any file system events to arrive basimplecobra. + // This will block any rebuild on config changes for the + // duration of the sleep. + time.Sleep(2 * time.Second) + }() + + defer c.r.timeTrack(time.Now(), "Rebuilt") + + err := c.reloadConfig() + if err != nil { + // Set the processing on pause until the state is recovered. + c.errState.setPaused(true) + c.handleBuildErr(err, "Failed to reload config") + } else { + c.errState.setPaused(false) + } + + if !c.errState.isPaused() { + _, err := c.copyStatic() + if err != nil { + c.r.logger.Errorln(err) + return + } + err = c.buildSites(false) + if err != nil { + c.r.logger.Errorln(err) + } else if c.s != nil && c.s.doLiveReload { + livereload.ForceRefresh() + } + } + }() +} + +func (c *hugoBuilder) handleBuildErr(err error, msg string) { + c.errState.setBuildErr(err) + c.r.logger.Errorln(msg + ": " + cleanErrorLog(err.Error())) +} + +func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher, staticSyncer *staticSyncer, evs []fsnotify.Event, configSet map[string]bool) { defer func() { - c.wasError = false + c.errState.setWasErr(false) }() var isHandled bool @@ -966,7 +646,8 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, } if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename { - for _, configFile := range c.configFiles { + configFiles := c.conf().configs.LoadingInfo.ConfigFiles + for _, configFile := range configFiles { counter := 0 for watcher.Add(configFile) != nil { counter++ @@ -989,7 +670,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, return } - if c.paused { + if c.errState.isPaused() { // Wait for the server to get into a consistent state before // we continue with processing. return @@ -1004,7 +685,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, return } - c.logger.Infoln("Received System Events:", evs) + c.r.logger.Infoln("Received System Events:", evs) staticEvents := []fsnotify.Event{} dynamicEvents := []fsnotify.Event{} @@ -1086,7 +767,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error { if f.IsDir() { - c.logger.Println("adding created directory to watchlist", path) + c.r.logger.Println("adding created directory to watchlist", path) if err := watcher.Add(path); err != nil { return err } @@ -1102,8 +783,8 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, // recursively add new directories to watch list // When mkdir -p is used, only the top directory triggers an event (at least on OSX) if ev.Op&fsnotify.Create == fsnotify.Create { - if s, err := c.Fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() { - _ = helpers.SymbolicWalk(c.Fs.Source, ev.Name, walkAdder) + if s, err := c.conf().fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() { + _ = helpers.SymbolicWalk(c.conf().fs.Source, ev.Name, walkAdder) } } @@ -1117,28 +798,29 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, if len(staticEvents) > 0 { c.printChangeDetected("Static files") - if c.Cfg.GetBool("forceSyncStatic") { - c.logger.Printf("Syncing all static files\n") + if c.r.forceSyncStatic { + c.r.logger.Printf("Syncing all static files\n") _, err := c.copyStatic() if err != nil { - c.logger.Errorln("Error copying static files to publish dir:", err) + c.r.logger.Errorln("Error copying static files to publish dir:", err) return } } else { if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil { - c.logger.Errorln("Error syncing static files to publish dir:", err) + c.r.logger.Errorln("Error syncing static files to publish dir:", err) return } } - if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") { + if c.s != nil && c.s.doLiveReload { // Will block forever trying to write to a channel that nobody is reading if livereload isn't initialized // force refresh when more than one file - if !c.wasError && len(staticEvents) == 1 { + if !c.errState.wasErr() && len(staticEvents) == 1 { ev := staticEvents[0] - path := c.hugo().BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) - path = c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(path), false) + h := c.hugo() + path := h.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name) + path = h.RelURL(helpers.ToSlashTrimLeading(path), false) livereload.RefreshPath(path) } else { @@ -1149,25 +831,24 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, if len(dynamicEvents) > 0 { partitionedEvents := partitionDynamicEvents( - c.firstPathSpec().BaseFs.SourceFilesystems, + c.hugo().BaseFs.SourceFilesystems, dynamicEvents) - doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") onePageName := pickOneWriteOrCreatePath(partitionedEvents.ContentEvents) c.printChangeDetected("") c.changeDetector.PrepareNew() func() { - defer c.timeTrack(time.Now(), "Total") + defer c.r.timeTrack(time.Now(), "Total") if err := c.rebuildSites(dynamicEvents); err != nil { c.handleBuildErr(err, "Rebuild failed") } }() - if doLiveReload { + if c.s != nil && c.s.doLiveReload { if len(partitionedEvents.ContentEvents) == 0 && len(partitionedEvents.AssetEvents) > 0 { - if c.wasError { + if c.errState.wasErr() { livereload.ForceRefresh() return } @@ -1176,7 +857,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, // Nothing has changed. return } else if len(changed) == 1 { - pathToRefresh := c.firstPathSpec().RelURL(helpers.ToSlashTrimLeading(changed[0]), false) + pathToRefresh := c.hugo().PathSpec.RelURL(helpers.ToSlashTrimLeading(changed[0]), false) livereload.RefreshPath(pathToRefresh) } else { livereload.ForceRefresh() @@ -1184,8 +865,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, } if len(partitionedEvents.ContentEvents) > 0 { - - navigate := c.Cfg.GetBool("navigateToChanged") + navigate := c.s != nil && c.s.navigateToChanged // We have fetched the same page above, but it may have // changed. var p page.Page @@ -1206,54 +886,108 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher, } } -// dynamicEvents contains events that is considered dynamic, as in "not static". -// Both of these categories will trigger a new build, but the asset events -// does not fit into the "navigate to changed" logic. -type dynamicEvents struct { - ContentEvents []fsnotify.Event - AssetEvents []fsnotify.Event -} - -func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) { - for _, e := range events { - if sourceFs.IsAsset(e.Name) { - de.AssetEvents = append(de.AssetEvents, e) - } else { - de.ContentEvents = append(de.ContentEvents, e) +func (c *hugoBuilder) hugo() *hugolib.HugoSites { + h, err := c.r.HugFromConfig(c.conf()) + if err != nil { + panic(err) + } + if c.s != nil { + // A running server, register the media types. + for _, s := range h.Sites { + s.RegisterMediaTypes() } } - return + return h } -func pickOneWriteOrCreatePath(events []fsnotify.Event) string { - name := "" +func (c *hugoBuilder) hugoTry() *hugolib.HugoSites { + h, _ := c.r.HugFromConfig(c.conf()) + return h +} - for _, ev := range events { - if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create { - if files.IsIndexContentFile(ev.Name) { - return ev.Name - } +func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error { + cfg := config.New() + cfg.Set("renderToDisk", (c.s == nil && !c.r.renderToMemory) || (c.s != nil && c.s.renderToDisk)) + watch := c.r.buildWatch || (c.s != nil && c.s.serverWatch) + cfg.Set("environment", c.r.environment) - if files.IsContentFile(ev.Name) { - name = ev.Name - } + cfg.Set("internal", maps.Params{ + "running": running, + "watch": watch, + "verbose": c.r.verbose, + }) + conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, cfg)) + if err != nil { + return err + } + c.setConf(conf) + if c.onConfigLoaded != nil { + if err := c.onConfigLoaded(false); err != nil { + return err } } - return name + return nil + } -func formatByteCount(b uint64) string { - const unit = 1000 - if b < unit { - return fmt.Sprintf("%d B", b) +func (c *hugoBuilder) printChangeDetected(typ string) { + msg := "\nChange" + if typ != "" { + msg += " of " + typ } - div, exp := int64(unit), 0 - for n := b / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", - float64(b)/float64(div), "kMGTPE"[exp]) + msg += " detected, rebuilding site." + + c.r.logger.Println(msg) + const layout = "2006-01-02 15:04:05.000 -0700" + c.r.logger.Println(htime.Now().Format(layout)) +} + +func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error { + if err := c.errState.buildErr(); err != nil { + ferrs := herrors.UnwrapFileErrorsWithErrorContext(err) + for _, err := range ferrs { + events = append(events, fsnotify.Event{Name: err.Position().Filename, Op: fsnotify.Write}) + } + } + c.errState.setBuildErr(nil) + visited := c.visitedURLs.PeekAllSet() + h := c.hugo() + if c.fastRenderMode { + // Make sure we always render the home pages + for _, l := range c.conf().configs.Languages { + langPath := h.GetLangSubDir(l.Lang) + if langPath != "" { + langPath = langPath + "/" + } + home := h.PrependBasePath("/"+langPath, false) + visited[home] = true + } + } + return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: visited, ErrRecovery: c.errState.wasErr()}, events...) +} + +func (c *hugoBuilder) reloadConfig() error { + c.r.Reset() + c.r.configVersionID.Add(1) + oldConf := c.conf() + conf, err := c.r.ConfigFromConfig(c.r.configVersionID.Load(), c.conf()) + if err != nil { + return err + } + sameLen := len(oldConf.configs.Languages) == len(conf.configs.Languages) + if !sameLen { + if oldConf.configs.IsMultihost || conf.configs.IsMultihost { + return errors.New("multihost change detected, please restart server") + } + } + c.setConf(conf) + if c.onConfigLoaded != nil { + if err := c.onConfigLoaded(true); err != nil { + return err + } + } + + return nil } diff --git a/commands/import_jekyll.go b/commands/import.go similarity index 69% rename from commands/import_jekyll.go rename to commands/import.go index 93991121d..20d23dfac 100644 --- a/commands/import_jekyll.go +++ b/commands/import.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,252 +15,96 @@ package commands import ( "bytes" + "context" "errors" "fmt" "io" "os" "path/filepath" "regexp" + + jww "github.com/spf13/jwalterweatherman" + "strconv" "strings" "time" "unicode" - "github.com/gohugoio/hugo/parser/pageparser" - + "github.com/bep/simplecobra" "github.com/gohugoio/hugo/common/htime" "github.com/gohugoio/hugo/common/hugio" - - "github.com/gohugoio/hugo/parser/metadecoders" - "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/parser/pageparser" "github.com/spf13/afero" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" ) -var _ cmder = (*importCmd)(nil) - -type importCmd struct { - *baseCmd -} - -func newImportCmd() *importCmd { - cc := &importCmd{} - - cc.baseCmd = newBaseCmd(&cobra.Command{ - Use: "import", - Short: "Import your site from others.", - Long: `Import your site from other web site generators like Jekyll. - -Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.", - RunE: nil, - }) - - importJekyllCmd := &cobra.Command{ - Use: "jekyll", - Short: "hugo import from Jekyll", - Long: `hugo import from Jekyll. - +func newImportCommand() *importCommand { + var c *importCommand + c = &importCommand{ + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "jekyll", + short: "hugo import from Jekyll", + long: `hugo import from Jekyll. + Import from Jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.", - RunE: cc.importFromJekyll, - } - - importJekyllCmd.Flags().Bool("force", false, "allow import into non-empty target directory") - - cc.cmd.AddCommand(importJekyllCmd) - - return cc -} - -func (i *importCmd) importFromJekyll(cmd *cobra.Command, args []string) error { - if len(args) < 2 { - return newUserError(`import from jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.") - } - - jekyllRoot, err := filepath.Abs(filepath.Clean(args[0])) - if err != nil { - return newUserError("path error:", args[0]) - } - - targetDir, err := filepath.Abs(filepath.Clean(args[1])) - if err != nil { - return newUserError("path error:", args[1]) - } - - jww.INFO.Println("Import Jekyll from:", jekyllRoot, "to:", targetDir) - - if strings.HasPrefix(filepath.Dir(targetDir), jekyllRoot) { - return newUserError("abort: target path should not be inside the Jekyll root") - } - - forceImport, _ := cmd.Flags().GetBool("force") - - fs := afero.NewOsFs() - jekyllPostDirs, hasAnyPost := i.getJekyllDirInfo(fs, jekyllRoot) - if !hasAnyPost { - return errors.New("abort: jekyll root contains neither posts nor drafts") - } - - err = i.createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs, forceImport) - - if err != nil { - return newUserError(err) - } - - jww.FEEDBACK.Println("Importing...") - - fileCount := 0 - callback := func(path string, fi hugofs.FileMetaInfo, err error) error { - if err != nil { - return err - } - - if fi.IsDir() { - return nil - } - - relPath, err := filepath.Rel(jekyllRoot, path) - if err != nil { - return newUserError("get rel path error:", path) - } - - relPath = filepath.ToSlash(relPath) - draft := false - - switch { - case strings.Contains(relPath, "_posts/"): - relPath = filepath.Join("content/post", strings.Replace(relPath, "_posts/", "", -1)) - case strings.Contains(relPath, "_drafts/"): - relPath = filepath.Join("content/draft", strings.Replace(relPath, "_drafts/", "", -1)) - draft = true - default: - return nil - } - - fileCount++ - return convertJekyllPost(path, relPath, targetDir, draft) - } - - for jekyllPostDir, hasAnyPostInDir := range jekyllPostDirs { - if hasAnyPostInDir { - if err = helpers.SymbolicWalk(hugofs.Os, filepath.Join(jekyllRoot, jekyllPostDir), callback); err != nil { - return err - } - } - } - - jww.FEEDBACK.Println("Congratulations!", fileCount, "post(s) imported!") - jww.FEEDBACK.Println("Now, start Hugo by yourself:\n" + - "$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove") - jww.FEEDBACK.Println("$ cd " + args[1] + "\n$ hugo server --theme=herring-cove") - - return nil -} - -func (i *importCmd) getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) { - postDirs := make(map[string]bool) - hasAnyPost := false - if entries, err := os.ReadDir(jekyllRoot); err == nil { - for _, entry := range entries { - if entry.IsDir() { - subDir := filepath.Join(jekyllRoot, entry.Name()) - if isPostDir, hasAnyPostInDir := i.retrieveJekyllPostDir(fs, subDir); isPostDir { - postDirs[entry.Name()] = hasAnyPostInDir - if hasAnyPostInDir { - hasAnyPost = true + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + if len(args) < 2 { + return newUserError(`import from jekyll requires two paths, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`.") } - } - } - } - } - return postDirs, hasAnyPost -} - -func (i *importCmd) retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) { - if strings.HasSuffix(dir, "_posts") || strings.HasSuffix(dir, "_drafts") { - isEmpty, _ := helpers.IsEmpty(dir, fs) - return true, !isEmpty - } - - if entries, err := os.ReadDir(dir); err == nil { - for _, entry := range entries { - if entry.IsDir() { - subDir := filepath.Join(dir, entry.Name()) - if isPostDir, hasAnyPost := i.retrieveJekyllPostDir(fs, subDir); isPostDir { - return isPostDir, hasAnyPost - } - } - } - } - - return false, true -} - -func (i *importCmd) createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[string]bool, force bool) error { - fs := &afero.OsFs{} - if exists, _ := helpers.Exists(targetDir, fs); exists { - if isDir, _ := helpers.IsDir(targetDir, fs); !isDir { - return errors.New("target path \"" + targetDir + "\" exists but is not a directory") - } - - isEmpty, _ := helpers.IsEmpty(targetDir, fs) - - if !isEmpty && !force { - return errors.New("target path \"" + targetDir + "\" exists and is not empty") - } - } - - jekyllConfig := i.loadJekyllConfig(fs, jekyllRoot) - - mkdir(targetDir, "layouts") - mkdir(targetDir, "content") - mkdir(targetDir, "archetypes") - mkdir(targetDir, "static") - mkdir(targetDir, "data") - mkdir(targetDir, "themes") - - i.createConfigFromJekyll(fs, targetDir, "yaml", jekyllConfig) - - i.copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"), jekyllPostDirs) - - return nil -} - -func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]any { - path := filepath.Join(jekyllRoot, "_config.yml") - - exists, err := helpers.Exists(path, fs) - - if err != nil || !exists { - jww.WARN.Println("_config.yaml not found: Is the specified Jekyll root correct?") - return nil - } - - f, err := fs.Open(path) - if err != nil { - return nil - } - - defer f.Close() - - b, err := io.ReadAll(f) - if err != nil { - return nil - } - - c, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.YAML) - if err != nil { - return nil + return c.importFromJekyll(args) + }, + withc: func(cmd *cobra.Command) { + cmd.Flags().BoolVar(&c.force, "force", false, "allow import into non-empty target directory") + }, + }, + }, } return c + } -func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]any) (err error) { +type importCommand struct { + r *rootCommand + + force bool + + commands []simplecobra.Commander +} + +func (c *importCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *importCommand) Name() string { + return "import" +} + +func (c *importCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + return nil +} + +func (c *importCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Import your site from others." + cmd.Long = `Import your site from other web site generators like Jekyll. + +Import requires a subcommand, e.g. ` + "`hugo import jekyll jekyll_root_path target_path`." + + return nil +} + +func (c *importCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + return nil +} + +func (i *importCommand) createConfigFromJekyll(fs afero.Fs, inpath string, kind metadecoders.Format, jekyllConfig map[string]any) (err error) { title := "My New Hugo Site" baseURL := "http://example.org/" @@ -293,10 +137,209 @@ func (i *importCmd) createConfigFromJekyll(fs afero.Fs, inpath string, kind meta return err } - return helpers.WriteToDisk(filepath.Join(inpath, "config."+string(kind)), &buf, fs) + return helpers.WriteToDisk(filepath.Join(inpath, "hugo."+string(kind)), &buf, fs) } -func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) { +func (c *importCommand) getJekyllDirInfo(fs afero.Fs, jekyllRoot string) (map[string]bool, bool) { + postDirs := make(map[string]bool) + hasAnyPost := false + if entries, err := os.ReadDir(jekyllRoot); err == nil { + for _, entry := range entries { + if entry.IsDir() { + subDir := filepath.Join(jekyllRoot, entry.Name()) + if isPostDir, hasAnyPostInDir := c.retrieveJekyllPostDir(fs, subDir); isPostDir { + postDirs[entry.Name()] = hasAnyPostInDir + if hasAnyPostInDir { + hasAnyPost = true + } + } + } + } + } + return postDirs, hasAnyPost +} + +func (c *importCommand) createSiteFromJekyll(jekyllRoot, targetDir string, jekyllPostDirs map[string]bool) error { + fs := &afero.OsFs{} + if exists, _ := helpers.Exists(targetDir, fs); exists { + if isDir, _ := helpers.IsDir(targetDir, fs); !isDir { + return errors.New("target path \"" + targetDir + "\" exists but is not a directory") + } + + isEmpty, _ := helpers.IsEmpty(targetDir, fs) + + if !isEmpty && !c.force { + return errors.New("target path \"" + targetDir + "\" exists and is not empty") + } + } + + jekyllConfig := c.loadJekyllConfig(fs, jekyllRoot) + + mkdir(targetDir, "layouts") + mkdir(targetDir, "content") + mkdir(targetDir, "archetypes") + mkdir(targetDir, "static") + mkdir(targetDir, "data") + mkdir(targetDir, "themes") + + c.createConfigFromJekyll(fs, targetDir, "yaml", jekyllConfig) + + c.copyJekyllFilesAndFolders(jekyllRoot, filepath.Join(targetDir, "static"), jekyllPostDirs) + + return nil +} + +func (c *importCommand) convertJekyllContent(m any, content string) (string, error) { + metadata, _ := maps.ToStringMapE(m) + + lines := strings.Split(content, "\n") + var resultLines []string + for _, line := range lines { + resultLines = append(resultLines, strings.Trim(line, "\r\n")) + } + + content = strings.Join(resultLines, "\n") + + excerptSep := "" + if value, ok := metadata["excerpt_separator"]; ok { + if str, strOk := value.(string); strOk { + content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1) + } + } + + replaceList := []struct { + re *regexp.Regexp + replace string + }{ + {regexp.MustCompile("(?i)"), ""}, + {regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"}, + {regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"}, + } + + for _, replace := range replaceList { + content = replace.re.ReplaceAllString(content, replace.replace) + } + + replaceListFunc := []struct { + re *regexp.Regexp + replace func(string) string + }{ + // Octopress image tag: http://octopress.org/docs/plugins/image-tag/ + {regexp.MustCompile(`{%\s+img\s*(.*?)\s*%}`), c.replaceImageTag}, + {regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), c.replaceHighlightTag}, + } + + for _, replace := range replaceListFunc { + content = replace.re.ReplaceAllStringFunc(content, replace.replace) + } + + var buf bytes.Buffer + if len(metadata) != 0 { + err := parser.InterfaceToFrontMatter(m, metadecoders.YAML, &buf) + if err != nil { + return "", err + } + } + buf.WriteString(content) + + return buf.String(), nil +} + +func (c *importCommand) convertJekyllMetaData(m any, postName string, postDate time.Time, draft bool) (any, error) { + metadata, err := maps.ToStringMapE(m) + if err != nil { + return nil, err + } + + if draft { + metadata["draft"] = true + } + + for key, value := range metadata { + lowerKey := strings.ToLower(key) + + switch lowerKey { + case "layout": + delete(metadata, key) + case "permalink": + if str, ok := value.(string); ok { + metadata["url"] = str + } + delete(metadata, key) + case "category": + if str, ok := value.(string); ok { + metadata["categories"] = []string{str} + } + delete(metadata, key) + case "excerpt_separator": + if key != lowerKey { + delete(metadata, key) + metadata[lowerKey] = value + } + case "date": + if str, ok := value.(string); ok { + re := regexp.MustCompile(`(\d+):(\d+):(\d+)`) + r := re.FindAllStringSubmatch(str, -1) + if len(r) > 0 { + hour, _ := strconv.Atoi(r[0][1]) + minute, _ := strconv.Atoi(r[0][2]) + second, _ := strconv.Atoi(r[0][3]) + postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC) + } + } + delete(metadata, key) + } + + } + + metadata["date"] = postDate.Format(time.RFC3339) + + return metadata, nil +} + +func (c *importCommand) convertJekyllPost(path, relPath, targetDir string, draft bool) error { + jww.TRACE.Println("Converting", path) + + filename := filepath.Base(path) + postDate, postName, err := c.parseJekyllFilename(filename) + if err != nil { + c.r.Printf("Failed to parse filename '%s': %s. Skipping.", filename, err) + return nil + } + + jww.TRACE.Println(filename, postDate, postName) + + targetFile := filepath.Join(targetDir, relPath) + targetParentDir := filepath.Dir(targetFile) + os.MkdirAll(targetParentDir, 0777) + + contentBytes, err := os.ReadFile(path) + if err != nil { + c.r.logger.Errorln("Read file error:", path) + return err + } + pf, err := pageparser.ParseFrontMatterAndContent(bytes.NewReader(contentBytes)) + if err != nil { + return fmt.Errorf("failed to parse file %q: %s", filename, err) + } + newmetadata, err := c.convertJekyllMetaData(pf.FrontMatter, postName, postDate, draft) + if err != nil { + return fmt.Errorf("failed to convert metadata for file %q: %s", filename, err) + } + + content, err := c.convertJekyllContent(newmetadata, string(pf.Content)) + if err != nil { + return fmt.Errorf("failed to convert content for file %q: %s", filename, err) + } + + fs := hugofs.Os + if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil { + return fmt.Errorf("failed to save file %q: %s", filename, err) + } + return nil +} + +func (c *importCommand) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPostDirs map[string]bool) (err error) { fs := hugofs.Os fi, err := fs.Stat(jekyllRoot) @@ -353,7 +396,116 @@ func (i *importCmd) copyJekyllFilesAndFolders(jekyllRoot, dest string, jekyllPos return nil } -func parseJekyllFilename(filename string) (time.Time, string, error) { +func (c *importCommand) importFromJekyll(args []string) error { + + jekyllRoot, err := filepath.Abs(filepath.Clean(args[0])) + if err != nil { + return newUserError("path error:", args[0]) + } + + targetDir, err := filepath.Abs(filepath.Clean(args[1])) + if err != nil { + return newUserError("path error:", args[1]) + } + + c.r.Println("Import Jekyll from:", jekyllRoot, "to:", targetDir) + + if strings.HasPrefix(filepath.Dir(targetDir), jekyllRoot) { + return newUserError("abort: target path should not be inside the Jekyll root") + } + + fs := afero.NewOsFs() + jekyllPostDirs, hasAnyPost := c.getJekyllDirInfo(fs, jekyllRoot) + if !hasAnyPost { + return errors.New("abort: jekyll root contains neither posts nor drafts") + } + + err = c.createSiteFromJekyll(jekyllRoot, targetDir, jekyllPostDirs) + if err != nil { + return newUserError(err) + } + + c.r.Println("Importing...") + + fileCount := 0 + callback := func(path string, fi hugofs.FileMetaInfo, err error) error { + if err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + relPath, err := filepath.Rel(jekyllRoot, path) + if err != nil { + return newUserError("get rel path error:", path) + } + + relPath = filepath.ToSlash(relPath) + draft := false + + switch { + case strings.Contains(relPath, "_posts/"): + relPath = filepath.Join("content/post", strings.Replace(relPath, "_posts/", "", -1)) + case strings.Contains(relPath, "_drafts/"): + relPath = filepath.Join("content/draft", strings.Replace(relPath, "_drafts/", "", -1)) + draft = true + default: + return nil + } + + fileCount++ + return c.convertJekyllPost(path, relPath, targetDir, draft) + } + + for jekyllPostDir, hasAnyPostInDir := range jekyllPostDirs { + if hasAnyPostInDir { + if err = helpers.SymbolicWalk(hugofs.Os, filepath.Join(jekyllRoot, jekyllPostDir), callback); err != nil { + return err + } + } + } + + c.r.Println("Congratulations!", fileCount, "post(s) imported!") + c.r.Println("Now, start Hugo by yourself:\n" + + "$ git clone https://github.com/spf13/herring-cove.git " + args[1] + "/themes/herring-cove") + c.r.Println("$ cd " + args[1] + "\n$ hugo server --theme=herring-cove") + + return nil +} + +func (c *importCommand) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]any { + path := filepath.Join(jekyllRoot, "_config.yml") + + exists, err := helpers.Exists(path, fs) + + if err != nil || !exists { + c.r.Println("_config.yaml not found: Is the specified Jekyll root correct?") + return nil + } + + f, err := fs.Open(path) + if err != nil { + return nil + } + + defer f.Close() + + b, err := io.ReadAll(f) + if err != nil { + return nil + } + + m, err := metadecoders.Default.UnmarshalToMap(b, metadecoders.YAML) + if err != nil { + return nil + } + + return m +} + +func (c *importCommand) parseJekyllFilename(filename string) (time.Time, string, error) { re := regexp.MustCompile(`(\d+-\d+-\d+)-(.+)\..*`) r := re.FindAllStringSubmatch(filename, -1) if len(r) == 0 { @@ -370,163 +522,7 @@ func parseJekyllFilename(filename string) (time.Time, string, error) { return postDate, postName, nil } -func convertJekyllPost(path, relPath, targetDir string, draft bool) error { - jww.TRACE.Println("Converting", path) - - filename := filepath.Base(path) - postDate, postName, err := parseJekyllFilename(filename) - if err != nil { - jww.WARN.Printf("Failed to parse filename '%s': %s. Skipping.", filename, err) - return nil - } - - jww.TRACE.Println(filename, postDate, postName) - - targetFile := filepath.Join(targetDir, relPath) - targetParentDir := filepath.Dir(targetFile) - os.MkdirAll(targetParentDir, 0777) - - contentBytes, err := os.ReadFile(path) - if err != nil { - jww.ERROR.Println("Read file error:", path) - return err - } - - pf, err := pageparser.ParseFrontMatterAndContent(bytes.NewReader(contentBytes)) - if err != nil { - jww.ERROR.Println("Parse file error:", path) - return err - } - - newmetadata, err := convertJekyllMetaData(pf.FrontMatter, postName, postDate, draft) - if err != nil { - jww.ERROR.Println("Convert metadata error:", path) - return err - } - - content, err := convertJekyllContent(newmetadata, string(pf.Content)) - if err != nil { - jww.ERROR.Println("Converting Jekyll error:", path) - return err - } - - fs := hugofs.Os - if err := helpers.WriteToDisk(targetFile, strings.NewReader(content), fs); err != nil { - return fmt.Errorf("failed to save file %q: %s", filename, err) - } - - return nil -} - -func convertJekyllMetaData(m any, postName string, postDate time.Time, draft bool) (any, error) { - metadata, err := maps.ToStringMapE(m) - if err != nil { - return nil, err - } - - if draft { - metadata["draft"] = true - } - - for key, value := range metadata { - lowerKey := strings.ToLower(key) - - switch lowerKey { - case "layout": - delete(metadata, key) - case "permalink": - if str, ok := value.(string); ok { - metadata["url"] = str - } - delete(metadata, key) - case "category": - if str, ok := value.(string); ok { - metadata["categories"] = []string{str} - } - delete(metadata, key) - case "excerpt_separator": - if key != lowerKey { - delete(metadata, key) - metadata[lowerKey] = value - } - case "date": - if str, ok := value.(string); ok { - re := regexp.MustCompile(`(\d+):(\d+):(\d+)`) - r := re.FindAllStringSubmatch(str, -1) - if len(r) > 0 { - hour, _ := strconv.Atoi(r[0][1]) - minute, _ := strconv.Atoi(r[0][2]) - second, _ := strconv.Atoi(r[0][3]) - postDate = time.Date(postDate.Year(), postDate.Month(), postDate.Day(), hour, minute, second, 0, time.UTC) - } - } - delete(metadata, key) - } - - } - - metadata["date"] = postDate.Format(time.RFC3339) - - return metadata, nil -} - -func convertJekyllContent(m any, content string) (string, error) { - metadata, _ := maps.ToStringMapE(m) - - lines := strings.Split(content, "\n") - var resultLines []string - for _, line := range lines { - resultLines = append(resultLines, strings.Trim(line, "\r\n")) - } - - content = strings.Join(resultLines, "\n") - - excerptSep := "" - if value, ok := metadata["excerpt_separator"]; ok { - if str, strOk := value.(string); strOk { - content = strings.Replace(content, strings.TrimSpace(str), excerptSep, -1) - } - } - - replaceList := []struct { - re *regexp.Regexp - replace string - }{ - {regexp.MustCompile("(?i)"), ""}, - {regexp.MustCompile(`\{%\s*raw\s*%\}\s*(.*?)\s*\{%\s*endraw\s*%\}`), "$1"}, - {regexp.MustCompile(`{%\s*endhighlight\s*%}`), "{{< / highlight >}}"}, - } - - for _, replace := range replaceList { - content = replace.re.ReplaceAllString(content, replace.replace) - } - - replaceListFunc := []struct { - re *regexp.Regexp - replace func(string) string - }{ - // Octopress image tag: http://octopress.org/docs/plugins/image-tag/ - {regexp.MustCompile(`{%\s+img\s*(.*?)\s*%}`), replaceImageTag}, - {regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`), replaceHighlightTag}, - } - - for _, replace := range replaceListFunc { - content = replace.re.ReplaceAllStringFunc(content, replace.replace) - } - - var buf bytes.Buffer - if len(metadata) != 0 { - err := parser.InterfaceToFrontMatter(m, metadecoders.YAML, &buf) - if err != nil { - return "", err - } - } - buf.WriteString(content) - - return buf.String(), nil -} - -func replaceHighlightTag(match string) string { +func (c *importCommand) replaceHighlightTag(match string) string { r := regexp.MustCompile(`{%\s*highlight\s*(.*?)\s*%}`) parts := r.FindStringSubmatch(match) lastQuote := rune(0) @@ -570,35 +566,55 @@ func replaceHighlightTag(match string) string { return result.String() } -func replaceImageTag(match string) string { +func (c *importCommand) replaceImageTag(match string) string { r := regexp.MustCompile(`{%\s+img\s*(\p{L}*)\s+([\S]*/[\S]+)\s+(\d*)\s*(\d*)\s*(.*?)\s*%}`) result := bytes.NewBufferString("{{< figure ") parts := r.FindStringSubmatch(match) // Index 0 is the entire string, ignore - replaceOptionalPart(result, "class", parts[1]) - replaceOptionalPart(result, "src", parts[2]) - replaceOptionalPart(result, "width", parts[3]) - replaceOptionalPart(result, "height", parts[4]) + c.replaceOptionalPart(result, "class", parts[1]) + c.replaceOptionalPart(result, "src", parts[2]) + c.replaceOptionalPart(result, "width", parts[3]) + c.replaceOptionalPart(result, "height", parts[4]) // title + alt part := parts[5] if len(part) > 0 { splits := strings.Split(part, "'") lenSplits := len(splits) if lenSplits == 1 { - replaceOptionalPart(result, "title", splits[0]) + c.replaceOptionalPart(result, "title", splits[0]) } else if lenSplits == 3 { - replaceOptionalPart(result, "title", splits[1]) + c.replaceOptionalPart(result, "title", splits[1]) } else if lenSplits == 5 { - replaceOptionalPart(result, "title", splits[1]) - replaceOptionalPart(result, "alt", splits[3]) + c.replaceOptionalPart(result, "title", splits[1]) + c.replaceOptionalPart(result, "alt", splits[3]) } } result.WriteString(">}}") return result.String() } -func replaceOptionalPart(buffer *bytes.Buffer, partName string, part string) { +func (c *importCommand) replaceOptionalPart(buffer *bytes.Buffer, partName string, part string) { if len(part) > 0 { buffer.WriteString(partName + "=\"" + part + "\" ") } } + +func (c *importCommand) retrieveJekyllPostDir(fs afero.Fs, dir string) (bool, bool) { + if strings.HasSuffix(dir, "_posts") || strings.HasSuffix(dir, "_drafts") { + isEmpty, _ := helpers.IsEmpty(dir, fs) + return true, !isEmpty + } + + if entries, err := os.ReadDir(dir); err == nil { + for _, entry := range entries { + if entry.IsDir() { + subDir := filepath.Join(dir, entry.Name()) + if isPostDir, hasAnyPost := c.retrieveJekyllPostDir(fs, subDir); isPostDir { + return isPostDir, hasAnyPost + } + } + } + } + + return false, true +} diff --git a/commands/import_jekyll_test.go b/commands/import_jekyll_test.go deleted file mode 100644 index dbe4e25d0..000000000 --- a/commands/import_jekyll_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "encoding/json" - "testing" - "time" - - qt "github.com/frankban/quicktest" -) - -func TestParseJekyllFilename(t *testing.T) { - c := qt.New(t) - filenameArray := []string{ - "2015-01-02-test.md", - "2012-03-15-中文.markup", - } - - expectResult := []struct { - postDate time.Time - postName string - }{ - {time.Date(2015, time.January, 2, 0, 0, 0, 0, time.UTC), "test"}, - {time.Date(2012, time.March, 15, 0, 0, 0, 0, time.UTC), "中文"}, - } - - for i, filename := range filenameArray { - postDate, postName, err := parseJekyllFilename(filename) - c.Assert(err, qt.IsNil) - c.Assert(expectResult[i].postDate.Format("2006-01-02"), qt.Equals, postDate.Format("2006-01-02")) - c.Assert(expectResult[i].postName, qt.Equals, postName) - } -} - -func TestConvertJekyllMetadata(t *testing.T) { - c := qt.New(t) - testDataList := []struct { - metadata any - postName string - postDate time.Time - draft bool - expect string - }{ - { - map[any]any{}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"date":"2015-10-01T00:00:00Z"}`, - }, - { - map[any]any{}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), true, - `{"date":"2015-10-01T00:00:00Z","draft":true}`, - }, - { - map[any]any{"Permalink": "/permalink.html", "layout": "post"}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`, - }, - { - map[any]any{"permalink": "/permalink.html"}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"date":"2015-10-01T00:00:00Z","url":"/permalink.html"}`, - }, - { - map[any]any{"category": nil, "permalink": 123}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"date":"2015-10-01T00:00:00Z"}`, - }, - { - map[any]any{"Excerpt_Separator": "sep"}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"date":"2015-10-01T00:00:00Z","excerpt_separator":"sep"}`, - }, - { - map[any]any{"category": "book", "layout": "post", "Others": "Goods", "Date": "2015-10-01 12:13:11"}, - "testPost", time.Date(2015, 10, 1, 0, 0, 0, 0, time.UTC), false, - `{"Others":"Goods","categories":["book"],"date":"2015-10-01T12:13:11Z"}`, - }, - } - - for _, data := range testDataList { - result, err := convertJekyllMetaData(data.metadata, data.postName, data.postDate, data.draft) - c.Assert(err, qt.IsNil) - jsonResult, err := json.Marshal(result) - c.Assert(err, qt.IsNil) - c.Assert(string(jsonResult), qt.Equals, data.expect) - } -} - -func TestConvertJekyllContent(t *testing.T) { - c := qt.New(t) - testDataList := []struct { - metadata any - content string - expect string - }{ - { - map[any]any{}, - "Test content\r\n\npart2 content", "Test content\n\npart2 content", - }, - { - map[any]any{}, - "Test content\n\npart2 content", "Test content\n\npart2 content", - }, - { - map[any]any{"excerpt_separator": ""}, - "Test content\n\npart2 content", - "---\nexcerpt_separator: \n---\nTest content\n\npart2 content", - }, - {map[any]any{}, "{% raw %}text{% endraw %}", "text"}, - {map[any]any{}, "{%raw%} text2 {%endraw %}", "text2"}, - { - map[any]any{}, - "{% highlight go %}\nvar s int\n{% endhighlight %}", - "{{< highlight go >}}\nvar s int\n{{< / highlight >}}", - }, - { - map[any]any{}, - "{% highlight go linenos hl_lines=\"1 2\" %}\nvar s string\nvar i int\n{% endhighlight %}", - "{{< highlight go \"linenos=table,hl_lines=1 2\" >}}\nvar s string\nvar i int\n{{< / highlight >}}", - }, - - // Octopress image tag - { - map[any]any{}, - "{% img http://placekitten.com/890/280 %}", - "{{< figure src=\"http://placekitten.com/890/280\" >}}", - }, - { - map[any]any{}, - "{% img left http://placekitten.com/320/250 Place Kitten #2 %}", - "{{< figure class=\"left\" src=\"http://placekitten.com/320/250\" title=\"Place Kitten #2\" >}}", - }, - { - map[any]any{}, - "{% img right http://placekitten.com/300/500 150 250 'Place Kitten #3' %}", - "{{< figure class=\"right\" src=\"http://placekitten.com/300/500\" width=\"150\" height=\"250\" title=\"Place Kitten #3\" >}}", - }, - { - map[any]any{}, - "{% img right http://placekitten.com/300/500 150 250 'Place Kitten #4' 'An image of a very cute kitten' %}", - "{{< figure class=\"right\" src=\"http://placekitten.com/300/500\" width=\"150\" height=\"250\" title=\"Place Kitten #4\" alt=\"An image of a very cute kitten\" >}}", - }, - { - map[any]any{}, - "{% img http://placekitten.com/300/500 150 250 'Place Kitten #4' 'An image of a very cute kitten' %}", - "{{< figure src=\"http://placekitten.com/300/500\" width=\"150\" height=\"250\" title=\"Place Kitten #4\" alt=\"An image of a very cute kitten\" >}}", - }, - { - map[any]any{}, - "{% img right /placekitten/300/500 'Place Kitten #4' 'An image of a very cute kitten' %}", - "{{< figure class=\"right\" src=\"/placekitten/300/500\" title=\"Place Kitten #4\" alt=\"An image of a very cute kitten\" >}}", - }, - { - map[any]any{"category": "book", "layout": "post", "Date": "2015-10-01 12:13:11"}, - "somecontent", - "---\nDate: \"2015-10-01 12:13:11\"\ncategory: book\nlayout: post\n---\nsomecontent", - }, - } - for _, data := range testDataList { - result, err := convertJekyllContent(data.metadata, data.content) - c.Assert(result, qt.Equals, data.expect) - c.Assert(err, qt.IsNil) - } -} diff --git a/commands/limit_darwin.go b/commands/limit_darwin.go deleted file mode 100644 index 6799f37b1..000000000 --- a/commands/limit_darwin.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "syscall" - - "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*limitCmd)(nil) - -type limitCmd struct { - *baseCmd -} - -func newLimitCmd() *limitCmd { - ccmd := &cobra.Command{ - Use: "ulimit", - Short: "Check system ulimit settings", - Long: `Hugo will inspect the current ulimit settings on the system. -This is primarily to ensure that Hugo can watch enough files on some OSs`, - RunE: func(cmd *cobra.Command, args []string) error { - var rLimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return newSystemError("Error Getting rlimit ", err) - } - - jww.FEEDBACK.Println("Current rLimit:", rLimit) - - if rLimit.Cur >= newRlimit { - return nil - } - - jww.FEEDBACK.Println("Attempting to increase limit") - rLimit.Cur = newRlimit - err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return newSystemError("Error Setting rLimit ", err) - } - err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - return newSystemError("Error Getting rLimit ", err) - } - jww.FEEDBACK.Println("rLimit after change:", rLimit) - - return nil - }, - } - - return &limitCmd{baseCmd: newBaseCmd(ccmd)} -} - -const newRlimit = 10240 - -func tweakLimit() { - var rLimit syscall.Rlimit - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - jww.WARN.Println("Unable to get rlimit:", err) - return - } - if rLimit.Cur < newRlimit { - rLimit.Cur = newRlimit - err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit) - if err != nil { - // This may not succeed, see https://github.com/golang/go/issues/30401 - jww.INFO.Println("Unable to increase number of open files limit:", err) - } - } -} diff --git a/commands/limit_others.go b/commands/limit_others.go deleted file mode 100644 index b141b7004..000000000 --- a/commands/limit_others.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !darwin -// +build !darwin - -package commands - -func tweakLimit() { - // nothing to do -} diff --git a/commands/list.go b/commands/list.go index 4b62c91c5..2f2e29887 100644 --- a/commands/list.go +++ b/commands/list.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,197 +14,154 @@ package commands import ( + "context" "encoding/csv" - "os" - "strconv" - "strings" "time" + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" ) -var _ cmder = (*listCmd)(nil) +// newListCommand creates a new list command and its subcommands. +func newListCommand() *listCommand { -type listCmd struct { - *baseBuilderCmd -} - -func (lc *listCmd) buildSites(config map[string]any) (*hugolib.HugoSites, error) { - cfgInit := func(c *commandeer) error { - for key, value := range config { - c.Set(key, value) + list := func(cd *simplecobra.Commandeer, r *rootCommand, createRecord func(page.Page) []string, opts ...any) error { + bcfg := hugolib.BuildCfg{SkipRender: true} + cfg := config.New() + for i := 0; i < len(opts); i += 2 { + cfg.Set(opts[i].(string), opts[i+1]) } + h, err := r.Build(cd, bcfg, cfg) + if err != nil { + return err + } + + writer := csv.NewWriter(r.Out) + defer writer.Flush() + + for _, p := range h.Pages() { + if record := createRecord(p); record != nil { + if err := writer.Write(record); err != nil { + return err + } + if err != nil { + return err + } + } + } + return nil + } - c, err := initializeConfig(true, true, false, &lc.hugoBuilderCommon, lc, cfgInit) - if err != nil { - return nil, err - } + return &listCommand{ + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "drafts", + short: "List all drafts", + long: `List all of the drafts in your content directory.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + createRecord := func(p page.Page) []string { + if !p.Draft() || p.File().IsZero() { + return nil + } + return []string{ + p.File().Path(), + p.PublishDate().Format(time.RFC3339)} - sites, err := hugolib.NewHugoSites(*c.DepsCfg) - if err != nil { - return nil, newSystemError("Error creating sites", err) - } - - if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return nil, newSystemError("Error Processing Source Content", err) - } - - return sites, nil -} - -func (b *commandsBuilder) newListCmd() *listCmd { - cc := &listCmd{} - - cmd := &cobra.Command{ - Use: "list", - Short: "Listing out various types of content", - Long: `Listing out various types of content. - -List requires a subcommand, e.g. ` + "`hugo list drafts`.", - RunE: nil, - } - - cmd.AddCommand( - &cobra.Command{ - Use: "drafts", - Short: "List all drafts", - Long: `List all of the drafts in your content directory.`, - RunE: func(cmd *cobra.Command, args []string) error { - sites, err := cc.buildSites(map[string]any{"buildDrafts": true}) - if err != nil { - return newSystemError("Error building sites", err) - } - - for _, p := range sites.Pages() { - if p.Draft() { - jww.FEEDBACK.Println(strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator))) } - } - - return nil + return list(cd, r, createRecord, "buildDrafts", true) + }, }, - }, - &cobra.Command{ - Use: "future", - Short: "List all posts dated in the future", - Long: `List all of the posts in your content directory which will be posted in the future.`, - RunE: func(cmd *cobra.Command, args []string) error { - sites, err := cc.buildSites(map[string]any{"buildFuture": true}) - if err != nil { - return newSystemError("Error building sites", err) - } - - if err != nil { - return newSystemError("Error building sites", err) - } - - writer := csv.NewWriter(os.Stdout) - defer writer.Flush() - - for _, p := range sites.Pages() { - if resource.IsFuture(p) { - err := writer.Write([]string{ - strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)), + &simpleCommand{ + name: "future", + short: "List all posts dated in the future", + long: `List all of the posts in your content directory which will be posted in the future.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + createRecord := func(p page.Page) []string { + if !resource.IsFuture(p) || p.File().IsZero() { + return nil + } + return []string{ + p.File().Path(), p.PublishDate().Format(time.RFC3339), - }) - if err != nil { - return newSystemError("Error writing future posts to stdout", err) } + } - } - - return nil + return list(cd, r, createRecord, "buildFuture", true) + }, }, - }, - &cobra.Command{ - Use: "expired", - Short: "List all posts already expired", - Long: `List all of the posts in your content directory which has already expired.`, - RunE: func(cmd *cobra.Command, args []string) error { - sites, err := cc.buildSites(map[string]any{"buildExpired": true}) - if err != nil { - return newSystemError("Error building sites", err) - } - - if err != nil { - return newSystemError("Error building sites", err) - } - - writer := csv.NewWriter(os.Stdout) - defer writer.Flush() - - for _, p := range sites.Pages() { - if resource.IsExpired(p) { - err := writer.Write([]string{ - strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)), - p.ExpiryDate().Format(time.RFC3339), - }) - if err != nil { - return newSystemError("Error writing expired posts to stdout", err) + &simpleCommand{ + name: "expired", + short: "List all posts already expired", + long: `List all of the posts in your content directory which has already expired.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + createRecord := func(p page.Page) []string { + if !resource.IsExpired(p) || p.File().IsZero() { + return nil + } + return []string{ + p.File().Path(), + p.PublishDate().Format(time.RFC3339), } - } - } - return nil + } + return list(cd, r, createRecord, "buildExpired", true) + }, + }, + &simpleCommand{ + name: "all", + short: "List all posts", + long: `List all of the posts in your content directory, include drafts, future and expired pages.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + createRecord := func(p page.Page) []string { + if p.File().IsZero() { + return nil + } + return []string{ + p.File().Path(), + p.PublishDate().Format(time.RFC3339), + } + + } + return list(cd, r, createRecord, "buildDrafts", true, "buildFuture", true, "buildExpired", true) + }, }, }, - &cobra.Command{ - Use: "all", - Short: "List all posts", - Long: `List all of the posts in your content directory, include drafts, future and expired pages.`, - RunE: func(cmd *cobra.Command, args []string) error { - sites, err := cc.buildSites(map[string]any{ - "buildExpired": true, - "buildDrafts": true, - "buildFuture": true, - }) - if err != nil { - return newSystemError("Error building sites", err) - } + } - writer := csv.NewWriter(os.Stdout) - defer writer.Flush() - - writer.Write([]string{ - "path", - "slug", - "title", - "date", - "expiryDate", - "publishDate", - "draft", - "permalink", - }) - for _, p := range sites.Pages() { - if !p.IsPage() { - continue - } - err := writer.Write([]string{ - strings.TrimPrefix(p.File().Filename(), sites.WorkingDir+string(os.PathSeparator)), - p.Slug(), - p.Title(), - p.Date().Format(time.RFC3339), - p.ExpiryDate().Format(time.RFC3339), - p.PublishDate().Format(time.RFC3339), - strconv.FormatBool(p.Draft()), - p.Permalink(), - }) - if err != nil { - return newSystemError("Error writing posts to stdout", err) - } - } - - return nil - }, - }, - ) - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc +} + +type listCommand struct { + commands []simplecobra.Commander +} + +func (c *listCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *listCommand) Name() string { + return "list" +} + +func (c *listCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + // Do nothing. + return nil +} + +func (c *listCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Listing out various types of content" + cmd.Long = `Listing out various types of content. + +List requires a subcommand, e.g. hugo list drafts` + + return nil +} + +func (c *listCommand) Init(cd, runner *simplecobra.Commandeer) error { + return nil } diff --git a/commands/list_test.go b/commands/list_test.go deleted file mode 100644 index 8b2535571..000000000 --- a/commands/list_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package commands - -import ( - "bytes" - "encoding/csv" - "io" - "os" - "path/filepath" - "strings" - "testing" - - qt "github.com/frankban/quicktest" -) - -func captureStdout(f func() error) (string, error) { - old := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - err := f() - - w.Close() - os.Stdout = old - - var buf bytes.Buffer - io.Copy(&buf, r) - return buf.String(), err -} - -func TestListAll(t *testing.T) { - c := qt.New(t) - dir := createSimpleTestSite(t, testSiteConfig{}) - - hugoCmd := newCommandsBuilder().addAll().build() - cmd := hugoCmd.getCommand() - - t.Cleanup(func() { - os.RemoveAll(dir) - }) - - cmd.SetArgs([]string{"-s=" + dir, "list", "all"}) - - out, err := captureStdout(func() error { - _, err := cmd.ExecuteC() - return err - }) - c.Assert(err, qt.IsNil) - - r := csv.NewReader(strings.NewReader(out)) - - header, err := r.Read() - - c.Assert(err, qt.IsNil) - c.Assert(header, qt.DeepEquals, []string{ - "path", "slug", "title", - "date", "expiryDate", "publishDate", - "draft", "permalink", - }) - - record, err := r.Read() - - c.Assert(err, qt.IsNil) - c.Assert(record, qt.DeepEquals, []string{ - filepath.Join("content", "p1.md"), "", "P1", - "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", "0001-01-01T00:00:00Z", - "false", "https://example.org/p1/", - }) -} diff --git a/commands/mod.go b/commands/mod.go index 44a48bf79..a0e488ecd 100644 --- a/commands/mod.go +++ b/commands/mod.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,87 +14,18 @@ package commands import ( + "context" "errors" - "fmt" "os" "path/filepath" - "regexp" - "github.com/gohugoio/hugo/hugolib" - - "github.com/gohugoio/hugo/modules" + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/modules/npm" "github.com/spf13/cobra" ) -var _ cmder = (*modCmd)(nil) - -type modCmd struct { - *baseBuilderCmd -} - -func (c *modCmd) newVerifyCmd() *cobra.Command { - var clean bool - - verifyCmd := &cobra.Command{ - Use: "verify", - Short: "Verify dependencies.", - Long: `Verify checks that the dependencies of the current module, which are stored in a local downloaded source cache, have not been modified since being downloaded. -`, - RunE: func(cmd *cobra.Command, args []string) error { - return c.withModsClient(true, func(c *modules.Client) error { - return c.Verify(clean) - }) - }, - } - - verifyCmd.Flags().BoolVarP(&clean, "clean", "", false, "delete module cache for dependencies that fail verification") - - return verifyCmd -} - -var moduleNotFoundRe = regexp.MustCompile("module.*not found") - -func (c *modCmd) newCleanCmd() *cobra.Command { - var pattern string - var all bool - cmd := &cobra.Command{ - Use: "clean", - Short: "Delete the Hugo Module cache for the current project.", - Long: `Delete the Hugo Module cache for the current project. - -Note that after you run this command, all of your dependencies will be re-downloaded next time you run "hugo". - -Also note that if you configure a positive maxAge for the "modules" file cache, it will also be cleaned as part of "hugo --gc". - -`, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - com, err := c.initConfig(false) - - if err != nil && com == nil { - return err - } - - count, err := com.hugo().FileCaches.ModulesCache().Prune(true) - com.logger.Printf("Deleted %d files from module cache.", count) - return err - } - return c.withModsClient(true, func(c *modules.Client) error { - return c.Clean(pattern) - }) - }, - } - - cmd.Flags().StringVarP(&pattern, "pattern", "", "", `pattern matching module paths to clean (all if not set), e.g. "**hugo*"`) - cmd.Flags().BoolVarP(&all, "all", "", false, "clean entire module cache") - - return cmd -} - -func (b *commandsBuilder) newModCmd() *modCmd { - c := &modCmd{} - - const commonUsage = ` +const commonUsageMod = ` Note that Hugo will always start out by resolving the components defined in the site configuration, provided by a _vendor directory (if no --ignoreVendorPaths flag provided), Go Modules, or a folder inside the themes directory, in that order. @@ -103,27 +34,156 @@ See https://gohugo.io/hugo-modules/ for more information. ` - cmd := &cobra.Command{ - Use: "mod", - Short: "Various Hugo Modules helpers.", - Long: `Various helpers to help manage the modules in your project's dependency graph. +// buildConfigCommands creates a new config command and its subcommands. +func newModCommands() *modCommands { + var ( + clean bool + pattern string + all bool + ) -Most operations here requires a Go version installed on your system (>= Go 1.12) and the relevant VCS client (typically Git). -This is not needed if you only operate on modules inside /themes or if you have vendored them via "hugo mod vendor". + npmCommand := &simpleCommand{ + name: "npm", + short: "Various npm helpers.", + long: `Various npm (Node package manager) helpers.`, + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "pack", + short: "Experimental: Prepares and writes a composite package.json file for your project.", + long: `Prepares and writes a composite package.json file for your project. -` + commonUsage, +On first run it creates a "package.hugo.json" in the project root if not already there. This file will be used as a template file +with the base dependency set. - RunE: nil, +This set will be merged with all "package.hugo.json" files found in the dependency tree, picking the version closest to the project. + +This command is marked as 'Experimental'. We think it's a great idea, so it's not likely to be +removed from Hugo, but we need to test this out in "real life" to get a feel of it, +so this may/will change in future versions of Hugo. +`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + return npm.Pack(h.BaseFs.SourceFs, h.BaseFs.Assets.Dirs) + }, + }, + }, } - cmd.AddCommand(newModNPMCmd(c)) + return &modCommands{ + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "init", + short: "Initialize this project as a Hugo Module.", + long: `Initialize this project as a Hugo Module. + It will try to guess the module path, but you may help by passing it as an argument, e.g: + + hugo mod init github.com/gohugoio/testshortcodes + + Note that Hugo Modules supports multi-module projects, so you can initialize a Hugo Module + inside a subfolder on GitHub, as one example. + `, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + var initPath string + if len(args) >= 1 { + initPath = args[0] + } + return h.Configs.ModulesClient.Init(initPath) + }, + }, + &simpleCommand{ + name: "verify", + short: "Verify dependencies.", + long: `Verify checks that the dependencies of the current module, which are stored in a local downloaded source cache, have not been modified since being downloaded.`, + withc: func(cmd *cobra.Command) { + cmd.Flags().BoolVarP(&clean, "clean", "", false, "delete module cache for dependencies that fail verification") + }, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, nil)) + if err != nil { + return err + } + client := conf.configs.ModulesClient + return client.Verify(clean) + }, + }, + &simpleCommand{ + name: "graph", + short: "Print a module dependency graph.", + long: `Print a module dependency graph with information about module status (disabled, vendored). +Note that for vendored modules, that is the version listed and not the one from go.mod. +`, + withc: func(cmd *cobra.Command) { + cmd.Flags().BoolVarP(&clean, "clean", "", false, "delete module cache for dependencies that fail verification") + }, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, nil)) + if err != nil { + return err + } + client := conf.configs.ModulesClient + return client.Graph(os.Stdout) + }, + }, + &simpleCommand{ + name: "clean", + short: "Delete the Hugo Module cache for the current project.", + long: `Delete the Hugo Module cache for the current project.`, + withc: func(cmd *cobra.Command) { + cmd.Flags().StringVarP(&pattern, "pattern", "", "", `pattern matching module paths to clean (all if not set), e.g. "**hugo*"`) + cmd.Flags().BoolVarP(&all, "all", "", false, "clean entire module cache") + }, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + if all { + modCache := h.ResourceSpec.FileCaches.ModulesCache() + count, err := modCache.Prune(true) + r.Printf("Deleted %d files from module cache.", count) + return err + } - cmd.AddCommand( - &cobra.Command{ - Use: "get", - DisableFlagParsing: true, - Short: "Resolves dependencies in your current Hugo Project.", - Long: ` + return h.Configs.ModulesClient.Clean(pattern) + }, + }, + &simpleCommand{ + name: "tidy", + short: "Remove unused entries in go.mod and go.sum.", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + return h.Configs.ModulesClient.Tidy() + }, + }, + &simpleCommand{ + name: "vendor", + short: "Vendor all module dependencies into the _vendor directory.", + long: `Vendor all module dependencies into the _vendor directory. + If a module is vendored, that is where Hugo will look for it's dependencies. + `, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + return h.Configs.ModulesClient.Vendor() + }, + }, + + &simpleCommand{ + name: "get", + short: "Resolves dependencies in your current Hugo Project.", + long: ` Resolves dependencies in your current Hugo Project. Some examples: @@ -142,152 +202,109 @@ Install the latest versions of all module dependencies: hugo mod get -u ./... (recursive) Run "go help get" for more information. All flags available for "go get" is also relevant here. -` + commonUsage, - RunE: func(cmd *cobra.Command, args []string) error { - // We currently just pass on the flags we get to Go and - // need to do the flag handling manually. - if len(args) == 1 && args[0] == "-h" { - return cmd.Help() - } - - var lastArg string - if len(args) != 0 { - lastArg = args[len(args)-1] - } - - if lastArg == "./..." { - args = args[:len(args)-1] - // Do a recursive update. - dirname, err := os.Getwd() - if err != nil { - return err +` + commonUsageMod, + withc: func(cmd *cobra.Command) { + cmd.DisableFlagParsing = true + }, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + // We currently just pass on the flags we get to Go and + // need to do the flag handling manually. + if len(args) == 1 && args[0] == "-h" { + return errHelp } - // Sanity check. We do recursive walking and want to avoid - // accidents. - if len(dirname) < 5 { - return errors.New("must not be run from the file system root") + var lastArg string + if len(args) != 0 { + lastArg = args[len(args)-1] } - filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil + if lastArg == "./..." { + args = args[:len(args)-1] + // Do a recursive update. + dirname, err := os.Getwd() + if err != nil { + return err } - if info.Name() == "go.mod" { - // Found a module. - dir := filepath.Dir(path) - fmt.Println("Update module in", dir) - c.source = dir - err := c.withModsClient(false, func(c *modules.Client) error { - if len(args) == 1 && args[0] == "-h" { - return cmd.Help() - } - return c.Get(args...) - }) - if err != nil { - return err + // Sanity chesimplecobra. We do recursive walking and want to avoid + // accidents. + if len(dirname) < 5 { + return errors.New("must not be run from the file system root") + } + + filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil } + if info.Name() == "go.mod" { + // Found a module. + dir := filepath.Dir(path) + r.Println("Update module in", dir) + cfg := config.New() + cfg.Set("workingDir", dir) + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, cfg)) + if err != nil { + return err + } + client := conf.configs.ModulesClient + return client.Get(args...) - } - + } + return nil + }) return nil - }) - - return nil - } - - return c.withModsClient(false, func(c *modules.Client) error { - return c.Get(args...) - }) + } else { + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, nil)) + if err != nil { + return err + } + client := conf.configs.ModulesClient + return client.Get(args...) + } + }, }, + npmCommand, }, - &cobra.Command{ - Use: "graph", - Short: "Print a module dependency graph.", - Long: `Print a module dependency graph with information about module status (disabled, vendored). -Note that for vendored modules, that is the version listed and not the one from go.mod. -`, - RunE: func(cmd *cobra.Command, args []string) error { - return c.withModsClient(true, func(c *modules.Client) error { - return c.Graph(os.Stdout) - }) - }, - }, - &cobra.Command{ - Use: "init", - Short: "Initialize this project as a Hugo Module.", - Long: `Initialize this project as a Hugo Module. -It will try to guess the module path, but you may help by passing it as an argument, e.g: + } - hugo mod init github.com/gohugoio/testshortcodes - -Note that Hugo Modules supports multi-module projects, so you can initialize a Hugo Module -inside a subfolder on GitHub, as one example. -`, - RunE: func(cmd *cobra.Command, args []string) error { - var path string - if len(args) >= 1 { - path = args[0] - } - return c.withModsClient(false, func(c *modules.Client) error { - return c.Init(path) - }) - }, - }, - &cobra.Command{ - Use: "vendor", - Short: "Vendor all module dependencies into the _vendor directory.", - Long: `Vendor all module dependencies into the _vendor directory. - -If a module is vendored, that is where Hugo will look for it's dependencies. -`, - RunE: func(cmd *cobra.Command, args []string) error { - return c.withModsClient(true, func(c *modules.Client) error { - return c.Vendor() - }) - }, - }, - c.newVerifyCmd(), - &cobra.Command{ - Use: "tidy", - Short: "Remove unused entries in go.mod and go.sum.", - RunE: func(cmd *cobra.Command, args []string) error { - return c.withModsClient(true, func(c *modules.Client) error { - return c.Tidy() - }) - }, - }, - c.newCleanCmd(), - ) - - c.baseBuilderCmd = b.newBuilderCmd(cmd) - - return c } -func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client) error) error { - com, err := c.initConfig(failOnMissingConfig) +type modCommands struct { + r *rootCommand + + commands []simplecobra.Commander +} + +func (c *modCommands) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *modCommands) Name() string { + return "mod" +} + +func (c *modCommands) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + _, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), nil) if err != nil { return err } + //config := conf.configs.Base - return f(com.hugo().ModulesClient) + return nil } -func (c *modCmd) withHugo(f func(*hugolib.HugoSites) error) error { - com, err := c.initConfig(true) - if err != nil { - return err - } +func (c *modCommands) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Various Hugo Modules helpers." + cmd.Long = `Various helpers to help manage the modules in your project's dependency graph. +Most operations here requires a Go version installed on your system (>= Go 1.12) and the relevant VCS client (typically Git). +This is not needed if you only operate on modules inside /themes or if you have vendored them via "hugo mod vendor". - return f(com.hugo()) +` + commonUsageMod + cmd.RunE = nil + return nil } -func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) { - com, err := initializeConfig(failOnNoConfig, false, false, &c.hugoBuilderCommon, c, nil) - if err != nil { - return nil, err - } - return com, nil +func (c *modCommands) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + return nil } diff --git a/commands/mod_npm.go b/commands/mod_npm.go deleted file mode 100644 index 852d98571..000000000 --- a/commands/mod_npm.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2020 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "github.com/gohugoio/hugo/hugolib" - "github.com/gohugoio/hugo/modules/npm" - "github.com/spf13/cobra" -) - -func newModNPMCmd(c *modCmd) *cobra.Command { - cmd := &cobra.Command{ - Use: "npm", - Short: "Various npm helpers.", - Long: `Various npm (Node package manager) helpers.`, - RunE: func(cmd *cobra.Command, args []string) error { - return c.withHugo(func(h *hugolib.HugoSites) error { - return nil - }) - }, - } - - cmd.AddCommand(&cobra.Command{ - Use: "pack", - Short: "Experimental: Prepares and writes a composite package.json file for your project.", - Long: `Prepares and writes a composite package.json file for your project. - -On first run it creates a "package.hugo.json" in the project root if not already there. This file will be used as a template file -with the base dependency set. - -This set will be merged with all "package.hugo.json" files found in the dependency tree, picking the version closest to the project. - -This command is marked as 'Experimental'. We think it's a great idea, so it's not likely to be -removed from Hugo, but we need to test this out in "real life" to get a feel of it, -so this may/will change in future versions of Hugo. -`, - RunE: func(cmd *cobra.Command, args []string) error { - return c.withHugo(func(h *hugolib.HugoSites) error { - return npm.Pack(h.BaseFs.SourceFs, h.BaseFs.Assets.Dirs) - }) - }, - }) - - return cmd -} diff --git a/commands/new.go b/commands/new.go index a6c2c8ca1..3a0e3ad71 100644 --- a/commands/new.go +++ b/commands/new.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,114 +15,351 @@ package commands import ( "bytes" - "os" + "context" + "errors" + "fmt" "path/filepath" "strings" + "github.com/bep/simplecobra" + "github.com/gohugoio/hugo/common/htime" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/create" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/parser" + "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/afero" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" ) -var _ cmder = (*newCmd)(nil) +func newNewCommand() *newCommand { + var ( + configFormat string + force bool + contentType string + ) -type newCmd struct { - contentEditor string - contentType string - force bool + var c *newCommand + c = &newCommand{ + commands: []simplecobra.Commander{ + &simpleCommand{ + name: "content", + use: "content [path]", + short: "Create new content for your site", + long: `Create a new content file and automatically set the date and title. + It will guess which kind of file to create based on the path provided. + + You can also specify the kind with ` + "`-k KIND`" + `. + + If archetypes are provided in your theme or site, they will be used. + + Ensure you run this within the root directory of your site.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + if len(args) < 1 { + return errors.New("path needs to be provided") + } + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + return create.NewContent(h, contentType, args[0], force) + }, + withc: func(cmd *cobra.Command) { + cmd.Flags().StringVarP(&contentType, "kind", "k", "", "content type to create") + cmd.Flags().String("editor", "", "edit new content with this editor, if provided") + cmd.Flags().BoolVarP(&force, "force", "f", false, "overwrite file if it already exists") + }, + }, + &simpleCommand{ + name: "site", + use: "site [path]", + short: "Create a new site (skeleton)", + long: `Create a new site in the provided directory. +The new site will have the correct structure, but no content or theme yet. +Use ` + "`hugo new [contentPath]`" + ` to create new content.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + if len(args) < 1 { + return errors.New("path needs to be provided") + } + createpath, err := filepath.Abs(filepath.Clean(args[0])) + if err != nil { + return err + } + + cfg := config.New() + cfg.Set("workingDir", createpath) + cfg.Set("publishDir", "public") + + conf, err := r.ConfigFromProvider(r.configVersionID.Load(), flagsToCfg(cd, cfg)) + if err != nil { + return err + } + sourceFs := conf.fs.Source + + archeTypePath := filepath.Join(createpath, "archetypes") + dirs := []string{ + archeTypePath, + filepath.Join(createpath, "assets"), + filepath.Join(createpath, "content"), + filepath.Join(createpath, "data"), + filepath.Join(createpath, "layouts"), + filepath.Join(createpath, "static"), + filepath.Join(createpath, "themes"), + } + + if exists, _ := helpers.Exists(createpath, sourceFs); exists { + if isDir, _ := helpers.IsDir(createpath, sourceFs); !isDir { + return errors.New(createpath + " already exists but not a directory") + } + + isEmpty, _ := helpers.IsEmpty(createpath, sourceFs) + + switch { + case !isEmpty && !force: + return errors.New(createpath + " already exists and is not empty. See --force.") + + case !isEmpty && force: + all := append(dirs, filepath.Join(createpath, "hugo."+configFormat)) + for _, path := range all { + if exists, _ := helpers.Exists(path, sourceFs); exists { + return errors.New(path + " already exists") + } + } + } + } + + for _, dir := range dirs { + if err := sourceFs.MkdirAll(dir, 0777); err != nil { + return fmt.Errorf("failed to create dir: %w", err) + } + } + + c.newSiteCreateConfig(sourceFs, createpath, configFormat) + + // Create a default archetype file. + helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"), + strings.NewReader(create.DefaultArchetypeTemplateTemplate), sourceFs) + + r.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", createpath) + r.Println(c.newSiteNextStepsText()) + + return nil + }, + withc: func(cmd *cobra.Command) { + cmd.Flags().StringVarP(&configFormat, "format", "f", "toml", "config file format") + cmd.Flags().BoolVar(&force, "force", false, "init inside non-empty directory") + }, + }, + &simpleCommand{ + name: "theme", + use: "theme [path]", + short: "Create a new site (skeleton)", + long: `Create a new site in the provided directory. +The new site will have the correct structure, but no content or theme yet. +Use ` + "`hugo new [contentPath]`" + ` to create new content.`, + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + h, err := r.Hugo(flagsToCfg(cd, nil)) + if err != nil { + return err + } + ps := h.PathSpec + sourceFs := ps.Fs.Source + themesDir := h.Configs.LoadingInfo.BaseConfig.ThemesDir + createpath := ps.AbsPathify(filepath.Join(themesDir, args[0])) + r.Println("Creating theme at", createpath) + + if x, _ := helpers.Exists(createpath, sourceFs); x { + return errors.New(createpath + " already exists") + } + + for _, filename := range []string{ + "index.html", + "404.html", + "_default/list.html", + "_default/single.html", + "partials/head.html", + "partials/header.html", + "partials/footer.html", + } { + touchFile(sourceFs, filepath.Join(createpath, "layouts", filename)) + } + + baseofDefault := []byte(` + + {{- partial "head.html" . -}} + + {{- partial "header.html" . -}} +
+ {{- block "main" . }}{{- end }} +
+ {{- partial "footer.html" . -}} + + +`) + + err = helpers.WriteToDisk(filepath.Join(createpath, "layouts", "_default", "baseof.html"), bytes.NewReader(baseofDefault), sourceFs) + if err != nil { + return err + } + + mkdir(createpath, "archetypes") + + archDefault := []byte("+++\n+++\n") + + err = helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), sourceFs) + if err != nil { + return err + } + + mkdir(createpath, "static", "js") + mkdir(createpath, "static", "css") + + by := []byte(`The MIT License (MIT) + +Copyright (c) ` + htime.Now().Format("2006") + ` YOUR_NAME_HERE + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +`) + + err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE"), bytes.NewReader(by), sourceFs) + if err != nil { + return err + } + + c.createThemeMD(ps.Fs.Source, createpath) + + return nil + }, + }, + }, + } + + return c - *baseBuilderCmd } -func (b *commandsBuilder) newNewCmd() *newCmd { - cmd := &cobra.Command{ - Use: "new [path]", - Short: "Create new content for your site", - Long: `Create a new content file and automatically set the date and title. +type newCommand struct { + rootCmd *rootCommand + + commands []simplecobra.Commander +} + +func (c *newCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *newCommand) Name() string { + return "new" +} + +func (c *newCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + return nil +} + +func (c *newCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Create new content for your site" + cmd.Long = `Create a new content file and automatically set the date and title. It will guess which kind of file to create based on the path provided. You can also specify the kind with ` + "`-k KIND`" + `. If archetypes are provided in your theme or site, they will be used. -Ensure you run this within the root directory of your site.`, - } - - cc := &newCmd{baseBuilderCmd: b.newBuilderCmd(cmd)} - - cmd.Flags().StringVarP(&cc.contentType, "kind", "k", "", "content type to create") - cmd.Flags().StringVar(&cc.contentEditor, "editor", "", "edit new content with this editor, if provided") - cmd.Flags().BoolVarP(&cc.force, "force", "f", false, "overwrite file if it already exists") - - cmd.AddCommand(b.newNewSiteCmd().getCommand()) - cmd.AddCommand(b.newNewThemeCmd().getCommand()) - - cmd.RunE = cc.newContent - - return cc +Ensure you run this within the root directory of your site.` + return nil } -func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { - cfgInit := func(c *commandeer) error { - if cmd.Flags().Changed("editor") { - c.Set("newContentEditor", n.contentEditor) - } - return nil +func (c *newCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.rootCmd = cd.Root.Command.(*rootCommand) + return nil +} + +func (c *newCommand) newSiteCreateConfig(fs afero.Fs, inpath string, kind string) (err error) { + in := map[string]string{ + "baseURL": "http://example.org/", + "title": "My New Hugo Site", + "languageCode": "en-us", } - c, err := initializeConfig(true, true, false, &n.hugoBuilderCommon, n, cfgInit) + var buf bytes.Buffer + err = parser.InterfaceToConfig(in, metadecoders.FormatFromString(kind), &buf) if err != nil { return err } - if len(args) < 1 { - return newUserError("path needs to be provided") - } - - return create.NewContent(c.hugo(), n.contentType, args[0], n.force) + return helpers.WriteToDisk(filepath.Join(inpath, "hugo."+kind), &buf, fs) } -func mkdir(x ...string) { - p := filepath.Join(x...) +func (c *newCommand) newSiteNextStepsText() string { + var nextStepsText bytes.Buffer - err := os.MkdirAll(p, 0777) // before umask + nextStepsText.WriteString(`Just a few more steps and you're ready to go: + +1. Download a theme into the same-named folder. + Choose a theme from https://themes.gohugo.io/ or + create your own with the "hugo new theme " command. +2. Perhaps you want to add some content. You can add single files + with "hugo new `) + + nextStepsText.WriteString(filepath.Join("", ".")) + + nextStepsText.WriteString(`". +3. Start the built-in live server via "hugo server". + +Visit https://gohugo.io/ for quickstart guide and full documentation.`) + + return nextStepsText.String() +} + +func (c *newCommand) createThemeMD(fs afero.Fs, inpath string) (err error) { + + by := []byte(`# theme.toml template for a Hugo theme +# See https://github.com/gohugoio/hugoThemes#themetoml for an example + +name = "` + strings.Title(helpers.MakeTitle(filepath.Base(inpath))) + `" +license = "MIT" +licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE" +description = "" +homepage = "http://example.com/" +tags = [] +features = [] +min_version = "0.41.0" + +[author] + name = "" + homepage = "" + +# If porting an existing theme +[original] + name = "" + homepage = "" + repo = "" +`) + + err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), fs) if err != nil { - jww.FATAL.Fatalln(err) + return } -} -func touchFile(fs afero.Fs, x ...string) { - inpath := filepath.Join(x...) - mkdir(filepath.Dir(inpath)) - err := helpers.WriteToDisk(inpath, bytes.NewReader([]byte{}), fs) + err = helpers.WriteToDisk(filepath.Join(inpath, "hugo.toml"), strings.NewReader("# Theme config.\n"), fs) if err != nil { - jww.FATAL.Fatalln(err) + return } -} - -func newContentPathSection(h *hugolib.HugoSites, path string) (string, string) { - // Forward slashes is used in all examples. Convert if needed. - // Issue #1133 - createpath := filepath.FromSlash(path) - - if h != nil { - for _, dir := range h.BaseFs.Content.Dirs { - createpath = strings.TrimPrefix(createpath, dir.Meta().Filename) - } - } - - var section string - // assume the first directory is the section (kind) - if strings.Contains(createpath[1:], helpers.FilePathSeparator) { - parts := strings.Split(strings.TrimPrefix(createpath, helpers.FilePathSeparator), helpers.FilePathSeparator) - if len(parts) > 0 { - section = parts[0] - } - - } - - return createpath, section + + return nil } diff --git a/commands/new_content_test.go b/commands/new_content_test.go deleted file mode 100644 index 42a7c968c..000000000 --- a/commands/new_content_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "path/filepath" - "testing" - - qt "github.com/frankban/quicktest" -) - -// Issue #1133 -func TestNewContentPathSectionWithForwardSlashes(t *testing.T) { - c := qt.New(t) - p, s := newContentPathSection(nil, "/post/new.md") - c.Assert(p, qt.Equals, filepath.FromSlash("/post/new.md")) - c.Assert(s, qt.Equals, "post") -} diff --git a/commands/new_site.go b/commands/new_site.go deleted file mode 100644 index fc4127f8b..000000000 --- a/commands/new_site.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "bytes" - "errors" - "fmt" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/parser/metadecoders" - - "github.com/gohugoio/hugo/create" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/parser" - "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*newSiteCmd)(nil) - -type newSiteCmd struct { - configFormat string - - *baseBuilderCmd -} - -func (b *commandsBuilder) newNewSiteCmd() *newSiteCmd { - cc := &newSiteCmd{} - - cmd := &cobra.Command{ - Use: "site [path]", - Short: "Create a new site (skeleton)", - Long: `Create a new site in the provided directory. -The new site will have the correct structure, but no content or theme yet. -Use ` + "`hugo new [contentPath]`" + ` to create new content.`, - RunE: cc.newSite, - } - - cmd.Flags().StringVarP(&cc.configFormat, "format", "f", "toml", "config file format") - cmd.Flags().Bool("force", false, "init inside non-empty directory") - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc -} - -func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error { - archeTypePath := filepath.Join(basepath, "archetypes") - dirs := []string{ - archeTypePath, - filepath.Join(basepath, "assets"), - filepath.Join(basepath, "content"), - filepath.Join(basepath, "data"), - filepath.Join(basepath, "layouts"), - filepath.Join(basepath, "static"), - filepath.Join(basepath, "themes"), - } - - if exists, _ := helpers.Exists(basepath, fs.Source); exists { - if isDir, _ := helpers.IsDir(basepath, fs.Source); !isDir { - return errors.New(basepath + " already exists but not a directory") - } - - isEmpty, _ := helpers.IsEmpty(basepath, fs.Source) - - switch { - case !isEmpty && !force: - return errors.New(basepath + " already exists and is not empty. See --force.") - - case !isEmpty && force: - // TODO(bep) eventually rename this to hugo. - all := append(dirs, filepath.Join(basepath, "config."+n.configFormat)) - for _, path := range all { - if exists, _ := helpers.Exists(path, fs.Source); exists { - return errors.New(path + " already exists") - } - } - } - } - - for _, dir := range dirs { - if err := fs.Source.MkdirAll(dir, 0777); err != nil { - return fmt.Errorf("Failed to create dir: %w", err) - } - } - - createConfig(fs, basepath, n.configFormat) - - // Create a default archetype file. - helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"), - strings.NewReader(create.DefaultArchetypeTemplateTemplate), fs.Source) - - jww.FEEDBACK.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", basepath) - jww.FEEDBACK.Println(nextStepsText()) - - return nil -} - -// newSite creates a new Hugo site and initializes a structured Hugo directory. -func (n *newSiteCmd) newSite(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return newUserError("path needs to be provided") - } - - createpath, err := filepath.Abs(filepath.Clean(args[0])) - if err != nil { - return newUserError(err) - } - - forceNew, _ := cmd.Flags().GetBool("force") - cfg := config.New() - cfg.Set("workingDir", createpath) - cfg.Set("publishDir", "public") - return n.doNewSite(hugofs.NewDefault(cfg), createpath, forceNew) -} - -func createConfig(fs *hugofs.Fs, inpath string, kind string) (err error) { - in := map[string]string{ - "baseURL": "http://example.org/", - "title": "My New Hugo Site", - "languageCode": "en-us", - } - - var buf bytes.Buffer - err = parser.InterfaceToConfig(in, metadecoders.FormatFromString(kind), &buf) - if err != nil { - return err - } - - return helpers.WriteToDisk(filepath.Join(inpath, "config."+kind), &buf, fs.Source) -} - -func nextStepsText() string { - var nextStepsText bytes.Buffer - - nextStepsText.WriteString(`Just a few more steps and you're ready to go: - -1. Download a theme into the same-named folder. - Choose a theme from https://themes.gohugo.io/ or - create your own with the "hugo new theme " command. -2. Perhaps you want to add some content. You can add single files - with "hugo new `) - - nextStepsText.WriteString(filepath.Join("", ".")) - - nextStepsText.WriteString(`". -3. Start the built-in live server via "hugo server". - -Visit https://gohugo.io/ for quickstart guide and full documentation.`) - - return nextStepsText.String() -} diff --git a/commands/new_theme.go b/commands/new_theme.go deleted file mode 100644 index 4e2357b55..000000000 --- a/commands/new_theme.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "bytes" - "errors" - "path/filepath" - "strings" - - "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/hugofs" - "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*newThemeCmd)(nil) - -type newThemeCmd struct { - *baseBuilderCmd -} - -func (b *commandsBuilder) newNewThemeCmd() *newThemeCmd { - cc := &newThemeCmd{} - - cmd := &cobra.Command{ - Use: "theme [name]", - Short: "Create a new theme", - Long: `Create a new theme (skeleton) called [name] in ./themes. -New theme is a skeleton. Please add content to the touched files. Add your -name to the copyright line in the license and adjust the theme.toml file -as you see fit.`, - RunE: cc.newTheme, - } - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc -} - -// newTheme creates a new Hugo theme template -func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error { - c, err := initializeConfig(false, false, false, &n.hugoBuilderCommon, n, nil) - if err != nil { - return err - } - - if len(args) < 1 { - return newUserError("theme name needs to be provided") - } - - createpath := c.hugo().PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0])) - jww.FEEDBACK.Println("Creating theme at", createpath) - - cfg := c.DepsCfg - - if x, _ := helpers.Exists(createpath, cfg.Fs.Source); x { - return errors.New(createpath + " already exists") - } - - mkdir(createpath, "layouts", "_default") - mkdir(createpath, "layouts", "partials") - - touchFile(cfg.Fs.Source, createpath, "layouts", "index.html") - touchFile(cfg.Fs.Source, createpath, "layouts", "404.html") - touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "list.html") - touchFile(cfg.Fs.Source, createpath, "layouts", "_default", "single.html") - - baseofDefault := []byte(` - - {{- partial "head.html" . -}} - - {{- partial "header.html" . -}} -
- {{- block "main" . }}{{- end }} -
- {{- partial "footer.html" . -}} - - -`) - err = helpers.WriteToDisk(filepath.Join(createpath, "layouts", "_default", "baseof.html"), bytes.NewReader(baseofDefault), cfg.Fs.Source) - if err != nil { - return err - } - - touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "head.html") - touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "header.html") - touchFile(cfg.Fs.Source, createpath, "layouts", "partials", "footer.html") - - mkdir(createpath, "archetypes") - - archDefault := []byte("+++\n+++\n") - - err = helpers.WriteToDisk(filepath.Join(createpath, "archetypes", "default.md"), bytes.NewReader(archDefault), cfg.Fs.Source) - if err != nil { - return err - } - - mkdir(createpath, "static", "js") - mkdir(createpath, "static", "css") - - by := []byte(`The MIT License (MIT) - -Copyright (c) ` + htime.Now().Format("2006") + ` YOUR_NAME_HERE - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -`) - - err = helpers.WriteToDisk(filepath.Join(createpath, "LICENSE"), bytes.NewReader(by), cfg.Fs.Source) - if err != nil { - return err - } - - n.createThemeMD(cfg.Fs, createpath) - - return nil -} - -func (n *newThemeCmd) createThemeMD(fs *hugofs.Fs, inpath string) (err error) { - by := []byte(`# theme.toml template for a Hugo theme -# See https://github.com/gohugoio/hugoThemes#themetoml for an example - -name = "` + strings.Title(helpers.MakeTitle(filepath.Base(inpath))) + `" -license = "MIT" -licenselink = "https://github.com/yourname/yourtheme/blob/master/LICENSE" -description = "" -homepage = "http://example.com/" -tags = [] -features = [] -min_version = "0.41.0" - -[author] - name = "" - homepage = "" - -# If porting an existing theme -[original] - name = "" - homepage = "" - repo = "" -`) - - err = helpers.WriteToDisk(filepath.Join(inpath, "theme.toml"), bytes.NewReader(by), fs.Source) - if err != nil { - return - } - - return nil -} diff --git a/commands/nodeploy.go b/commands/nodeploy.go deleted file mode 100644 index 061ea503e..000000000 --- a/commands/nodeploy.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build nodeploy -// +build nodeploy - -package commands - -import ( - "errors" - - "github.com/spf13/cobra" -) - -var _ cmder = (*deployCmd)(nil) - -// deployCmd supports deploying sites to Cloud providers. -type deployCmd struct { - *baseBuilderCmd -} - -func (b *commandsBuilder) newDeployCmd() *deployCmd { - cc := &deployCmd{} - - cmd := &cobra.Command{ - Use: "deploy", - Short: "Deploy your site to a Cloud provider.", - Long: `Deploy your site to a Cloud provider. - -See https://gohugo.io/hosting-and-deployment/hugo-deploy/ for detailed -documentation. -`, - RunE: func(cmd *cobra.Command, args []string) error { - return errors.New("build without HUGO_BUILD_TAGS=nodeploy to use this command") - }, - } - - cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) - - return cc -} diff --git a/commands/release.go b/commands/release.go index 2072f3eb2..fe3c5efb6 100644 --- a/commands/release.go +++ b/commands/release.go @@ -1,7 +1,4 @@ -//go:build release -// +build release - -// Copyright 2017-present The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,55 +14,39 @@ package commands import ( - "github.com/gohugoio/hugo/config" + "context" + + "github.com/bep/simplecobra" "github.com/gohugoio/hugo/releaser" "github.com/spf13/cobra" ) -var _ cmder = (*releaseCommandeer)(nil) +// Note: This is a command only meant for internal use and must be run +// via "go run -tags release main.go release" on the actual code base that is in the release. +func newReleaseCommand() simplecobra.Commander { -type releaseCommandeer struct { - cmd *cobra.Command + var ( + step int + skipPush bool + try bool + ) - step int - skipPush bool - try bool -} + return &simpleCommand{ + name: "release", + short: "Release a new version of Hugo.", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + rel, err := releaser.New(skipPush, try, step) + if err != nil { + return err + } -func createReleaser() cmder { - // Note: This is a command only meant for internal use and must be run - // via "go run -tags release main.go release" on the actual code base that is in the release. - r := &releaseCommandeer{ - cmd: &cobra.Command{ - Use: "release", - Short: "Release a new version of Hugo.", - Hidden: true, + return rel.Run() + }, + withc: func(cmd *cobra.Command) { + cmd.Hidden = true + cmd.PersistentFlags().BoolVarP(&skipPush, "skip-push", "", false, "skip pushing to remote") + cmd.PersistentFlags().BoolVarP(&try, "try", "", false, "no changes") + cmd.PersistentFlags().IntVarP(&step, "step", "", 0, "step to run (1: set new version 2: prepare next dev version)") }, } - - r.cmd.RunE = func(cmd *cobra.Command, args []string) error { - return r.release() - } - - r.cmd.PersistentFlags().BoolVarP(&r.skipPush, "skip-push", "", false, "skip pushing to remote") - r.cmd.PersistentFlags().BoolVarP(&r.try, "try", "", false, "no changes") - r.cmd.PersistentFlags().IntVarP(&r.step, "step", "", 0, "step to run (1: set new version 2: prepare next dev version)") - - return r -} - -func (c *releaseCommandeer) getCommand() *cobra.Command { - return c.cmd -} - -func (c *releaseCommandeer) flagsToConfig(cfg config.Provider) { -} - -func (r *releaseCommandeer) release() error { - rel, err := releaser.New(r.skipPush, r.try, r.step) - if err != nil { - return err - } - - return rel.Run() } diff --git a/commands/release_noop.go b/commands/release_noop.go deleted file mode 100644 index 176dc9794..000000000 --- a/commands/release_noop.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !release -// +build !release - -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -func createReleaser() cmder { - return &nilCommand{} -} diff --git a/commands/server.go b/commands/server.go index 121a649d4..81a5120ef 100644 --- a/commands/server.go +++ b/commands/server.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,357 +16,217 @@ package commands import ( "bytes" "context" + "encoding/json" + "errors" "fmt" "io" + "io/ioutil" "net" "net/http" "net/url" "os" + "sync" + "sync/atomic" + "os/signal" "path" "path/filepath" "regexp" - "runtime" "strconv" "strings" - "sync" "syscall" "time" - "github.com/gohugoio/hugo/common/htime" - "github.com/gohugoio/hugo/common/paths" - "github.com/gohugoio/hugo/hugolib" - "github.com/gohugoio/hugo/tpl" - "golang.org/x/sync/errgroup" - - "github.com/gohugoio/hugo/livereload" - - "github.com/gohugoio/hugo/config" + "github.com/bep/debounce" + "github.com/bep/simplecobra" + "github.com/fsnotify/fsnotify" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/common/urls" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + "github.com/gohugoio/hugo/hugofs/files" + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/hugolib/filesystems" + "github.com/gohugoio/hugo/livereload" + "github.com/gohugoio/hugo/tpl" + "github.com/gohugoio/hugo/transform" + "github.com/gohugoio/hugo/transform/livereloadinject" "github.com/spf13/afero" "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" + "github.com/spf13/fsync" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" ) -type serverCmd struct { - // Can be used to stop the server. Useful in tests - stop chan bool +var ( + logErrorRe = regexp.MustCompile(`(?s)ERROR \d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} `) + logDuplicateTemplateExecuteRe = regexp.MustCompile(`: template: .*?:\d+:\d+: executing ".*?"`) + logDuplicateTemplateParseRe = regexp.MustCompile(`: template: .*?:\d+:\d*`) +) - disableLiveReload bool - navigateToChanged bool - renderToDisk bool - renderStaticToDisk bool - serverAppend bool - serverInterface string - serverPort int - liveReloadPort int - serverWatch bool - noHTTPCache bool +var logReplacer = strings.NewReplacer( + "can't", "can’t", // Chroma lexer doesn't do well with "can't" + "*hugolib.pageState", "page.Page", // Page is the public interface. + "Rebuild failed:", "", +) - disableFastRender bool - disableBrowserError bool +const ( + configChangeConfig = "config file" + configChangeGoMod = "go.mod file" + configChangeGoWork = "go work file" +) - *baseBuilderCmd -} - -func (b *commandsBuilder) newServerCmd() *serverCmd { - return b.newServerCmdSignaled(nil) -} - -func (b *commandsBuilder) newServerCmdSignaled(stop chan bool) *serverCmd { - cc := &serverCmd{stop: stop} - - cc.baseBuilderCmd = b.newBuilderCmd(&cobra.Command{ - Use: "server", - Aliases: []string{"serve"}, - Short: "A high performance webserver", - Long: `Hugo provides its own webserver which builds and serves the site. -While hugo server is high performance, it is a webserver with limited options. -Many run it in production, but the standard behavior is for people to use it -in development and use a more full featured server such as Nginx or Caddy. - -'hugo server' will avoid writing the rendered and served content to disk, -preferring to store it in memory. - -By default hugo will also watch your files for any changes you make and -automatically rebuild the site. It will then live reload any open browser pages -and push the latest content to them. As most Hugo sites are built in a fraction -of a second, you will be able to save and see your changes nearly instantly.`, - RunE: func(cmd *cobra.Command, args []string) error { - err := cc.server(cmd, args) - if err != nil && cc.stop != nil { - cc.stop <- true +func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder { + return &hugoBuilder{ + r: r, + s: s, + visitedURLs: types.NewEvictingStringQueue(100), + fullRebuildSem: semaphore.NewWeighted(1), + debounce: debounce.New(4 * time.Second), + onConfigLoaded: func(reloaded bool) error { + for _, wc := range onConfigLoaded { + if err := wc(reloaded); err != nil { + return err + } } - return err - }, - }) - - cc.cmd.Flags().IntVarP(&cc.serverPort, "port", "p", 1313, "port on which the server will listen") - cc.cmd.Flags().IntVar(&cc.liveReloadPort, "liveReloadPort", -1, "port for live reloading (i.e. 443 in HTTPS proxy situations)") - cc.cmd.Flags().StringVarP(&cc.serverInterface, "bind", "", "127.0.0.1", "interface to which the server will bind") - cc.cmd.Flags().BoolVarP(&cc.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed") - cc.cmd.Flags().BoolVar(&cc.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching") - cc.cmd.Flags().BoolVarP(&cc.serverAppend, "appendPort", "", true, "append port to baseURL") - cc.cmd.Flags().BoolVar(&cc.disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild") - cc.cmd.Flags().BoolVar(&cc.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload") - cc.cmd.Flags().BoolVar(&cc.renderToDisk, "renderToDisk", false, "serve all files from disk (default is from memory)") - cc.cmd.Flags().BoolVar(&cc.renderStaticToDisk, "renderStaticToDisk", false, "serve static files from disk and dynamic files from memory") - cc.cmd.Flags().BoolVar(&cc.disableFastRender, "disableFastRender", false, "enables full re-renders on changes") - cc.cmd.Flags().BoolVar(&cc.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser") - - cc.cmd.Flags().String("memstats", "", "log memory usage to this file") - cc.cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".") - - return cc -} - -type filesOnlyFs struct { - fs http.FileSystem -} - -type noDirFile struct { - http.File -} - -func (fs filesOnlyFs) Open(name string) (http.File, error) { - f, err := fs.fs.Open(name) - if err != nil { - return nil, err - } - return noDirFile{f}, nil -} - -func (f noDirFile) Readdir(count int) ([]os.FileInfo, error) { - return nil, nil -} - -func (sc *serverCmd) server(cmd *cobra.Command, args []string) error { - // If a Destination is provided via flag write to disk - destination, _ := cmd.Flags().GetString("destination") - if destination != "" { - sc.renderToDisk = true - } - - var serverCfgInit sync.Once - - cfgInit := func(c *commandeer) (rerr error) { - c.Set("renderToMemory", !(sc.renderToDisk || sc.renderStaticToDisk)) - c.Set("renderStaticToDisk", sc.renderStaticToDisk) - if cmd.Flags().Changed("navigateToChanged") { - c.Set("navigateToChanged", sc.navigateToChanged) - } - if cmd.Flags().Changed("disableLiveReload") { - c.Set("disableLiveReload", sc.disableLiveReload) - } - if cmd.Flags().Changed("disableFastRender") { - c.Set("disableFastRender", sc.disableFastRender) - } - if cmd.Flags().Changed("disableBrowserError") { - c.Set("disableBrowserError", sc.disableBrowserError) - } - if sc.serverWatch { - c.Set("watch", true) - } - - // TODO(bep) see issue 9901 - // cfgInit is called twice, before and after the languages have been initialized. - // The servers (below) can not be initialized before we - // know if we're configured in a multihost setup. - if len(c.languages) == 0 { return nil + }, + } +} + +func newServerCommand() *serverCommand { + var c *serverCommand + c = &serverCommand{ + quit: make(chan bool), + } + return c +} + +type countingStatFs struct { + afero.Fs + statCounter uint64 +} + +func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) { + f, err := fs.Fs.Stat(name) + if err == nil { + if !f.IsDir() { + atomic.AddUint64(&fs.statCounter, 1) } + } + return f, err +} - // We can only do this once. - serverCfgInit.Do(func() { - c.serverPorts = make([]serverPortListener, 1) +// dynamicEvents contains events that is considered dynamic, as in "not static". +// Both of these categories will trigger a new build, but the asset events +// does not fit into the "navigate to changed" logic. +type dynamicEvents struct { + ContentEvents []fsnotify.Event + AssetEvents []fsnotify.Event +} - if c.languages.IsMultihost() { - if !sc.serverAppend { - rerr = newSystemError("--appendPort=false not supported when in multihost mode") - } - c.serverPorts = make([]serverPortListener, len(c.languages)) - } +type fileChangeDetector struct { + sync.Mutex + current map[string]string + prev map[string]string - currentServerPort := sc.serverPort + irrelevantRe *regexp.Regexp +} - for i := 0; i < len(c.serverPorts); i++ { - l, err := net.Listen("tcp", net.JoinHostPort(sc.serverInterface, strconv.Itoa(currentServerPort))) - if err == nil { - c.serverPorts[i] = serverPortListener{ln: l, p: currentServerPort} - } else { - if i == 0 && sc.cmd.Flags().Changed("port") { - // port set explicitly by user -- he/she probably meant it! - rerr = newSystemErrorF("Server startup failed: %s", err) - return - } - c.logger.Println("port", sc.serverPort, "already in use, attempting to use an available port") - l, sp, err := helpers.TCPListen() - if err != nil { - rerr = newSystemError("Unable to find alternative port to use:", err) - return - } - c.serverPorts[i] = serverPortListener{ln: l, p: sp.Port} - } - - currentServerPort = c.serverPorts[i].p + 1 - } - }) - - if rerr != nil { - return - } - - c.Set("port", sc.serverPort) - if sc.liveReloadPort != -1 { - c.Set("liveReloadPort", sc.liveReloadPort) - } else { - c.Set("liveReloadPort", c.serverPorts[0].p) - } - - isMultiHost := c.languages.IsMultihost() - for i, language := range c.languages { - var serverPort int - if isMultiHost { - serverPort = c.serverPorts[i].p - } else { - serverPort = c.serverPorts[0].p - } - - baseURL, err := sc.fixURL(language, sc.baseURL, serverPort) - if err != nil { - return nil - } - if isMultiHost { - language.Set("baseURL", baseURL) - } - if i == 0 { - c.Set("baseURL", baseURL) - } - } +func (f *fileChangeDetector) OnFileClose(name, md5sum string) { + f.Lock() + defer f.Unlock() + f.current[name] = md5sum +} +func (f *fileChangeDetector) PrepareNew() { + if f == nil { return } - if err := memStats(); err != nil { - jww.WARN.Println("memstats error:", err) + f.Lock() + defer f.Unlock() + + if f.current == nil { + f.current = make(map[string]string) + f.prev = make(map[string]string) + return } - // silence errors in cobra so we can handle them here - cmd.SilenceErrors = true - - c, err := initializeConfig(true, true, true, &sc.hugoBuilderCommon, sc, cfgInit) - if err != nil { - cmd.PrintErrln("Error:", err.Error()) - return err + f.prev = make(map[string]string) + for k, v := range f.current { + f.prev[k] = v } - - err = func() error { - defer c.timeTrack(time.Now(), "Built") - err := c.serverBuild() - if err != nil { - cmd.PrintErrln("Error:", err.Error()) - } - return err - }() - if err != nil { - return err - } - - // Watch runs its own server as part of the routine - if sc.serverWatch { - - watchDirs, err := c.getDirList() - if err != nil { - return err - } - - watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs) - - for _, group := range watchGroups { - jww.FEEDBACK.Printf("Watching for changes in %s\n", group) - } - watcher, err := c.newWatcher(sc.poll, watchDirs...) - if err != nil { - return err - } - - defer watcher.Close() - - } - - return c.serve(sc) + f.current = make(map[string]string) } -func getRootWatchDirsStr(baseDir string, watchDirs []string) string { - relWatchDirs := make([]string, len(watchDirs)) - for i, dir := range watchDirs { - relWatchDirs[i], _ = paths.GetRelativePath(dir, baseDir) +func (f *fileChangeDetector) changed() []string { + if f == nil { + return nil + } + f.Lock() + defer f.Unlock() + var c []string + for k, v := range f.current { + vv, found := f.prev[k] + if !found || v != vv { + c = append(c, k) + } } - return strings.Join(helpers.UniqueStringsSorted(helpers.ExtractRootPaths(relWatchDirs)), ",") + return f.filterIrrelevant(c) +} + +func (f *fileChangeDetector) filterIrrelevant(in []string) []string { + var filtered []string + for _, v := range in { + if !f.irrelevantRe.MatchString(v) { + filtered = append(filtered, v) + } + } + return filtered } type fileServer struct { baseURLs []string roots []string errorTemplate func(err any) (io.Reader, error) - c *commandeer - s *serverCmd -} - -func (f *fileServer) rewriteRequest(r *http.Request, toPath string) *http.Request { - r2 := new(http.Request) - *r2 = *r - r2.URL = new(url.URL) - *r2.URL = *r.URL - r2.URL.Path = toPath - r2.Header.Set("X-Rewrite-Original-URI", r.URL.RequestURI()) - - return r2 + c *serverCommand } func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string, string, error) { + r := f.c.r + conf := f.c.conf() baseURL := f.baseURLs[i] root := f.roots[i] port := f.c.serverPorts[i].p listener := f.c.serverPorts[i].ln + logger := f.c.r.logger - // For logging only. - // TODO(bep) consolidate. - publishDir := f.c.Cfg.GetString("publishDir") - publishDirStatic := f.c.Cfg.GetString("publishDirStatic") - workingDir := f.c.Cfg.GetString("workingDir") - - if root != "" { - publishDir = filepath.Join(publishDir, root) - publishDirStatic = filepath.Join(publishDirStatic, root) - } - absPublishDir := paths.AbsPathify(workingDir, publishDir) - absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic) - - jww.FEEDBACK.Printf("Environment: %q", f.c.hugo().Deps.Site.Hugo().Environment) + r.Printf("Environment: %q", f.c.hugoTry().Deps.Site.Hugo().Environment) if i == 0 { - if f.s.renderToDisk { - jww.FEEDBACK.Println("Serving pages from " + absPublishDir) - } else if f.s.renderStaticToDisk { - jww.FEEDBACK.Println("Serving pages from memory and static files from " + absPublishDirStatic) + if f.c.renderToDisk { + r.Println("Serving pages from disk") + } else if f.c.renderStaticToDisk { + r.Println("Serving pages from memory and static files from disk") } else { - jww.FEEDBACK.Println("Serving pages from memory") + r.Println("Serving pages from memory") } } - httpFs := afero.NewHttpFs(f.c.publishDirServerFs) + httpFs := afero.NewHttpFs(conf.fs.PublishDirServer) fs := filesOnlyFs{httpFs.Dir(path.Join("/", root))} - if i == 0 && f.c.fastRenderMode { - jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender") + r.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender") } // We're only interested in the path u, err := url.Parse(baseURL) if err != nil { - return nil, nil, "", "", fmt.Errorf("Invalid baseURL: %w", err) + return nil, nil, "", "", fmt.Errorf("invalid baseURL: %w", err) } decorate := func(h http.Handler) http.Handler { @@ -375,16 +235,16 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string // First check the error state err := f.c.getErrorWithContext() if err != nil { - f.c.wasError = true + f.c.errState.setWasErr(false) w.WriteHeader(500) r, err := f.errorTemplate(err) if err != nil { - f.c.logger.Errorln(err) + logger.Errorln(err) } port = 1313 - if !f.c.paused { - port = f.c.Cfg.GetInt("liveReloadPort") + if !f.c.errState.isPaused() { + port = conf.configs.Base.Internal.LiveReloadPort } lr := *u lr.Host = fmt.Sprintf("%s:%d", lr.Hostname(), port) @@ -394,19 +254,21 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string } } - if f.s.noHTTPCache { + if f.c.noHTTPCache { w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0") w.Header().Set("Pragma", "no-cache") } + serverConfig := f.c.conf().configs.Base.Server + // Ignore any query params for the operations below. requestURI, _ := url.PathUnescape(strings.TrimSuffix(r.RequestURI, "?"+r.URL.RawQuery)) - for _, header := range f.c.serverConfig.MatchHeaders(requestURI) { + for _, header := range serverConfig.MatchHeaders(requestURI) { w.Header().Set(header.Key, header.Value) } - if redirect := f.c.serverConfig.MatchRedirect(requestURI); !redirect.IsZero() { + if redirect := serverConfig.MatchRedirect(requestURI); !redirect.IsZero() { // fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))) doRedirect := true // This matches Netlify's behaviour and is needed for SPA behaviour. @@ -416,7 +278,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string if root != "" { path = filepath.Join(root, path) } - fs := f.c.publishDirServerFs + fs := f.c.conf().getFs().PublishDir fi, err := fs.Stat(path) @@ -459,7 +321,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string } - if f.c.fastRenderMode && f.c.buildErr == nil { + if f.c.fastRenderMode && f.c.errState.buildErr() == nil { if strings.HasSuffix(requestURI, "/") || strings.HasSuffix(requestURI, "html") || strings.HasSuffix(requestURI, "htm") { if !f.c.visitedURLs.Contains(requestURI) { // If not already on stack, re-render that single page. @@ -488,196 +350,302 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string } else { mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver)) } + if r.IsTestRun() { + var shutDownOnce sync.Once + mu.HandleFunc("/__stop", func(w http.ResponseWriter, r *http.Request) { + shutDownOnce.Do(func() { + close(f.c.quit) + }) + }) + } - endpoint := net.JoinHostPort(f.s.serverInterface, strconv.Itoa(port)) + endpoint := net.JoinHostPort(f.c.serverInterface, strconv.Itoa(port)) return mu, listener, u.String(), endpoint, nil } -var ( - logErrorRe = regexp.MustCompile(`(?s)ERROR \d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} `) - logDuplicateTemplateExecuteRe = regexp.MustCompile(`: template: .*?:\d+:\d+: executing ".*?"`) - logDuplicateTemplateParseRe = regexp.MustCompile(`: template: .*?:\d+:\d*`) -) +func (f *fileServer) rewriteRequest(r *http.Request, toPath string) *http.Request { + r2 := new(http.Request) + *r2 = *r + r2.URL = new(url.URL) + *r2.URL = *r.URL + r2.URL.Path = toPath + r2.Header.Set("X-Rewrite-Original-URI", r.URL.RequestURI()) -func removeErrorPrefixFromLog(content string) string { - return logErrorRe.ReplaceAllLiteralString(content, "") + return r2 } -var logReplacer = strings.NewReplacer( - "can't", "can’t", // Chroma lexer doesn't do well with "can't" - "*hugolib.pageState", "page.Page", // Page is the public interface. - "Rebuild failed:", "", -) - -func cleanErrorLog(content string) string { - content = strings.ReplaceAll(content, "\n", " ") - content = logReplacer.Replace(content) - content = logDuplicateTemplateExecuteRe.ReplaceAllString(content, "") - content = logDuplicateTemplateParseRe.ReplaceAllString(content, "") - seen := make(map[string]bool) - parts := strings.Split(content, ": ") - keep := make([]string, 0, len(parts)) - for _, part := range parts { - if seen[part] { - continue - } - seen[part] = true - keep = append(keep, part) - } - return strings.Join(keep, ": ") +type filesOnlyFs struct { + fs http.FileSystem } -func (c *commandeer) serve(s *serverCmd) error { - isMultiHost := c.hugo().IsMultihost() - - var ( - baseURLs []string - roots []string - ) - - if isMultiHost { - for _, s := range c.hugo().Sites { - baseURLs = append(baseURLs, s.BaseURL.String()) - roots = append(roots, s.Language().Lang) - } - } else { - s := c.hugo().Sites[0] - baseURLs = []string{s.BaseURL.String()} - roots = []string{""} - } - - // Cache it here. The HugoSites object may be unavailable later on due to intermittent configuration errors. - // To allow the en user to change the error template while the server is running, we use - // the freshest template we can provide. - var ( - errTempl tpl.Template - templHandler tpl.TemplateHandler - ) - getErrorTemplateAndHandler := func(h *hugolib.HugoSites) (tpl.Template, tpl.TemplateHandler) { - if h == nil { - return errTempl, templHandler - } - templHandler := h.Tmpl() - errTempl, found := templHandler.Lookup("_server/error.html") - if !found { - panic("template server/error.html not found") - } - return errTempl, templHandler - } - errTempl, templHandler = getErrorTemplateAndHandler(c.hugo()) - - srv := &fileServer{ - baseURLs: baseURLs, - roots: roots, - c: c, - s: s, - errorTemplate: func(ctx any) (io.Reader, error) { - // hugoTry does not block, getErrorTemplateAndHandler will fall back - // to cached values if nil. - templ, handler := getErrorTemplateAndHandler(c.hugoTry()) - b := &bytes.Buffer{} - err := handler.ExecuteWithContext(context.Background(), templ, b, ctx) - return b, err - }, - } - - doLiveReload := !c.Cfg.GetBool("disableLiveReload") - - if doLiveReload { - livereload.Initialize() - } - - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - var servers []*http.Server - - wg1, ctx := errgroup.WithContext(context.Background()) - - for i := range baseURLs { - mu, listener, serverURL, endpoint, err := srv.createEndpoint(i) - srv := &http.Server{ - Addr: endpoint, - Handler: mu, - } - servers = append(servers, srv) - - if doLiveReload { - u, err := url.Parse(helpers.SanitizeURL(baseURLs[i])) - if err != nil { - return err - } - - mu.HandleFunc(u.Path+"/livereload.js", livereload.ServeJS) - mu.HandleFunc(u.Path+"/livereload", livereload.Handler) - } - jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", serverURL, s.serverInterface) - wg1.Go(func() error { - err = srv.Serve(listener) - if err != nil && err != http.ErrServerClosed { - return err - } - return nil - }) - } - - jww.FEEDBACK.Println("Press Ctrl+C to stop") - - err := func() error { - if s.stop != nil { - for { - select { - case <-sigs: - return nil - case <-s.stop: - return nil - case <-ctx.Done(): - return ctx.Err() - } - } - } else { - for { - select { - case <-sigs: - return nil - case <-ctx.Done(): - return ctx.Err() - } - } - } - }() - +func (fs filesOnlyFs) Open(name string) (http.File, error) { + f, err := fs.fs.Open(name) if err != nil { - jww.ERROR.Println("Error:", err) + return nil, err + } + return noDirFile{f}, nil +} + +type noDirFile struct { + http.File +} + +func (f noDirFile) Readdir(count int) ([]os.FileInfo, error) { + return nil, nil +} + +type serverCommand struct { + r *rootCommand + + commands []simplecobra.Commander + + *hugoBuilder + + quit chan bool // Closed when the server should shut down. Used in tests only. + serverPorts []serverPortListener + doLiveReload bool + + // Flags. + + renderToDisk bool + renderStaticToDisk bool + navigateToChanged bool + serverAppend bool + serverInterface string + serverPort int + liveReloadPort int + serverWatch bool + noHTTPCache bool + disableLiveReload bool + disableFastRender bool + disableBrowserError bool +} + +func (c *serverCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *serverCommand) Name() string { + return "server" +} + +func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + err := func() error { + defer c.r.timeTrack(time.Now(), "Built") + err := c.build() + if err != nil { + c.r.Println("Error:", err.Error()) + } + return err + }() + if err != nil { + return err } - if h := c.hugoTry(); h != nil { - h.Close() + // Watch runs its own server as part of the routine + if c.serverWatch { + + watchDirs, err := c.getDirList() + if err != nil { + return err + } + + watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs) + + for _, group := range watchGroups { + c.r.Printf("Watching for changes in %s\n", group) + } + watcher, err := c.newWatcher(c.r.poll, watchDirs...) + if err != nil { + return err + } + + defer watcher.Close() + } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - wg2, ctx := errgroup.WithContext(ctx) - for _, srv := range servers { - srv := srv - wg2.Go(func() error { - return srv.Shutdown(ctx) - }) + return c.serve() +} + +func (c *serverCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "A high performance webserver" + cmd.Long = `Hugo provides its own webserver which builds and serves the site. +While hugo server is high performance, it is a webserver with limited options. +Many run it in production, but the standard behavior is for people to use it +in development and use a more full featured server such as Nginx or Caddy. + +'hugo server' will avoid writing the rendered and served content to disk, +preferring to store it in memory. + +By default hugo will also watch your files for any changes you make and +automatically rebuild the site. It will then live reload any open browser pages +and push the latest content to them. As most Hugo sites are built in a fraction +of a second, you will be able to save and see your changes nearly instantly.` + cmd.Aliases = []string{"serve"} + + cmd.Flags().IntVarP(&c.serverPort, "port", "p", 1313, "port on which the server will listen") + cmd.Flags().IntVar(&c.liveReloadPort, "liveReloadPort", -1, "port for live reloading (i.e. 443 in HTTPS proxy situations)") + cmd.Flags().StringVarP(&c.serverInterface, "bind", "", "127.0.0.1", "interface to which the server will bind") + cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed") + cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching") + cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL") + cmd.Flags().BoolVar(&c.disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild") + cmd.Flags().BoolVar(&c.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload") + cmd.Flags().BoolVar(&c.renderToDisk, "renderToDisk", false, "serve all files from disk (default is from memory)") + cmd.Flags().BoolVar(&c.renderStaticToDisk, "renderStaticToDisk", false, "serve static files from disk and dynamic files from memory") + cmd.Flags().BoolVar(&c.disableFastRender, "disableFastRender", false, "enables full re-renders on changes") + cmd.Flags().BoolVar(&c.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser") + + cmd.Flags().String("memstats", "", "log memory usage to this file") + cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".") + return nil +} + +func (c *serverCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + + c.hugoBuilder = newHugoBuilder( + c.r, + c, + func(reloaded bool) error { + if !reloaded { + if err := c.createServerPorts(cd); err != nil { + return err + } + } + if err := c.setBaseURLsInConfig(); err != nil { + return err + } + + if !reloaded && c.fastRenderMode { + c.conf().fs.PublishDir = hugofs.NewHashingFs(c.conf().fs.PublishDir, c.changeDetector) + c.conf().fs.PublishDirStatic = hugofs.NewHashingFs(c.conf().fs.PublishDirStatic, c.changeDetector) + } + + return nil + }, + ) + + destinationFlag := cd.CobraCommand.Flags().Lookup("destination") + c.renderToDisk = c.renderToDisk || (destinationFlag != nil && destinationFlag.Changed) + c.doLiveReload = !c.disableLiveReload + c.fastRenderMode = !c.disableFastRender + c.showErrorInBrowser = c.doLiveReload && !c.disableBrowserError + if c.r.environment == "" { + c.r.environment = hugo.EnvironmentDevelopment } - err1, err2 := wg1.Wait(), wg2.Wait() - if err1 != nil { - return err1 + if c.fastRenderMode { + // For now, fast render mode only. It should, however, be fast enough + // for the full variant, too. + c.changeDetector = &fileChangeDetector{ + // We use this detector to decide to do a Hot reload of a single path or not. + // We need to filter out source maps and possibly some other to be able + // to make that decision. + irrelevantRe: regexp.MustCompile(`\.map$`), + } + + c.changeDetector.PrepareNew() + } - return err2 + + err := c.loadConfig(cd, true) + if err != nil { + return err + } + + return nil +} + +func (c *serverCommand) setBaseURLsInConfig() error { + if len(c.serverPorts) == 0 { + panic("no server ports set") + } + isMultiHost := c.conf().configs.IsMultihost + for i, language := range c.conf().configs.Languages { + var serverPort int + if isMultiHost { + serverPort = c.serverPorts[i].p + } else { + serverPort = c.serverPorts[0].p + } + langConfig := c.conf().configs.LanguageConfigMap[language.Lang] + baseURLStr, err := c.fixURL(langConfig.BaseURL, c.r.baseURL, serverPort) + if err != nil { + return nil + } + baseURL, err := urls.NewBaseURLFromString(baseURLStr) + if err != nil { + return fmt.Errorf("failed to create baseURL from %q: %s", baseURLStr, err) + } + + baseURLLiveReload := baseURL + if c.liveReloadPort != -1 { + baseURLLiveReload, _ = baseURLLiveReload.WithPort(c.liveReloadPort) + } + langConfig.C.SetBaseURL(baseURL, baseURLLiveReload) + } + return nil +} + +func (c *serverCommand) getErrorWithContext() any { + errCount := c.errCount() + + if errCount == 0 { + return nil + } + + m := make(map[string]any) + + //xwm["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.r.logger.Errors()))) + m["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.r.logger.Errors()))) + m["Version"] = hugo.BuildVersionString() + ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.errState.buildErr()) + m["Files"] = ferrors + + return m +} + +func (c *serverCommand) createServerPorts(cd *simplecobra.Commandeer) error { + flags := cd.CobraCommand.Flags() + isMultiHost := c.conf().configs.IsMultihost + c.serverPorts = make([]serverPortListener, 1) + if isMultiHost { + if !c.serverAppend { + return errors.New("--appendPort=false not supported when in multihost mode") + } + c.serverPorts = make([]serverPortListener, len(c.conf().configs.Languages)) + } + currentServerPort := c.serverPort + for i := 0; i < len(c.serverPorts); i++ { + l, err := net.Listen("tcp", net.JoinHostPort(c.serverInterface, strconv.Itoa(currentServerPort))) + if err == nil { + c.serverPorts[i] = serverPortListener{ln: l, p: currentServerPort} + } else { + if i == 0 && flags.Changed("port") { + // port set explicitly by user -- he/she probably meant it! + return fmt.Errorf("server startup failed: %s", err) + } + c.r.Println("port", currentServerPort, "already in use, attempting to use an available port") + l, sp, err := helpers.TCPListen() + if err != nil { + return fmt.Errorf("unable to find alternative port to use: %s", err) + } + c.serverPorts[i] = serverPortListener{ln: l, p: sp.Port} + } + + currentServerPort = c.serverPorts[i].p + 1 + } + return nil } // fixURL massages the baseURL into a form needed for serving // all pages correctly. -func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, error) { +func (c *serverCommand) fixURL(baseURL, s string, port int) (string, error) { useLocalhost := false if s == "" { - s = cfg.GetString("baseURL") + s = baseURL useLocalhost = true } @@ -709,11 +677,11 @@ func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, er u.Host = "localhost" } - if sc.serverAppend { + if c.serverAppend { if strings.Contains(u.Host, ":") { u.Host, _, err = net.SplitHostPort(u.Host) if err != nil { - return "", fmt.Errorf("Failed to split baseURL hostpost: %w", err) + return "", fmt.Errorf("failed to split baseURL hostport: %w", err) } } u.Host += fmt.Sprintf(":%d", port) @@ -722,39 +690,368 @@ func (sc *serverCmd) fixURL(cfg config.Provider, s string, port int) (string, er return u.String(), nil } -func memStats() error { - b := newCommandsBuilder() - sc := b.newServerCmd().getCommand() - memstats := sc.Flags().Lookup("memstats").Value.String() - if memstats != "" { - interval, err := time.ParseDuration(sc.Flags().Lookup("meminterval").Value.String()) - if err != nil { - interval, _ = time.ParseDuration("100ms") - } - - fileMemStats, err := os.Create(memstats) - if err != nil { - return err - } - - fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n") - - go func() { - var stats runtime.MemStats - - start := htime.Now().UnixNano() - - for { - runtime.ReadMemStats(&stats) - if fileMemStats != nil { - fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n", - (htime.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased)) - time.Sleep(interval) - } else { - break - } - } - }() +func (c *serverCommand) partialReRender(urls ...string) error { + defer func() { + c.errState.setWasErr(false) + }() + c.errState.setBuildErr(nil) + visited := make(map[string]bool) + for _, url := range urls { + visited[url] = true } - return nil + + // Note: We do not set NoBuildLock as the file lock is not acquired at this stage. + return c.hugo().Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyVisited: visited, PartialReRender: true, ErrRecovery: c.errState.wasErr()}) +} + +func (c *serverCommand) serve() error { + isMultiHost := c.conf().configs.IsMultihost + var err error + h, err := c.r.HugFromConfig(c.conf()) + if err != nil { + return err + } + r := c.r + + var ( + baseURLs []string + roots []string + ) + + if isMultiHost { + for _, l := range c.conf().configs.ConfigLangs() { + baseURLs = append(baseURLs, l.BaseURL().String()) + roots = append(roots, l.Language().Lang) + } + } else { + l := c.conf().configs.GetFirstLanguageConfig() + baseURLs = []string{l.BaseURL().String()} + roots = []string{""} + } + + // Cache it here. The HugoSites object may be unavailable later on due to intermittent configuration errors. + // To allow the en user to change the error template while the server is running, we use + // the freshest template we can provide. + var ( + errTempl tpl.Template + templHandler tpl.TemplateHandler + ) + getErrorTemplateAndHandler := func(h *hugolib.HugoSites) (tpl.Template, tpl.TemplateHandler) { + if h == nil { + return errTempl, templHandler + } + templHandler := h.Tmpl() + errTempl, found := templHandler.Lookup("_server/error.html") + if !found { + panic("template server/error.html not found") + } + return errTempl, templHandler + } + errTempl, templHandler = getErrorTemplateAndHandler(h) + + srv := &fileServer{ + baseURLs: baseURLs, + roots: roots, + c: c, + errorTemplate: func(ctx any) (io.Reader, error) { + // hugoTry does not block, getErrorTemplateAndHandler will fall back + // to cached values if nil. + templ, handler := getErrorTemplateAndHandler(c.hugoTry()) + b := &bytes.Buffer{} + err := handler.ExecuteWithContext(context.Background(), templ, b, ctx) + return b, err + }, + } + + doLiveReload := !c.disableLiveReload + + if doLiveReload { + livereload.Initialize() + } + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + var servers []*http.Server + + wg1, ctx := errgroup.WithContext(context.Background()) + + for i := range baseURLs { + mu, listener, serverURL, endpoint, err := srv.createEndpoint(i) + srv := &http.Server{ + Addr: endpoint, + Handler: mu, + } + servers = append(servers, srv) + + if doLiveReload { + u, err := url.Parse(helpers.SanitizeURL(baseURLs[i])) + if err != nil { + return err + } + + mu.HandleFunc(u.Path+"/livereload.js", livereload.ServeJS) + mu.HandleFunc(u.Path+"/livereload", livereload.Handler) + } + r.Printf("Web Server is available at %s (bind address %s)\n", serverURL, c.serverInterface) + wg1.Go(func() error { + err = srv.Serve(listener) + if err != nil && err != http.ErrServerClosed { + return err + } + return nil + }) + } + + if c.r.IsTestRun() { + // Write a .ready file to disk to signal ready status. + // This is where the test is run from. + testInfo := map[string]any{ + "baseURLs": srv.baseURLs, + } + + dir := os.Getenv("WORK") + if dir != "" { + readyFile := filepath.Join(dir, ".ready") + // encode the test info as JSON into the .ready file. + b, err := json.Marshal(testInfo) + if err != nil { + return err + } + err = ioutil.WriteFile(readyFile, b, 0777) + if err != nil { + return err + } + } + + } + + r.Println("Press Ctrl+C to stop") + + err = func() error { + for { + select { + case <-c.quit: + return nil + case <-sigs: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + }() + + if err != nil { + r.Println("Error:", err) + } + + if h := c.hugoTry(); h != nil { + h.Close() + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + wg2, ctx := errgroup.WithContext(ctx) + for _, srv := range servers { + srv := srv + wg2.Go(func() error { + return srv.Shutdown(ctx) + }) + } + + err1, err2 := wg1.Wait(), wg2.Wait() + if err1 != nil { + return err1 + } + return err2 +} + +type serverPortListener struct { + p int + ln net.Listener +} + +type staticSyncer struct { + c *hugoBuilder +} + +func (s *staticSyncer) isStatic(filename string) bool { + return s.c.hugo().BaseFs.SourceFilesystems.IsStatic(filename) +} + +func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { + c := s.c + + syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) { + publishDir := helpers.FilePathSeparator + + if sourceFs.PublishFolder != "" { + publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) + } + + conf := s.c.conf().configs.Base + fs := s.c.conf().fs + syncer := fsync.NewSyncer() + syncer.NoTimes = conf.NoTimes + syncer.NoChmod = conf.NoChmod + syncer.ChmodFilter = chmodFilter + syncer.SrcFs = sourceFs.Fs + syncer.DestFs = fs.PublishDir + if c.s != nil && c.s.renderStaticToDisk { + syncer.DestFs = fs.PublishDirStatic + } + + // prevent spamming the log on changes + logger := helpers.NewDistinctErrorLogger() + + for _, ev := range staticEvents { + // Due to our approach of layering both directories and the content's rendered output + // into one we can't accurately remove a file not in one of the source directories. + // If a file is in the local static dir and also in the theme static dir and we remove + // it from one of those locations we expect it to still exist in the destination + // + // If Hugo generates a file (from the content dir) over a static file + // the content generated file should take precedence. + // + // Because we are now watching and handling individual events it is possible that a static + // event that occupies the same path as a content generated file will take precedence + // until a regeneration of the content takes places. + // + // Hugo assumes that these cases are very rare and will permit this bad behavior + // The alternative is to track every single file and which pipeline rendered it + // and then to handle conflict resolution on every event. + + fromPath := ev.Name + + relPath, found := sourceFs.MakePathRelative(fromPath) + + if !found { + // Not member of this virtual host. + continue + } + + // Remove || rename is harder and will require an assumption. + // Hugo takes the following approach: + // If the static file exists in any of the static source directories after this event + // Hugo will re-sync it. + // If it does not exist in all of the static directories Hugo will remove it. + // + // This assumes that Hugo has not generated content on top of a static file and then removed + // the source of that static file. In this case Hugo will incorrectly remove that file + // from the published directory. + if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { + if _, err := sourceFs.Fs.Stat(relPath); herrors.IsNotExist(err) { + // If file doesn't exist in any static dir, remove it + logger.Println("File no longer exists in static dir, removing", relPath) + _ = c.conf().fs.PublishDirStatic.RemoveAll(relPath) + + } else if err == nil { + // If file still exists, sync it + logger.Println("Syncing", relPath, "to", publishDir) + + if err := syncer.Sync(relPath, relPath); err != nil { + c.r.logger.Errorln(err) + } + } else { + c.r.logger.Errorln(err) + } + + continue + } + + // For all other event operations Hugo will sync static. + logger.Println("Syncing", relPath, "to", publishDir) + if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { + c.r.logger.Errorln(err) + } + } + + return 0, nil + } + + _, err := c.doWithPublishDirs(syncFn) + return err +} + +func chmodFilter(dst, src os.FileInfo) bool { + // Hugo publishes data from multiple sources, potentially + // with overlapping directory structures. We cannot sync permissions + // for directories as that would mean that we might end up with write-protected + // directories inside /public. + // One example of this would be syncing from the Go Module cache, + // which have 0555 directories. + return src.IsDir() +} + +func cleanErrorLog(content string) string { + content = strings.ReplaceAll(content, "\n", " ") + content = logReplacer.Replace(content) + content = logDuplicateTemplateExecuteRe.ReplaceAllString(content, "") + content = logDuplicateTemplateParseRe.ReplaceAllString(content, "") + seen := make(map[string]bool) + parts := strings.Split(content, ": ") + keep := make([]string, 0, len(parts)) + for _, part := range parts { + if seen[part] { + continue + } + seen[part] = true + keep = append(keep, part) + } + return strings.Join(keep, ": ") +} + +func injectLiveReloadScript(src io.Reader, baseURL url.URL) string { + var b bytes.Buffer + chain := transform.Chain{livereloadinject.New(baseURL)} + chain.Apply(&b, src) + + return b.String() +} + +func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) { + for _, e := range events { + if sourceFs.IsAsset(e.Name) { + de.AssetEvents = append(de.AssetEvents, e) + } else { + de.ContentEvents = append(de.ContentEvents, e) + } + } + return +} + +func pickOneWriteOrCreatePath(events []fsnotify.Event) string { + name := "" + + for _, ev := range events { + if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create { + if files.IsIndexContentFile(ev.Name) { + return ev.Name + } + + if files.IsContentFile(ev.Name) { + name = ev.Name + } + + } + } + + return name +} + +func removeErrorPrefixFromLog(content string) string { + return logErrorRe.ReplaceAllLiteralString(content, "") +} + +func formatByteCount(b uint64) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", + float64(b)/float64(div), "kMGTPE"[exp]) } diff --git a/commands/server_errors.go b/commands/server_errors.go deleted file mode 100644 index edf658156..000000000 --- a/commands/server_errors.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "bytes" - "io" - "net/url" - - "github.com/gohugoio/hugo/transform" - "github.com/gohugoio/hugo/transform/livereloadinject" -) - -func injectLiveReloadScript(src io.Reader, baseURL url.URL) string { - var b bytes.Buffer - chain := transform.Chain{livereloadinject.New(baseURL)} - chain.Apply(&b, src) - - return b.String() -} diff --git a/commands/server_test.go b/commands/server_test.go deleted file mode 100644 index 010208067..000000000 --- a/commands/server_test.go +++ /dev/null @@ -1,429 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "context" - "fmt" - "net/http" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/htesting" - "golang.org/x/sync/errgroup" - - qt "github.com/frankban/quicktest" -) - -// Issue 9518 -func TestServerPanicOnConfigError(t *testing.T) { - c := qt.New(t) - - config := ` -[markup] -[markup.highlight] -linenos='table' -` - - r := runServerTest(c, - serverTestOptions{ - config: config, - }, - ) - - c.Assert(r.err, qt.IsNotNil) - c.Assert(r.err.Error(), qt.Contains, "cannot parse 'Highlight.LineNos' as bool:") -} - -func TestServer404(t *testing.T) { - c := qt.New(t) - - r := runServerTest(c, - serverTestOptions{ - pathsToGet: []string{"this/does/not/exist"}, - getNumHomes: 1, - }, - ) - - c.Assert(r.err, qt.IsNil) - pr := r.pathsResults["this/does/not/exist"] - c.Assert(pr.statusCode, qt.Equals, http.StatusNotFound) - c.Assert(pr.body, qt.Contains, "404: 404 Page not found|Not Found.") -} - -func TestServerPathEncodingIssues(t *testing.T) { - c := qt.New(t) - - // Issue 10287 - c.Run("Unicode paths", func(c *qt.C) { - r := runServerTest(c, - serverTestOptions{ - pathsToGet: []string{"hügö/"}, - getNumHomes: 1, - }, - ) - - c.Assert(r.err, qt.IsNil) - c.Assert(r.pathsResults["hügö/"].body, qt.Contains, "This is hügö") - }) - - // Issue 10314 - c.Run("Windows multilingual 404", func(c *qt.C) { - config := ` -baseURL = 'https://example.org/' -title = 'Hugo Forum Topic #40568' - -defaultContentLanguageInSubdir = true - -[languages.en] -contentDir = 'content/en' -languageCode = 'en-US' -languageName = 'English' -weight = 1 - -[languages.es] -contentDir = 'content/es' -languageCode = 'es-ES' -languageName = 'Espanol' -weight = 2 - -[server] -[[server.redirects]] -from = '/en/**' -to = '/en/404.html' -status = 404 - -[[server.redirects]] -from = '/es/**' -to = '/es/404.html' -status = 404 -` - r := runServerTest(c, - serverTestOptions{ - config: config, - pathsToGet: []string{"en/this/does/not/exist", "es/this/does/not/exist"}, - getNumHomes: 1, - }, - ) - - c.Assert(r.err, qt.IsNil) - pr1 := r.pathsResults["en/this/does/not/exist"] - pr2 := r.pathsResults["es/this/does/not/exist"] - c.Assert(pr1.statusCode, qt.Equals, http.StatusNotFound) - c.Assert(pr2.statusCode, qt.Equals, http.StatusNotFound) - c.Assert(pr1.body, qt.Contains, "404: 404 Page not found|Not Found.") - c.Assert(pr2.body, qt.Contains, "404: 404 Page not found|Not Found.") - - }) - -} -func TestServerFlags(t *testing.T) { - c := qt.New(t) - - assertPublic := func(c *qt.C, r serverTestResult, renderStaticToDisk bool) { - c.Assert(r.err, qt.IsNil) - c.Assert(r.homesContent[0], qt.Contains, "Environment: development") - c.Assert(r.publicDirnames["myfile.txt"], qt.Equals, renderStaticToDisk) - - } - - for _, test := range []struct { - flag string - assert func(c *qt.C, r serverTestResult) - }{ - {"", func(c *qt.C, r serverTestResult) { - assertPublic(c, r, false) - }}, - {"--renderToDisk", func(c *qt.C, r serverTestResult) { - assertPublic(c, r, true) - }}, - {"--renderStaticToDisk", func(c *qt.C, r serverTestResult) { - assertPublic(c, r, true) - }}, - } { - c.Run(test.flag, func(c *qt.C) { - config := ` -baseURL="https://example.org" -` - - var args []string - if test.flag != "" { - args = strings.Split(test.flag, "=") - } - - opts := serverTestOptions{ - config: config, - args: args, - getNumHomes: 1, - } - - r := runServerTest(c, opts) - - test.assert(c, r) - - }) - - } - -} - -func TestServerBugs(t *testing.T) { - // TODO(bep) this is flaky on Windows on GH Actions. - if htesting.IsGitHubAction() && runtime.GOOS == "windows" { - t.Skip("skipping on windows") - } - c := qt.New(t) - - for _, test := range []struct { - name string - config string - flag string - numservers int - assert func(c *qt.C, r serverTestResult) - }{ - {"PostProcess, memory", "", "", 1, func(c *qt.C, r serverTestResult) { - c.Assert(r.err, qt.IsNil) - c.Assert(r.homesContent[0], qt.Contains, "PostProcess: /foo.min.css") - }}, - // Issue 9788 - {"PostProcess, memory", "", "", 1, func(c *qt.C, r serverTestResult) { - c.Assert(r.err, qt.IsNil) - c.Assert(r.homesContent[0], qt.Contains, "PostProcess: /foo.min.css") - }}, - {"PostProcess, disk", "", "--renderToDisk", 1, func(c *qt.C, r serverTestResult) { - c.Assert(r.err, qt.IsNil) - c.Assert(r.homesContent[0], qt.Contains, "PostProcess: /foo.min.css") - }}, - // Issue 9901 - {"Multihost", ` -defaultContentLanguage = 'en' -[languages] -[languages.en] -baseURL = 'https://example.com' -title = 'My blog' -weight = 1 -[languages.fr] -baseURL = 'https://example.fr' -title = 'Mon blogue' -weight = 2 -`, "", 2, func(c *qt.C, r serverTestResult) { - c.Assert(r.err, qt.IsNil) - for i, s := range []string{"My blog", "Mon blogue"} { - c.Assert(r.homesContent[i], qt.Contains, s) - } - }}, - } { - c.Run(test.name, func(c *qt.C) { - if test.config == "" { - test.config = ` -baseURL="https://example.org" -` - } - - var args []string - if test.flag != "" { - args = strings.Split(test.flag, "=") - } - - opts := serverTestOptions{ - config: test.config, - getNumHomes: test.numservers, - pathsToGet: []string{"this/does/not/exist"}, - args: args, - } - - r := runServerTest(c, opts) - pr := r.pathsResults["this/does/not/exist"] - c.Assert(pr.statusCode, qt.Equals, http.StatusNotFound) - c.Assert(pr.body, qt.Contains, "404: 404 Page not found|Not Found.") - test.assert(c, r) - - }) - - } - -} - -type serverTestResult struct { - err error - homesContent []string - content404 string - publicDirnames map[string]bool - pathsResults map[string]pathResult -} - -type pathResult struct { - statusCode int - body string -} - -type serverTestOptions struct { - getNumHomes int - config string - pathsToGet []string - args []string -} - -func runServerTest(c *qt.C, opts serverTestOptions) serverTestResult { - dir := createSimpleTestSite(c, testSiteConfig{configTOML: opts.config}) - result := serverTestResult{ - publicDirnames: make(map[string]bool), - pathsResults: make(map[string]pathResult), - } - - sp, err := helpers.FindAvailablePort() - c.Assert(err, qt.IsNil) - port := sp.Port - - defer func() { - os.RemoveAll(dir) - }() - - stop := make(chan bool) - - b := newCommandsBuilder() - scmd := b.newServerCmdSignaled(stop) - - cmd := scmd.getCommand() - args := append([]string{"-s=" + dir, fmt.Sprintf("-p=%d", port)}, opts.args...) - cmd.SetArgs(args) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - wg, ctx := errgroup.WithContext(ctx) - - wg.Go(func() error { - _, err := cmd.ExecuteC() - return err - }) - - if opts.getNumHomes > 0 { - // Esp. on slow CI machines, we need to wait a little before the web - // server is ready. - wait := 567 * time.Millisecond - if os.Getenv("CI") != "" { - wait = 2 * time.Second - } - time.Sleep(wait) - result.homesContent = make([]string, opts.getNumHomes) - for i := 0; i < opts.getNumHomes; i++ { - func() { - resp, err := http.Get(fmt.Sprintf("http://localhost:%d/", port+i)) - c.Assert(err, qt.IsNil) - c.Assert(resp.StatusCode, qt.Equals, http.StatusOK) - if err == nil { - defer resp.Body.Close() - result.homesContent[i] = helpers.ReaderToString(resp.Body) - } - }() - } - } - - for _, path := range opts.pathsToGet { - func() { - resp, err := http.Get(fmt.Sprintf("http://localhost:%d/%s", port, path)) - c.Assert(err, qt.IsNil) - pr := pathResult{ - statusCode: resp.StatusCode, - } - - if err == nil { - defer resp.Body.Close() - pr.body = helpers.ReaderToString(resp.Body) - } - result.pathsResults[path] = pr - }() - } - - time.Sleep(1 * time.Second) - - select { - case <-stop: - case stop <- true: - } - - pubFiles, err := os.ReadDir(filepath.Join(dir, "public")) - c.Assert(err, qt.IsNil) - for _, f := range pubFiles { - result.publicDirnames[f.Name()] = true - } - - result.err = wg.Wait() - - return result - -} - -func TestFixURL(t *testing.T) { - type data struct { - TestName string - CLIBaseURL string - CfgBaseURL string - AppendPort bool - Port int - Result string - } - tests := []data{ - {"Basic http localhost", "", "http://foo.com", true, 1313, "http://localhost:1313/"}, - {"Basic https production, http localhost", "", "https://foo.com", true, 1313, "http://localhost:1313/"}, - {"Basic subdir", "", "http://foo.com/bar", true, 1313, "http://localhost:1313/bar/"}, - {"Basic production", "http://foo.com", "http://foo.com", false, 80, "http://foo.com/"}, - {"Production subdir", "http://foo.com/bar", "http://foo.com/bar", false, 80, "http://foo.com/bar/"}, - {"No http", "", "foo.com", true, 1313, "//localhost:1313/"}, - {"Override configured port", "", "foo.com:2020", true, 1313, "//localhost:1313/"}, - {"No http production", "foo.com", "foo.com", false, 80, "//foo.com/"}, - {"No http production with port", "foo.com", "foo.com", true, 2020, "//foo.com:2020/"}, - {"No config", "", "", true, 1313, "//localhost:1313/"}, - } - - for _, test := range tests { - t.Run(test.TestName, func(t *testing.T) { - b := newCommandsBuilder() - s := b.newServerCmd() - v := config.NewWithTestDefaults() - baseURL := test.CLIBaseURL - v.Set("baseURL", test.CfgBaseURL) - s.serverAppend = test.AppendPort - s.serverPort = test.Port - result, err := s.fixURL(v, baseURL, s.serverPort) - if err != nil { - t.Errorf("Unexpected error %s", err) - } - if result != test.Result { - t.Errorf("Expected %q, got %q", test.Result, result) - } - }) - } -} - -func TestRemoveErrorPrefixFromLog(t *testing.T) { - c := qt.New(t) - content := `ERROR 2018/10/07 13:11:12 Error while rendering "home": template: _default/baseof.html:4:3: executing "main" at : error calling partial: template: partials/logo.html:5:84: executing "partials/logo.html" at <$resized.AHeight>: can't evaluate field AHeight in type *resource.Image -ERROR 2018/10/07 13:11:12 Rebuild failed: logged 1 error(s) -` - - withoutError := removeErrorPrefixFromLog(content) - - c.Assert(strings.Contains(withoutError, "ERROR"), qt.Equals, false) -} - -func isWindowsCI() bool { - return runtime.GOOS == "windows" && os.Getenv("CI") != "" -} diff --git a/commands/static_syncer.go b/commands/static_syncer.go deleted file mode 100644 index c248ca152..000000000 --- a/commands/static_syncer.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "path/filepath" - - "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/hugolib/filesystems" - - "github.com/fsnotify/fsnotify" - "github.com/gohugoio/hugo/helpers" - "github.com/spf13/fsync" -) - -type staticSyncer struct { - c *commandeer -} - -func newStaticSyncer(c *commandeer) (*staticSyncer, error) { - return &staticSyncer{c: c}, nil -} - -func (s *staticSyncer) isStatic(filename string) bool { - return s.c.hugo().BaseFs.SourceFilesystems.IsStatic(filename) -} - -func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { - c := s.c - - syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) { - publishDir := helpers.FilePathSeparator - - if sourceFs.PublishFolder != "" { - publishDir = filepath.Join(publishDir, sourceFs.PublishFolder) - } - - syncer := fsync.NewSyncer() - syncer.NoTimes = c.Cfg.GetBool("noTimes") - syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.ChmodFilter = chmodFilter - syncer.SrcFs = sourceFs.Fs - syncer.DestFs = c.Fs.PublishDir - if c.renderStaticToDisk { - syncer.DestFs = c.Fs.PublishDirStatic - } - - // prevent spamming the log on changes - logger := helpers.NewDistinctErrorLogger() - - for _, ev := range staticEvents { - // Due to our approach of layering both directories and the content's rendered output - // into one we can't accurately remove a file not in one of the source directories. - // If a file is in the local static dir and also in the theme static dir and we remove - // it from one of those locations we expect it to still exist in the destination - // - // If Hugo generates a file (from the content dir) over a static file - // the content generated file should take precedence. - // - // Because we are now watching and handling individual events it is possible that a static - // event that occupies the same path as a content generated file will take precedence - // until a regeneration of the content takes places. - // - // Hugo assumes that these cases are very rare and will permit this bad behavior - // The alternative is to track every single file and which pipeline rendered it - // and then to handle conflict resolution on every event. - - fromPath := ev.Name - - relPath, found := sourceFs.MakePathRelative(fromPath) - - if !found { - // Not member of this virtual host. - continue - } - - // Remove || rename is harder and will require an assumption. - // Hugo takes the following approach: - // If the static file exists in any of the static source directories after this event - // Hugo will re-sync it. - // If it does not exist in all of the static directories Hugo will remove it. - // - // This assumes that Hugo has not generated content on top of a static file and then removed - // the source of that static file. In this case Hugo will incorrectly remove that file - // from the published directory. - if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { - if _, err := sourceFs.Fs.Stat(relPath); herrors.IsNotExist(err) { - // If file doesn't exist in any static dir, remove it - logger.Println("File no longer exists in static dir, removing", relPath) - _ = c.Fs.PublishDirStatic.RemoveAll(relPath) - - } else if err == nil { - // If file still exists, sync it - logger.Println("Syncing", relPath, "to", publishDir) - - if err := syncer.Sync(relPath, relPath); err != nil { - c.logger.Errorln(err) - } - } else { - c.logger.Errorln(err) - } - - continue - } - - // For all other event operations Hugo will sync static. - logger.Println("Syncing", relPath, "to", publishDir) - if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { - c.logger.Errorln(err) - } - } - - return 0, nil - } - - _, err := c.doWithPublishDirs(syncFn) - return err -} diff --git a/commands/version.go b/commands/version.go deleted file mode 100644 index 287950a2d..000000000 --- a/commands/version.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2015 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package commands - -import ( - "github.com/gohugoio/hugo/common/hugo" - "github.com/spf13/cobra" - jww "github.com/spf13/jwalterweatherman" -) - -var _ cmder = (*versionCmd)(nil) - -type versionCmd struct { - *baseCmd -} - -func newVersionCmd() *versionCmd { - return &versionCmd{ - newBaseCmd(&cobra.Command{ - Use: "version", - Short: "Print the version number of Hugo", - Long: `All software has versions. This is Hugo's.`, - RunE: func(cmd *cobra.Command, args []string) error { - printHugoVersion() - return nil - }, - }), - } -} - -func printHugoVersion() { - jww.FEEDBACK.Println(hugo.BuildVersionString()) -} diff --git a/commands/xcommand_template.go b/commands/xcommand_template.go new file mode 100644 index 000000000..6bb507a5e --- /dev/null +++ b/commands/xcommand_template.go @@ -0,0 +1,78 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "context" + "fmt" + + "github.com/bep/simplecobra" + "github.com/spf13/cobra" +) + +func newSimpleTemplateCommand() simplecobra.Commander { + return &simpleCommand{ + name: "template", + run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error { + + return nil + }, + withc: func(cmd *cobra.Command) { + + }, + } + +} + +func newTemplateCommand() *templateCommand { + return &templateCommand{ + commands: []simplecobra.Commander{}, + } + +} + +type templateCommand struct { + r *rootCommand + + commands []simplecobra.Commander +} + +func (c *templateCommand) Commands() []simplecobra.Commander { + return c.commands +} + +func (c *templateCommand) Name() string { + return "template" +} + +func (c *templateCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error { + conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, nil)) + if err != nil { + return err + } + fmt.Println("templateCommand.Run", conf) + + return nil +} + +func (c *templateCommand) WithCobraCommand(cmd *cobra.Command) error { + cmd.Short = "Print the site configuration" + cmd.Long = `Print the site configuration, both default and custom settings.` + return nil +} + +func (c *templateCommand) Init(cd, runner *simplecobra.Commandeer) error { + c.r = cd.Root.Command.(*rootCommand) + return nil +} diff --git a/common/hstrings/strings.go b/common/hstrings/strings.go new file mode 100644 index 000000000..6c0f820fe --- /dev/null +++ b/common/hstrings/strings.go @@ -0,0 +1,57 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hstrings + +import ( + "fmt" + "strings" + + "github.com/gohugoio/hugo/compare" +) + +var _ compare.Eqer = StringEqualFold("") + +// StringEqualFold is a string that implements the compare.Eqer interface and considers +// two strings equal if they are equal when folded to lower case. +// The compare.Eqer interface is used in Hugo to compare values in templates (e.g. using the eq template function). +type StringEqualFold string + +func (s StringEqualFold) EqualFold(s2 string) bool { + return strings.EqualFold(string(s), s2) +} + +func (s StringEqualFold) String() string { + return string(s) +} + +func (s StringEqualFold) Eq(s2 any) bool { + switch ss := s2.(type) { + case string: + return s.EqualFold(ss) + case fmt.Stringer: + return s.EqualFold(ss.String()) + } + + return false +} + +// EqualAny returns whether a string is equal to any of the given strings. +func EqualAny(a string, b ...string) bool { + for _, s := range b { + if a == s { + return true + } + } + return false +} diff --git a/config/compositeConfig_test.go b/common/hstrings/strings_test.go similarity index 53% rename from config/compositeConfig_test.go rename to common/hstrings/strings_test.go index 60644102f..dc2eae6f2 100644 --- a/config/compositeConfig_test.go +++ b/common/hstrings/strings_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package config +package hstrings import ( "testing" @@ -19,22 +19,18 @@ import ( qt "github.com/frankban/quicktest" ) -func TestCompositeConfig(t *testing.T) { +func TestStringEqualFold(t *testing.T) { c := qt.New(t) - c.Run("Set and get", func(c *qt.C) { - base, layer := New(), New() - cfg := NewCompositeConfig(base, layer) + s1 := "A" + s2 := "a" - layer.Set("a1", "av") - base.Set("b1", "bv") - cfg.Set("c1", "cv") + c.Assert(StringEqualFold(s1).EqualFold(s2), qt.Equals, true) + c.Assert(StringEqualFold(s1).EqualFold(s1), qt.Equals, true) + c.Assert(StringEqualFold(s2).EqualFold(s1), qt.Equals, true) + c.Assert(StringEqualFold(s2).EqualFold(s2), qt.Equals, true) + c.Assert(StringEqualFold(s1).EqualFold("b"), qt.Equals, false) + c.Assert(StringEqualFold(s1).Eq(s2), qt.Equals, true) + c.Assert(StringEqualFold(s1).Eq("b"), qt.Equals, false) - c.Assert(cfg.Get("a1"), qt.Equals, "av") - c.Assert(cfg.Get("b1"), qt.Equals, "bv") - c.Assert(cfg.Get("c1"), qt.Equals, "cv") - c.Assert(cfg.IsSet("c1"), qt.IsTrue) - c.Assert(layer.IsSet("c1"), qt.IsTrue) - c.Assert(base.IsSet("c1"), qt.IsFalse) - }) } diff --git a/common/htime/time.go b/common/htime/time.go index d30ecf7e1..961962b60 100644 --- a/common/htime/time.go +++ b/common/htime/time.go @@ -14,6 +14,7 @@ package htime import ( + "log" "strings" "time" @@ -163,3 +164,11 @@ func Since(t time.Time) time.Duration { type AsTimeProvider interface { AsTime(zone *time.Location) time.Time } + +// StopWatch is a simple helper to measure time during development. +func StopWatch(name string) func() { + start := time.Now() + return func() { + log.Printf("StopWatch %q took %s", name, time.Since(start)) + } +} diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index efcb470a3..6402d7b88 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -46,8 +46,8 @@ var ( vendorInfo string ) -// Info contains information about the current Hugo environment -type Info struct { +// HugoInfo contains information about the current Hugo environment +type HugoInfo struct { CommitHash string BuildDate string @@ -64,30 +64,30 @@ type Info struct { } // Version returns the current version as a comparable version string. -func (i Info) Version() VersionString { +func (i HugoInfo) Version() VersionString { return CurrentVersion.Version() } // Generator a Hugo meta generator HTML tag. -func (i Info) Generator() template.HTML { +func (i HugoInfo) Generator() template.HTML { return template.HTML(fmt.Sprintf(``, CurrentVersion.String())) } -func (i Info) IsProduction() bool { +func (i HugoInfo) IsProduction() bool { return i.Environment == EnvironmentProduction } -func (i Info) IsExtended() bool { +func (i HugoInfo) IsExtended() bool { return IsExtended } // Deps gets a list of dependencies for this Hugo build. -func (i Info) Deps() []*Dependency { +func (i HugoInfo) Deps() []*Dependency { return i.deps } // NewInfo creates a new Hugo Info object. -func NewInfo(environment string, deps []*Dependency) Info { +func NewInfo(environment string, deps []*Dependency) HugoInfo { if environment == "" { environment = EnvironmentProduction } @@ -104,7 +104,7 @@ func NewInfo(environment string, deps []*Dependency) Info { goVersion = bi.GoVersion } - return Info{ + return HugoInfo{ CommitHash: commitHash, BuildDate: buildDate, Environment: environment, @@ -115,7 +115,7 @@ func NewInfo(environment string, deps []*Dependency) Info { // GetExecEnviron creates and gets the common os/exec environment used in the // external programs we interact with via os/exec, e.g. postcss. -func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string { +func GetExecEnviron(workDir string, cfg config.AllProvider, fs afero.Fs) []string { var env []string nodepath := filepath.Join(workDir, "node_modules") if np := os.Getenv("NODE_PATH"); np != "" { @@ -123,10 +123,9 @@ func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string { } config.SetEnvVars(&env, "NODE_PATH", nodepath) config.SetEnvVars(&env, "PWD", workDir) - config.SetEnvVars(&env, "HUGO_ENVIRONMENT", cfg.GetString("environment")) - config.SetEnvVars(&env, "HUGO_ENV", cfg.GetString("environment")) - - config.SetEnvVars(&env, "HUGO_PUBLISHDIR", filepath.Join(workDir, cfg.GetString("publishDirOrig"))) + config.SetEnvVars(&env, "HUGO_ENVIRONMENT", cfg.Environment()) + config.SetEnvVars(&env, "HUGO_ENV", cfg.Environment()) + config.SetEnvVars(&env, "HUGO_PUBLISHDIR", filepath.Join(workDir, cfg.BaseConfig().PublishDir)) if fs != nil { fis, err := afero.ReadDir(fs, files.FolderJSConfig) diff --git a/common/loggers/ignorableLogger.go b/common/loggers/ignorableLogger.go index 5040d1036..c8aba560e 100644 --- a/common/loggers/ignorableLogger.go +++ b/common/loggers/ignorableLogger.go @@ -15,7 +15,6 @@ package loggers import ( "fmt" - "strings" ) // IgnorableLogger is a logger that ignores certain log statements. @@ -31,14 +30,13 @@ type ignorableLogger struct { } // NewIgnorableLogger wraps the given logger and ignores the log statement IDs given. -func NewIgnorableLogger(logger Logger, statements ...string) IgnorableLogger { - statementsSet := make(map[string]bool) - for _, s := range statements { - statementsSet[strings.ToLower(s)] = true +func NewIgnorableLogger(logger Logger, statements map[string]bool) IgnorableLogger { + if statements == nil { + statements = make(map[string]bool) } return ignorableLogger{ Logger: logger, - statements: statementsSet, + statements: statements, } } diff --git a/common/maps/maps.go b/common/maps/maps.go index 2d8a122ca..6aefde927 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -43,25 +43,25 @@ func ToStringMapE(in any) (map[string]any, error) { // ToParamsAndPrepare converts in to Params and prepares it for use. // If in is nil, an empty map is returned. // See PrepareParams. -func ToParamsAndPrepare(in any) (Params, bool) { +func ToParamsAndPrepare(in any) (Params, error) { if types.IsNil(in) { - return Params{}, true + return Params{}, nil } m, err := ToStringMapE(in) if err != nil { - return nil, false + return nil, err } PrepareParams(m) - return m, true + return m, nil } // MustToParamsAndPrepare calls ToParamsAndPrepare and panics if it fails. func MustToParamsAndPrepare(in any) Params { - if p, ok := ToParamsAndPrepare(in); ok { - return p - } else { - panic(fmt.Sprintf("cannot convert %T to maps.Params", in)) + p, err := ToParamsAndPrepare(in) + if err != nil { + panic(fmt.Sprintf("cannot convert %T to maps.Params: %s", in, err)) } + return p } // ToStringMap converts in to map[string]interface{}. @@ -96,6 +96,8 @@ func ToSliceStringMap(in any) ([]map[string]any, error) { switch v := in.(type) { case []map[string]any: return v, nil + case Params: + return []map[string]any{v}, nil case []any: var s []map[string]any for _, entry := range v { @@ -123,6 +125,23 @@ func LookupEqualFold[T any | string](m map[string]T, key string) (T, bool) { return s, false } +// MergeShallow merges src into dst, but only if the key does not already exist in dst. +// The keys are compared case insensitively. +func MergeShallow(dst, src map[string]any) { + for k, v := range src { + found := false + for dk := range dst { + if strings.EqualFold(dk, k) { + found = true + break + } + } + if !found { + dst[k] = v + } + } +} + type keyRename struct { pattern glob.Glob newKey string diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go index 0b84d2dd7..0e8589d34 100644 --- a/common/maps/maps_test.go +++ b/common/maps/maps_test.go @@ -116,11 +116,11 @@ func TestToSliceStringMap(t *testing.T) { func TestToParamsAndPrepare(t *testing.T) { c := qt.New(t) - _, ok := ToParamsAndPrepare(map[string]any{"A": "av"}) - c.Assert(ok, qt.IsTrue) + _, err := ToParamsAndPrepare(map[string]any{"A": "av"}) + c.Assert(err, qt.IsNil) - params, ok := ToParamsAndPrepare(nil) - c.Assert(ok, qt.IsTrue) + params, err := ToParamsAndPrepare(nil) + c.Assert(err, qt.IsNil) c.Assert(params, qt.DeepEquals, Params{}) } diff --git a/common/maps/params.go b/common/maps/params.go index 4bf95f43b..eb60fbbfc 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -23,30 +23,37 @@ import ( // Params is a map where all keys are lower case. type Params map[string]any -// Get does a lower case and nested search in this map. +// KeyParams is an utility struct for the WalkParams method. +type KeyParams struct { + Key string + Params Params +} + +// GetNested does a lower case and nested search in this map. // It will return nil if none found. -func (p Params) Get(indices ...string) any { +// Make all of these methods internal somehow. +func (p Params) GetNested(indices ...string) any { v, _, _ := getNested(p, indices) return v } -// Set overwrites values in p with values in pp for common or new keys. +// Set overwrites values in dst with values in src for common or new keys. // This is done recursively. -func (p Params) Set(pp Params) { - for k, v := range pp { - vv, found := p[k] +func SetParams(dst, src Params) { + for k, v := range src { + vv, found := dst[k] if !found { - p[k] = v + dst[k] = v } else { switch vvv := vv.(type) { case Params: if pv, ok := v.(Params); ok { - vvv.Set(pv) + SetParams(vvv, pv) } else { - p[k] = v + dst[k] = v } default: - p[k] = v + dst[k] = v } } } @@ -70,18 +77,17 @@ func (p Params) IsZero() bool { } -// Merge transfers values from pp to p for new keys. +// MergeParamsWithStrategy transfers values from src to dst for new keys using the merge strategy given. // This is done recursively. -func (p Params) Merge(pp Params) { - p.merge("", pp) +func MergeParamsWithStrategy(strategy string, dst, src Params) { + dst.merge(ParamsMergeStrategy(strategy), src) } -// MergeRoot transfers values from pp to p for new keys where p is the -// root of the tree. +// MergeParamsWithStrategy transfers values from src to dst for new keys using the merge encoded in dst. // This is done recursively. -func (p Params) MergeRoot(pp Params) { - ms, _ := p.GetMergeStrategy() - p.merge(ms, pp) +func MergeParams(dst, src Params) { + ms, _ := dst.GetMergeStrategy() + dst.merge(ms, src) } func (p Params) merge(ps ParamsMergeStrategy, pp Params) { @@ -116,6 +122,7 @@ func (p Params) merge(ps ParamsMergeStrategy, pp Params) { } } +// For internal use. func (p Params) GetMergeStrategy() (ParamsMergeStrategy, bool) { if v, found := p[mergeStrategyKey]; found { if s, ok := v.(ParamsMergeStrategy); ok { @@ -125,6 +132,7 @@ func (p Params) GetMergeStrategy() (ParamsMergeStrategy, bool) { return ParamsMergeStrategyShallow, false } +// For internal use. func (p Params) DeleteMergeStrategy() bool { if _, found := p[mergeStrategyKey]; found { delete(p, mergeStrategyKey) @@ -133,7 +141,8 @@ func (p Params) DeleteMergeStrategy() bool { return false } -func (p Params) SetDefaultMergeStrategy(s ParamsMergeStrategy) { +// For internal use. +func (p Params) SetMergeStrategy(s ParamsMergeStrategy) { switch s { case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow: default: @@ -187,7 +196,7 @@ func GetNestedParam(keyStr, separator string, candidates ...Params) (any, error) keySegments := strings.Split(keyStr, separator) for _, m := range candidates { - if v := m.Get(keySegments...); v != nil { + if v := m.GetNested(keySegments...); v != nil { return v, nil } } @@ -236,6 +245,55 @@ const ( mergeStrategyKey = "_merge" ) +// CleanConfigStringMapString removes any processing instructions from m, +// m will never be modified. +func CleanConfigStringMapString(m map[string]string) map[string]string { + if m == nil || len(m) == 0 { + return m + } + if _, found := m[mergeStrategyKey]; !found { + return m + } + // Create a new map and copy all the keys except the merge strategy key. + m2 := make(map[string]string, len(m)-1) + for k, v := range m { + if k != mergeStrategyKey { + m2[k] = v + } + } + return m2 +} + +// CleanConfigStringMap is the same as CleanConfigStringMapString but for +// map[string]any. +func CleanConfigStringMap(m map[string]any) map[string]any { + if m == nil || len(m) == 0 { + return m + } + if _, found := m[mergeStrategyKey]; !found { + return m + } + // Create a new map and copy all the keys except the merge strategy key. + m2 := make(map[string]any, len(m)-1) + for k, v := range m { + if k != mergeStrategyKey { + m2[k] = v + } + switch v2 := v.(type) { + case map[string]any: + m2[k] = CleanConfigStringMap(v2) + case Params: + var p Params = CleanConfigStringMap(v2) + m2[k] = p + case map[string]string: + m2[k] = CleanConfigStringMapString(v2) + } + + } + return m2 + +} + func toMergeStrategy(v any) ParamsMergeStrategy { s := ParamsMergeStrategy(cast.ToString(v)) switch s { diff --git a/common/maps/params_test.go b/common/maps/params_test.go index a070e6f60..7e1dbbae7 100644 --- a/common/maps/params_test.go +++ b/common/maps/params_test.go @@ -81,7 +81,7 @@ func TestParamsSetAndMerge(t *testing.T) { p1, p2 := createParamsPair() - p1.Set(p2) + SetParams(p1, p2) c.Assert(p1, qt.DeepEquals, Params{ "a": "abv", @@ -97,7 +97,7 @@ func TestParamsSetAndMerge(t *testing.T) { p1, p2 = createParamsPair() - p1.Merge(p2) + MergeParamsWithStrategy("", p1, p2) // Default is to do a shallow merge. c.Assert(p1, qt.DeepEquals, Params{ @@ -111,8 +111,8 @@ func TestParamsSetAndMerge(t *testing.T) { }) p1, p2 = createParamsPair() - p1.SetDefaultMergeStrategy(ParamsMergeStrategyNone) - p1.Merge(p2) + p1.SetMergeStrategy(ParamsMergeStrategyNone) + MergeParamsWithStrategy("", p1, p2) p1.DeleteMergeStrategy() c.Assert(p1, qt.DeepEquals, Params{ @@ -125,8 +125,8 @@ func TestParamsSetAndMerge(t *testing.T) { }) p1, p2 = createParamsPair() - p1.SetDefaultMergeStrategy(ParamsMergeStrategyShallow) - p1.Merge(p2) + p1.SetMergeStrategy(ParamsMergeStrategyShallow) + MergeParamsWithStrategy("", p1, p2) p1.DeleteMergeStrategy() c.Assert(p1, qt.DeepEquals, Params{ @@ -140,8 +140,8 @@ func TestParamsSetAndMerge(t *testing.T) { }) p1, p2 = createParamsPair() - p1.SetDefaultMergeStrategy(ParamsMergeStrategyDeep) - p1.Merge(p2) + p1.SetMergeStrategy(ParamsMergeStrategyDeep) + MergeParamsWithStrategy("", p1, p2) p1.DeleteMergeStrategy() c.Assert(p1, qt.DeepEquals, Params{ diff --git a/hugolib/paths/baseURL.go b/common/urls/baseURL.go similarity index 62% rename from hugolib/paths/baseURL.go rename to common/urls/baseURL.go index a3c7e9d27..df26730ec 100644 --- a/hugolib/paths/baseURL.go +++ b/common/urls/baseURL.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,32 +11,37 @@ // See the License for the specific language governing permissions and // limitations under the License. -package paths +package urls import ( "fmt" "net/url" + "strconv" "strings" ) // A BaseURL in Hugo is normally on the form scheme://path, but the // form scheme: is also valid (mailto:hugo@rules.com). type BaseURL struct { - url *url.URL - urlStr string + url *url.URL + WithPath string + WithoutPath string + BasePath string } func (b BaseURL) String() string { - if b.urlStr != "" { - return b.urlStr - } - return b.url.String() + return b.WithPath } func (b BaseURL) Path() string { return b.url.Path } +func (b BaseURL) Port() int { + p, _ := strconv.Atoi(b.url.Port()) + return p +} + // HostURL returns the URL to the host root without any path elements. func (b BaseURL) HostURL() string { return strings.TrimSuffix(b.String(), b.Path()) @@ -44,7 +49,7 @@ func (b BaseURL) HostURL() string { // WithProtocol returns the BaseURL prefixed with the given protocol. // The Protocol is normally of the form "scheme://", i.e. "webcal://". -func (b BaseURL) WithProtocol(protocol string) (string, error) { +func (b BaseURL) WithProtocol(protocol string) (BaseURL, error) { u := b.URL() scheme := protocol @@ -62,10 +67,16 @@ func (b BaseURL) WithProtocol(protocol string) (string, error) { if isFullProtocol && u.Opaque != "" { u.Opaque = "//" + u.Opaque } else if isOpaqueProtocol && u.Opaque == "" { - return "", fmt.Errorf("cannot determine BaseURL for protocol %q", protocol) + return BaseURL{}, fmt.Errorf("cannot determine BaseURL for protocol %q", protocol) } - return u.String(), nil + return newBaseURLFromURL(u) +} + +func (b BaseURL) WithPort(port int) (BaseURL, error) { + u := b.URL() + u.Host = u.Hostname() + ":" + strconv.Itoa(port) + return newBaseURLFromURL(u) } // URL returns a copy of the internal URL. @@ -75,13 +86,25 @@ func (b BaseURL) URL() *url.URL { return &c } -func newBaseURLFromString(b string) (BaseURL, error) { - var result BaseURL - - base, err := url.Parse(b) +func NewBaseURLFromString(b string) (BaseURL, error) { + u, err := url.Parse(b) if err != nil { - return result, err + return BaseURL{}, err + } + return newBaseURLFromURL(u) + +} + +func newBaseURLFromURL(u *url.URL) (BaseURL, error) { + baseURL := BaseURL{url: u, WithPath: u.String()} + var baseURLNoPath = baseURL.URL() + baseURLNoPath.Path = "" + baseURL.WithoutPath = baseURLNoPath.String() + + basePath := u.Path + if basePath != "" && basePath != "/" { + baseURL.BasePath = basePath } - return BaseURL{url: base, urlStr: base.String()}, nil + return baseURL, nil } diff --git a/hugolib/paths/baseURL_test.go b/common/urls/baseURL_test.go similarity index 74% rename from hugolib/paths/baseURL_test.go rename to common/urls/baseURL_test.go index 77095bb7d..95dc73339 100644 --- a/hugolib/paths/baseURL_test.go +++ b/common/urls/baseURL_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package paths +package urls import ( "testing" @@ -21,46 +21,46 @@ import ( func TestBaseURL(t *testing.T) { c := qt.New(t) - b, err := newBaseURLFromString("http://example.com") + b, err := NewBaseURLFromString("http://example.com") c.Assert(err, qt.IsNil) c.Assert(b.String(), qt.Equals, "http://example.com") p, err := b.WithProtocol("webcal://") c.Assert(err, qt.IsNil) - c.Assert(p, qt.Equals, "webcal://example.com") + c.Assert(p.String(), qt.Equals, "webcal://example.com") p, err = b.WithProtocol("webcal") c.Assert(err, qt.IsNil) - c.Assert(p, qt.Equals, "webcal://example.com") + c.Assert(p.String(), qt.Equals, "webcal://example.com") _, err = b.WithProtocol("mailto:") c.Assert(err, qt.Not(qt.IsNil)) - b, err = newBaseURLFromString("mailto:hugo@rules.com") + b, err = NewBaseURLFromString("mailto:hugo@rules.com") c.Assert(err, qt.IsNil) c.Assert(b.String(), qt.Equals, "mailto:hugo@rules.com") // These are pretty constructed p, err = b.WithProtocol("webcal") c.Assert(err, qt.IsNil) - c.Assert(p, qt.Equals, "webcal:hugo@rules.com") + c.Assert(p.String(), qt.Equals, "webcal:hugo@rules.com") p, err = b.WithProtocol("webcal://") c.Assert(err, qt.IsNil) - c.Assert(p, qt.Equals, "webcal://hugo@rules.com") + c.Assert(p.String(), qt.Equals, "webcal://hugo@rules.com") // Test with "non-URLs". Some people will try to use these as a way to get // relative URLs working etc. - b, err = newBaseURLFromString("/") + b, err = NewBaseURLFromString("/") c.Assert(err, qt.IsNil) c.Assert(b.String(), qt.Equals, "/") - b, err = newBaseURLFromString("") + b, err = NewBaseURLFromString("") c.Assert(err, qt.IsNil) c.Assert(b.String(), qt.Equals, "") // BaseURL with sub path - b, err = newBaseURLFromString("http://example.com/sub") + b, err = NewBaseURLFromString("http://example.com/sub") c.Assert(err, qt.IsNil) c.Assert(b.String(), qt.Equals, "http://example.com/sub") c.Assert(b.HostURL(), qt.Equals, "http://example.com") diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go new file mode 100644 index 000000000..4daae3ccb --- /dev/null +++ b/config/allconfig/allconfig.go @@ -0,0 +1,813 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package allconfig contains the full configuration for Hugo. +// { "name": "Configuration", "description": "This section holds all configiration options in Hugo." } +package allconfig + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/urls" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/privacy" + "github.com/gohugoio/hugo/config/security" + "github.com/gohugoio/hugo/config/services" + "github.com/gohugoio/hugo/deploy" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/markup/markup_config" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/minifiers" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/navigation" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/related" + "github.com/gohugoio/hugo/resources/images" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/spf13/afero" + + xmaps "golang.org/x/exp/maps" +) + +// InternalConfig is the internal configuration for Hugo, not read from any user provided config file. +type InternalConfig struct { + // Server mode? + Running bool + + Quiet bool + Verbose bool + Clock string + Watch bool + DisableLiveReload bool + LiveReloadPort int +} + +type Config struct { + // For internal use only. + Internal InternalConfig `mapstructure:"-" json:"-"` + // For internal use only. + C ConfigCompiled `mapstructure:"-" json:"-"` + + RootConfig + + // Author information. + Author map[string]any + + // Social links. + Social map[string]string + + // The build configuration section contains build-related configuration options. + // {"identifiers": ["build"] } + Build config.BuildConfig `mapstructure:"-"` + + // The caches configuration section contains cache-related configuration options. + // {"identifiers": ["caches"] } + Caches filecache.Configs `mapstructure:"-"` + + // The markup configuration section contains markup-related configuration options. + // {"identifiers": ["markup"] } + Markup markup_config.Config `mapstructure:"-"` + + // The mediatypes configuration section maps the MIME type (a string) to a configuration object for that type. + // {"identifiers": ["mediatypes"], "refs": ["types:media:type"] } + MediaTypes *config.ConfigNamespace[map[string]media.MediaTypeConfig, media.Types] `mapstructure:"-"` + + Imaging *config.ConfigNamespace[images.ImagingConfig, images.ImagingConfigInternal] `mapstructure:"-"` + + // The outputformats configuration sections maps a format name (a string) to a configuration object for that format. + OutputFormats *config.ConfigNamespace[map[string]output.OutputFormatConfig, output.Formats] `mapstructure:"-"` + + // The outputs configuration section maps a Page Kind (a string) to a slice of output formats. + // This can be overridden in the front matter. + Outputs map[string][]string `mapstructure:"-"` + + // The cascade configuration section contains the top level front matter cascade configuration options, + // a slice of page matcher and params to apply to those pages. + Cascade *config.ConfigNamespace[[]page.PageMatcherParamsConfig, map[page.PageMatcher]maps.Params] `mapstructure:"-"` + + // Menu configuration. + // {"refs": ["config:languages:menus"] } + Menus *config.ConfigNamespace[map[string]navigation.MenuConfig, navigation.Menus] `mapstructure:"-"` + + // The deployment configuration section contains for hugo deploy. + Deployment deploy.DeployConfig `mapstructure:"-"` + + // Module configuration. + Module modules.Config `mapstructure:"-"` + + // Front matter configuration. + Frontmatter pagemeta.FrontmatterConfig `mapstructure:"-"` + + // Minification configuration. + Minify minifiers.MinifyConfig `mapstructure:"-"` + + // Permalink configuration. + Permalinks map[string]string `mapstructure:"-"` + + // Taxonomy configuration. + Taxonomies map[string]string `mapstructure:"-"` + + // Sitemap configuration. + Sitemap config.SitemapConfig `mapstructure:"-"` + + // Related content configuration. + Related related.Config `mapstructure:"-"` + + // Server configuration. + Server config.Server `mapstructure:"-"` + + // Privacy configuration. + Privacy privacy.Config `mapstructure:"-"` + + // Security configuration. + Security security.Config `mapstructure:"-"` + + // Services configuration. + Services services.Config `mapstructure:"-"` + + // User provided parameters. + // {"refs": ["config:languages:params"] } + Params maps.Params `mapstructure:"-"` + + // The languages configuration sections maps a language code (a string) to a configuration object for that language. + Languages map[string]langs.LanguageConfig `mapstructure:"-"` + + // UglyURLs configuration. Either a boolean or a sections map. + UglyURLs any `mapstructure:"-"` +} + +type configCompiler interface { + CompileConfig() error +} + +func (c Config) cloneForLang() *Config { + x := c + // Collapse all static dirs to one. + x.StaticDir = x.staticDirs() + // These will go away soon ... + x.StaticDir0 = nil + x.StaticDir1 = nil + x.StaticDir2 = nil + x.StaticDir3 = nil + x.StaticDir4 = nil + x.StaticDir5 = nil + x.StaticDir6 = nil + x.StaticDir7 = nil + x.StaticDir8 = nil + x.StaticDir9 = nil + x.StaticDir10 = nil + + return &x +} + +func (c *Config) CompileConfig() error { + s := c.Timeout + if _, err := strconv.Atoi(s); err == nil { + // A number, assume seconds. + s = s + "s" + } + timeout, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("failed to parse timeout: %s", err) + } + disabledKinds := make(map[string]bool) + for _, kind := range c.DisableKinds { + disabledKinds[strings.ToLower(kind)] = true + } + kindOutputFormats := make(map[string]output.Formats) + isRssDisabled := disabledKinds["rss"] + outputFormats := c.OutputFormats.Config + for kind, formats := range c.Outputs { + if disabledKinds[kind] { + continue + } + for _, format := range formats { + if isRssDisabled && format == "rss" { + // Legacy config. + continue + } + f, found := outputFormats.GetByName(format) + if !found { + return fmt.Errorf("unknown output format %q for kind %q", format, kind) + } + kindOutputFormats[kind] = append(kindOutputFormats[kind], f) + } + } + + disabledLangs := make(map[string]bool) + for _, lang := range c.DisableLanguages { + if lang == c.DefaultContentLanguage { + return fmt.Errorf("cannot disable default content language %q", lang) + } + disabledLangs[lang] = true + } + + ignoredErrors := make(map[string]bool) + for _, err := range c.IgnoreErrors { + ignoredErrors[strings.ToLower(err)] = true + } + + baseURL, err := urls.NewBaseURLFromString(c.BaseURL) + if err != nil { + return err + } + + isUglyURL := func(section string) bool { + switch v := c.UglyURLs.(type) { + case bool: + return v + case map[string]bool: + return v[section] + default: + return false + } + } + + ignoreFile := func(s string) bool { + return false + } + if len(c.IgnoreFiles) > 0 { + regexps := make([]*regexp.Regexp, len(c.IgnoreFiles)) + for i, pattern := range c.IgnoreFiles { + var err error + regexps[i], err = regexp.Compile(pattern) + if err != nil { + return fmt.Errorf("failed to compile ignoreFiles pattern %q: %s", pattern, err) + } + } + ignoreFile = func(s string) bool { + for _, r := range regexps { + if r.MatchString(s) { + return true + } + } + return false + } + } + + var clock time.Time + if c.Internal.Clock != "" { + var err error + clock, err = time.Parse(time.RFC3339, c.Internal.Clock) + if err != nil { + return fmt.Errorf("failed to parse clock: %s", err) + } + } + + c.C = ConfigCompiled{ + Timeout: timeout, + BaseURL: baseURL, + BaseURLLiveReload: baseURL, + DisabledKinds: disabledKinds, + DisabledLanguages: disabledLangs, + IgnoredErrors: ignoredErrors, + KindOutputFormats: kindOutputFormats, + CreateTitle: helpers.GetTitleFunc(c.TitleCaseStyle), + IsUglyURLSection: isUglyURL, + IgnoreFile: ignoreFile, + MainSections: c.MainSections, + Clock: clock, + } + + for _, s := range allDecoderSetups { + if getCompiler := s.getCompiler; getCompiler != nil { + if err := getCompiler(c).CompileConfig(); err != nil { + return err + } + } + } + + return nil +} + +func (c Config) IsKindEnabled(kind string) bool { + return !c.C.DisabledKinds[kind] +} + +func (c Config) IsLangDisabled(lang string) bool { + return c.C.DisabledLanguages[lang] +} + +// ConfigCompiled holds values and functions that are derived from the config. +type ConfigCompiled struct { + Timeout time.Duration + BaseURL urls.BaseURL + BaseURLLiveReload urls.BaseURL + KindOutputFormats map[string]output.Formats + DisabledKinds map[string]bool + DisabledLanguages map[string]bool + IgnoredErrors map[string]bool + CreateTitle func(s string) string + IsUglyURLSection func(section string) bool + IgnoreFile func(filename string) bool + MainSections []string + Clock time.Time +} + +// This may be set after the config is compiled. +func (c *ConfigCompiled) SetMainSections(sections []string) { + c.MainSections = sections +} + +// This is set after the config is compiled by the server command. +func (c *ConfigCompiled) SetBaseURL(baseURL, baseURLLiveReload urls.BaseURL) { + c.BaseURL = baseURL + c.BaseURLLiveReload = baseURLLiveReload +} + +// RootConfig holds all the top-level configuration options in Hugo +type RootConfig struct { + + // The base URL of the site. + // Note that the default value is empty, but Hugo requires a valid URL (e.g. "https://example.com/") to work properly. + // {"identifiers": ["URL"] } + BaseURL string + + // Whether to build content marked as draft.X + // {"identifiers": ["draft"] } + BuildDrafts bool + + // Whether to build content with expiryDate in the past. + // {"identifiers": ["expiryDate"] } + BuildExpired bool + + // Whether to build content with publishDate in the future. + // {"identifiers": ["publishDate"] } + BuildFuture bool + + // Copyright information. + Copyright string + + // The language to apply to content without any Clolanguage indicator. + DefaultContentLanguage string + + // By defefault, we put the default content language in the root and the others below their language ID, e.g. /no/. + // Set this to true to put all languages below their language ID. + DefaultContentLanguageInSubdir bool + + // Disable creation of alias redirect pages. + DisableAliases bool + + // Disable lower casing of path segments. + DisablePathToLower bool + + // Disable page kinds from build. + DisableKinds []string + + // A list of languages to disable. + DisableLanguages []string + + // Disable the injection of the Hugo generator tag on the home page. + DisableHugoGeneratorInject bool + + // Enable replacement in Pages' Content of Emoji shortcodes with their equivalent Unicode characters. + // {"identifiers": ["Content", "Unicode"] } + EnableEmoji bool + + // THe main section(s) of the site. + // If not set, Hugo will try to guess this from the content. + MainSections []string + + // Enable robots.txt generation. + EnableRobotsTXT bool + + // When enabled, Hugo will apply Git version information to each Page if possible, which + // can be used to keep lastUpdated in synch and to print version information. + // {"identifiers": ["Page"] } + EnableGitInfo bool + + // Enable to track, calculate and print metrics. + TemplateMetrics bool + + // Enable to track, print and calculate metric hints. + TemplateMetricsHints bool + + // Enable to disable the build lock file. + NoBuildLock bool + + // A list of error IDs to ignore. + IgnoreErrors []string + + // A list of regexps that match paths to ignore. + // Deprecated: Use the settings on module imports. + IgnoreFiles []string + + // Ignore cache. + IgnoreCache bool + + // Enable to print greppable placeholders (on the form "[i18n] TRANSLATIONID") for missing translation strings. + EnableMissingTranslationPlaceholders bool + + // Enable to print warnings for missing translation strings. + LogI18nWarnings bool + + // ENable to print warnings for multiple files published to the same destination. + LogPathWarnings bool + + // The configured environment. Default is "development" for server and "production" for build. + Environment string + + // The default language code. + LanguageCode string + + // Enable if the site content has CJK language (Chinese, Japanese, or Korean). This affects how Hugo counts words. + HasCJKLanguage bool + + // The default number of pages per page when paginating. + Paginate int + + // The path to use when creating pagination URLs, e.g. "page" in /page/2/. + PaginatePath string + + // Whether to pluralize default list titles. + // Note that this currently only works for English, but you can provide your own title in the content file's front matter. + PluralizeListTitles bool + + // Make all relative URLs absolute using the baseURL. + // {"identifiers": ["baseURL"] } + CanonifyURLs bool + + // Enable this to make all relative URLs relative to content root. Note that this does not affect absolute URLs. + RelativeURLs bool + + // Removes non-spacing marks from composite characters in content paths. + RemovePathAccents bool + + // Whether to track and print unused templates during the build. + PrintUnusedTemplates bool + + // URL to be used as a placeholder when a page reference cannot be found in ref or relref. Is used as-is. + RefLinksNotFoundURL string + + // When using ref or relref to resolve page links and a link cannot be resolved, it will be logged with this log level. + // Valid values are ERROR (default) or WARNING. Any ERROR will fail the build (exit -1). + RefLinksErrorLevel string + + // This will create a menu with all the sections as menu items and all the sections’ pages as “shadow-members”. + SectionPagesMenu string + + // The length of text in words to show in a .Summary. + SummaryLength int + + // The site title. + Title string + + // The theme(s) to use. + // See Modules for more a more flexible way to load themes. + Theme []string + + // Timeout for generating page contents, specified as a duration or in milliseconds. + Timeout string + + // The time zone (or location), e.g. Europe/Oslo, used to parse front matter dates without such information and in the time function. + TimeZone string + + // Set titleCaseStyle to specify the title style used by the title template function and the automatic section titles in Hugo. + // It defaults to AP Stylebook for title casing, but you can also set it to Chicago or Go (every word starts with a capital letter). + TitleCaseStyle string + + // The editor used for opening up new content. + NewContentEditor string + + // Don't sync modification time of files for the static mounts. + NoTimes bool + + // Don't sync modification time of files for the static mounts. + NoChmod bool + + // Clean the destination folder before a new build. + // This currently only handles static files. + CleanDestinationDir bool + + // A Glob pattern of module paths to ignore in the _vendor folder. + IgnoreVendorPaths string + + config.CommonDirs `mapstructure:",squash"` + + // The odd constructs below are kept for backwards compatibility. + // Deprecated: Use module mount config instead. + StaticDir []string + // Deprecated: Use module mount config instead. + StaticDir0 []string + // Deprecated: Use module mount config instead. + StaticDir1 []string + // Deprecated: Use module mount config instead. + StaticDir2 []string + // Deprecated: Use module mount config instead. + StaticDir3 []string + // Deprecated: Use module mount config instead. + StaticDir4 []string + // Deprecated: Use module mount config instead. + StaticDir5 []string + // Deprecated: Use module mount config instead. + StaticDir6 []string + // Deprecated: Use module mount config instead. + StaticDir7 []string + // Deprecated: Use module mount config instead. + StaticDir8 []string + // Deprecated: Use module mount config instead. + StaticDir9 []string + // Deprecated: Use module mount config instead. + StaticDir10 []string +} + +func (c RootConfig) staticDirs() []string { + var dirs []string + dirs = append(dirs, c.StaticDir...) + dirs = append(dirs, c.StaticDir0...) + dirs = append(dirs, c.StaticDir1...) + dirs = append(dirs, c.StaticDir2...) + dirs = append(dirs, c.StaticDir3...) + dirs = append(dirs, c.StaticDir4...) + dirs = append(dirs, c.StaticDir5...) + dirs = append(dirs, c.StaticDir6...) + dirs = append(dirs, c.StaticDir7...) + dirs = append(dirs, c.StaticDir8...) + dirs = append(dirs, c.StaticDir9...) + dirs = append(dirs, c.StaticDir10...) + return helpers.UniqueStringsReuse(dirs) +} + +type Configs struct { + Base *Config + LoadingInfo config.LoadConfigResult + LanguageConfigMap map[string]*Config + LanguageConfigSlice []*Config + + IsMultihost bool + Languages langs.Languages + LanguagesDefaultFirst langs.Languages + + Modules modules.Modules + ModulesClient *modules.Client + + configLangs []config.AllProvider +} + +func (c *Configs) IsZero() bool { + // A config always has at least one language. + return c == nil || len(c.Languages) == 0 +} + +func (c *Configs) Init() error { + c.configLangs = make([]config.AllProvider, len(c.Languages)) + for i, l := range c.LanguagesDefaultFirst { + c.configLangs[i] = ConfigLanguage{ + m: c, + config: c.LanguageConfigMap[l.Lang], + baseConfig: c.LoadingInfo.BaseConfig, + language: l, + } + } + + if len(c.Modules) == 0 { + return errors.New("no modules loaded (ned at least the main module)") + } + + // Apply default project mounts. + if err := modules.ApplyProjectConfigDefaults(c.Modules[0], c.configLangs...); err != nil { + return err + } + + return nil +} + +func (c Configs) ConfigLangs() []config.AllProvider { + return c.configLangs +} + +func (c Configs) GetFirstLanguageConfig() config.AllProvider { + return c.configLangs[0] +} + +func (c Configs) GetByLang(lang string) config.AllProvider { + for _, l := range c.configLangs { + if l.Language().Lang == lang { + return l + } + } + return nil +} + +// FromLoadConfigResult creates a new Config from res. +func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, error) { + if !res.Cfg.IsSet("languages") { + // We need at least one + lang := res.Cfg.GetString("defaultContentLanguage") + res.Cfg.Set("languages", maps.Params{lang: maps.Params{}}) + } + bcfg := res.BaseConfig + cfg := res.Cfg + + all := &Config{} + err := decodeConfigFromParams(fs, bcfg, cfg, all, nil) + if err != nil { + return nil, err + } + + langConfigMap := make(map[string]*Config) + var langConfigs []*Config + + languagesConfig := cfg.GetStringMap("languages") + var isMultiHost bool + + if err := all.CompileConfig(); err != nil { + return nil, err + } + + for k, v := range languagesConfig { + mergedConfig := config.New() + var differentRootKeys []string + switch x := v.(type) { + case maps.Params: + for kk, vv := range x { + if kk == "baseurl" { + // baseURL configure don the language level is a multihost setup. + isMultiHost = true + } + mergedConfig.Set(kk, vv) + if cfg.IsSet(kk) { + rootv := cfg.Get(kk) + // This overrides a root key and potentially needs a merge. + if !reflect.DeepEqual(rootv, vv) { + switch vvv := vv.(type) { + case maps.Params: + differentRootKeys = append(differentRootKeys, kk) + + // Use the language value as base. + mergedConfigEntry := xmaps.Clone(vvv) + // Merge in the root value. + maps.MergeParams(mergedConfigEntry, rootv.(maps.Params)) + + mergedConfig.Set(kk, mergedConfigEntry) + default: + // Apply new values to the root. + differentRootKeys = append(differentRootKeys, "") + } + } + } else { + // Apply new values to the root. + differentRootKeys = append(differentRootKeys, "") + } + } + differentRootKeys = helpers.UniqueStringsSorted(differentRootKeys) + + if len(differentRootKeys) == 0 { + langConfigMap[k] = all + continue + } + + // Create a copy of the complete config and replace the root keys with the language specific ones. + clone := all.cloneForLang() + if err := decodeConfigFromParams(fs, bcfg, mergedConfig, clone, differentRootKeys); err != nil { + return nil, fmt.Errorf("failed to decode config for language %q: %w", k, err) + } + if err := clone.CompileConfig(); err != nil { + return nil, err + } + langConfigMap[k] = clone + case maps.ParamsMergeStrategy: + default: + panic(fmt.Sprintf("unknown type in languages config: %T", v)) + + } + } + + var languages langs.Languages + defaultContentLanguage := all.DefaultContentLanguage + for k, v := range langConfigMap { + languageConf := v.Languages[k] + language, err := langs.NewLanguage(k, defaultContentLanguage, v.TimeZone, languageConf) + if err != nil { + return nil, err + } + languages = append(languages, language) + } + + // Sort the sites by language weight (if set) or lang. + sort.Slice(languages, func(i, j int) bool { + li := languages[i] + lj := languages[j] + if li.Weight != lj.Weight { + return li.Weight < lj.Weight + } + return li.Lang < lj.Lang + }) + + for _, l := range languages { + langConfigs = append(langConfigs, langConfigMap[l.Lang]) + } + + var languagesDefaultFirst langs.Languages + for _, l := range languages { + if l.Lang == defaultContentLanguage { + languagesDefaultFirst = append(languagesDefaultFirst, l) + } + } + for _, l := range languages { + if l.Lang != defaultContentLanguage { + languagesDefaultFirst = append(languagesDefaultFirst, l) + } + } + + bcfg.PublishDir = all.PublishDir + res.BaseConfig = bcfg + + cm := &Configs{ + Base: all, + LanguageConfigMap: langConfigMap, + LanguageConfigSlice: langConfigs, + LoadingInfo: res, + IsMultihost: isMultiHost, + Languages: languages, + LanguagesDefaultFirst: languagesDefaultFirst, + } + + return cm, nil +} + +func decodeConfigFromParams(fs afero.Fs, bcfg config.BaseConfig, p config.Provider, target *Config, keys []string) error { + + var decoderSetups []decodeWeight + + if len(keys) == 0 { + for _, v := range allDecoderSetups { + decoderSetups = append(decoderSetups, v) + } + } else { + for _, key := range keys { + if v, found := allDecoderSetups[key]; found { + decoderSetups = append(decoderSetups, v) + } else { + return fmt.Errorf("unknown config key %q", key) + } + } + } + + // Sort them to get the dependency order right. + sort.Slice(decoderSetups, func(i, j int) bool { + ki, kj := decoderSetups[i], decoderSetups[j] + if ki.weight == kj.weight { + return ki.key < kj.key + } + return ki.weight < kj.weight + }) + + for _, v := range decoderSetups { + p := decodeConfig{p: p, c: target, fs: fs, bcfg: bcfg} + if err := v.decode(v, p); err != nil { + return fmt.Errorf("failed to decode %q: %w", v.key, err) + } + } + + return nil +} + +func createDefaultOutputFormats(allFormats output.Formats) map[string][]string { + if len(allFormats) == 0 { + panic("no output formats") + } + rssOut, rssFound := allFormats.GetByName(output.RSSFormat.Name) + htmlOut, _ := allFormats.GetByName(output.HTMLFormat.Name) + + defaultListTypes := []string{htmlOut.Name} + if rssFound { + defaultListTypes = append(defaultListTypes, rssOut.Name) + } + + m := map[string][]string{ + page.KindPage: {htmlOut.Name}, + page.KindHome: defaultListTypes, + page.KindSection: defaultListTypes, + page.KindTerm: defaultListTypes, + page.KindTaxonomy: defaultListTypes, + } + + // May be disabled + if rssFound { + m["rss"] = []string{rssOut.Name} + } + + return m +} diff --git a/config/allconfig/alldecoders.go b/config/allconfig/alldecoders.go new file mode 100644 index 000000000..e8536b667 --- /dev/null +++ b/config/allconfig/alldecoders.go @@ -0,0 +1,325 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package allconfig + +import ( + "fmt" + "strings" + + "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/privacy" + "github.com/gohugoio/hugo/config/security" + "github.com/gohugoio/hugo/config/services" + "github.com/gohugoio/hugo/deploy" + "github.com/gohugoio/hugo/langs" + "github.com/gohugoio/hugo/markup/markup_config" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/minifiers" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/navigation" + "github.com/gohugoio/hugo/output" + "github.com/gohugoio/hugo/related" + "github.com/gohugoio/hugo/resources/images" + "github.com/gohugoio/hugo/resources/page" + "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/mitchellh/mapstructure" + "github.com/spf13/afero" + "github.com/spf13/cast" +) + +type decodeConfig struct { + p config.Provider + c *Config + fs afero.Fs + bcfg config.BaseConfig +} + +type decodeWeight struct { + key string + decode func(decodeWeight, decodeConfig) error + getCompiler func(c *Config) configCompiler + weight int +} + +var allDecoderSetups = map[string]decodeWeight{ + "": { + key: "", + weight: -100, // Always first. + decode: func(d decodeWeight, p decodeConfig) error { + return mapstructure.WeakDecode(p.p.Get(""), &p.c.RootConfig) + }, + }, + "imaging": { + key: "imaging", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Imaging, err = images.DecodeConfig(p.p.GetStringMap(d.key)) + return err + }, + }, + "caches": { + key: "caches", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Caches, err = filecache.DecodeConfig(p.fs, p.bcfg, p.p.GetStringMap(d.key)) + if p.c.IgnoreCache { + // Set MaxAge in all caches to 0. + for k, cache := range p.c.Caches { + cache.MaxAge = 0 + p.c.Caches[k] = cache + } + } + return err + }, + }, + "build": { + key: "build", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Build = config.DecodeBuildConfig(p.p) + return nil + }, + }, + "frontmatter": { + key: "frontmatter", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Frontmatter, err = pagemeta.DecodeFrontMatterConfig(p.p) + return err + }, + }, + "markup": { + key: "markup", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Markup, err = markup_config.Decode(p.p) + return err + }, + }, + "server": { + key: "server", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Server, err = config.DecodeServer(p.p) + return err + }, + getCompiler: func(c *Config) configCompiler { + return &c.Server + }, + }, + "minify": { + key: "minify", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Minify, err = minifiers.DecodeConfig(p.p.Get(d.key)) + return err + }, + }, + "mediaTypes": { + key: "mediaTypes", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.MediaTypes, err = media.DecodeTypes(p.p.GetStringMap(d.key)) + return err + }, + }, + "outputs": { + key: "outputs", + decode: func(d decodeWeight, p decodeConfig) error { + defaults := createDefaultOutputFormats(p.c.OutputFormats.Config) + m := p.p.GetStringMap("outputs") + p.c.Outputs = make(map[string][]string) + for k, v := range m { + s := types.ToStringSlicePreserveString(v) + for i, v := range s { + s[i] = strings.ToLower(v) + } + p.c.Outputs[k] = s + } + // Apply defaults. + for k, v := range defaults { + if _, found := p.c.Outputs[k]; !found { + p.c.Outputs[k] = v + } + } + return nil + }, + }, + "outputFormats": { + key: "outputFormats", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.OutputFormats, err = output.DecodeConfig(p.c.MediaTypes.Config, p.p.Get(d.key)) + return err + }, + }, + "params": { + key: "params", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Params = maps.CleanConfigStringMap(p.p.GetStringMap("params")) + if p.c.Params == nil { + p.c.Params = make(map[string]any) + } + + // Before Hugo 0.112.0 this was configured via site Params. + if mainSections, found := p.c.Params["mainsections"]; found { + p.c.MainSections = types.ToStringSlicePreserveString(mainSections) + } + + return nil + }, + }, + "module": { + key: "module", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Module, err = modules.DecodeConfig(p.p) + return err + }, + }, + "permalinks": { + key: "permalinks", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Permalinks = maps.CleanConfigStringMapString(p.p.GetStringMapString(d.key)) + return nil + }, + }, + "sitemap": { + key: "sitemap", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Sitemap, err = config.DecodeSitemap(config.SitemapConfig{Priority: -1, Filename: "sitemap.xml"}, p.p.GetStringMap(d.key)) + return err + }, + }, + "taxonomies": { + key: "taxonomies", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Taxonomies = maps.CleanConfigStringMapString(p.p.GetStringMapString(d.key)) + return nil + }, + }, + "related": { + key: "related", + weight: 100, // This needs to be decoded after taxonomies. + decode: func(d decodeWeight, p decodeConfig) error { + if p.p.IsSet(d.key) { + var err error + p.c.Related, err = related.DecodeConfig(p.p.GetParams(d.key)) + if err != nil { + return fmt.Errorf("failed to decode related config: %w", err) + } + } else { + p.c.Related = related.DefaultConfig + if _, found := p.c.Taxonomies["tag"]; found { + p.c.Related.Add(related.IndexConfig{Name: "tags", Weight: 80}) + } + } + return nil + }, + }, + "languages": { + key: "languages", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Languages, err = langs.DecodeConfig(p.p.GetStringMap(d.key)) + return err + }, + }, + "cascade": { + key: "cascade", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Cascade, err = page.DecodeCascadeConfig(p.p.Get(d.key)) + return err + }, + }, + "menus": { + key: "menus", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Menus, err = navigation.DecodeConfig(p.p.Get(d.key)) + return err + }, + }, + "privacy": { + key: "privacy", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Privacy, err = privacy.DecodeConfig(p.p) + return err + }, + }, + "security": { + key: "security", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Security, err = security.DecodeConfig(p.p) + return err + }, + }, + "services": { + key: "services", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Services, err = services.DecodeConfig(p.p) + return err + }, + }, + "deployment": { + key: "deployment", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Deployment, err = deploy.DecodeConfig(p.p) + return err + }, + }, + "author": { + key: "author", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Author = p.p.GetStringMap(d.key) + return nil + }, + }, + "social": { + key: "social", + decode: func(d decodeWeight, p decodeConfig) error { + p.c.Social = p.p.GetStringMapString(d.key) + return nil + }, + }, + "uglyurls": { + key: "uglyurls", + decode: func(d decodeWeight, p decodeConfig) error { + v := p.p.Get(d.key) + switch vv := v.(type) { + case bool: + p.c.UglyURLs = vv + case string: + p.c.UglyURLs = vv == "true" + default: + p.c.UglyURLs = cast.ToStringMapBool(v) + } + return nil + }, + }, + "internal": { + key: "internal", + decode: func(d decodeWeight, p decodeConfig) error { + return mapstructure.WeakDecode(p.p.GetStringMap(d.key), &p.c.Internal) + }, + }, +} diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go new file mode 100644 index 000000000..b28d54769 --- /dev/null +++ b/config/allconfig/configlanguage.go @@ -0,0 +1,216 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package allconfig + +import ( + "time" + + "github.com/gohugoio/hugo/common/urls" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/langs" +) + +type ConfigLanguage struct { + config *Config + baseConfig config.BaseConfig + + m *Configs + language *langs.Language +} + +func (c ConfigLanguage) Language() *langs.Language { + return c.language +} + +func (c ConfigLanguage) Languages() langs.Languages { + return c.m.Languages +} + +func (c ConfigLanguage) LanguagesDefaultFirst() langs.Languages { + return c.m.LanguagesDefaultFirst +} + +func (c ConfigLanguage) BaseURL() urls.BaseURL { + return c.config.C.BaseURL +} + +func (c ConfigLanguage) BaseURLLiveReload() urls.BaseURL { + return c.config.C.BaseURLLiveReload +} + +func (c ConfigLanguage) Environment() string { + return c.config.Environment +} + +func (c ConfigLanguage) IsMultihost() bool { + return c.m.IsMultihost +} + +func (c ConfigLanguage) IsMultiLingual() bool { + return len(c.m.Languages) > 1 +} + +func (c ConfigLanguage) TemplateMetrics() bool { + return c.config.TemplateMetrics +} + +func (c ConfigLanguage) TemplateMetricsHints() bool { + return c.config.TemplateMetricsHints +} + +func (c ConfigLanguage) IsLangDisabled(lang string) bool { + return c.config.C.DisabledLanguages[lang] +} + +func (c ConfigLanguage) IgnoredErrors() map[string]bool { + return c.config.C.IgnoredErrors +} + +func (c ConfigLanguage) NoBuildLock() bool { + return c.config.NoBuildLock +} + +func (c ConfigLanguage) NewContentEditor() string { + return c.config.NewContentEditor +} + +func (c ConfigLanguage) Timeout() time.Duration { + return c.config.C.Timeout +} + +func (c ConfigLanguage) BaseConfig() config.BaseConfig { + return c.baseConfig +} + +func (c ConfigLanguage) Dirs() config.CommonDirs { + return c.config.CommonDirs +} + +func (c ConfigLanguage) DirsBase() config.CommonDirs { + return c.m.Base.CommonDirs +} + +func (c ConfigLanguage) Quiet() bool { + return c.m.Base.Internal.Quiet +} + +// GetConfigSection is mostly used in tests. The switch statement isn't complete, but what's in use. +func (c ConfigLanguage) GetConfigSection(s string) any { + switch s { + case "security": + return c.config.Security + case "build": + return c.config.Build + case "frontmatter": + return c.config.Frontmatter + case "caches": + return c.config.Caches + case "markup": + return c.config.Markup + case "mediaTypes": + return c.config.MediaTypes.Config + case "outputFormats": + return c.config.OutputFormats.Config + case "permalinks": + return c.config.Permalinks + case "minify": + return c.config.Minify + case "activeModules": + return c.m.Modules + case "deployment": + return c.config.Deployment + default: + panic("not implemented: " + s) + } +} + +func (c ConfigLanguage) GetConfig() any { + return c.config +} + +func (c ConfigLanguage) CanonifyURLs() bool { + return c.config.CanonifyURLs +} + +func (c ConfigLanguage) IsUglyURLs(section string) bool { + return c.config.C.IsUglyURLSection(section) +} + +func (c ConfigLanguage) IgnoreFile(s string) bool { + return c.config.C.IgnoreFile(s) +} + +func (c ConfigLanguage) DisablePathToLower() bool { + return c.config.DisablePathToLower +} + +func (c ConfigLanguage) RemovePathAccents() bool { + return c.config.RemovePathAccents +} + +func (c ConfigLanguage) DefaultContentLanguage() string { + return c.config.DefaultContentLanguage +} + +func (c ConfigLanguage) DefaultContentLanguageInSubdir() bool { + return c.config.DefaultContentLanguageInSubdir +} + +func (c ConfigLanguage) SummaryLength() int { + return c.config.SummaryLength +} + +func (c ConfigLanguage) BuildExpired() bool { + return c.config.BuildExpired +} + +func (c ConfigLanguage) BuildFuture() bool { + return c.config.BuildFuture +} + +func (c ConfigLanguage) BuildDrafts() bool { + return c.config.BuildDrafts +} + +func (c ConfigLanguage) Running() bool { + return c.config.Internal.Running +} + +func (c ConfigLanguage) PrintUnusedTemplates() bool { + return c.config.PrintUnusedTemplates +} + +func (c ConfigLanguage) EnableMissingTranslationPlaceholders() bool { + return c.config.EnableMissingTranslationPlaceholders +} + +func (c ConfigLanguage) LogI18nWarnings() bool { + return c.config.LogI18nWarnings +} + +func (c ConfigLanguage) CreateTitle(s string) string { + return c.config.C.CreateTitle(s) +} + +func (c ConfigLanguage) Paginate() int { + return c.config.Paginate +} + +func (c ConfigLanguage) PaginatePath() string { + return c.config.PaginatePath +} + +func (c ConfigLanguage) StaticDirs() []string { + return c.config.staticDirs() +} diff --git a/config/allconfig/integration_test.go b/config/allconfig/integration_test.go new file mode 100644 index 000000000..e96dbd296 --- /dev/null +++ b/config/allconfig/integration_test.go @@ -0,0 +1,71 @@ +package allconfig_test + +import ( + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config/allconfig" + "github.com/gohugoio/hugo/hugolib" +) + +func TestDirsMount(t *testing.T) { + + files := ` +-- hugo.toml -- +baseURL = "https://example.com" +disableKinds = ["taxonomy", "term"] +[languages] +[languages.en] +weight = 1 +[languages.sv] +weight = 2 +[[module.mounts]] +source = 'content/en' +target = 'content' +lang = 'en' +[[module.mounts]] +source = 'content/sv' +target = 'content' +lang = 'sv' +-- content/en/p1.md -- +--- +title: "p1" +--- +-- content/sv/p1.md -- +--- +title: "p1" +--- +-- layouts/_default/single.html -- +Title: {{ .Title }} + ` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{T: t, TxtarString: files}, + ).Build() + + //b.AssertFileContent("public/p1/index.html", "Title: p1") + + sites := b.H.Sites + b.Assert(len(sites), qt.Equals, 2) + + configs := b.H.Configs + mods := configs.Modules + b.Assert(len(mods), qt.Equals, 1) + mod := mods[0] + b.Assert(mod.Mounts(), qt.HasLen, 8) + + enConcp := sites[0].Conf + enConf := enConcp.GetConfig().(*allconfig.Config) + + b.Assert(enConcp.BaseURL().String(), qt.Equals, "https://example.com") + modConf := enConf.Module + b.Assert(modConf.Mounts, qt.HasLen, 2) + b.Assert(modConf.Mounts[0].Source, qt.Equals, filepath.FromSlash("content/en")) + b.Assert(modConf.Mounts[0].Target, qt.Equals, "content") + b.Assert(modConf.Mounts[0].Lang, qt.Equals, "en") + b.Assert(modConf.Mounts[1].Source, qt.Equals, filepath.FromSlash("content/sv")) + b.Assert(modConf.Mounts[1].Target, qt.Equals, "content") + b.Assert(modConf.Mounts[1].Lang, qt.Equals, "sv") + +} diff --git a/config/allconfig/load.go b/config/allconfig/load.go new file mode 100644 index 000000000..9f27e867e --- /dev/null +++ b/config/allconfig/load.go @@ -0,0 +1,559 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package allconfig contains the full configuration for Hugo. +package allconfig + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/common/hugo" + "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" + hglob "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/spf13/afero" +) + +var ErrNoConfigFile = errors.New("Unable to locate config file or config directory. Perhaps you need to create a new site.\n Run `hugo help new` for details.\n") + +func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) { + if len(d.Environ) == 0 && !hugo.IsRunningAsTest() { + d.Environ = os.Environ() + } + + l := &configLoader{ConfigSourceDescriptor: d, cfg: config.New()} + // Make sure we always do this, even in error situations, + // as we have commands (e.g. "hugo mod init") that will + // use a partial configuration to do its job. + defer l.deleteMergeStrategies() + res, _, err := l.loadConfigMain(d) + if err != nil { + return nil, fmt.Errorf("failed to load config: %w", err) + } + + configs, err := FromLoadConfigResult(d.Fs, res) + if err != nil { + return nil, fmt.Errorf("failed to create config from result: %w", err) + } + + moduleConfig, modulesClient, err := l.loadModules(configs) + if err != nil { + return nil, fmt.Errorf("failed to load modules: %w", err) + } + if len(l.ModulesConfigFiles) > 0 { + // Config merged in from modules. + // Re-read the config. + configs, err = FromLoadConfigResult(d.Fs, res) + if err != nil { + return nil, fmt.Errorf("failed to create config: %w", err) + } + } + + configs.Modules = moduleConfig.ActiveModules + configs.ModulesClient = modulesClient + + if err := configs.Init(); err != nil { + return nil, fmt.Errorf("failed to init config: %w", err) + } + + return configs, nil + +} + +// ConfigSourceDescriptor describes where to find the config (e.g. config.toml etc.). +type ConfigSourceDescriptor struct { + Fs afero.Fs + Logger loggers.Logger + + // Config received from the command line. + // These will override any config file settings. + Flags config.Provider + + // Path to the config file to use, e.g. /my/project/config.toml + Filename string + + // The (optional) directory for additional configuration files. + ConfigDir string + + // production, development + Environment string + + // Defaults to os.Environ if not set. + Environ []string +} + +func (d ConfigSourceDescriptor) configFilenames() []string { + if d.Filename == "" { + return nil + } + return strings.Split(d.Filename, ",") +} + +type configLoader struct { + cfg config.Provider + BaseConfig config.BaseConfig + ConfigSourceDescriptor + + // collected + ModulesConfig modules.ModulesConfig + ModulesConfigFiles []string +} + +// Handle some legacy values. +func (l configLoader) applyConfigAliases() error { + aliases := []types.KeyValueStr{{Key: "taxonomies", Value: "indexes"}} + + for _, alias := range aliases { + if l.cfg.IsSet(alias.Key) { + vv := l.cfg.Get(alias.Key) + l.cfg.Set(alias.Value, vv) + } + } + + return nil +} + +func (l configLoader) applyDefaultConfig() error { + defaultSettings := maps.Params{ + "baseURL": "", + "cleanDestinationDir": false, + "watch": false, + "contentDir": "content", + "resourceDir": "resources", + "publishDir": "public", + "publishDirOrig": "public", + "themesDir": "themes", + "assetDir": "assets", + "layoutDir": "layouts", + "i18nDir": "i18n", + "dataDir": "data", + "archetypeDir": "archetypes", + "configDir": "config", + "staticDir": "static", + "buildDrafts": false, + "buildFuture": false, + "buildExpired": false, + "params": maps.Params{}, + "environment": hugo.EnvironmentProduction, + "uglyURLs": false, + "verbose": false, + "ignoreCache": false, + "canonifyURLs": false, + "relativeURLs": false, + "removePathAccents": false, + "titleCaseStyle": "AP", + "taxonomies": maps.Params{"tag": "tags", "category": "categories"}, + "permalinks": maps.Params{}, + "sitemap": maps.Params{"priority": -1, "filename": "sitemap.xml"}, + "menus": maps.Params{}, + "disableLiveReload": false, + "pluralizeListTitles": true, + "forceSyncStatic": false, + "footnoteAnchorPrefix": "", + "footnoteReturnLinkContents": "", + "newContentEditor": "", + "paginate": 10, + "paginatePath": "page", + "summaryLength": 70, + "rssLimit": -1, + "sectionPagesMenu": "", + "disablePathToLower": false, + "hasCJKLanguage": false, + "enableEmoji": false, + "defaultContentLanguage": "en", + "defaultContentLanguageInSubdir": false, + "enableMissingTranslationPlaceholders": false, + "enableGitInfo": false, + "ignoreFiles": make([]string, 0), + "disableAliases": false, + "debug": false, + "disableFastRender": false, + "timeout": "30s", + "timeZone": "", + "enableInlineShortcodes": false, + } + + l.cfg.SetDefaults(defaultSettings) + + return nil +} + +func (l configLoader) normalizeCfg(cfg config.Provider) error { + minify := cfg.Get("minify") + if b, ok := minify.(bool); ok && b { + cfg.Set("minify", maps.Params{"minifyOutput": true}) + } + + // Simplify later merge. + languages := cfg.GetStringMap("languages") + for _, v := range languages { + switch m := v.(type) { + case maps.Params: + // params have merge strategy deep by default. + // The languages config key has strategy none by default. + // This means that if these two sections does not exist on the left side, + // they will not get merged in, so just create some empty maps. + if _, ok := m["params"]; !ok { + m["params"] = maps.Params{} + } + } + + } + + return nil +} + +func (l configLoader) cleanExternalConfig(cfg config.Provider) error { + if cfg.IsSet("internal") { + cfg.Set("internal", nil) + } + return nil +} + +func (l configLoader) applyFlagsOverrides(cfg config.Provider) error { + for _, k := range cfg.Keys() { + l.cfg.Set(k, cfg.Get(k)) + } + return nil +} + +func (l configLoader) applyOsEnvOverrides(environ []string) error { + if len(environ) == 0 { + return nil + } + + const delim = "__env__delim" + + // Extract all that start with the HUGO prefix. + // The delimiter is the following rune, usually "_". + const hugoEnvPrefix = "HUGO" + var hugoEnv []types.KeyValueStr + for _, v := range environ { + key, val := config.SplitEnvVar(v) + if strings.HasPrefix(key, hugoEnvPrefix) { + delimiterAndKey := strings.TrimPrefix(key, hugoEnvPrefix) + if len(delimiterAndKey) < 2 { + continue + } + // Allow delimiters to be case sensitive. + // It turns out there isn't that many allowed special + // chars in environment variables when used in Bash and similar, + // so variables on the form HUGOxPARAMSxFOO=bar is one option. + key := strings.ReplaceAll(delimiterAndKey[1:], delimiterAndKey[:1], delim) + key = strings.ToLower(key) + hugoEnv = append(hugoEnv, types.KeyValueStr{ + Key: key, + Value: val, + }) + + } + } + + for _, env := range hugoEnv { + existing, nestedKey, owner, err := maps.GetNestedParamFn(env.Key, delim, l.cfg.Get) + if err != nil { + return err + } + + if existing != nil { + val, err := metadecoders.Default.UnmarshalStringTo(env.Value, existing) + if err != nil { + continue + } + + if owner != nil { + owner[nestedKey] = val + } else { + l.cfg.Set(env.Key, val) + } + } else if nestedKey != "" { + owner[nestedKey] = env.Value + } else { + // The container does not exist yet. + l.cfg.Set(strings.ReplaceAll(env.Key, delim, "."), env.Value) + } + } + + return nil +} + +func (l *configLoader) loadConfigMain(d ConfigSourceDescriptor) (config.LoadConfigResult, modules.ModulesConfig, error) { + var res config.LoadConfigResult + + if d.Flags != nil { + if err := l.normalizeCfg(d.Flags); err != nil { + return res, l.ModulesConfig, err + } + } + + if d.Fs == nil { + return res, l.ModulesConfig, errors.New("no filesystem provided") + } + + if d.Flags != nil { + if err := l.applyFlagsOverrides(d.Flags); err != nil { + return res, l.ModulesConfig, err + } + workingDir := filepath.Clean(l.cfg.GetString("workingDir")) + + l.BaseConfig = config.BaseConfig{ + WorkingDir: workingDir, + ThemesDir: paths.AbsPathify(workingDir, l.cfg.GetString("themesDir")), + } + + } + + names := d.configFilenames() + + if names != nil { + for _, name := range names { + var filename string + filename, err := l.loadConfig(name) + if err == nil { + res.ConfigFiles = append(res.ConfigFiles, filename) + } else if err != ErrNoConfigFile { + return res, l.ModulesConfig, l.wrapFileError(err, filename) + } + } + } else { + for _, name := range config.DefaultConfigNames { + var filename string + filename, err := l.loadConfig(name) + if err == nil { + res.ConfigFiles = append(res.ConfigFiles, filename) + break + } else if err != ErrNoConfigFile { + return res, l.ModulesConfig, l.wrapFileError(err, filename) + } + } + } + + if d.ConfigDir != "" { + absConfigDir := paths.AbsPathify(l.BaseConfig.WorkingDir, d.ConfigDir) + dcfg, dirnames, err := config.LoadConfigFromDir(l.Fs, absConfigDir, l.Environment) + if err == nil { + if len(dirnames) > 0 { + if err := l.normalizeCfg(dcfg); err != nil { + return res, l.ModulesConfig, err + } + if err := l.cleanExternalConfig(dcfg); err != nil { + return res, l.ModulesConfig, err + } + l.cfg.Set("", dcfg.Get("")) + res.ConfigFiles = append(res.ConfigFiles, dirnames...) + } + } else if err != ErrNoConfigFile { + if len(dirnames) > 0 { + return res, l.ModulesConfig, l.wrapFileError(err, dirnames[0]) + } + return res, l.ModulesConfig, err + } + } + + res.Cfg = l.cfg + + if err := l.applyDefaultConfig(); err != nil { + return res, l.ModulesConfig, err + } + + // Some settings are used before we're done collecting all settings, + // so apply OS environment both before and after. + if err := l.applyOsEnvOverrides(d.Environ); err != nil { + return res, l.ModulesConfig, err + } + + workingDir := filepath.Clean(l.cfg.GetString("workingDir")) + + l.BaseConfig = config.BaseConfig{ + WorkingDir: workingDir, + CacheDir: l.cfg.GetString("cacheDir"), + ThemesDir: paths.AbsPathify(workingDir, l.cfg.GetString("themesDir")), + } + + var err error + l.BaseConfig.CacheDir, err = helpers.GetCacheDir(l.Fs, l.BaseConfig.CacheDir) + if err != nil { + return res, l.ModulesConfig, err + } + + res.BaseConfig = l.BaseConfig + + l.cfg.SetDefaultMergeStrategy() + + res.ConfigFiles = append(res.ConfigFiles, l.ModulesConfigFiles...) + + if d.Flags != nil { + if err := l.applyFlagsOverrides(d.Flags); err != nil { + return res, l.ModulesConfig, err + } + } + + if err := l.applyOsEnvOverrides(d.Environ); err != nil { + return res, l.ModulesConfig, err + } + + if err = l.applyConfigAliases(); err != nil { + return res, l.ModulesConfig, err + } + + return res, l.ModulesConfig, err +} + +func (l *configLoader) loadModules(configs *Configs) (modules.ModulesConfig, *modules.Client, error) { + bcfg := configs.LoadingInfo.BaseConfig + conf := configs.Base + workingDir := bcfg.WorkingDir + themesDir := bcfg.ThemesDir + + cfg := configs.LoadingInfo.Cfg + + var ignoreVendor glob.Glob + if s := conf.IgnoreVendorPaths; s != "" { + ignoreVendor, _ = hglob.GetGlob(hglob.NormalizePath(s)) + } + + ex := hexec.New(conf.Security) + + hook := func(m *modules.ModulesConfig) error { + for _, tc := range m.ActiveModules { + if len(tc.ConfigFilenames()) > 0 { + if tc.Watch() { + l.ModulesConfigFiles = append(l.ModulesConfigFiles, tc.ConfigFilenames()...) + } + + // Merge in the theme config using the configured + // merge strategy. + cfg.Merge("", tc.Cfg().Get("")) + + } + } + + return nil + } + + modulesClient := modules.NewClient(modules.ClientConfig{ + Fs: l.Fs, + Logger: l.Logger, + Exec: ex, + HookBeforeFinalize: hook, + WorkingDir: workingDir, + ThemesDir: themesDir, + Environment: l.Environment, + CacheDir: conf.Caches.CacheDirModules(), + ModuleConfig: conf.Module, + IgnoreVendor: ignoreVendor, + }) + + moduleConfig, err := modulesClient.Collect() + + // We want to watch these for changes and trigger rebuild on version + // changes etc. + if moduleConfig.GoModulesFilename != "" { + l.ModulesConfigFiles = append(l.ModulesConfigFiles, moduleConfig.GoModulesFilename) + } + + if moduleConfig.GoWorkspaceFilename != "" { + l.ModulesConfigFiles = append(l.ModulesConfigFiles, moduleConfig.GoWorkspaceFilename) + } + + return moduleConfig, modulesClient, err +} + +func (l configLoader) loadConfig(configName string) (string, error) { + baseDir := l.BaseConfig.WorkingDir + var baseFilename string + if filepath.IsAbs(configName) { + baseFilename = configName + } else { + baseFilename = filepath.Join(baseDir, configName) + } + + var filename string + if paths.ExtNoDelimiter(configName) != "" { + exists, _ := helpers.Exists(baseFilename, l.Fs) + if exists { + filename = baseFilename + } + } else { + for _, ext := range config.ValidConfigFileExtensions { + filenameToCheck := baseFilename + "." + ext + exists, _ := helpers.Exists(filenameToCheck, l.Fs) + if exists { + filename = filenameToCheck + break + } + } + } + + if filename == "" { + return "", ErrNoConfigFile + } + + m, err := config.FromFileToMap(l.Fs, filename) + if err != nil { + return filename, err + } + + // Set overwrites keys of the same name, recursively. + l.cfg.Set("", m) + + if err := l.normalizeCfg(l.cfg); err != nil { + return filename, err + } + + if err := l.cleanExternalConfig(l.cfg); err != nil { + return filename, err + } + + return filename, nil +} + +func (l configLoader) deleteMergeStrategies() { + l.cfg.WalkParams(func(params ...maps.KeyParams) bool { + params[len(params)-1].Params.DeleteMergeStrategy() + return false + }) +} + +func (l configLoader) loadModulesConfig() (modules.Config, error) { + modConfig, err := modules.DecodeConfig(l.cfg) + if err != nil { + return modules.Config{}, err + } + + return modConfig, nil +} + +func (l configLoader) wrapFileError(err error, filename string) error { + fe := herrors.UnwrapFileError(err) + if fe != nil { + pos := fe.Position() + pos.Filename = filename + fe.UpdatePosition(pos) + return err + } + return herrors.NewFileErrorFromFile(err, filename, l.Fs, nil) +} diff --git a/config/allconfig/load_test.go b/config/allconfig/load_test.go new file mode 100644 index 000000000..153a59c44 --- /dev/null +++ b/config/allconfig/load_test.go @@ -0,0 +1,67 @@ +package allconfig + +import ( + "os" + "path/filepath" + "testing" + + "github.com/spf13/afero" +) + +func BenchmarkLoad(b *testing.B) { + tempDir := b.TempDir() + configFilename := filepath.Join(tempDir, "hugo.toml") + config := ` +baseURL = "https://example.com" +defaultContentLanguage = 'en' + +[module] +[[module.mounts]] +source = 'content/en' +target = 'content/en' +lang = 'en' +[[module.mounts]] +source = 'content/nn' +target = 'content/nn' +lang = 'nn' +[[module.mounts]] +source = 'content/no' +target = 'content/no' +lang = 'no' +[[module.mounts]] +source = 'content/sv' +target = 'content/sv' +lang = 'sv' +[[module.mounts]] +source = 'layouts' +target = 'layouts' + +[languages] +[languages.en] +title = "English" +weight = 1 +[languages.nn] +title = "Nynorsk" +weight = 2 +[languages.no] +title = "Norsk" +weight = 3 +[languages.sv] +title = "Svenska" +weight = 4 +` + if err := os.WriteFile(configFilename, []byte(config), 0666); err != nil { + b.Fatal(err) + } + d := ConfigSourceDescriptor{ + Fs: afero.NewOsFs(), + Filename: configFilename, + } + + for i := 0; i < b.N; i++ { + _, err := LoadConfig(d) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/config/commonConfig.go b/config/commonConfig.go index 31705841e..8cac2e1e5 100644 --- a/config/commonConfig.go +++ b/config/commonConfig.go @@ -17,7 +17,6 @@ import ( "fmt" "sort" "strings" - "sync" "github.com/gohugoio/hugo/common/types" @@ -25,16 +24,66 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" - jww "github.com/spf13/jwalterweatherman" ) -var DefaultBuild = Build{ +type BaseConfig struct { + WorkingDir string + CacheDir string + ThemesDir string + PublishDir string +} + +type CommonDirs struct { + // The directory where Hugo will look for themes. + ThemesDir string + + // Where to put the generated files. + PublishDir string + + // The directory to put the generated resources files. This directory should in most situations be considered temporary + // and not be committed to version control. But there may be cached content in here that you want to keep, + // e.g. resources/_gen/images for performance reasons or CSS built from SASS when your CI server doesn't have the full setup. + ResourceDir string + + // The project root directory. + WorkingDir string + + // The root directory for all cache files. + CacheDir string + + // The content source directory. + // Deprecated: Use module mounts. + ContentDir string + // Deprecated: Use module mounts. + // The data source directory. + DataDir string + // Deprecated: Use module mounts. + // The layout source directory. + LayoutDir string + // Deprecated: Use module mounts. + // The i18n source directory. + I18nDir string + // Deprecated: Use module mounts. + // The archetypes source directory. + ArcheTypeDir string + // Deprecated: Use module mounts. + // The assets source directory. + AssetDir string +} + +type LoadConfigResult struct { + Cfg Provider + ConfigFiles []string + BaseConfig BaseConfig +} + +var DefaultBuild = BuildConfig{ UseResourceCacheWhen: "fallback", WriteStats: false, } -// Build holds some build related configuration. -type Build struct { +// BuildConfig holds some build related configuration. +type BuildConfig struct { UseResourceCacheWhen string // never, fallback, always. Default is fallback // When enabled, will collect and write a hugo_stats.json with some build @@ -46,7 +95,7 @@ type Build struct { NoJSConfigInAssets bool } -func (b Build) UseResourceCache(err error) bool { +func (b BuildConfig) UseResourceCache(err error) bool { if b.UseResourceCacheWhen == "never" { return false } @@ -58,7 +107,7 @@ func (b Build) UseResourceCache(err error) bool { return true } -func DecodeBuild(cfg Provider) Build { +func DecodeBuildConfig(cfg Provider) BuildConfig { m := cfg.GetStringMap("build") b := DefaultBuild if m == nil { @@ -79,28 +128,19 @@ func DecodeBuild(cfg Provider) Build { return b } -// Sitemap configures the sitemap to be generated. -type Sitemap struct { +// SitemapConfig configures the sitemap to be generated. +type SitemapConfig struct { + // The page change frequency. ChangeFreq string - Priority float64 - Filename string + // The priority of the page. + Priority float64 + // The sitemap filename. + Filename string } -func DecodeSitemap(prototype Sitemap, input map[string]any) Sitemap { - for key, value := range input { - switch key { - case "changefreq": - prototype.ChangeFreq = cast.ToString(value) - case "priority": - prototype.Priority = cast.ToFloat64(value) - case "filename": - prototype.Filename = cast.ToString(value) - default: - jww.WARN.Printf("Unknown Sitemap field: %s\n", key) - } - } - - return prototype +func DecodeSitemap(prototype SitemapConfig, input map[string]any) (SitemapConfig, error) { + err := mapstructure.WeakDecode(input, &prototype) + return prototype, err } // Config for the dev server. @@ -108,25 +148,24 @@ type Server struct { Headers []Headers Redirects []Redirect - compiledInit sync.Once compiledHeaders []glob.Glob compiledRedirects []glob.Glob } -func (s *Server) init() { - s.compiledInit.Do(func() { - for _, h := range s.Headers { - s.compiledHeaders = append(s.compiledHeaders, glob.MustCompile(h.For)) - } - for _, r := range s.Redirects { - s.compiledRedirects = append(s.compiledRedirects, glob.MustCompile(r.From)) - } - }) +func (s *Server) CompileConfig() error { + if s.compiledHeaders != nil { + return nil + } + for _, h := range s.Headers { + s.compiledHeaders = append(s.compiledHeaders, glob.MustCompile(h.For)) + } + for _, r := range s.Redirects { + s.compiledRedirects = append(s.compiledRedirects, glob.MustCompile(r.From)) + } + return nil } func (s *Server) MatchHeaders(pattern string) []types.KeyValueStr { - s.init() - if s.compiledHeaders == nil { return nil } @@ -150,8 +189,6 @@ func (s *Server) MatchHeaders(pattern string) []types.KeyValueStr { } func (s *Server) MatchRedirect(pattern string) Redirect { - s.init() - if s.compiledRedirects == nil { return Redirect{} } @@ -195,14 +232,10 @@ func (r Redirect) IsZero() bool { return r.From == "" } -func DecodeServer(cfg Provider) (*Server, error) { - m := cfg.GetStringMap("server") +func DecodeServer(cfg Provider) (Server, error) { s := &Server{} - if m == nil { - return s, nil - } - _ = mapstructure.WeakDecode(m, s) + _ = mapstructure.WeakDecode(cfg.GetStringMap("server"), s) for i, redir := range s.Redirects { // Get it in line with the Hugo server for OK responses. @@ -213,7 +246,7 @@ func DecodeServer(cfg Provider) (*Server, error) { // There are some tricky infinite loop situations when dealing // when the target does not have a trailing slash. // This can certainly be handled better, but not time for that now. - return nil, fmt.Errorf("unsupported redirect to value %q in server config; currently this must be either a remote destination or a local folder, e.g. \"/blog/\" or \"/blog/index.html\"", redir.To) + return Server{}, fmt.Errorf("unsupported redirect to value %q in server config; currently this must be either a remote destination or a local folder, e.g. \"/blog/\" or \"/blog/index.html\"", redir.To) } } s.Redirects[i] = redir @@ -231,5 +264,5 @@ func DecodeServer(cfg Provider) (*Server, error) { } - return s, nil + return *s, nil } diff --git a/config/commonConfig_test.go b/config/commonConfig_test.go index 4ff2e8ed5..f05664448 100644 --- a/config/commonConfig_test.go +++ b/config/commonConfig_test.go @@ -31,7 +31,7 @@ func TestBuild(t *testing.T) { "useResourceCacheWhen": "always", }) - b := DecodeBuild(v) + b := DecodeBuildConfig(v) c.Assert(b.UseResourceCacheWhen, qt.Equals, "always") @@ -39,7 +39,7 @@ func TestBuild(t *testing.T) { "useResourceCacheWhen": "foo", }) - b = DecodeBuild(v) + b = DecodeBuildConfig(v) c.Assert(b.UseResourceCacheWhen, qt.Equals, "fallback") @@ -91,6 +91,7 @@ status = 301 s, err := DecodeServer(cfg) c.Assert(err, qt.IsNil) + c.Assert(s.CompileConfig(), qt.IsNil) c.Assert(s.MatchHeaders("/foo.jpg"), qt.DeepEquals, []types.KeyValueStr{ {Key: "X-Content-Type-Options", Value: "nosniff"}, diff --git a/config/compositeConfig.go b/config/compositeConfig.go deleted file mode 100644 index 395b2d585..000000000 --- a/config/compositeConfig.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2021 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/gohugoio/hugo/common/maps" -) - -// NewCompositeConfig creates a new composite Provider with a read-only base -// and a writeable layer. -func NewCompositeConfig(base, layer Provider) Provider { - return &compositeConfig{ - base: base, - layer: layer, - } -} - -// compositeConfig contains a read only config base with -// a possibly writeable config layer on top. -type compositeConfig struct { - base Provider - layer Provider -} - -func (c *compositeConfig) GetBool(key string) bool { - if c.layer.IsSet(key) { - return c.layer.GetBool(key) - } - return c.base.GetBool(key) -} - -func (c *compositeConfig) GetInt(key string) int { - if c.layer.IsSet(key) { - return c.layer.GetInt(key) - } - return c.base.GetInt(key) -} - -func (c *compositeConfig) Merge(key string, value any) { - c.layer.Merge(key, value) -} - -func (c *compositeConfig) GetParams(key string) maps.Params { - if c.layer.IsSet(key) { - return c.layer.GetParams(key) - } - return c.base.GetParams(key) -} - -func (c *compositeConfig) GetStringMap(key string) map[string]any { - if c.layer.IsSet(key) { - return c.layer.GetStringMap(key) - } - return c.base.GetStringMap(key) -} - -func (c *compositeConfig) GetStringMapString(key string) map[string]string { - if c.layer.IsSet(key) { - return c.layer.GetStringMapString(key) - } - return c.base.GetStringMapString(key) -} - -func (c *compositeConfig) GetStringSlice(key string) []string { - if c.layer.IsSet(key) { - return c.layer.GetStringSlice(key) - } - return c.base.GetStringSlice(key) -} - -func (c *compositeConfig) Get(key string) any { - if c.layer.IsSet(key) { - return c.layer.Get(key) - } - return c.base.Get(key) -} - -func (c *compositeConfig) IsSet(key string) bool { - if c.layer.IsSet(key) { - return true - } - return c.base.IsSet(key) -} - -func (c *compositeConfig) GetString(key string) string { - if c.layer.IsSet(key) { - return c.layer.GetString(key) - } - return c.base.GetString(key) -} - -func (c *compositeConfig) Set(key string, value any) { - c.layer.Set(key, value) -} - -func (c *compositeConfig) SetDefaults(params maps.Params) { - c.layer.SetDefaults(params) -} - -func (c *compositeConfig) WalkParams(walkFn func(params ...KeyParams) bool) { - panic("not supported") -} - -func (c *compositeConfig) SetDefaultMergeStrategy() { - panic("not supported") -} diff --git a/config/configLoader.go b/config/configLoader.go index 95594fc62..6e520b9cc 100644 --- a/config/configLoader.go +++ b/config/configLoader.go @@ -57,6 +57,14 @@ func IsValidConfigFilename(filename string) bool { return validConfigFileExtensionsMap[ext] } +func FromTOMLConfigString(config string) Provider { + cfg, err := FromConfigString(config, "toml") + if err != nil { + panic(err) + } + return cfg +} + // FromConfigString creates a config from the given YAML, JSON or TOML config. This is useful in tests. func FromConfigString(config, configType string) (Provider, error) { m, err := readConfig(metadecoders.FormatFromString(configType), []byte(config)) diff --git a/config/configProvider.go b/config/configProvider.go index 01a2e8c54..ac00c7476 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -14,10 +14,58 @@ package config import ( + "time" + "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/common/urls" + "github.com/gohugoio/hugo/langs" ) +// AllProvider is a sub set of all config settings. +type AllProvider interface { + Language() *langs.Language + Languages() langs.Languages + LanguagesDefaultFirst() langs.Languages + BaseURL() urls.BaseURL + BaseURLLiveReload() urls.BaseURL + Environment() string + IsMultihost() bool + IsMultiLingual() bool + NoBuildLock() bool + BaseConfig() BaseConfig + Dirs() CommonDirs + Quiet() bool + DirsBase() CommonDirs + GetConfigSection(string) any + GetConfig() any + CanonifyURLs() bool + DisablePathToLower() bool + RemovePathAccents() bool + IsUglyURLs(section string) bool + DefaultContentLanguage() string + DefaultContentLanguageInSubdir() bool + IsLangDisabled(string) bool + SummaryLength() int + Paginate() int + PaginatePath() string + BuildExpired() bool + BuildFuture() bool + BuildDrafts() bool + Running() bool + PrintUnusedTemplates() bool + EnableMissingTranslationPlaceholders() bool + TemplateMetrics() bool + TemplateMetricsHints() bool + LogI18nWarnings() bool + CreateTitle(s string) string + IgnoreFile(s string) bool + NewContentEditor() string + Timeout() time.Duration + StaticDirs() []string + IgnoredErrors() map[string]bool +} + // Provider provides the configuration settings for Hugo. type Provider interface { GetString(key string) string @@ -29,10 +77,11 @@ type Provider interface { GetStringSlice(key string) []string Get(key string) any Set(key string, value any) + Keys() []string Merge(key string, value any) SetDefaults(params maps.Params) SetDefaultMergeStrategy() - WalkParams(walkFn func(params ...KeyParams) bool) + WalkParams(walkFn func(params ...maps.KeyParams) bool) IsSet(key string) bool } @@ -44,22 +93,6 @@ func GetStringSlicePreserveString(cfg Provider, key string) []string { return types.ToStringSlicePreserveString(sd) } -// SetBaseTestDefaults provides some common config defaults used in tests. -func SetBaseTestDefaults(cfg Provider) Provider { - setIfNotSet(cfg, "baseURL", "https://example.org") - setIfNotSet(cfg, "resourceDir", "resources") - setIfNotSet(cfg, "contentDir", "content") - setIfNotSet(cfg, "dataDir", "data") - setIfNotSet(cfg, "i18nDir", "i18n") - setIfNotSet(cfg, "layoutDir", "layouts") - setIfNotSet(cfg, "assetDir", "assets") - setIfNotSet(cfg, "archetypeDir", "archetypes") - setIfNotSet(cfg, "publishDir", "public") - setIfNotSet(cfg, "workingDir", "") - setIfNotSet(cfg, "defaultContentLanguage", "en") - return cfg -} - func setIfNotSet(cfg Provider, key string, value any) { if !cfg.IsSet(key) { cfg.Set(key, value) diff --git a/config/defaultConfigProvider.go b/config/defaultConfigProvider.go index 822f421fa..e8a08e281 100644 --- a/config/defaultConfigProvider.go +++ b/config/defaultConfigProvider.go @@ -19,6 +19,8 @@ import ( "strings" "sync" + xmaps "golang.org/x/exp/maps" + "github.com/spf13/cast" "github.com/gohugoio/hugo/common/maps" @@ -75,11 +77,6 @@ func NewFrom(params maps.Params) Provider { } } -// NewWithTestDefaults is used in tests only. -func NewWithTestDefaults() Provider { - return SetBaseTestDefaults(New()) -} - // defaultConfigProvider is a Provider backed by a map where all keys are lower case. // All methods are thread safe. type defaultConfigProvider struct { @@ -160,9 +157,9 @@ func (c *defaultConfigProvider) Set(k string, v any) { k = strings.ToLower(k) if k == "" { - if p, ok := maps.ToParamsAndPrepare(v); ok { + if p, err := maps.ToParamsAndPrepare(v); err == nil { // Set the values directly in root. - c.root.Set(p) + maps.SetParams(c.root, p) } else { c.root[k] = v } @@ -184,7 +181,7 @@ func (c *defaultConfigProvider) Set(k string, v any) { if existing, found := m[key]; found { if p1, ok := existing.(maps.Params); ok { if p2, ok := v.(maps.Params); ok { - p1.Set(p2) + maps.SetParams(p1, p2) return } } @@ -208,12 +205,6 @@ func (c *defaultConfigProvider) Merge(k string, v any) { defer c.mu.Unlock() k = strings.ToLower(k) - const ( - languagesKey = "languages" - paramsKey = "params" - menusKey = "menus" - ) - if k == "" { rs, f := c.root.GetMergeStrategy() if f && rs == maps.ParamsMergeStrategyNone { @@ -222,7 +213,7 @@ func (c *defaultConfigProvider) Merge(k string, v any) { return } - if p, ok := maps.ToParamsAndPrepare(v); ok { + if p, err := maps.ToParamsAndPrepare(v); err == nil { // As there may be keys in p not in root, we need to handle // those as a special case. var keysToDelete []string @@ -230,49 +221,14 @@ func (c *defaultConfigProvider) Merge(k string, v any) { if pp, ok := vv.(maps.Params); ok { if pppi, ok := c.root[kk]; ok { ppp := pppi.(maps.Params) - if kk == languagesKey { - // Languages is currently a special case. - // We may have languages with menus or params in the - // right map that is not present in the left map. - // With the default merge strategy those items will not - // be passed over. - var hasParams, hasMenus bool - for _, rv := range pp { - if lkp, ok := rv.(maps.Params); ok { - _, hasMenus = lkp[menusKey] - _, hasParams = lkp[paramsKey] - } - } - - if hasMenus || hasParams { - for _, lv := range ppp { - if lkp, ok := lv.(maps.Params); ok { - if hasMenus { - if _, ok := lkp[menusKey]; !ok { - p := maps.Params{} - p.SetDefaultMergeStrategy(maps.ParamsMergeStrategyShallow) - lkp[menusKey] = p - } - } - if hasParams { - if _, ok := lkp[paramsKey]; !ok { - p := maps.Params{} - p.SetDefaultMergeStrategy(maps.ParamsMergeStrategyShallow) - lkp[paramsKey] = p - } - } - } - } - } - } - ppp.Merge(pp) + maps.MergeParamsWithStrategy("", ppp, pp) } else { // We need to use the default merge strategy for // this key. np := make(maps.Params) - strategy := c.determineMergeStrategy(KeyParams{Key: "", Params: c.root}, KeyParams{Key: kk, Params: np}) - np.SetDefaultMergeStrategy(strategy) - np.Merge(pp) + strategy := c.determineMergeStrategy(maps.KeyParams{Key: "", Params: c.root}, maps.KeyParams{Key: kk, Params: np}) + np.SetMergeStrategy(strategy) + maps.MergeParamsWithStrategy("", np, pp) c.root[kk] = np if np.IsZero() { // Just keep it until merge is done. @@ -282,7 +238,7 @@ func (c *defaultConfigProvider) Merge(k string, v any) { } } // Merge the rest. - c.root.MergeRoot(p) + maps.MergeParams(c.root, p) for _, k := range keysToDelete { delete(c.root, k) } @@ -307,7 +263,7 @@ func (c *defaultConfigProvider) Merge(k string, v any) { if existing, found := m[key]; found { if p1, ok := existing.(maps.Params); ok { if p2, ok := v.(maps.Params); ok { - p1.Merge(p2) + maps.MergeParamsWithStrategy("", p1, p2) } } } else { @@ -315,9 +271,15 @@ func (c *defaultConfigProvider) Merge(k string, v any) { } } -func (c *defaultConfigProvider) WalkParams(walkFn func(params ...KeyParams) bool) { - var walk func(params ...KeyParams) - walk = func(params ...KeyParams) { +func (c *defaultConfigProvider) Keys() []string { + c.mu.RLock() + defer c.mu.RUnlock() + return xmaps.Keys(c.root) +} + +func (c *defaultConfigProvider) WalkParams(walkFn func(params ...maps.KeyParams) bool) { + var walk func(params ...maps.KeyParams) + walk = func(params ...maps.KeyParams) { if walkFn(params...) { return } @@ -325,17 +287,17 @@ func (c *defaultConfigProvider) WalkParams(walkFn func(params ...KeyParams) bool i := len(params) for k, v := range p1.Params { if p2, ok := v.(maps.Params); ok { - paramsplus1 := make([]KeyParams, i+1) + paramsplus1 := make([]maps.KeyParams, i+1) copy(paramsplus1, params) - paramsplus1[i] = KeyParams{Key: k, Params: p2} + paramsplus1[i] = maps.KeyParams{Key: k, Params: p2} walk(paramsplus1...) } } } - walk(KeyParams{Key: "", Params: c.root}) + walk(maps.KeyParams{Key: "", Params: c.root}) } -func (c *defaultConfigProvider) determineMergeStrategy(params ...KeyParams) maps.ParamsMergeStrategy { +func (c *defaultConfigProvider) determineMergeStrategy(params ...maps.KeyParams) maps.ParamsMergeStrategy { if len(params) == 0 { return maps.ParamsMergeStrategyNone } @@ -391,13 +353,8 @@ func (c *defaultConfigProvider) determineMergeStrategy(params ...KeyParams) maps return strategy } -type KeyParams struct { - Key string - Params maps.Params -} - func (c *defaultConfigProvider) SetDefaultMergeStrategy() { - c.WalkParams(func(params ...KeyParams) bool { + c.WalkParams(func(params ...maps.KeyParams) bool { if len(params) == 0 { return false } @@ -409,7 +366,7 @@ func (c *defaultConfigProvider) SetDefaultMergeStrategy() { } strategy := c.determineMergeStrategy(params...) if strategy != "" { - p.SetDefaultMergeStrategy(strategy) + p.SetMergeStrategy(strategy) } return false }) diff --git a/config/namespace.go b/config/namespace.go new file mode 100644 index 000000000..3ecd01014 --- /dev/null +++ b/config/namespace.go @@ -0,0 +1,76 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "encoding/json" + + "github.com/gohugoio/hugo/identity" +) + +func DecodeNamespace[S, C any](configSource any, buildConfig func(any) (C, any, error)) (*ConfigNamespace[S, C], error) { + + // Calculate the hash of the input (not including any defaults applied later). + // This allows us to introduce new config options without breaking the hash. + h := identity.HashString(configSource) + + // Build the config + c, ext, err := buildConfig(configSource) + if err != nil { + return nil, err + } + + if ext == nil { + ext = configSource + } + + if ext == nil { + panic("ext is nil") + } + + ns := &ConfigNamespace[S, C]{ + SourceStructure: ext, + SourceHash: h, + Config: c, + } + + return ns, nil +} + +// ConfigNamespace holds a Hugo configuration namespace. +// The construct looks a little odd, but it's built to make the configuration elements +// both self-documenting and contained in a common structure. +type ConfigNamespace[S, C any] struct { + // SourceStructure represents the source configuration with any defaults applied. + // This is used for documentation and printing of the configuration setup to the user. + SourceStructure any + + // SourceHash is a hash of the source configuration before any defaults gets applied. + SourceHash string + + // Config is the final configuration as used by Hugo. + Config C +} + +// MarshalJSON marshals the source structure. +func (ns *ConfigNamespace[S, C]) MarshalJSON() ([]byte, error) { + return json.Marshal(ns.SourceStructure) +} + +// Signature returns the signature of the source structure. +// Note that this is for documentation purposes only and SourceStructure may not always be cast to S (it's usually just a map). +func (ns *ConfigNamespace[S, C]) Signature() S { + var s S + return s +} diff --git a/config/namespace_test.go b/config/namespace_test.go new file mode 100644 index 000000000..008237c13 --- /dev/null +++ b/config/namespace_test.go @@ -0,0 +1,68 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "strings" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/maps" + "github.com/mitchellh/mapstructure" +) + +func TestNamespace(t *testing.T) { + c := qt.New(t) + c.Assert(true, qt.Equals, true) + + //ns, err := config.DecodeNamespace[map[string]DocsMediaTypeConfig](in, defaultMediaTypesConfig, buildConfig) + + ns, err := DecodeNamespace[[]*tstNsExt]( + map[string]interface{}{"foo": "bar"}, + func(v any) (*tstNsExt, any, error) { + t := &tstNsExt{} + m, err := maps.ToStringMapE(v) + if err != nil { + return nil, nil, err + } + return t, nil, mapstructure.WeakDecode(m, t) + }, + ) + + c.Assert(err, qt.IsNil) + c.Assert(ns, qt.Not(qt.IsNil)) + c.Assert(ns.SourceStructure, qt.DeepEquals, map[string]interface{}{"foo": "bar"}) + c.Assert(ns.SourceHash, qt.Equals, "14368731254619220105") + c.Assert(ns.Config, qt.DeepEquals, &tstNsExt{Foo: "bar"}) + c.Assert(ns.Signature(), qt.DeepEquals, []*tstNsExt(nil)) + +} + +type ( + tstNsExt struct { + Foo string + } + tstNsInt struct { + Foo string + } +) + +func (t *tstNsExt) Init() error { + t.Foo = strings.ToUpper(t.Foo) + return nil +} +func (t *tstNsInt) Compile(ext *tstNsExt) error { + t.Foo = ext.Foo + " qux" + return nil +} diff --git a/config/security/securityConfig.go b/config/security/securityConfig.go index 4b0e07086..66e89fb97 100644 --- a/config/security/securityConfig.go +++ b/config/security/securityConfig.go @@ -54,14 +54,16 @@ var DefaultConfig = Config{ } // Config is the top level security config. +// {"name": "security", "description": "This section holds the top level security config.", "newIn": "0.91.0" } type Config struct { - // Restricts access to os.Exec. + // Restricts access to os.Exec.... + // { "newIn": "0.91.0" } Exec Exec `json:"exec"` // Restricts access to certain template funcs. Funcs Funcs `json:"funcs"` - // Restricts access to resources.Get, getJSON, getCSV. + // Restricts access to resources.GetRemote, getJSON, getCSV. HTTP HTTP `json:"http"` // Allow inline shortcodes diff --git a/config/services/servicesConfig_test.go b/config/services/servicesConfig_test.go index 826255e73..12b042a5a 100644 --- a/config/services/servicesConfig_test.go +++ b/config/services/servicesConfig_test.go @@ -54,7 +54,7 @@ disableInlineCSS = true func TestUseSettingsFromRootIfSet(t *testing.T) { c := qt.New(t) - cfg := config.NewWithTestDefaults() + cfg := config.New() cfg.Set("disqusShortname", "root_short") cfg.Set("googleAnalytics", "ga_root") diff --git a/config/testconfig/testconfig.go b/config/testconfig/testconfig.go new file mode 100644 index 000000000..4b47d82d1 --- /dev/null +++ b/config/testconfig/testconfig.go @@ -0,0 +1,84 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This package should only be used for testing. +package testconfig + +import ( + _ "unsafe" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/allconfig" + "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/hugofs" + toml "github.com/pelletier/go-toml/v2" + "github.com/spf13/afero" +) + +func GetTestConfigs(fs afero.Fs, cfg config.Provider) *allconfig.Configs { + if fs == nil { + fs = afero.NewMemMapFs() + } + if cfg == nil { + cfg = config.New() + } + // Make sure that the workingDir exists. + workingDir := cfg.GetString("workingDir") + if workingDir != "" { + if err := fs.MkdirAll(workingDir, 0777); err != nil { + panic(err) + } + } + + configs, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{Fs: fs, Flags: cfg}) + if err != nil { + panic(err) + } + return configs + +} + +func GetTestConfig(fs afero.Fs, cfg config.Provider) config.AllProvider { + return GetTestConfigs(fs, cfg).GetFirstLanguageConfig() +} + +func GetTestDeps(fs afero.Fs, cfg config.Provider, beforeInit ...func(*deps.Deps)) *deps.Deps { + if fs == nil { + fs = afero.NewMemMapFs() + } + conf := GetTestConfig(fs, cfg) + d := &deps.Deps{ + Conf: conf, + Fs: hugofs.NewFrom(fs, conf.BaseConfig()), + } + for _, f := range beforeInit { + f(d) + } + if err := d.Init(); err != nil { + panic(err) + } + return d +} + +func GetTestConfigSectionFromStruct(section string, v any) config.AllProvider { + data, err := toml.Marshal(v) + if err != nil { + panic(err) + } + p := maps.Params{ + section: config.FromTOMLConfigString(string(data)).Get(""), + } + cfg := config.NewFrom(p) + return GetTestConfig(nil, cfg) +} diff --git a/create/content.go b/create/content.go index f8629a778..55159c24c 100644 --- a/create/content.go +++ b/create/content.go @@ -340,7 +340,7 @@ func (b *contentBuilder) mapArcheTypeDir() error { } func (b *contentBuilder) openInEditorIfConfigured(filename string) error { - editor := b.h.Cfg.GetString("newContentEditor") + editor := b.h.Conf.NewContentEditor() if editor == "" { return nil } diff --git a/create/content_test.go b/create/content_test.go index fdfee6e68..77c6ca6c9 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -21,6 +21,8 @@ import ( "testing" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/allconfig" + "github.com/gohugoio/hugo/config/testconfig" "github.com/gohugoio/hugo/deps" @@ -80,7 +82,8 @@ func TestNewContentFromFile(t *testing.T) { mm := afero.NewMemMapFs() c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + conf := testconfig.GetTestConfigs(fs.Source, cfg) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) c.Assert(err, qt.IsNil) err = create.NewContent(h, cas.kind, cas.path, false) @@ -141,7 +144,8 @@ i18n: {{ T "hugo" }} c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + conf := testconfig.GetTestConfigs(fs.Source, cfg) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) c.Assert(err, qt.IsNil) c.Assert(len(h.Sites), qt.Equals, 2) @@ -183,7 +187,8 @@ site RegularPages: {{ len site.RegularPages }} c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + conf := testconfig.GetTestConfigs(fs.Source, cfg) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) c.Assert(err, qt.IsNil) c.Assert(len(h.Sites), qt.Equals, 2) @@ -232,8 +237,8 @@ i18n: {{ T "hugo" }} c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) - - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + conf := testconfig.GetTestConfigs(fs.Source, cfg) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) c.Assert(err, qt.IsNil) c.Assert(len(h.Sites), qt.Equals, 2) @@ -264,7 +269,8 @@ func TestNewContentForce(t *testing.T) { c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) - h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + conf := testconfig.GetTestConfigs(fs.Source, cfg) + h, err := hugolib.NewHugoSites(deps.DepsCfg{Configs: conf, Fs: fs}) c.Assert(err, qt.IsNil) c.Assert(len(h.Sites), qt.Equals, 2) @@ -461,8 +467,8 @@ other = "Hugo Rokkar!"`), 0o755), qt.IsNil) c.Assert(afero.WriteFile(mm, "config.toml", []byte(cfg), 0o755), qt.IsNil) - v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"}) + res, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"}) c.Assert(err, qt.IsNil) - return v, hugofs.NewFrom(mm, v) + return res.LoadingInfo.Cfg, hugofs.NewFrom(mm, res.LoadingInfo.BaseConfig) } diff --git a/deploy/deploy.go b/deploy/deploy.go index 2d3d3b552..db88996a9 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -55,17 +55,12 @@ type Deployer struct { localFs afero.Fs bucket *blob.Bucket - target *target // the target to deploy to - matchers []*matcher // matchers to apply to uploaded files - mediaTypes media.Types // Hugo's MediaType to guess ContentType - ordering []*regexp.Regexp // orders uploads - quiet bool // true reduces STDOUT - confirm bool // true enables confirmation before making changes - dryRun bool // true skips conformations and prints changes instead of applying them - force bool // true forces upload of all files - invalidateCDN bool // true enables invalidate CDN cache (if possible) - maxDeletes int // caps the # of files to delete; -1 to disable - workers int // The number of workers to transfer files + mediaTypes media.Types // Hugo's MediaType to guess ContentType + quiet bool // true reduces STDOUT + + cfg DeployConfig + + target *Target // the target to deploy to // For tests... summary deploySummary // summary of latest Deploy results @@ -78,21 +73,18 @@ type deploySummary struct { const metaMD5Hash = "md5chksum" // the meta key to store md5hash in // New constructs a new *Deployer. -func New(cfg config.Provider, localFs afero.Fs) (*Deployer, error) { - targetName := cfg.GetString("target") +func New(cfg config.AllProvider, localFs afero.Fs) (*Deployer, error) { - // Load the [deployment] section of the config. - dcfg, err := decodeConfig(cfg) - if err != nil { - return nil, err - } + dcfg := cfg.GetConfigSection(deploymentConfigKey).(DeployConfig) + targetName := dcfg.Target if len(dcfg.Targets) == 0 { return nil, errors.New("no deployment targets found") } + mediaTypes := cfg.GetConfigSection("mediaTypes").(media.Types) // Find the target to deploy to. - var tgt *target + var tgt *Target if targetName == "" { // Default to the first target. tgt = dcfg.Targets[0] @@ -108,18 +100,11 @@ func New(cfg config.Provider, localFs afero.Fs) (*Deployer, error) { } return &Deployer{ - localFs: localFs, - target: tgt, - matchers: dcfg.Matchers, - ordering: dcfg.ordering, - mediaTypes: dcfg.mediaTypes, - quiet: cfg.GetBool("quiet"), - confirm: cfg.GetBool("confirm"), - dryRun: cfg.GetBool("dryRun"), - force: cfg.GetBool("force"), - invalidateCDN: cfg.GetBool("invalidateCDN"), - maxDeletes: cfg.GetInt("maxDeletes"), - workers: cfg.GetInt("workers"), + localFs: localFs, + target: tgt, + quiet: cfg.BuildExpired(), + mediaTypes: mediaTypes, + cfg: dcfg, }, nil } @@ -138,12 +123,16 @@ func (d *Deployer) Deploy(ctx context.Context) error { return err } + if d.cfg.Workers <= 0 { + d.cfg.Workers = 10 + } + // Load local files from the source directory. var include, exclude glob.Glob if d.target != nil { include, exclude = d.target.includeGlob, d.target.excludeGlob } - local, err := walkLocal(d.localFs, d.matchers, include, exclude, d.mediaTypes) + local, err := walkLocal(d.localFs, d.cfg.Matchers, include, exclude, d.mediaTypes) if err != nil { return err } @@ -159,7 +148,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { d.summary.NumRemote = len(remote) // Diff local vs remote to see what changes need to be applied. - uploads, deletes := findDiffs(local, remote, d.force) + uploads, deletes := findDiffs(local, remote, d.cfg.Force) d.summary.NumUploads = len(uploads) d.summary.NumDeletes = len(deletes) if len(uploads)+len(deletes) == 0 { @@ -173,7 +162,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { } // Ask for confirmation before proceeding. - if d.confirm && !d.dryRun { + if d.cfg.Confirm && !d.cfg.DryRun { fmt.Printf("Continue? (Y/n) ") var confirm string if _, err := fmt.Scanln(&confirm); err != nil { @@ -186,15 +175,9 @@ func (d *Deployer) Deploy(ctx context.Context) error { // Order the uploads. They are organized in groups; all uploads in a group // must be complete before moving on to the next group. - uploadGroups := applyOrdering(d.ordering, uploads) + uploadGroups := applyOrdering(d.cfg.ordering, uploads) - // Apply the changes in parallel, using an inverted worker - // pool (https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s). - // sem prevents more than nParallel concurrent goroutines. - if d.workers <= 0 { - d.workers = 10 - } - nParallel := d.workers + nParallel := d.cfg.Workers var errs []error var errMu sync.Mutex // protects errs @@ -207,7 +190,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { // Within the group, apply uploads in parallel. sem := make(chan struct{}, nParallel) for _, upload := range uploads { - if d.dryRun { + if d.cfg.DryRun { if !d.quiet { jww.FEEDBACK.Printf("[DRY RUN] Would upload: %v\n", upload) } @@ -230,15 +213,15 @@ func (d *Deployer) Deploy(ctx context.Context) error { } } - if d.maxDeletes != -1 && len(deletes) > d.maxDeletes { - jww.WARN.Printf("Skipping %d deletes because it is more than --maxDeletes (%d). If this is expected, set --maxDeletes to a larger number, or -1 to disable this check.\n", len(deletes), d.maxDeletes) + if d.cfg.MaxDeletes != -1 && len(deletes) > d.cfg.MaxDeletes { + jww.WARN.Printf("Skipping %d deletes because it is more than --maxDeletes (%d). If this is expected, set --maxDeletes to a larger number, or -1 to disable this check.\n", len(deletes), d.cfg.MaxDeletes) d.summary.NumDeletes = 0 } else { // Apply deletes in parallel. sort.Slice(deletes, func(i, j int) bool { return deletes[i] < deletes[j] }) sem := make(chan struct{}, nParallel) for _, del := range deletes { - if d.dryRun { + if d.cfg.DryRun { if !d.quiet { jww.FEEDBACK.Printf("[DRY RUN] Would delete %s\n", del) } @@ -264,6 +247,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { sem <- struct{}{} } } + if len(errs) > 0 { if !d.quiet { jww.FEEDBACK.Printf("Encountered %d errors.\n", len(errs)) @@ -274,9 +258,9 @@ func (d *Deployer) Deploy(ctx context.Context) error { jww.FEEDBACK.Println("Success!") } - if d.invalidateCDN { + if d.cfg.InvalidateCDN { if d.target.CloudFrontDistributionID != "" { - if d.dryRun { + if d.cfg.DryRun { if !d.quiet { jww.FEEDBACK.Printf("[DRY RUN] Would invalidate CloudFront CDN with ID %s\n", d.target.CloudFrontDistributionID) } @@ -289,7 +273,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { } } if d.target.GoogleCloudCDNOrigin != "" { - if d.dryRun { + if d.cfg.DryRun { if !d.quiet { jww.FEEDBACK.Printf("[DRY RUN] Would invalidate Google Cloud CDN with origin %s\n", d.target.GoogleCloudCDNOrigin) } @@ -356,14 +340,14 @@ type localFile struct { UploadSize int64 fs afero.Fs - matcher *matcher + matcher *Matcher md5 []byte // cache gzipped bytes.Buffer // cached of gzipped contents if gzipping mediaTypes media.Types } // newLocalFile initializes a *localFile. -func newLocalFile(fs afero.Fs, nativePath, slashpath string, m *matcher, mt media.Types) (*localFile, error) { +func newLocalFile(fs afero.Fs, nativePath, slashpath string, m *Matcher, mt media.Types) (*localFile, error) { f, err := fs.Open(nativePath) if err != nil { return nil, err @@ -448,7 +432,7 @@ func (lf *localFile) ContentType() string { ext := filepath.Ext(lf.NativePath) if mimeType, _, found := lf.mediaTypes.GetFirstBySuffix(strings.TrimPrefix(ext, ".")); found { - return mimeType.Type() + return mimeType.Type } return mime.TypeByExtension(ext) @@ -495,7 +479,7 @@ func knownHiddenDirectory(name string) bool { // walkLocal walks the source directory and returns a flat list of files, // using localFile.SlashPath as the map keys. -func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob, mediaTypes media.Types) (map[string]*localFile, error) { +func walkLocal(fs afero.Fs, matchers []*Matcher, include, exclude glob.Glob, mediaTypes media.Types) (map[string]*localFile, error) { retval := map[string]*localFile{} err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { if err != nil { @@ -534,7 +518,7 @@ func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob, med } // Find the first matching matcher (if any). - var m *matcher + var m *Matcher for _, cur := range matchers { if cur.Matches(slashpath) { m = cur diff --git a/deploy/deployConfig.go b/deploy/deployConfig.go index 477751d33..3f5465171 100644 --- a/deploy/deployConfig.go +++ b/deploy/deployConfig.go @@ -25,23 +25,37 @@ import ( "github.com/gobwas/glob" "github.com/gohugoio/hugo/config" hglob "github.com/gohugoio/hugo/hugofs/glob" - "github.com/gohugoio/hugo/media" "github.com/mitchellh/mapstructure" ) const deploymentConfigKey = "deployment" -// deployConfig is the complete configuration for deployment. -type deployConfig struct { - Targets []*target - Matchers []*matcher +// DeployConfig is the complete configuration for deployment. +type DeployConfig struct { + Targets []*Target + Matchers []*Matcher Order []string - ordering []*regexp.Regexp // compiled Order - mediaTypes media.Types + // Usually set via flags. + // Target deployment Name; defaults to the first one. + Target string + // Show a confirm prompt before deploying. + Confirm bool + // DryRun will try the deployment without any remote changes. + DryRun bool + // Force will re-upload all files. + Force bool + // Invalidate the CDN cache listed in the deployment target. + InvalidateCDN bool + // MaxDeletes is the maximum number of files to delete. + MaxDeletes int + // Number of concurrent workers to use when uploading files. + Workers int + + ordering []*regexp.Regexp // compiled Order } -type target struct { +type Target struct { Name string URL string @@ -61,7 +75,7 @@ type target struct { excludeGlob glob.Glob } -func (tgt *target) parseIncludeExclude() error { +func (tgt *Target) parseIncludeExclude() error { var err error if tgt.Include != "" { tgt.includeGlob, err = hglob.GetGlob(tgt.Include) @@ -78,9 +92,9 @@ func (tgt *target) parseIncludeExclude() error { return nil } -// matcher represents configuration to be applied to files whose paths match +// Matcher represents configuration to be applied to files whose paths match // a specified pattern. -type matcher struct { +type Matcher struct { // Pattern is the string pattern to match against paths. // Matching is done against paths converted to use / as the path separator. Pattern string @@ -109,15 +123,14 @@ type matcher struct { re *regexp.Regexp } -func (m *matcher) Matches(path string) bool { +func (m *Matcher) Matches(path string) bool { return m.re.MatchString(path) } -// decode creates a config from a given Hugo configuration. -func decodeConfig(cfg config.Provider) (deployConfig, error) { +// DecodeConfig creates a config from a given Hugo configuration. +func DecodeConfig(cfg config.Provider) (DeployConfig, error) { var ( - mediaTypesConfig []map[string]any - dcfg deployConfig + dcfg DeployConfig ) if !cfg.IsSet(deploymentConfigKey) { @@ -126,8 +139,13 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) { if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil { return dcfg, err } + + if dcfg.Workers <= 0 { + dcfg.Workers = 10 + } + for _, tgt := range dcfg.Targets { - if *tgt == (target{}) { + if *tgt == (Target{}) { return dcfg, errors.New("empty deployment target") } if err := tgt.parseIncludeExclude(); err != nil { @@ -136,7 +154,7 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) { } var err error for _, m := range dcfg.Matchers { - if *m == (matcher{}) { + if *m == (Matcher{}) { return dcfg, errors.New("empty deployment matcher") } m.re, err = regexp.Compile(m.Pattern) @@ -152,13 +170,5 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) { dcfg.ordering = append(dcfg.ordering, re) } - if cfg.IsSet("mediaTypes") { - mediaTypesConfig = append(mediaTypesConfig, cfg.GetStringMap("mediaTypes")) - } - - dcfg.mediaTypes, err = media.DecodeTypes(mediaTypesConfig...) - if err != nil { - return dcfg, err - } return dcfg, nil } diff --git a/deploy/deployConfig_test.go b/deploy/deployConfig_test.go index ed03d57db..2dbe18715 100644 --- a/deploy/deployConfig_test.go +++ b/deploy/deployConfig_test.go @@ -84,7 +84,7 @@ force = true cfg, err := config.FromConfigString(tomlConfig, "toml") c.Assert(err, qt.IsNil) - dcfg, err := decodeConfig(cfg) + dcfg, err := DecodeConfig(cfg) c.Assert(err, qt.IsNil) // Order. @@ -139,7 +139,7 @@ order = ["["] # invalid regular expression cfg, err := config.FromConfigString(tomlConfig, "toml") c.Assert(err, qt.IsNil) - _, err = decodeConfig(cfg) + _, err = DecodeConfig(cfg) c.Assert(err, qt.Not(qt.IsNil)) } @@ -157,14 +157,14 @@ Pattern = "[" # invalid regular expression cfg, err := config.FromConfigString(tomlConfig, "toml") c.Assert(err, qt.IsNil) - _, err = decodeConfig(cfg) + _, err = DecodeConfig(cfg) c.Assert(err, qt.Not(qt.IsNil)) } func TestDecodeConfigDefault(t *testing.T) { c := qt.New(t) - dcfg, err := decodeConfig(config.New()) + dcfg, err := DecodeConfig(config.New()) c.Assert(err, qt.IsNil) c.Assert(len(dcfg.Targets), qt.Equals, 0) c.Assert(len(dcfg.Matchers), qt.Equals, 0) @@ -180,7 +180,7 @@ func TestEmptyTarget(t *testing.T) { cfg, err := config.FromConfigString(tomlConfig, "toml") c.Assert(err, qt.IsNil) - _, err = decodeConfig(cfg) + _, err = DecodeConfig(cfg) c.Assert(err, qt.Not(qt.IsNil)) } @@ -194,6 +194,6 @@ func TestEmptyMatcher(t *testing.T) { cfg, err := config.FromConfigString(tomlConfig, "toml") c.Assert(err, qt.IsNil) - _, err = decodeConfig(cfg) + _, err = DecodeConfig(cfg) c.Assert(err, qt.Not(qt.IsNil)) } diff --git a/deploy/deploy_test.go b/deploy/deploy_test.go index 5c436abf2..fe874fbbd 100644 --- a/deploy/deploy_test.go +++ b/deploy/deploy_test.go @@ -108,7 +108,7 @@ func TestFindDiffs(t *testing.T) { { Description: "local == remote with route.Force true -> diffs", Local: []*localFile{ - {NativePath: "aaa", SlashPath: "aaa", UploadSize: 1, matcher: &matcher{Force: true}, md5: hash1}, + {NativePath: "aaa", SlashPath: "aaa", UploadSize: 1, matcher: &Matcher{Force: true}, md5: hash1}, makeLocal("bbb", 2, hash1), }, Remote: []*blob.ListObject{ @@ -289,8 +289,8 @@ func TestLocalFile(t *testing.T) { tests := []struct { Description string Path string - Matcher *matcher - MediaTypesConfig []map[string]any + Matcher *Matcher + MediaTypesConfig map[string]any WantContent []byte WantSize int64 WantMD5 []byte @@ -315,7 +315,7 @@ func TestLocalFile(t *testing.T) { { Description: "CacheControl from matcher", Path: "foo.txt", - Matcher: &matcher{CacheControl: "max-age=630720000"}, + Matcher: &Matcher{CacheControl: "max-age=630720000"}, WantContent: contentBytes, WantSize: contentLen, WantMD5: contentMD5[:], @@ -324,7 +324,7 @@ func TestLocalFile(t *testing.T) { { Description: "ContentEncoding from matcher", Path: "foo.txt", - Matcher: &matcher{ContentEncoding: "foobar"}, + Matcher: &Matcher{ContentEncoding: "foobar"}, WantContent: contentBytes, WantSize: contentLen, WantMD5: contentMD5[:], @@ -333,7 +333,7 @@ func TestLocalFile(t *testing.T) { { Description: "ContentType from matcher", Path: "foo.txt", - Matcher: &matcher{ContentType: "foo/bar"}, + Matcher: &Matcher{ContentType: "foo/bar"}, WantContent: contentBytes, WantSize: contentLen, WantMD5: contentMD5[:], @@ -342,7 +342,7 @@ func TestLocalFile(t *testing.T) { { Description: "gzipped content", Path: "foo.txt", - Matcher: &matcher{Gzip: true}, + Matcher: &Matcher{Gzip: true}, WantContent: gzBytes, WantSize: gzLen, WantMD5: gzMD5[:], @@ -351,11 +351,9 @@ func TestLocalFile(t *testing.T) { { Description: "Custom MediaType", Path: "foo.hugo", - MediaTypesConfig: []map[string]any{ - { - "hugo/custom": map[string]any{ - "suffixes": []string{"hugo"}, - }, + MediaTypesConfig: map[string]any{ + "hugo/custom": map[string]any{ + "suffixes": []string{"hugo"}, }, }, WantContent: contentBytes, @@ -373,11 +371,11 @@ func TestLocalFile(t *testing.T) { } mediaTypes := media.DefaultTypes if len(tc.MediaTypesConfig) > 0 { - mt, err := media.DecodeTypes(tc.MediaTypesConfig...) + mt, err := media.DecodeTypes(tc.MediaTypesConfig) if err != nil { t.Fatal(err) } - mediaTypes = mt + mediaTypes = mt.Config } lf, err := newLocalFile(fs, tc.Path, filepath.ToSlash(tc.Path), tc.Matcher, mediaTypes) if err != nil { @@ -556,9 +554,9 @@ func TestEndToEndSync(t *testing.T) { } deployer := &Deployer{ localFs: test.fs, - maxDeletes: -1, bucket: test.bucket, mediaTypes: media.DefaultTypes, + cfg: DeployConfig{MaxDeletes: -1}, } // Initial deployment should sync remote with local. @@ -639,9 +637,9 @@ func TestMaxDeletes(t *testing.T) { } deployer := &Deployer{ localFs: test.fs, - maxDeletes: -1, bucket: test.bucket, mediaTypes: media.DefaultTypes, + cfg: DeployConfig{MaxDeletes: -1}, } // Sync remote with local. @@ -662,7 +660,7 @@ func TestMaxDeletes(t *testing.T) { } // A deployment with maxDeletes=0 shouldn't change anything. - deployer.maxDeletes = 0 + deployer.cfg.MaxDeletes = 0 if err := deployer.Deploy(ctx); err != nil { t.Errorf("deploy failed: %v", err) } @@ -672,7 +670,7 @@ func TestMaxDeletes(t *testing.T) { } // A deployment with maxDeletes=1 shouldn't change anything either. - deployer.maxDeletes = 1 + deployer.cfg.MaxDeletes = 1 if err := deployer.Deploy(ctx); err != nil { t.Errorf("deploy failed: %v", err) } @@ -682,7 +680,7 @@ func TestMaxDeletes(t *testing.T) { } // A deployment with maxDeletes=2 should make the changes. - deployer.maxDeletes = 2 + deployer.cfg.MaxDeletes = 2 if err := deployer.Deploy(ctx); err != nil { t.Errorf("deploy failed: %v", err) } @@ -700,7 +698,7 @@ func TestMaxDeletes(t *testing.T) { } // A deployment with maxDeletes=-1 should make the changes. - deployer.maxDeletes = -1 + deployer.cfg.MaxDeletes = -1 if err := deployer.Deploy(ctx); err != nil { t.Errorf("deploy failed: %v", err) } @@ -762,7 +760,7 @@ func TestIncludeExclude(t *testing.T) { if err != nil { t.Fatal(err) } - tgt := &target{ + tgt := &Target{ Include: test.Include, Exclude: test.Exclude, } @@ -770,9 +768,8 @@ func TestIncludeExclude(t *testing.T) { t.Error(err) } deployer := &Deployer{ - localFs: fsTest.fs, - maxDeletes: -1, - bucket: fsTest.bucket, + localFs: fsTest.fs, + cfg: DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket, target: tgt, mediaTypes: media.DefaultTypes, } @@ -828,9 +825,8 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) { t.Fatal(err) } deployer := &Deployer{ - localFs: fsTest.fs, - maxDeletes: -1, - bucket: fsTest.bucket, + localFs: fsTest.fs, + cfg: DeployConfig{MaxDeletes: -1}, bucket: fsTest.bucket, mediaTypes: media.DefaultTypes, } @@ -848,7 +844,7 @@ func TestIncludeExcludeRemoteDelete(t *testing.T) { } // Second sync - tgt := &target{ + tgt := &Target{ Include: test.Include, Exclude: test.Exclude, } @@ -882,7 +878,7 @@ func TestCompression(t *testing.T) { deployer := &Deployer{ localFs: test.fs, bucket: test.bucket, - matchers: []*matcher{{Pattern: ".*", Gzip: true, re: regexp.MustCompile(".*")}}, + cfg: DeployConfig{MaxDeletes: -1, Matchers: []*Matcher{{Pattern: ".*", Gzip: true, re: regexp.MustCompile(".*")}}}, mediaTypes: media.DefaultTypes, } @@ -937,7 +933,7 @@ func TestMatching(t *testing.T) { deployer := &Deployer{ localFs: test.fs, bucket: test.bucket, - matchers: []*matcher{{Pattern: "^subdir/aaa$", Force: true, re: regexp.MustCompile("^subdir/aaa$")}}, + cfg: DeployConfig{MaxDeletes: -1, Matchers: []*Matcher{{Pattern: "^subdir/aaa$", Force: true, re: regexp.MustCompile("^subdir/aaa$")}}}, mediaTypes: media.DefaultTypes, } @@ -962,7 +958,7 @@ func TestMatching(t *testing.T) { } // Repeat with a matcher that should now match 3 files. - deployer.matchers = []*matcher{{Pattern: "aaa", Force: true, re: regexp.MustCompile("aaa")}} + deployer.cfg.Matchers = []*Matcher{{Pattern: "aaa", Force: true, re: regexp.MustCompile("aaa")}} if err := deployer.Deploy(ctx); err != nil { t.Errorf("no-op deploy with triple force matcher: %v", err) } diff --git a/deps/deps.go b/deps/deps.go index 511ee885c..9cb8557a5 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -4,30 +4,27 @@ import ( "context" "fmt" "path/filepath" + "sort" "strings" "sync" "sync/atomic" - "time" - "github.com/gohugoio/hugo/cache/filecache" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/allconfig" "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/postpub" "github.com/gohugoio/hugo/metrics" - "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" - "github.com/spf13/cast" + "github.com/spf13/afero" jww "github.com/spf13/jwalterweatherman" ) @@ -45,10 +42,7 @@ type Deps struct { ExecHelper *hexec.Exec // The templates to use. This will usually implement the full tpl.TemplateManager. - tmpl tpl.TemplateHandler - - // We use this to parse and execute ad-hoc text templates. - textTmpl tpl.TemplateParseFinder + tmplHandlers *tpl.TemplateHandlers // The file systems to use. Fs *hugofs.Fs `json:"-"` @@ -66,56 +60,170 @@ type Deps struct { ResourceSpec *resources.Spec // The configuration to use - Cfg config.Provider `json:"-"` - - // The file cache to use. - FileCaches filecache.Caches + Conf config.AllProvider `json:"-"` // The translation func to use Translate func(ctx context.Context, translationID string, templateData any) string `json:"-"` - // The language in use. TODO(bep) consolidate with site - Language *langs.Language - // The site building. Site page.Site - // All the output formats available for the current site. - OutputFormatsConfig output.Formats - - // FilenameHasPostProcessPrefix is a set of filenames in /public that - // contains a post-processing prefix. - FilenameHasPostProcessPrefix []string - - templateProvider ResourceProvider - WithTemplate func(templ tpl.TemplateManager) error `json:"-"` - + TemplateProvider ResourceProvider // Used in tests OverloadedTemplateFuncs map[string]any - translationProvider ResourceProvider + TranslationProvider ResourceProvider Metrics metrics.Provider - // Timeout is configurable in site config. - Timeout time.Duration - // BuildStartListeners will be notified before a build starts. BuildStartListeners *Listeners // Resources that gets closed when the build is done or the server shuts down. BuildClosers *Closers - // Atomic values set during a build. // This is common/global for all sites. BuildState *BuildState - // Whether we are in running (server) mode - Running bool - *globalErrHandler } +func (d Deps) Clone(s page.Site, conf config.AllProvider) (*Deps, error) { + d.Conf = conf + d.Site = s + d.ExecHelper = nil + d.ContentSpec = nil + + if err := d.Init(); err != nil { + return nil, err + } + + return &d, nil + +} + +func (d *Deps) SetTempl(t *tpl.TemplateHandlers) { + d.tmplHandlers = t +} + +func (d *Deps) Init() error { + if d.Conf == nil { + panic("conf is nil") + } + + if d.Fs == nil { + // For tests. + d.Fs = hugofs.NewFrom(afero.NewMemMapFs(), d.Conf.BaseConfig()) + } + + if d.Log == nil { + d.Log = loggers.NewErrorLogger() + } + + if d.LogDistinct == nil { + d.LogDistinct = helpers.NewDistinctLogger(d.Log) + } + + if d.globalErrHandler == nil { + d.globalErrHandler = &globalErrHandler{} + } + + if d.BuildState == nil { + d.BuildState = &BuildState{} + } + + if d.BuildStartListeners == nil { + d.BuildStartListeners = &Listeners{} + } + + if d.BuildClosers == nil { + d.BuildClosers = &Closers{} + } + + if d.Metrics == nil && d.Conf.TemplateMetrics() { + d.Metrics = metrics.NewProvider(d.Conf.TemplateMetricsHints()) + } + + if d.ExecHelper == nil { + d.ExecHelper = hexec.New(d.Conf.GetConfigSection("security").(security.Config)) + } + + if d.PathSpec == nil { + hashBytesReceiverFunc := func(name string, match bool) { + if !match { + return + } + d.BuildState.AddFilenameWithPostPrefix(name) + } + + // Skip binary files. + mediaTypes := d.Conf.GetConfigSection("mediaTypes").(media.Types) + hashBytesSHouldCheck := func(name string) bool { + ext := strings.TrimPrefix(filepath.Ext(name), ".") + return mediaTypes.IsTextSuffix(ext) + } + d.Fs.PublishDir = hugofs.NewHasBytesReceiver(d.Fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix)) + pathSpec, err := helpers.NewPathSpec(d.Fs, d.Conf, d.Log) + if err != nil { + return err + } + d.PathSpec = pathSpec + } else { + var err error + d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, d.Conf, d.Log, d.PathSpec.BaseFs) + if err != nil { + return err + } + } + + if d.ContentSpec == nil { + contentSpec, err := helpers.NewContentSpec(d.Conf, d.Log, d.Content.Fs, d.ExecHelper) + if err != nil { + return err + } + d.ContentSpec = contentSpec + } + + if d.SourceSpec == nil { + d.SourceSpec = source.NewSourceSpec(d.PathSpec, nil, d.Fs.Source) + } + + var common *resources.SpecCommon + if d.ResourceSpec != nil { + common = d.ResourceSpec.SpecCommon + } + resourceSpec, err := resources.NewSpec(d.PathSpec, common, d.BuildState, d.Log, d, d.ExecHelper) + if err != nil { + return fmt.Errorf("failed to create resource spec: %w", err) + } + d.ResourceSpec = resourceSpec + + return nil +} + +func (d *Deps) Compile(prototype *Deps) error { + var err error + if prototype == nil { + if err = d.TemplateProvider.NewResource(d); err != nil { + return err + } + if err = d.TranslationProvider.NewResource(d); err != nil { + return err + } + return nil + } + + if err = d.TemplateProvider.CloneResource(d, prototype); err != nil { + return err + } + + if err = d.TranslationProvider.CloneResource(d, prototype); err != nil { + return err + } + + return nil +} + type globalErrHandler struct { // Channel for some "hard to get to" build errors buildErrors chan error @@ -181,236 +289,22 @@ func (b *Listeners) Notify() { // ResourceProvider is used to create and refresh, and clone resources needed. type ResourceProvider interface { - Update(deps *Deps) error - Clone(deps *Deps) error + NewResource(dst *Deps) error + CloneResource(dst, src *Deps) error } func (d *Deps) Tmpl() tpl.TemplateHandler { - return d.tmpl + return d.tmplHandlers.Tmpl } func (d *Deps) TextTmpl() tpl.TemplateParseFinder { - return d.textTmpl -} - -func (d *Deps) SetTmpl(tmpl tpl.TemplateHandler) { - d.tmpl = tmpl -} - -func (d *Deps) SetTextTmpl(tmpl tpl.TemplateParseFinder) { - d.textTmpl = tmpl -} - -// LoadResources loads translations and templates. -func (d *Deps) LoadResources() error { - // Note that the translations need to be loaded before the templates. - if err := d.translationProvider.Update(d); err != nil { - return fmt.Errorf("loading translations: %w", err) - } - - if err := d.templateProvider.Update(d); err != nil { - return fmt.Errorf("loading templates: %w", err) - } - - return nil -} - -// New initializes a Dep struct. -// Defaults are set for nil values, -// but TemplateProvider, TranslationProvider and Language are always required. -func New(cfg DepsCfg) (*Deps, error) { - var ( - logger = cfg.Logger - fs = cfg.Fs - d *Deps - ) - - if cfg.TemplateProvider == nil { - panic("Must have a TemplateProvider") - } - - if cfg.TranslationProvider == nil { - panic("Must have a TranslationProvider") - } - - if cfg.Language == nil { - panic("Must have a Language") - } - - if logger == nil { - logger = loggers.NewErrorLogger() - } - - if fs == nil { - // Default to the production file system. - fs = hugofs.NewDefault(cfg.Language) - } - - if cfg.MediaTypes == nil { - cfg.MediaTypes = media.DefaultTypes - } - - if cfg.OutputFormats == nil { - cfg.OutputFormats = output.DefaultFormats - } - - securityConfig, err := security.DecodeConfig(cfg.Cfg) - if err != nil { - return nil, fmt.Errorf("failed to create security config from configuration: %w", err) - } - execHelper := hexec.New(securityConfig) - - var filenameHasPostProcessPrefixMu sync.Mutex - hashBytesReceiverFunc := func(name string, match bool) { - if !match { - return - } - filenameHasPostProcessPrefixMu.Lock() - d.FilenameHasPostProcessPrefix = append(d.FilenameHasPostProcessPrefix, name) - filenameHasPostProcessPrefixMu.Unlock() - } - - // Skip binary files. - hashBytesSHouldCheck := func(name string) bool { - ext := strings.TrimPrefix(filepath.Ext(name), ".") - mime, _, found := cfg.MediaTypes.GetBySuffix(ext) - if !found { - return false - } - switch mime.MainType { - case "text", "application": - return true - default: - return false - } - } - fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix)) - - ps, err := helpers.NewPathSpec(fs, cfg.Language, logger) - if err != nil { - return nil, fmt.Errorf("create PathSpec: %w", err) - } - - fileCaches, err := filecache.NewCaches(ps) - if err != nil { - return nil, fmt.Errorf("failed to create file caches from configuration: %w", err) - } - - errorHandler := &globalErrHandler{} - buildState := &BuildState{} - - resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, execHelper, cfg.OutputFormats, cfg.MediaTypes) - if err != nil { - return nil, err - } - - contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs, execHelper) - if err != nil { - return nil, err - } - - sp := source.NewSourceSpec(ps, nil, fs.Source) - - timeout := 30 * time.Second - if cfg.Cfg.IsSet("timeout") { - v := cfg.Cfg.Get("timeout") - d, err := types.ToDurationE(v) - if err == nil { - timeout = d - } - } - ignoreErrors := cast.ToStringSlice(cfg.Cfg.Get("ignoreErrors")) - ignorableLogger := loggers.NewIgnorableLogger(logger, ignoreErrors...) - - logDistinct := helpers.NewDistinctLogger(logger) - - d = &Deps{ - Fs: fs, - Log: ignorableLogger, - LogDistinct: logDistinct, - ExecHelper: execHelper, - templateProvider: cfg.TemplateProvider, - translationProvider: cfg.TranslationProvider, - WithTemplate: cfg.WithTemplate, - OverloadedTemplateFuncs: cfg.OverloadedTemplateFuncs, - PathSpec: ps, - ContentSpec: contentSpec, - SourceSpec: sp, - ResourceSpec: resourceSpec, - Cfg: cfg.Language, - Language: cfg.Language, - Site: cfg.Site, - FileCaches: fileCaches, - BuildStartListeners: &Listeners{}, - BuildClosers: &Closers{}, - BuildState: buildState, - Running: cfg.Running, - Timeout: timeout, - globalErrHandler: errorHandler, - } - - if cfg.Cfg.GetBool("templateMetrics") { - d.Metrics = metrics.NewProvider(cfg.Cfg.GetBool("templateMetricsHints")) - } - - return d, nil + return d.tmplHandlers.TxtTmpl } func (d *Deps) Close() error { return d.BuildClosers.Close() } -// ForLanguage creates a copy of the Deps with the language dependent -// parts switched out. -func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, error) { - l := cfg.Language - var err error - - d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.Log, d.BaseFs) - if err != nil { - return nil, err - } - - d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs, d.ExecHelper) - if err != nil { - return nil, err - } - - d.Site = cfg.Site - - // These are common for all sites, so reuse. - // TODO(bep) clean up these inits. - resourceCache := d.ResourceSpec.ResourceCache - postBuildAssets := d.ResourceSpec.PostBuildAssets - d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, d.ExecHelper, cfg.OutputFormats, cfg.MediaTypes) - if err != nil { - return nil, err - } - d.ResourceSpec.ResourceCache = resourceCache - d.ResourceSpec.PostBuildAssets = postBuildAssets - - d.Cfg = l - d.Language = l - - if onCreated != nil { - if err = onCreated(&d); err != nil { - return nil, err - } - } - - if err := d.translationProvider.Clone(&d); err != nil { - return nil, err - } - - if err := d.templateProvider.Clone(&d); err != nil { - return nil, err - } - - d.BuildStartListeners = &Listeners{} - - return &d, nil -} - // DepsCfg contains configuration options that can be used to configure Hugo // on a global level, i.e. logging etc. // Nil values will be given default values. @@ -422,47 +316,53 @@ type DepsCfg struct { // The file systems to use Fs *hugofs.Fs - // The language to use. - Language *langs.Language - // The Site in use Site page.Site - // The configuration to use. - Cfg config.Provider - - // The media types configured. - MediaTypes media.Types - - // The output formats configured. - OutputFormats output.Formats + Configs *allconfig.Configs // Template handling. TemplateProvider ResourceProvider - WithTemplate func(templ tpl.TemplateManager) error - // Used in tests - OverloadedTemplateFuncs map[string]any // i18n handling. TranslationProvider ResourceProvider - - // Whether we are in running (server) mode - Running bool } -// BuildState are flags that may be turned on during a build. +// BuildState are state used during a build. type BuildState struct { counter uint64 + + mu sync.Mutex // protects state below. + + // A set of ilenames in /public that + // contains a post-processing prefix. + filenamesWithPostPrefix map[string]bool +} + +func (b *BuildState) AddFilenameWithPostPrefix(filename string) { + b.mu.Lock() + defer b.mu.Unlock() + if b.filenamesWithPostPrefix == nil { + b.filenamesWithPostPrefix = make(map[string]bool) + } + b.filenamesWithPostPrefix[filename] = true +} + +func (b *BuildState) GetFilenamesWithPostPrefix() []string { + b.mu.Lock() + defer b.mu.Unlock() + var filenames []string + for filename := range b.filenamesWithPostPrefix { + filenames = append(filenames, filename) + } + sort.Strings(filenames) + return filenames } func (b *BuildState) Incr() int { return int(atomic.AddUint64(&b.counter, uint64(1))) } -func NewBuildState() BuildState { - return BuildState{} -} - type Closer interface { Close() error } diff --git a/deps/deps_test.go b/deps/deps_test.go index d68276732..e92ed2327 100644 --- a/deps/deps_test.go +++ b/deps/deps_test.go @@ -11,17 +11,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deps +package deps_test import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/deps" ) func TestBuildFlags(t *testing.T) { c := qt.New(t) - var bf BuildState + var bf deps.BuildState bf.Incr() bf.Incr() bf.Incr() diff --git a/go.mod b/go.mod index 9acee286d..171d3921f 100644 --- a/go.mod +++ b/go.mod @@ -47,12 +47,12 @@ require ( github.com/niklasfasching/go-org v1.6.6 github.com/olekukonko/tablewriter v0.0.5 github.com/pelletier/go-toml/v2 v2.0.6 - github.com/rogpeppe/go-internal v1.9.0 + github.com/rogpeppe/go-internal v1.10.1-0.20230508101108-a4f6fabd84c5 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/sanity-io/litter v1.5.5 github.com/spf13/afero v1.9.3 github.com/spf13/cast v1.5.1 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/fsync v0.9.0 github.com/spf13/jwalterweatherman v1.1.0 github.com/spf13/pflag v1.0.5 @@ -94,6 +94,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect github.com/aws/smithy-go v1.8.0 // indirect + github.com/bep/helpers v0.4.0 // indirect + github.com/bep/simplecobra v0.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -106,7 +108,7 @@ require ( github.com/googleapis/gax-go/v2 v2.3.0 // indirect github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -119,6 +121,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.3.0 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/oauth2 v0.2.0 // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 29e8d7475..30af63f39 100644 --- a/go.sum +++ b/go.sum @@ -179,10 +179,14 @@ github.com/bep/golibsass v1.1.0 h1:pjtXr00IJZZaOdfryNa9wARTB3Q0BmxC3/V1KNcgyTw= github.com/bep/golibsass v1.1.0/go.mod h1:DL87K8Un/+pWUS75ggYv41bliGiolxzDKWJAq3eJ1MA= github.com/bep/gowebp v0.2.0 h1:ZVfK8i9PpZqKHEmthQSt3qCnnHycbLzBPEsVtk2ch2Q= github.com/bep/gowebp v0.2.0/go.mod h1:ZhFodwdiFp8ehGJpF4LdPl6unxZm9lLFjxD3z2h2AgI= +github.com/bep/helpers v0.4.0 h1:ab9veaAiWY4ST48Oxp5usaqivDmYdB744fz+tcZ3Ifs= +github.com/bep/helpers v0.4.0/go.mod h1:/QpHdmcPagDw7+RjkLFCvnlUc8lQ5kg4KDrEkb2Yyco= github.com/bep/lazycache v0.2.0 h1:HKrlZTrDxHIrNKqmnurH42ryxkngCMYLfBpyu40VcwY= github.com/bep/lazycache v0.2.0/go.mod h1:xUIsoRD824Vx0Q/n57+ZO7kmbEhMBOnTjM/iPixNGbg= github.com/bep/overlayfs v0.6.0 h1:sgLcq/qtIzbaQNl2TldGXOkHvqeZB025sPvHOQL+DYo= github.com/bep/overlayfs v0.6.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM= +github.com/bep/simplecobra v0.2.0 h1:gfdZZ8QlPBMC9R9DRzUsxExR3FyuNtRkqMJqK98SBno= +github.com/bep/simplecobra v0.2.0/go.mod h1:EOp6bCKuuHmwA9bQcRC8LcDB60co2Cmht5X4xMIOwf0= github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI= github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0= github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg= @@ -408,6 +412,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jdkato/prose v1.2.1 h1:Fp3UnJmLVISmlc57BgKUzdjr0lOtjqTZicL3PaYy6cU= @@ -493,6 +499,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.1-0.20230508101108-a4f6fabd84c5 h1:Tb1D114RozKzV2dDfarvSZn8lVYvjcGSCDaMQ+b4I+E= +github.com/rogpeppe/go-internal v1.10.1-0.20230508101108-a4f6fabd84c5/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= @@ -510,6 +518,8 @@ github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/fsync v0.9.0 h1:f9CEt3DOB2mnHxZaftmEOFWjABEvKM/xpf3cUwJrGOY= github.com/spf13/fsync v0.9.0/go.mod h1:fNtJEfG3HiltN3y4cPOz6MLjos9+2pIEqLIgszqhp/0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= @@ -628,6 +638,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/helpers/content.go b/helpers/content.go index d04e34a07..510d496b9 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -50,30 +50,18 @@ type ContentSpec struct { anchorNameSanitizer converter.AnchorNameSanitizer getRenderer func(t hooks.RendererType, id any) any - // SummaryLength is the length of the summary that Hugo extracts from a content. - summaryLength int - - BuildFuture bool - BuildExpired bool - BuildDrafts bool - - Cfg config.Provider + Cfg config.AllProvider } // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs, ex *hexec.Exec) (*ContentSpec, error) { +func NewContentSpec(cfg config.AllProvider, logger loggers.Logger, contentFs afero.Fs, ex *hexec.Exec) (*ContentSpec, error) { spec := &ContentSpec{ - summaryLength: cfg.GetInt("summaryLength"), - BuildFuture: cfg.GetBool("buildFuture"), - BuildExpired: cfg.GetBool("buildExpired"), - BuildDrafts: cfg.GetBool("buildDrafts"), - Cfg: cfg, } converterProvider, err := markup.NewConverterProvider(converter.ProviderConfig{ - Cfg: cfg, + Conf: cfg, ContentFs: contentFs, Logger: logger, Exec: ex, @@ -157,6 +145,9 @@ func (c *ContentSpec) SanitizeAnchorName(s string) string { } func (c *ContentSpec) ResolveMarkup(in string) string { + if c == nil { + panic("nil ContentSpec") + } in = strings.ToLower(in) switch in { case "md", "markdown", "mdown": @@ -194,17 +185,17 @@ func (c *ContentSpec) TruncateWordsByRune(in []string) (string, bool) { count := 0 for index, word := range words { - if count >= c.summaryLength { + if count >= c.Cfg.SummaryLength() { return strings.Join(words[:index], " "), true } runeCount := utf8.RuneCountInString(word) if len(word) == runeCount { count++ - } else if count+runeCount < c.summaryLength { + } else if count+runeCount < c.Cfg.SummaryLength() { count += runeCount } else { for ri := range word { - if count >= c.summaryLength { + if count >= c.Cfg.SummaryLength() { truncatedWords := append(words[:index], word[:ri]) return strings.Join(truncatedWords, " "), true } @@ -229,7 +220,7 @@ func (c *ContentSpec) TruncateWordsToWholeSentence(s string) (string, bool) { wordCount++ lastWordIndex = i - if wordCount >= c.summaryLength { + if wordCount >= c.Cfg.SummaryLength() { break } @@ -283,19 +274,19 @@ func isEndOfSentence(r rune) bool { func (c *ContentSpec) truncateWordsToWholeSentenceOld(content string) (string, bool) { words := strings.Fields(content) - if c.summaryLength >= len(words) { + if c.Cfg.SummaryLength() >= len(words) { return strings.Join(words, " "), false } - for counter, word := range words[c.summaryLength:] { + for counter, word := range words[c.Cfg.SummaryLength():] { if strings.HasSuffix(word, ".") || strings.HasSuffix(word, "?") || strings.HasSuffix(word, ".\"") || strings.HasSuffix(word, "!") { - upper := c.summaryLength + counter + 1 + upper := c.Cfg.SummaryLength() + counter + 1 return strings.Join(words[:upper], " "), (upper < len(words)) } } - return strings.Join(words[:c.summaryLength], " "), true + return strings.Join(words[:c.Cfg.SummaryLength()], " "), true } diff --git a/helpers/content_test.go b/helpers/content_test.go index 54b7ef3f9..2909c0266 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2023 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package helpers +package helpers_test import ( "bytes" @@ -19,12 +19,9 @@ import ( "strings" "testing" - "github.com/spf13/afero" - - "github.com/gohugoio/hugo/common/loggers" - "github.com/gohugoio/hugo/config" - qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" ) const tstHTMLContent = "

This is some text.
And some more.

" @@ -43,7 +40,7 @@ func TestTrimShortHTML(t *testing.T) { {[]byte("

Hello

\n
    \n
  • list1
    • \n
  • list2
    • \n
"), []byte("

Hello

\n
    \n
  • list1
    • \n
  • list2
    • \n
")}, } - c := newTestContentSpec() + c := newTestContentSpec(nil) for i, test := range tests { output := c.TrimShortHTML(test.input) if !bytes.Equal(test.output, output) { @@ -52,55 +49,23 @@ func TestTrimShortHTML(t *testing.T) { } } -func TestStripEmptyNav(t *testing.T) { - c := qt.New(t) - cleaned := stripEmptyNav([]byte("do\n\nbedobedo")) - c.Assert(cleaned, qt.DeepEquals, []byte("dobedobedo")) -} - func TestBytesToHTML(t *testing.T) { c := qt.New(t) - c.Assert(BytesToHTML([]byte("dobedobedo")), qt.Equals, template.HTML("dobedobedo")) -} - -func TestNewContentSpec(t *testing.T) { - cfg := config.NewWithTestDefaults() - c := qt.New(t) - - cfg.Set("summaryLength", 32) - cfg.Set("buildFuture", true) - cfg.Set("buildExpired", true) - cfg.Set("buildDrafts", true) - - spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) - - c.Assert(err, qt.IsNil) - c.Assert(spec.summaryLength, qt.Equals, 32) - c.Assert(spec.BuildFuture, qt.Equals, true) - c.Assert(spec.BuildExpired, qt.Equals, true) - c.Assert(spec.BuildDrafts, qt.Equals, true) + c.Assert(helpers.BytesToHTML([]byte("dobedobedo")), qt.Equals, template.HTML("dobedobedo")) } var benchmarkTruncateString = strings.Repeat("This is a sentence about nothing.", 20) func BenchmarkTestTruncateWordsToWholeSentence(b *testing.B) { - c := newTestContentSpec() + c := newTestContentSpec(nil) b.ResetTimer() for i := 0; i < b.N; i++ { c.TruncateWordsToWholeSentence(benchmarkTruncateString) } } -func BenchmarkTestTruncateWordsToWholeSentenceOld(b *testing.B) { - c := newTestContentSpec() - b.ResetTimer() - for i := 0; i < b.N; i++ { - c.truncateWordsToWholeSentenceOld(benchmarkTruncateString) - } -} - func TestTruncateWordsToWholeSentence(t *testing.T) { - c := newTestContentSpec() + type test struct { input, expected string max int @@ -118,7 +83,9 @@ func TestTruncateWordsToWholeSentence(t *testing.T) { {"This... is a more difficult test?", "This... is a more difficult test?", 1, false}, } for i, d := range data { - c.summaryLength = d.max + cfg := config.New() + cfg.Set("summaryLength", d.max) + c := newTestContentSpec(cfg) output, truncated := c.TruncateWordsToWholeSentence(d.input) if d.expected != output { t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, output) @@ -131,7 +98,7 @@ func TestTruncateWordsToWholeSentence(t *testing.T) { } func TestTruncateWordsByRune(t *testing.T) { - c := newTestContentSpec() + type test struct { input, expected string max int @@ -153,7 +120,9 @@ func TestTruncateWordsByRune(t *testing.T) { {" \nThis is not a sentence\n ", "This is not", 3, true}, } for i, d := range data { - c.summaryLength = d.max + cfg := config.New() + cfg.Set("summaryLength", d.max) + c := newTestContentSpec(cfg) output, truncated := c.TruncateWordsByRune(strings.Fields(d.input)) if d.expected != output { t.Errorf("Test %d failed. Expected %q got %q", i, d.expected, output) @@ -168,7 +137,7 @@ func TestTruncateWordsByRune(t *testing.T) { func TestExtractTOCNormalContent(t *testing.T) { content := []byte("