From 85ba9bfffba9bfd0b095cb766f72700d4c211e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 9 Sep 2020 22:31:43 +0200 Subject: [PATCH] Add "hugo mod npm pack" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit also introduces a convention where these common JS config files, including `package.hugo.json`, gets mounted into: ``` assets/_jsconfig ยด`` These files mapped to their real filename will be added to the environment when running PostCSS, Babel etc., so you can do `process.env.HUGO_FILE_TAILWIND_CONFIG_JS` to resolve the real filename. But do note that `assets` is a composite/union filesystem, so if your config file is not meant to be overridden, name them something specific. This commit also adds adds `workDir/node_modules` to `NODE_PATH` and `HUGO_WORKDIR` to the env when running the JS tools above. Fixes #7644 Fixes #7656 Fixes #7675 --- commands/mod.go | 15 +- commands/mod_npm.go | 58 +++++ common/hugo/hugo.go | 24 +- docs/content/en/hugo-pipes/babel.md | 24 ++ hugofs/files/classifier.go | 10 + hugofs/rootmapping_fs.go | 3 - hugofs/walk_test.go | 8 - hugolib/filesystems/basefs.go | 35 ++- hugolib/hugo_modules_test.go | 159 +++++++++++- hugolib/resource_chain_test.go | 6 + modules/collect.go | 43 ++++ modules/config.go | 13 +- modules/npm/package_builder.go | 230 ++++++++++++++++++ modules/npm/package_builder_test.go | 95 ++++++++ .../resource_transformers/babel/babel.go | 25 +- .../resource_transformers/postcss/postcss.go | 19 +- 16 files changed, 721 insertions(+), 46 deletions(-) create mode 100644 commands/mod_npm.go create mode 100644 modules/npm/package_builder.go create mode 100644 modules/npm/package_builder_test.go diff --git a/commands/mod.go b/commands/mod.go index 81f660f43..b390d1e75 100644 --- a/commands/mod.go +++ b/commands/mod.go @@ -1,4 +1,4 @@ -// Copyright 2019 The Hugo Authors. All rights reserved. +// Copyright 2020 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ import ( "path/filepath" "regexp" + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/modules" "github.com/spf13/cobra" ) @@ -114,6 +116,8 @@ This is not needed if you only operate on modules inside /themes or if you have RunE: nil, } + cmd.AddCommand(newModNPMCmd(c)) + cmd.AddCommand( &cobra.Command{ Use: "get", @@ -272,6 +276,15 @@ func (c *modCmd) withModsClient(failOnMissingConfig bool, f func(*modules.Client return f(com.hugo().ModulesClient) } +func (c *modCmd) withHugo(f func(*hugolib.HugoSites) error) error { + com, err := c.initConfig(true) + if err != nil { + return err + } + + return f(com.hugo()) +} + func (c *modCmd) initConfig(failOnNoConfig bool) (*commandeer, error) { com, err := initializeConfig(failOnNoConfig, false, &c.hugoBuilderCommon, c, nil) if err != nil { diff --git a/commands/mod_npm.go b/commands/mod_npm.go new file mode 100644 index 000000000..15c875d8e --- /dev/null +++ b/commands/mod_npm.go @@ -0,0 +1,58 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "github.com/gohugoio/hugo/hugolib" + "github.com/gohugoio/hugo/modules/npm" + "github.com/spf13/cobra" +) + +func newModNPMCmd(c *modCmd) *cobra.Command { + + cmd := &cobra.Command{ + Use: "npm", + Short: "Various npm helpers.", + Long: `Various npm (Node package manager) helpers.`, + RunE: func(cmd *cobra.Command, args []string) error { + return c.withHugo(func(h *hugolib.HugoSites) error { + return nil + }) + }, + } + + cmd.AddCommand(&cobra.Command{ + Use: "pack", + Short: "Experimental: Prepares and writes a composite package.json file for your project.", + Long: `Prepares and writes a composite package.json file for your project. + +On first run it creates a "package.hugo.json" in the project root if not alread there. This file will be used as a template file +with the base dependency set. + +This set will be merged with all "package.hugo.json" files found in the dependency tree, picking the version closest to the project. + +This command is marked as 'Experimental'. We think it's a great idea, so it's not likely to be +removed from Hugo, but we need to test this out in "real life" to get a feel of it, +so this may/will change in future versions of Hugo. +`, + RunE: func(cmd *cobra.Command, args []string) error { + + return c.withHugo(func(h *hugolib.HugoSites) error { + return npm.Pack(h.BaseFs.SourceFs, h.BaseFs.Assets.Dirs) + }) + }, + }) + + return cmd +} diff --git a/common/hugo/hugo.go b/common/hugo/hugo.go index 6e07f69c3..ac75e6bca 100644 --- a/common/hugo/hugo.go +++ b/common/hugo/hugo.go @@ -17,8 +17,15 @@ import ( "fmt" "html/template" "os" + "path/filepath" + "strings" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/spf13/afero" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/hugofs" ) const ( @@ -73,8 +80,23 @@ func NewInfo(environment string) Info { } } -func GetExecEnviron(cfg config.Provider) []string { +func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string { env := os.Environ() + nodepath := filepath.Join(workDir, "node_modules") + if np := os.Getenv("NODE_PATH"); np != "" { + nodepath = workDir + string(os.PathListSeparator) + np + } + config.SetEnvVars(&env, "NODE_PATH", nodepath) + config.SetEnvVars(&env, "HUGO_WORKDIR", workDir) config.SetEnvVars(&env, "HUGO_ENVIRONMENT", cfg.GetString("environment")) + fis, err := afero.ReadDir(fs, files.FolderJSConfig) + if err == nil { + for _, fi := range fis { + key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_")) + value := fi.(hugofs.FileMetaInfo).Meta().Filename() + config.SetEnvVars(&env, key, value) + } + } + return env } diff --git a/docs/content/en/hugo-pipes/babel.md b/docs/content/en/hugo-pipes/babel.md index 5fb5e11e1..9688626d9 100755 --- a/docs/content/en/hugo-pipes/babel.md +++ b/docs/content/en/hugo-pipes/babel.md @@ -24,6 +24,30 @@ Hugo Pipe's Babel requires the `@babel/cli` and `@babel/core` JavaScript package If you are using the Hugo Snap package, Babel and plugin(s) need to be installed locally within your Hugo site directory, e.g., `npm install @babel/cli @babel/core --save-dev` without the `-g` flag. {{% /note %}} + +### Config + +{{< new-in "v0.75.0" >}} + +In Hugo `v0.75` we improved the way we resolve JS configuration and dependencies. One of them is that we now adds the main project's `node_modules` to `NODE_PATH` when running Babel and similar tools. There are some known [issues](https://github.com/babel/babel/issues/5618) with Babel in this area, so if you have a `babel.config.js` living in a Hugo Module (and not in the project itself), we recommend using `require` to load the presets/plugins, e.g.: + + +```js +module.exports = { + presets: [ + [ + require('@babel/preset-env'), + { + useBuiltIns: 'entry', + corejs: 3 + } + ] + ] +}; +``` + + + ### Options config [string] diff --git a/hugofs/files/classifier.go b/hugofs/files/classifier.go index 5e26bbac0..35e416c8f 100644 --- a/hugofs/files/classifier.go +++ b/hugofs/files/classifier.go @@ -26,6 +26,13 @@ import ( "github.com/spf13/afero" ) +const ( + // The NPM package.json "template" file. + FilenamePackageHugoJSON = "package.hugo.json" + // The NPM package file. + FilenamePackageJSON = "package.json" +) + var ( // This should be the only list of valid extensions for content files. contentFileExtensions = []string{ @@ -163,9 +170,12 @@ const ( ComponentFolderI18n = "i18n" FolderResources = "resources" + FolderJSConfig = "_jsconfig" // Mounted below /assets with postcss.config.js etc. ) var ( + JsConfigFolderMountPrefix = filepath.Join(ComponentFolderAssets, FolderJSConfig) + ComponentFolders = []string{ ComponentFolderArchetypes, ComponentFolderStatic, diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index ea3ef003e..2c4f0df52 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -42,9 +42,6 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { (&rm).clean() fromBase := files.ResolveComponentFolder(rm.From) - if fromBase == "" { - panic("unrecognised component folder in" + rm.From) - } if len(rm.To) < 2 { panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go index 0c08968c6..c38c6044f 100644 --- a/hugofs/walk_test.go +++ b/hugofs/walk_test.go @@ -21,8 +21,6 @@ import ( "strings" "testing" - "github.com/gohugoio/hugo/common/hugo" - "github.com/pkg/errors" "github.com/gohugoio/hugo/htesting" @@ -129,12 +127,6 @@ func TestWalkSymbolicLink(t *testing.T) { }) t.Run("BasePath Fs", func(t *testing.T) { - if hugo.GoMinorVersion() < 12 { - // https://github.com/golang/go/issues/30520 - // This is fixed in Go 1.13 and in the latest Go 1.12 - t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") - - } c := qt.New(t) docsFs := afero.NewBasePathFs(fs, docsDir) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 57a95a037..76d49055c 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -49,6 +49,9 @@ type BaseFs struct { // SourceFilesystems contains the different source file systems. *SourceFilesystems + // The project source. + SourceFs afero.Fs + // The filesystem used to publish the rendered site. // This usually maps to /my-project/public. PublishFs afero.Fs @@ -100,6 +103,23 @@ func (b *BaseFs) RelContentDir(filename string) string { return filename } +// ResolveJSConfigFile resolves the JS-related config file to a absolute +// filename. One example of such would be postcss.config.js. +func (fs *BaseFs) ResolveJSConfigFile(name string) string { + // First look in assets/_jsconfig + fi, err := fs.Assets.Fs.Stat(filepath.Join(files.FolderJSConfig, name)) + if err == nil { + return fi.(hugofs.FileMetaInfo).Meta().Filename() + } + // Fall back to the work dir. + fi, err = fs.Work.Stat(name) + if err == nil { + return fi.(hugofs.FileMetaInfo).Meta().Filename() + } + + return "" +} + // SourceFilesystems contains the different source file systems. These can be // composite file systems (theme and project etc.), and they have all root // set to the source type the provides: data, i18n, static, layouts. @@ -346,8 +366,10 @@ func NewBase(p *paths.Paths, logger *loggers.Logger, options ...func(*BaseFs) er } publishFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)) + sourceFs := hugofs.NewBaseFileDecorator(afero.NewBasePathFs(fs.Source, p.WorkingDir)) b := &BaseFs{ + SourceFs: sourceFs, PublishFs: publishFs, } @@ -696,11 +718,16 @@ type filesystemsCollector struct { func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) { for _, componentFolder := range files.ComponentFolders { - dirs, err := rfs.Dirs(componentFolder) + c.addDir(rfs, componentFolder) + } - if err == nil { - c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) - } +} + +func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder string) { + dirs, err := rfs.Dirs(componentFolder) + + if err == nil { + c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...) } } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 0ed4fceb0..c3358a0c2 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -22,6 +22,8 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/modules/npm" + "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" @@ -38,7 +40,6 @@ import ( "github.com/spf13/viper" ) -// https://github.com/gohugoio/hugo/issues/6730 func TestHugoModulesVariants(t *testing.T) { if !isCI() { t.Skip("skip (relative) long running modules test when running locally") @@ -60,8 +61,10 @@ path="github.com/gohugoio/hugoTestModule2" newTestBuilder := func(t testing.TB, moduleOpts string) (*sitesBuilder, func()) { b := newTestSitesBuilder(t) - workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-variants") + tempDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-variants") b.Assert(err, qt.IsNil) + workingDir := filepath.Join(tempDir, "myhugosite") + b.Assert(os.MkdirAll(workingDir, 0777), qt.IsNil) b.Fs = hugofs.NewDefault(viper.New()) b.WithWorkingDir(workingDir).WithConfigFile("toml", createConfig(workingDir, moduleOpts)) b.WithTemplates( @@ -129,6 +132,158 @@ JS imported in module: | `) }) + t.Run("Create package.json", func(t *testing.T) { + + b, clean := newTestBuilder(t, "") + defer clean() + + b.WithSourceFile("package.json", `{ + "name": "mypack", + "version": "1.2.3", + "scripts": {}, + "dependencies": { + "nonon": "error" + } +}`) + + b.WithSourceFile("package.hugo.json", `{ + "name": "mypack", + "version": "1.2.3", + "scripts": {}, + "dependencies": { + "foo": "1.2.3" + }, + "devDependencies": { + "postcss-cli": "7.8.0", + "tailwindcss": "1.8.0" + + } +}`) + + b.Build(BuildCfg{}) + b.Assert(npm.Pack(b.H.BaseFs.SourceFs, b.H.BaseFs.Assets.Dirs), qt.IsNil) + + b.AssertFileContentFn("package.json", func(s string) bool { + return s == `{ + "comments": { + "dependencies": { + "foo": "project", + "react-dom": "github.com/gohugoio/hugoTestModule2" + }, + "devDependencies": { + "@babel/cli": "github.com/gohugoio/hugoTestModule2", + "@babel/core": "github.com/gohugoio/hugoTestModule2", + "@babel/preset-env": "github.com/gohugoio/hugoTestModule2", + "postcss-cli": "project", + "tailwindcss": "project" + } + }, + "dependencies": { + "foo": "1.2.3", + "react-dom": "^16.13.1" + }, + "devDependencies": { + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.8.0", + "tailwindcss": "1.8.0" + }, + "name": "mypack", + "scripts": {}, + "version": "1.2.3" +}` + }) + }) + + t.Run("Create package.json, no default", func(t *testing.T) { + + b, clean := newTestBuilder(t, "") + defer clean() + + b.WithSourceFile("package.json", `{ + "name": "mypack", + "version": "1.2.3", + "scripts": {}, + "dependencies": { + "moo": "1.2.3" + } +}`) + + b.Build(BuildCfg{}) + b.Assert(npm.Pack(b.H.BaseFs.SourceFs, b.H.BaseFs.Assets.Dirs), qt.IsNil) + + b.AssertFileContentFn("package.json", func(s string) bool { + return s == `{ + "comments": { + "dependencies": { + "moo": "project", + "react-dom": "github.com/gohugoio/hugoTestModule2" + }, + "devDependencies": { + "@babel/cli": "github.com/gohugoio/hugoTestModule2", + "@babel/core": "github.com/gohugoio/hugoTestModule2", + "@babel/preset-env": "github.com/gohugoio/hugoTestModule2", + "postcss-cli": "github.com/gohugoio/hugoTestModule2", + "tailwindcss": "github.com/gohugoio/hugoTestModule2" + } + }, + "dependencies": { + "moo": "1.2.3", + "react-dom": "^16.13.1" + }, + "devDependencies": { + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.1.0", + "tailwindcss": "1.2.0" + }, + "name": "mypack", + "scripts": {}, + "version": "1.2.3" +}` + }) + }) + + t.Run("Create package.json, no default, no package.json", func(t *testing.T) { + + b, clean := newTestBuilder(t, "") + defer clean() + + b.Build(BuildCfg{}) + b.Assert(npm.Pack(b.H.BaseFs.SourceFs, b.H.BaseFs.Assets.Dirs), qt.IsNil) + + b.AssertFileContentFn("package.json", func(s string) bool { + return s == `{ + "comments": { + "dependencies": { + "react-dom": "github.com/gohugoio/hugoTestModule2" + }, + "devDependencies": { + "@babel/cli": "github.com/gohugoio/hugoTestModule2", + "@babel/core": "github.com/gohugoio/hugoTestModule2", + "@babel/preset-env": "github.com/gohugoio/hugoTestModule2", + "postcss-cli": "github.com/gohugoio/hugoTestModule2", + "tailwindcss": "github.com/gohugoio/hugoTestModule2" + } + }, + "dependencies": { + "react-dom": "^16.13.1" + }, + "devDependencies": { + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.1.0", + "tailwindcss": "1.2.0" + }, + "name": "myhugosite", + "version": "0.1.0" +}` + }) + }) + } // TODO(bep) this fails when testmodBuilder is also building ... diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 64484f1f4..7573199aa 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -873,6 +873,10 @@ func TestResourceChainPostCSS(t *testing.T) { postcssConfig := ` console.error("Hugo Environment:", process.env.HUGO_ENVIRONMENT ); +// https://github.com/gohugoio/hugo/issues/7656 +console.error("package.json:", process.env.HUGO_FILE_PACKAGE_JSON ); +console.error("PostCSS Config File:", process.env.HUGO_FILE_POSTCSS_CONFIG_JS ); + module.exports = { plugins: [ @@ -954,6 +958,8 @@ class-in-b { // Make sure Node sees this. b.Assert(logBuf.String(), qt.Contains, "Hugo Environment: production") + b.Assert(logBuf.String(), qt.Contains, fmt.Sprintf("PostCSS Config File: %s/postcss.config.js", workDir)) + b.Assert(logBuf.String(), qt.Contains, fmt.Sprintf("package.json: %s/package.json", workDir)) b.AssertFileContent("public/index.html", ` Styles RelPermalink: /css/styles.css diff --git a/modules/collect.go b/modules/collect.go index b82d395fd..8959572d6 100644 --- a/modules/collect.go +++ b/modules/collect.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "time" @@ -382,6 +383,11 @@ func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error { return err } + mounts, err = c.mountCommonJSConfig(mod, mounts) + if err != nil { + return err + } + mod.mounts = mounts return nil } @@ -549,6 +555,43 @@ func (c *collector) loadModules() error { return nil } +// Matches postcss.config.js etc. +var commonJSConfigs = regexp.MustCompile(`(babel|postcss|tailwind)\.config\.js`) + +func (c *collector) mountCommonJSConfig(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { + for _, m := range mounts { + if strings.HasPrefix(m.Target, files.JsConfigFolderMountPrefix) { + // This follows the convention of the other component types (assets, content, etc.), + // if one or more is specificed by the user, we skip the defaults. + // These mounts were added to Hugo in 0.75. + return mounts, nil + } + } + + // Mount the common JS config files. + fis, err := afero.ReadDir(c.fs, owner.Dir()) + if err != nil { + return mounts, err + } + + for _, fi := range fis { + n := fi.Name() + + should := n == files.FilenamePackageHugoJSON || n == files.FilenamePackageJSON + should = should || commonJSConfigs.MatchString(n) + + if should { + mounts = append(mounts, Mount{ + Source: n, + Target: filepath.Join(files.ComponentFolderAssets, files.FolderJSConfig, n), + }) + } + + } + + return mounts, nil +} + func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mount, error) { var out []Mount dir := owner.Dir() diff --git a/modules/config.go b/modules/config.go index 1964479f4..e0a0ea060 100644 --- a/modules/config.go +++ b/modules/config.go @@ -56,7 +56,9 @@ func ApplyProjectConfigDefaults(cfg config.Provider, mod Module) error { // the basic level. componentsConfigured := make(map[string]bool) for _, mnt := range moda.mounts { - componentsConfigured[mnt.Component()] = true + if !strings.HasPrefix(mnt.Target, files.JsConfigFolderMountPrefix) { + componentsConfigured[mnt.Component()] = true + } } type dirKeyComponent struct { @@ -318,12 +320,21 @@ type Mount struct { Target string // relative target path, e.g. "assets/bootstrap/scss" Lang string // any language code associated with this mount. + } func (m Mount) Component() string { return strings.Split(m.Target, fileSeparator)[0] } +func (m Mount) ComponentAndName() (string, string) { + k := strings.Index(m.Target, fileSeparator) + if k == -1 { + return m.Target, "" + } + return m.Target[:k], m.Target[k+1:] +} + func getStaticDirs(cfg config.Provider) []string { var staticDirs []string for i := -1; i <= 10; i++ { diff --git a/modules/npm/package_builder.go b/modules/npm/package_builder.go new file mode 100644 index 000000000..23aac7246 --- /dev/null +++ b/modules/npm/package_builder.go @@ -0,0 +1,230 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package npm + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/gohugoio/hugo/hugofs/files" + + "github.com/pkg/errors" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/afero" + + "github.com/spf13/cast" + + "github.com/gohugoio/hugo/helpers" +) + +const ( + dependenciesKey = "dependencies" + devDependenciesKey = "devDependencies" + + packageJSONName = "package.json" + + packageJSONTemplate = `{ + "name": "%s", + "version": "%s" +}` +) + +func Pack(fs afero.Fs, fis []hugofs.FileMetaInfo) error { + + var b *packageBuilder + + // Have a package.hugo.json? + fi, err := fs.Stat(files.FilenamePackageHugoJSON) + if err != nil { + // Have a package.json? + fi, err = fs.Stat(packageJSONName) + if err != nil { + // Create one. + name := "project" + // Use the Hugo site's folder name as the default name. + // The owner can change it later. + rfi, err := fs.Stat("") + if err == nil { + name = rfi.Name() + } + packageJSONContent := fmt.Sprintf(packageJSONTemplate, name, "0.1.0") + if err = afero.WriteFile(fs, files.FilenamePackageHugoJSON, []byte(packageJSONContent), 0666); err != nil { + return err + } + fi, err = fs.Stat(files.FilenamePackageHugoJSON) + if err != nil { + return err + } + } + } + + meta := fi.(hugofs.FileMetaInfo).Meta() + masterFilename := meta.Filename() + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "npm pack: failed to open package file") + } + b = newPackageBuilder(meta.Module(), f) + f.Close() + + for _, fi := range fis { + if fi.IsDir() { + // We only care about the files in the root. + continue + } + + if fi.Name() != files.FilenamePackageHugoJSON { + continue + } + + meta := fi.(hugofs.FileMetaInfo).Meta() + + if meta.Filename() == masterFilename { + continue + } + + f, err := meta.Open() + if err != nil { + return errors.Wrap(err, "npm pack: failed to open package file") + } + b.Add(meta.Module(), f) + f.Close() + } + + if b.Err() != nil { + return errors.Wrap(b.Err(), "npm pack: failed to build") + } + + // Replace the dependencies in the original template with the merged set. + b.originalPackageJSON[dependenciesKey] = b.dependencies + b.originalPackageJSON[devDependenciesKey] = b.devDependencies + var commentsm map[string]interface{} + comments, found := b.originalPackageJSON["comments"] + if found { + commentsm = cast.ToStringMap(comments) + } else { + commentsm = make(map[string]interface{}) + } + commentsm[dependenciesKey] = b.dependenciesComments + commentsm[devDependenciesKey] = b.devDependenciesComments + b.originalPackageJSON["comments"] = commentsm + + // Write it out to the project package.json + packageJSONData, err := json.MarshalIndent(b.originalPackageJSON, "", " ") + if err != nil { + return errors.Wrap(err, "npm pack: failed to marshal JSON") + } + + if err := afero.WriteFile(fs, packageJSONName, packageJSONData, 0666); err != nil { + return errors.Wrap(err, "npm pack: failed to write package.json") + } + + return nil + +} + +func newPackageBuilder(source string, first io.Reader) *packageBuilder { + b := &packageBuilder{ + devDependencies: make(map[string]interface{}), + devDependenciesComments: make(map[string]interface{}), + dependencies: make(map[string]interface{}), + dependenciesComments: make(map[string]interface{}), + } + + m := b.unmarshal(first) + if b.err != nil { + return b + } + + b.addm(source, m) + b.originalPackageJSON = m + + return b +} + +type packageBuilder struct { + err error + + // The original package.hugo.json. + originalPackageJSON map[string]interface{} + + devDependencies map[string]interface{} + devDependenciesComments map[string]interface{} + dependencies map[string]interface{} + dependenciesComments map[string]interface{} +} + +func (b *packageBuilder) Add(source string, r io.Reader) *packageBuilder { + if b.err != nil { + return b + } + + m := b.unmarshal(r) + if b.err != nil { + return b + } + + b.addm(source, m) + + return b +} + +func (b *packageBuilder) addm(source string, m map[string]interface{}) { + if source == "" { + source = "project" + } + + // The version selection is currently very simple. + // We may consider minimal version selection or something + // after testing this out. + // + // But for now, the first version string for a given dependency wins. + // These packages will be added by order of import (project, module1, module2...), + // so that should at least give the project control over the situation. + if devDeps, found := m[devDependenciesKey]; found { + mm := cast.ToStringMapString(devDeps) + for k, v := range mm { + if _, added := b.devDependencies[k]; !added { + b.devDependencies[k] = v + b.devDependenciesComments[k] = source + } + } + } + + if deps, found := m[dependenciesKey]; found { + mm := cast.ToStringMapString(deps) + for k, v := range mm { + if _, added := b.dependencies[k]; !added { + b.dependencies[k] = v + b.dependenciesComments[k] = source + } + } + } + +} + +func (b *packageBuilder) unmarshal(r io.Reader) map[string]interface{} { + m := make(map[string]interface{}) + err := json.Unmarshal(helpers.ReaderToBytes(r), &m) + if err != nil { + b.err = err + } + return m +} + +func (b *packageBuilder) Err() error { + return b.err +} diff --git a/modules/npm/package_builder_test.go b/modules/npm/package_builder_test.go new file mode 100644 index 000000000..510a04776 --- /dev/null +++ b/modules/npm/package_builder_test.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package npm + +import ( + "strings" + "testing" + + qt "github.com/frankban/quicktest" +) + +const templ = `{ + "name": "foo", + "version": "0.1.1", + "scripts": {}, + "dependencies": { + "react-dom": "1.1.1", + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5" + }, + "devDependencies": { + "postcss-cli": "7.1.0", + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5" + } +}` + +func TestPackageBuilder(t *testing.T) { + c := qt.New(t) + + b := newPackageBuilder("", strings.NewReader(templ)) + c.Assert(b.Err(), qt.IsNil) + + b.Add("mymod", strings.NewReader(`{ +"dependencies": { + "react-dom": "9.1.1", + "add1": "1.1.1" +}, +"devDependencies": { + "tailwindcss": "error", + "add2": "2.1.1" +} +}`)) + + b.Add("mymod", strings.NewReader(`{ +"dependencies": { + "react-dom": "error", + "add1": "error", + "add3": "3.1.1" +}, +"devDependencies": { + "tailwindcss": "error", + "add2": "error", + "add4": "4.1.1" + +} +}`)) + + c.Assert(b.Err(), qt.IsNil) + + c.Assert(b.dependencies, qt.DeepEquals, map[string]interface{}{ + "@babel/cli": "7.8.4", + "add1": "1.1.1", + "add3": "3.1.1", + "@babel/core": "7.9.0", + "@babel/preset-env": "7.9.5", + "react-dom": "1.1.1", + "tailwindcss": "1.2.0", + }) + + c.Assert(b.devDependencies, qt.DeepEquals, map[string]interface{}{ + "tailwindcss": "1.2.0", + "@babel/cli": "7.8.4", + "@babel/core": "7.9.0", + "add2": "2.1.1", + "add4": "4.1.1", + "@babel/preset-env": "7.9.5", + "postcss-cli": "7.1.0", + }) +} diff --git a/resources/resource_transformers/babel/babel.go b/resources/resource_transformers/babel/babel.go index c5ab48dd8..4255210c3 100644 --- a/resources/resource_transformers/babel/babel.go +++ b/resources/resource_transformers/babel/babel.go @@ -14,6 +14,7 @@ package babel import ( + "bytes" "io" "os/exec" "path/filepath" @@ -27,7 +28,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/gohugoio/hugo/common/herrors" - "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" "github.com/pkg/errors" @@ -120,6 +120,9 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx var configFile string logger := t.rs.Logger + var errBuf bytes.Buffer + infoW := loggers.LoggerToWriterWithPrefix(logger.INFO, "babel") + if t.options.Config != "" { configFile = t.options.Config } else { @@ -130,16 +133,10 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx // We need an abolute filename to the config file. if !filepath.IsAbs(configFile) { - // We resolve this against the virtual Work filesystem, to allow - // this config file to live in one of the themes if needed. - fi, err := t.rs.BaseFs.Work.Stat(configFile) - if err != nil { - if t.options.Config != "" { - // Only fail if the user specificed config file is not found. - return errors.Wrapf(err, "babel config %q not found:", configFile) - } - } else { - configFile = fi.(hugofs.FileMetaInfo).Meta().Filename() + configFile = t.rs.BaseFs.ResolveJSConfigFile(configFile) + if configFile == "" && t.options.Config != "" { + // Only fail if the user specificed config file is not found. + return errors.Errorf("babel config %q not found:", configFile) } } @@ -158,8 +155,8 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx cmd := exec.Command(binary, cmdArgs...) cmd.Stdout = ctx.To - cmd.Stderr = loggers.LoggerToWriterWithPrefix(logger.INFO, "babel") - cmd.Env = hugo.GetExecEnviron(t.rs.Cfg) + cmd.Stderr = io.MultiWriter(infoW, &errBuf) + cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) stdin, err := cmd.StdinPipe() if err != nil { @@ -173,7 +170,7 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx err = cmd.Run() if err != nil { - return err + return errors.Wrap(err, errBuf.String()) } return nil diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index 258185a40..41472fe98 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -170,17 +170,11 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC // We need an abolute filename to the config file. if !filepath.IsAbs(configFile) { - // We resolve this against the virtual Work filesystem, to allow - // this config file to live in one of the themes if needed. - fi, err := t.rs.BaseFs.Work.Stat(configFile) - if err != nil { - if t.options.Config != "" { - // Only fail if the user specificed config file is not found. - return errors.Wrapf(err, "postcss config %q not found:", configFile) - } - configFile = "" - } else { - configFile = fi.(hugofs.FileMetaInfo).Meta().Filename() + configFile = t.rs.BaseFs.ResolveJSConfigFile(configFile) + if configFile == "" && t.options.Config != "" { + // Only fail if the user specificed config file is not found. + return errors.Errorf("postcss config %q not found:", configFile) + } } @@ -202,7 +196,8 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC cmd.Stdout = ctx.To cmd.Stderr = io.MultiWriter(infoW, &errBuf) - cmd.Env = hugo.GetExecEnviron(t.rs.Cfg) + + cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) stdin, err := cmd.StdinPipe() if err != nil {