From d4611c4322dabfd8d2520232be578388029867db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Wed, 9 Sep 2020 21:10:28 +0200 Subject: [PATCH] modules: Add noVendor to module config Fixes #7647 --- cache/filecache/filecache_test.go | 5 +- docs/content/en/hugo-modules/configuration.md | 5 + hugolib/filesystems/basefs_test.go | 5 +- modules/client.go | 19 ++- modules/client_test.go | 144 +++++++++++------- modules/config.go | 4 + 6 files changed, 122 insertions(+), 60 deletions(-) diff --git a/cache/filecache/filecache_test.go b/cache/filecache/filecache_test.go index 5a5dac983..00b4e5573 100644 --- a/cache/filecache/filecache_test.go +++ b/cache/filecache/filecache_test.go @@ -25,6 +25,8 @@ import ( "testing" "time" + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/modules" @@ -314,12 +316,13 @@ func initConfig(fs afero.Fs, cfg config.Provider) error { if !filepath.IsAbs(themesDir) { themesDir = filepath.Join(workingDir, themesDir) } + globAll := glob.MustCompile("**", '/') modulesClient := modules.NewClient(modules.ClientConfig{ Fs: fs, WorkingDir: workingDir, ThemesDir: themesDir, ModuleConfig: modConfig, - IgnoreVendor: true, + IgnoreVendor: globAll, }) moduleConfig, err := modulesClient.Collect() diff --git a/docs/content/en/hugo-modules/configuration.md b/docs/content/en/hugo-modules/configuration.md index c97e8d5a2..2fb4ec5e6 100644 --- a/docs/content/en/hugo-modules/configuration.md +++ b/docs/content/en/hugo-modules/configuration.md @@ -18,11 +18,16 @@ toc: true {{< code-toggle file="config">}} [module] +noVendor = "" proxy = "direct" noProxy = "none" private = "*.*" {{< /code-toggle >}} + +noVendor {{< new-in "0.75.0" >}} +: A optional Glob pattern matching module paths to skip when vendoring, e.g. "github.com/**" + proxy : Defines the proxy server to use to download remote modules. Default is `direct`, which means "git clone" and similar. diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index e3222af48..633c8fe08 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -21,6 +21,8 @@ import ( "strings" "testing" + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/langs" @@ -49,12 +51,13 @@ func initConfig(fs afero.Fs, cfg config.Provider) error { if !filepath.IsAbs(themesDir) { themesDir = filepath.Join(workingDir, themesDir) } + globAll := glob.MustCompile("**", '/') modulesClient := modules.NewClient(modules.ClientConfig{ Fs: fs, WorkingDir: workingDir, ThemesDir: themesDir, ModuleConfig: modConfig, - IgnoreVendor: true, + IgnoreVendor: globAll, }) moduleConfig, err := modulesClient.Collect() diff --git a/modules/client.go b/modules/client.go index 86f8a2caa..d71c48f0c 100644 --- a/modules/client.go +++ b/modules/client.go @@ -26,9 +26,10 @@ import ( "path/filepath" "regexp" - "github.com/gobwas/glob" hglob "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/files" @@ -100,10 +101,16 @@ func NewClient(cfg ClientConfig) *Client { logger = loggers.NewWarningLogger() } + var noVendor glob.Glob + if cfg.ModuleConfig.NoVendor != "" { + noVendor, _ = hglob.GetGlob(hglob.NormalizePath(cfg.ModuleConfig.NoVendor)) + } + return &Client{ fs: fs, ccfg: cfg, logger: logger, + noVendor: noVendor, moduleConfig: mcfg, environ: env, GoModulesFilename: goModFilename} @@ -114,6 +121,8 @@ type Client struct { fs afero.Fs logger *loggers.Logger + noVendor glob.Glob + ccfg ClientConfig // The top level module config @@ -220,6 +229,10 @@ func (c *Client) Vendor() error { continue } + if !c.shouldVendor(t.Path()) { + continue + } + if !t.IsGoMod() && !t.Vendor() { // We currently do not vendor components living in the // theme directory, see https://github.com/gohugoio/hugo/issues/5993 @@ -596,6 +609,10 @@ func (c *Client) tidy(mods Modules, goModOnly bool) error { return nil } +func (c *Client) shouldVendor(path string) bool { + return c.noVendor == nil || !c.noVendor.Match(path) +} + // ClientConfig configures the module Client. type ClientConfig struct { Fs afero.Fs diff --git a/modules/client_test.go b/modules/client_test.go index d5da621d1..41509a9ed 100644 --- a/modules/client_test.go +++ b/modules/client_test.go @@ -19,8 +19,6 @@ import ( "github.com/gohugoio/hugo/hugofs/glob" - "github.com/gohugoio/hugo/common/hugo" - "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/hugofs" @@ -29,77 +27,109 @@ import ( ) func TestClient(t *testing.T) { - if hugo.GoMinorVersion() < 12 { - // https://github.com/golang/go/issues/26794 - // There were some concurrent issues with Go modules in < Go 12. - t.Skip("skip this for Go <= 1.11 due to a bug in Go's stdlib") - } - - t.Parallel() modName := "hugo-modules-basic-test" modPath := "github.com/gohugoio/tests/" + modName - modConfig := DefaultModuleConfig - modConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}} - - c := qt.New(t) - - workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName) - c.Assert(err, qt.IsNil) - defer clean() - - client := NewClient(ClientConfig{ - Fs: hugofs.Os, - WorkingDir: workingDir, - ModuleConfig: modConfig, - }) - - // Test Init - c.Assert(client.Init(modPath), qt.IsNil) - - // Test Collect - mc, err := client.Collect() - c.Assert(err, qt.IsNil) - c.Assert(len(mc.AllModules), qt.Equals, 4) - for _, m := range mc.AllModules { - c.Assert(m, qt.Not(qt.IsNil)) - } - - // Test Graph - var graphb bytes.Buffer - c.Assert(client.Graph(&graphb), qt.IsNil) - expect := `github.com/gohugoio/tests/hugo-modules-basic-test github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0 ` - c.Assert(graphb.String(), qt.Equals, expect) + c := qt.New(t) - // Test Vendor - c.Assert(client.Vendor(), qt.IsNil) - graphb.Reset() - c.Assert(client.Graph(&graphb), qt.IsNil) - expectVendored := `project github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor + newClient := func(c *qt.C, withConfig func(cfg *ClientConfig)) (*Client, func()) { + + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, modName) + c.Assert(err, qt.IsNil) + + ccfg := ClientConfig{ + Fs: hugofs.Os, + WorkingDir: workingDir, + } + + withConfig(&ccfg) + ccfg.ModuleConfig.Imports = []Import{Import{Path: "github.com/gohugoio/hugoTestModules1_darwin/modh2_2"}} + client := NewClient(ccfg) + + return client, clean + } + + c.Run("All", func(c *qt.C) { + client, clean := newClient(c, func(cfg *ClientConfig) { + cfg.ModuleConfig = DefaultModuleConfig + }) + defer clean() + + // Test Init + c.Assert(client.Init(modPath), qt.IsNil) + + // Test Collect + mc, err := client.Collect() + c.Assert(err, qt.IsNil) + c.Assert(len(mc.AllModules), qt.Equals, 4) + for _, m := range mc.AllModules { + c.Assert(m, qt.Not(qt.IsNil)) + } + + // Test Graph + var graphb bytes.Buffer + c.Assert(client.Graph(&graphb), qt.IsNil) + + c.Assert(graphb.String(), qt.Equals, expect) + + // Test Vendor + c.Assert(client.Vendor(), qt.IsNil) + graphb.Reset() + c.Assert(client.Graph(&graphb), qt.IsNil) + + expectVendored := `project github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0+vendor project github.com/gohugoio/hugoTestModules1_darwin/modh2_2_1v@v1.3.0+vendor project github.com/gohugoio/hugoTestModules1_darwin/modh2_2_2@v1.3.0+vendor ` - c.Assert(graphb.String(), qt.Equals, expectVendored) - // Test the ignoreVendor setting - clientIgnoreVendor := NewClient(ClientConfig{ - Fs: hugofs.Os, - WorkingDir: workingDir, - ModuleConfig: modConfig, - IgnoreVendor: globAll, + c.Assert(graphb.String(), qt.Equals, expectVendored) + + // Test Tidy + c.Assert(client.Tidy(), qt.IsNil) + }) - graphb.Reset() - c.Assert(clientIgnoreVendor.Graph(&graphb), qt.IsNil) - c.Assert(graphb.String(), qt.Equals, expect) + c.Run("IgnoreVendor", func(c *qt.C) { + client, clean := newClient( + c, func(cfg *ClientConfig) { + cfg.ModuleConfig = DefaultModuleConfig + cfg.IgnoreVendor = globAll + }) + defer clean() - // Test Tidy - c.Assert(client.Tidy(), qt.IsNil) + c.Assert(client.Init(modPath), qt.IsNil) + _, err := client.Collect() + c.Assert(err, qt.IsNil) + c.Assert(client.Vendor(), qt.IsNil) + + var graphb bytes.Buffer + c.Assert(client.Graph(&graphb), qt.IsNil) + c.Assert(graphb.String(), qt.Equals, expect) + }) + + c.Run("NoVendor", func(c *qt.C) { + mcfg := DefaultModuleConfig + mcfg.NoVendor = "**" + client, clean := newClient( + c, func(cfg *ClientConfig) { + cfg.ModuleConfig = mcfg + }) + defer clean() + + c.Assert(client.Init(modPath), qt.IsNil) + _, err := client.Collect() + c.Assert(err, qt.IsNil) + c.Assert(client.Vendor(), qt.IsNil) + + var graphb bytes.Buffer + c.Assert(client.Graph(&graphb), qt.IsNil) + c.Assert(graphb.String(), qt.Equals, expect) + }) } diff --git a/modules/config.go b/modules/config.go index 5e95f0ac1..1964479f4 100644 --- a/modules/config.go +++ b/modules/config.go @@ -227,6 +227,10 @@ type Config struct { // Will be validated against the running Hugo version. HugoVersion HugoVersion + // A optional Glob pattern matching module paths to skip when vendoring, e.g. + // "github.com/**". + NoVendor string + // Configures GOPROXY. Proxy string // Configures GONOPROXY.