From 05a74eaec0d944a4b29445c878a431cd6ae12277 Mon Sep 17 00:00:00 2001 From: Robert van Gent Date: Wed, 26 Feb 2020 22:26:05 -0800 Subject: [PATCH] deploy: Implement include/exclude filters for deploy Fixes #6922 --- deploy/deploy.go | 21 ++++- deploy/deployConfig.go | 33 ++++++++ deploy/deployConfig_test.go | 13 +++ deploy/deploy_test.go | 80 +++++++++++++++++++ .../en/hosting-and-deployment/hugo-deploy.md | 9 ++- 5 files changed, 151 insertions(+), 5 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index 1d911f29b..c0c6ed4f3 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -31,6 +31,7 @@ import ( "sync" "github.com/dustin/go-humanize" + "github.com/gobwas/glob" "github.com/gohugoio/hugo/config" "github.com/pkg/errors" "github.com/spf13/afero" @@ -125,7 +126,11 @@ func (d *Deployer) Deploy(ctx context.Context) error { } // Load local files from the source directory. - local, err := walkLocal(d.localFs, d.matchers) + var include, exclude glob.Glob + if d.target != nil { + include, exclude = d.target.includeGlob, d.target.excludeGlob + } + local, err := walkLocal(d.localFs, d.matchers, include, exclude) if err != nil { return err } @@ -437,7 +442,7 @@ func (lf *localFile) MD5() []byte { // walkLocal walks the source directory and returns a flat list of files, // using localFile.SlashPath as the map keys. -func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error) { +func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (map[string]*localFile, error) { retval := map[string]*localFile{} err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { if err != nil { @@ -461,8 +466,18 @@ func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error) path = norm.NFC.String(path) } - // Find the first matching matcher (if any). + // Check include/exclude matchers. slashpath := filepath.ToSlash(path) + if include != nil && !include.Match(slashpath) { + jww.INFO.Printf(" dropping %q due to include\n", slashpath) + return nil + } + if exclude != nil && exclude.Match(slashpath) { + jww.INFO.Printf(" dropping %q due to exclude\n", slashpath) + return nil + } + + // Find the first matching matcher (if any). var m *matcher for _, cur := range matchers { if cur.Matches(slashpath) { diff --git a/deploy/deployConfig.go b/deploy/deployConfig.go index 3bc51294d..ecfabb7a4 100644 --- a/deploy/deployConfig.go +++ b/deploy/deployConfig.go @@ -17,7 +17,9 @@ import ( "fmt" "regexp" + "github.com/gobwas/glob" "github.com/gohugoio/hugo/config" + hglob "github.com/gohugoio/hugo/hugofs/glob" "github.com/mitchellh/mapstructure" ) @@ -41,6 +43,32 @@ type target struct { // GoogleCloudCDNOrigin specifies the Google Cloud project and CDN origin to // invalidate when deploying this target. It is specified as /. GoogleCloudCDNOrigin string + + // Optional patterns of files to include/exclude for this target. + // Parsed using github.com/gobwas/glob. + Include string + Exclude string + + // Parsed versions of Include/Exclude. + includeGlob glob.Glob + excludeGlob glob.Glob +} + +func (tgt *target) parseIncludeExclude() error { + var err error + if tgt.Include != "" { + tgt.includeGlob, err = hglob.GetGlob(tgt.Include) + if err != nil { + return fmt.Errorf("invalid deployment.target.include %q: %v", tgt.Include, err) + } + } + if tgt.Exclude != "" { + tgt.excludeGlob, err = hglob.GetGlob(tgt.Exclude) + if err != nil { + return fmt.Errorf("invalid deployment.target.exclude %q: %v", tgt.Exclude, err) + } + } + return nil } // matcher represents configuration to be applied to files whose paths match @@ -87,6 +115,11 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) { if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil { return dcfg, err } + for _, tgt := range dcfg.Targets { + if err := tgt.parseIncludeExclude(); err != nil { + return dcfg, err + } + } var err error for _, m := range dcfg.Matchers { m.re, err = regexp.Compile(m.Pattern) diff --git a/deploy/deployConfig_test.go b/deploy/deployConfig_test.go index f4aaa5eaf..c385510fe 100644 --- a/deploy/deployConfig_test.go +++ b/deploy/deployConfig_test.go @@ -38,18 +38,21 @@ order = ["o1", "o2"] name = "name0" url = "url0" cloudfrontdistributionid = "cdn0" +include = "*.html" # All uppercase. [[deployment.targets]] NAME = "name1" URL = "url1" CLOUDFRONTDISTRIBUTIONID = "cdn1" +INCLUDE = "*.jpg" # Camelcase. [[deployment.targets]] name = "name2" url = "url2" cloudFrontDistributionID = "cdn2" +exclude = "*.png" # All lowercase. [[deployment.matchers]] @@ -90,11 +93,21 @@ force = true // Targets. c.Assert(len(dcfg.Targets), qt.Equals, 3) + wantInclude := []string{"*.html", "*.jpg", ""} + wantExclude := []string{"", "", "*.png"} for i := 0; i < 3; i++ { tgt := dcfg.Targets[i] c.Assert(tgt.Name, qt.Equals, fmt.Sprintf("name%d", i)) c.Assert(tgt.URL, qt.Equals, fmt.Sprintf("url%d", i)) c.Assert(tgt.CloudFrontDistributionID, qt.Equals, fmt.Sprintf("cdn%d", i)) + c.Assert(tgt.Include, qt.Equals, wantInclude[i]) + if wantInclude[i] != "" { + c.Assert(tgt.includeGlob, qt.Not(qt.IsNil)) + } + c.Assert(tgt.Exclude, qt.Equals, wantExclude[i]) + if wantExclude[i] != "" { + c.Assert(tgt.excludeGlob, qt.Not(qt.IsNil)) + } } // Matchers. diff --git a/deploy/deploy_test.go b/deploy/deploy_test.go index ed20daef4..be1a628d2 100644 --- a/deploy/deploy_test.go +++ b/deploy/deploy_test.go @@ -640,6 +640,86 @@ func TestMaxDeletes(t *testing.T) { } } +// TestIncludeExclude verifies that the include/exclude options for targets work. +func TestIncludeExclude(t *testing.T) { + ctx := context.Background() + + tests := []struct { + Include string + Exclude string + Want deploySummary + }{ + { + Want: deploySummary{NumLocal: 5, NumUploads: 5}, + }, + { + Include: "**aaa", + Want: deploySummary{NumLocal: 3, NumUploads: 3}, + }, + { + Include: "**bbb", + Want: deploySummary{NumLocal: 2, NumUploads: 2}, + }, + { + Include: "aaa", + Want: deploySummary{NumLocal: 1, NumUploads: 1}, + }, + { + Exclude: "**aaa", + Want: deploySummary{NumLocal: 2, NumUploads: 2}, + }, + { + Exclude: "**bbb", + Want: deploySummary{NumLocal: 3, NumUploads: 3}, + }, + { + Exclude: "aaa", + Want: deploySummary{NumLocal: 4, NumUploads: 4}, + }, + { + Include: "**aaa", + Exclude: "**nested**", + Want: deploySummary{NumLocal: 2, NumUploads: 2}, + }, + } + for _, test := range tests { + t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) { + fsTests, cleanup, err := initFsTests() + if err != nil { + t.Fatal(err) + } + defer cleanup() + fsTest := fsTests[1] // just do file-based test + + _, err = initLocalFs(ctx, fsTest.fs) + if err != nil { + t.Fatal(err) + } + tgt := &target{ + Include: test.Include, + Exclude: test.Exclude, + } + if err := tgt.parseIncludeExclude(); err != nil { + t.Error(err) + } + deployer := &Deployer{ + localFs: fsTest.fs, + maxDeletes: -1, + bucket: fsTest.bucket, + target: tgt, + } + + // Sync remote with local. + if err := deployer.Deploy(ctx); err != nil { + t.Errorf("deploy: failed: %v", err) + } + if !cmp.Equal(deployer.summary, test.Want) { + t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want) + } + }) + } +} + // TestCompression verifies that gzip compression works correctly. // In particular, MD5 hashes must be of the compressed content. func TestCompression(t *testing.T) { diff --git a/docs/content/en/hosting-and-deployment/hugo-deploy.md b/docs/content/en/hosting-and-deployment/hugo-deploy.md index 81436b7f3..a571d366d 100644 --- a/docs/content/en/hosting-and-deployment/hugo-deploy.md +++ b/docs/content/en/hosting-and-deployment/hugo-deploy.md @@ -82,8 +82,13 @@ name = "mydeployment" # If you are using a CloudFront CDN, deploy will invalidate the cache as needed. cloudFrontDistributionID = - -# ... add more [[deployment.targets]] sections ... +# Optionally, you can include or exclude specific files. +# See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax. +# If non-empty, the pattern is matched against the local path. +# If exclude is non-empty, and a file's path matches it, that file is dropped. +# If include is non-empty, and a file's path does not match it, that file is dropped. +# include = "**.html" # would only include files with ".html" suffix +# exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix # [[deployment.matchers]] configure behavior for files that match the Pattern.