deploy: Implement include/exclude filters for deploy

Fixes #6922
This commit is contained in:
Robert van Gent 2020-02-26 22:26:05 -08:00 committed by GitHub
parent 33ae621083
commit 05a74eaec0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 151 additions and 5 deletions

View file

@ -31,6 +31,7 @@ import (
"sync" "sync"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/spf13/afero" "github.com/spf13/afero"
@ -125,7 +126,11 @@ func (d *Deployer) Deploy(ctx context.Context) error {
} }
// Load local files from the source directory. // Load local files from the source directory.
local, err := walkLocal(d.localFs, d.matchers) var include, exclude glob.Glob
if d.target != nil {
include, exclude = d.target.includeGlob, d.target.excludeGlob
}
local, err := walkLocal(d.localFs, d.matchers, include, exclude)
if err != nil { if err != nil {
return err return err
} }
@ -437,7 +442,7 @@ func (lf *localFile) MD5() []byte {
// walkLocal walks the source directory and returns a flat list of files, // walkLocal walks the source directory and returns a flat list of files,
// using localFile.SlashPath as the map keys. // using localFile.SlashPath as the map keys.
func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error) { func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (map[string]*localFile, error) {
retval := map[string]*localFile{} retval := map[string]*localFile{}
err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
@ -461,8 +466,18 @@ func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error)
path = norm.NFC.String(path) path = norm.NFC.String(path)
} }
// Find the first matching matcher (if any). // Check include/exclude matchers.
slashpath := filepath.ToSlash(path) slashpath := filepath.ToSlash(path)
if include != nil && !include.Match(slashpath) {
jww.INFO.Printf(" dropping %q due to include\n", slashpath)
return nil
}
if exclude != nil && exclude.Match(slashpath) {
jww.INFO.Printf(" dropping %q due to exclude\n", slashpath)
return nil
}
// Find the first matching matcher (if any).
var m *matcher var m *matcher
for _, cur := range matchers { for _, cur := range matchers {
if cur.Matches(slashpath) { if cur.Matches(slashpath) {

View file

@ -17,7 +17,9 @@ import (
"fmt" "fmt"
"regexp" "regexp"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config"
hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/mitchellh/mapstructure" "github.com/mitchellh/mapstructure"
) )
@ -41,6 +43,32 @@ type target struct {
// GoogleCloudCDNOrigin specifies the Google Cloud project and CDN origin to // GoogleCloudCDNOrigin specifies the Google Cloud project and CDN origin to
// invalidate when deploying this target. It is specified as <project>/<origin>. // invalidate when deploying this target. It is specified as <project>/<origin>.
GoogleCloudCDNOrigin string GoogleCloudCDNOrigin string
// Optional patterns of files to include/exclude for this target.
// Parsed using github.com/gobwas/glob.
Include string
Exclude string
// Parsed versions of Include/Exclude.
includeGlob glob.Glob
excludeGlob glob.Glob
}
func (tgt *target) parseIncludeExclude() error {
var err error
if tgt.Include != "" {
tgt.includeGlob, err = hglob.GetGlob(tgt.Include)
if err != nil {
return fmt.Errorf("invalid deployment.target.include %q: %v", tgt.Include, err)
}
}
if tgt.Exclude != "" {
tgt.excludeGlob, err = hglob.GetGlob(tgt.Exclude)
if err != nil {
return fmt.Errorf("invalid deployment.target.exclude %q: %v", tgt.Exclude, err)
}
}
return nil
} }
// matcher represents configuration to be applied to files whose paths match // matcher represents configuration to be applied to files whose paths match
@ -87,6 +115,11 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) {
if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil { if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil {
return dcfg, err return dcfg, err
} }
for _, tgt := range dcfg.Targets {
if err := tgt.parseIncludeExclude(); err != nil {
return dcfg, err
}
}
var err error var err error
for _, m := range dcfg.Matchers { for _, m := range dcfg.Matchers {
m.re, err = regexp.Compile(m.Pattern) m.re, err = regexp.Compile(m.Pattern)

View file

@ -38,18 +38,21 @@ order = ["o1", "o2"]
name = "name0" name = "name0"
url = "url0" url = "url0"
cloudfrontdistributionid = "cdn0" cloudfrontdistributionid = "cdn0"
include = "*.html"
# All uppercase. # All uppercase.
[[deployment.targets]] [[deployment.targets]]
NAME = "name1" NAME = "name1"
URL = "url1" URL = "url1"
CLOUDFRONTDISTRIBUTIONID = "cdn1" CLOUDFRONTDISTRIBUTIONID = "cdn1"
INCLUDE = "*.jpg"
# Camelcase. # Camelcase.
[[deployment.targets]] [[deployment.targets]]
name = "name2" name = "name2"
url = "url2" url = "url2"
cloudFrontDistributionID = "cdn2" cloudFrontDistributionID = "cdn2"
exclude = "*.png"
# All lowercase. # All lowercase.
[[deployment.matchers]] [[deployment.matchers]]
@ -90,11 +93,21 @@ force = true
// Targets. // Targets.
c.Assert(len(dcfg.Targets), qt.Equals, 3) c.Assert(len(dcfg.Targets), qt.Equals, 3)
wantInclude := []string{"*.html", "*.jpg", ""}
wantExclude := []string{"", "", "*.png"}
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
tgt := dcfg.Targets[i] tgt := dcfg.Targets[i]
c.Assert(tgt.Name, qt.Equals, fmt.Sprintf("name%d", i)) c.Assert(tgt.Name, qt.Equals, fmt.Sprintf("name%d", i))
c.Assert(tgt.URL, qt.Equals, fmt.Sprintf("url%d", i)) c.Assert(tgt.URL, qt.Equals, fmt.Sprintf("url%d", i))
c.Assert(tgt.CloudFrontDistributionID, qt.Equals, fmt.Sprintf("cdn%d", i)) c.Assert(tgt.CloudFrontDistributionID, qt.Equals, fmt.Sprintf("cdn%d", i))
c.Assert(tgt.Include, qt.Equals, wantInclude[i])
if wantInclude[i] != "" {
c.Assert(tgt.includeGlob, qt.Not(qt.IsNil))
}
c.Assert(tgt.Exclude, qt.Equals, wantExclude[i])
if wantExclude[i] != "" {
c.Assert(tgt.excludeGlob, qt.Not(qt.IsNil))
}
} }
// Matchers. // Matchers.

View file

@ -640,6 +640,86 @@ func TestMaxDeletes(t *testing.T) {
} }
} }
// TestIncludeExclude verifies that the include/exclude options for targets work.
func TestIncludeExclude(t *testing.T) {
ctx := context.Background()
tests := []struct {
Include string
Exclude string
Want deploySummary
}{
{
Want: deploySummary{NumLocal: 5, NumUploads: 5},
},
{
Include: "**aaa",
Want: deploySummary{NumLocal: 3, NumUploads: 3},
},
{
Include: "**bbb",
Want: deploySummary{NumLocal: 2, NumUploads: 2},
},
{
Include: "aaa",
Want: deploySummary{NumLocal: 1, NumUploads: 1},
},
{
Exclude: "**aaa",
Want: deploySummary{NumLocal: 2, NumUploads: 2},
},
{
Exclude: "**bbb",
Want: deploySummary{NumLocal: 3, NumUploads: 3},
},
{
Exclude: "aaa",
Want: deploySummary{NumLocal: 4, NumUploads: 4},
},
{
Include: "**aaa",
Exclude: "**nested**",
Want: deploySummary{NumLocal: 2, NumUploads: 2},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
fsTests, cleanup, err := initFsTests()
if err != nil {
t.Fatal(err)
}
defer cleanup()
fsTest := fsTests[1] // just do file-based test
_, err = initLocalFs(ctx, fsTest.fs)
if err != nil {
t.Fatal(err)
}
tgt := &target{
Include: test.Include,
Exclude: test.Exclude,
}
if err := tgt.parseIncludeExclude(); err != nil {
t.Error(err)
}
deployer := &Deployer{
localFs: fsTest.fs,
maxDeletes: -1,
bucket: fsTest.bucket,
target: tgt,
}
// Sync remote with local.
if err := deployer.Deploy(ctx); err != nil {
t.Errorf("deploy: failed: %v", err)
}
if !cmp.Equal(deployer.summary, test.Want) {
t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
}
})
}
}
// TestCompression verifies that gzip compression works correctly. // TestCompression verifies that gzip compression works correctly.
// In particular, MD5 hashes must be of the compressed content. // In particular, MD5 hashes must be of the compressed content.
func TestCompression(t *testing.T) { func TestCompression(t *testing.T) {

View file

@ -82,8 +82,13 @@ name = "mydeployment"
# If you are using a CloudFront CDN, deploy will invalidate the cache as needed. # If you are using a CloudFront CDN, deploy will invalidate the cache as needed.
cloudFrontDistributionID = <ID> cloudFrontDistributionID = <ID>
# Optionally, you can include or exclude specific files.
# ... add more [[deployment.targets]] sections ... # See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax.
# If non-empty, the pattern is matched against the local path.
# If exclude is non-empty, and a file's path matches it, that file is dropped.
# If include is non-empty, and a file's path does not match it, that file is dropped.
# include = "**.html" # would only include files with ".html" suffix
# exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix
# [[deployment.matchers]] configure behavior for files that match the Pattern. # [[deployment.matchers]] configure behavior for files that match the Pattern.