switch transfers to workers

This commit is contained in:
davidejones 2023-03-04 09:18:43 +00:00 committed by Bjørn Erik Pedersen
parent e6f029bdee
commit bebb2b8d0a
4 changed files with 8 additions and 8 deletions

View file

@ -31,7 +31,7 @@ type deployCmd struct {
invalidateCDN bool invalidateCDN bool
maxDeletes int maxDeletes int
transfers int workers int
} }
// TODO: In addition to the "deploy" command, consider adding a "--deploy" // TODO: In addition to the "deploy" command, consider adding a "--deploy"
@ -60,7 +60,7 @@ documentation.
cfgInit := func(c *commandeer) error { cfgInit := func(c *commandeer) error {
c.Set("invalidateCDN", cc.invalidateCDN) c.Set("invalidateCDN", cc.invalidateCDN)
c.Set("maxDeletes", cc.maxDeletes) c.Set("maxDeletes", cc.maxDeletes)
c.Set("transfers", cc.transfers) c.Set("workers", cc.workers)
return nil return nil
} }
comm, err := initializeConfig(true, true, false, &cc.hugoBuilderCommon, cc, cfgInit) comm, err := initializeConfig(true, true, false, &cc.hugoBuilderCommon, cc, cfgInit)
@ -81,7 +81,7 @@ documentation.
cmd.Flags().Bool("force", false, "force upload of all files") cmd.Flags().Bool("force", false, "force upload of all files")
cmd.Flags().BoolVar(&cc.invalidateCDN, "invalidateCDN", true, "invalidate the CDN cache listed in the deployment target") cmd.Flags().BoolVar(&cc.invalidateCDN, "invalidateCDN", true, "invalidate the CDN cache listed in the deployment target")
cmd.Flags().IntVar(&cc.maxDeletes, "maxDeletes", 256, "maximum # of files to delete, or -1 to disable") cmd.Flags().IntVar(&cc.maxDeletes, "maxDeletes", 256, "maximum # of files to delete, or -1 to disable")
cmd.Flags().IntVar(&cc.transfers, "transfers", 10, "number of file transfers to run in parallel. defaults to 10") cmd.Flags().IntVar(&cc.workers, "workers", 10, "number of workers to transfer files. defaults to 10")
cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd) cc.baseBuilderCmd = b.newBuilderBasicCmd(cmd)

View file

@ -236,9 +236,9 @@ func initializeFlags(cmd *cobra.Command, cfg config.Provider) {
"target", "target",
"theme", "theme",
"themesDir", "themesDir",
"transfers",
"verbose", "verbose",
"verboseLog", "verboseLog",
"workers",
"duplicateTargetPaths", "duplicateTargetPaths",
} }

View file

@ -65,7 +65,7 @@ type Deployer struct {
force bool // true forces upload of all files force bool // true forces upload of all files
invalidateCDN bool // true enables invalidate CDN cache (if possible) invalidateCDN bool // true enables invalidate CDN cache (if possible)
maxDeletes int // caps the # of files to delete; -1 to disable maxDeletes int // caps the # of files to delete; -1 to disable
transfers int // The number of file transfers to run in parallel workers int // The number of workers to transfer files
// For tests... // For tests...
summary deploySummary // summary of latest Deploy results summary deploySummary // summary of latest Deploy results
@ -119,7 +119,7 @@ func New(cfg config.Provider, localFs afero.Fs) (*Deployer, error) {
force: cfg.GetBool("force"), force: cfg.GetBool("force"),
invalidateCDN: cfg.GetBool("invalidateCDN"), invalidateCDN: cfg.GetBool("invalidateCDN"),
maxDeletes: cfg.GetInt("maxDeletes"), maxDeletes: cfg.GetInt("maxDeletes"),
transfers: cfg.GetInt("transfers"), workers: cfg.GetInt("workers"),
}, nil }, nil
} }
@ -191,7 +191,7 @@ func (d *Deployer) Deploy(ctx context.Context) error {
// Apply the changes in parallel, using an inverted worker // Apply the changes in parallel, using an inverted worker
// pool (https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s). // pool (https://www.youtube.com/watch?v=5zXAHh5tJqQ&t=26m58s).
// sem prevents more than nParallel concurrent goroutines. // sem prevents more than nParallel concurrent goroutines.
nParallel := d.transfers nParallel := d.workers
var errs []error var errs []error
var errMu sync.Mutex // protects errs var errMu sync.Mutex // protects errs

View file

@ -31,7 +31,7 @@ hugo deploy [flags]
--ignoreVendorPaths string ignores any _vendor for module paths matching the given Glob pattern --ignoreVendorPaths string ignores any _vendor for module paths matching the given Glob pattern
--invalidateCDN invalidate the CDN cache listed in the deployment target (default true) --invalidateCDN invalidate the CDN cache listed in the deployment target (default true)
--maxDeletes int maximum # of files to delete, or -1 to disable (default 256) --maxDeletes int maximum # of files to delete, or -1 to disable (default 256)
--transfers int number of file transfers to run in parallel (default 10) --workers int number of workers to transfer files. (default 10)
-s, --source string filesystem path to read files relative from -s, --source string filesystem path to read files relative from
--target string target deployment from deployments section in config file; defaults to the first one --target string target deployment from deployments section in config file; defaults to the first one
--themesDir string filesystem path to themes directory --themesDir string filesystem path to themes directory