diff --git a/commands/hugo.go b/commands/hugo.go index 1a40a4d61..b186057c8 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -42,6 +42,7 @@ import ( "github.com/spf13/nitro" "github.com/spf13/viper" "gopkg.in/fsnotify.v1" + "github.com/spf13/afero" ) var mainSite *hugolib.Site @@ -434,6 +435,46 @@ func build(watches ...bool) error { return nil } +func getStaticSourceFs() afero.Fs { + source := hugofs.SourceFs + themeDir, err := helpers.GetThemeStaticDirPath() + staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator + + useTheme := true + useStatic := true + + if err != nil { + jww.WARN.Println(err) + useTheme = false + } else { + if _, err := source.Stat(themeDir); os.IsNotExist(err) { + jww.WARN.Println("Unable to find Theme Static Directory:", themeDir) + useTheme = false + } + } + + if _, err := source.Stat(staticDir); os.IsNotExist(err) { + jww.WARN.Println("Unable to find Static Directory:", staticDir) + useStatic = false + } + + if !useStatic && !useTheme { + return nil + } + + if !useStatic { + return afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir)) + } + + if !useTheme { + return afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir)) + } + + base := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, themeDir)) + overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(hugofs.SourceFs, staticDir)) + return afero.NewCopyOnWriteFs(base, overlay) +} + func copyStatic() error { publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator @@ -442,31 +483,51 @@ func copyStatic() error { publishDir = helpers.FilePathSeparator } + staticSourceFs := getStaticSourceFs() + + if staticSourceFs == nil { + jww.WARN.Println("No static directories found to sync") + return nil + } + syncer := fsync.NewSyncer() syncer.NoTimes = viper.GetBool("notimes") - syncer.SrcFs = hugofs.SourceFs + syncer.SrcFs = staticSourceFs syncer.DestFs = hugofs.DestinationFS + // Now that we are using a unionFs for the static directories + // We can effectively clean the publishDir on initial sync + syncer.Delete = true + jww.INFO.Println("syncing static files to", publishDir) - themeDir, err := helpers.GetThemeStaticDirPath() - if err != nil { - jww.WARN.Println(err) - } - - // Copy the theme's static directory - if themeDir != "" { - jww.INFO.Println("syncing from", themeDir, "to", publishDir) - utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir)) - } - - // Copy the site's own static directory - staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator - if _, err := os.Stat(staticDir); err == nil { - jww.INFO.Println("syncing from", staticDir, "to", publishDir) - return syncer.Sync(publishDir, staticDir) - } else if os.IsNotExist(err) { - jww.WARN.Println("Unable to find Static Directory:", staticDir) - } + // because we are using a baseFs (to get the union right). Sync from the root + syncer.Sync(publishDir, helpers.FilePathSeparator) return nil +// +// themeDir, err := helpers.GetThemeStaticDirPath() +// if err != nil { +// jww.WARN.Println(err) +// } +// +// staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator +// if _, err := os.Stat(staticDir); os.IsNotExist(err) { +// jww.WARN.Println("Unable to find Static Directory:", staticDir) +// } +// +// // Copy the theme's static directory +// if themeDir != "" { +// jww.INFO.Println("syncing from", themeDir, "to", publishDir) +// utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir)) +// } +// +// // Copy the site's own static directory +// staticDir := helpers.GetStaticDirPath() + helpers.FilePathSeparator +// if _, err := os.Stat(staticDir); err == nil { +// jww.INFO.Println("syncing from", staticDir, "to", publishDir) +// return syncer.Sync(publishDir, staticDir) +// } else if os.IsNotExist(err) { +// jww.WARN.Println("Unable to find Static Directory:", staticDir) +// } +// return nil } // getDirList provides NewWatcher() with a list of directories to watch for changes. @@ -597,6 +658,11 @@ func NewWatcher(port int) error { continue } + // Sometimes during rm -rf operations a '"": REMOVE' is triggered. Just ignore these + if ev.Name == "" { + continue + } + // Write and rename operations are often followed by CHMOD. // There may be valid use cases for rebuilding the site on CHMOD, // but that will require more complex logic than this simple conditional. @@ -609,10 +675,19 @@ func NewWatcher(port int) error { continue } - // add new directory to watch list - if s, err := os.Stat(ev.Name); err == nil && s.Mode().IsDir() { - if ev.Op&fsnotify.Create == fsnotify.Create { - watcher.Add(ev.Name) + walkAdder := func (path string, f os.FileInfo, err error) error { + if f.IsDir() { + jww.FEEDBACK.Println("adding created directory to watchlist", path) + watcher.Add(path) + } + return nil + } + + // recursively add new directories to watch list + // When mkdir -p is used, only the top directory triggers an event (at least on OSX) + if ev.Op&fsnotify.Create == fsnotify.Create { + if s, err := hugofs.SourceFs.Stat(ev.Name); err == nil && s.Mode().IsDir() { + afero.Walk(hugofs.SourceFs, ev.Name, walkAdder) } } @@ -627,7 +702,18 @@ func NewWatcher(port int) error { } if len(staticEvents) > 0 { - jww.FEEDBACK.Printf("Static file changed, syncing\n") + publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator + + // If root, remove the second '/' + if publishDir == "//" { + publishDir = helpers.FilePathSeparator + } + + jww.FEEDBACK.Println("\n Static file changes detected") + jww.FEEDBACK.Println("syncing to", publishDir) + const layout = "2006-01-02 15:04 -0700" + fmt.Println(time.Now().Format(layout)) + if viper.GetBool("ForceSyncStatic") { jww.FEEDBACK.Printf("Syncing all static files\n") err := copyStatic() @@ -636,26 +722,38 @@ func NewWatcher(port int) error { utils.StopOnErr(err, fmt.Sprintf("Error copying static files to %s", helpers.AbsPathify(viper.GetString("PublishDir")))) } } else { - syncer := fsync.NewSyncer() - syncer.NoTimes = viper.GetBool("notimes") - syncer.SrcFs = hugofs.SourceFs - syncer.DestFs = hugofs.DestinationFS + staticSourceFs := getStaticSourceFs() - publishDir := helpers.AbsPathify(viper.GetString("PublishDir")) + helpers.FilePathSeparator - - if publishDir == "//" || publishDir == helpers.FilePathSeparator { - publishDir = "" + if staticSourceFs == nil { + jww.WARN.Println("No static directories found to sync") + return } - staticDir := helpers.GetStaticDirPath() - themeStaticDir := helpers.GetThemesDirPath() + syncer := fsync.NewSyncer() + syncer.NoTimes = viper.GetBool("notimes") + syncer.SrcFs = staticSourceFs + syncer.DestFs = hugofs.DestinationFS - jww.FEEDBACK.Printf("Syncing from: \n \tStaticDir: '%s'\n\tThemeStaticDir: '%s'\n", staticDir, themeStaticDir) for _, ev := range staticEvents { + // Due to our approach of layering both directories and the content's rendered output + // into one we can't accurately remove a file not in one of the source directories. + // If a file is in the local static dir and also in the theme static dir and we remove + // it from one of those locations we expect it to still exist in the destination + // If a file is generated by the content over a static file we expect it to remain as well. + // Because we are never certain if the file was overwritten by the content generation + // We can't effectively remove anything. + // + // This leads to two approaches: + // 1. Not overwrite anything + // 2. Assume these cases are rare and overwrite anyway. If things get out of sync + // a clean sync will be needed. + // There is an alternative which is quite heavy. We would have to track every single file + // placed into the publishedPath and which pipeline put it there. + // We have chosen to take the 2nd approach fmt.Println(ev) + fromPath := ev.Name - var publishPath string // If we are here we already know the event took place in a static dir relPath, err := helpers.MakeStaticPathRelative(fromPath) @@ -663,28 +761,42 @@ func NewWatcher(port int) error { fmt.Println(err) continue } + fmt.Println("relpath", relPath) - if strings.HasPrefix(fromPath, staticDir) { - publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir)) - } else if strings.HasPrefix(relPath, themeStaticDir) { - publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir)) - } - jww.FEEDBACK.Println("Syncing file", relPath) - // Due to our approach of layering many directories onto one we can't accurately - // remove file not in one of the source directories. - // If a file is in the local static dir and also in the theme static dir and we remove - // it from one of those locations we expect it to still exist in the destination - - // if remove or rename ignore + // if remove or rename ignore.. as in leave the old file in the publishDir if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { + // What about the case where a file in the theme is moved so the local static file can + // take it's place. + if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) { + // If file doesn't exist in any static dir, remove it + toRemove :=filepath.Join(publishDir, relPath) + jww.FEEDBACK.Println("File no longer exists in static dir, removing", toRemove) + hugofs.DestinationFS.Remove(toRemove) + } else if err == nil { + // If file still exists, sync it + jww.FEEDBACK.Println("Syncing", relPath, "to", publishDir) + syncer.Sync(filepath.Join(publishDir, relPath), relPath) + } else { + jww.ERROR.Println(err) + } + continue } - jww.INFO.Println("syncing from ", fromPath, " to ", publishPath) - if er := syncer.Sync(publishPath, fromPath); er != nil { - jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er) - } +// if strings.HasPrefix(fromPath, staticDir) { +// publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir)) +// } else if strings.HasPrefix(relPath, themeStaticDir) { +// publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir)) +// } + jww.FEEDBACK.Println("Syncing", relPath, "to", publishDir) + syncer.Sync(filepath.Join(publishDir, relPath), relPath) + + +// jww.INFO.Println("syncing from ", fromPath, " to ", publishPath) +// if er := syncer.Sync(publishPath, fromPath); er != nil { +// jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er) +// } } } @@ -692,7 +804,7 @@ func NewWatcher(port int) error { // Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized // force refresh when more than one file - if len(staticEvents) == 1 { + if len(staticEvents) > 0 { for _, ev := range staticEvents { path, _ := helpers.MakeStaticPathRelative(ev.Name) livereload.RefreshPath(path) @@ -704,7 +816,7 @@ func NewWatcher(port int) error { } } - if len(dynamicEvents) >0 { + if len(dynamicEvents) > 0 { fmt.Print("\nChange detected, rebuilding site\n") const layout = "2006-01-02 15:04 -0700" fmt.Println(time.Now().Format(layout))