From 7e196a82944148ed3f78f334303b452ab2bd4078 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Mon, 11 Jan 2016 12:06:52 -0500 Subject: [PATCH] Handle remove & rename source operations incrementally --- commands/hugo.go | 93 +++++++++++++++++++++++++----------------------- hugolib/page.go | 18 +++++----- hugolib/site.go | 59 +++++++++++++++++++++--------- 3 files changed, 100 insertions(+), 70 deletions(-) diff --git a/commands/hugo.go b/commands/hugo.go index eb35c8002..1a40a4d61 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -546,9 +546,9 @@ func buildSite(watching ...bool) (err error) { return nil } -func rebuildSite(changes map[string]bool) error { +func rebuildSite(events []fsnotify.Event) error { startTime := time.Now() - err := mainSite.ReBuild(changes) + err := mainSite.ReBuild(events) if err != nil { return err } @@ -585,12 +585,10 @@ func NewWatcher(port int) error { for { select { case evs := <-watcher.Events: - jww.INFO.Println("File System Event:", evs) + jww.INFO.Println("Recieved System Events:", evs) - staticChanged := false - dynamicChanged := false - staticFilesChanged := make(map[string]bool) - dynamicFilesChanged := make(map[string]bool) + staticEvents := []fsnotify.Event{} //ev make(map[string]bool) + dynamicEvents := []fsnotify.Event{} //make(map[string]bool) for _, ev := range evs { ext := filepath.Ext(ev.Name) @@ -598,10 +596,6 @@ func NewWatcher(port int) error { if istemp { continue } - // renames are always followed with Create/Modify - if ev.Op&fsnotify.Rename == fsnotify.Rename { - continue - } // Write and rename operations are often followed by CHMOD. // There may be valid use cases for rebuilding the site on CHMOD, @@ -615,27 +609,24 @@ func NewWatcher(port int) error { continue } - isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath())) - staticChanged = staticChanged || isstatic - dynamicChanged = dynamicChanged || !isstatic - - if isstatic { - if staticPath, err := helpers.MakeStaticPathRelative(ev.Name); err == nil { - staticFilesChanged[staticPath] = true - } - } else { - dynamicFilesChanged[ev.Name] = true - } - // add new directory to watch list if s, err := os.Stat(ev.Name); err == nil && s.Mode().IsDir() { if ev.Op&fsnotify.Create == fsnotify.Create { watcher.Add(ev.Name) } } + + isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath())) + + if isstatic { + staticEvents = append(staticEvents, ev) +// } + } else { + dynamicEvents = append(dynamicEvents, ev) + } } - if staticChanged { + if len(staticEvents) > 0 { jww.FEEDBACK.Printf("Static file changed, syncing\n") if viper.GetBool("ForceSyncStatic") { jww.FEEDBACK.Printf("Syncing all static files\n") @@ -645,7 +636,6 @@ func NewWatcher(port int) error { utils.StopOnErr(err, fmt.Sprintf("Error copying static files to %s", helpers.AbsPathify(viper.GetString("PublishDir")))) } } else { - syncer := fsync.NewSyncer() syncer.NoTimes = viper.GetBool("notimes") syncer.SrcFs = hugofs.SourceFs @@ -660,24 +650,40 @@ func NewWatcher(port int) error { staticDir := helpers.GetStaticDirPath() themeStaticDir := helpers.GetThemesDirPath() - jww.FEEDBACK.Printf("StaticDir '%s'\nThemeStaticDir '%s'\n", staticDir, themeStaticDir) + jww.FEEDBACK.Printf("Syncing from: \n \tStaticDir: '%s'\n\tThemeStaticDir: '%s'\n", staticDir, themeStaticDir) - for path := range staticFilesChanged { + for _, ev := range staticEvents { + fmt.Println(ev) + fromPath := ev.Name var publishPath string - if strings.HasPrefix(path, staticDir) { - publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, staticDir)) - } else if strings.HasPrefix(path, themeStaticDir) { - publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, themeStaticDir)) + // If we are here we already know the event took place in a static dir + relPath, err := helpers.MakeStaticPathRelative(fromPath) + if err != nil { + fmt.Println(err) + continue } - jww.FEEDBACK.Printf("Syncing file '%s'", path) - if _, err := os.Stat(path); err == nil { - jww.INFO.Println("syncing from ", path, " to ", publishPath) - err := syncer.Sync(publishPath, path) - if err != nil { - jww.FEEDBACK.Printf("Error on syncing file '%s'\n", path) - } + if strings.HasPrefix(fromPath, staticDir) { + publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir)) + } else if strings.HasPrefix(relPath, themeStaticDir) { + publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir)) + } + jww.FEEDBACK.Println("Syncing file", relPath) + + // Due to our approach of layering many directories onto one we can't accurately + // remove file not in one of the source directories. + // If a file is in the local static dir and also in the theme static dir and we remove + // it from one of those locations we expect it to still exist in the destination + + // if remove or rename ignore + if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { + continue + } + + jww.INFO.Println("syncing from ", fromPath, " to ", publishPath) + if er := syncer.Sync(publishPath, fromPath); er != nil { + jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er) } } } @@ -686,8 +692,9 @@ func NewWatcher(port int) error { // Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized // force refresh when more than one file - if len(staticFilesChanged) == 1 { - for path := range staticFilesChanged { + if len(staticEvents) == 1 { + for _, ev := range staticEvents { + path, _ := helpers.MakeStaticPathRelative(ev.Name) livereload.RefreshPath(path) } @@ -697,14 +704,12 @@ func NewWatcher(port int) error { } } - if dynamicChanged { + if len(dynamicEvents) >0 { fmt.Print("\nChange detected, rebuilding site\n") const layout = "2006-01-02 15:04 -0700" fmt.Println(time.Now().Format(layout)) - //TODO here - // utils.CheckErr(buildSite(true)) - rebuildSite(dynamicFilesChanged) + rebuildSite(dynamicEvents) if !BuildWatch && !viper.GetBool("DisableLiveReload") { // Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized diff --git a/hugolib/page.go b/hugolib/page.go index 1a0f2985e..58496e81b 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -112,20 +112,20 @@ type Pages []*Page // } //} -//func (ps Pages) FindPageByFilePath(inPath string) *Page { -// for _, x := range ps { -// if x.Source.LogicalName() == inPath { -// return x -// } -// } -// return nil -//} +func (ps Pages) FindPagePosByFilePath(inPath string) int { + for i, x := range ps { + if x.Source.Path() == inPath { + return i + } + } + return -1 +} // FindPagePos Given a page, it will find the position in Pages // will return -1 if not found func (ps Pages) FindPagePos(page *Page) int { for i, x := range ps { - if x.Source.LogicalName() == page.Source.LogicalName() { + if x.Source.Path() == page.Source.Path() { return i } } diff --git a/hugolib/site.go b/hugolib/site.go index 0376645ca..1d1f1cc1b 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -43,6 +43,7 @@ import ( jww "github.com/spf13/jwalterweatherman" "github.com/spf13/nitro" "github.com/spf13/viper" + "gopkg.in/fsnotify.v1" ) var _ = transform.AbsURL @@ -426,28 +427,29 @@ func (s *Site) Build() (err error) { return nil } -func (s *Site) ReBuild(changed map[string]bool) error { +func (s *Site) ReBuild(events []fsnotify.Event) error { s.timerStep("initialize rebuild") // First we need to determine what changed - sourceChanged := []string{} - tmplChanged := []string{} - dataChanged := []string{} + sourceChanged := []fsnotify.Event{} + tmplChanged := []fsnotify.Event{} + dataChanged := []fsnotify.Event{} + var err error - for f := range changed { + for _, ev := range events { // Need to re-read source - if strings.HasPrefix(f, s.absContentDir()) { - fmt.Println("Source changed", f) - sourceChanged = append(sourceChanged, f) + if strings.HasPrefix(ev.Name, s.absContentDir()) { + fmt.Println("Source changed", ev) + sourceChanged = append(sourceChanged, ev) } - if strings.HasPrefix(f, s.absLayoutDir()) || strings.HasPrefix(f, s.absThemeDir()) { - fmt.Println("Template changed", f) - tmplChanged = append(tmplChanged, f) + if strings.HasPrefix(ev.Name, s.absLayoutDir()) || strings.HasPrefix(ev.Name, s.absThemeDir()) { + fmt.Println("Template changed", ev) + tmplChanged = append(tmplChanged, ev) } - if strings.HasPrefix(f, s.absDataDir()) { - fmt.Println("Data changed", f) - dataChanged = append(dataChanged, f) + if strings.HasPrefix(ev.Name, s.absDataDir()) { + fmt.Println("Data changed", ev) + dataChanged = append(dataChanged,ev) } } @@ -497,8 +499,15 @@ func (s *Site) ReBuild(changed map[string]bool) error { go incrementalReadCollator(s, readResults, pageChan, fileConvChan, coordinator, errs) go converterCollator(s, convertResults, errs) - for _, x := range sourceChanged { - file, err := s.ReReadFile(x) + for _, ev := range sourceChanged { + if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { + //remove the file & a create will follow + path, _ := helpers.GetRelativePath(ev.Name, s.absContentDir()) + s.RemovePageByPath(path) + continue + } + + file, err := s.ReReadFile(ev.Name) if err != nil { errs <- err } @@ -540,7 +549,6 @@ func (s *Site) ReBuild(changed map[string]bool) error { if err = s.Render(); err != nil { // Better reporting when the template is missing (commit 2bbecc7b) jww.ERROR.Printf("Error rendering site: %s", err) - jww.ERROR.Printf("Available templates:") var keys []string for _, template := range s.Tmpl.Templates() { @@ -1005,6 +1013,23 @@ func (s *Site) AddPage(page *Page) { } } + +func (s *Site) RemovePageByPath(path string) { + if i := s.Pages.FindPagePosByFilePath(path); i >= 0 { + page := s.Pages[i] + + if page.IsDraft() { + s.draftCount-- + } + + if page.IsFuture() { + s.futureCount-- + } + + s.Pages = append(s.Pages[:i], s.Pages[i+1:]...) + } +} + func (s *Site) RemovePage(page *Page) { if i := s.Pages.FindPagePos(page); i >= 0 { if page.IsDraft() {