From 9f3796a31dc217b2b8094f948266b23ac3808aa6 Mon Sep 17 00:00:00 2001 From: Steve Francia Date: Thu, 7 Jan 2016 21:48:13 -0500 Subject: [PATCH] Read/reread individual source content files next is incremental conversion --- commands/hugo.go | 4 +- docs/content/meta/roadmap.md | 1 - hugolib/handler_page.go | 1 + hugolib/page.go | 69 ++++++++++---- hugolib/site.go | 168 ++++++++++++++++++++++++++++++----- source/file.go | 12 ++- source/filesystem.go | 79 ++++++++-------- 7 files changed, 253 insertions(+), 81 deletions(-) diff --git a/commands/hugo.go b/commands/hugo.go index ab847c01e..eb35c8002 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -594,7 +594,7 @@ func NewWatcher(port int) error { for _, ev := range evs { ext := filepath.Ext(ev.Name) - istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___")|| strings.HasSuffix(ext, "jb_bak___") + istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___") || strings.HasSuffix(ext, "jb_bak___") if istemp { continue } @@ -703,7 +703,7 @@ func NewWatcher(port int) error { fmt.Println(time.Now().Format(layout)) //TODO here - // utils.CheckErr(buildSite(true)) + // utils.CheckErr(buildSite(true)) rebuildSite(dynamicFilesChanged) if !BuildWatch && !viper.GetBool("DisableLiveReload") { diff --git a/docs/content/meta/roadmap.md b/docs/content/meta/roadmap.md index 5cd515aa2..8ed9da1ad 100644 --- a/docs/content/meta/roadmap.md +++ b/docs/content/meta/roadmap.md @@ -19,7 +19,6 @@ In no particular order, here is what we are working on: * Import from other website systems * from Drupal (See https://bitbucket.org/rickb777/drupal2hugo by Rick Beton (@rickb777)) * from WordPress (See [#100][], especially https://github.com/SchumacherFM/wordpress-to-hugo-exporter by Cyrill Schumacher (@SchumacherFM), but volunteers are needed to make it work with latest versions of WordPress.) - * from Jekyll (See [#101][]) * An interactive web based editor (See http://discuss.gohugo.io/t/web-based-editor/155) * Additional [themes](https://github.com/spf13/hugoThemes) (always on-going, contributions welcome!) * Dynamic image resizing via shortcodes diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go index 9e4c9ed05..cc8ac6bb1 100644 --- a/hugolib/handler_page.go +++ b/hugolib/handler_page.go @@ -32,6 +32,7 @@ type basicPageHandler Handle func (b basicPageHandler) Read(f *source.File, s *Site) HandledResult { page, err := NewPage(f.Path()) + if err != nil { return HandledResult{file: f, err: err} } diff --git a/hugolib/page.go b/hugolib/page.go index a37d21663..1a0f2985e 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -48,20 +48,19 @@ var ( ) type Page struct { - Params map[string]interface{} - Content template.HTML - Summary template.HTML - Aliases []string - Status string - Images []Image - Videos []Video - TableOfContents template.HTML - Truncated bool - Draft bool - PublishDate time.Time - Tmpl tpl.Template - Markup string - + Params map[string]interface{} + Content template.HTML + Summary template.HTML + Aliases []string + Status string + Images []Image + Videos []Video + TableOfContents template.HTML + Truncated bool + Draft bool + PublishDate time.Time + Tmpl tpl.Template + Markup string extension string contentType string renderable bool @@ -77,13 +76,13 @@ type Page struct { plainSecondaryInit sync.Once renderingConfig *helpers.Blackfriday renderingConfigInit sync.Once + pageMenus PageMenus + pageMenusInit sync.Once + isCJKLanguage bool PageMeta Source Position `json:"-"` Node - pageMenus PageMenus - pageMenusInit sync.Once - isCJKLanguage bool } type Source struct { @@ -106,6 +105,42 @@ type Position struct { } type Pages []*Page +// +//func (ps Pages) Replace(page *Page) { +// if i := ps.FindPagePos(page); i >= 0 { +// ps[i] = page +// } +//} + +//func (ps Pages) FindPageByFilePath(inPath string) *Page { +// for _, x := range ps { +// if x.Source.LogicalName() == inPath { +// return x +// } +// } +// return nil +//} + +// FindPagePos Given a page, it will find the position in Pages +// will return -1 if not found +func (ps Pages) FindPagePos(page *Page) int { + for i, x := range ps { + if x.Source.LogicalName() == page.Source.LogicalName() { + return i + } + } + return -1 +} + +// FindPage Given a page, it will return the page in Pages +// will return nil if not found +//func (ps Pages) FindPage(page *Page) *Page { +// if i := ps.FindPagePos(page); i >= 0 { +// return ps[i] +// } +// +// return nil +//} func (p *Page) Plain() string { p.initPlain() diff --git a/hugolib/site.go b/hugolib/site.go index a1e5a7868..caa9a31f1 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -451,7 +451,6 @@ func (s *Site) ReBuild(changed map[string]bool) error { } } - if len(tmplChanged) > 0 { s.prepTemplates() s.Tmpl.PrintErrors() @@ -462,10 +461,41 @@ func (s *Site) ReBuild(changed map[string]bool) error { s.ReadDataFromSourceFS() } - if len (sourceChanged) > 0 { - if err = s.CreatePages(); err != nil { - return err + if len(sourceChanged) > 0 { + + results := make(chan HandledResult) + filechan := make(chan *source.File) + errs := make(chan error) + wg := &sync.WaitGroup{} + + wg.Add(2) + for i := 0; i < 2; i++ { + go sourceReader(s, filechan, results, wg) } + + go incrementalReadCollator(s, results, errs) + + for _, x := range sourceChanged { + file, err := s.ReReadFile(x) + if err != nil { + errs <- err + } + + filechan <- file + } + + close(filechan) + wg.Wait() + close(results) + + s.timerStep("read pages from source") + + //renderErrs := <-s.ConvertSource() + s.timerStep("convert source") + // TODO(spf13) port this + + fmt.Errorf("%s", errs) + s.setupPrevNext() if err = s.BuildSiteMeta(); err != nil { return err @@ -497,7 +527,6 @@ func (s *Site) ReBuild(changed map[string]bool) error { return nil } - func (s *Site) Analyze() error { if err := s.Process(); err != nil { return err @@ -764,6 +793,47 @@ type pageResult struct { err error } +// ReReadFile resets file to be read from disk again +func (s *Site) ReReadFile(absFilePath string) (*source.File, error) { + fmt.Println("rereading", absFilePath) + var file *source.File + + reader, err := source.NewLazyFileReader(absFilePath) + if err != nil { + return nil, err + } + fmt.Println(s.absDataDir()) + + file, err = source.NewFileFromAbs(s.absContentDir(), absFilePath, reader) + + fmt.Println("file created", file.Path()) + + if err != nil { + return nil, err + } + + // maybe none of this rest needs to be here. + // replaced := false + + // fmt.Println(len(s.Files)) + + // for i, x := range s.Files { + // fmt.Println(x) + // fmt.Println("*** COMPARING:") + // fmt.Println(" ", x.LogicalName()) + // fmt.Println(" ", absFilePath) + // if x.LogicalName() == absFilePath { + // s.Files[i] = file + // replaced = true + // } + // } + + // if !replaced { + // s.Files = append(s.Files, file) + // } + return file, nil +} + func (s *Site) ReadPagesFromSource() chan error { if s.Source == nil { panic(fmt.Sprintf("s.Source not set %s", s.absContentDir())) @@ -856,12 +926,17 @@ func (s *Site) CreatePages() error { func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResult, wg *sync.WaitGroup) { defer wg.Done() for file := range files { - h := NewMetaHandler(file.Extension()) - if h != nil { - h.Read(file, s, results) - } else { - jww.ERROR.Println("Unsupported File Type", file.Path()) - } + fmt.Println("reading", file.Path()) + readSourceFile(s, file, results) + } +} + +func readSourceFile(s *Site, file *source.File, results chan<- HandledResult) { + h := NewMetaHandler(file.Extension()) + if h != nil { + h.Read(file, s, results) + } else { + jww.ERROR.Println("Unsupported File Type", file.Path()) } } @@ -905,6 +980,65 @@ func converterCollator(s *Site, results <-chan HandledResult, errs chan<- error) errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n")) } +func (s *Site) AddPage(page *Page) { + if page.ShouldBuild() { + s.Pages = append(s.Pages, page) + } + + if page.IsDraft() { + s.draftCount++ + } + + if page.IsFuture() { + s.futureCount++ + } +} + +func (s *Site) RemovePage(page *Page) { + if i := s.Pages.FindPagePos(page); i >= 0 { + if page.IsDraft() { + s.draftCount-- + } + + if page.IsFuture() { + s.futureCount-- + } + + s.Pages = append(s.Pages[:i], s.Pages[i+1:]...) + } +} + +func (s *Site) ReplacePage(page *Page) { + // will find existing page that matches filepath and remove it + s.RemovePage(page) + s.AddPage(page) +} + +func incrementalReadCollator(s *Site, results <-chan HandledResult, errs chan<- error) { + errMsgs := []string{} + for r := range results { + if r.err != nil { + errMsgs = append(errMsgs, r.Error()) + continue + } + + // !page == file + if r.page == nil { + // TODO(spf13): Make this incremental as well + s.Files = append(s.Files, r.file) + } else { + s.ReplacePage(r.page) + } + } + + s.Pages.Sort() + if len(errMsgs) == 0 { + errs <- nil + return + } + errs <- fmt.Errorf("Errors reading pages: %s", strings.Join(errMsgs, "\n")) +} + func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) { errMsgs := []string{} for r := range results { @@ -917,17 +1051,7 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) { if r.page == nil { s.Files = append(s.Files, r.file) } else { - if r.page.ShouldBuild() { - s.Pages = append(s.Pages, r.page) - } - - if r.page.IsDraft() { - s.draftCount++ - } - - if r.page.IsFuture() { - s.futureCount++ - } + s.AddPage(r.page) } } diff --git a/source/file.go b/source/file.go index a132cefde..efe604912 100644 --- a/source/file.go +++ b/source/file.go @@ -14,14 +14,16 @@ package source import ( - "github.com/spf13/hugo/helpers" "io" "path/filepath" "strings" + + "github.com/spf13/hugo/helpers" ) +// All paths are relative from the source directory base type File struct { - relpath string // Original Full Path eg. /Users/Home/Hugo/foo.txt + relpath string // Original Full Path eg. content/foo.txt logicalName string // foo.txt Contents io.Reader section string // The first directory @@ -30,6 +32,7 @@ type File struct { uniqueID string // MD5 of the filename } +// UniqueID: MD5 of the filename func (f *File) UniqueID() string { return f.uniqueID } @@ -42,15 +45,17 @@ func (f *File) Bytes() []byte { return helpers.ReaderToBytes(f.Contents) } -// Filename without extension +// BaseFileName Filename without extension func (f *File) BaseFileName() string { return helpers.Filename(f.LogicalName()) } +// Section The first directory func (f *File) Section() string { return f.section } +// LogicalName The filename and extension of the file func (f *File) LogicalName() string { return f.logicalName } @@ -71,6 +76,7 @@ func (f *File) Ext() string { return f.Extension() } +// Path the relative path including file name and extension from the base of the source directory func (f *File) Path() string { return f.relpath } diff --git a/source/filesystem.go b/source/filesystem.go index 75cf09d56..09118c27a 100644 --- a/source/filesystem.go +++ b/source/filesystem.go @@ -14,13 +14,14 @@ package source import ( - "github.com/spf13/viper" "io" "os" "path/filepath" "regexp" "strings" + "github.com/spf13/viper" + "github.com/spf13/hugo/helpers" jww "github.com/spf13/jwalterweatherman" ) @@ -59,14 +60,11 @@ func (f *Filesystem) Files() []*File { return f.files } +// add populates a file in the Filesystem.files func (f *Filesystem) add(name string, reader io.Reader) (err error) { var file *File - //if f.Base == "" { - //file = NewFileWithContents(name, reader) - //} else { file, err = NewFileFromAbs(f.Base, name, reader) - //} if err == nil { f.files = append(f.files, file) @@ -79,50 +77,59 @@ func (f *Filesystem) getRelativePath(name string) (final string, err error) { } func (f *Filesystem) captureFiles() { - walker := func(filePath string, fi os.FileInfo, err error) error { if err != nil { return nil } - if fi.Mode()&os.ModeSymlink == os.ModeSymlink { - link, err := filepath.EvalSymlinks(filePath) - if err != nil { - jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err) - return nil - } - linkfi, err := os.Stat(link) - if err != nil { - jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) - return nil - } - if !linkfi.Mode().IsRegular() { - jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath) - } - return nil - } - - if fi.IsDir() { - if f.avoid(filePath) || isNonProcessablePath(filePath) { - return filepath.SkipDir - } - return nil - } - - if isNonProcessablePath(filePath) { - return nil - } - rd, err := NewLazyFileReader(filePath) + b, err := f.shouldRead(filePath, fi) if err != nil { return err } - f.add(filePath, rd) - return nil + if b { + rd, err := NewLazyFileReader(filePath) + if err != nil { + return err + } + f.add(filePath, rd) + } + return err } filepath.Walk(f.Base, walker) } +func (f *Filesystem) shouldRead(filePath string, fi os.FileInfo) (bool, error) { + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + link, err := filepath.EvalSymlinks(filePath) + if err != nil { + jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err) + return false, nil + } + linkfi, err := os.Stat(link) + if err != nil { + jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err) + return false, nil + } + if !linkfi.Mode().IsRegular() { + jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath) + } + return false, nil + } + + if fi.IsDir() { + if f.avoid(filePath) || isNonProcessablePath(filePath) { + return false, filepath.SkipDir + } + return false, nil + } + + if isNonProcessablePath(filePath) { + return false, nil + } + return true, nil +} + func (f *Filesystem) avoid(filePath string) bool { for _, avoid := range f.AvoidPaths { if avoid == filePath {