Read/reread individual source content files

next is incremental conversion
This commit is contained in:
Steve Francia 2016-01-07 21:48:13 -05:00
parent ca6ca4f4fc
commit 9f3796a31d
7 changed files with 253 additions and 81 deletions

View file

@ -19,7 +19,6 @@ In no particular order, here is what we are working on:
* Import from other website systems
* from Drupal (See https://bitbucket.org/rickb777/drupal2hugo by Rick Beton (@rickb777))
* from WordPress (See [#100][], especially https://github.com/SchumacherFM/wordpress-to-hugo-exporter by Cyrill Schumacher (@SchumacherFM), but volunteers are needed to make it work with latest versions of WordPress.)
* from Jekyll (See [#101][])
* An interactive web based editor (See http://discuss.gohugo.io/t/web-based-editor/155)
* Additional [themes](https://github.com/spf13/hugoThemes) (always on-going, contributions welcome!)
* Dynamic image resizing via shortcodes

View file

@ -32,6 +32,7 @@ type basicPageHandler Handle
func (b basicPageHandler) Read(f *source.File, s *Site) HandledResult {
page, err := NewPage(f.Path())
if err != nil {
return HandledResult{file: f, err: err}
}

View file

@ -61,7 +61,6 @@ type Page struct {
PublishDate time.Time
Tmpl tpl.Template
Markup string
extension string
contentType string
renderable bool
@ -77,13 +76,13 @@ type Page struct {
plainSecondaryInit sync.Once
renderingConfig *helpers.Blackfriday
renderingConfigInit sync.Once
pageMenus PageMenus
pageMenusInit sync.Once
isCJKLanguage bool
PageMeta
Source
Position `json:"-"`
Node
pageMenus PageMenus
pageMenusInit sync.Once
isCJKLanguage bool
}
type Source struct {
@ -106,6 +105,42 @@ type Position struct {
}
type Pages []*Page
//
//func (ps Pages) Replace(page *Page) {
// if i := ps.FindPagePos(page); i >= 0 {
// ps[i] = page
// }
//}
//func (ps Pages) FindPageByFilePath(inPath string) *Page {
// for _, x := range ps {
// if x.Source.LogicalName() == inPath {
// return x
// }
// }
// return nil
//}
// FindPagePos Given a page, it will find the position in Pages
// will return -1 if not found
func (ps Pages) FindPagePos(page *Page) int {
for i, x := range ps {
if x.Source.LogicalName() == page.Source.LogicalName() {
return i
}
}
return -1
}
// FindPage Given a page, it will return the page in Pages
// will return nil if not found
//func (ps Pages) FindPage(page *Page) *Page {
// if i := ps.FindPagePos(page); i >= 0 {
// return ps[i]
// }
//
// return nil
//}
func (p *Page) Plain() string {
p.initPlain()

View file

@ -451,7 +451,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
}
}
if len(tmplChanged) > 0 {
s.prepTemplates()
s.Tmpl.PrintErrors()
@ -463,9 +462,40 @@ func (s *Site) ReBuild(changed map[string]bool) error {
}
if len(sourceChanged) > 0 {
if err = s.CreatePages(); err != nil {
return err
results := make(chan HandledResult)
filechan := make(chan *source.File)
errs := make(chan error)
wg := &sync.WaitGroup{}
wg.Add(2)
for i := 0; i < 2; i++ {
go sourceReader(s, filechan, results, wg)
}
go incrementalReadCollator(s, results, errs)
for _, x := range sourceChanged {
file, err := s.ReReadFile(x)
if err != nil {
errs <- err
}
filechan <- file
}
close(filechan)
wg.Wait()
close(results)
s.timerStep("read pages from source")
//renderErrs := <-s.ConvertSource()
s.timerStep("convert source")
// TODO(spf13) port this
fmt.Errorf("%s", errs)
s.setupPrevNext()
if err = s.BuildSiteMeta(); err != nil {
return err
@ -497,7 +527,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
return nil
}
func (s *Site) Analyze() error {
if err := s.Process(); err != nil {
return err
@ -764,6 +793,47 @@ type pageResult struct {
err error
}
// ReReadFile resets file to be read from disk again
func (s *Site) ReReadFile(absFilePath string) (*source.File, error) {
fmt.Println("rereading", absFilePath)
var file *source.File
reader, err := source.NewLazyFileReader(absFilePath)
if err != nil {
return nil, err
}
fmt.Println(s.absDataDir())
file, err = source.NewFileFromAbs(s.absContentDir(), absFilePath, reader)
fmt.Println("file created", file.Path())
if err != nil {
return nil, err
}
// maybe none of this rest needs to be here.
// replaced := false
// fmt.Println(len(s.Files))
// for i, x := range s.Files {
// fmt.Println(x)
// fmt.Println("*** COMPARING:")
// fmt.Println(" ", x.LogicalName())
// fmt.Println(" ", absFilePath)
// if x.LogicalName() == absFilePath {
// s.Files[i] = file
// replaced = true
// }
// }
// if !replaced {
// s.Files = append(s.Files, file)
// }
return file, nil
}
func (s *Site) ReadPagesFromSource() chan error {
if s.Source == nil {
panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
@ -856,6 +926,12 @@ func (s *Site) CreatePages() error {
func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResult, wg *sync.WaitGroup) {
defer wg.Done()
for file := range files {
fmt.Println("reading", file.Path())
readSourceFile(s, file, results)
}
}
func readSourceFile(s *Site, file *source.File, results chan<- HandledResult) {
h := NewMetaHandler(file.Extension())
if h != nil {
h.Read(file, s, results)
@ -863,7 +939,6 @@ func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResu
jww.ERROR.Println("Unsupported File Type", file.Path())
}
}
}
func pageConverter(s *Site, pages <-chan *Page, results HandleResults, wg *sync.WaitGroup) {
defer wg.Done()
@ -905,6 +980,65 @@ func converterCollator(s *Site, results <-chan HandledResult, errs chan<- error)
errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
}
func (s *Site) AddPage(page *Page) {
if page.ShouldBuild() {
s.Pages = append(s.Pages, page)
}
if page.IsDraft() {
s.draftCount++
}
if page.IsFuture() {
s.futureCount++
}
}
func (s *Site) RemovePage(page *Page) {
if i := s.Pages.FindPagePos(page); i >= 0 {
if page.IsDraft() {
s.draftCount--
}
if page.IsFuture() {
s.futureCount--
}
s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
}
}
func (s *Site) ReplacePage(page *Page) {
// will find existing page that matches filepath and remove it
s.RemovePage(page)
s.AddPage(page)
}
func incrementalReadCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
errMsgs := []string{}
for r := range results {
if r.err != nil {
errMsgs = append(errMsgs, r.Error())
continue
}
// !page == file
if r.page == nil {
// TODO(spf13): Make this incremental as well
s.Files = append(s.Files, r.file)
} else {
s.ReplacePage(r.page)
}
}
s.Pages.Sort()
if len(errMsgs) == 0 {
errs <- nil
return
}
errs <- fmt.Errorf("Errors reading pages: %s", strings.Join(errMsgs, "\n"))
}
func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
errMsgs := []string{}
for r := range results {
@ -917,17 +1051,7 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
if r.page == nil {
s.Files = append(s.Files, r.file)
} else {
if r.page.ShouldBuild() {
s.Pages = append(s.Pages, r.page)
}
if r.page.IsDraft() {
s.draftCount++
}
if r.page.IsFuture() {
s.futureCount++
}
s.AddPage(r.page)
}
}

View file

@ -14,14 +14,16 @@
package source
import (
"github.com/spf13/hugo/helpers"
"io"
"path/filepath"
"strings"
"github.com/spf13/hugo/helpers"
)
// All paths are relative from the source directory base
type File struct {
relpath string // Original Full Path eg. /Users/Home/Hugo/foo.txt
relpath string // Original Full Path eg. content/foo.txt
logicalName string // foo.txt
Contents io.Reader
section string // The first directory
@ -30,6 +32,7 @@ type File struct {
uniqueID string // MD5 of the filename
}
// UniqueID: MD5 of the filename
func (f *File) UniqueID() string {
return f.uniqueID
}
@ -42,15 +45,17 @@ func (f *File) Bytes() []byte {
return helpers.ReaderToBytes(f.Contents)
}
// Filename without extension
// BaseFileName Filename without extension
func (f *File) BaseFileName() string {
return helpers.Filename(f.LogicalName())
}
// Section The first directory
func (f *File) Section() string {
return f.section
}
// LogicalName The filename and extension of the file
func (f *File) LogicalName() string {
return f.logicalName
}
@ -71,6 +76,7 @@ func (f *File) Ext() string {
return f.Extension()
}
// Path the relative path including file name and extension from the base of the source directory
func (f *File) Path() string {
return f.relpath
}

View file

@ -14,13 +14,14 @@
package source
import (
"github.com/spf13/viper"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/spf13/viper"
"github.com/spf13/hugo/helpers"
jww "github.com/spf13/jwalterweatherman"
)
@ -59,14 +60,11 @@ func (f *Filesystem) Files() []*File {
return f.files
}
// add populates a file in the Filesystem.files
func (f *Filesystem) add(name string, reader io.Reader) (err error) {
var file *File
//if f.Base == "" {
//file = NewFileWithContents(name, reader)
//} else {
file, err = NewFileFromAbs(f.Base, name, reader)
//}
if err == nil {
f.files = append(f.files, file)
@ -79,50 +77,59 @@ func (f *Filesystem) getRelativePath(name string) (final string, err error) {
}
func (f *Filesystem) captureFiles() {
walker := func(filePath string, fi os.FileInfo, err error) error {
if err != nil {
return nil
}
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filePath)
b, err := f.shouldRead(filePath, fi)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
return nil
}
linkfi, err := os.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return nil
}
if !linkfi.Mode().IsRegular() {
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
}
return nil
}
if fi.IsDir() {
if f.avoid(filePath) || isNonProcessablePath(filePath) {
return filepath.SkipDir
}
return nil
}
if isNonProcessablePath(filePath) {
return nil
return err
}
if b {
rd, err := NewLazyFileReader(filePath)
if err != nil {
return err
}
f.add(filePath, rd)
return nil
}
return err
}
filepath.Walk(f.Base, walker)
}
func (f *Filesystem) shouldRead(filePath string, fi os.FileInfo) (bool, error) {
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
link, err := filepath.EvalSymlinks(filePath)
if err != nil {
jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
return false, nil
}
linkfi, err := os.Stat(link)
if err != nil {
jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
return false, nil
}
if !linkfi.Mode().IsRegular() {
jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
}
return false, nil
}
if fi.IsDir() {
if f.avoid(filePath) || isNonProcessablePath(filePath) {
return false, filepath.SkipDir
}
return false, nil
}
if isNonProcessablePath(filePath) {
return false, nil
}
return true, nil
}
func (f *Filesystem) avoid(filePath string) bool {
for _, avoid := range f.AvoidPaths {
if avoid == filePath {