Fix CreatePages

This fixes #450.  There are two problems:

1.) We're creating a new goroutine for every page.
2.) We're calling s.Pages = append(s.Pages, page) inside each goroutine.

1 is a problem if in that if you have a ton of pages, that's a ton of goroutines.  It's not really useful to have more than a few goroutines at a time, and lots can actually make your code much slower, and, evidently, crash.

2 is a problem in that append is not thread safe. Sometimes it returns a new slice with a larger capacity, when the original slice isn't large enough.  This can cause problems if two goroutines do this at the same time.

The solution for 1 is to use a limited number of workers (I chose 2*GOMAXPROCS as a nice guess).
The solution for 2 is to serialize access to s.Pages, which I did by doing it in a single goroutine.
This commit is contained in:
Nate Finch 2014-08-29 13:40:21 -04:00 committed by spf13
parent 93addfcbee
commit 47c91a4ca2

View file

@ -19,6 +19,7 @@ import (
"html/template" "html/template"
"io" "io"
"os" "os"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -311,60 +312,107 @@ func (s *Site) checkDirectories() (err error) {
return return
} }
func (s *Site) CreatePages() (err error) { type pageRenderResult struct {
page *Page
err error
}
func (s *Site) CreatePages() error {
if s.Source == nil { if s.Source == nil {
panic(fmt.Sprintf("s.Source not set %s", s.absContentDir())) panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
} }
if len(s.Source.Files()) < 1 { if len(s.Source.Files()) < 1 {
return return nil
} }
var wg sync.WaitGroup files := s.Source.Files()
for _, fi := range s.Source.Files() {
results := make(chan pageRenderResult)
input := make(chan *source.File)
procs := getGoMaxProcs()
wg := &sync.WaitGroup{}
for i := 0; i < procs*2; i++ {
wg.Add(1) wg.Add(1)
go func(file *source.File) (err error) { go pageRenderer(s, input, results, wg)
defer wg.Done()
page, err := NewPage(file.LogicalName)
if err != nil {
return err
}
err = page.ReadFrom(file.Contents)
if err != nil {
return err
}
page.Site = &s.Info
page.Tmpl = s.Tmpl
page.Section = file.Section
page.Dir = file.Dir
//Handling short codes prior to Conversion to HTML
page.ProcessShortcodes(s.Tmpl)
err = page.Convert()
if err != nil {
return err
}
if page.ShouldBuild() {
s.Pages = append(s.Pages, page)
}
if page.IsDraft() {
s.draftCount += 1
}
if page.IsFuture() {
s.futureCount += 1
}
return
}(fi)
} }
errs := make(chan error)
// we can only have exactly one result collator, since it makes changes that
// must be synchronized.
go resultCollator(s, results, errs)
for _, fi := range files {
input <- fi
}
close(input)
wg.Wait() wg.Wait()
close(results)
return <-errs
}
func pageRenderer(s *Site, input <-chan *source.File, results chan<- pageRenderResult, wg *sync.WaitGroup) {
for file := range input {
page, err := NewPage(file.LogicalName)
if err != nil {
results <- pageRenderResult{nil, err}
continue
}
err = page.ReadFrom(file.Contents)
if err != nil {
results <- pageRenderResult{nil, err}
continue
}
page.Site = &s.Info
page.Tmpl = s.Tmpl
page.Section = file.Section
page.Dir = file.Dir
//Handling short codes prior to Conversion to HTML
page.ProcessShortcodes(s.Tmpl)
err = page.Convert()
if err != nil {
results <- pageRenderResult{nil, err}
continue
}
results <- pageRenderResult{page, nil}
}
wg.Done()
}
func resultCollator(s *Site, results <-chan pageRenderResult, errs chan<- error) {
errMsgs := []string{}
for r := range results {
if r.err != nil {
errMsgs = append(errMsgs, r.err.Error())
continue
}
if r.page.ShouldBuild() {
s.Pages = append(s.Pages, r.page)
}
if r.page.IsDraft() {
s.draftCount += 1
}
if r.page.IsFuture() {
s.futureCount += 1
}
}
s.Pages.Sort() s.Pages.Sort()
return if len(errMsgs) == 0 {
errs <- nil
}
errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
} }
func (s *Site) BuildSiteMeta() (err error) { func (s *Site) BuildSiteMeta() (err error) {
@ -1010,3 +1058,12 @@ func (s *Site) futureStats() string {
return "0 of " + msg return "0 of " + msg
} }
func getGoMaxProcs() int {
if gmp := os.Getenv("GOMAXPROCS"); gmp != "" {
if p, err := strconv.Atoi(gmp); err != nil {
return p
}
}
return 1
}