limit the number of goroutines used in taxonomy rendering to gomaxprocs*4

This commit is contained in:
Nate Finch 2014-09-14 07:23:03 -04:00
parent 49045b1cbc
commit b590de6917

View file

@ -15,6 +15,7 @@ package hugolib
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"html/template" "html/template"
"io" "io"
@ -664,7 +665,7 @@ func (s *Site) RenderAliases() error {
} }
// Render pages each corresponding to a markdown file // Render pages each corresponding to a markdown file
func (s *Site) RenderPages() (err error) { func (s *Site) RenderPages() error {
results := make(chan error) results := make(chan error)
pages := make(chan *Page) pages := make(chan *Page)
@ -680,7 +681,7 @@ func (s *Site) RenderPages() (err error) {
errs := make(chan error) errs := make(chan error)
go renderCollator(s, results, errs) go errorCollator(results, errs)
for _, page := range s.Pages { for _, page := range s.Pages {
pages <- page pages <- page
@ -692,7 +693,11 @@ func (s *Site) RenderPages() (err error) {
close(results) close(results)
return <-errs err := <-errs
if err != nil {
return fmt.Errorf("Error(s) rendering pages: %s", err)
}
return nil
} }
func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.WaitGroup) { func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.WaitGroup) {
@ -717,19 +722,19 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
} }
} }
func renderCollator(s *Site, results <-chan error, errs chan<- error) { func errorCollator(results <-chan error, errs chan<- error) {
errMsgs := []string{} errMsgs := []string{}
for err := range results { for err := range results {
if err != nil { if err != nil {
errMsgs = append(errMsgs, err.Error()) errMsgs = append(errMsgs, err.Error())
continue
} }
} }
if len(errMsgs) == 0 { if len(errMsgs) == 0 {
errs <- nil errs <- nil
return } else {
errs <- errors.New(strings.Join(errMsgs, "\n"))
} }
errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n")) close(errs)
} }
func (s *Site) appendThemeTemplates(in []string) []string { func (s *Site) appendThemeTemplates(in []string) []string {
@ -760,47 +765,82 @@ func (s *Site) appendThemeTemplates(in []string) []string {
} }
} }
type taxRenderInfo struct {
key string
pages WeightedPages
singular string
plural string
}
// Render the listing pages based on the meta data // Render the listing pages based on the meta data
// each unique term within a taxonomy will have a page created // each unique term within a taxonomy will have a page created
func (s *Site) RenderTaxonomiesLists() (err error) { func (s *Site) RenderTaxonomiesLists() error {
var wg sync.WaitGroup wg := &sync.WaitGroup{}
taxes := make(chan taxRenderInfo)
results := make(chan error)
procs := getGoMaxProcs()
for i := 0; i < procs*4; i++ {
wg.Add(1)
go taxonomyRenderer(s, taxes, results, wg)
}
errs := make(chan error)
go errorCollator(results, errs)
taxonomies := viper.GetStringMapString("Taxonomies") taxonomies := viper.GetStringMapString("Taxonomies")
for sing, pl := range taxonomies { for singular, plural := range taxonomies {
for key, oo := range s.Taxonomies[pl] { for key, pages := range s.Taxonomies[plural] {
wg.Add(1) taxes <- taxRenderInfo{key, pages, singular, plural}
go func(k string, o WeightedPages, singular string, plural string) (err error) {
defer wg.Done()
base := plural + "/" + k
n := s.NewNode()
n.Title = strings.Title(k)
s.setUrls(n, base)
n.Date = o[0].Page.Date
n.Data[singular] = o
n.Data["Pages"] = o.Pages()
layouts := []string{"taxonomy/" + singular + ".html", "indexes/" + singular + ".html", "_default/taxonomy.html", "_default/list.html"}
err = s.render(n, base+".html", s.appendThemeTemplates(layouts)...)
if err != nil {
return err
}
if !viper.GetBool("DisableRSS") {
// XML Feed
s.setUrls(n, base+".xml")
rssLayouts := []string{"taxonomy/" + singular + ".rss.xml", "_default/rss.xml", "rss.xml", "_internal/_default/rss.xml"}
err := s.render(n, base+".xml", s.appendThemeTemplates(rssLayouts)...)
if err != nil {
return err
}
}
return
}(key, oo, sing, pl)
} }
} }
close(taxes)
wg.Wait() wg.Wait()
close(results)
err := <-errs
if err != nil {
return fmt.Errorf("Error(s) rendering taxonomies: %s", err)
}
return nil return nil
} }
func taxonomyRenderer(s *Site, taxes <-chan taxRenderInfo, results chan<- error, wg *sync.WaitGroup) {
defer wg.Done()
for t := range taxes {
base := t.plural + "/" + t.key
n := s.NewNode()
n.Title = strings.Title(t.key)
s.setUrls(n, base)
n.Date = t.pages[0].Page.Date
n.Data[t.singular] = t.pages
n.Data["Pages"] = t.pages.Pages()
layouts := []string{"taxonomy/" + t.singular + ".html", "indexes/" + t.singular + ".html", "_default/taxonomy.html", "_default/list.html"}
err := s.render(n, base+".html", s.appendThemeTemplates(layouts)...)
if err != nil {
results <- err
continue
}
if !viper.GetBool("DisableRSS") {
// XML Feed
s.setUrls(n, base+".xml")
rssLayouts := []string{"taxonomy/" + t.singular + ".rss.xml", "_default/rss.xml", "rss.xml", "_internal/_default/rss.xml"}
err := s.render(n, base+".xml", s.appendThemeTemplates(rssLayouts)...)
if err != nil {
results <- err
continue
}
}
results <- nil
}
}
// Render a page per taxonomy that lists the terms for that taxonomy // Render a page per taxonomy that lists the terms for that taxonomy
func (s *Site) RenderListsOfTaxonomyTerms() (err error) { func (s *Site) RenderListsOfTaxonomyTerms() (err error) {
taxonomies := viper.GetStringMapString("Taxonomies") taxonomies := viper.GetStringMapString("Taxonomies")