Parallelizing the largest build processes (templates, pages, indexes). Seeing 300%+ improvement of total build time.

hugolib/site.go

@@ -27,6 +27,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -249,29 +250,39 @@ func (s *Site) CreatePages() (err error) {
 	if len(s.Source.Files()) < 1 {
 		return fmt.Errorf("No source files found in %s", s.absContentDir())
 	}
-	for _, file := range s.Source.Files() {
-		page, err := ReadFrom(file.Contents, file.LogicalName)
-		if err != nil {
-			return err
-		}
-		page.Site = s.Info
-		page.Tmpl = s.Tmpl
-		page.Section = file.Section
-		page.Dir = file.Dir
 
-		//Handling short codes prior to Conversion to HTML
+	var wg sync.WaitGroup
-		page.ProcessShortcodes(s.Tmpl)
+	for _, fi := range s.Source.Files() {
+		wg.Add(1)
+		go func(file *source.File) (err error) {
+			defer wg.Done()
 
-		err = page.Convert()
+			page, err := ReadFrom(file.Contents, file.LogicalName)
-		if err != nil {
+			if err != nil {
-			return err
+				return err
-		}
+			}
+			page.Site = s.Info
+			page.Tmpl = s.Tmpl
+			page.Section = file.Section
+			page.Dir = file.Dir
 
-		if s.Config.BuildDrafts || !page.Draft {
+			//Handling short codes prior to Conversion to HTML
-			s.Pages = append(s.Pages, page)
+			page.ProcessShortcodes(s.Tmpl)
-		}
+
+			err = page.Convert()
+			if err != nil {
+				return err
+			}
+
+			if s.Config.BuildDrafts || !page.Draft {
+				s.Pages = append(s.Pages, page)
+			}
+
+			return
+		}(fi)
 	}
 
+	wg.Wait()
 	s.Pages.Sort()
 	return
 }
@@ -368,55 +379,70 @@ func (s *Site) RenderAliases() error {
 }
 
 func (s *Site) RenderPages() (err error) {
-	for _, p := range s.Pages {
+	var wg sync.WaitGroup
-		var layout []string
+	for _, page := range s.Pages {
+		wg.Add(1)
+		go func(p *Page) (err error) {
+			var layout []string
+			defer wg.Done()
 
-		if !p.IsRenderable() {
+			if !p.IsRenderable() {
-			self := "__" + p.TargetPath()
+				self := "__" + p.TargetPath()
-			_, err := s.Tmpl.New(self).Parse(string(p.Content))
+				_, err := s.Tmpl.New(self).Parse(string(p.Content))
-			if err != nil {
+				if err != nil {
-				return err
+					return err
+				}
+				layout = append(layout, self)
+			} else {
+				layout = append(layout, p.Layout()...)
+				layout = append(layout, "_default/single.html")
 			}
-			layout = append(layout, self)
-		} else {
-			layout = append(layout, p.Layout()...)
-			layout = append(layout, "_default/single.html")
-		}
 
-		err := s.render(p, p.TargetPath(), layout...)
+			return s.render(p, p.TargetPath(), layout...)
-		if err != nil {
+		}(page)
-			return err
+	}
-		}
+	wg.Wait()
+
+	if err != nil {
+		return err
 	}
 	return nil
 }
 
-func (s *Site) RenderIndexes() error {
+func (s *Site) RenderIndexes() (err error) {
+	var wg sync.WaitGroup
 	for singular, plural := range s.Config.Indexes {
-		for k, o := range s.Indexes[plural] {
+		for key, oo := range s.Indexes[plural] {
-			n := s.NewNode()
+			wg.Add(1)
-			n.Title = strings.Title(k)
-			base := plural + "/" + k
-			s.setUrls(n, base)
-			n.Date = o[0].Page.Date
-			n.Data[singular] = o
-			n.Data["Pages"] = o.Pages()
-			layout := "indexes/" + singular + ".html"
-			err := s.render(n, base+".html", layout)
-			if err != nil {
-				return err
-			}
 
-			if a := s.Tmpl.Lookup("rss.xml"); a != nil {
+			go func(k string, o WeightedPages) (err error) {
-				// XML Feed
+				defer wg.Done()
-				s.setUrls(n, base+".xml")
+				n := s.NewNode()
-				err := s.render(n, base+".xml", "rss.xml")
+				n.Title = strings.Title(k)
+				base := plural + "/" + k
+				s.setUrls(n, base)
+				n.Date = o[0].Page.Date
+				n.Data[singular] = o
+				n.Data["Pages"] = o.Pages()
+				layout := "indexes/" + singular + ".html"
+				err = s.render(n, base+".html", layout)
 				if err != nil {
 					return err
 				}
-			}
+
+				if a := s.Tmpl.Lookup("rss.xml"); a != nil {
+					// XML Feed
+					s.setUrls(n, base+".xml")
+					err := s.render(n, base+".xml", "rss.xml")
+					if err != nil {
+						return err
+					}
+				}
+				return
+			}(key, oo)
 		}
 	}
+	wg.Wait()
 	return nil
 }
 

main.go

@@ -15,8 +15,10 @@ package main
 
 import (
 	"github.com/spf13/hugo/commands"
+	"runtime"
 )
 
 func main() {
+	runtime.GOMAXPROCS(runtime.NumCPU())
 	commands.Execute()
 }