hugo/hugolib/hugo_sites.go

// Copyright 2016-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"errors"
	"io"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/gohugoio/hugo/resource"

	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"

	"github.com/gohugoio/hugo/i18n"
	"github.com/gohugoio/hugo/tpl"
	"github.com/gohugoio/hugo/tpl/tplimpl"
)

// HugoSites represents the sites to build. Each site represents a language.
type HugoSites struct {
	Sites []*Site

	multilingual *Multilingual

	// Multihost is set if multilingual and baseURL set on the language level.
	multihost bool

	// If this is running in the dev server.
	running bool

	*deps.Deps

	// Keeps track of bundle directories and symlinks to enable partial rebuilding.
	ContentChanges *contentChangeMap

	// If enabled, keeps a revision map for all content.
	gitInfo *gitInfo
}

func (h *HugoSites) IsMultihost() bool {
	return h != nil && h.multihost
}

func (h *HugoSites) PrintProcessingStats(w io.Writer) {
	stats := make([]*helpers.ProcessingStats, len(h.Sites))
	for i := 0; i < len(h.Sites); i++ {
		stats[i] = h.Sites[i].PathSpec.ProcessingStats
	}
	helpers.ProcessingStatsTable(w, stats...)
}

func (h *HugoSites) langSite() map[string]*Site {
	m := make(map[string]*Site)
	for _, s := range h.Sites {
		m[s.Language.Lang] = s
	}
	return m
}

// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) *Page {
	for _, s := range h.Sites {
		pos := s.rawAllPages.findPagePosByFilename(filename)
		if pos == -1 {
			continue
		}
		return s.rawAllPages[pos]
	}

	// If not found already, this may be bundled in another content file.
	dir := filepath.Dir(filename)

	for _, s := range h.Sites {
		pos := s.rawAllPages.findPagePosByFilnamePrefix(dir)
		if pos == -1 {
			continue
		}
		return s.rawAllPages[pos]
	}
	return nil
}

// NewHugoSites creates a new collection of sites given the input sites, building
// a language configuration based on those.
func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {

	if cfg.Language != nil {
		return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
	}

	langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)

	if err != nil {
		return nil, err
	}

	var contentChangeTracker *contentChangeMap

	h := &HugoSites{
		running:      cfg.Running,
		multilingual: langConfig,
		multihost:    cfg.Cfg.GetBool("multihost"),
		Sites:        sites}

	for _, s := range sites {
		s.owner = h
	}

	if err := applyDepsIfNeeded(cfg, sites...); err != nil {
		return nil, err
	}

	h.Deps = sites[0].Deps

	// Only needed in server mode.
	// TODO(bep) clean up the running vs watching terms
	if cfg.Running {
		contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}
		h.ContentChanges = contentChangeTracker
	}

	if err := h.initGitInfo(); err != nil {
		return nil, err
	}

	return h, nil
}

func (h *HugoSites) initGitInfo() error {
	if h.Cfg.GetBool("enableGitInfo") {
		gi, err := newGitInfo(h.Cfg)
		if err != nil {
			h.Log.ERROR.Println("Failed to read Git log:", err)
		} else {
			h.gitInfo = gi
		}
	}
	return nil
}

func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error {
	if cfg.TemplateProvider == nil {
		cfg.TemplateProvider = tplimpl.DefaultTemplateProvider
	}

	if cfg.TranslationProvider == nil {
		cfg.TranslationProvider = i18n.NewTranslationProvider()
	}

	var (
		d   *deps.Deps
		err error
	)

	for _, s := range sites {
		if s.Deps != nil {
			continue
		}

		if d == nil {
			cfg.Language = s.Language
			cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)

			var err error
			d, err = deps.New(cfg)
			if err != nil {
				return err
			}

			d.OutputFormatsConfig = s.outputFormatsConfig
			s.Deps = d

			if err = d.LoadResources(); err != nil {
				return err
			}

		} else {
			d, err = d.ForLanguage(s.Language)
			if err != nil {
				return err
			}
			d.OutputFormatsConfig = s.outputFormatsConfig
			s.Deps = d
		}

		s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig)
		if err != nil {
			return err
		}

	}

	return nil
}

// NewHugoSites creates HugoSites from the given config.
func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
	sites, err := createSitesFromConfig(cfg)
	if err != nil {
		return nil, err
	}
	return newHugoSites(cfg, sites...)
}

func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error {
	return func(templ tpl.TemplateHandler) error {
		templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "")
		if s.PathSpec.ThemeSet() {
			templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme")
		}

		for _, wt := range withTemplates {
			if wt == nil {
				continue
			}
			if err := wt(templ); err != nil {
				return err
			}
		}

		return nil
	}
}

func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {

	var (
		sites []*Site
	)

	languages := getLanguages(cfg.Cfg)

	for _, lang := range languages {
		if lang.Disabled {
			continue
		}
		var s *Site
		var err error
		cfg.Language = lang
		s, err = newSite(cfg)

		if err != nil {
			return nil, err
		}

		sites = append(sites, s)
	}

	return sites, nil
}

// Reset resets the sites and template caches, making it ready for a full rebuild.
func (h *HugoSites) reset() {
	for i, s := range h.Sites {
		h.Sites[i] = s.reset()
	}
}

func (h *HugoSites) createSitesFromConfig() error {
	oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages)

	if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil {
		return err
	}

	depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg}

	sites, err := createSitesFromConfig(depsCfg)

	if err != nil {
		return err
	}

	langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...)

	if err != nil {
		return err
	}

	h.Sites = sites

	for _, s := range sites {
		s.owner = h
	}

	if err := applyDepsIfNeeded(depsCfg, sites...); err != nil {
		return err
	}

	h.Deps = sites[0].Deps

	h.multilingual = langConfig
	h.multihost = h.Deps.Cfg.GetBool("multihost")

	return nil
}

func (h *HugoSites) toSiteInfos() []*SiteInfo {
	infos := make([]*SiteInfo, len(h.Sites))
	for i, s := range h.Sites {
		infos[i] = &s.Info
	}
	return infos
}

// BuildCfg holds build options used to, as an example, skip the render step.
type BuildCfg struct {
	// Reset site state before build. Use to force full rebuilds.
	ResetState bool
	// Re-creates the sites from configuration before a build.
	// This is needed if new languages are added.
	CreateSitesFromConfig bool
	// Skip rendering. Useful for testing.
	SkipRender bool
	// Use this to indicate what changed (for rebuilds).
	whatChanged *whatChanged
	// Recently visited URLs. This is used for partial re-rendering.
	RecentlyVisited map[string]bool
}

// shouldRender is used in the Fast Render Mode to determine if we need to re-render
// a Page: If it is recently visited (the home pages will always be in this set) or changed.
// Note that a page does not have to have a content page / file.
// For regular builds, this will allways return true.
func (cfg *BuildCfg) shouldRender(p *Page) bool {
	if len(cfg.RecentlyVisited) == 0 {
		return true
	}

	if cfg.RecentlyVisited[p.RelPermalink()] {
		return true
	}

	if cfg.whatChanged != nil && p.File != nil {
		return cfg.whatChanged.files[p.File.Filename()]
	}

	return false
}

func (h *HugoSites) renderCrossSitesArtifacts() error {

	if !h.multilingual.enabled() || h.IsMultihost() {
		return nil
	}

	sitemapEnabled := false
	for _, s := range h.Sites {
		if s.isEnabled(kindSitemap) {
			sitemapEnabled = true
			break
		}
	}

	if !sitemapEnabled {
		return nil
	}

	// TODO(bep) DRY
	sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap"))

	s := h.Sites[0]

	smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}

	return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex",
		sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...)
}

func (h *HugoSites) assignMissingTranslations() error {

	// This looks heavy, but it should be a small number of nodes by now.
	allPages := h.findAllPagesByKindNotIn(KindPage)
	for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} {
		nodes := h.findPagesByKindIn(nodeType, allPages)

		// Assign translations
		for _, t1 := range nodes {
			for _, t2 := range nodes {
				if t1.isNewTranslation(t2) {
					t1.translations = append(t1.translations, t2)
				}
			}
		}
	}

	// Now we can sort the translations.
	for _, p := range allPages {
		if len(p.translations) > 0 {
			pageBy(languagePageSort).Sort(p.translations)
		}
	}
	return nil

}

// createMissingPages creates home page, taxonomies etc. that isnt't created as an
// effect of having a content file.
func (h *HugoSites) createMissingPages() error {
	var newPages Pages

	for _, s := range h.Sites {
		if s.isEnabled(KindHome) {
			// home pages
			home := s.findPagesByKind(KindHome)
			if len(home) > 1 {
				panic("Too many homes")
			}
			if len(home) == 0 {
				n := s.newHomePage()
				s.Pages = append(s.Pages, n)
				newPages = append(newPages, n)
			}
		}

		// Will create content-less root sections.
		newSections := s.assembleSections()
		s.Pages = append(s.Pages, newSections...)
		newPages = append(newPages, newSections...)

		// taxonomy list and terms pages
		taxonomies := s.Language.GetStringMapString("taxonomies")
		if len(taxonomies) > 0 {
			taxonomyPages := s.findPagesByKind(KindTaxonomy)
			taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm)
			for _, plural := range taxonomies {
				if s.isEnabled(KindTaxonomyTerm) {
					foundTaxonomyTermsPage := false
					for _, p := range taxonomyTermsPages {
						if p.sections[0] == plural {
							foundTaxonomyTermsPage = true
							break
						}
					}

					if !foundTaxonomyTermsPage {
						n := s.newTaxonomyTermsPage(plural)
						s.Pages = append(s.Pages, n)
						newPages = append(newPages, n)
					}
				}

				if s.isEnabled(KindTaxonomy) {
					for key := range s.Taxonomies[plural] {
						foundTaxonomyPage := false
						origKey := key

						if s.Info.preserveTaxonomyNames {
							key = s.PathSpec.MakePathSanitized(key)
						}
						for _, p := range taxonomyPages {
							// Some people may have /authors/MaxMustermann etc. as paths.
							// p.sections contains the raw values from the file system.
							// See https://github.com/gohugoio/hugo/issues/4238
							singularKey := s.PathSpec.MakePathSanitized(p.sections[1])
							if p.sections[0] == plural && singularKey == key {
								foundTaxonomyPage = true
								break
							}
						}

						if !foundTaxonomyPage {
							n := s.newTaxonomyPage(plural, origKey)
							s.Pages = append(s.Pages, n)
							newPages = append(newPages, n)
						}
					}
				}
			}
		}
	}

	if len(newPages) > 0 {
		// This resorting is unfortunate, but it also needs to be sorted
		// when sections are created.
		first := h.Sites[0]

		first.AllPages = append(first.AllPages, newPages...)

		first.AllPages.Sort()

		for _, s := range h.Sites {
			s.Pages.Sort()
		}

		for i := 1; i < len(h.Sites); i++ {
			h.Sites[i].AllPages = first.AllPages
		}
	}

	return nil
}

func (h *HugoSites) removePageByFilename(filename string) {
	for _, s := range h.Sites {
		s.removePageFilename(filename)
	}
}

func (h *HugoSites) setupTranslations() {
	for _, s := range h.Sites {
		for _, p := range s.rawAllPages {
			if p.Kind == kindUnknown {
				p.Kind = p.s.kindFromSections(p.sections)
			}

			if !p.s.isEnabled(p.Kind) {
				continue
			}

			shouldBuild := p.shouldBuild()
			s.updateBuildStats(p)
			if shouldBuild {
				if p.headless {
					s.headlessPages = append(s.headlessPages, p)
				} else {
					s.Pages = append(s.Pages, p)
				}
			}
		}
	}

	allPages := make(Pages, 0)

	for _, s := range h.Sites {
		allPages = append(allPages, s.Pages...)
	}

	allPages.Sort()

	for _, s := range h.Sites {
		s.AllPages = allPages
	}

	// Pull over the collections from the master site
	for i := 1; i < len(h.Sites); i++ {
		h.Sites[i].Data = h.Sites[0].Data
	}

	if len(h.Sites) > 1 {
		allTranslations := pagesToTranslationsMap(allPages)
		assignTranslationsToPages(allTranslations, allPages)
	}
}

type pagesRenderPreparer struct {
	numWorkers int
	s          *Site
	cfg        *BuildCfg
	wg         *sync.WaitGroup
	pages      chan *Page
}

func newStartedRenderPreparator(s *Site, cfg *BuildCfg) *pagesRenderPreparer {
	numWorkers := getGoMaxProcs() * 4
	pp := &pagesRenderPreparer{
		s:          s,
		cfg:        cfg,
		numWorkers: numWorkers,
		wg:         &sync.WaitGroup{},
		pages:      make(chan *Page),
	}

	pp.start()
	return pp
}

func (pp *pagesRenderPreparer) start() {
	for i := 0; i < pp.numWorkers; i++ {
		pp.wg.Add(1)
		go func() {
			defer pp.wg.Done()
			for p := range pp.pages {
				if err := p.prepareForRender(pp.cfg); err != nil {
					pp.s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)

				}
			}
		}()
	}
}

func (pp *pagesRenderPreparer) add(p *Page) {
	pp.pages <- p
}

func (pp *pagesRenderPreparer) done() {
	close(pp.pages)
	pp.wg.Wait()
}

func (s *Site) preparePagesForRender(cfg *BuildCfg) {

	// For the content from other pages in shortcodes there are some chicken and
	// egg dependencies that is hard to get around. But we can improve on this
	// by preparing the pages in a certain order.
	// So the headless pages goes first. These are typically collection of
	// pages and images etc. used by others.
	batch := newStartedRenderPreparator(s, cfg)
	for _, p := range s.headlessPages {
		batch.add(p)
	}

	batch.done()

	// Then the rest in the following order:
	order := []bundleDirType{bundleLeaf, bundleNot, bundleBranch}

	for _, tp := range order {
		batch = newStartedRenderPreparator(s, cfg)
		for _, p := range s.Pages {
			// sanity check
			if p.bundleType < 0 || p.bundleType > bundleBranch {
				panic("unknown bundle type")
			}
			if p.bundleType == tp {
				batch.add(p)
			}
		}
		batch.done()
	}

}

// Pages returns all pages for all sites.
func (h *HugoSites) Pages() Pages {
	return h.Sites[0].AllPages
}

func handleShortcodes(p *Page, rawContentCopy []byte) ([]byte, error) {
	if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
		err := p.shortcodeState.executeShortcodesForDelta(p)

		if err != nil {
			return rawContentCopy, err
		}

		rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes)

		if err != nil {
			p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
		}
	}

	return rawContentCopy, nil
}

func (s *Site) updateBuildStats(page *Page) {
	if page.IsDraft() {
		s.draftCount++
	}

	if page.IsFuture() {
		s.futureCount++
	}

	if page.IsExpired() {
		s.expiredCount++
	}
}

func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages {
	return h.Sites[0].findPagesByKindNotIn(kind, inPages)
}

func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages {
	return h.Sites[0].findPagesByKindIn(kind, inPages)
}

func (h *HugoSites) findAllPagesByKind(kind string) Pages {
	return h.findPagesByKindIn(kind, h.Sites[0].AllPages)
}

func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages {
	return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages)
}

func (h *HugoSites) findPagesByShortcode(shortcode string) Pages {
	var pages Pages
	for _, s := range h.Sites {
		pages = append(pages, s.findPagesByShortcode(shortcode)...)
	}
	return pages
}

// Used in partial reloading to determine if the change is in a bundle.
type contentChangeMap struct {
	mu       sync.RWMutex
	branches []string
	leafs    []string

	pathSpec *helpers.PathSpec

	// Hugo supports symlinked content (both directories and files). This
	// can lead to situations where the same file can be referenced from several
	// locations in /content -- which is really cool, but also means we have to
	// go an extra mile to handle changes.
	// This map is only used in watch mode.
	// It maps either file to files or the real dir to a set of content directories where it is in use.
	symContent   map[string]map[string]bool
	symContentMu sync.Mutex
}

func (m *contentChangeMap) add(filename string, tp bundleDirType) {
	m.mu.Lock()
	dir := filepath.Dir(filename) + helpers.FilePathSeparator
	dir = strings.TrimPrefix(dir, ".")
	switch tp {
	case bundleBranch:
		m.branches = append(m.branches, dir)
	case bundleLeaf:
		m.leafs = append(m.leafs, dir)
	default:
		panic("invalid bundle type")
	}
	m.mu.Unlock()
}

// Track the addition of bundle dirs.
func (m *contentChangeMap) handleBundles(b *bundleDirs) {
	for _, bd := range b.bundles {
		m.add(bd.fi.Path(), bd.tp)
	}
}

// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant.
// It also removes the entry from the map. It will be re-added again by the partial
// build if it still is a bundle.
func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) {
	m.mu.RLock()
	defer m.mu.RUnlock()

	// Bundles share resources, so we need to start from the virtual root.
	relPath, _ := m.pathSpec.RelContentDir(filename)
	dir, name := filepath.Split(relPath)
	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
		dir += helpers.FilePathSeparator
	}

	fileTp, isContent := classifyBundledFile(name)

	// This may be a member of a bundle. Start with branch bundles, the most specific.
	if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) {
		for i, b := range m.branches {
			if b == dir {
				m.branches = append(m.branches[:i], m.branches[i+1:]...)
				return dir, b, bundleBranch
			}
		}
	}

	// And finally the leaf bundles, which can contain anything.
	for i, l := range m.leafs {
		if strings.HasPrefix(dir, l) {
			m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)
			return dir, l, bundleLeaf
		}
	}

	// Not part of any bundle
	return dir, filename, bundleNot
}

func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) {
	m.symContentMu.Lock()
	mm, found := m.symContent[from]
	if !found {
		mm = make(map[string]bool)
		m.symContent[from] = mm
	}
	mm[to] = true
	m.symContentMu.Unlock()
}

func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string {
	mm, found := m.symContent[dir]
	if !found {
		return nil
	}
	dirs := make([]string, len(mm))
	i := 0
	for dir := range mm {
		dirs[i] = dir
		i++
	}

	sort.Strings(dirs)
	return dirs
}