From 2e0465764b5dacc511b977b1c9aa07324ad0ee9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Thu, 2 Nov 2017 08:25:20 +0100 Subject: [PATCH] Add multilingual multihost support This commit adds multihost support when more than one language is configured and `baseURL` is set per language. Updates #4027 --- commands/commandeer.go | 4 + commands/hugo.go | 20 +++ commands/server.go | 182 ++++++++++++++++++++------- commands/server_test.go | 2 +- helpers/language.go | 24 +++- helpers/path.go | 1 - hugolib/config.go | 28 ++++- hugolib/hugo_sites.go | 76 ++++++----- hugolib/hugo_sites_build_test.go | 2 +- hugolib/hugo_sites_multihost_test.go | 72 +++++++++++ hugolib/multilingual.go | 8 ++ hugolib/page.go | 5 + hugolib/page_paths.go | 4 + hugolib/site_render.go | 2 +- 14 files changed, 350 insertions(+), 80 deletions(-) create mode 100644 hugolib/hugo_sites_multihost_test.go diff --git a/commands/commandeer.go b/commands/commandeer.go index d07a3d5bb..63fc0a663 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -41,6 +41,10 @@ func (c *commandeer) PathSpec() *helpers.PathSpec { return c.pathSpec } +func (c *commandeer) languages() helpers.Languages { + return c.Cfg.Get("languagesSorted").(helpers.Languages) +} + func (c *commandeer) initFs(fs *hugofs.Fs) error { c.DepsCfg.Fs = fs ps, err := helpers.NewPathSpec(fs, c.Cfg) diff --git a/commands/hugo.go b/commands/hugo.go index 63c5e3159..1714c8035 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -526,6 +526,7 @@ func (c *commandeer) watchConfig() { func (c *commandeer) build(watches ...bool) error { if err := c.copyStatic(); err != nil { + // TODO(bep) multihost return fmt.Errorf("Error copying static files to %s: %s", c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")), err) } watch := false @@ -593,6 +594,24 @@ func (c *commandeer) getStaticSourceFs() afero.Fs { func (c *commandeer) copyStatic() error { publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator + roots := c.roots() + + if len(roots) == 0 { + return c.copyStaticTo(publishDir) + } + + for _, root := range roots { + dir := filepath.Join(publishDir, root) + if err := c.copyStaticTo(dir); err != nil { + return err + } + } + + return nil + +} + +func (c *commandeer) copyStaticTo(publishDir string) error { // If root, remove the second '/' if publishDir == "//" { @@ -893,6 +912,7 @@ func (c *commandeer) newWatcher(port int) error { if c.Cfg.GetBool("forceSyncStatic") { c.Logger.FEEDBACK.Printf("Syncing all static files\n") + // TODO(bep) multihost err := c.copyStatic() if err != nil { utils.StopOnErr(c.Logger, err, fmt.Sprintf("Error copying static files to %s", publishDir)) diff --git a/commands/server.go b/commands/server.go index e2d37f165..bd45e7054 100644 --- a/commands/server.go +++ b/commands/server.go @@ -19,12 +19,14 @@ import ( "net/http" "net/url" "os" + "path/filepath" "runtime" "strconv" "strings" "time" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -137,34 +139,58 @@ func server(cmd *cobra.Command, args []string) error { c.watchConfig() } - l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(serverPort))) - if err == nil { - l.Close() - } else { - if serverCmd.Flags().Changed("port") { - // port set explicitly by user -- he/she probably meant it! - return newSystemErrorF("Server startup failed: %s", err) + languages := c.languages() + serverPorts := make([]int, 1) + + if languages.IsMultihost() { + serverPorts = make([]int, len(languages)) + } + + currentServerPort := serverPort + + for i := 0; i < len(serverPorts); i++ { + l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(currentServerPort))) + if err == nil { + l.Close() + serverPorts[i] = currentServerPort + } else { + if i == 0 && serverCmd.Flags().Changed("port") { + // port set explicitly by user -- he/she probably meant it! + return newSystemErrorF("Server startup failed: %s", err) + } + jww.ERROR.Println("port", serverPort, "already in use, attempting to use an available port") + sp, err := helpers.FindAvailablePort() + if err != nil { + return newSystemError("Unable to find alternative port to use:", err) + } + serverPorts[i] = sp.Port } - jww.ERROR.Println("port", serverPort, "already in use, attempting to use an available port") - sp, err := helpers.FindAvailablePort() - if err != nil { - return newSystemError("Unable to find alternative port to use:", err) - } - serverPort = sp.Port + + currentServerPort = serverPorts[i] + 1 } c.Set("port", serverPort) if liveReloadPort != -1 { c.Set("liveReloadPort", liveReloadPort) } else { - c.Set("liveReloadPort", serverPort) + c.Set("liveReloadPort", serverPorts[0]) } - baseURL, err = fixURL(c.Cfg, baseURL) - if err != nil { - return err + if languages.IsMultihost() { + for i, language := range languages { + baseURL, err = fixURL(language, baseURL, serverPorts[i]) + if err != nil { + return err + } + language.Set("baseURL", baseURL) + } + } else { + baseURL, err = fixURL(c.Cfg, baseURL, serverPorts[0]) + if err != nil { + return err + } + c.Cfg.Set("baseURL", baseURL) } - c.Set("baseURL", baseURL) if err := memStats(); err != nil { jww.ERROR.Println("memstats error:", err) @@ -208,28 +234,52 @@ func server(cmd *cobra.Command, args []string) error { } } - c.serve(serverPort) - return nil } -func (c *commandeer) serve(port int) { +type fileServer struct { + basePort int + baseURLs []string + roots []string + c *commandeer +} + +func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) { + baseURL := f.baseURLs[i] + root := f.roots[i] + port := f.basePort + i + + publishDir := f.c.Cfg.GetString("publishDir") + + if root != "" { + publishDir = filepath.Join(publishDir, root) + } + + absPublishDir := f.c.PathSpec().AbsPathify(publishDir) + + // TODO(bep) multihost unify feedback if renderToDisk { - jww.FEEDBACK.Println("Serving pages from " + c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir"))) + jww.FEEDBACK.Println("Serving pages from " + absPublishDir) } else { jww.FEEDBACK.Println("Serving pages from memory") } - httpFs := afero.NewHttpFs(c.Fs.Destination) - fs := filesOnlyFs{httpFs.Dir(c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")))} + httpFs := afero.NewHttpFs(f.c.Fs.Destination) + fs := filesOnlyFs{httpFs.Dir(absPublishDir)} - doLiveReload := !buildWatch && !c.Cfg.GetBool("disableLiveReload") - fastRenderMode := doLiveReload && !c.Cfg.GetBool("disableFastRender") + doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload") + fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender") if fastRenderMode { jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender") } + // We're only interested in the path + u, err := url.Parse(baseURL) + if err != nil { + return nil, "", fmt.Errorf("Invalid baseURL: %s", err) + } + decorate := func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if noHTTPCache { @@ -240,7 +290,7 @@ func (c *commandeer) serve(port int) { if fastRenderMode { p := r.RequestURI if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") { - c.visitedURLs.Add(p) + f.c.visitedURLs.Add(p) } } h.ServeHTTP(w, r) @@ -248,32 +298,78 @@ func (c *commandeer) serve(port int) { } fileserver := decorate(http.FileServer(fs)) + mu := http.NewServeMux() - // We're only interested in the path - u, err := url.Parse(c.Cfg.GetString("baseURL")) - if err != nil { - jww.ERROR.Fatalf("Invalid baseURL: %s", err) - } if u.Path == "" || u.Path == "/" { - http.Handle("/", fileserver) + mu.Handle("/", fileserver) } else { - http.Handle(u.Path, http.StripPrefix(u.Path, fileserver)) + mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver)) } - jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface) - jww.FEEDBACK.Println("Press Ctrl+C to stop") - endpoint := net.JoinHostPort(serverInterface, strconv.Itoa(port)) - err = http.ListenAndServe(endpoint, nil) - if err != nil { - jww.ERROR.Printf("Error: %s\n", err.Error()) - os.Exit(1) + + return mu, endpoint, nil +} + +func (c *commandeer) roots() []string { + var roots []string + languages := c.languages() + isMultiHost := languages.IsMultihost() + if !isMultiHost { + return roots } + + for _, l := range languages { + roots = append(roots, l.Lang) + } + return roots +} + +func (c *commandeer) serve(port int) { + // TODO(bep) multihost + isMultiHost := Hugo.IsMultihost() + + var ( + baseURLs []string + roots []string + ) + + if isMultiHost { + for _, s := range Hugo.Sites { + baseURLs = append(baseURLs, s.BaseURL.String()) + roots = append(roots, s.Language.Lang) + } + } else { + baseURLs = []string{Hugo.Sites[0].BaseURL.String()} + roots = []string{""} + } + + srv := &fileServer{ + basePort: port, + baseURLs: baseURLs, + roots: roots, + c: c, + } + + for i, _ := range baseURLs { + mu, endpoint, err := srv.createEndpoint(i) + + go func() { + err = http.ListenAndServe(endpoint, mu) + if err != nil { + jww.ERROR.Printf("Error: %s\n", err.Error()) + os.Exit(1) + } + }() + } + + // TODO(bep) multihost jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface) + jww.FEEDBACK.Println("Press Ctrl+C to stop") } // fixURL massages the baseURL into a form needed for serving // all pages correctly. -func fixURL(cfg config.Provider, s string) (string, error) { +func fixURL(cfg config.Provider, s string, port int) (string, error) { useLocalhost := false if s == "" { s = cfg.GetString("baseURL") @@ -315,7 +411,7 @@ func fixURL(cfg config.Provider, s string) (string, error) { return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err) } } - u.Host += fmt.Sprintf(":%d", serverPort) + u.Host += fmt.Sprintf(":%d", port) } return u.String(), nil diff --git a/commands/server_test.go b/commands/server_test.go index 3f1518aaa..ce6dc078b 100644 --- a/commands/server_test.go +++ b/commands/server_test.go @@ -47,7 +47,7 @@ func TestFixURL(t *testing.T) { v.Set("baseURL", test.CfgBaseURL) serverAppend = test.AppendPort serverPort = test.Port - result, err := fixURL(v, baseURL) + result, err := fixURL(v, baseURL, serverPort) if err != nil { t.Errorf("Test #%d %s: unexpected error %s", i, test.TestName, err) } diff --git a/helpers/language.go b/helpers/language.go index 6b4119a9b..d9f3f69ed 100644 --- a/helpers/language.go +++ b/helpers/language.go @@ -102,6 +102,17 @@ func (l *Language) Params() map[string]interface{} { return l.params } +// IsMultihost returns whether the languages has baseURL specificed on the +// language level. +func (l Languages) IsMultihost() bool { + for _, lang := range l { + if lang.GetLocal("baseURL") != nil { + return true + } + } + return false +} + // SetParam sets param with the given key and value. // SetParam is case-insensitive. func (l *Language) SetParam(k string, v interface{}) { @@ -132,6 +143,17 @@ func (l *Language) GetStringMapString(key string) map[string]string { // // Get returns an interface. For a specific value use one of the Get____ methods. func (l *Language) Get(key string) interface{} { + local := l.GetLocal(key) + if local != nil { + return local + } + return l.Cfg.Get(key) +} + +// GetLocal gets a configuration value set on language level. It will +// not fall back to any global value. +// It will return nil if a value with the given key cannot be found. +func (l *Language) GetLocal(key string) interface{} { if l == nil { panic("language not set") } @@ -141,7 +163,7 @@ func (l *Language) Get(key string) interface{} { return v } } - return l.Cfg.Get(key) + return nil } // Set sets the value for the key in the language's params. diff --git a/helpers/path.go b/helpers/path.go index fccba2238..a9e2567c6 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -158,7 +158,6 @@ func (p *PathSpec) AbsPathify(inPath string) string { return filepath.Clean(inPath) } - // TODO(bep): Consider moving workingDir to argument list return filepath.Join(p.workingDir, inPath) } diff --git a/hugolib/config.go b/hugolib/config.go index acfa0704d..db59253cd 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -19,6 +19,7 @@ import ( "io" "strings" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" "github.com/spf13/viper" @@ -80,11 +81,34 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper. helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", false) } - loadDefaultSettingsFor(v) + if err := loadDefaultSettingsFor(v); err != nil { + return v, err + } return v, nil } +func loadLanguageSettings(cfg config.Provider) error { + multilingual := cfg.GetStringMap("languages") + var ( + langs helpers.Languages + err error + ) + + if len(multilingual) == 0 { + langs = append(langs, helpers.NewDefaultLanguage(cfg)) + } else { + langs, err = toSortedLanguages(cfg, multilingual) + if err != nil { + return fmt.Errorf("Failed to parse multilingual config: %s", err) + } + } + + cfg.Set("languagesSorted", langs) + + return nil +} + func loadDefaultSettingsFor(v *viper.Viper) error { c, err := helpers.NewContentSpec(v) @@ -154,5 +178,5 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("debug", false) v.SetDefault("disableFastRender", false) - return nil + return loadLanguageSettings(v) } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 6e2340903..e0697507b 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -15,7 +15,6 @@ package hugolib import ( "errors" - "fmt" "strings" "sync" @@ -37,9 +36,16 @@ type HugoSites struct { multilingual *Multilingual + // Multihost is set if multilingual and baseURL set on the language level. + multihost bool + *deps.Deps } +func (h *HugoSites) IsMultihost() bool { + return h != nil && h.multihost +} + // GetContentPage finds a Page with content given the absolute filename. // Returns nil if none found. func (h *HugoSites) GetContentPage(filename string) *Page { @@ -92,6 +98,31 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h.Deps = sites[0].Deps + // The baseURL may be provided at the language level. If that is true, + // then every language must have a baseURL. In this case we always render + // to a language sub folder, which is then stripped from all the Permalink URLs etc. + var baseURLFromLang bool + + for _, s := range sites { + burl := s.Language.GetLocal("baseURL") + if baseURLFromLang && burl == nil { + return h, errors.New("baseURL must be set on all or none of the languages") + } + + if burl != nil { + baseURLFromLang = true + } + } + + if baseURLFromLang { + for _, s := range sites { + // TODO(bep) multihost check + s.Info.defaultContentLanguageInSubdir = true + s.Cfg.Set("defaultContentLanguageInSubdir", true) + } + h.multihost = true + } + return h, nil } @@ -180,41 +211,21 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { sites []*Site ) - multilingual := cfg.Cfg.GetStringMap("languages") + languages := getLanguages(cfg.Cfg) + + for _, lang := range languages { + var s *Site + var err error + cfg.Language = lang + s, err = newSite(cfg) - if len(multilingual) == 0 { - l := helpers.NewDefaultLanguage(cfg.Cfg) - cfg.Language = l - s, err := newSite(cfg) if err != nil { return nil, err } + sites = append(sites, s) } - if len(multilingual) > 0 { - var err error - - languages, err := toSortedLanguages(cfg.Cfg, multilingual) - - if err != nil { - return nil, fmt.Errorf("Failed to parse multilingual config: %s", err) - } - - for _, lang := range languages { - var s *Site - var err error - cfg.Language = lang - s, err = newSite(cfg) - - if err != nil { - return nil, err - } - - sites = append(sites, s) - } - } - return sites, nil } @@ -227,7 +238,12 @@ func (h *HugoSites) reset() { func (h *HugoSites) createSitesFromConfig() error { + if err := loadLanguageSettings(h.Cfg); err != nil { + return err + } + depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg} + sites, err := createSitesFromConfig(depsCfg) if err != nil { @@ -286,7 +302,7 @@ type BuildCfg struct { func (h *HugoSites) renderCrossSitesArtifacts() error { - if !h.multilingual.enabled() { + if !h.multilingual.enabled() || h.IsMultihost() { return nil } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 96e2c66b2..079f0fcfa 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -1269,7 +1269,7 @@ lag: t.Fatalf("Failed to create sites: %s", err) } - if len(sites.Sites) != 4 { + if len(sites.Sites) == 0 { t.Fatalf("Got %d sites", len(sites.Sites)) } diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go new file mode 100644 index 000000000..864d52c71 --- /dev/null +++ b/hugolib/hugo_sites_multihost_test.go @@ -0,0 +1,72 @@ +package hugolib + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func TestMultihosts(t *testing.T) { + t.Parallel() + + var multiSiteTOMLConfigTemplate = ` +paginate = 1 +disablePathToLower = true +defaultContentLanguage = "{{ .DefaultContentLanguage }}" +defaultContentLanguageInSubdir = {{ .DefaultContentLanguageInSubdir }} + +[permalinks] +other = "/somewhere/else/:filename" + +[Taxonomies] +tag = "tags" + +[Languages] +[Languages.en] +baseURL = "https://example.com" +weight = 10 +title = "In English" +languageName = "English" + +[Languages.fr] +baseURL = "https://example.fr" +weight = 20 +title = "Le Français" +languageName = "Français" + +[Languages.nn] +baseURL = "https://example.no" +weight = 30 +title = "På nynorsk" +languageName = "Nynorsk" + +` + + siteConfig := testSiteConfig{Fs: afero.NewMemMapFs(), DefaultContentLanguage: "fr", DefaultContentLanguageInSubdir: false} + sites := createMultiTestSites(t, siteConfig, multiSiteTOMLConfigTemplate) + fs := sites.Fs + cfg := BuildCfg{Watching: true} + th := testHelper{sites.Cfg, fs, t} + assert := require.New(t) + + err := sites.Build(cfg) + assert.NoError(err) + + th.assertFileContent("public/en/sect/doc1-slug/index.html", "Hello") + + s1 := sites.Sites[0] + + s1h := s1.getPage(KindHome) + assert.True(s1h.IsTranslated()) + assert.Len(s1h.Translations(), 2) + assert.Equal("https://example.com/", s1h.Permalink()) + + s2 := sites.Sites[1] + s2h := s2.getPage(KindHome) + assert.Equal("https://example.fr/", s2h.Permalink()) + + th.assertFileContentStraight("public/fr/index.html", "French Home Page") + th.assertFileContentStraight("public/en/index.html", "Default Home Page") + +} diff --git a/hugolib/multilingual.go b/hugolib/multilingual.go index e0245fa2b..589df66e0 100644 --- a/hugolib/multilingual.go +++ b/hugolib/multilingual.go @@ -47,6 +47,14 @@ func (ml *Multilingual) Language(lang string) *helpers.Language { return ml.langMap[lang] } +func getLanguages(cfg config.Provider) helpers.Languages { + if cfg.IsSet("languagesSorted") { + return cfg.Get("languagesSorted").(helpers.Languages) + } + + return helpers.Languages{helpers.NewDefaultLanguage(cfg)} +} + func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) { languages := make(helpers.Languages, len(sites)) diff --git a/hugolib/page.go b/hugolib/page.go index d928b92f9..7da77f192 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1754,6 +1754,11 @@ func (p *Page) shouldAddLanguagePrefix() bool { return false } + if p.s.owner.IsMultihost() { + // TODO(bep) multihost check vs lang below + return true + } + if p.Lang() == "" { return false } diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go index e18e0f10e..993ad0780 100644 --- a/hugolib/page_paths.go +++ b/hugolib/page_paths.go @@ -257,6 +257,10 @@ func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string { tp = strings.TrimSuffix(tp, f.BaseFilename()) } + if p.s.owner.IsMultihost() { + tp = strings.TrimPrefix(tp, helpers.FilePathSeparator+p.s.Info.Language.Lang) + } + return p.s.PathSpec.URLizeFilename(tp) } diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 4118f3eef..b4d688bda 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -387,7 +387,7 @@ func (s *Site) renderAliases() error { } } - if s.owner.multilingual.enabled() { + if s.owner.multilingual.enabled() && !s.owner.IsMultihost() { mainLang := s.owner.multilingual.DefaultLang if s.Info.defaultContentLanguageInSubdir { mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false)