From 9a6dc6c791f47e1d410956cb16cc42a0117f7246 Mon Sep 17 00:00:00 2001 From: Ivan Fraixedes Date: Tue, 8 Dec 2015 21:13:09 +0000 Subject: [PATCH] Add embeded template for robots.txt --- commands/hugo.go | 7 ++- docs/content/commands/hugo.md | 1 + docs/content/extras/robots-txt.md | 34 +++++++++++++ docs/content/extras/urls.md | 2 +- docs/content/overview/configuration.md | 2 + hugolib/robotstxt_test.go | 67 ++++++++++++++++++++++++++ hugolib/site.go | 26 ++++++++++ hugolib/sitemap_test.go | 4 ++ tpl/template_embedded.go | 2 + 9 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 docs/content/extras/robots-txt.md create mode 100644 hugolib/robotstxt_test.go diff --git a/commands/hugo.go b/commands/hugo.go index 4d3f9bab6..3b2e172c6 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -112,7 +112,7 @@ Complete documentation is available at http://gohugo.io/.`, var hugoCmdV *cobra.Command // Flags that are to be added to commands. -var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool +var BuildWatch, IgnoreCache, Draft, Future, UglyURLs, CanonifyURLs, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, DisableRobotsTXT, PluralizeListTitles, PreserveTaxonomyNames, NoTimes, ForceSync bool var Source, CacheDir, Destination, Theme, BaseURL, CfgFile, LogFile, Editor string // Execute adds all child commands to the root command HugoCmd and sets flags appropriately. @@ -159,6 +159,7 @@ func initCoreCommonFlags(cmd *cobra.Command) { cmd.Flags().BoolVarP(&Future, "buildFuture", "F", false, "include content with publishdate in the future") cmd.Flags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files") cmd.Flags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file") + cmd.Flags().BoolVar(&DisableRobotsTXT, "disableRobotsTXT", false, "Do not build Robots TXT file") cmd.Flags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from") cmd.Flags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") cmd.Flags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it") @@ -204,6 +205,7 @@ func LoadDefaultSettings() { viper.SetDefault("MetaDataFormat", "toml") viper.SetDefault("DisableRSS", false) viper.SetDefault("DisableSitemap", false) + viper.SetDefault("DisableRobotsTXT", false) viper.SetDefault("ContentDir", "content") viper.SetDefault("LayoutDir", "layouts") viper.SetDefault("StaticDir", "static") @@ -294,6 +296,9 @@ func InitializeConfig(subCmdVs ...*cobra.Command) error { if cmdV.Flags().Lookup("disableSitemap").Changed { viper.Set("DisableSitemap", DisableSitemap) } + if cmdV.Flags().Lookup("disableRobotsTXT").Changed { + viper.Set("DisableRobotsTXT", DisableRobotsTXT) + } if cmdV.Flags().Lookup("pluralizeListTitles").Changed { viper.Set("PluralizeListTitles", PluralizeListTitles) } diff --git a/docs/content/commands/hugo.md b/docs/content/commands/hugo.md index d36cb0bab..190f67a14 100644 --- a/docs/content/commands/hugo.md +++ b/docs/content/commands/hugo.md @@ -34,6 +34,7 @@ hugo -d, --destination="": filesystem path to write files to --disableRSS[=false]: Do not build RSS files --disableSitemap[=false]: Do not build Sitemap file + --disableRobotsTXT[=false]: Do not build robots TXT file --editor="": edit new content with this editor, if provided --ignoreCache[=false]: Ignores the cache directory for reading but still writes to it --log[=false]: Enable Logging diff --git a/docs/content/extras/robots-txt.md b/docs/content/extras/robots-txt.md new file mode 100644 index 000000000..336999d1c --- /dev/null +++ b/docs/content/extras/robots-txt.md @@ -0,0 +1,34 @@ +--- +date: 2013-07-09 +menu: + main: + parent: extras +next: /community/mailing-list +prev: /extras/urls +title: Table of Contents +weight: 120 +--- + +Hugo can generated customized [robots.txt](http://www.robotstxt.org/) in the +[same way than any other template]({{< ref "templates/go-templates.md" >}}). + +By default it generates a robots.txt which allows everything, it looks exactly + + User-agent: * + +To disable it just set `disableRobotsTXT` option to false in the [command line]({{< ref "commands/hugo.md" >}}) or [configuration file]({{< ref "overview/configuration.md" >}}). + +Hugo will use the template `robots.txt` following the list starting with the one with more priority + +* /layouts/robots.txt +* /themes/`THEME`/layout/robots.txt + +An example of a robots.txt layout is: + + User-agent: * + + {{range .Data.Pages}} + Disallow: {{.RelPermalink}}{{end}} + + +This template disallows and all the pages of the site creating one `Disallow` entry for each one. diff --git a/docs/content/extras/urls.md b/docs/content/extras/urls.md index 497caa0fe..fb53b8f13 100644 --- a/docs/content/extras/urls.md +++ b/docs/content/extras/urls.md @@ -5,7 +5,7 @@ date: 2014-01-03 menu: main: parent: extras -next: /community/mailing-list +next: /extras/robots-txt notoc: true prev: /extras/toc title: URLs diff --git a/docs/content/overview/configuration.md b/docs/content/overview/configuration.md index 89a2642c5..d5db7520e 100644 --- a/docs/content/overview/configuration.md +++ b/docs/content/overview/configuration.md @@ -96,6 +96,8 @@ Following is a list of Hugo-defined variables that you can configure and their c disableRSS: false # Do not build Sitemap file disableSitemap: false + # Do not build robots.txt file + disableRobotsTXT: false # edit new content with this editor, if provided editor: "" footnoteAnchorPrefix: "" diff --git a/hugolib/robotstxt_test.go b/hugolib/robotstxt_test.go new file mode 100644 index 000000000..c964c0231 --- /dev/null +++ b/hugolib/robotstxt_test.go @@ -0,0 +1,67 @@ +package hugolib + +import ( + "bytes" + "testing" + + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/hugofs" + "github.com/spf13/hugo/source" + "github.com/spf13/viper" +) + +const ROBOTSTXT_TEMPLATE = `User-agent: Googlebot + {{ range .Data.Pages }} + Disallow: {{.RelPermalink}} + {{ end }} +` + +func TestRobotsTXTOutput(t *testing.T) { + viper.Reset() + defer viper.Reset() + + hugofs.DestinationFS = new(afero.MemMapFs) + + viper.Set("baseurl", "http://auth/bub/") + + s := &Site{ + Source: &source.InMemorySource{ByteSource: WEIGHTED_SOURCES}, + } + + s.initializeSiteInfo() + + s.prepTemplates() + s.addTemplate("robots.txt", ROBOTSTXT_TEMPLATE) + + if err := s.CreatePages(); err != nil { + t.Fatalf("Unable to create pages: %s", err) + } + + if err := s.BuildSiteMeta(); err != nil { + t.Fatalf("Unable to build site metadata: %s", err) + } + + if err := s.RenderHomePage(); err != nil { + t.Fatalf("Unable to RenderHomePage: %s", err) + } + + if err := s.RenderSitemap(); err != nil { + t.Fatalf("Unable to RenderSitemap: %s", err) + } + + if err := s.RenderRobotsTXT(); err != nil { + t.Fatalf("Unable to RenderRobotsTXT :%s", err) + } + + robotsFile, err := hugofs.DestinationFS.Open("robots.txt") + + if err != nil { + t.Fatalf("Unable to locate: robots.txt") + } + + robots := helpers.ReaderToBytes(robotsFile) + if !bytes.HasPrefix(robots, []byte("User-agent: Googlebot")) { + t.Errorf("Robots file should start with 'User-agentL Googlebot'. %s", robots) + } +} diff --git a/hugolib/site.go b/hugolib/site.go index 648ba6454..957e8ed4c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -583,6 +583,12 @@ func (s *Site) Render() (err error) { return } s.timerStep("render and write Sitemap") + + if err = s.RenderRobotsTXT(); err != nil { + return + } + s.timerStep("render and write robots.txt") + return } @@ -1561,6 +1567,26 @@ func (s *Site) RenderSitemap() error { return nil } +func (s *Site) RenderRobotsTXT() error { + if viper.GetBool("DisableRobotsTXT") { + return nil + } + + n := s.NewNode() + n.Data["Pages"] = s.Pages + + rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"} + outBuffer := bp.GetBuffer() + defer bp.PutBuffer(outBuffer) + err := s.render("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...) + + if err == nil { + err = s.WriteDestFile("robots.txt", outBuffer) + } + + return err +} + func (s *Site) Stats() { jww.FEEDBACK.Println(s.draftStats()) jww.FEEDBACK.Println(s.futureStats()) diff --git a/hugolib/sitemap_test.go b/hugolib/sitemap_test.go index 7bc16b482..13bc92a18 100644 --- a/hugolib/sitemap_test.go +++ b/hugolib/sitemap_test.go @@ -68,6 +68,10 @@ func TestSitemapOutput(t *testing.T) { t.Fatalf("Unable to RenderSitemap: %s", err) } + if err := s.RenderRobotsTXT(); err != nil { + t.Fatalf("Unable to RenderRobotsTXT :%s", err) + } + sitemapFile, err := hugofs.DestinationFS.Open("sitemap.xml") if err != nil { diff --git a/tpl/template_embedded.go b/tpl/template_embedded.go index a2fde4b10..14364f6db 100644 --- a/tpl/template_embedded.go +++ b/tpl/template_embedded.go @@ -242,4 +242,6 @@ ga('send', 'pageview'); {{ end }}`) + + t.AddInternalTemplate("_default", "robots.txt", "User-agent: *") }