Add robots.txt and sitemap

website/config.rb

@@ -2,6 +2,8 @@
 # Configure Middleman
 #-------------------------------------------------------------------------
 
+set :base_url, "https://www.consul.io/"
+
 activate :hashicorp do |h|
   h.version      = '0.4.0'
   h.bintray_repo = 'mitchellh/consul'

website/source/404.html.erb

@@ -1 +1,5 @@
+---
+noindex: true
+---
+
 <h2>Page Not Found</h2>

website/source/robots.txt

@@ -0,0 +1,7 @@
+---
+layout: false
+---
+
+User-agent: *
+Disallow: /404
+Disallow: /500

website/source/sitemap.xml.builder

@@ -0,0 +1,19 @@
+---
+layout: false
+---
+
+xml.instruct!
+xml.urlset 'xmlns' => "http://www.sitemaps.org/schemas/sitemap/0.9" do
+  sitemap
+    .resources
+    .select { |page| page.path =~ /\.html/ }
+    .select { |page| !page.data.noindex }
+    .each do |page|
+      xml.url do
+        xml.loc File.join(base_url, page.url)
+        xml.lastmod Date.today.to_time.iso8601
+        xml.changefreq page.data.changefreq || "monthly"
+        xml.priority page.data.priority || "0.5"
+      end
+  end
+end