Add robots.txt

src/site.rs

@@ -21,6 +21,7 @@ lazy_static! {
         tera.add_raw_templates(vec![
             ("rss.xml", include_str!("templates/rss.xml")),
             ("sitemap.xml", include_str!("templates/sitemap.xml")),
+            ("robots.txt", include_str!("templates/robots.txt")),
         ]).unwrap();
         tera
     };
@@ -296,10 +297,19 @@ impl Site {
             self.render_rss_feed()?;
         }
 
+        self.render_robots()?;
+
         self.render_sections()?;
         self.copy_static_directory()
     }
 
+    fn render_robots(&self) -> Result<()> {
+        create_file(
+            self.output_path.join("robots.txt"),
+            &self.templates.render("robots.txt", &Context::new())?
+        )
+    }
+
     /// Render the /{categories, list} pages and each individual category/tag page
     /// They are the same thing fundamentally, a list of pages with something in common
     fn render_categories_and_tags(&self, kind: RenderList) -> Result<()> {

src/templates/robots.txt

@@ -0,0 +1 @@
+User-agent: *

tests/site.rs

@@ -99,6 +99,7 @@ fn test_can_build_site_without_live_reload() {
 
     assert!(file_exists!(public, "index.html"));
     assert!(file_exists!(public, "sitemap.xml"));
+    assert!(file_exists!(public, "robots.txt"));
     assert!(file_exists!(public, "a-fixed-url/index.html"));
 
     assert!(file_exists!(public, "posts/python/index.html"));
@@ -140,6 +141,7 @@ fn test_can_build_site_with_live_reload() {
 
     assert!(file_exists!(public, "index.html"));
     assert!(file_exists!(public, "sitemap.xml"));
+    assert!(file_exists!(public, "robots.txt"));
     assert!(file_exists!(public, "a-fixed-url/index.html"));
 
     assert!(file_exists!(public, "posts/python/index.html"));
@@ -186,6 +188,7 @@ fn test_can_build_site_with_categories() {
 
     assert!(file_exists!(public, "index.html"));
     assert!(file_exists!(public, "sitemap.xml"));
+    assert!(file_exists!(public, "robots.txt"));
     assert!(file_exists!(public, "a-fixed-url/index.html"));
 
     assert!(file_exists!(public, "posts/python/index.html"));
@@ -237,6 +240,7 @@ fn test_can_build_site_with_tags() {
 
     assert!(file_exists!(public, "index.html"));
     assert!(file_exists!(public, "sitemap.xml"));
+    assert!(file_exists!(public, "robots.txt"));
     assert!(file_exists!(public, "a-fixed-url/index.html"));
 
     assert!(file_exists!(public, "posts/python/index.html"));