Fix load_toml date handling and fix bug in date conversion

CHANGELOG.md

@@ -11,6 +11,7 @@ Tera function
 - Table of content now strips HTML from the titles to avoid various issues
 
 ### Others
+- Many many times faster (x5-x40) for most sites
 - Update dependencies, fixing a few bugs with templates
 - Load only .html files in themes from the templates folder
 - Background colour is set fewer times when highlighting syntaxes, resulting in smaller HTML filesize
@@ -18,7 +19,6 @@ Tera function
 - Load table and footnote markdown extensions in `markdown` filter
 - `get_url` now defaults to not adding a trailing slash
 - Fix `--base-url` not overriding processed images URLs
-- Many many times faster (x5-x40) for most sites
 - Add more Emacs temp file to the ignored patterns in `gutenberg serve`
 - Files starting with `.` are not considered pages anymore even if they end with `.md`
 - `_processed_images` folder for image processing has been renamed `processed_images` to avoid issues with GitHub Pages

Cargo.lock

@@ -48,8 +48,8 @@ dependencies = [
  "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -638,7 +638,7 @@ name = "errors"
 version = "0.1.0"
 dependencies = [
  "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "syntect 3.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -719,6 +719,7 @@ dependencies = [
  "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utils 0.1.0",
 ]
 
 [[package]]
@@ -945,7 +946,7 @@ dependencies = [
 
 [[package]]
 name = "image"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -965,7 +966,7 @@ name = "imageproc"
 version = "0.1.0"
 dependencies = [
  "errors 0.1.0",
- "image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2518,7 +2519,7 @@ dependencies = [
 
 [[package]]
 name = "trust-dns-proto"
-version = "0.5.0-alpha.5"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2559,7 +2560,7 @@ dependencies = [
 
 [[package]]
 name = "trust-dns-resolver"
-version = "0.10.0-alpha.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2572,7 +2573,7 @@ dependencies = [
  "resolv-conf 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -2717,8 +2718,10 @@ name = "utils"
 version = "0.1.0"
 dependencies = [
  "errors 0.1.0",
+ "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "walkdir 2.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -3000,7 +3003,7 @@ dependencies = [
 "checksum hyper 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)" = "95ffee0d1d30de4313fdaaa485891ce924991d45bbc18adfc8ac5b1639e62fbb"
 "checksum hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cd73f14ad370d3b4d4b7dce08f69b81536c82e39fcc89731930fe5788cd661"
 "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e"
-"checksum image 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60710fd3cb40c2434451d8d5147bcf39bbb68aae0741041133e09439cb2401e3"
+"checksum image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44665b4395d1844c96e7dc8ed5754782a1cdfd9ef458a80bbe45702681450504"
 "checksum indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7e81a7c05f79578dbc15793d8b619db9ba32b4577003ef3af1a91c416798c58d"
 "checksum inflate 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6f53b811ee8e2057ccf9643ca6b4277de90efaf5e61e55fd5254576926bb4245"
 "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718"
@@ -3154,8 +3157,8 @@ dependencies = [
 "checksum toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4a2ecc31b0351ea18b3fe11274b8db6e4d82bce861bbb22e6dbed40417902c65"
 "checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987"
 "checksum trust-dns-proto 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f1525ca4e26f5a09d81b79584f19225e7dba5606ae3a416311c2751c5cea60bb"
-"checksum trust-dns-proto 0.5.0-alpha.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3daf083dfce3c3554e82b768078331864df1ab220e6c3c1dbb4c847173d33ff6"
-"checksum trust-dns-resolver 0.10.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "476d293db1c1027727a1683b681550a24207277e96bb1fb763053e1164621971"
+"checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74"
+"checksum trust-dns-resolver 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4e913a5df94658858e548cc95a3212797ee524e487ede091c32f27ca26e11620"
 "checksum trust-dns-resolver 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4a821ad51a29816420b8cac4b026756b81c023630b97eaa4c8090637ee3508bd"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
 "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"

components/front_matter/Cargo.toml

@@ -13,3 +13,4 @@ regex = "1"
 lazy_static = "1"
 
 errors = { path = "../errors" }
+utils = { path = "../utils" }

components/front_matter/src/lib.rs

@@ -10,6 +10,7 @@ extern crate toml;
 
 #[macro_use]
 extern crate errors;
+extern crate utils;
 
 use errors::{Result, ResultExt};
 use regex::Regex;

components/front_matter/src/page.rs

@@ -1,66 +1,11 @@
 use std::collections::HashMap;
-use std::result::Result as StdResult;
 
 use chrono::prelude::*;
-use serde::{Deserialize, Deserializer};
 use tera::{Map, Value};
 use toml;
 
 use errors::Result;
-
-fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult<Option<String>, D::Error>
-where
-    D: Deserializer<'de>,
-{
-    toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
-}
-
-/// Returns key/value for a converted date from TOML.
-/// If the table itself is the TOML struct, only return its value without the key
-fn convert_toml_date(table: Map<String, Value>) -> Value {
-    let mut new = Map::new();
-
-    for (k, v) in table {
-        if k == "$__toml_private_datetime" {
-            return v;
-        }
-
-        match v {
-            Value::Object(mut o) => {
-                // that was a toml datetime object, just return the date
-                if let Some(toml_date) = o.remove("$__toml_private_datetime") {
-                    new.insert(k, toml_date);
-                    return Value::Object(new);
-                }
-                new.insert(k, convert_toml_date(o));
-            }
-            _ => {
-                new.insert(k, v);
-            }
-        }
-    }
-
-    Value::Object(new)
-}
-
-/// TOML datetimes will be serialized as a struct but we want the
-/// stringified version for json, otherwise they are going to be weird
-fn fix_toml_dates(table: Map<String, Value>) -> Value {
-    let mut new = Map::new();
-
-    for (key, value) in table {
-        match value {
-            Value::Object(mut o) => {
-                new.insert(key, convert_toml_date(o));
-            }
-            _ => {
-                new.insert(key, value);
-            }
-        }
-    }
-
-    Value::Object(new)
-}
+use utils::de::fix_toml_dates;
 
 /// The front matter of every page
 #[derive(Debug, Clone, PartialEq, Deserialize)]
@@ -71,7 +16,7 @@ pub struct PageFrontMatter {
     /// Description in <meta> that appears when linked, e.g. on twitter
     pub description: Option<String>,
     /// Date if we want to order pages (ie blog post)
-    #[serde(default, deserialize_with = "from_toml_datetime")]
+    #[serde(default, deserialize_with = "utils::de::from_toml_datetime")]
     pub date: Option<String>,
     /// Chrono converted datetime
     #[serde(default, skip_deserializing)]

components/templates/src/global_fns/load_data.rs

@@ -1,6 +1,7 @@
 extern crate serde_json;
 extern crate toml;
 
+use utils::de::fix_toml_dates;
 use utils::fs::{get_file_time, is_path_in_directory, read_file};
 
 use reqwest::{header, Client};
@@ -236,7 +237,12 @@ fn load_json(json_data: String) -> Result<Value> {
 /// Parse a TOML string and convert it to a Tera Value
 fn load_toml(toml_data: String) -> Result<Value> {
     let toml_content: toml::Value = toml::from_str(&toml_data).map_err(|e| format!("{:?}", e))?;
-    to_value(toml_content).map_err(|e| e.into())
+    let toml_value = to_value(toml_content).expect("Got invalid JSON that was valid TOML somehow");
+
+    match toml_value {
+        Value::Object(m) => Ok(fix_toml_dates(m)),
+        _ => unreachable!("Loaded something other than a TOML object"),
+    }
 }
 
 /// Parse a CSV string and convert it to a Tera Value
@@ -409,15 +415,12 @@ mod tests {
         args.insert("path".to_string(), to_value("test.toml").unwrap());
         let result = static_fn(args.clone()).unwrap();
 
-        //TOML does not load in order, and also dates are not returned as strings, but
-        //rather as another object with a key and value
+        //TOML does not load in order
         assert_eq!(
             result,
             json!({
             "category": {
-                "date": {
-                    "$__toml_private_datetime": "1979-05-27T07:32:00Z"
-                },
+                "date": "1979-05-27T07:32:00Z",
                 "key": "value"
             },
         })

components/utils/Cargo.toml

@@ -8,6 +8,8 @@ errors = { path = "../errors" }
 tera = "0.11"
 unicode-segmentation = "1.2"
 walkdir = "2"
+toml = "0.4"
+serde = "1"
 
 [dev-dependencies]
 tempfile = "3"

components/utils/src/de.rs

@@ -0,0 +1,53 @@
+use serde::{Deserialize, Deserializer};
+use tera::{Map, Value};
+use toml;
+
+/// Used as an attribute when we want to convert from TOML to a string date
+pub fn from_toml_datetime<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    toml::value::Datetime::deserialize(deserializer).map(|s| Some(s.to_string()))
+}
+
+/// Returns key/value for a converted date from TOML.
+/// If the table itself is the TOML struct, only return its value without the key
+fn convert_toml_date(table: Map<String, Value>) -> Value {
+    let mut new = Map::new();
+
+    for (k, v) in table {
+        if k == "$__toml_private_datetime" {
+            return v;
+        }
+
+        match v {
+            Value::Object(o) => {
+                new.insert(k, convert_toml_date(o));
+            }
+            _ => {
+                new.insert(k, v);
+            }
+        }
+    }
+
+    Value::Object(new)
+}
+
+/// TOML datetimes will be serialized as a struct but we want the
+/// stringified version for json, otherwise they are going to be weird
+pub fn fix_toml_dates(table: Map<String, Value>) -> Value {
+    let mut new = Map::new();
+
+    for (key, value) in table {
+        match value {
+            Value::Object(mut o) => {
+                new.insert(key, convert_toml_date(o));
+            }
+            _ => {
+                new.insert(key, value);
+            }
+        }
+    }
+
+    Value::Object(new)
+}

components/utils/src/lib.rs

@@ -1,12 +1,15 @@
 #[macro_use]
 extern crate errors;
 
+extern crate serde;
 #[cfg(test)]
 extern crate tempfile;
 extern crate tera;
+extern crate toml;
 extern crate unicode_segmentation;
 extern crate walkdir;
 
+pub mod de;
 pub mod fs;
 pub mod net;
 pub mod site;