diff --git a/Cargo.lock b/Cargo.lock index f969f12..67c057c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -449,6 +449,23 @@ name = "crossbeam-utils" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "csv" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "csv-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ctrlc" version = "3.1.1" @@ -2167,12 +2184,16 @@ version = "0.1.0" dependencies = [ "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", "config 0.1.0", + "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "errors 0.1.0", "imageproc 0.1.0", "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "library 0.1.0", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", "tera 0.11.17 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "utils 0.1.0", ] @@ -2875,6 +2896,8 @@ dependencies = [ "checksum crossbeam-epoch 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c90f1474584f38e270b5b613e898c8c328aa4f3dea85e0a27ac2e642f009416" "checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" +"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452" +"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105" "checksum ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "630391922b1b893692c6334369ff528dcc3a9d8061ccf4c803aa8f83cb13db5e" "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum deflate 0.7.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8a6abb26e16e8d419b5c78662aa9f82857c2386a073da266840e474d5055ec86" diff --git a/README.md b/README.md index 7acdfe0..fa923cf 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ in the `docs/content` folder of the repository. | Pagination | ✔ | ✕ | ✔ | ✔ | | Custom taxonomies | ✔ | ✕ | ✔ | ✕ | | Search | ✔ | ✕ | ✕ | ✔ | -| Data files | ✕ | ✔ | ✔ | ✕ | +| Data files | ✔ | ✔ | ✔ | ✕ | | LiveReload | ✔ | ✕ | ✔ | ✔ | | Netlify support | ✔ | ✕ | ✔ | ✕ | diff --git a/components/site/src/lib.rs b/components/site/src/lib.rs index 2d756d9..e6204a2 100644 --- a/components/site/src/lib.rs +++ b/components/site/src/lib.rs @@ -307,6 +307,7 @@ impl Site { "get_taxonomy_url", global_fns::make_get_taxonomy_url(&self.taxonomies), ); + self.tera.register_function("load_data", global_fns::make_load_data(self.content_path.clone())); } /// Add a page to the site diff --git a/components/templates/Cargo.toml b/components/templates/Cargo.toml index 1645fde..c39963b 100644 --- a/components/templates/Cargo.toml +++ b/components/templates/Cargo.toml @@ -8,6 +8,10 @@ tera = "0.11" base64 = "0.9" lazy_static = "1" pulldown-cmark = "0" +toml = "0.4" +csv = "1" +serde_json = "1.0" +error-chain = "0.12" errors = { path = "../errors" } utils = { path = "../utils" } diff --git a/components/templates/src/global_fns.rs b/components/templates/src/global_fns.rs index 872ed98..23efd2e 100644 --- a/components/templates/src/global_fns.rs +++ b/components/templates/src/global_fns.rs @@ -1,14 +1,21 @@ +extern crate toml; +extern crate serde_json; +extern crate error_chain; + use std::collections::HashMap; use std::sync::{Arc, Mutex}; +use std::path::PathBuf; + +use csv::Reader; -use tera::{GlobalFn, Value, from_value, to_value, Result}; +use tera::{GlobalFn, Value, from_value, to_value, Result, Map}; use library::{Taxonomy, Library}; use config::Config; use utils::site::resolve_internal_link; +use utils::fs::read_file; use imageproc; - macro_rules! required_arg { ($ty: ty, $e: expr, $err: expr) => { match $e { @@ -264,12 +271,132 @@ pub fn make_resize_image(imageproc: Arc>) -> GlobalF }) } +/// A global function to load data from a data file. +/// Currently the supported formats are json, toml and csv +pub fn make_load_data(content_path: PathBuf) -> GlobalFn { + Box::new(move |args| -> Result { + let path_arg: String = required_arg!( + String, + args.get("path"), + "`load_data`: requires a `path` argument with a string value, being a path to a file" + ); + let kind_arg = optional_arg!( + String, + args.get("kind"), + "`load_data`: `kind` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)" + ); + + let full_path = content_path.join(&path_arg); + + let extension = match full_path.extension() { + Some(value) => value.to_str().unwrap().to_lowercase(), + None => return Err(format!("`load_data`: Cannot parse file extension of specified file: {}", path_arg).into()) + }; + + let file_kind = kind_arg.unwrap_or(extension); + + let result_value: Result = match file_kind.as_str() { + "toml" => load_toml(&full_path), + "csv" => load_csv(&full_path), + "json" => load_json(&full_path), + _ => return Err(format!("'load_data': {} - is an unsupported file kind", file_kind).into()) + }; + + result_value + }) +} + +/// load/parse a json file from the given path and place it into a +/// tera value +fn load_json(json_path: &PathBuf) -> Result { + + let content_string: String = read_file(json_path) + .map_err(|e| format!("`load_data`: error {} loading json file {}", json_path.to_str().unwrap(), e))?; + + let json_content = serde_json::from_str(content_string.as_str()).unwrap(); + let tera_value: Value = json_content; + + return Ok(tera_value); +} + +/// load/parse a toml file from the given path, and place it into a +/// tera Value +fn load_toml(toml_path: &PathBuf) -> Result { + let content_string: String = read_file(toml_path) + .map_err(|e| format!("`load_data`: error {} loading toml file {}", toml_path.to_str().unwrap(), e))?; + + let toml_content: toml::Value = toml::from_str(&content_string) + .map_err(|e| format!("'load_data': {} - {}", toml_path.to_str().unwrap(), e))?; + + to_value(toml_content).map_err(|err| err.into()) +} + +/// Load/parse a csv file from the given path, and place it into a +/// tera Value. +/// +/// An example csv file `example.csv` could be: +/// ```csv +/// Number, Title +/// 1,Gutenberg +/// 2,Printing +/// ``` +/// The json value output would be: +/// ```json +/// { +/// "headers": ["Number", "Title"], +/// "records": [ +/// ["1", "Gutenberg"], +/// ["2", "Printing"] +/// ], +/// } +/// ``` +fn load_csv(csv_path: &PathBuf) -> Result { + let mut reader = Reader::from_path(csv_path.clone()) + .map_err(|e| format!("'load_data': {} - {}", csv_path.to_str().unwrap(), e))?; + + let mut csv_map = Map::new(); + + { + let hdrs = reader.headers() + .map_err(|e| format!("'load_data': {} - {} - unable to read CSV header line (line 1) for CSV file", csv_path.to_str().unwrap(), e))?; + + let headers_array = hdrs.iter() + .map(|v| Value::String(v.to_string())) + .collect(); + + csv_map.insert(String::from("headers"), Value::Array(headers_array)); + } + + { + let records = reader.records(); + + let mut records_array: Vec = Vec::new(); + + for result in records { + let record = result.unwrap(); + + let mut elements_array: Vec = Vec::new(); + + for e in record.into_iter() { + elements_array.push(Value::String(String::from(e))); + } + + records_array.push(Value::Array(elements_array)); + } + + csv_map.insert(String::from("records"), Value::Array(records_array)); + } + + let csv_value: Value = Value::Object(csv_map); + to_value(csv_value).map_err(|err| err.into()) +} #[cfg(test)] mod tests { - use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans}; + use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans, make_load_data}; use std::collections::HashMap; + use std::path::PathBuf; use tera::{to_value, Value}; @@ -422,4 +549,58 @@ title = "A title" args.insert("lang".to_string(), to_value("fr").unwrap()); assert_eq!(static_fn(args.clone()).unwrap(), "Un titre"); } + + #[test] + fn can_load_toml() + { + let static_fn = make_load_data(PathBuf::from("../utils/test-files")); + let mut args = HashMap::new(); + args.insert("path".to_string(), to_value("test.toml").unwrap()); + let result = static_fn(args.clone()).unwrap(); + + //TOML does not load in order, and also dates are not returned as strings, but + //rather as another object with a key and value + assert_eq!(result, json!({ + "category": { + "date": { + "$__toml_private_datetime": "1979-05-27T07:32:00Z" + }, + "key": "value" + }, + })); + } + + #[test] + fn can_load_csv() + { + let static_fn = make_load_data(PathBuf::from("../utils/test-files")); + let mut args = HashMap::new(); + args.insert("path".to_string(), to_value("test.csv").unwrap()); + let result = static_fn(args.clone()).unwrap(); + + assert_eq!(result, json!({ + "headers": ["Number", "Title"], + "records": [ + ["1", "Gutenberg"], + ["2", "Printing"] + ], + })) + } + + #[test] + fn can_load_json() + { + let static_fn = make_load_data(PathBuf::from("../utils/test-files")); + let mut args = HashMap::new(); + args.insert("path".to_string(), to_value("test.json").unwrap()); + let result = static_fn(args.clone()).unwrap(); + + assert_eq!(result, json!({ + "key": "value", + "array": [1, 2, 3], + "subpackage": { + "subkey": 5 + } + })) + } } diff --git a/components/templates/src/lib.rs b/components/templates/src/lib.rs index 31630b6..5b3080c 100644 --- a/components/templates/src/lib.rs +++ b/components/templates/src/lib.rs @@ -4,6 +4,13 @@ extern crate lazy_static; extern crate tera; extern crate base64; extern crate pulldown_cmark; +extern crate csv; + +#[cfg(test)] +#[macro_use] +extern crate serde_json; +#[cfg(not(test))] +extern crate serde_json; extern crate errors; extern crate utils; diff --git a/components/utils/test-files/test.csv b/components/utils/test-files/test.csv new file mode 100644 index 0000000..62380d5 --- /dev/null +++ b/components/utils/test-files/test.csv @@ -0,0 +1,3 @@ +Number,Title +1,Gutenberg +2,Printing \ No newline at end of file diff --git a/components/utils/test-files/test.json b/components/utils/test-files/test.json new file mode 100644 index 0000000..fc0c34a --- /dev/null +++ b/components/utils/test-files/test.json @@ -0,0 +1,7 @@ +{ + "key": "value", + "array": [1, 2, 3], + "subpackage": { + "subkey": 5 + } +} \ No newline at end of file diff --git a/components/utils/test-files/test.toml b/components/utils/test-files/test.toml new file mode 100644 index 0000000..6473320 --- /dev/null +++ b/components/utils/test-files/test.toml @@ -0,0 +1,3 @@ +[category] +key = "value" +date = 1979-05-27T07:32:00Z \ No newline at end of file diff --git a/docs/content/documentation/templates/overview.md b/docs/content/documentation/templates/overview.md index 019d992..6d09cce 100644 --- a/docs/content/documentation/templates/overview.md +++ b/docs/content/documentation/templates/overview.md @@ -142,6 +142,52 @@ Gets the whole taxonomy of a specific kind. {% set categories = get_taxonomy_url(kind="categories") %} ``` +### `load_data` +Loads data from a file. Supported file types include *toml*, *json* and *csv*. + +The `path` argument specifies the path to the data file relative to your content directory. + +```jinja2 +{% set data = load_data(path="blog/story/data.toml") %} +``` + +The optional `kind` argument allows you to specify and override which data type is contained +within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"* +or *"csv"*. + +```jinja2 +{% set data = load_data(path="blog/story/data.txt", kind="json") %} +``` + +For *toml* and *json* the data is loaded into a structure matching the original data file, +however for *csv* there is no native notion of such a structure. Instead the data is seperated +into a data structure containing *headers* and *records*. See the example below to see +how this works. + +In the template: +```jinja2 +{% set data = load_data(path="blog/story/data.csv") %} +``` + +In the *blog/story/data.csv* file: + ```csv +Number, Title +1,Gutenberg +2,Printing +``` + +The equivalent json value of the parsed data would be stored in the `data` variable in the +template: +```json +{ + "headers": ["Number", "Title"], + "records": [ + ["1", "Gutenberg"], + ["2", "Printing"] + ], +} + ``` + ### `trans` Gets the translation of the given `key`, for the `default_language` or the `language given