Browse Source

CSV and TOML loading global functions (#379)

Local CSV/TOML/JSON loading Tera function
index-subcmd
Luke Frisken Vincent Prouillet 5 years ago
parent
commit
1baa7750f3
10 changed files with 279 additions and 4 deletions
  1. +23
    -0
      Cargo.lock
  2. +1
    -1
      README.md
  3. +1
    -0
      components/site/src/lib.rs
  4. +4
    -0
      components/templates/Cargo.toml
  5. +184
    -3
      components/templates/src/global_fns.rs
  6. +7
    -0
      components/templates/src/lib.rs
  7. +3
    -0
      components/utils/test-files/test.csv
  8. +7
    -0
      components/utils/test-files/test.json
  9. +3
    -0
      components/utils/test-files/test.toml
  10. +46
    -0
      docs/content/documentation/templates/overview.md

+ 23
- 0
Cargo.lock View File

@@ -449,6 +449,23 @@ name = "crossbeam-utils"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "csv"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "csv-core"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "ctrlc"
version = "3.1.1"
@@ -2167,12 +2184,16 @@ version = "0.1.0"
dependencies = [
"base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
"config 0.1.0",
"csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"errors 0.1.0",
"imageproc 0.1.0",
"lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"library 0.1.0",
"pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)",
"tera 0.11.17 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"utils 0.1.0",
]

@@ -2875,6 +2896,8 @@ dependencies = [
"checksum crossbeam-epoch 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c90f1474584f38e270b5b613e898c8c328aa4f3dea85e0a27ac2e642f009416"
"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452"
"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105"
"checksum ctrlc 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "630391922b1b893692c6334369ff528dcc3a9d8061ccf4c803aa8f83cb13db5e"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum deflate 0.7.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8a6abb26e16e8d419b5c78662aa9f82857c2386a073da266840e474d5055ec86"


+ 1
- 1
README.md View File

@@ -29,7 +29,7 @@ in the `docs/content` folder of the repository.
| Pagination | ✔ | ✕ | ✔ | ✔ |
| Custom taxonomies | ✔ | ✕ | ✔ | ✕ |
| Search | ✔ | ✕ | ✕ | ✔ |
| Data files | | ✔ | ✔ | ✕ |
| Data files | | ✔ | ✔ | ✕ |
| LiveReload | ✔ | ✕ | ✔ | ✔ |
| Netlify support | ✔ | ✕ | ✔ | ✕ |



+ 1
- 0
components/site/src/lib.rs View File

@@ -307,6 +307,7 @@ impl Site {
"get_taxonomy_url",
global_fns::make_get_taxonomy_url(&self.taxonomies),
);
self.tera.register_function("load_data", global_fns::make_load_data(self.content_path.clone()));
}

/// Add a page to the site


+ 4
- 0
components/templates/Cargo.toml View File

@@ -8,6 +8,10 @@ tera = "0.11"
base64 = "0.9"
lazy_static = "1"
pulldown-cmark = "0"
toml = "0.4"
csv = "1"
serde_json = "1.0"
error-chain = "0.12"

errors = { path = "../errors" }
utils = { path = "../utils" }


+ 184
- 3
components/templates/src/global_fns.rs View File

@@ -1,14 +1,21 @@
extern crate toml;
extern crate serde_json;
extern crate error_chain;

use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::path::PathBuf;

use csv::Reader;

use tera::{GlobalFn, Value, from_value, to_value, Result};
use tera::{GlobalFn, Value, from_value, to_value, Result, Map};

use library::{Taxonomy, Library};
use config::Config;
use utils::site::resolve_internal_link;
use utils::fs::read_file;
use imageproc;


macro_rules! required_arg {
($ty: ty, $e: expr, $err: expr) => {
match $e {
@@ -264,12 +271,132 @@ pub fn make_resize_image(imageproc: Arc<Mutex<imageproc::Processor>>) -> GlobalF
})
}

/// A global function to load data from a data file.
/// Currently the supported formats are json, toml and csv
pub fn make_load_data(content_path: PathBuf) -> GlobalFn {
Box::new(move |args| -> Result<Value> {
let path_arg: String = required_arg!(
String,
args.get("path"),
"`load_data`: requires a `path` argument with a string value, being a path to a file"
);
let kind_arg = optional_arg!(
String,
args.get("kind"),
"`load_data`: `kind` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)"
);

let full_path = content_path.join(&path_arg);

let extension = match full_path.extension() {
Some(value) => value.to_str().unwrap().to_lowercase(),
None => return Err(format!("`load_data`: Cannot parse file extension of specified file: {}", path_arg).into())
};

let file_kind = kind_arg.unwrap_or(extension);

let result_value: Result<Value> = match file_kind.as_str() {
"toml" => load_toml(&full_path),
"csv" => load_csv(&full_path),
"json" => load_json(&full_path),
_ => return Err(format!("'load_data': {} - is an unsupported file kind", file_kind).into())
};

result_value
})
}

/// load/parse a json file from the given path and place it into a
/// tera value
fn load_json(json_path: &PathBuf) -> Result<Value> {
let content_string: String = read_file(json_path)
.map_err(|e| format!("`load_data`: error {} loading json file {}", json_path.to_str().unwrap(), e))?;

let json_content = serde_json::from_str(content_string.as_str()).unwrap();
let tera_value: Value = json_content;

return Ok(tera_value);
}

/// load/parse a toml file from the given path, and place it into a
/// tera Value
fn load_toml(toml_path: &PathBuf) -> Result<Value> {
let content_string: String = read_file(toml_path)
.map_err(|e| format!("`load_data`: error {} loading toml file {}", toml_path.to_str().unwrap(), e))?;

let toml_content: toml::Value = toml::from_str(&content_string)
.map_err(|e| format!("'load_data': {} - {}", toml_path.to_str().unwrap(), e))?;

to_value(toml_content).map_err(|err| err.into())
}

/// Load/parse a csv file from the given path, and place it into a
/// tera Value.
///
/// An example csv file `example.csv` could be:
/// ```csv
/// Number, Title
/// 1,Gutenberg
/// 2,Printing
/// ```
/// The json value output would be:
/// ```json
/// {
/// "headers": ["Number", "Title"],
/// "records": [
/// ["1", "Gutenberg"],
/// ["2", "Printing"]
/// ],
/// }
/// ```
fn load_csv(csv_path: &PathBuf) -> Result<Value> {
let mut reader = Reader::from_path(csv_path.clone())
.map_err(|e| format!("'load_data': {} - {}", csv_path.to_str().unwrap(), e))?;

let mut csv_map = Map::new();

{
let hdrs = reader.headers()
.map_err(|e| format!("'load_data': {} - {} - unable to read CSV header line (line 1) for CSV file", csv_path.to_str().unwrap(), e))?;

let headers_array = hdrs.iter()
.map(|v| Value::String(v.to_string()))
.collect();

csv_map.insert(String::from("headers"), Value::Array(headers_array));
}

{
let records = reader.records();

let mut records_array: Vec<Value> = Vec::new();

for result in records {
let record = result.unwrap();

let mut elements_array: Vec<Value> = Vec::new();

for e in record.into_iter() {
elements_array.push(Value::String(String::from(e)));
}

records_array.push(Value::Array(elements_array));
}

csv_map.insert(String::from("records"), Value::Array(records_array));
}

let csv_value: Value = Value::Object(csv_map);
to_value(csv_value).map_err(|err| err.into())
}

#[cfg(test)]
mod tests {
use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans};
use super::{make_get_url, make_get_taxonomy, make_get_taxonomy_url, make_trans, make_load_data};

use std::collections::HashMap;
use std::path::PathBuf;

use tera::{to_value, Value};

@@ -422,4 +549,58 @@ title = "A title"
args.insert("lang".to_string(), to_value("fr").unwrap());
assert_eq!(static_fn(args.clone()).unwrap(), "Un titre");
}

#[test]
fn can_load_toml()
{
let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.toml").unwrap());
let result = static_fn(args.clone()).unwrap();

//TOML does not load in order, and also dates are not returned as strings, but
//rather as another object with a key and value
assert_eq!(result, json!({
"category": {
"date": {
"$__toml_private_datetime": "1979-05-27T07:32:00Z"
},
"key": "value"
},
}));
}

#[test]
fn can_load_csv()
{
let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.csv").unwrap());
let result = static_fn(args.clone()).unwrap();

assert_eq!(result, json!({
"headers": ["Number", "Title"],
"records": [
["1", "Gutenberg"],
["2", "Printing"]
],
}))
}

#[test]
fn can_load_json()
{
let static_fn = make_load_data(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.json").unwrap());
let result = static_fn(args.clone()).unwrap();

assert_eq!(result, json!({
"key": "value",
"array": [1, 2, 3],
"subpackage": {
"subkey": 5
}
}))
}
}

+ 7
- 0
components/templates/src/lib.rs View File

@@ -4,6 +4,13 @@ extern crate lazy_static;
extern crate tera;
extern crate base64;
extern crate pulldown_cmark;
extern crate csv;

#[cfg(test)]
#[macro_use]
extern crate serde_json;
#[cfg(not(test))]
extern crate serde_json;

extern crate errors;
extern crate utils;


+ 3
- 0
components/utils/test-files/test.csv View File

@@ -0,0 +1,3 @@
Number,Title
1,Gutenberg
2,Printing

+ 7
- 0
components/utils/test-files/test.json View File

@@ -0,0 +1,7 @@
{
"key": "value",
"array": [1, 2, 3],
"subpackage": {
"subkey": 5
}
}

+ 3
- 0
components/utils/test-files/test.toml View File

@@ -0,0 +1,3 @@
[category]
key = "value"
date = 1979-05-27T07:32:00Z

+ 46
- 0
docs/content/documentation/templates/overview.md View File

@@ -142,6 +142,52 @@ Gets the whole taxonomy of a specific kind.
{% set categories = get_taxonomy_url(kind="categories") %}
```

### `load_data`
Loads data from a file. Supported file types include *toml*, *json* and *csv*.

The `path` argument specifies the path to the data file relative to your content directory.

```jinja2
{% set data = load_data(path="blog/story/data.toml") %}
```

The optional `kind` argument allows you to specify and override which data type is contained
within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"*
or *"csv"*.

```jinja2
{% set data = load_data(path="blog/story/data.txt", kind="json") %}
```

For *toml* and *json* the data is loaded into a structure matching the original data file,
however for *csv* there is no native notion of such a structure. Instead the data is seperated
into a data structure containing *headers* and *records*. See the example below to see
how this works.

In the template:
```jinja2
{% set data = load_data(path="blog/story/data.csv") %}
```

In the *blog/story/data.csv* file:
```csv
Number, Title
1,Gutenberg
2,Printing
```

The equivalent json value of the parsed data would be stored in the `data` variable in the
template:
```json
{
"headers": ["Number", "Title"],
"records": [
["1", "Gutenberg"],
["2", "Printing"]
],
}
```

### `trans`
Gets the translation of the given `key`, for the `default_language` or the `language given



Loading…
Cancel
Save