Browse Source

load_data now defaults to plain type + fix bug with get_taxonomy fn

index-subcmd
Vincent Prouillet 5 years ago
parent
commit
c63b7fde44
6 changed files with 48 additions and 51 deletions
  1. +3
    -1
      CHANGELOG.md
  2. +2
    -6
      components/rendering/src/markdown.rs
  3. +16
    -11
      components/site/src/lib.rs
  4. +8
    -22
      components/templates/src/global_fns/load_data.rs
  5. +4
    -1
      components/templates/src/lib.rs
  6. +15
    -10
      docs/content/documentation/templates/overview.md

+ 3
- 1
CHANGELOG.md View File

@@ -20,7 +20,9 @@ rendering it and not anymore on the `page`/`section` variable
- Fix sections with render=false being shown in sitemap - Fix sections with render=false being shown in sitemap
- Sitemap is now split when there are more than 30 000 links in it - Sitemap is now split when there are more than 30 000 links in it
- Add link to sitemap in robots.txt - Add link to sitemap in robots.txt
- Markdown rendering is now fully CommonMark commpliant
- Markdown rendering is now fully CommonMark compliant
- `load_data` now defaults to loading file as plain text, unless `format` is passed
or the extension matches csv/toml/json




## 0.5.1 (2018-12-14) ## 0.5.1 (2018-12-14)


+ 2
- 6
components/rendering/src/markdown.rs View File

@@ -14,7 +14,7 @@ use table_of_contents::{make_table_of_contents, Header};
use utils::site::resolve_internal_link; use utils::site::resolve_internal_link;
use utils::vec::InsertMany; use utils::vec::InsertMany;


use self::cmark::{Event, Options, Parser, Tag, LinkType};
use self::cmark::{Event, LinkType, Options, Parser, Tag};


const CONTINUE_READING: &str = const CONTINUE_READING: &str =
"<p id=\"zola-continue-reading\"><a name=\"continue-reading\"></a></p>\n"; "<p id=\"zola-continue-reading\"><a name=\"continue-reading\"></a></p>\n";
@@ -200,11 +200,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
Event::Start(Tag::Image(link_type, src, title)) => { Event::Start(Tag::Image(link_type, src, title)) => {
if is_colocated_asset_link(&src) { if is_colocated_asset_link(&src) {
let link = format!("{}{}", context.current_page_permalink, &*src); let link = format!("{}{}", context.current_page_permalink, &*src);
return Event::Start(Tag::Image(
link_type,
link.into(),
title,
));
return Event::Start(Tag::Image(link_type, link.into(), title));
} }


Event::Start(Tag::Image(link_type, src, title)) Event::Start(Tag::Image(link_type, src, title))


+ 16
- 11
components/site/src/lib.rs View File

@@ -246,10 +246,12 @@ impl Site {
self.add_page(p, false)?; self.add_page(p, false)?;
} }


// taxonomy Tera fns are loaded in `register_early_global_fns`
// so we do need to populate it first.
self.populate_taxonomies()?;
self.register_early_global_fns(); self.register_early_global_fns();
self.populate_sections(); self.populate_sections();
self.render_markdown()?; self.render_markdown()?;
self.populate_taxonomies()?;
self.register_tera_global_fns(); self.register_tera_global_fns();


Ok(()) Ok(())
@@ -347,10 +349,7 @@ impl Site {
"resize_image", "resize_image",
global_fns::ResizeImage::new(self.imageproc.clone()), global_fns::ResizeImage::new(self.imageproc.clone()),
); );
self.tera.register_function(
"load_data",
global_fns::LoadData::new(self.base_path.clone()),
);
self.tera.register_function("load_data", global_fns::LoadData::new(self.base_path.clone()));
self.tera.register_function("trans", global_fns::Trans::new(self.config.clone())); self.tera.register_function("trans", global_fns::Trans::new(self.config.clone()));
self.tera.register_function( self.tera.register_function(
"get_taxonomy_url", "get_taxonomy_url",
@@ -862,7 +861,6 @@ impl Site {
taxonomies.push(terms); taxonomies.push(terms);
} }



let mut all_sitemap_entries = HashSet::new(); let mut all_sitemap_entries = HashSet::new();
for p in pages { for p in pages {
all_sitemap_entries.insert(p); all_sitemap_entries.insert(p);
@@ -886,25 +884,32 @@ impl Site {
context.insert("entries", &all_sitemap_entries); context.insert("entries", &all_sitemap_entries);
let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?; let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?;
create_file(&self.output_path.join("sitemap.xml"), sitemap)?; create_file(&self.output_path.join("sitemap.xml"), sitemap)?;
return Ok(())
return Ok(());
} }


// Create multiple sitemaps (max 30000 urls each) // Create multiple sitemaps (max 30000 urls each)
let mut sitemap_index = Vec::new(); let mut sitemap_index = Vec::new();
for (i, chunk) in all_sitemap_entries.iter().collect::<Vec<_>>().chunks(sitemap_limit).enumerate() {
for (i, chunk) in
all_sitemap_entries.iter().collect::<Vec<_>>().chunks(sitemap_limit).enumerate()
{
let mut context = Context::new(); let mut context = Context::new();
context.insert("entries", &chunk); context.insert("entries", &chunk);
let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?; let sitemap = &render_template("sitemap.xml", &self.tera, context, &self.config.theme)?;
let file_name = format!("sitemap{}.xml", i+1);
let file_name = format!("sitemap{}.xml", i + 1);
create_file(&self.output_path.join(&file_name), sitemap)?; create_file(&self.output_path.join(&file_name), sitemap)?;
let mut sitemap_url:String = self.config.make_permalink(&file_name);
let mut sitemap_url: String = self.config.make_permalink(&file_name);
sitemap_url.pop(); // Remove trailing slash sitemap_url.pop(); // Remove trailing slash
sitemap_index.push(sitemap_url); sitemap_index.push(sitemap_url);
} }
// Create main sitemap that reference numbered sitemaps // Create main sitemap that reference numbered sitemaps
let mut main_context = Context::new(); let mut main_context = Context::new();
main_context.insert("sitemaps", &sitemap_index); main_context.insert("sitemaps", &sitemap_index);
let sitemap = &render_template("split_sitemap_index.xml", &self.tera, main_context, &self.config.theme)?;
let sitemap = &render_template(
"split_sitemap_index.xml",
&self.tera,
main_context,
&self.config.theme,
)?;
create_file(&self.output_path.join("sitemap.xml"), sitemap)?; create_file(&self.output_path.join("sitemap.xml"), sitemap)?;


Ok(()) Ok(())


+ 8
- 22
components/templates/src/global_fns/load_data.rs View File

@@ -151,7 +151,7 @@ fn get_output_format_from_args(
let format_arg = optional_arg!( let format_arg = optional_arg!(
String, String,
args.get("format"), args.get("format"),
"`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml)"
"`load_data`: `format` needs to be an argument with a string value, being one of the supported `load_data` file types (csv, json, toml, plain)"
); );


if let Some(format) = format_arg { if let Some(format) = format_arg {
@@ -159,11 +159,7 @@ fn get_output_format_from_args(
} }


let from_extension = if let DataSource::Path(path) = data_source { let from_extension = if let DataSource::Path(path) = data_source {
let extension_result: Result<&str> =
path.extension().map(|extension| extension.to_str().unwrap()).ok_or_else(|| {
format!("Could not determine format for {} from extension", path.display()).into()
});
extension_result?
path.extension().map(|extension| extension.to_str().unwrap()).unwrap_or_else(|| "plain")
} else { } else {
"plain" "plain"
}; };
@@ -332,8 +328,7 @@ mod tests {


#[test] #[test]
fn fails_when_missing_file() { fn fails_when_missing_file() {
let static_fn =
LoadData::new(PathBuf::from("../utils"));
let static_fn = LoadData::new(PathBuf::from("../utils"));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("../../../READMEE.md").unwrap()); args.insert("path".to_string(), to_value("../../../READMEE.md").unwrap());
let result = static_fn.call(&args); let result = static_fn.call(&args);
@@ -343,8 +338,7 @@ mod tests {


#[test] #[test]
fn cant_load_outside_content_dir() { fn cant_load_outside_content_dir() {
let static_fn =
LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let static_fn = LoadData::new(PathBuf::from(PathBuf::from("../utils")));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("../../README.md").unwrap()); args.insert("path".to_string(), to_value("../../README.md").unwrap());
args.insert("format".to_string(), to_value("plain").unwrap()); args.insert("format".to_string(), to_value("plain").unwrap());
@@ -421,9 +415,7 @@ mod tests {


#[test] #[test]
fn can_load_toml() { fn can_load_toml() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.toml").unwrap()); args.insert("path".to_string(), to_value("test.toml").unwrap());
let result = static_fn.call(&args.clone()).unwrap(); let result = static_fn.call(&args.clone()).unwrap();
@@ -442,9 +434,7 @@ mod tests {


#[test] #[test]
fn can_load_csv() { fn can_load_csv() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.csv").unwrap()); args.insert("path".to_string(), to_value("test.csv").unwrap());
let result = static_fn.call(&args.clone()).unwrap(); let result = static_fn.call(&args.clone()).unwrap();
@@ -464,9 +454,7 @@ mod tests {
// Test points to bad csv file with uneven row lengths // Test points to bad csv file with uneven row lengths
#[test] #[test]
fn bad_csv_should_result_in_error() { fn bad_csv_should_result_in_error() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("uneven_rows.csv").unwrap()); args.insert("path".to_string(), to_value("uneven_rows.csv").unwrap());
let result = static_fn.call(&args.clone()); let result = static_fn.call(&args.clone());
@@ -486,9 +474,7 @@ mod tests {


#[test] #[test]
fn can_load_json() { fn can_load_json() {
let static_fn = LoadData::new(
PathBuf::from("../utils/test-files"),
);
let static_fn = LoadData::new(PathBuf::from("../utils/test-files"));
let mut args = HashMap::new(); let mut args = HashMap::new();
args.insert("path".to_string(), to_value("test.json").unwrap()); args.insert("path".to_string(), to_value("test.json").unwrap());
let result = static_fn.call(&args.clone()).unwrap(); let result = static_fn.call(&args.clone()).unwrap();


+ 4
- 1
components/templates/src/lib.rs View File

@@ -35,7 +35,10 @@ lazy_static! {
("__zola_builtins/rss.xml", include_str!("builtins/rss.xml")), ("__zola_builtins/rss.xml", include_str!("builtins/rss.xml")),
("__zola_builtins/sitemap.xml", include_str!("builtins/sitemap.xml")), ("__zola_builtins/sitemap.xml", include_str!("builtins/sitemap.xml")),
("__zola_builtins/robots.txt", include_str!("builtins/robots.txt")), ("__zola_builtins/robots.txt", include_str!("builtins/robots.txt")),
("__zola_builtins/split_sitemap_index.xml", include_str!("builtins/split_sitemap_index.xml")),
(
"__zola_builtins/split_sitemap_index.xml",
include_str!("builtins/split_sitemap_index.xml"),
),
("__zola_builtins/anchor-link.html", include_str!("builtins/anchor-link.html")), ("__zola_builtins/anchor-link.html", include_str!("builtins/anchor-link.html")),
( (
"__zola_builtins/shortcodes/youtube.html", "__zola_builtins/shortcodes/youtube.html",


+ 15
- 10
docs/content/documentation/templates/overview.md View File

@@ -146,34 +146,37 @@ Gets the whole taxonomy of a specific kind.
``` ```


### `load_data` ### `load_data`
Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*.
Loads data from a file or URL. Supported file types include *toml*, *json* and *csv*.
Any other file type will be loaded as plain text.


The `path` argument specifies the path to the data file relative to your content directory.
The `path` argument specifies the path to the data file relative to your base directory, where your `config.toml` is.
As a security precaution, If this file is outside of the main site directory, your site will fail to build. As a security precaution, If this file is outside of the main site directory, your site will fail to build.


```jinja2 ```jinja2
{% set data = load_data(path="blog/story/data.toml") %}
{% set data = load_data(path="content/blog/story/data.toml") %}
``` ```


The optional `format` argument allows you to specify and override which data type is contained The optional `format` argument allows you to specify and override which data type is contained
within the file specified in the `path` argument. Valid entries are *"toml"*, *"json"*, *"csv"*
or *"plain"*. If the `format` argument isn't specified, then the paths extension is used.
within the file specified in the `path` argument. Valid entries are `toml`, `json`, `csv`
or `plain`. If the `format` argument isn't specified, then the paths extension is used.


```jinja2 ```jinja2
{% set data = load_data(path="blog/story/data.txt", format="json") %}
{% set data = load_data(path="content/blog/story/data.txt", format="json") %}
``` ```


Use the `plain` format for when your file has a toml/json/csv extension but you want to load it as plain text.

For *toml* and *json* the data is loaded into a structure matching the original data file, For *toml* and *json* the data is loaded into a structure matching the original data file,
however for *csv* there is no native notion of such a structure. Instead the data is seperated
however for *csv* there is no native notion of such a structure. Instead the data is separated
into a data structure containing *headers* and *records*. See the example below to see into a data structure containing *headers* and *records*. See the example below to see
how this works. how this works.


In the template: In the template:
```jinja2 ```jinja2
{% set data = load_data(path="blog/story/data.csv") %}
{% set data = load_data(path="content/blog/story/data.csv") %}
``` ```


In the *blog/story/data.csv* file:
In the *content/blog/story/data.csv* file:
```csv ```csv
Number, Title Number, Title
1,Gutenberg 1,Gutenberg
@@ -211,7 +214,9 @@ By default, the response body will be returned with no parsing. This can be chan


#### Data Caching #### Data Caching


Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint. URLs are cached based on the URL, and data files are cached based on the files modified time. The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats.
Data file loading and remote requests are cached in memory during build, so multiple requests aren't made to the same endpoint.
URLs are cached based on the URL, and data files are cached based on the files modified time.
The format is also taken into account when caching, so a request will be sent twice if it's loaded with 2 different formats.


### `trans` ### `trans`
Gets the translation of the given `key`, for the `default_language` or the `language given Gets the translation of the given `key`, for the `default_language` or the `language given


Loading…
Cancel
Save