Browse Source

Use the new shortcode parser

index-subcmd
Vincent Prouillet 6 years ago
parent
commit
85e13483eb
18 changed files with 321 additions and 578 deletions
  1. +3
    -2
      Cargo.lock
  2. +2
    -2
      README.md
  3. +8
    -6
      components/content/src/page.rs
  4. +4
    -5
      components/content/src/section.rs
  5. +1
    -1
      components/front_matter/src/page.rs
  6. +0
    -0
      components/highlighting/examples/generate_sublime.rs
  7. +14
    -1
      components/highlighting/src/lib.rs
  8. +0
    -2
      components/rendering/Cargo.toml
  9. +25
    -16
      components/rendering/benches/all.rs
  10. +11
    -17
      components/rendering/src/context.rs
  11. +15
    -7
      components/rendering/src/lib.rs
  12. +117
    -244
      components/rendering/src/markdown.rs
  13. +0
    -190
      components/rendering/src/short_code.rs
  14. +23
    -7
      components/rendering/src/shortcode.rs
  15. +5
    -7
      components/rendering/src/table_of_contents.rs
  16. +93
    -71
      components/rendering/tests/markdown.rs
  17. BIN
      sublime_syntaxes/newlines.packdump
  18. BIN
      sublime_syntaxes/nonewlines.packdump

+ 3
- 2
Cargo.lock View File

@@ -1113,12 +1113,13 @@ dependencies = [
name = "rendering" name = "rendering"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"config 0.1.0",
"errors 0.1.0", "errors 0.1.0",
"front_matter 0.1.0", "front_matter 0.1.0",
"highlighting 0.1.0", "highlighting 0.1.0",
"lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_derive 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
"slug 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "slug 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",


+ 2
- 2
README.md View File

@@ -52,7 +52,7 @@ If you want a feature added or modified, please open an issue to discuss it befo
Syntax highlighting depends on submodules so ensure you load them first: Syntax highlighting depends on submodules so ensure you load them first:


```bash ```bash
$ git submodule update --init
$ git submodule update --init
``` ```


Gutenberg only works with syntaxes in the `.sublime-syntax` format. If your syntax Gutenberg only works with syntaxes in the `.sublime-syntax` format. If your syntax
@@ -75,7 +75,7 @@ You can check for any updates to the current packages by running:
$ git submodule update --remote --merge $ git submodule update --remote --merge
``` ```


And finally from the root of the components/rendering crate run the following command:
And finally from the root of the components/highlighting crate run the following command:


```bash ```bash
$ cargo run --example generate_sublime synpack ../../sublime_syntaxes ../../sublime_syntaxes/newlines.packdump ../../sublime_syntaxes/nonewlines.packdump $ cargo run --example generate_sublime synpack ../../sublime_syntaxes ../../sublime_syntaxes/newlines.packdump ../../sublime_syntaxes/nonewlines.packdump


+ 8
- 6
components/content/src/page.rs View File

@@ -14,7 +14,7 @@ use utils::fs::{read_file, find_related_assets};
use utils::site::get_reading_analytics; use utils::site::get_reading_analytics;
use utils::templates::render_template; use utils::templates::render_template;
use front_matter::{PageFrontMatter, InsertAnchor, split_page_content}; use front_matter::{PageFrontMatter, InsertAnchor, split_page_content};
use rendering::{Context, Header, markdown_to_html};
use rendering::{RenderContext, Header, render_content};


use file_info::FileInfo; use file_info::FileInfo;


@@ -162,21 +162,23 @@ impl Page {
/// We need access to all pages url to render links relative to content /// We need access to all pages url to render links relative to content
/// so that can't happen at the same time as parsing /// so that can't happen at the same time as parsing
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config, anchor_insert: InsertAnchor) -> Result<()> { pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config, anchor_insert: InsertAnchor) -> Result<()> {
let context = Context::new(
let context = RenderContext::new(
tera, tera,
config.highlight_code,
config.highlight_theme.clone(),
config,
&self.permalink, &self.permalink,
permalinks, permalinks,
anchor_insert anchor_insert
); );
let res = markdown_to_html(&self.raw_content.replacen("<!-- more -->", "<a name=\"continue-reading\"></a>", 1), &context)?;
let res = render_content(
&self.raw_content.replacen("<!-- more -->", "<a name=\"continue-reading\"></a>", 1),
&context
)?;
self.content = res.0; self.content = res.0;
self.toc = res.1; self.toc = res.1;
if self.raw_content.contains("<!-- more -->") { if self.raw_content.contains("<!-- more -->") {
self.summary = Some({ self.summary = Some({
let summary = self.raw_content.splitn(2, "<!-- more -->").collect::<Vec<&str>>()[0]; let summary = self.raw_content.splitn(2, "<!-- more -->").collect::<Vec<&str>>()[0];
markdown_to_html(summary, &context)?.0
render_content(summary, &context)?.0
}) })
} }




+ 4
- 5
components/content/src/section.rs View File

@@ -11,7 +11,7 @@ use errors::{Result, ResultExt};
use utils::fs::read_file; use utils::fs::read_file;
use utils::templates::render_template; use utils::templates::render_template;
use utils::site::get_reading_analytics; use utils::site::get_reading_analytics;
use rendering::{Context, Header, markdown_to_html};
use rendering::{RenderContext, Header, render_content};


use page::Page; use page::Page;
use file_info::FileInfo; use file_info::FileInfo;
@@ -98,15 +98,14 @@ impl Section {
/// We need access to all pages url to render links relative to content /// We need access to all pages url to render links relative to content
/// so that can't happen at the same time as parsing /// so that can't happen at the same time as parsing
pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config) -> Result<()> { pub fn render_markdown(&mut self, permalinks: &HashMap<String, String>, tera: &Tera, config: &Config) -> Result<()> {
let context = Context::new(
let context = RenderContext::new(
tera, tera,
config.highlight_code,
config.highlight_theme.clone(),
config,
&self.permalink, &self.permalink,
permalinks, permalinks,
self.meta.insert_anchor_links, self.meta.insert_anchor_links,
); );
let res = markdown_to_html(&self.raw_content, &context)?;
let res = render_content(&self.raw_content, &context)?;
self.content = res.0; self.content = res.0;
self.toc = res.1; self.toc = res.1;
Ok(()) Ok(())


+ 1
- 1
components/front_matter/src/page.rs View File

@@ -21,7 +21,7 @@ fn from_toml_datetime<'de, D>(deserializer: D) -> StdResult<Option<String>, D::E
fn convert_toml_date(table: Map<String, Value>) -> Value { fn convert_toml_date(table: Map<String, Value>) -> Value {
let mut new = Map::new(); let mut new = Map::new();


for (k, v) in table.into_iter() {
for (k, v) in table {
if k == "$__toml_private_datetime" { if k == "$__toml_private_datetime" {
return v; return v;
} }


components/rendering/examples/generate_sublime.rs → components/highlighting/examples/generate_sublime.rs View File


+ 14
- 1
components/highlighting/src/lib.rs View File

@@ -4,7 +4,8 @@ extern crate syntect;


use syntect::dumps::from_binary; use syntect::dumps::from_binary;
use syntect::parsing::SyntaxSet; use syntect::parsing::SyntaxSet;
use syntect::highlighting::ThemeSet;
use syntect::highlighting::{ThemeSet, Theme};
use syntect::easy::HighlightLines;


thread_local!{ thread_local!{
pub static SYNTAX_SET: SyntaxSet = { pub static SYNTAX_SET: SyntaxSet = {
@@ -17,3 +18,15 @@ thread_local!{
lazy_static!{ lazy_static!{
pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../../sublime_themes/all.themedump")); pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../../sublime_themes/all.themedump"));
} }


pub fn get_highlighter<'a>(theme: &'a Theme, info: &str) -> HighlightLines<'a> {
SYNTAX_SET.with(|ss| {
let syntax = info
.split(' ')
.next()
.and_then(|lang| ss.find_syntax_by_token(lang))
.unwrap_or_else(|| ss.find_syntax_plain_text());
HighlightLines::new(syntax, theme)
})
}

+ 0
- 2
components/rendering/Cargo.toml View File

@@ -5,8 +5,6 @@ authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]


[dependencies] [dependencies]
tera = "0.11" tera = "0.11"
regex = "1"
lazy_static = "1"
syntect = "2" syntect = "2"
pulldown-cmark = "0" pulldown-cmark = "0"
slug = "0.1" slug = "0.1"


+ 25
- 16
components/rendering/benches/all.rs View File

@@ -9,7 +9,7 @@ extern crate front_matter;
use std::collections::HashMap; use std::collections::HashMap;


use tera::Tera; use tera::Tera;
use rendering::{Context, markdown_to_html, render_shortcodes};
use rendering::{RenderContext, render_content, render_shortcodes};
use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use config::Config; use config::Config;


@@ -86,34 +86,43 @@ if __name__ == "__main__":
"#; "#;


#[bench] #[bench]
fn bench_markdown_to_html_with_highlighting(b: &mut test::Bencher) {
let tera_ctx = Tera::default();
fn bench_render_content_with_highlighting(b: &mut test::Bencher) {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
b.iter(|| markdown_to_html(CONTENT, &context));
let config = Config::default();
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
b.iter(|| render_content(CONTENT, &context).unwrap());
} }


#[bench] #[bench]
fn bench_markdown_to_html_without_highlighting(b: &mut test::Bencher) {
let tera_ctx = Tera::default();
fn bench_render_content_without_highlighting(b: &mut test::Bencher) {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, false, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
b.iter(|| markdown_to_html(CONTENT, &context));
let mut config = Config::default();
config.highlight_code = false;
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);
b.iter(|| render_content(CONTENT, &context).unwrap());
} }


#[bench] #[bench]
fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
fn bench_render_content_no_shortcode(b: &mut test::Bencher) {
let tera = Tera::default();
let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, "");
let mut config = Config::default();
config.highlight_code = false;
let permalinks_ctx = HashMap::new();
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);


b.iter(|| render_shortcodes(CONTENT, &tera, &Config::default()));
b.iter(|| render_content(&content2, &context).unwrap());
} }


#[bench] #[bench]
fn bench_render_shortcodes_none(b: &mut test::Bencher) {
fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
let mut tera = Tera::default(); let mut tera = Tera::default();
tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap(); tera.add_raw_template("shortcodes/youtube.html", "{{id}}").unwrap();
let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, "");


b.iter(|| render_shortcodes(&content2, &tera, &Config::default()));
b.iter(|| render_shortcodes(CONTENT, &tera, &Config::default()));
} }


+ 11
- 17
components/rendering/src/context.rs View File

@@ -1,41 +1,35 @@
use std::collections::HashMap; use std::collections::HashMap;


use tera::Tera; use tera::Tera;

use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use config::Config;




/// All the information from the gutenberg site that is needed to render HTML from markdown /// All the information from the gutenberg site that is needed to render HTML from markdown
#[derive(Debug)] #[derive(Debug)]
pub struct Context<'a> {
pub struct RenderContext<'a> {
pub tera: &'a Tera, pub tera: &'a Tera,
pub highlight_code: bool,
pub highlight_theme: String,
pub current_page_permalink: String,
pub config: &'a Config,
pub current_page_permalink: &'a str,
pub permalinks: &'a HashMap<String, String>, pub permalinks: &'a HashMap<String, String>,
pub insert_anchor: InsertAnchor, pub insert_anchor: InsertAnchor,
} }


impl<'a> Context<'a> {
impl<'a> RenderContext<'a> {
pub fn new( pub fn new(
tera: &'a Tera, tera: &'a Tera,
highlight_code: bool,
highlight_theme: String,
current_page_permalink: &str,
config: &'a Config,
current_page_permalink: &'a str,
permalinks: &'a HashMap<String, String>, permalinks: &'a HashMap<String, String>,
insert_anchor: InsertAnchor, insert_anchor: InsertAnchor,
) -> Context<'a> {
Context {
) -> RenderContext<'a> {
RenderContext {
tera, tera,
current_page_permalink: current_page_permalink.to_string(),
current_page_permalink,
permalinks, permalinks,
insert_anchor, insert_anchor,
highlight_code,
highlight_theme,
config,
} }
} }


pub fn should_insert_anchor(&self) -> bool {
self.insert_anchor != InsertAnchor::None
}
} }

+ 15
- 7
components/rendering/src/lib.rs View File

@@ -1,6 +1,3 @@
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate tera; extern crate tera;
extern crate syntect; extern crate syntect;
extern crate pulldown_cmark; extern crate pulldown_cmark;
@@ -12,7 +9,7 @@ extern crate pest;
#[macro_use] #[macro_use]
extern crate pest_derive; extern crate pest_derive;


#[macro_use]
extern crate errors; extern crate errors;
extern crate front_matter; extern crate front_matter;
extern crate highlighting; extern crate highlighting;
@@ -26,9 +23,20 @@ mod context;
mod markdown; mod markdown;
mod table_of_contents; mod table_of_contents;
mod shortcode; mod shortcode;
mod short_code;


pub use context::Context;
pub use markdown::markdown_to_html;
use errors::Result;

use markdown::markdown_to_html;
pub use table_of_contents::Header; pub use table_of_contents::Header;
pub use shortcode::render_shortcodes; pub use shortcode::render_shortcodes;
pub use context::RenderContext;

pub fn render_content(content: &str, context: &RenderContext) -> Result<(String, Vec<Header>)> {
// Don't do anything if there is nothing like a shortcode in the content
if content.contains("{{") || content.contains("{%") {
let rendered = render_shortcodes(content, context.tera, context.config)?;
return markdown_to_html(&rendered, context);
}

markdown_to_html(&content, context)
}

+ 117
- 244
components/rendering/src/markdown.rs View File

@@ -8,37 +8,36 @@ use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, Includ


use errors::Result; use errors::Result;
use utils::site::resolve_internal_link; use utils::site::resolve_internal_link;
use context::Context;
use highlighting::{SYNTAX_SET, THEME_SET};
use short_code::{SHORTCODE_RE, ShortCode, parse_shortcode, render_simple_shortcode};
use highlighting::{get_highlighter, THEME_SET};

use table_of_contents::{TempHeader, Header, make_table_of_contents}; use table_of_contents::{TempHeader, Header, make_table_of_contents};
use context::RenderContext;

// We might have cases where the slug is already present in our list of anchor
// for example an article could have several titles named Example
// We add a counter after the slug if the slug is already present, which
// means we will have example, example-1, example-2 etc
fn find_anchor(anchors: &[String], name: String, level: u8) -> String {
if level == 0 && !anchors.contains(&name) {
return name.to_string();
}

let new_anchor = format!("{}-{}", name, level + 1);
if !anchors.contains(&new_anchor) {
return new_anchor;
}

find_anchor(anchors, name, level + 1)
}




pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec<Header>)> {
// We try to be smart about highlighting code as it can be time-consuming
// If the global config disables it, then we do nothing. However,
// if we see a code block in the content, we assume that this page needs
// to be highlighted. It could potentially have false positive if the content
// has ``` in it but that seems kind of unlikely
let should_highlight = if context.highlight_code {
content.contains("```")
} else {
false
};
pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<(String, Vec<Header>)> {
// the rendered html
let mut html = String::new();
// Set while parsing // Set while parsing
let mut error = None; let mut error = None;

let mut highlighter: Option<HighlightLines> = None; let mut highlighter: Option<HighlightLines> = None;
// the markdown parser will send several Text event if a markdown character
// is present in it, for example `hello_test` will be split in 2: hello and _test.
// Since we can use those chars in shortcode arguments, we need to collect
// the full shortcode somehow first
let mut current_shortcode = String::new();
let mut shortcode_block = None;
// shortcodes live outside of paragraph so we need to ensure we don't close
// a paragraph that has already been closed
let mut added_shortcode = false;
// Don't transform things that look like shortcodes in code blocks
let mut in_code_block = false;
// If we get text in header, we need to insert the id and a anchor // If we get text in header, we need to insert the id and a anchor
let mut in_header = false; let mut in_header = false;
// pulldown_cmark can send several text events for a title if there are markdown // pulldown_cmark can send several text events for a title if there are markdown
@@ -46,254 +45,128 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
let mut header_created = false; let mut header_created = false;
let mut anchors: Vec<String> = vec![]; let mut anchors: Vec<String> = vec![];


// the rendered html
let mut html = String::new();

// We might have cases where the slug is already present in our list of anchor
// for example an article could have several titles named Example
// We add a counter after the slug if the slug is already present, which
// means we will have example, example-1, example-2 etc
fn find_anchor(anchors: &[String], name: String, level: u8) -> String {
if level == 0 && !anchors.contains(&name) {
return name.to_string();
}

let new_anchor = format!("{}-{}", name, level + 1);
if !anchors.contains(&new_anchor) {
return new_anchor;
}

find_anchor(anchors, name, level + 1)
}

let mut headers = vec![]; let mut headers = vec![];
// Defaults to a 0 level so not a real header // Defaults to a 0 level so not a real header
// It should be an Option ideally but not worth the hassle to update // It should be an Option ideally but not worth the hassle to update
let mut temp_header = TempHeader::default(); let mut temp_header = TempHeader::default();
let mut clear_shortcode_block = false;


let mut opts = Options::empty(); let mut opts = Options::empty();
opts.insert(OPTION_ENABLE_TABLES); opts.insert(OPTION_ENABLE_TABLES);
opts.insert(OPTION_ENABLE_FOOTNOTES); opts.insert(OPTION_ENABLE_FOOTNOTES);


{ {

let parser = Parser::new_ext(content, opts).map(|event| { let parser = Parser::new_ext(content, opts).map(|event| {
if clear_shortcode_block {
clear_shortcode_block = false;
shortcode_block = None;
}

match event { match event {
Event::Text(mut text) => {
// Header first
if in_header {
if header_created {
temp_header.push(&text);
Event::Text(text) => {
// Header first
if in_header {
if header_created {
temp_header.push(&text);
return Event::Html(Owned(String::new()));
}
let id = find_anchor(&anchors, slugify(&text), 0);
anchors.push(id.clone());
// update the header and add it to the list
temp_header.id = id.clone();
// += as we might have some <code> or other things already there
temp_header.title += &text;
temp_header.permalink = format!("{}#{}", context.current_page_permalink, id);
header_created = true;
return Event::Html(Owned(String::new())); return Event::Html(Owned(String::new()));
} }
let id = find_anchor(&anchors, slugify(&text), 0);
anchors.push(id.clone());
// update the header and add it to the list
temp_header.id = id.clone();
// += as we might have some <code> or other things already there
temp_header.title += &text;
temp_header.permalink = format!("{}#{}", context.current_page_permalink, id);
header_created = true;
return Event::Html(Owned(String::new()));
}


// if we are in the middle of a code block
if let Some(ref mut highlighter) = highlighter {
let highlighted = &highlighter.highlight(&text);
let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes);
return Event::Html(Owned(html));
}

if in_code_block {
return Event::Text(text);
}

// Are we in the middle of a shortcode that somehow got cut off
// by the markdown parser?
if current_shortcode.is_empty() {
if text.starts_with("{{") && !text.ends_with("}}") {
current_shortcode += &text;
} else if text.starts_with("{%") && !text.ends_with("%}") {
current_shortcode += &text;
// if we are in the middle of a code block
if let Some(ref mut highlighter) = highlighter {
let highlighted = &highlighter.highlight(&text);
let html = styles_to_coloured_html(highlighted, IncludeBackground::Yes);
return Event::Html(Owned(html));
} }
} else {
current_shortcode += &text;
}

if current_shortcode.ends_with("}}") || current_shortcode.ends_with("%}") {
text = Owned(current_shortcode.clone());
current_shortcode = String::new();
}


// Shortcode without body
if shortcode_block.is_none() && text.starts_with("{{") && text.ends_with("}}") && SHORTCODE_RE.is_match(&text) {
let (name, args) = parse_shortcode(&text);

added_shortcode = true;
match render_simple_shortcode(context.tera, &name, &args) {
// Make before and after cleaning up of extra <p> / </p> tags more parallel.
// Or, in other words:
// TERRIBLE HORRIBLE NO GOOD VERY BAD HACK
Ok(s) => return Event::Html(Owned(format!("</p>{}<p>", s))),
Err(e) => {
error = Some(e);
return Event::Html(Owned(String::new()));
}
// Business as usual
Event::Text(text)
},
Event::Start(Tag::CodeBlock(ref info)) => {
if !context.config.highlight_code {
return Event::Html(Owned("<pre><code>".to_owned()));
} }
}

// Shortcode with a body
if shortcode_block.is_none() && text.starts_with("{%") && text.ends_with("%}") {
if SHORTCODE_RE.is_match(&text) {
let (name, args) = parse_shortcode(&text);
shortcode_block = Some(ShortCode::new(&name, args));
let theme = &THEME_SET.themes[&context.config.highlight_theme];
highlighter = Some(get_highlighter(&theme, info));
let snippet = start_coloured_html_snippet(theme);
Event::Html(Owned(snippet))
},
Event::End(Tag::CodeBlock(_)) => {
if !context.config.highlight_code {
return Event::Html(Owned("</code></pre>\n".to_owned()))
} }
// Don't return anything
return Event::Text(Owned(String::new()));
}
// If we have some text while in a shortcode, it's either the body
// or the end tag
if shortcode_block.is_some() {
if let Some(ref mut shortcode) = shortcode_block {
if text.trim() == "{% end %}" {
added_shortcode = true;
clear_shortcode_block = true;
match shortcode.render(context.tera) {
Ok(s) => return Event::Html(Owned(format!("</p>{}", s))),
Err(e) => {
error = Some(e);
return Event::Html(Owned(String::new()));
}
// reset highlight and close the code block
highlighter = None;
Event::Html(Owned("</pre>".to_owned()))
},
// Need to handle relative links
Event::Start(Tag::Link(ref link, ref title)) => {
if in_header {
return Event::Html(Owned("".to_owned()));
}
if link.starts_with("./") {
match resolve_internal_link(link, context.permalinks) {
Ok(url) => {
return Event::Start(Tag::Link(Owned(url), title.clone()));
},
Err(_) => {
error = Some(format!("Relative link {} not found.", link).into());
return Event::Html(Owned("".to_string()));
} }
} else {
shortcode.append(&text);
return Event::Html(Owned(String::new()));
}
};
} }
}

// Business as usual
Event::Text(text)
},
Event::Start(Tag::CodeBlock(ref info)) => {
in_code_block = true;
if !should_highlight {
return Event::Html(Owned("<pre><code>".to_owned()));
}
let theme = &THEME_SET.themes[&context.highlight_theme];
highlighter = SYNTAX_SET.with(|ss| {
let syntax = info
.split(' ')
.next()
.and_then(|lang| ss.find_syntax_by_token(lang))
.unwrap_or_else(|| ss.find_syntax_plain_text());
Some(HighlightLines::new(syntax, theme))
});
let snippet = start_coloured_html_snippet(theme);
Event::Html(Owned(snippet))
},
Event::End(Tag::CodeBlock(_)) => {
in_code_block = false;
if !should_highlight{
return Event::Html(Owned("</code></pre>\n".to_owned()))
}
// reset highlight and close the code block
highlighter = None;
Event::Html(Owned("</pre>".to_owned()))
},
// Need to handle relative links
Event::Start(Tag::Link(ref link, ref title)) => {
if in_header {
return Event::Html(Owned("".to_owned()));
}
if link.starts_with("./") {
match resolve_internal_link(link, context.permalinks) {
Ok(url) => {
return Event::Start(Tag::Link(Owned(url), title.clone()));
},
Err(_) => {
error = Some(format!("Relative link {} not found.", link).into());
return Event::Html(Owned("".to_string()));
}
};
}


Event::Start(Tag::Link(link.clone(), title.clone()))
},
Event::End(Tag::Link(_, _)) => {
if in_header {
return Event::Html(Owned("".to_owned()));
Event::Start(Tag::Link(link.clone(), title.clone()))
},
Event::End(Tag::Link(_, _)) => {
if in_header {
return Event::Html(Owned("".to_owned()));
}
event
} }
event
Event::Start(Tag::Code) => {
if in_header {
temp_header.push("<code>");
return Event::Html(Owned(String::new()));
}
event
},
Event::End(Tag::Code) => {
if in_header {
temp_header.push("</code>");
return Event::Html(Owned(String::new()));
}
event
},
Event::Start(Tag::Header(num)) => {
in_header = true;
temp_header = TempHeader::new(num);
Event::Html(Owned(String::new()))
},
Event::End(Tag::Header(_)) => {
// End of a header, reset all the things and return the stringified version of the header
in_header = false;
header_created = false;
let val = temp_header.to_string(context.tera, context.insert_anchor);
headers.push(temp_header.clone());
temp_header = TempHeader::default();
Event::Html(Owned(val))
},
_ => event,
} }
// need to know when we are in a code block to disable shortcodes in them
Event::Start(Tag::Code) => {
in_code_block = true;
if in_header {
temp_header.push("<code>");
return Event::Html(Owned(String::new()));
}
event
},
Event::End(Tag::Code) => {
in_code_block = false;
if in_header {
temp_header.push("</code>");
return Event::Html(Owned(String::new()));
}
event
},
Event::Start(Tag::Header(num)) => {
in_header = true;
temp_header = TempHeader::new(num);
Event::Html(Owned(String::new()))
},
Event::End(Tag::Header(_)) => {
// End of a header, reset all the things and return the stringified version of the header
in_header = false;
header_created = false;
let val = temp_header.to_string(context);
headers.push(temp_header.clone());
temp_header = TempHeader::default();
Event::Html(Owned(val))
},
// If we added shortcodes, don't close a paragraph since there's none
Event::End(Tag::Paragraph) => {
if added_shortcode {
added_shortcode = false;
return Event::Html(Owned("".to_owned()));
}
event
},
// Ignore softbreaks inside shortcodes
Event::SoftBreak => {
if shortcode_block.is_some() {
return Event::Html(Owned("".to_owned()));
}
event
},
_ => {
// println!("event = {:?}", event);
event
},
}});
});


cmark::html::push_html(&mut html, parser); cmark::html::push_html(&mut html, parser);
} }


if !current_shortcode.is_empty() {
return Err(format!("A shortcode was not closed properly:\n{:?}", current_shortcode).into());
}

match error { match error {
Some(e) => Err(e), Some(e) => Err(e),
None => Ok((html.replace("<p></p>", "").replace("</p></p>", "</p>"), make_table_of_contents(&headers))),
None => Ok((
html.replace("<p></p>", "").replace("</p></p>", "</p>"),
make_table_of_contents(&headers)
)),
} }
} }

+ 0
- 190
components/rendering/src/short_code.rs View File

@@ -1,190 +0,0 @@
use std::collections::HashMap;

use regex::Regex;
use tera::{Tera, Context, Value, to_value};

use errors::{Result, ResultExt};

lazy_static!{
// Does this look like a shortcode?
pub static ref SHORTCODE_RE: Regex = Regex::new(
r#"\{(?:%|\{)\s+(\w+?)\((\w+?="?(?:.|\n)+?"?)?\)\s+(?:%|\})\}"#
).unwrap();

// Parse the shortcode args with capture groups named after their type
pub static ref SHORTCODE_ARGS_RE: Regex = Regex::new(
r#"(?P<name>\w+)=\s*((?P<str>".*?")|(?P<float>[-+]?[0-9]+\.[0-9]+)|(?P<int>[-+]?[0-9]+)|(?P<bool>true|false))"#
).unwrap();
}

/// A shortcode that has a body
/// Called by having some content like {% ... %} body {% end %}
/// We need the struct to hold the data while we're processing the markdown
#[derive(Debug)]
pub struct ShortCode {
name: String,
args: HashMap<String, Value>,
body: String,
}

impl ShortCode {
pub fn new(name: &str, args: HashMap<String, Value>) -> ShortCode {
ShortCode {
name: name.to_string(),
args,
body: String::new(),
}
}

pub fn append(&mut self, text: &str) {
self.body.push_str(text)
}

pub fn render(&self, tera: &Tera) -> Result<String> {
let mut context = Context::new();
for (key, value) in &self.args {
context.add(key, value);
}
context.add("body", &self.body);
let tpl_name = format!("shortcodes/{}.html", self.name);
tera.render(&tpl_name, &context)
.chain_err(|| format!("Failed to render {} shortcode", self.name))
}
}

/// Parse a shortcode without a body
pub fn parse_shortcode(input: &str) -> (String, HashMap<String, Value>) {
let mut args = HashMap::new();
let caps = SHORTCODE_RE.captures(input).unwrap();
// caps[0] is the full match
let name = &caps[1];

if let Some(arg_list) = caps.get(2) {
for arg_cap in SHORTCODE_ARGS_RE.captures_iter(arg_list.as_str()) {
let arg_name = arg_cap["name"].trim().to_string();

if let Some(arg_val) = arg_cap.name("str") {
args.insert(arg_name, to_value(arg_val.as_str().replace("\"", "")).unwrap());
continue;
}

if let Some(arg_val) = arg_cap.name("int") {
args.insert(arg_name, to_value(arg_val.as_str().parse::<i64>().unwrap()).unwrap());
continue;
}

if let Some(arg_val) = arg_cap.name("float") {
args.insert(arg_name, to_value(arg_val.as_str().parse::<f64>().unwrap()).unwrap());
continue;
}

if let Some(arg_val) = arg_cap.name("bool") {
args.insert(arg_name, to_value(arg_val.as_str() == "true").unwrap());
continue;
}
}
}

(name.to_string(), args)
}

/// Renders a shortcode or return an error
pub fn render_simple_shortcode(tera: &Tera, name: &str, args: &HashMap<String, Value>) -> Result<String> {
let mut context = Context::new();
for (key, value) in args.iter() {
context.add(key, value);
}
let tpl_name = format!("shortcodes/{}.html", name);

tera.render(&tpl_name, &context).chain_err(|| format!("Failed to render {} shortcode", name))
}


#[cfg(test)]
mod tests {
use super::{parse_shortcode, SHORTCODE_RE};

#[test]
fn can_match_all_kinds_of_shortcode() {
let inputs = vec![
"{{ basic() }}",
"{{ basic(ho=1) }}",
"{{ basic(ho=\"hey\") }}",
"{{ basic(ho=\"hey_underscore\") }}",
"{{ basic(ho=\"hey-dash\") }}",
"{% basic(ho=\"hey-dash\") %}",
"{% basic(ho=\"hey_underscore\") %}",
"{% basic() %}",
"{% quo_te(author=\"Bob\") %}",
"{{ quo_te(author=\"Bob\") }}",
// https://github.com/Keats/gutenberg/issues/229
r#"{{ youtube(id="dQw4w9WgXcQ",

autoplay=true) }}"#,
];

for i in inputs {
println!("{}", i);
assert!(SHORTCODE_RE.is_match(i));
}
}

// https://github.com/Keats/gutenberg/issues/228
#[test]
fn doesnt_panic_on_invalid_shortcode() {
let (name, args) = parse_shortcode(r#"{{ youtube(id="dQw4w9WgXcQ", autoplay) }}"#);
assert_eq!(name, "youtube");
assert_eq!(args["id"], "dQw4w9WgXcQ");
assert!(args.get("autoplay").is_none());
}

#[test]
fn can_parse_simple_shortcode_no_arg() {
let (name, args) = parse_shortcode(r#"{{ basic() }}"#);
assert_eq!(name, "basic");
assert!(args.is_empty());
}

#[test]
fn can_parse_simple_shortcode_one_arg() {
let (name, args) = parse_shortcode(r#"{{ youtube(id="w7Ft2ymGmfc") }}"#);
assert_eq!(name, "youtube");
assert_eq!(args["id"], "w7Ft2ymGmfc");
}

#[test]
fn can_parse_simple_shortcode_several_arg() {
let (name, args) = parse_shortcode(r#"{{ youtube(id="w7Ft2ymGmfc", autoplay=true) }}"#);
assert_eq!(name, "youtube");
assert_eq!(args["id"], "w7Ft2ymGmfc");
assert_eq!(args["autoplay"], true);
}

#[test]
fn can_parse_block_shortcode_several_arg() {
let (name, args) = parse_shortcode(r#"{% youtube(id="w7Ft2ymGmfc", autoplay=true) %}"#);
assert_eq!(name, "youtube");
assert_eq!(args["id"], "w7Ft2ymGmfc");
assert_eq!(args["autoplay"], true);
}

#[test]
fn can_parse_shortcode_number() {
let (name, args) = parse_shortcode(r#"{% test(int=42, float=42.0, autoplay=false) %}"#);
assert_eq!(name, "test");
assert_eq!(args["int"], 42);
assert_eq!(args["float"], 42.0);
assert_eq!(args["autoplay"], false);
}

// https://github.com/Keats/gutenberg/issues/249
#[test]
fn can_parse_shortcode_with_comma_in_it() {
let (name, args) = parse_shortcode(
r#"{% quote(author="C++ Standard Core Language Defect Reports and Accepted Issues, Revision 82, delete and user-written deallocation function", href="http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#348") %}"#
);
assert_eq!(name, "quote");
assert_eq!(args["author"], "C++ Standard Core Language Defect Reports and Accepted Issues, Revision 82, delete and user-written deallocation function");
assert_eq!(args["href"], "http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#348");
}
}

+ 23
- 7
components/rendering/src/shortcode.rs View File

@@ -90,7 +90,8 @@ fn render_shortcode(name: String, args: Map<String, Value>, tera: &Tera, config:
context.insert(key, value); context.insert(key, value);
} }
if let Some(ref b) = body { if let Some(ref b) = body {
context.insert("body", b);
// Trimming right to avoid most shortcodes with bodies ending up with a HTML new line
context.insert("body", b.trim_right());
} }
context.insert("config", config); context.insert("config", config);
let tpl_name = format!("shortcodes/{}.html", name); let tpl_name = format!("shortcodes/{}.html", name);
@@ -99,16 +100,25 @@ fn render_shortcode(name: String, args: Map<String, Value>, tera: &Tera, config:
} }


pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result<String> { pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result<String> {
// Don't do anything if there is nothing like a shortcode in the content
if !content.contains("{{") && !content.contains("{%") {
return Ok(content.to_string());
}

let mut res = String::with_capacity(content.len()); let mut res = String::with_capacity(content.len());


let mut pairs = match ContentParser::parse(Rule::page, content) { let mut pairs = match ContentParser::parse(Rule::page, content) {
Ok(p) => p, Ok(p) => p,
Err(_) => panic!("TODO"), // TODO: error handling
Err(e) => {
let fancy_e = e.renamed_rules(|rule| {
match *rule {
Rule::int => "an integer".to_string(),
Rule::float => "a float".to_string(),
Rule::string => "a string".to_string(),
Rule::literal => "a literal (int, float, string, bool)".to_string(),
Rule::array => "an array".to_string(),
Rule::kwarg => "a keyword argument".to_string(),
Rule::ident => "an identifier".to_string(),
_ => format!("TODO error: {:?}", rule).to_string(),
}
});
bail!("{}", fancy_e);
},
}; };


// We have at least a `page` pair // We have at least a `page` pair
@@ -333,4 +343,10 @@ Hello World
let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap(); let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap();
assert_eq!(res, "Body\n Hey!"); assert_eq!(res, "Body\n Hey!");
} }

#[test]
fn errors_on_unterminated_shortcode() {
let res = render_shortcodes("{{ youtube(", &Tera::default(), &Config::default());
assert!(res.is_err());
}
} }

+ 5
- 7
components/rendering/src/table_of_contents.rs View File

@@ -1,8 +1,6 @@
use tera::{Context as TeraContext};
use tera::{Tera, Context as TeraContext};
use front_matter::InsertAnchor; use front_matter::InsertAnchor;


use context::Context;



#[derive(Debug, PartialEq, Clone, Serialize)] #[derive(Debug, PartialEq, Clone, Serialize)]
pub struct Header { pub struct Header {
@@ -50,16 +48,16 @@ impl TempHeader {
} }


/// Transform all the information we have about this header into the HTML string for it /// Transform all the information we have about this header into the HTML string for it
pub fn to_string(&self, context: &Context) -> String {
let anchor_link = if context.should_insert_anchor() {
pub fn to_string(&self, tera: &Tera, insert_anchor: InsertAnchor) -> String {
let anchor_link = if insert_anchor != InsertAnchor::None {
let mut c = TeraContext::new(); let mut c = TeraContext::new();
c.add("id", &self.id); c.add("id", &self.id);
context.tera.render("anchor-link.html", &c).unwrap()
tera.render("anchor-link.html", &c).unwrap()
} else { } else {
String::new() String::new()
}; };


match context.insert_anchor {
match insert_anchor {
InsertAnchor::None => format!("<h{lvl} id=\"{id}\">{t}</h{lvl}>\n", lvl=self.level, t=self.title, id=self.id), InsertAnchor::None => format!("<h{lvl} id=\"{id}\">{t}</h{lvl}>\n", lvl=self.level, t=self.title, id=self.id),
InsertAnchor::Left => format!("<h{lvl} id=\"{id}\">{a}{t}</h{lvl}>\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id), InsertAnchor::Left => format!("<h{lvl} id=\"{id}\">{a}{t}</h{lvl}>\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id),
InsertAnchor::Right => format!("<h{lvl} id=\"{id}\">{t}{a}</h{lvl}>\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id), InsertAnchor::Right => format!("<h{lvl} id=\"{id}\">{t}{a}</h{lvl}>\n", lvl=self.level, a=anchor_link, t=self.title, id=self.id),


+ 93
- 71
components/rendering/tests/markdown.rs View File

@@ -2,22 +2,25 @@ extern crate tera;
extern crate front_matter; extern crate front_matter;
extern crate templates; extern crate templates;
extern crate rendering; extern crate rendering;
extern crate config;


use std::collections::HashMap; use std::collections::HashMap;


use tera::Tera; use tera::Tera;


use config::Config;
use front_matter::InsertAnchor; use front_matter::InsertAnchor;
use templates::GUTENBERG_TERA; use templates::GUTENBERG_TERA;
use rendering::{Context, markdown_to_html};
use rendering::{RenderContext, render_content};




#[test] #[test]
fn can_do_markdown_to_html_simple() {
fn can_do_render_content_simple() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("hello", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("hello", &context).unwrap();
assert_eq!(res.0, "<p>hello</p>\n"); assert_eq!(res.0, "<p>hello</p>\n");
} }


@@ -25,9 +28,10 @@ fn can_do_markdown_to_html_simple() {
fn doesnt_highlight_code_block_with_highlighting_off() { fn doesnt_highlight_code_block_with_highlighting_off() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let mut context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
context.highlight_code = false;
let res = markdown_to_html("```\n$ gutenberg server\n```", &context).unwrap();
let mut config = Config::default();
config.highlight_code = false;
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("```\n$ gutenberg server\n```", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<pre><code>$ gutenberg server\n</code></pre>\n" "<pre><code>$ gutenberg server\n</code></pre>\n"
@@ -38,8 +42,9 @@ fn doesnt_highlight_code_block_with_highlighting_off() {
fn can_highlight_code_block_no_lang() { fn can_highlight_code_block_no_lang() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("```\n$ gutenberg server\n$ ping\n```", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("```\n$ gutenberg server\n$ ping\n```", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<pre style=\"background-color:#2b303b\">\n<span style=\"background-color:#2b303b;color:#c0c5ce;\">$ gutenberg server\n</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">$ ping\n</span></pre>" "<pre style=\"background-color:#2b303b\">\n<span style=\"background-color:#2b303b;color:#c0c5ce;\">$ gutenberg server\n</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">$ ping\n</span></pre>"
@@ -50,8 +55,9 @@ fn can_highlight_code_block_no_lang() {
fn can_highlight_code_block_with_lang() { fn can_highlight_code_block_with_lang() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("```python\nlist.append(1)\n```", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("```python\nlist.append(1)\n```", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<pre style=\"background-color:#2b303b\">\n<span style=\"background-color:#2b303b;color:#c0c5ce;\">list.</span><span style=\"background-color:#2b303b;color:#bf616a;\">append</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">(</span><span style=\"background-color:#2b303b;color:#d08770;\">1</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">)\n</span></pre>" "<pre style=\"background-color:#2b303b\">\n<span style=\"background-color:#2b303b;color:#c0c5ce;\">list.</span><span style=\"background-color:#2b303b;color:#bf616a;\">append</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">(</span><span style=\"background-color:#2b303b;color:#d08770;\">1</span><span style=\"background-color:#2b303b;color:#c0c5ce;\">)\n</span></pre>"
@@ -62,8 +68,9 @@ fn can_highlight_code_block_with_lang() {
fn can_higlight_code_block_with_unknown_lang() { fn can_higlight_code_block_with_unknown_lang() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("```yolo\nlist.append(1)\n```", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("```yolo\nlist.append(1)\n```", &context).unwrap();
// defaults to plain text // defaults to plain text
assert_eq!( assert_eq!(
res.0, res.0,
@@ -74,8 +81,9 @@ fn can_higlight_code_block_with_unknown_lang() {
#[test] #[test]
fn can_render_shortcode() { fn can_render_shortcode() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html(r#"
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content(r#"
Hello Hello


{{ youtube(id="ub36ffWAqgQ") }} {{ youtube(id="ub36ffWAqgQ") }}
@@ -87,7 +95,8 @@ Hello
#[test] #[test]
fn can_render_shortcode_with_markdown_char_in_args_name() { fn can_render_shortcode_with_markdown_char_in_args_name() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let input = vec![ let input = vec![
"name", "name",
"na_me", "na_me",
@@ -95,7 +104,7 @@ fn can_render_shortcode_with_markdown_char_in_args_name() {
"n1", "n1",
]; ];
for i in input { for i in input {
let res = markdown_to_html(&format!("{{{{ youtube(id=\"hey\", {}=1) }}}}", i), &context).unwrap();
let res = render_content(&format!("{{{{ youtube(id=\"hey\", {}=1) }}}}", i), &context).unwrap();
assert!(res.0.contains(r#"<iframe src="https://www.youtube.com/embed/hey""#)); assert!(res.0.contains(r#"<iframe src="https://www.youtube.com/embed/hey""#));
} }
} }
@@ -103,7 +112,8 @@ fn can_render_shortcode_with_markdown_char_in_args_name() {
#[test] #[test]
fn can_render_shortcode_with_markdown_char_in_args_value() { fn can_render_shortcode_with_markdown_char_in_args_value() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let input = vec![ let input = vec![
"ub36ffWAqgQ-hey", "ub36ffWAqgQ-hey",
"ub36ffWAqgQ_hey", "ub36ffWAqgQ_hey",
@@ -112,7 +122,7 @@ fn can_render_shortcode_with_markdown_char_in_args_value() {
"ub36ffWAqgQ#hey", "ub36ffWAqgQ#hey",
]; ];
for i in input { for i in input {
let res = markdown_to_html(&format!("{{{{ youtube(id=\"{}\") }}}}", i), &context).unwrap();
let res = render_content(&format!("{{{{ youtube(id=\"{}\") }}}}", i), &context).unwrap();
assert!(res.0.contains(&format!(r#"<iframe src="https://www.youtube.com/embed/{}""#, i))); assert!(res.0.contains(&format!(r#"<iframe src="https://www.youtube.com/embed/{}""#, i)));
} }
} }
@@ -126,12 +136,13 @@ fn can_render_body_shortcode_with_markdown_char_in_name() {
"quo_te", "quo_te",
"qu_o_te", "qu_o_te",
]; ];
let config = Config::default();


for i in input { for i in input {
tera.add_raw_template(&format!("shortcodes/{}.html", i), "<blockquote>{{ body }} - {{ author}}</blockquote>").unwrap(); tera.add_raw_template(&format!("shortcodes/{}.html", i), "<blockquote>{{ body }} - {{ author}}</blockquote>").unwrap();
let context = Context::new(&tera, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);


let res = markdown_to_html(&format!("{{% {}(author=\"Bob\") %}}\nhey\n{{% end %}}", i), &context).unwrap();
let res = render_content(&format!("{{% {}(author=\"Bob\") %}}\nhey\n{{% end %}}", i), &context).unwrap();
println!("{:?}", res); println!("{:?}", res);
assert!(res.0.contains("<blockquote>hey - Bob</blockquote>")); assert!(res.0.contains("<blockquote>hey - Bob</blockquote>"));
} }
@@ -157,9 +168,10 @@ Here is another paragraph.
"; ";


tera.add_raw_template(&format!("shortcodes/{}.html", "figure"), shortcode).unwrap(); tera.add_raw_template(&format!("shortcodes/{}.html", "figure"), shortcode).unwrap();
let context = Context::new(&tera, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let config = Config::default();
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);


let res = markdown_to_html(markdown_string, &context).unwrap();
let res = render_content(markdown_string, &context).unwrap();
println!("{:?}", res); println!("{:?}", res);
assert_eq!(res.0, expected); assert_eq!(res.0, expected);
} }
@@ -189,9 +201,10 @@ Here is another paragraph.
"; ";


tera.add_raw_template(&format!("shortcodes/{}.html", "figure"), shortcode).unwrap(); tera.add_raw_template(&format!("shortcodes/{}.html", "figure"), shortcode).unwrap();
let context = Context::new(&tera, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let config = Config::default();
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);


let res = markdown_to_html(markdown_string, &context).unwrap();
let res = render_content(markdown_string, &context).unwrap();
println!("{:?}", res); println!("{:?}", res);
assert_eq!(res.0, expected); assert_eq!(res.0, expected);
} }
@@ -199,8 +212,9 @@ Here is another paragraph.
#[test] #[test]
fn can_render_several_shortcode_in_row() { fn can_render_several_shortcode_in_row() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html(r#"
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content(r#"
Hello Hello


{{ youtube(id="ub36ffWAqgQ") }} {{ youtube(id="ub36ffWAqgQ") }}
@@ -222,18 +236,12 @@ Hello
} }


#[test] #[test]
fn errors_if_unterminated_shortcode() {
fn doesnt_render_ignored_shortcodes() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html(r#"{{ youtube(id="w7Ft2ym_a"#, &context);
assert!(res.is_err());
}

#[test]
fn doesnt_render_shortcode_in_code_block() {
let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html(r#"```{{ youtube(id="w7Ft2ymGmfc") }}```"#, &context).unwrap();
let mut config = Config::default();
config.highlight_code = false;
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content(r#"```{{/* youtube(id="w7Ft2ymGmfc") */}}```"#, &context).unwrap();
assert_eq!(res.0, "<p><code>{{ youtube(id=&quot;w7Ft2ymGmfc&quot;) }}</code></p>\n"); assert_eq!(res.0, "<p><code>{{ youtube(id=&quot;w7Ft2ymGmfc&quot;) }}</code></p>\n");
} }


@@ -243,23 +251,25 @@ fn can_render_shortcode_with_body() {
tera.extend(&GUTENBERG_TERA).unwrap(); tera.extend(&GUTENBERG_TERA).unwrap();
tera.add_raw_template("shortcodes/quote.html", "<blockquote>{{ body }} - {{ author }}</blockquote>").unwrap(); tera.add_raw_template("shortcodes/quote.html", "<blockquote>{{ body }} - {{ author }}</blockquote>").unwrap();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let config = Config::default();
let context = RenderContext::new(&tera, &config, "", &permalinks_ctx, InsertAnchor::None);


let res = markdown_to_html(r#"
let res = render_content(r#"
Hello Hello
{% quote(author="Keats") %} {% quote(author="Keats") %}
A quote A quote
{% end %} {% end %}
"#, &context).unwrap(); "#, &context).unwrap();
assert_eq!(res.0, "<p>Hello\n</p><blockquote>A quote - Keats</blockquote>");
assert_eq!(res.0, "<p>Hello</p>\n<blockquote>A quote - Keats</blockquote>\n");
} }


#[test] #[test]
fn errors_rendering_unknown_shortcode() { fn errors_rendering_unknown_shortcode() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("{{ hello(flash=true) }}", &context);
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("{{ hello(flash=true) }}", &context);
assert!(res.is_err()); assert!(res.is_err());
} }


@@ -268,8 +278,9 @@ fn can_make_valid_relative_link() {
let mut permalinks = HashMap::new(); let mut permalinks = HashMap::new();
permalinks.insert("pages/about.md".to_string(), "https://vincent.is/about".to_string()); permalinks.insert("pages/about.md".to_string(), "https://vincent.is/about".to_string());
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks, InsertAnchor::None);
let res = markdown_to_html(
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks, InsertAnchor::None);
let res = render_content(
r#"[rel link](./pages/about.md), [abs link](https://vincent.is/about)"#, r#"[rel link](./pages/about.md), [abs link](https://vincent.is/about)"#,
&context &context
).unwrap(); ).unwrap();
@@ -284,8 +295,9 @@ fn can_make_relative_links_with_anchors() {
let mut permalinks = HashMap::new(); let mut permalinks = HashMap::new();
permalinks.insert("pages/about.md".to_string(), "https://vincent.is/about".to_string()); permalinks.insert("pages/about.md".to_string(), "https://vincent.is/about".to_string());
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks, InsertAnchor::None);
let res = markdown_to_html(r#"[rel link](./pages/about.md#cv)"#, &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks, InsertAnchor::None);
let res = render_content(r#"[rel link](./pages/about.md#cv)"#, &context).unwrap();


assert!( assert!(
res.0.contains(r#"<p><a href="https://vincent.is/about#cv">rel link</a></p>"#) res.0.contains(r#"<p><a href="https://vincent.is/about#cv">rel link</a></p>"#)
@@ -296,8 +308,9 @@ fn can_make_relative_links_with_anchors() {
fn errors_relative_link_inexistant() { fn errors_relative_link_inexistant() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("[rel link](./pages/about.md)", &context);
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("[rel link](./pages/about.md)", &context);
assert!(res.is_err()); assert!(res.is_err());
} }


@@ -305,8 +318,9 @@ fn errors_relative_link_inexistant() {
fn can_add_id_to_headers() { fn can_add_id_to_headers() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html(r#"# Hello"#, &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content(r#"# Hello"#, &context).unwrap();
assert_eq!(res.0, "<h1 id=\"hello\">Hello</h1>\n"); assert_eq!(res.0, "<h1 id=\"hello\">Hello</h1>\n");
} }


@@ -314,16 +328,18 @@ fn can_add_id_to_headers() {
fn can_add_id_to_headers_same_slug() { fn can_add_id_to_headers_same_slug() {
let tera_ctx = Tera::default(); let tera_ctx = Tera::default();
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&tera_ctx, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("# Hello\n# Hello", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&tera_ctx, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("# Hello\n# Hello", &context).unwrap();
assert_eq!(res.0, "<h1 id=\"hello\">Hello</h1>\n<h1 id=\"hello-1\">Hello</h1>\n"); assert_eq!(res.0, "<h1 id=\"hello\">Hello</h1>\n<h1 id=\"hello-1\">Hello</h1>\n");
} }


#[test] #[test]
fn can_insert_anchor_left() { fn can_insert_anchor_left() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::Left);
let res = markdown_to_html("# Hello", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::Left);
let res = render_content("# Hello", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello</h1>\n" "<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello</h1>\n"
@@ -333,8 +349,9 @@ fn can_insert_anchor_left() {
#[test] #[test]
fn can_insert_anchor_right() { fn can_insert_anchor_right() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::Right);
let res = markdown_to_html("# Hello", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::Right);
let res = render_content("# Hello", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h1 id=\"hello\">Hello<a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\n</h1>\n" "<h1 id=\"hello\">Hello<a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\n</h1>\n"
@@ -345,8 +362,9 @@ fn can_insert_anchor_right() {
#[test] #[test]
fn can_insert_anchor_with_exclamation_mark() { fn can_insert_anchor_with_exclamation_mark() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::Left);
let res = markdown_to_html("# Hello!", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::Left);
let res = render_content("# Hello!", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello!</h1>\n" "<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello!</h1>\n"
@@ -357,8 +375,9 @@ fn can_insert_anchor_with_exclamation_mark() {
#[test] #[test]
fn can_insert_anchor_with_link() { fn can_insert_anchor_with_link() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::Left);
let res = markdown_to_html("## [](#xresources)Xresources", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::Left);
let res = render_content("## [](#xresources)Xresources", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h2 id=\"xresources\"><a class=\"gutenberg-anchor\" href=\"#xresources\" aria-label=\"Anchor link for: xresources\">๐Ÿ”—</a>\nXresources</h2>\n" "<h2 id=\"xresources\"><a class=\"gutenberg-anchor\" href=\"#xresources\" aria-label=\"Anchor link for: xresources\">๐Ÿ”—</a>\nXresources</h2>\n"
@@ -368,8 +387,9 @@ fn can_insert_anchor_with_link() {
#[test] #[test]
fn can_insert_anchor_with_other_special_chars() { fn can_insert_anchor_with_other_special_chars() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::Left);
let res = markdown_to_html("# Hello*_()", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::Left);
let res = render_content("# Hello*_()", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello*_()</h1>\n" "<h1 id=\"hello\"><a class=\"gutenberg-anchor\" href=\"#hello\" aria-label=\"Anchor link for: hello\">๐Ÿ”—</a>\nHello*_()</h1>\n"
@@ -379,16 +399,16 @@ fn can_insert_anchor_with_other_special_chars() {
#[test] #[test]
fn can_make_toc() { fn can_make_toc() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(
let config = Config::default();
let context = RenderContext::new(
&GUTENBERG_TERA, &GUTENBERG_TERA,
true,
"base16-ocean-dark".to_string(),
&config,
"https://mysite.com/something", "https://mysite.com/something",
&permalinks_ctx, &permalinks_ctx,
InsertAnchor::Left InsertAnchor::Left
); );


let res = markdown_to_html(r#"
let res = render_content(r#"
# Header 1 # Header 1


## Header 2 ## Header 2
@@ -408,8 +428,9 @@ fn can_make_toc() {
#[test] #[test]
fn can_understand_backtick_in_titles() { fn can_understand_backtick_in_titles() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("# `Hello`", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("# `Hello`", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<h1 id=\"hello\"><code>Hello</code></h1>\n" "<h1 id=\"hello\"><code>Hello</code></h1>\n"
@@ -419,8 +440,9 @@ fn can_understand_backtick_in_titles() {
#[test] #[test]
fn can_understand_backtick_in_paragraphs() { fn can_understand_backtick_in_paragraphs() {
let permalinks_ctx = HashMap::new(); let permalinks_ctx = HashMap::new();
let context = Context::new(&GUTENBERG_TERA, true, "base16-ocean-dark".to_string(), "", &permalinks_ctx, InsertAnchor::None);
let res = markdown_to_html("Hello `world`", &context).unwrap();
let config = Config::default();
let context = RenderContext::new(&GUTENBERG_TERA, &config, "", &permalinks_ctx, InsertAnchor::None);
let res = render_content("Hello `world`", &context).unwrap();
assert_eq!( assert_eq!(
res.0, res.0,
"<p>Hello <code>world</code></p>\n" "<p>Hello <code>world</code></p>\n"


BIN
sublime_syntaxes/newlines.packdump View File


BIN
sublime_syntaxes/nonewlines.packdump View File


Loadingโ€ฆ
Cancel
Save