You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

416 lines
15KB

  1. use pest::iterators::Pair;
  2. use pest::Parser;
  3. use regex::Regex;
  4. use tera::{to_value, Context, Map, Value};
  5. use context::RenderContext;
  6. use errors::{Error, Result};
  7. // This include forces recompiling this source file if the grammar file changes.
  8. // Uncomment it when doing changes to the .pest file
  9. const _GRAMMAR: &str = include_str!("content.pest");
  10. #[derive(Parser)]
  11. #[grammar = "content.pest"]
  12. pub struct ContentParser;
  13. lazy_static! {
  14. static ref MULTIPLE_NEWLINE_RE: Regex = Regex::new(r"\n\s*\n").unwrap();
  15. }
  16. fn replace_string_markers(input: &str) -> String {
  17. match input.chars().next().unwrap() {
  18. '"' => input.replace('"', ""),
  19. '\'' => input.replace('\'', ""),
  20. '`' => input.replace('`', ""),
  21. _ => unreachable!("How did you even get there"),
  22. }
  23. }
  24. fn parse_literal(pair: Pair<Rule>) -> Value {
  25. let mut val = None;
  26. for p in pair.into_inner() {
  27. match p.as_rule() {
  28. Rule::boolean => match p.as_str() {
  29. "true" => val = Some(Value::Bool(true)),
  30. "false" => val = Some(Value::Bool(false)),
  31. _ => unreachable!(),
  32. },
  33. Rule::string => val = Some(Value::String(replace_string_markers(p.as_str()))),
  34. Rule::float => {
  35. val = Some(to_value(p.as_str().parse::<f64>().unwrap()).unwrap());
  36. }
  37. Rule::int => {
  38. val = Some(to_value(p.as_str().parse::<i64>().unwrap()).unwrap());
  39. }
  40. _ => unreachable!("Unknown literal: {:?}", p),
  41. };
  42. }
  43. val.unwrap()
  44. }
  45. /// Returns (shortcode_name, kwargs)
  46. fn parse_shortcode_call(pair: Pair<Rule>) -> (String, Map<String, Value>) {
  47. let mut name = None;
  48. let mut args = Map::new();
  49. for p in pair.into_inner() {
  50. match p.as_rule() {
  51. Rule::ident => {
  52. name = Some(p.into_span().as_str().to_string());
  53. }
  54. Rule::kwarg => {
  55. let mut arg_name = None;
  56. let mut arg_val = None;
  57. for p2 in p.into_inner() {
  58. match p2.as_rule() {
  59. Rule::ident => {
  60. arg_name = Some(p2.into_span().as_str().to_string());
  61. }
  62. Rule::literal => {
  63. arg_val = Some(parse_literal(p2));
  64. }
  65. Rule::array => {
  66. let mut vals = vec![];
  67. for p3 in p2.into_inner() {
  68. match p3.as_rule() {
  69. Rule::literal => vals.push(parse_literal(p3)),
  70. _ => unreachable!(
  71. "Got something other than literal in an array: {:?}",
  72. p3
  73. ),
  74. }
  75. }
  76. arg_val = Some(Value::Array(vals));
  77. }
  78. _ => unreachable!("Got something unexpected in a kwarg: {:?}", p2),
  79. }
  80. }
  81. args.insert(arg_name.unwrap(), arg_val.unwrap());
  82. }
  83. _ => unreachable!("Got something unexpected in a shortcode: {:?}", p),
  84. }
  85. }
  86. (name.unwrap(), args)
  87. }
  88. fn render_shortcode(
  89. name: &str,
  90. args: &Map<String, Value>,
  91. context: &RenderContext,
  92. body: Option<&str>,
  93. ) -> Result<String> {
  94. let mut tera_context = Context::new();
  95. for (key, value) in args.iter() {
  96. tera_context.insert(key, value);
  97. }
  98. if let Some(ref b) = body {
  99. // Trimming right to avoid most shortcodes with bodies ending up with a HTML new line
  100. tera_context.insert("body", b.trim_end());
  101. }
  102. tera_context.extend(context.tera_context.clone());
  103. let template_name = format!("shortcodes/{}.html", name);
  104. let res =
  105. utils::templates::render_template(&template_name, &context.tera, tera_context, &None)
  106. .map_err(|e| Error::chain(format!("Failed to render {} shortcode", name), e))?;
  107. // Small hack to avoid having multiple blank lines because of Tera tags for example
  108. // A blank like will cause the markdown parser to think we're out of HTML and start looking
  109. // at indentation, making the output a code block.
  110. let res = MULTIPLE_NEWLINE_RE.replace_all(&res, "\n");
  111. Ok(res.to_string())
  112. }
  113. pub fn render_shortcodes(content: &str, context: &RenderContext) -> Result<String> {
  114. let mut res = String::with_capacity(content.len());
  115. let mut pairs = match ContentParser::parse(Rule::page, content) {
  116. Ok(p) => p,
  117. Err(e) => {
  118. let fancy_e = e.renamed_rules(|rule| match *rule {
  119. Rule::int => "an integer".to_string(),
  120. Rule::float => "a float".to_string(),
  121. Rule::string => "a string".to_string(),
  122. Rule::literal => "a literal (int, float, string, bool)".to_string(),
  123. Rule::array => "an array".to_string(),
  124. Rule::kwarg => "a keyword argument".to_string(),
  125. Rule::ident => "an identifier".to_string(),
  126. Rule::inline_shortcode => "an inline shortcode".to_string(),
  127. Rule::ignored_inline_shortcode => "an ignored inline shortcode".to_string(),
  128. Rule::sc_body_start => "the start of a shortcode".to_string(),
  129. Rule::ignored_sc_body_start => "the start of an ignored shortcode".to_string(),
  130. Rule::text => "some text".to_string(),
  131. Rule::EOI => "end of input".to_string(),
  132. Rule::double_quoted_string => "double quoted string".to_string(),
  133. Rule::single_quoted_string => "single quoted string".to_string(),
  134. Rule::backquoted_quoted_string => "backquoted quoted string".to_string(),
  135. Rule::boolean => "a boolean (true, false)".to_string(),
  136. Rule::all_chars => "a alphanumerical character".to_string(),
  137. Rule::kwargs => "a list of keyword arguments".to_string(),
  138. Rule::sc_def => "a shortcode definition".to_string(),
  139. Rule::shortcode_with_body => "a shortcode with body".to_string(),
  140. Rule::ignored_shortcode_with_body => "an ignored shortcode with body".to_string(),
  141. Rule::sc_body_end => "{% end %}".to_string(),
  142. Rule::ignored_sc_body_end => "{%/* end */%}".to_string(),
  143. Rule::text_in_body_sc => "text in a shortcode body".to_string(),
  144. Rule::text_in_ignored_body_sc => "text in an ignored shortcode body".to_string(),
  145. Rule::content => "some content".to_string(),
  146. Rule::page => "a page".to_string(),
  147. Rule::WHITESPACE => "whitespace".to_string(),
  148. });
  149. bail!("{}", fancy_e);
  150. }
  151. };
  152. // We have at least a `page` pair
  153. for p in pairs.next().unwrap().into_inner() {
  154. match p.as_rule() {
  155. Rule::text => res.push_str(p.into_span().as_str()),
  156. Rule::inline_shortcode => {
  157. let (name, args) = parse_shortcode_call(p);
  158. res.push_str(&render_shortcode(&name, &args, context, None)?);
  159. }
  160. Rule::shortcode_with_body => {
  161. let mut inner = p.into_inner();
  162. // 3 items in inner: call, body, end
  163. // we don't care about the closing tag
  164. let (name, args) = parse_shortcode_call(inner.next().unwrap());
  165. let body = inner.next().unwrap().into_span().as_str();
  166. res.push_str(&render_shortcode(&name, &args, context, Some(body))?);
  167. }
  168. Rule::ignored_inline_shortcode => {
  169. res.push_str(
  170. &p.into_span().as_str().replacen("{{/*", "{{", 1).replacen("*/}}", "}}", 1),
  171. );
  172. }
  173. Rule::ignored_shortcode_with_body => {
  174. for p2 in p.into_inner() {
  175. match p2.as_rule() {
  176. Rule::ignored_sc_body_start | Rule::ignored_sc_body_end => {
  177. res.push_str(
  178. &p2.into_span()
  179. .as_str()
  180. .replacen("{%/*", "{%", 1)
  181. .replacen("*/%}", "%}", 1),
  182. );
  183. }
  184. Rule::text_in_ignored_body_sc => res.push_str(p2.into_span().as_str()),
  185. _ => unreachable!("Got something weird in an ignored shortcode: {:?}", p2),
  186. }
  187. }
  188. }
  189. Rule::EOI => (),
  190. _ => unreachable!("unexpected page rule: {:?}", p.as_rule()),
  191. }
  192. }
  193. Ok(res)
  194. }
  195. #[cfg(test)]
  196. mod tests {
  197. use std::collections::HashMap;
  198. use super::*;
  199. use config::Config;
  200. use front_matter::InsertAnchor;
  201. use tera::Tera;
  202. macro_rules! assert_lex_rule {
  203. ($rule: expr, $input: expr) => {
  204. let res = ContentParser::parse($rule, $input);
  205. println!("{:?}", $input);
  206. println!("{:#?}", res);
  207. if res.is_err() {
  208. println!("{}", res.unwrap_err());
  209. panic!();
  210. }
  211. assert!(res.is_ok());
  212. assert_eq!(res.unwrap().last().unwrap().into_span().end(), $input.len());
  213. };
  214. }
  215. fn render_shortcodes(code: &str, tera: &Tera) -> String {
  216. let config = Config::default();
  217. let permalinks = HashMap::new();
  218. let context = RenderContext::new(&tera, &config, "", &permalinks, InsertAnchor::None);
  219. super::render_shortcodes(code, &context).unwrap()
  220. }
  221. #[test]
  222. fn lex_text() {
  223. let inputs = vec!["Hello world", "HEllo \n world", "Hello 1 2 true false 'hey'"];
  224. for i in inputs {
  225. assert_lex_rule!(Rule::text, i);
  226. }
  227. }
  228. #[test]
  229. fn lex_inline_shortcode() {
  230. let inputs = vec![
  231. "{{ youtube() }}",
  232. "{{ youtube(id=1, autoplay=true, url='hey') }}",
  233. "{{ youtube(id=1, \nautoplay=true, url='hey') }}",
  234. ];
  235. for i in inputs {
  236. assert_lex_rule!(Rule::inline_shortcode, i);
  237. }
  238. }
  239. #[test]
  240. fn lex_inline_ignored_shortcode() {
  241. let inputs = vec![
  242. "{{/* youtube() */}}",
  243. "{{/* youtube(id=1, autoplay=true, url='hey') */}}",
  244. "{{/* youtube(id=1, \nautoplay=true, \nurl='hey') */}}",
  245. ];
  246. for i in inputs {
  247. assert_lex_rule!(Rule::ignored_inline_shortcode, i);
  248. }
  249. }
  250. #[test]
  251. fn lex_shortcode_with_body() {
  252. let inputs = vec![
  253. r#"{% youtube() %}
  254. Some text
  255. {% end %}"#,
  256. r#"{% youtube(id=1,
  257. autoplay=true, url='hey') %}
  258. Some text
  259. {% end %}"#,
  260. ];
  261. for i in inputs {
  262. assert_lex_rule!(Rule::shortcode_with_body, i);
  263. }
  264. }
  265. #[test]
  266. fn lex_ignored_shortcode_with_body() {
  267. let inputs = vec![
  268. r#"{%/* youtube() */%}
  269. Some text
  270. {%/* end */%}"#,
  271. r#"{%/* youtube(id=1,
  272. autoplay=true, url='hey') */%}
  273. Some text
  274. {%/* end */%}"#,
  275. ];
  276. for i in inputs {
  277. assert_lex_rule!(Rule::ignored_shortcode_with_body, i);
  278. }
  279. }
  280. #[test]
  281. fn lex_page() {
  282. let inputs = vec![
  283. "Some text and a shortcode `{{/* youtube() */}}`",
  284. "{{ youtube(id=1, autoplay=true, url='hey') }}",
  285. "{{ youtube(id=1, \nautoplay=true, url='hey') }} that's it",
  286. r#"
  287. This is a test
  288. {% hello() %}
  289. Body {{ var }}
  290. {% end %}
  291. "#,
  292. ];
  293. for i in inputs {
  294. assert_lex_rule!(Rule::page, i);
  295. }
  296. }
  297. #[test]
  298. fn does_nothing_with_no_shortcodes() {
  299. let res = render_shortcodes("Hello World", &Tera::default());
  300. assert_eq!(res, "Hello World");
  301. }
  302. #[test]
  303. fn can_unignore_inline_shortcode() {
  304. let res = render_shortcodes("Hello World {{/* youtube() */}}", &Tera::default());
  305. assert_eq!(res, "Hello World {{ youtube() }}");
  306. }
  307. #[test]
  308. fn can_unignore_shortcode_with_body() {
  309. let res = render_shortcodes(
  310. r#"
  311. Hello World
  312. {%/* youtube() */%}Some body {{ hello() }}{%/* end */%}"#,
  313. &Tera::default(),
  314. );
  315. assert_eq!(res, "\nHello World\n{% youtube() %}Some body {{ hello() }}{% end %}");
  316. }
  317. // https://github.com/Keats/gutenberg/issues/383
  318. #[test]
  319. fn unignore_shortcode_with_body_does_not_swallow_initial_whitespace() {
  320. let res = render_shortcodes(
  321. r#"
  322. Hello World
  323. {%/* youtube() */%}
  324. Some body {{ hello() }}{%/* end */%}"#,
  325. &Tera::default(),
  326. );
  327. assert_eq!(res, "\nHello World\n{% youtube() %}\nSome body {{ hello() }}{% end %}");
  328. }
  329. #[test]
  330. fn can_parse_shortcode_arguments() {
  331. let inputs = vec![
  332. ("{{ youtube() }}", "youtube", Map::new()),
  333. ("{{ youtube(id=1, autoplay=true, hello='salut', float=1.2) }}", "youtube", {
  334. let mut m = Map::new();
  335. m.insert("id".to_string(), to_value(1).unwrap());
  336. m.insert("autoplay".to_string(), to_value(true).unwrap());
  337. m.insert("hello".to_string(), to_value("salut").unwrap());
  338. m.insert("float".to_string(), to_value(1.2).unwrap());
  339. m
  340. }),
  341. ("{{ gallery(photos=['something', 'else'], fullscreen=true) }}", "gallery", {
  342. let mut m = Map::new();
  343. m.insert("photos".to_string(), to_value(["something", "else"]).unwrap());
  344. m.insert("fullscreen".to_string(), to_value(true).unwrap());
  345. m
  346. }),
  347. ];
  348. for (i, n, a) in inputs {
  349. let mut res = ContentParser::parse(Rule::inline_shortcode, i).unwrap();
  350. let (name, args) = parse_shortcode_call(res.next().unwrap());
  351. assert_eq!(name, n);
  352. assert_eq!(args, a);
  353. }
  354. }
  355. #[test]
  356. fn can_render_inline_shortcodes() {
  357. let mut tera = Tera::default();
  358. tera.add_raw_template("shortcodes/youtube.html", "Hello {{id}}").unwrap();
  359. let res = render_shortcodes("Inline {{ youtube(id=1) }}.", &tera);
  360. assert_eq!(res, "Inline Hello 1.");
  361. }
  362. #[test]
  363. fn can_render_shortcodes_with_body() {
  364. let mut tera = Tera::default();
  365. tera.add_raw_template("shortcodes/youtube.html", "{{body}}").unwrap();
  366. let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera);
  367. assert_eq!(res, "Body\n Hey!");
  368. }
  369. // https://github.com/Keats/gutenberg/issues/462
  370. #[test]
  371. fn shortcodes_with_body_do_not_eat_newlines() {
  372. let mut tera = Tera::default();
  373. tera.add_raw_template("shortcodes/youtube.html", "{{body | safe}}").unwrap();
  374. let res = render_shortcodes("Body\n {% youtube() %}\nHello \n World{% end %}", &tera);
  375. assert_eq!(res, "Body\n Hello \n World");
  376. }
  377. }