Update shortcode parsing and docs

Cargo.lock

@@ -1583,6 +1583,11 @@ name = "unicode-normalization"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.5"
@@ -1646,6 +1651,7 @@ dependencies = [
  "errors 0.1.0",
  "tempfile 3.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "walkdir 2.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1909,6 +1915,7 @@ dependencies = [
 "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33"
 "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
 "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25"
+"checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1"
 "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526"
 "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"

components/rendering/src/content.pest

@@ -45,8 +45,6 @@ kwarg   = { ident ~ "=" ~ (literal | array) }
 kwargs  = _{ kwarg ~ ("," ~ kwarg )* }
 sc_def  = _{ ident ~ "(" ~ kwargs* ~ ")" }
 
-//sc_start = _{ "{{/*" | "{{" | "{%/*" | "{%" }
-
 inline_shortcode         = !{ "{{" ~ sc_def ~ "}}" }
 ignored_inline_shortcode = !{ "{{/*" ~ sc_def ~ "*/}}" }
 
@@ -58,9 +56,9 @@ ignored_sc_body_end   = !{ "{%/*" ~ "end" ~ "*/%}" }
 shortcode_with_body         = !{ sc_body_start ~ text_in_body_sc ~ sc_body_end }
 ignored_shortcode_with_body = !{ ignored_sc_body_start ~ text_in_ignored_body_sc ~ ignored_sc_body_end }
 
-text_in_body_sc         = ${ (!("{%") ~ any)+ }
-text_in_ignored_body_sc = ${ (!("{%/*") ~ any)+ }
-text                    = ${ (!("{{/*" | "{{" | "{%/*" | "{%") ~ any)+ }
+text_in_body_sc         = ${ (!(sc_body_end) ~ any)+ }
+text_in_ignored_body_sc = ${ (!(ignored_sc_body_end) ~ any)+ }
+text                    = ${ (!(inline_shortcode | ignored_inline_shortcode | sc_body_start | ignored_sc_body_start) ~ any)+ }
 
 content = _{
     ignored_inline_shortcode |

components/rendering/src/shortcode.rs

@@ -7,7 +7,7 @@ use config::Config;
 
 // This include forces recompiling this source file if the grammar file changes.
 // Uncomment it when doing changes to the .pest file
-// const _GRAMMAR: &str = include_str!("content.pest");
+ const _GRAMMAR: &str = include_str!("content.pest");
 
 #[derive(Parser)]
 #[grammar = "content.pest"]
@@ -114,6 +114,11 @@ pub fn render_shortcodes(content: &str, tera: &Tera, config: &Config) -> Result<
                     Rule::array => "an array".to_string(),
                     Rule::kwarg => "a keyword argument".to_string(),
                     Rule::ident => "an identifier".to_string(),
+                    Rule::inline_shortcode => "an inline shortcode".to_string(),
+                    Rule::ignored_inline_shortcode => "an ignored inline shortcode".to_string(),
+                    Rule::sc_body_start => "the start of a shortcode".to_string(),
+                    Rule::ignored_sc_body_start => "the start of an ignored shortcode".to_string(),
+                    Rule::text => "some text".to_string(),
                     _ => format!("TODO error: {:?}", rule).to_string(),
                 }
             });
@@ -343,10 +348,4 @@ Hello World
         let res = render_shortcodes("Body\n {% youtube() %}Hey!{% end %}", &tera, &Config::default()).unwrap();
         assert_eq!(res, "Body\n Hey!");
     }
-
-    #[test]
-    fn errors_on_unterminated_shortcode() {
-        let res = render_shortcodes("{{ youtube(", &Tera::default(), &Config::default());
-        assert!(res.is_err());
-    }
 }

docs/content/documentation/content/shortcodes.md

@@ -6,7 +6,7 @@ weight = 40
 While Markdown is good at writing, it isn't great when you need write inline
 HTML to add some styling for example.
 
-To solve this, Gutenberg borrows the concept of [shortcodes](https://codex.wordpress.org/Shortcode_API) 
+To solve this, Gutenberg borrows the concept of [shortcodes](https://codex.wordpress.org/Shortcode_API)
 from WordPress.
 In our case, the shortcode corresponds to a template that is defined in the `templates/shortcodes` directory or a built-in one.
 
@@ -17,10 +17,10 @@ following:
 
 ```jinja2
 <div {% if class %}class="{{class}}"{% endif %}>
-    <iframe 
-        src="https://www.youtube.com/embed/{{id}}{% if autoplay %}?autoplay=1{% endif %}" 
-        webkitallowfullscreen 
-        mozallowfullscreen 
+    <iframe
+        src="https://www.youtube.com/embed/{{id}}{% if autoplay %}?autoplay=1{% endif %}"
+        webkitallowfullscreen
+        mozallowfullscreen
         allowfullscreen>
     </iframe>
 </div>
@@ -34,7 +34,7 @@ That's it, Gutenberg will now recognise this template as a shortcode named `yout
 
 ## Using shortcodes
 
-There are two kinds of shortcodes: 
+There are two kinds of shortcodes:
 
 - ones that do not take a body like the YouTube example above
 - ones that do, a quote for example
@@ -43,7 +43,7 @@ In both cases, their arguments must be named and they will all be passed to the
 
 Any shortcodes in code blocks will be ignored.
 
-Lastly, a shortcode name (and thus the corresponding `.html` file) as well as the arguments name 
+Lastly, a shortcode name (and thus the corresponding `.html` file) as well as the arguments name
 can only contain numbers, letters and underscores, or in Regex terms the following: `[0-9A-Za-z_]`.
 While theoretically an argument name could be a number, it will not be possible to use it in the template in that case.
 
@@ -64,11 +64,11 @@ calls of the YouTube shortcode.
 ```md
 Here is a YouTube video:
 
-{{ youtube(id="dQw4w9WgXcQ") }}
+{{/* youtube(id="dQw4w9WgXcQ") */}}
 
-{{ youtube(id="dQw4w9WgXcQ", autoplay=true) }}
+{{/* youtube(id="dQw4w9WgXcQ", autoplay=true) */}}
 
-{{ youtube(id="dQw4w9WgXcQ", autoplay=true, class="youtube") }}
+{{/* youtube(id="dQw4w9WgXcQ", autoplay=true, class="youtube") */}}
 ```
 
 ### Shortcodes with body
@@ -86,9 +86,9 @@ We could use it in our markup file like so:
 ```md
 As someone said:
 
-{% quote(author="Vincent") %}
+{%/* quote(author="Vincent") */%}
 A quote
-{% end %}
+{%/* end */%}
 ```
 
 The body of the shortcode will be automatically passed down to the rendering context as the `body` variable and needs
@@ -112,11 +112,11 @@ The arguments are:
 Usage example:
 
 ```md
-{{ youtube(id="dQw4w9WgXcQ") }}
+{{/* youtube(id="dQw4w9WgXcQ") */}}
 
-{{ youtube(id="dQw4w9WgXcQ", autoplay=true) }}
+{{/* youtube(id="dQw4w9WgXcQ", autoplay=true) */}}
 
-{{ youtube(id="dQw4w9WgXcQ", autoplay=true, class="youtube") }}
+{{/* youtube(id="dQw4w9WgXcQ", autoplay=true, class="youtube") */}}
 ```
 
 Result example:
@@ -134,9 +134,9 @@ The arguments are:
 Usage example:
 
 ```md
-{{ vimeo(id="124313553") }}
+{{/* vimeo(id="124313553") */}}
 
-{{ vimeo(id="124313553", class="vimeo") }}
+{{/* vimeo(id="124313553", class="vimeo") */}}
 ```
 
 Result example:
@@ -145,7 +145,7 @@ Result example:
 
 ### Streamable
 Embed a player for a Streamable video.
- 
+
 The arguments are:
 
 - `id`: the video id (mandatory)
@@ -154,9 +154,9 @@ The arguments are:
 Usage example:
 
 ```md
-{{ streamable(id="2zt0") }}
+{{/* streamable(id="2zt0") */}}
 
-{{ streamable(id="2zt0", class="streamble") }}
+{{/* streamable(id="2zt0", class="streamble") */}}
 ```
 
 Result example:
@@ -164,7 +164,7 @@ Result example:
 {{ streamable(id="2zt0") }}
 
 ### Gist
-Embed a [Github gist]().
+Embed a [Github gist](https://gist.github.com).
 
 The arguments are:
 
@@ -175,9 +175,9 @@ The arguments are:
 Usage example:
 
 ```md
-{{ gist(url="https://gist.github.com/Keats/e5fb6aad409f28721c0ba14161644c57") }}
+{{/* gist(url="https://gist.github.com/Keats/e5fb6aad409f28721c0ba14161644c57") */}}
 
-{{ gist(url="https://gist.github.com/Keats/e5fb6aad409f28721c0ba14161644c57", class="gist") }}
+{{/* gist(url="https://gist.github.com/Keats/e5fb6aad409f28721c0ba14161644c57", class="gist") */}}
 ```
 
 Result example:

docs/content/documentation/templates/overview.md

@@ -74,14 +74,14 @@ This can also be used to get the permalinks for static assets for example if
 we want to link to the file that is located at `static/css/app.css`:
 
 ```jinja2
-{{ get_url(path="css/app.css") }}
+{{/* get_url(path="css/app.css") */}}
 ```
 
 For assets it is reccommended that you pass `trailing_slash=false` to the `get_url` function. This prevents errors
 when dealing with certain hosting providers. An example is:
 
 ```jinja2
-{{ get_url(path="css/app.css", trailing_slash=false) }}
+{{/* get_url(path="css/app.css", trailing_slash=false) */}}
 ```
 
 In the case of non-internal links, you can also add a cachebust of the format `?t=1290192` at the end of a URL
@@ -102,6 +102,6 @@ the value should be the same as the one in the front-matter, not the slugified v
 Gets the translation of the given `key`, for the `default_language` or the `language given
 
 ```jinja2
-{{ trans(key="title") }}
-{{ trans(key="title", lang="fr") }}
+{{/* trans(key="title") */}}
+{{/* trans(key="title", lang="fr") */}}
 ```