Browse Source

Update pest for shortcode parser

index-subcmd
Vincent Prouillet 5 years ago
parent
commit
0b0b4a86db
4 changed files with 144 additions and 15 deletions
  1. +115
    -2
      Cargo.lock
  2. +2
    -2
      components/rendering/Cargo.toml
  3. +8
    -9
      components/rendering/src/content.pest
  4. +19
    -2
      components/rendering/src/shortcode.rs

+ 115
- 2
Cargo.lock View File

@@ -119,6 +119,11 @@ dependencies = [
"winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "arrayref"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "arrayvec"
version = "0.4.7"
@@ -205,11 +210,25 @@ name = "bitflags"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "block-buffer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "build_const"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "byte-tools"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "byteorder"
version = "1.2.6"
@@ -435,6 +454,14 @@ name = "deunicode"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "digest"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "dtoa"
version = "0.4.3"
@@ -580,6 +607,11 @@ dependencies = [
"synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "filetime"
version = "0.2.1"
@@ -691,6 +723,14 @@ dependencies = [
"num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "generic-array"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "getopts"
version = "0.2.18"
@@ -1408,6 +1448,14 @@ name = "pest"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "pest"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ucd-trie 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "pest_derive"
version = "1.0.8"
@@ -1418,6 +1466,37 @@ dependencies = [
"syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "pest_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"pest 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_generator 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "pest_generator"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"pest 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_meta 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "pest_meta"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pest 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "phf"
version = "0.7.23"
@@ -1654,8 +1733,8 @@ dependencies = [
"errors 0.1.0",
"front_matter 0.1.0",
"link_checker 0.1.0",
"pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
"pest 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pest_derive 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1860,6 +1939,17 @@ dependencies = [
"url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "sha-1"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
"fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "sha1"
version = "0.6.0"
@@ -2404,6 +2494,16 @@ name = "try-lock"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "typenum"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "ucd-trie"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "ucd-util"
version = "0.1.1"
@@ -2716,6 +2816,7 @@ dependencies = [
"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a"
"checksum ammonia 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a8b93ecb80665873703bf3b0a77f369c96b183d8e0afaf30a3ff5ff07dfc6409"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee"
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652"
"checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f"
@@ -2726,7 +2827,9 @@ dependencies = [
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5"
"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12"
"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab"
"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40"
"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781"
"checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62"
"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
@@ -2751,6 +2854,7 @@ dependencies = [
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum deflate 0.7.19 (registry+https://github.com/rust-lang/crates.io-index)" = "8a6abb26e16e8d419b5c78662aa9f82857c2386a073da266840e474d5055ec86"
"checksum deunicode 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "850878694b7933ca4c9569d30a34b55031b9b139ee1fc7b94a527c4ef960d690"
"checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90"
"checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd"
"checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0"
"checksum elasticlunr-rs 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4837d77a1e157489a3933b743fd774ae75074e0e390b2b7f071530048a0d87ee"
@@ -2767,6 +2871,7 @@ dependencies = [
"checksum error-chain 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46"
"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9"
"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426"
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f"
"checksum flate2 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4af030962d89d62aa52cd9492083b1cd9b2d1a77764878102a6c0f86b4d5444d"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@@ -2780,6 +2885,7 @@ dependencies = [
"checksum futf 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b"
"checksum futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)" = "49e7653e374fe0d0c12de4250f0bdb60680b8c80eed558c5c7538eec9c89e21b"
"checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d"
"checksum getopts 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0a7292d30132fb5424b354f5dc02512a86e4c516fe544bb7a25e7f266951b797"
"checksum gif 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4bca55ac1f213920ce3527ccd62386f1f15fa3f1714aeee1cf93f2c416903f"
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
@@ -2851,7 +2957,11 @@ dependencies = [
"checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c"
"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
"checksum pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc"
"checksum pest 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c3abb0d36ede865dcc689fd3bee2ff39094eff6e57a814f4a53c3c6108088353"
"checksum pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4"
"checksum pest_derive 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b76f477146419bc539a63f4ef40e902166cb43b3e51cecc71d9136fd12c567e7"
"checksum pest_generator 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ebee4e9680be4fd162e6f3394ae4192a6b60b1e4d17d845e631f0c68d1a3386"
"checksum pest_meta 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1f6d5f6f0e6082578c86af197d780dc38328e3f768cec06aac9bc46d714e8221"
"checksum phf 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "cec29da322b242f4c3098852c77a0ca261c9c01b806cae85a5572a1eb94db9a6"
"checksum phf_codegen 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "7d187f00cd98d5afbcd8898f6cf181743a449162aeb329dcd2f3849009e605ad"
"checksum phf_generator 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "03dc191feb9b08b0dc1330d6549b795b9d81aec19efe6b4a45aec8d4caee0c4b"
@@ -2899,6 +3009,7 @@ dependencies = [
"checksum serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "31569d901045afbff7a9479f793177fe9259819aff10ab4f89ef69bbc5f567fe"
"checksum serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "43344e7ce05d0d8280c5940cabb4964bea626aa58b1ec0e8c73fa2a8512a38ce"
"checksum serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aaed41d9fb1e2f587201b863356590c90c1157495d811430a0c0325fe8169650"
"checksum sha-1 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "51b9d1f3b5de8a167ab06834a7c883bd197f2191e1dda1a22d9ccfeedbf9aded"
"checksum sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"
"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
"checksum slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5f9776d6b986f77b35c6cf846c11ad986ff128fe0b2b63a3628e3755e8d3102d"
@@ -2947,6 +3058,8 @@ dependencies = [
"checksum trust-dns-proto 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35ba79f839f2ca927c2581a92286b18d4281fe3cd60984c4e48c68eb29c38793"
"checksum trust-dns-resolver 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4a821ad51a29816420b8cac4b026756b81c023630b97eaa4c8090637ee3508bd"
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169"
"checksum ucd-trie 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "71a9c5b1fe77426cf144cc30e49e955270f5086e31a6441dfa8b32efc09b9d77"
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
"checksum unic-char-property 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce36d3f7ce754afdbccccf8ff0dd0134e50fb44aaae579f96218856e9e5dbd1e"
"checksum unic-char-range 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab85fab42ad1b26cafc03bf891f69cb4d6e15f491030e89a0122197baa8ae8"


+ 2
- 2
components/rendering/Cargo.toml View File

@@ -10,8 +10,8 @@ pulldown-cmark = "0"
slug = "0.1"
serde = "1"
serde_derive = "1"
pest = "1"
pest_derive = "1"
pest = "2"
pest_derive = "2"

errors = { path = "../errors" }
front_matter = { path = "../front_matter" }


+ 8
- 9
components/rendering/src/content.pest View File

@@ -1,6 +1,5 @@
// Partly taken from Tera

whitespace = _{ " " | "\t" | "\r" | "\n" }
WHITESPACE = _{ " " | "\t" | "\r" | "\n" }

/// LITERALS
int = @{ "-" ? ~ ("0" | '1'..'9' ~ '0'..'9' * ) }
@@ -12,11 +11,11 @@ float = @{
)
}
// matches anything between 2 double quotes
double_quoted_string = @{ "\"" ~ (!("\"") ~ any)* ~ "\""}
double_quoted_string = @{ "\"" ~ (!("\"") ~ ANY)* ~ "\""}
// matches anything between 2 single quotes
single_quoted_string = @{ "\'" ~ (!("\'") ~ any)* ~ "\'"}
single_quoted_string = @{ "\'" ~ (!("\'") ~ ANY)* ~ "\'"}
// matches anything between 2 backquotes\backticks
backquoted_quoted_string = @{ "`" ~ (!("`") ~ any)* ~ "`"}
backquoted_quoted_string = @{ "`" ~ (!("`") ~ ANY)* ~ "`"}

string = @{
double_quoted_string |
@@ -56,9 +55,9 @@ ignored_sc_body_end = !{ "{%/*" ~ "end" ~ "*/%}" }
shortcode_with_body = !{ sc_body_start ~ text_in_body_sc ~ sc_body_end }
ignored_shortcode_with_body = { ignored_sc_body_start ~ text_in_ignored_body_sc ~ ignored_sc_body_end }

text_in_body_sc = ${ (!(sc_body_end) ~ any)+ }
text_in_ignored_body_sc = ${ (!(ignored_sc_body_end) ~ any)+ }
text = ${ (!(inline_shortcode | ignored_inline_shortcode | sc_body_start | ignored_sc_body_start) ~ any)+ }
text_in_body_sc = ${ (!(sc_body_end) ~ ANY)+ }
text_in_ignored_body_sc = ${ (!(ignored_sc_body_end) ~ ANY)+ }
text = ${ (!(inline_shortcode | ignored_inline_shortcode | sc_body_start | ignored_sc_body_start) ~ ANY)+ }

content = _{
ignored_inline_shortcode |
@@ -69,4 +68,4 @@ content = _{
}


page = ${ soi ~ content* ~ eoi }
page = ${ SOI ~ content* ~ EOI }

+ 19
- 2
components/rendering/src/shortcode.rs View File

@@ -125,7 +125,23 @@ pub fn render_shortcodes(content: &str, context: &RenderContext) -> Result<Strin
Rule::sc_body_start => "the start of a shortcode".to_string(),
Rule::ignored_sc_body_start => "the start of an ignored shortcode".to_string(),
Rule::text => "some text".to_string(),
_ => format!("TODO error: {:?}", rule).to_string(),
Rule::EOI => "end of input".to_string(),
Rule::double_quoted_string => "double quoted string".to_string(),
Rule::single_quoted_string => "single quoted string".to_string(),
Rule::backquoted_quoted_string => "backquoted quoted string".to_string(),
Rule::boolean => "a boolean (true, false)".to_string(),
Rule::all_chars => "a alphanumerical character".to_string(),
Rule::kwargs => "a list of keyword arguments".to_string(),
Rule::sc_def => "a shortcode definition".to_string(),
Rule::shortcode_with_body => "a shortcode with body".to_string(),
Rule::ignored_shortcode_with_body => "an ignored shortcode with body".to_string(),
Rule::sc_body_end => "{% end %}".to_string(),
Rule::ignored_sc_body_end => "{%/* end */%}".to_string(),
Rule::text_in_body_sc => "text in a shortcode body".to_string(),
Rule::text_in_ignored_body_sc => "text in an ignored shortcode body".to_string(),
Rule::content => "some content".to_string(),
Rule::page => "a page".to_string(),
Rule::WHITESPACE => "whitespace".to_string(),
}
});
bail!("{}", fancy_e);
@@ -169,7 +185,8 @@ pub fn render_shortcodes(content: &str, context: &RenderContext) -> Result<Strin
_ => unreachable!("Got something weird in an ignored shortcode: {:?}", p2),
}
}
}
},
Rule::EOI => (),
_ => unreachable!("unexpected page rule: {:?}", p.as_rule()),
}
}


Loading…
Cancel
Save