Merge pull request #253 from Keats/search

[WIP] Search

.gitignore

@@ -1,6 +1,6 @@
 target
 .idea/
-components/site/test_site/public
+test_site/public
 docs/public
 
 small-blog

CHANGELOG.md

@@ -8,6 +8,7 @@
 to the public directory
 - Do not require themes to have a static folder
 - Now supports indented Sass syntax
+- Add search index building
 
 ## 0.3.2 (2018-03-05)
 

Cargo.lock

@@ -6,6 +6,19 @@ dependencies = [
  "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ammonia"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "html5ever 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "ansi_term"
 version = "0.11.0"
@@ -149,7 +162,7 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "2.31.1"
+version = "2.31.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -241,6 +254,14 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "debug_unreachable"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "dtoa"
 version = "0.4.2"
@@ -262,6 +283,21 @@ name = "either"
 version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "elasticlunr-rs"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strum 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strum_macros 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "error-chain"
 version = "0.11.0"
@@ -354,6 +390,15 @@ name = "fuchsia-zircon-sys"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "futf"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "gcc"
 version = "0.3.54"
@@ -386,7 +431,7 @@ name = "gutenberg"
 version = "0.3.3"
 dependencies = [
  "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "clap 2.31.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "content 0.1.0",
  "ctrlc 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "errors 0.1.0",
@@ -412,6 +457,18 @@ dependencies = [
  "syntect 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "html5ever"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "httparse"
 version = "1.2.4"
@@ -547,6 +604,31 @@ dependencies = [
  "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "maplit"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "markup5ever"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "matches"
 version = "0.1.6"
@@ -881,6 +963,11 @@ dependencies = [
  "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "proc-macro2"
 version = "0.2.3"
@@ -1031,6 +1118,15 @@ dependencies = [
  "utils 0.1.0",
 ]
 
+[[package]]
+name = "rust-stemmers"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.7"
@@ -1073,6 +1169,17 @@ name = "scopeguard"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "search"
+version = "0.1.0"
+dependencies = [
+ "ammonia 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "content 0.1.0",
+ "elasticlunr-rs 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "errors 0.1.0",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "sequence_trie"
 version = "0.3.5"
@@ -1146,6 +1253,7 @@ dependencies = [
  "pagination 0.1.0",
  "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "search 0.1.0",
  "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
  "taxonomies 0.1.0",
@@ -1153,7 +1261,6 @@ dependencies = [
  "templates 0.1.0",
  "tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "utils 0.1.0",
- "walkdir 2.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1190,11 +1297,55 @@ dependencies = [
  "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "string_cache"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "string_cache_shared"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "strsim"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "strum"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "strum_macros"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "syn"
 version = "0.11.11"
@@ -1282,6 +1433,16 @@ dependencies = [
  "utils 0.1.0",
 ]
 
+[[package]]
+name = "tendril"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "futf 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf-8 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "tera"
 version = "0.11.5"
@@ -1427,6 +1588,14 @@ name = "unidecode"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "unreachable"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "unreachable"
 version = "1.0.0"
@@ -1453,6 +1622,11 @@ dependencies = [
  "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "utf-8"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "utf8-ranges"
 version = "1.0.0"
@@ -1465,6 +1639,7 @@ dependencies = [
  "errors 0.1.0",
  "tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "walkdir 2.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1563,6 +1738,7 @@ dependencies = [
 
 [metadata]
 "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
+"checksum ammonia 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd4c682378117e4186a492b2252b9537990e1617f44aed9788b9a1149de45477"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
 "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
 "checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4"
@@ -1582,15 +1758,17 @@ dependencies = [
 "checksum cc 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "d9324127e719125ec8a16e6e509abc4c641e773621b50aea695af3f005656d61"
 "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de"
 "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9"
-"checksum clap 2.31.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dc18f6f4005132120d9711636b32c46a233fad94df6217fa1d81c5e97a9f200"
+"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536"
 "checksum cmake 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "56d741ea7a69e577f6d06b36b7dff4738f680593dc27a701ffa8506b73ce28bb"
 "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3"
 "checksum crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "59796cc6cbbdc6bb319161349db0c3250ec73ec7fcb763a51065ec4e2e158552"
 "checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
 "checksum ctrlc 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "653abc99aa905f693d89df4797fadc08085baee379db92be9f2496cefe8a6f2c"
+"checksum debug_unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3"
 "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
 "checksum duct 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8c553d79f40e74f7f611e49bf3429b6760cff79596b61818291c27cc0b18549d"
 "checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3"
+"checksum elasticlunr-rs 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18c5413e83a7be7e86ce0afce03b23508db56082edbb2b28a178e6275fa3957d"
 "checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3"
 "checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f"
 "checksum flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fac2277e84e5e858483756647a9d0aa8d9a2b7cba517fd84325a0aaa69a0909"
@@ -1600,10 +1778,12 @@ dependencies = [
 "checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
+"checksum futf 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "51f93f3de6ba1794dcd5810b3546d004600a59a98266487c8407bc4b24e398f3"
 "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb"
 "checksum getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "b900c08c1939860ce8b54dc6a89e26e00c04c380fd0e09796799bd7f12861e05"
 "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
 "checksum globset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1e96ab92362c06811385ae9a34d2698e8a1160745e0c78fbb434a44c8de3fabc"
+"checksum html5ever 0.22.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e579ac8647178ab915d400d7d22938bda5cd351c6c62e1c294d56884ccfc75fe"
 "checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37"
 "checksum humansize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6cab2627acfc432780848602f3f558f7e9dd427352224b0d9324025796d2a5e"
 "checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2"
@@ -1622,6 +1802,9 @@ dependencies = [
 "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e"
 "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
 "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2"
+"checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+"checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43"
+"checksum markup5ever 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfedc97d5a503e96816d10fedcd5b42f760b2e525ce2f7ec71f6a41780548475"
 "checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
 "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
 "checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
@@ -1657,6 +1840,7 @@ dependencies = [
 "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
 "checksum plist 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c61ac2afed2856590ae79d6f358a24b85ece246d2aa134741a66d589519b7503"
 "checksum plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0"
+"checksum precomputed-hash 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 "checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0"
 "checksum pulldown-cmark 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d6fdf85cda6cadfae5428a54661d431330b312bc767ddbc57adbedc24da66e32"
 "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
@@ -1671,6 +1855,7 @@ dependencies = [
 "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
 "checksum regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2550876c31dc914696a6c2e01cbce8afba79a93c8ae979d2fe051c0230b3756"
 "checksum remove_dir_all 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b5d2f806b0fcdabd98acd380dc8daef485e22bcb7cddc811d1337967f2528cf5"
+"checksum rust-stemmers 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fbf06149ec391025664a5634200ced1afb489f0f3f8a140d515ebc0eb04b4bc0"
 "checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb"
 "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
 "checksum same-file 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cfb6eded0b06a0b512c8ddbcf04089138c9b4362c2f696f3c3d76039d68f3637"
@@ -1690,12 +1875,18 @@ dependencies = [
 "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d"
 "checksum slug 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f5ff4b43cb07b86c5f9236c92714a22cdf9e5a27a7d85e398e2c9403328cb8"
 "checksum staticfile 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "babd3fa68bb7e3994ce181c5f21ff3ff5fffef7b18b8a10163b45e4dafc6fb86"
+"checksum string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39cb4173bcbd1319da31faa5468a7e3870683d7a237150b0b0aaafd546f6ad12"
+"checksum string_cache_codegen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "479cde50c3539481f33906a387f2bd17c8e87cb848c35b6021d41fb81ff9b4d7"
+"checksum string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
+"checksum strum 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "099e21b5dd6dd07b5adcf8c4b723a7c0b7efd7a9359bf963d58c0caae8532545"
+"checksum strum_macros 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd9bd569e88028750e3ae5c25616b8278ac16a8e61aba4339195c72396d49e1"
 "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
 "checksum syn 0.12.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8c5bc2d6ff27891209efa5f63e9de78648d7801f085e4653701a692ce938d6fd"
 "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6"
 "checksum syntect 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db9fffcb25a761118df53811bd1cfcd54cf57fcbc51e1ea3167ae263477129ad"
 "checksum tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f73eebdb68c14bcb24aef74ea96079830e7fa7b31a6106e42ea7ee887c1e134e"
+"checksum tendril 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9de21546595a0873061940d994bbbc5c35f024ae4fd61ec5c5b159115684f508"
 "checksum tera 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1a35d04c2444875b1319293fbc72c00215ae6220f8c70f9f14fefa5eaae0c6"
 "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1"
 "checksum term-painter 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "dcaa948f0e3e38470cd8dc8dcfe561a75c9e43f28075bb183845be2b9b3c08cf"
@@ -1715,9 +1906,11 @@ dependencies = [
 "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc"
 "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
 "checksum unidecode 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2adb95ee07cd579ed18131f2d9e7a17c25a4b76022935c7f2460d2bfae89fd2"
+"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91"
 "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
 "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f"
 "checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7"
+"checksum utf-8 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f1262dfab4c30d5cb7c07026be00ee343a6cf5027fdc0104a9160f354e5db75c"
 "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
 "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c"
 "checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d"

Cargo.toml

@@ -52,4 +52,5 @@ members = [
     "components/taxonomies",
     "components/templates",
     "components/utils",
+    "components/search",
 ]

components/config/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "config"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 toml = "0.4"

components/content/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "content"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/content/src/page.rs

@@ -75,7 +75,7 @@ impl Page {
     }
 
     pub fn is_draft(&self) -> bool {
-        self.meta.draft.unwrap_or(false)
+        self.meta.draft
     }
 
     /// Parse a page given the content of the .md file

components/content/src/section.rs

@@ -104,7 +104,7 @@ impl Section {
             config.highlight_theme.clone(),
             &self.permalink,
             permalinks,
-            self.meta.insert_anchor_links.unwrap()
+            self.meta.insert_anchor_links,
         );
         let res = markdown_to_html(&self.raw_content, &context)?;
         self.content = res.0;

components/content/src/sorting.rs

@@ -149,7 +149,7 @@ mod tests {
     fn create_draft_page_with_order(order: usize) -> Page {
         let mut front_matter = PageFrontMatter::default();
         front_matter.order = Some(order);
-        front_matter.draft = Some(true);
+        front_matter.draft = true;
         Page::new("content/hello.md", front_matter)
     }
 

components/errors/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "errors"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 error-chain = "0.11"

components/front_matter/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "front_matter"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/front_matter/src/page.rs

@@ -62,6 +62,7 @@ fn fix_toml_dates(table: Map<String, Value>) -> Value {
 
 /// The front matter of every page
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
 pub struct PageFrontMatter {
     /// <title> of the page
     pub title: Option<String>,
@@ -71,7 +72,7 @@ pub struct PageFrontMatter {
     #[serde(default, deserialize_with = "from_toml_datetime")]
     pub date: Option<String>,
     /// Whether this page is a draft and should be ignored for pagination etc
-    pub draft: Option<bool>,
+    pub draft: bool,
     /// The page slug. Will be used instead of the filename if present
     /// Can't be an empty string if present
     pub slug: Option<String>,
@@ -90,12 +91,15 @@ pub struct PageFrontMatter {
     /// All aliases for that page. Gutenberg will create HTML templates that will
     /// redirect to this
     #[serde(skip_serializing)]
-    pub aliases: Option<Vec<String>>,
+    pub aliases: Vec<String>,
     /// Specify a template different from `page.html` to use for that page
     #[serde(skip_serializing)]
     pub template: Option<String>,
+    /// Whether the page is included in the search index
+    /// Defaults to `true` but is only used if search if explicitly enabled in the config.
+    #[serde(skip_serializing)]
+    pub in_search_index: bool,
     /// Any extra parameter present in the front matter
-    #[serde(default)]
     pub extra: Map<String, Value>,
 }
 
@@ -166,14 +170,15 @@ impl Default for PageFrontMatter {
             title: None,
             description: None,
             date: None,
-            draft: None,
+            draft: false,
             slug: None,
             path: None,
             tags: None,
             category: None,
             order: None,
             weight: None,
-            aliases: None,
+            aliases: Vec::new(),
+            in_search_index: true,
             template: None,
             extra: Map::new(),
         }

components/front_matter/src/section.rs

@@ -12,6 +12,7 @@ static DEFAULT_PAGINATE_PATH: &'static str = "page";
 
 /// The front matter of every section
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(default)]
 pub struct SectionFrontMatter {
     /// <title> of the page
     pub title: Option<String>,
@@ -19,11 +20,11 @@ pub struct SectionFrontMatter {
     pub description: Option<String>,
     /// Whether to sort by "date", "order", "weight" or "none". Defaults to `none`.
     #[serde(skip_serializing)]
-    pub sort_by: Option<SortBy>,
+    pub sort_by: SortBy,
     /// Used by the parent section to order its subsections.
-    /// Higher values means it will be at the end.
+    /// Higher values means it will be at the end. Defaults to `0`
     #[serde(skip_serializing)]
-    pub weight: Option<usize>,
+    pub weight: usize,
     /// Optional template, if we want to specify which template to render for that section
     #[serde(skip_serializing)]
     pub template: Option<String>,
@@ -32,59 +33,38 @@ pub struct SectionFrontMatter {
     pub paginate_by: Option<usize>,
     /// Path to be used by pagination: the page number will be appended after it. Defaults to `page`.
     #[serde(skip_serializing)]
-    pub paginate_path: Option<String>,
+    pub paginate_path: String,
     /// Whether to insert a link for each header like the ones you can see in this site if you hover one
     /// The default template can be overridden by creating a `anchor-link.html` in the `templates` directory
-    pub insert_anchor_links: Option<InsertAnchor>,
+    pub insert_anchor_links: InsertAnchor,
     /// Whether to render that section or not. Defaults to `true`.
     /// Useful when the section is only there to organize things but is not meant
     /// to be used directly, like a posts section in a personal site
     #[serde(skip_serializing)]
-    pub render: Option<bool>,
+    pub render: bool,
     /// Whether to redirect when landing on that section. Defaults to `None`.
     /// Useful for the same reason as `render` but when you don't want a 404 when
     /// landing on the root section page
     #[serde(skip_serializing)]
     pub redirect_to: Option<String>,
+    /// Whether the section content and its pages/subsections are included in the index.
+    /// Defaults to `true` but is only used if search if explicitly enabled in the config.
+    #[serde(skip_serializing)]
+    pub in_search_index: bool,
     /// Any extra parameter present in the front matter
-    pub extra: Option<HashMap<String, Value>>,
+    pub extra: HashMap<String, Value>,
 }
 
 impl SectionFrontMatter {
     pub fn parse(toml: &str) -> Result<SectionFrontMatter> {
-        let mut f: SectionFrontMatter = match toml::from_str(toml) {
+        let f: SectionFrontMatter = match toml::from_str(toml) {
             Ok(d) => d,
             Err(e) => bail!(e),
         };
 
-        if f.paginate_path.is_none() {
-            f.paginate_path = Some(DEFAULT_PAGINATE_PATH.to_string());
-        }
-
-        if f.render.is_none() {
-            f.render = Some(true);
-        }
-
-        if f.sort_by.is_none() {
-            f.sort_by = Some(SortBy::None);
-        }
-
-        if f.insert_anchor_links.is_none() {
-            f.insert_anchor_links = Some(InsertAnchor::None);
-        }
-
-        if f.weight.is_none() {
-            f.weight = Some(0);
-        }
-
         Ok(f)
     }
 
-    /// Returns the current sorting method, defaults to `None` (== no sorting)
-    pub fn sort_by(&self) -> SortBy {
-        self.sort_by.unwrap()
-    }
-
     /// Only applies to section, whether it is paginated or not.
     pub fn is_paginated(&self) -> bool {
         match self.paginate_by {
@@ -92,10 +72,6 @@ impl SectionFrontMatter {
             None => false
         }
     }
-
-    pub fn should_render(&self) -> bool {
-        self.render.unwrap()
-    }
 }
 
 impl Default for SectionFrontMatter {
@@ -103,15 +79,16 @@ impl Default for SectionFrontMatter {
         SectionFrontMatter {
             title: None,
             description: None,
-            sort_by: Some(SortBy::None),
-            weight: Some(0),
+            sort_by: SortBy::None,
+            weight: 0,
             template: None,
             paginate_by: None,
-            paginate_path: Some(DEFAULT_PAGINATE_PATH.to_string()),
-            render: Some(true),
+            paginate_path: DEFAULT_PAGINATE_PATH.to_string(),
+            render: true,
             redirect_to: None,
-            insert_anchor_links: Some(InsertAnchor::None),
-            extra: None,
+            insert_anchor_links: InsertAnchor::None,
+            in_search_index: true,
+            extra: HashMap::new(),
         }
     }
 }

components/highlighting/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "highlighting"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 lazy_static = "1"

components/pagination/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pagination"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/pagination/src/lib.rs

@@ -72,13 +72,9 @@ impl<'a> Paginator<'a> {
     /// It will always at least create one pager (the first) even if there are no pages to paginate
     pub fn new(all_pages: &'a [Page], section: &'a Section) -> Paginator<'a> {
         let paginate_by = section.meta.paginate_by.unwrap();
-        let paginate_path = match section.meta.paginate_path {
-            Some(ref p) => p,
-            None => unreachable!(),
-        };
-
         let mut pages = vec![];
         let mut current_page = vec![];
+
         for page in all_pages {
             current_page.push(page);
 
@@ -99,7 +95,7 @@ impl<'a> Paginator<'a> {
                 continue;
             }
 
-            let page_path = format!("{}/{}/", paginate_path, index + 1);
+            let page_path = format!("{}/{}/", section.meta.paginate_path, index + 1);
             let permalink = format!("{}{}", section.permalink, page_path);
             let pager_path = if section.is_index() {
                 page_path
@@ -189,7 +185,7 @@ mod tests {
     fn create_section(is_index: bool) -> Section {
         let mut f = SectionFrontMatter::default();
         f.paginate_by = Some(2);
-        f.paginate_path = Some("page".to_string());
+        f.paginate_path = "page".to_string();
         let mut s = Section::new("content/_index.md", f);
         if !is_index {
             s.path = "posts/".to_string();

components/rebuild/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "rebuild"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 errors = { path = "../errors" }

components/rebuild/src/lib.rs

@@ -60,7 +60,7 @@ fn find_section_front_matter_changes(current: &SectionFrontMatter, new: &Section
 
     // We want to hide the section
     // TODO: what to do on redirect_path change?
-    if current.should_render() && !new.should_render() {
+    if current.render && !new.render {
         changes_needed.push(SectionChangesNeeded::Delete);
         // Nothing else we can do
         return changes_needed;
@@ -383,14 +383,14 @@ mod tests {
 
     #[test]
     fn can_find_sort_changes_in_section_frontmatter() {
-        let new = SectionFrontMatter { sort_by: Some(SortBy::Date), ..SectionFrontMatter::default() };
+        let new = SectionFrontMatter { sort_by: SortBy::Date, ..SectionFrontMatter::default() };
         let changes = find_section_front_matter_changes(&SectionFrontMatter::default(), &new);
         assert_eq!(changes, vec![SectionChangesNeeded::Sort, SectionChangesNeeded::Render]);
     }
 
     #[test]
     fn can_find_render_changes_in_section_frontmatter() {
-        let new = SectionFrontMatter { render: Some(false), ..SectionFrontMatter::default() };
+        let new = SectionFrontMatter { render: false, ..SectionFrontMatter::default() };
         let changes = find_section_front_matter_changes(&SectionFrontMatter::default(), &new);
         assert_eq!(changes, vec![SectionChangesNeeded::Delete]);
     }

components/rendering/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "rendering"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/search/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "search"
+version = "0.1.0"
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
+
+[dependencies]
+elasticlunr-rs = "2"
+ammonia = "1"
+lazy_static = "1"
+
+errors = { path = "../errors" }
+content = { path = "../content" }

components/search/src/lib.rs

@@ -0,0 +1,80 @@
+extern crate elasticlunr;
+#[macro_use]
+extern crate lazy_static;
+extern crate ammonia;
+#[macro_use]
+extern crate errors;
+extern crate content;
+
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use elasticlunr::{Index, Language};
+
+use content::Section;
+use errors::Result;
+
+
+pub const ELASTICLUNR_JS: &'static str = include_str!("elasticlunr.min.js");
+
+lazy_static! {
+    static ref AMMONIA: ammonia::Builder<'static> = {
+        let mut clean_content = HashSet::new();
+        clean_content.insert("script");
+        clean_content.insert("style");
+        let mut builder = ammonia::Builder::new();
+        builder
+            .tags(HashSet::new())
+            .tag_attributes(HashMap::new())
+            .generic_attributes(HashSet::new())
+            .link_rel(None)
+            .allowed_classes(HashMap::new())
+            .clean_content_tags(clean_content);
+        builder
+    };
+}
+
+
+/// Returns the generated JSON index with all the documents of the site added using
+/// the language given
+/// Errors if the language given is not available in Elasticlunr
+/// TODO: is making `in_search_index` apply to subsections of a `false` section useful?
+pub fn build_index(sections: &HashMap<PathBuf, Section>, lang: &str) -> Result<String> {
+    let language = match Language::from_code(lang) {
+        Some(l) => l,
+        None => { bail!("Tried to build search index for language {} which is not supported", lang); }
+    };
+
+    let mut index = Index::with_language(language, &["title", "body"]);
+
+    for section in sections.values() {
+        add_section_to_index(&mut index, section);
+    }
+
+    Ok(index.to_json())
+}
+
+fn add_section_to_index(index: &mut Index, section: &Section) {
+    if !section.meta.in_search_index {
+        return;
+    }
+
+    // Don't index redirecting sections
+    if section.meta.redirect_to.is_none() {
+        index.add_doc(
+            &section.permalink,
+            &[&section.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(&section.content).to_string()],
+        );
+    }
+
+    for page in &section.pages {
+        if !page.meta.in_search_index || page.meta.draft {
+            continue;
+        }
+
+        index.add_doc(
+            &page.permalink,
+            &[&page.meta.title.clone().unwrap_or(String::new()), &AMMONIA.clean(&page.content).to_string()],
+        );
+    }
+}

components/site/Cargo.toml

@@ -1,12 +1,11 @@
 [package]
 name = "site"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"
 glob = "0.2"
-walkdir = "2"
 rayon = "1"
 serde = "1"
 serde_derive = "1"
@@ -20,6 +19,7 @@ front_matter = { path = "../front_matter" }
 pagination = { path = "../pagination" }
 taxonomies = { path = "../taxonomies" }
 content = { path = "../content" }
+search = { path = "../search" }
 
 [dev-dependencies]
 tempdir = "0.3"

components/site/src/lib.rs

@@ -1,7 +1,6 @@
 extern crate tera;
 extern crate rayon;
 extern crate glob;
-extern crate walkdir;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
@@ -16,23 +15,23 @@ extern crate templates;
 extern crate pagination;
 extern crate taxonomies;
 extern crate content;
+extern crate search;
 
 #[cfg(test)]
 extern crate tempdir;
 
 use std::collections::HashMap;
-use std::fs::{remove_dir_all, copy, create_dir_all};
+use std::fs::{create_dir_all, remove_dir_all, copy};
 use std::mem;
 use std::path::{Path, PathBuf};
 
 use glob::glob;
 use tera::{Tera, Context};
-use walkdir::WalkDir;
 use sass_rs::{Options as SassOptions, OutputStyle, compile_file};
 
 use errors::{Result, ResultExt};
 use config::{Config, get_config};
-use utils::fs::{create_file, create_directory, ensure_directory_exists};
+use utils::fs::{create_file, copy_directory, create_directory, ensure_directory_exists};
 use utils::templates::{render_template, rewrite_theme_paths};
 use content::{Page, Section, populate_previous_and_next_pages, sort_pages};
 use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
@@ -67,7 +66,7 @@ pub struct Site {
     pub sections: HashMap<PathBuf, Section>,
     pub tera: Tera,
     live_reload: bool,
-    output_path: PathBuf,
+    pub output_path: PathBuf,
     pub static_path: PathBuf,
     pub tags: Option<Taxonomy>,
     pub categories: Option<Taxonomy>,
@@ -125,6 +124,11 @@ impl Site {
         Ok(site)
     }
 
+    /// The index section is ALWAYS at that path
+    pub fn index_section_path(&self) -> PathBuf {
+        self.base_path.join("content").join("_index.md")
+    }
+
     /// What the function name says
     pub fn enable_live_reload(&mut self) {
         self.live_reload = true;
@@ -198,7 +202,17 @@ impl Site {
 
         // Insert a default index section if necessary so we don't need to create
         // a _index.md to render the index page
-        let index_path = self.base_path.join("content").join("_index.md");
+        let index_path = self.index_section_path();
+        if let Some(ref index_section) = self.sections.get(&index_path) {
+            if self.config.build_search_index && !index_section.meta.in_search_index {
+                bail!(
+                    "You have enabled search in the config but disabled it in the index section: \
+                    either turn off the search in the config or remote `in_search_index = true` from the \
+                    section front-matter."
+                )
+            }
+        }
+        // Not in else because of borrow checker
         if !self.sections.contains_key(&index_path) {
             let mut index_section = Section::default();
             index_section.permalink = self.config.make_permalink("");
@@ -308,7 +322,7 @@ impl Site {
     /// Defaults to `AnchorInsert::None` if no parent section found
     pub fn find_parent_section_insert_anchor(&self, parent_path: &PathBuf) -> InsertAnchor {
         match self.sections.get(&parent_path.join("_index.md")) {
-            Some(s) => s.meta.insert_anchor_links.unwrap(),
+            Some(s) => s.meta.insert_anchor_links,
             None => InsertAnchor::None
         }
     }
@@ -350,7 +364,7 @@ impl Site {
                     .map(|p| sections[p].clone())
                     .collect::<Vec<_>>();
                 section.subsections
-                    .sort_by(|a, b| a.meta.weight.unwrap().cmp(&b.meta.weight.unwrap()));
+                    .sort_by(|a, b| a.meta.weight.cmp(&b.meta.weight));
             }
         }
     }
@@ -365,7 +379,7 @@ impl Site {
                 }
             }
             let pages = mem::replace(&mut section.pages, vec![]);
-            let (sorted_pages, cannot_be_sorted_pages) = sort_pages(pages, section.meta.sort_by());
+            let (sorted_pages, cannot_be_sorted_pages) = sort_pages(pages, section.meta.sort_by);
             section.pages = populate_previous_and_next_pages(&sorted_pages);
             section.ignored_pages = cannot_be_sorted_pages;
         }
@@ -409,45 +423,18 @@ impl Site {
         html
     }
 
-    /// Copy the file at the given path into the public folder
-    pub fn copy_static_file<P: AsRef<Path>>(&self, path: P, base_path: &PathBuf) -> Result<()> {
-        let relative_path = path.as_ref().strip_prefix(base_path).unwrap();
-        let target_path = self.output_path.join(relative_path);
-        if let Some(parent_directory) = target_path.parent() {
-            create_dir_all(parent_directory)?;
-        }
-        copy(path.as_ref(), &target_path)?;
-        Ok(())
-    }
-
-    /// Copy the content of the given folder into the `public` folder
-    fn copy_static_directory(&self, path: &PathBuf) -> Result<()> {
-        for entry in WalkDir::new(path).into_iter().filter_map(|e| e.ok()) {
-            let relative_path = entry.path().strip_prefix(path).unwrap();
-            let target_path = self.output_path.join(relative_path);
-            if entry.path().is_dir() {
-                if !target_path.exists() {
-                    create_directory(&target_path)?;
-                }
-            } else {
-                let entry_fullpath = self.base_path.join(entry.path());
-                self.copy_static_file(entry_fullpath, path)?;
-            }
-        }
-        Ok(())
-    }
-
     /// Copy the main `static` folder and the theme `static` folder if a theme is used
     pub fn copy_static_directories(&self) -> Result<()> {
         // The user files will overwrite the theme files
         if let Some(ref theme) = self.config.theme {
-            self.copy_static_directory(
-                &self.base_path.join("themes").join(theme).join("static")
+            copy_directory(
+                &self.base_path.join("themes").join(theme).join("static"),
+                &self.output_path
             )?;
         }
         // We're fine with missing static folders
         if self.static_path.exists() {
-            self.copy_static_directory(&self.static_path)?;
+            copy_directory(&self.static_path, &self.output_path)?;
         }
 
         Ok(())
@@ -522,7 +509,32 @@ impl Site {
             self.compile_sass(&self.base_path)?;
         }
 
-        self.copy_static_directories()
+        self.copy_static_directories()?;
+
+        if self.config.build_search_index {
+            self.build_search_index()?;
+        }
+
+        Ok(())
+    }
+
+    pub fn build_search_index(&self) -> Result<()> {
+        // index first
+        create_file(
+            &self.output_path.join(&format!("search_index.{}.js", self.config.default_language)),
+            &format!(
+                "window.searchIndex = {};",
+                search::build_index(&self.sections, &self.config.default_language)?
+            ),
+        )?;
+
+        // then elasticlunr.min.js
+        create_file(
+            &self.output_path.join("elasticlunr.min.js"),
+            search::ELASTICLUNR_JS,
+        )?;
+
+        Ok(())
     }
 
     pub fn compile_sass(&self, base_path: &Path) -> Result<()> {
@@ -585,18 +597,16 @@ impl Site {
 
     pub fn render_aliases(&self) -> Result<()> {
         for page in self.pages.values() {
-            if let Some(ref aliases) = page.meta.aliases {
-                for alias in aliases {
-                    let mut output_path = self.output_path.to_path_buf();
-                    for component in alias.split('/') {
-                        output_path.push(&component);
-
-                        if !output_path.exists() {
-                            create_directory(&output_path)?;
-                        }
+            for alias in &page.meta.aliases {
+                let mut output_path = self.output_path.to_path_buf();
+                for component in alias.split('/') {
+                    output_path.push(&component);
+
+                    if !output_path.exists() {
+                        create_directory(&output_path)?;
                     }
-                    create_file(&output_path.join("index.html"), &render_redirect_template(&page.permalink, &self.tera)?)?;
                 }
+                create_file(&output_path.join("index.html"), &render_redirect_template(&page.permalink, &self.tera)?)?;
             }
         }
         Ok(())
@@ -773,7 +783,7 @@ impl Site {
                 .reduce(|| Ok(()), Result::and)?;
         }
 
-        if !section.meta.should_render() {
+        if !section.meta.render {
             return Ok(());
         }
 
@@ -827,13 +837,8 @@ impl Site {
     pub fn render_paginated(&self, output_path: &Path, section: &Section) -> Result<()> {
         ensure_directory_exists(&self.output_path)?;
 
-        let paginate_path = match section.meta.paginate_path {
-            Some(ref s) => s.clone(),
-            None => unreachable!()
-        };
-
         let paginator = Paginator::new(&section.pages, section);
-        let folder_path = output_path.join(&paginate_path);
+        let folder_path = output_path.join(&section.meta.paginate_path);
         create_directory(&folder_path)?;
 
         paginator

components/site/tests/site.rs

@@ -445,3 +445,21 @@ fn can_build_rss_feed() {
     // Next is posts/python.md
     assert!(file_contains!(public, "rss.xml", "Python in posts"));
 }
+
+
+#[test]
+fn can_build_search_index() {
+    let mut path = env::current_dir().unwrap().parent().unwrap().parent().unwrap().to_path_buf();
+    path.push("test_site");
+    let mut site = Site::new(&path, "config.toml").unwrap();
+    site.load().unwrap();
+    site.config.build_search_index = true;
+    let tmp_dir = TempDir::new("example").expect("create temp dir");
+    let public = &tmp_dir.path().join("public");
+    site.set_output_path(&public);
+    site.build().unwrap();
+
+    assert!(Path::new(&public).exists());
+    assert!(file_exists!(public, "elasticlunr.min.js"));
+    assert!(file_exists!(public, "search_index.en.js"));
+}

components/taxonomies/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "taxonomies"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/templates/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "templates"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 tera = "0.11"

components/utils/Cargo.toml

@@ -1,11 +1,12 @@
 [package]
 name = "utils"
 version = "0.1.0"
-authors = ["Vincent Prouillet <vincent@wearewizards.io>"]
+authors = ["Vincent Prouillet <prouillet.vincent@gmail.com>"]
 
 [dependencies]
 errors = { path = "../errors" }
 tera = "0.11"
+walkdir = "2"
 
 
 [dev-dependencies]

components/utils/src/fs.rs

@@ -1,9 +1,12 @@
 use std::io::prelude::*;
-use std::fs::{File, create_dir_all, read_dir};
+use std::fs::{File, create_dir_all, read_dir, copy};
 use std::path::{Path, PathBuf};
 
+use walkdir::WalkDir;
+
 use errors::{Result, ResultExt};
 
+
 /// Create a file with the content given
 pub fn create_file(path: &Path, content: &str) -> Result<()> {
     let mut file = File::create(&path)?;
@@ -60,6 +63,36 @@ pub fn find_related_assets(path: &Path) -> Vec<PathBuf> {
     assets
 }
 
+/// Copy a file but takes into account where to start the copy as
+/// there might be folders we need to create on the way
+pub fn copy_file(src: &Path, dest: &PathBuf, base_path: &PathBuf) -> Result<()> {
+    let relative_path = src.strip_prefix(base_path).unwrap();
+    let target_path = dest.join(relative_path);
+
+    if let Some(parent_directory) = target_path.parent() {
+        create_dir_all(parent_directory)?;
+    }
+
+    copy(src, target_path)?;
+    Ok(())
+}
+
+pub fn copy_directory(src: &PathBuf, dest: &PathBuf) -> Result<()> {
+    for entry in WalkDir::new(src).into_iter().filter_map(|e| e.ok()) {
+        let relative_path = entry.path().strip_prefix(src).unwrap();
+        let target_path = dest.join(relative_path);
+
+        if entry.path().is_dir() {
+            if !target_path.exists() {
+                create_directory(&target_path)?;
+            }
+        } else {
+            copy_file(entry.path(), dest, src)?;
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use std::fs::File;

components/utils/src/lib.rs

@@ -4,6 +4,7 @@ extern crate errors;
 #[cfg(test)]
 extern crate tempdir;
 extern crate tera;
+extern crate walkdir;
 
 pub mod fs;
 pub mod site;

docs/config.toml

@@ -6,6 +6,7 @@ compile_sass = true
 highlight_code = true
 insert_anchor_links = true
 highlight_theme = "kronuz"
+build_search_index = true
 
 [extra]
 author = "Vincent Prouillet"

docs/content/documentation/content/page.md

@@ -54,6 +54,11 @@ weight = 0
 # current one. This takes an array of path, not URLs.
 aliases = []
 
+# Whether the page should be in the search index. This is only used if
+# `build_search_index` is set to true in the config and the parent section 
+# hasn't set `in_search_index` to false in its front-matter
+in_search_index = true
+
 # Template to use to render this page
 template = "page.html"
 

docs/content/documentation/content/search.md

@@ -0,0 +1,22 @@
++++
+title = "Search"
+weight = 100
++++
+
+Gutenberg can build a search index from the sections and pages content to
+be used by a JavaScript library: [elasticlunr](http://elasticlunr.com/). 
+
+To enable it, you only need to set `build_search_index = true` in your `config.toml` and Gutenberg will
+generate an index for the `default_language` set for all pages not excluded from the search index.
+
+It is very important to set the `default_language` in your `config.toml` if you are writing a site not in 
+English: the index building pipelines are very different depending on the language.
+
+After `gutenberg build` or `gutenberg serve`, you should see two files in your static directory:
+
+- `search_index.${default_language}.js`: so `search_index.en.js` for a default setup
+- `elasticlunr.min.js`
+
+As each site will be different, Gutenberg makes no assumptions about how your search and doesn't provide
+the JavaScript/CSS code to do an actual search and display results. You can however look at how this very site
+is implementing it to have an idea: [search.js](https://github.com/Keats/gutenberg/tree/master/docs/static/search.js).

docs/content/documentation/content/section.md

@@ -52,6 +52,10 @@ paginate_path = "page"
 # Options are "left", "right" and "none"
 insert_anchor_links = "none"
 
+# Whether the section pages should be in the search index. This is only used if
+# `build_search_index` is set to true in the config
+in_search_index = true
+
 # Whether to render that section homepage or not. 
 # Useful when the section is only there to organize things but is not meant
 # to be used directly

docs/content/documentation/getting-started/configuration.md

@@ -51,6 +51,10 @@ generate_categories_pages = false
 # Whether to compile the Sass files found in the `sass` directory
 compile_sass = false
 
+# Whether to build a search index out of the pages and section
+# content for the `default_language`
+build_search_index = false
+
 # A list of glob patterns specifying asset files to ignore when
 # processing the content directory. 
 # Defaults to none, which means all asset files are copied over to the public folder.

docs/sass/_search.scss

@@ -0,0 +1,47 @@
+.search-container {
+  display: inline-block;
+  position: relative;
+  width: 300px;
+
+  input {
+    width: 100%;
+    padding: 0.5rem;
+  }
+}
+
+.search-results {
+  display: none;
+  position: absolute;
+  background: white;
+  color: black;
+  padding: 1rem;
+  box-shadow: 2px 2px 2px 0 rgba(0, 0, 0, 0.5);
+  max-height: 500px;
+  overflow: auto;
+  width: 150%;
+  right: 0;
+
+  &__items {
+    list-style: none;
+  }
+
+  li {
+    margin-top: 1rem;
+    border-bottom: 1px solid #ccc;
+    font-size: 0.9rem;
+
+    &:first-of-type {
+      margin-top: 0;
+    }
+  }
+
+  &__item {
+    margin-bottom: 1rem;
+
+    a {
+      font-size: 1.2rem;
+      display: inline-block;
+      margin-bottom: 0.5rem;
+    }
+  }
+}

docs/sass/site.scss

@@ -16,3 +16,4 @@ $link-color: #007CBC;
 @import "index";
 @import "docs";
 @import "themes";
+@import "search";

docs/static/search.js

@@ -0,0 +1,180 @@
+function debounce(func, wait) {
+  var timeout;
+
+  return function () {
+    var context = this;
+    var args = arguments;
+    clearTimeout(timeout);
+
+    timeout = setTimeout(function () {
+      timeout = null;
+      func.apply(context, args);
+    }, wait);
+  };
+}
+
+// Taken from mdbook
+// The strategy is as follows:
+// First, assign a value to each word in the document:
+//  Words that correspond to search terms (stemmer aware): 40
+//  Normal words: 2
+//  First word in a sentence: 8
+// Then use a sliding window with a constant number of words and count the
+// sum of the values of the words within the window. Then use the window that got the
+// maximum sum. If there are multiple maximas, then get the last one.
+// Enclose the terms in <b>.
+function makeTeaser(body, terms) {
+  var TERM_WEIGHT = 40;
+  var NORMAL_WORD_WEIGHT = 2;
+  var FIRST_WORD_WEIGHT = 8;
+  var TEASER_MAX_WORDS = 30;
+
+  var stemmedTerms = terms.map(function (w) {
+    return elasticlunr.stemmer(w.toLowerCase());
+  });
+  var termFound = false;
+  var index = 0;
+  var weighted = []; // contains elements of ["word", weight, index_in_document]
+
+  // split in sentences, then words
+  var sentences = body.toLowerCase().split(". ");
+
+  for (var i in sentences) {
+    var words = sentences[i].split(" ");
+    var value = FIRST_WORD_WEIGHT;
+
+    for (var j in words) {
+      var word = words[j];
+
+      if (word.length > 0) {
+        for (var k in stemmedTerms) {
+          if (elasticlunr.stemmer(word).startsWith(stemmedTerms[k])) {
+            value = TERM_WEIGHT;
+            termFound = true;
+          }
+        }
+        weighted.push([word, value, index]);
+        value = NORMAL_WORD_WEIGHT;
+      }
+
+      index += word.length;
+      index += 1;  // ' ' or '.' if last word in sentence
+    }
+
+    index += 1;  // because we split at a two-char boundary '. '
+  }
+
+  if (weighted.length === 0) {
+    return body;
+  }
+
+  var windowWeights = [];
+  var windowSize = Math.min(weighted.length, TEASER_MAX_WORDS);
+  // We add a window with all the weights first
+  var curSum = 0;
+  for (var i = 0; i < windowSize; i++) {
+    curSum += weighted[i][1];
+  }
+  windowWeights.push(curSum);
+
+  for (var i = 0; i < weighted.length - windowSize; i++) {
+    curSum -= weighted[i][1];
+    curSum += weighted[i + windowSize][1];
+    windowWeights.push(curSum);
+  }
+
+  // If we didn't find the term, just pick the first window
+  var maxSumIndex = 0;
+  if (termFound) {
+    var maxFound = 0;
+    // backwards
+    for (var i = windowWeights.length - 1; i >= 0; i--) {
+      if (windowWeights[i] > maxFound) {
+        maxFound = windowWeights[i];
+        maxSumIndex = i;
+      }
+    }
+  }
+
+  var teaser = [];
+  var startIndex = weighted[maxSumIndex][2];
+  for (var i = maxSumIndex; i < maxSumIndex + windowSize; i++) {
+    var word = weighted[i];
+    if (startIndex < word[2]) {
+      // missing text from index to start of `word`
+      teaser.push(body.substring(startIndex, word[2]));
+      startIndex = word[2];
+    }
+
+    // add <em/> around search terms
+    if (word[1] === TERM_WEIGHT) {
+      teaser.push("<b>");
+    }
+    startIndex = word[2] + word[0].length;
+    teaser.push(body.substring(word[2], startIndex));
+
+    if (word[1] === TERM_WEIGHT) {
+      teaser.push("</b>");
+    }
+  }
+  teaser.push("…");
+  return teaser.join("");
+}
+
+function formatSearchResultItem(item, terms) {
+  return '<div class="search-results__item">'
+  + `<a href="${item.ref}">${item.doc.title}</a>`
+  + `<div>${makeTeaser(item.doc.body, terms)}</div>`
+  + '</div>';
+}
+
+function initSearch() {
+  var $searchInput = document.getElementById("search");
+  var $searchResults = document.querySelector(".search-results");
+  var $searchResultsItems = document.querySelector(".search-results__items");
+  var MAX_ITEMS = 10;
+
+  var options = {
+    bool: "AND",
+    fields: {
+      title: {boost: 2},
+      body: {boost: 1},
+    }
+  };
+  var currentTerm = "";
+  var index = elasticlunr.Index.load(window.searchIndex);
+
+  $searchInput.addEventListener("keyup", debounce(function() {
+    var term = $searchInput.value.trim();
+    if (term === currentTerm || !index) {
+      return;
+    }
+    $searchResults.style.display = term === "" ? "none" : "block";
+    $searchResultsItems.innerHTML = "";
+    if (term === "") {
+      return;
+    }
+
+    var results = index.search(term, options);
+    if (results.length === 0) {
+      $searchResults.style.display = "none";
+      return;
+    }
+
+    currentTerm = term;
+    for (var i = 0; i < Math.min(results.length, MAX_ITEMS); i++) {
+      var item = document.createElement("li");
+      item.innerHTML = formatSearchResultItem(results[i], term.split(" "));
+      $searchResultsItems.appendChild(item);
+    }
+  }, 150));
+}
+
+
+if (document.readyState === "complete" ||
+    (document.readyState !== "loading" && !document.documentElement.doScroll)
+) {
+  initSearch();
+} else {
+  document.addEventListener("DOMContentLoaded", initSearch);
+}

docs/templates/index.html

@@ -18,6 +18,14 @@
                 <a class="white" href="{{ get_url(path="./documentation/_index.md") }}" class="nav-link">Docs</a>
                 <a class="white" href="{{ get_url(path="./themes/_index.md") }}" class="nav-link">Themes</a>
                 <a class="white" href="https://github.com/Keats/gutenberg" class="nav-link">GitHub</a>
+
+                <div class="search-container">
+                    <input id="search" type="search" placeholder="🔎 Search the docs">
+
+                    <div class="search-results">
+                        <div class="search-results__items"></div>
+                    </div>
+                </div>
             </nav>
         </header>
 
@@ -93,5 +101,9 @@
         <footer>
             ©2017-2018 — <a class="white" href="https://vincent.is">Vincent Prouillet</a> and <a class="white" href="https://github.com/Keats/gutenberg/graphs/contributors">contributors</a>
         </footer>
+
+        <script type="text/javascript" src="{{ get_url(path="elasticlunr.min.js", trailing_slash=false) }}"></script>
+        <script type="text/javascript" src="{{ get_url(path="search_index.en.js", trailing_slash=false) }}"></script>
+        <script type="text/javascript" src="{{ get_url(path="search.js", trailing_slash=false) }}"></script>
     </body>
 </html>

src/cmd/init.rs

@@ -58,7 +58,6 @@ pub fn create_new_project(name: &str) -> Result<()> {
     if compile_sass {
         create_dir(path.join("sass"))?;
     }
-    // TODO: if search == true, copy a lunr js file embedded in gutenberg
 
     println!();
     console::success(&format!("Done! Your site was created in {:?}", canonicalize(path).unwrap()));

src/cmd/serve.rs

@@ -38,6 +38,7 @@ use ctrlc;
 
 use site::Site;
 use errors::{Result, ResultExt};
+use utils::fs::copy_file;
 
 use console;
 use rebuild;
@@ -207,7 +208,7 @@ pub fn serve(interface: &str, port: &str, output_dir: &str, base_url: &str, conf
                             (ChangeKind::StaticFiles, p) => {
                                 if path.is_file() {
                                     console::info(&format!("-> Static file changes detected {}", path.display()));
-                                    rebuild_done_handling(&broadcaster, site.copy_static_file(&path, &site.static_path), &p);
+                                    rebuild_done_handling(&broadcaster, copy_file(&path, &site.output_path, &site.static_path), &p);
                                 }
                             },
                             (ChangeKind::Sass, p) => {