jstrong
/
zola
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Merge pull request #89 from williamyaoh/add-parallel 

Parallelize site rendering using rayon
index-subcmd
Vincent Prouillet GitHub 6 years ago
parent
414457ed92 40cfd6fff8
commit
b45f8c3116
7 changed files with 96 additions and 32 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      CHANGELOG.md
  2. +52
    -0
      Cargo.lock
  3. +1
    -0
      Cargo.toml
  4. +1
    -0
      src/lib.rs
  5. +10
    -0
      src/rendering/highlighting.rs
  6. +9
    -24
      src/rendering/markdown.rs
  7. +22
    -8
      src/site.rs

+ 1
- 0
CHANGELOG.md View File

@@ -2,6 +2,7 @@

## 0.0.8 (unreleased)

- Parallelize page rendering using rayon

## 0.0.7 (2017-06-19)



+ 52
- 0
Cargo.lock View File

@@ -12,6 +12,7 @@ dependencies = [
"mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"notify 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -170,6 +171,15 @@ dependencies = [
"gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "coco"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "conduit-mime-types"
version = "0.7.3"
@@ -192,6 +202,11 @@ name = "dtoa"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "either"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "error"
version = "0.1.9"
@@ -249,6 +264,11 @@ dependencies = [
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "futures"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "gcc"
version = "0.3.51"
@@ -629,6 +649,27 @@ dependencies = [
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "rayon"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "rayon-core"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.24 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "redox_syscall"
version = "0.1.18"
@@ -675,6 +716,11 @@ dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "scopeguard"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "sequence_trie"
version = "0.2.1"
@@ -1073,9 +1119,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum chrono 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d9123be86fd2a8f627836c235ecdf331fdd067ecf7ac05aa1a68fbcf2429f056"
"checksum clap 2.24.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8f69e518f967224e628896b54e41ff6acfb4dcfefc5076325c36525dac900f"
"checksum cmake 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "b8ebbb35d3dc9cd09497168f33de1acb79b265d350ab0ac34133b98f8509af1f"
"checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd"
"checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90"
"checksum either 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18785c1ba806c258137c937e44ada9ee7e69a37e3c72077542cd2f069d78562a"
"checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc"
"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
@@ -1083,6 +1131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum fnv 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc484842f1e2884faf56f529f960cc12ad8c71ce96cc7abba0a067c98fee344"
"checksum fsevent 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfe593ebcfc76884138b25426999890b10da8e6a46d01b499d7c54c604672c38"
"checksum fsevent-sys 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a772d36c338d07a032d5375a36f15f9a7043bf0cb8ce7cee658e037c6032874"
"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a"
"checksum gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)" = "120d07f202dcc3f72859422563522b66fe6463a4c513df062874daad05f85f0a"
"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
"checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb"
@@ -1128,6 +1177,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum pulldown-cmark 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9ab1e588ef8efd702c7ed9d2bd774db5e6f4d878bb5a1a9f371828fbdff6973"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum rayon 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "705cf28d52a26a9ab548930a9a3d9799eb77cf84d66d7cc6e52fa222ca662424"
"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493"
"checksum redox_syscall 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "3041aeb6000db123d2c9c751433f526e1f404b23213bd733167ab770c3989b4d"
"checksum regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1731164734096285ec2a5ec7fea5248ae2f5485b3feeb0115af4fda2183b2d1b"
"checksum regex-syntax 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad890a5eef7953f55427c50575c680c42841653abd2b028b68cd223d157f62db"
@@ -1135,6 +1186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
"checksum same-file 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d931a44fdaa43b8637009e7632a02adc4f2b2e0733c08caa4cf00e8da4a117a7"
"checksum scopeguard 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c79eb2c3ac4bc2507cda80e7f3ac5b88bd8eae4c0914d5663e6a8933994be918"
"checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed"
"checksum serde 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f530d36fb84ec48fb7146936881f026cdbf4892028835fd9398475f82c1bb4"
"checksum serde_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "10552fad5500771f3902d0c5ba187c5881942b811b7ba0d8fbbfbf84d80806d3"


+ 1
- 0
Cargo.toml View File

@@ -31,6 +31,7 @@ chrono = "0.3"
toml = "0.4"
term-painter = "0.2"
base64 = "0.6"
rayon = "0.8"

# Below is for the serve cmd
staticfile = "0.4"


+ 1
- 0
src/lib.rs View File

@@ -16,6 +16,7 @@ extern crate syntect;
extern crate slug;
extern crate chrono;
extern crate base64;
extern crate rayon;
#[cfg(test)]
extern crate tempdir;



+ 10
- 0
src/rendering/highlighting.rs View File

@@ -1,6 +1,16 @@
use syntect::dumps::from_binary;
use syntect::parsing::SyntaxSet;
use syntect::highlighting::ThemeSet;

thread_local!{
pub static SYNTAX_SET: SyntaxSet = {
let mut ss: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
ss.link_syntaxes();
ss
};
}

lazy_static!{
pub static ref THEME_SET: ThemeSet = from_binary(include_bytes!("../../sublime_themes/all.themedump"));
}


+ 9
- 24
src/rendering/markdown.rs View File

@@ -4,9 +4,7 @@ use pulldown_cmark as cmark;
use self::cmark::{Parser, Event, Tag, Options, OPTION_ENABLE_TABLES, OPTION_ENABLE_FOOTNOTES};
use regex::Regex;
use slug::slugify;
use syntect::dumps::from_binary;
use syntect::easy::HighlightLines;
use syntect::parsing::SyntaxSet;
use syntect::html::{start_coloured_html_snippet, styles_to_coloured_html, IncludeBackground};
use tera::{Context as TeraContext};

@@ -14,27 +12,12 @@ use errors::{Result};
use site::resolve_internal_link;
use front_matter::InsertAnchor;
use rendering::context::Context;
use rendering::highlighting::THEME_SET;
use rendering::highlighting::{SYNTAX_SET, THEME_SET};
use rendering::short_code::{ShortCode, parse_shortcode, render_simple_shortcode};
use content::{TempHeader, Header, make_table_of_contents};

// We need to put those in a struct to impl Send and sync
pub struct Setup {
pub syntax_set: SyntaxSet,
}

unsafe impl Send for Setup {}
unsafe impl Sync for Setup {}

lazy_static!{
static ref SHORTCODE_RE: Regex = Regex::new(r#"\{(?:%|\{)\s+([[:alnum:]]+?)\(([[:alnum:]]+?="?.+?"?)\)\s+(?:%|\})\}"#).unwrap();
pub static ref SETUP: Setup = Setup {
syntax_set: {
let mut ps: SyntaxSet = from_binary(include_bytes!("../../sublime_syntaxes/newlines.packdump"));
ps.link_syntaxes();
ps
},
};
}


@@ -190,12 +173,14 @@ pub fn markdown_to_html(content: &str, context: &Context) -> Result<(String, Vec
return Event::Html(Owned("<pre><code>".to_owned()));
}
let theme = &THEME_SET.themes[&context.highlight_theme];
let syntax = info
.split(' ')
.next()
.and_then(|lang| SETUP.syntax_set.find_syntax_by_token(lang))
.unwrap_or_else(|| SETUP.syntax_set.find_syntax_plain_text());
highlighter = Some(HighlightLines::new(syntax, theme));
highlighter = SYNTAX_SET.with(|ss| {
let syntax = info
.split(' ')
.next()
.and_then(|lang| ss.find_syntax_by_token(lang))
.unwrap_or_else(|| ss.find_syntax_plain_text());
Some(HighlightLines::new(syntax, theme))
});
let snippet = start_coloured_html_snippet(theme);
Event::Html(Owned(snippet))
},


+ 22
- 8
src/site.rs View File

@@ -13,6 +13,7 @@ use content::{Page, Section, Paginator, SortBy, Taxonomy, populate_previous_and_
use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};
use front_matter::InsertAnchor;

use rayon::prelude::*;

#[derive(Debug)]
pub struct Site {
@@ -118,14 +119,27 @@ impl Site {
pages_insert_anchors.insert(page.file.path.clone(), self.find_parent_section_insert_anchor(&page.file.parent.clone()));
}

// TODO: make that parallel
for page in self.pages.values_mut() {
let insert_anchor = pages_insert_anchors[&page.file.path];
page.render_markdown(&self.permalinks, &self.tera, &self.config, insert_anchor)?;
}
// TODO: make that parallel
for section in self.sections.values_mut() {
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
{
// Another silly thing needed to not borrow &self in parallel and
// make the borrow checker happy
let permalinks = &self.permalinks;
let tera = &self.tera;
let config = &self.config;

self.pages.par_iter_mut()
.map(|(_, page)| page)
.map(|page| {
let insert_anchor = pages_insert_anchors[&page.file.path];
page.render_markdown(&permalinks, &tera, &config, insert_anchor)
})
.fold(|| Ok(()), Result::and)
.reduce(|| Ok(()), Result::and)?;

self.sections.par_iter_mut()
.map(|(_, section)| section)
.map(|section| section.render_markdown(permalinks, tera, config))
.fold(|| Ok(()), Result::and)
.reduce(|| Ok(()), Result::and)?;
}

self.populate_sections();


Write Preview
Loading…
Cancel
Save