Browse Source

Smarter rebuild on content change

Fix #59
index-subcmd
Vincent Prouillet 7 years ago
parent
commit
09d5e74a65
8 changed files with 383 additions and 144 deletions
  1. +4
    -1
      CHANGELOG.md
  2. +3
    -2
      src/bin/cmd/serve.rs
  3. +1
    -0
      src/bin/gutenberg.rs
  4. +230
    -0
      src/bin/rebuild.rs
  5. +15
    -2
      src/front_matter/page.rs
  6. +1
    -8
      src/page.rs
  7. +17
    -0
      src/section.rs
  8. +112
    -131
      src/site.rs

+ 4
- 1
CHANGELOG.md View File

@@ -4,11 +4,14 @@

- Fix XML templates overriding and reloading
- `title` and `description` are now optional in the front matter
- Add GenericConfig, Vim syntax
- Add GenericConfig, Vim, Jinja2 syntax
- Add `_index.md` for homepage as well and make that into a normal section
- Allow sorting by `none`, `date` and `order` for sections
- Add pagination
- Add a `get_page` global function to tera
- Revamp index page, no more `pages` variables
- Fix livereload stopping randomly
- Smarter re-rendering in `serve` command

## 0.0.4 (2017-04-23)



+ 3
- 2
src/bin/cmd/serve.rs View File

@@ -14,6 +14,7 @@ use gutenberg::Site;
use gutenberg::errors::{Result, ResultExt};

use console;
use rebuild;

#[derive(Debug, PartialEq)]
enum ChangeKind {
@@ -137,12 +138,12 @@ pub fn serve(interface: &str, port: &str, config_file: &str) -> Result<()> {
(ChangeKind::Content, _) => {
console::info(&format!("-> Content changed {}", path.display()));
// Force refresh
rebuild_done_handling(&broadcaster, site.rebuild_after_content_change(&path), "/x.js");
rebuild_done_handling(&broadcaster, rebuild::after_content_change(&mut site, &path), "/x.js");
},
(ChangeKind::Templates, _) => {
console::info(&format!("-> Template changed {}", path.display()));
// Force refresh
rebuild_done_handling(&broadcaster, site.rebuild_after_template_change(&path), "/x.js");
rebuild_done_handling(&broadcaster, rebuild::after_template_change(&mut site, &path), "/x.js");
},
(ChangeKind::StaticFiles, p) => {
if path.is_file() {


+ 1
- 0
src/bin/gutenberg.rs View File

@@ -16,6 +16,7 @@ use std::time::Instant;

mod cmd;
mod console;
mod rebuild;


fn main() {


+ 230
- 0
src/bin/rebuild.rs View File

@@ -0,0 +1,230 @@
use std::path::Path;

use gutenberg::{Site, SectionFrontMatter, PageFrontMatter};
use gutenberg::errors::Result;

#[derive(Debug, Clone, Copy, PartialEq)]
enum PageChangesNeeded {
/// Editing `tags`
Tags,
/// Editing `categories`
Categories,
/// Editing `date` or `order`
Sort,
/// Editing anything else
Render,
}

// TODO: seems like editing sort_by/render do weird stuff
#[derive(Debug, Clone, Copy, PartialEq)]
enum SectionChangesNeeded {
/// Editing `sort_by`
Sort,
/// Editing `title`, `description`, `extra`, `template` or setting `render` to true
Render,
/// Editing `paginate_by` or `paginate_path`
RenderWithPages,
/// Setting `render` to false
Delete,
}

/// Evaluates all the params in the front matter that changed so we can do the smallest
/// delta in the serve command
fn find_section_front_matter_changes(current: &SectionFrontMatter, other: &SectionFrontMatter) -> Vec<SectionChangesNeeded> {
let mut changes_needed = vec![];

if current.sort_by != other.sort_by {
changes_needed.push(SectionChangesNeeded::Sort);
}

if !current.should_render() && other.should_render() {
changes_needed.push(SectionChangesNeeded::Delete);
// Nothing else we can do
return changes_needed;
}

if current.paginate_by != other.paginate_by || current.paginate_path != other.paginate_path {
changes_needed.push(SectionChangesNeeded::RenderWithPages);
// Nothing else we can do
return changes_needed;
}

// Any other change will trigger a re-rendering of the section page only
changes_needed.push(SectionChangesNeeded::Render);
changes_needed
}

/// Evaluates all the params in the front matter that changed so we can do the smallest
/// delta in the serve command
fn find_page_front_matter_changes(current: &PageFrontMatter, other: &PageFrontMatter) -> Vec<PageChangesNeeded> {
let mut changes_needed = vec![];

if current.tags != other.tags {
changes_needed.push(PageChangesNeeded::Tags);
}

if current.category != other.category {
changes_needed.push(PageChangesNeeded::Categories);
}

if current.date != other.date || current.order != other.order {
changes_needed.push(PageChangesNeeded::Sort);
}

changes_needed.push(PageChangesNeeded::Render);
changes_needed
}

// What happens when a section or a page is changed
pub fn after_content_change(site: &mut Site, path: &Path) -> Result<()> {
let is_section = path.file_name().unwrap() == "_index.md";

// A page or section got deleted
if !path.exists() {
if is_section {
// A section was deleted, many things can be impacted:
// - the pages of the section are becoming orphans
// - any page that was referencing the section (index, etc)
let relative_path = site.sections[path].relative_path.clone();
// Remove the link to it and the section itself from the Site
site.permalinks.remove(&relative_path);
site.sections.remove(path);
site.populate_sections();
} else {
// A page was deleted, many things can be impacted:
// - the section the page is in
// - any page that was referencing the section (index, etc)
let relative_path = site.pages[path].relative_path.clone();
site.permalinks.remove(&relative_path);
match site.pages.remove(path) {
Some(p) => {
if p.meta.has_tags() || p.meta.category.is_some() {
site.populate_tags_and_categories();
}

if site.find_parent_section(&p).is_some() {
site.populate_sections();
}
},
None => ()
};
}
// Deletion is something that doesn't happen all the time so we
// don't need to optimise it too much
return site.build();
}

// A section was edited
if is_section {
match site.add_section(path, true)? {
Some(prev) => {
// Updating a section
let current_meta = site.sections[path].meta.clone();
// Front matter didn't change, only content did
// so we render only the section page, not its pages
if current_meta == prev.meta {
return site.render_section(&site.sections[path], false);
}

// Front matter changed
for changes in find_section_front_matter_changes(&current_meta, &prev.meta) {
// Sort always comes first if present so the rendering will be fine
match changes {
SectionChangesNeeded::Sort => site.sort_sections_pages(Some(path)),
SectionChangesNeeded::Render => site.render_section(&site.sections[path], false)?,
SectionChangesNeeded::RenderWithPages => site.render_section(&site.sections[path], true)?,
// can't be arsed to make the Delete efficient, it's not a common enough operation
SectionChangesNeeded::Delete => {
site.populate_sections();
site.build()?;
},
};
}
return Ok(());
},
None => {
// New section, only render that one
site.populate_sections();
return site.render_section(&site.sections[path], true);
}
};
}

// A page was edited
match site.add_page(path, true)? {
Some(prev) => {
// Updating a page
let current = site.pages[path].clone();
// Front matter didn't change, only content did
// so we render only the section page, not its pages
if current.meta == prev.meta {
return site.render_page(&site.pages[path]);
}

// Front matter changed
for changes in find_page_front_matter_changes(&current.meta, &prev.meta) {
// Sort always comes first if present so the rendering will be fine
match changes {
PageChangesNeeded::Tags => {
site.populate_tags_and_categories();
site.render_tags()?;
},
PageChangesNeeded::Categories => {
site.populate_tags_and_categories();
site.render_categories()?;
},
PageChangesNeeded::Sort => {
let section_path = match site.find_parent_section(&site.pages[path]) {
Some(s) => s.file_path.clone(),
None => continue // Do nothing if it's an orphan page
};
site.populate_sections();
site.sort_sections_pages(Some(&section_path));
site.render_index()?;
},
PageChangesNeeded::Render => {
site.render_page(&site.pages[path])?;
},
};
}
return Ok(());

},
None => {
// It's a new page!
site.populate_sections();
site.populate_tags_and_categories();
// No need to optimise that yet, we can revisit if it becomes an issue
site.build()?;
}
}

Ok(())
}

/// What happens when a template is changed
pub fn after_template_change(site: &mut Site, path: &Path) -> Result<()> {
site.tera.full_reload()?;

match path.file_name().unwrap().to_str().unwrap() {
"sitemap.xml" => site.render_sitemap(),
"rss.xml" => site.render_rss_feed(),
"robots.txt" => site.render_robots(),
"categories.html" | "category.html" => site.render_categories(),
"tags.html" | "tag.html" => site.render_tags(),
"page.html" => {
site.render_sections()?;
site.render_orphan_pages()
},
"section.html" => site.render_sections(),
// Either the index or some unknown template changed
// We can't really know what this change affects so rebuild all
// the things
_ => {
site.render_sections()?;
site.render_orphan_pages()?;
site.render_categories()?;
site.render_tags()
},
}
}

+ 15
- 2
src/front_matter/page.rs View File

@@ -26,11 +26,11 @@ pub struct PageFrontMatter {
pub tags: Option<Vec<String>>,
/// Whether this page is a draft and should be published or not
pub draft: Option<bool>,
/// Only one category allowed
/// Only one category allowed. Can't be an empty string if present
pub category: Option<String>,
/// Integer to use to order content. Lowest is at the bottom, highest first
pub order: Option<usize>,
/// Optional template, if we want to specify which template to render for that page
/// Specify a template different from `page.html` to use for that page
#[serde(skip_serializing)]
pub template: Option<String>,
/// Any extra parameter present in the front matter
@@ -56,6 +56,12 @@ impl PageFrontMatter {
}
}

if let Some(ref category) = f.category {
if category == "" {
bail!("`category` can't be empty if present")
}
}

Ok(f)
}

@@ -76,6 +82,13 @@ impl PageFrontMatter {
pub fn order(&self) -> usize {
self.order.unwrap()
}

pub fn has_tags(&self) -> bool {
match self.tags {
Some(ref t) => !t.is_empty(),
None => false
}
}
}

impl Default for PageFrontMatter {


+ 1
- 8
src/page.rs View File

@@ -297,14 +297,7 @@ pub fn sort_pages(pages: Vec<Page>, sort_by: SortBy) -> (Vec<Page>, Vec<Page>) {

(can_be_sorted, cannot_be_sorted)
},
SortBy::None => {
let mut p = vec![];
for page in pages {
p.push(page);
}

(p, vec![])
},
SortBy::None => (pages, vec![])
}
}



+ 17
- 0
src/section.rs View File

@@ -134,6 +134,23 @@ impl Section {
paths.extend(self.ignored_pages.iter().map(|p| p.file_path.clone()));
paths
}

/// Whether the page given belongs to that section
pub fn is_child_page(&self, page: &Page) -> bool {
for p in &self.pages {
if p.file_path == page.file_path {
return true;
}
}

for p in &self.ignored_pages {
if p.file_path == page.file_path {
return true;
}
}

false
}
}

impl ser::Serialize for Section {


+ 112
- 131
src/site.rs View File

@@ -18,8 +18,6 @@ use front_matter::{SortBy};
use templates::{GUTENBERG_TERA, global_fns, render_redirect_template};




#[derive(Debug, PartialEq)]
enum RenderList {
Tags,
@@ -56,6 +54,8 @@ pub struct Site {
static_path: PathBuf,
pub tags: HashMap<String, Vec<PathBuf>>,
pub categories: HashMap<String, Vec<PathBuf>>,
/// A map of all .md files (section and pages) and their permalink
/// We need that if there are relative links in the content that need to be resolved
pub permalinks: HashMap<String, String>,
}

@@ -92,6 +92,7 @@ impl Site {
}

/// Gets the path of all ignored pages in the site
/// Used for reporting them in the CLI
pub fn get_ignored_pages(&self) -> Vec<PathBuf> {
self.sections
.values()
@@ -117,6 +118,17 @@ impl Site {
orphans
}

/// Finds the section that contains the page given if there is one
pub fn find_parent_section(&self, page: &Page) -> Option<&Section> {
for section in self.sections.values() {
if section.is_child_page(page) {
return Some(section)
}
}

None
}

/// Used by tests to change the output path to a tmp dir
#[doc(hidden)]
pub fn set_output_path<P: AsRef<Path>>(&mut self, path: P) {
@@ -132,9 +144,9 @@ impl Site {
for entry in glob(&content_glob).unwrap().filter_map(|e| e.ok()) {
let path = entry.as_path();
if path.file_name().unwrap() == "_index.md" {
self.add_section(path)?;
self.add_section(path, false)?;
} else {
self.add_page(path)?;
self.add_page(path, false)?;
}
}
// Insert a default index section so we don't need to create a _index.md to render
@@ -146,27 +158,15 @@ impl Site {
self.sections.insert(index_path, index_section);
}

// A map of all .md files (section and pages) and their permalink
// We need that if there are relative links in the content that need to be resolved
let mut permalinks = HashMap::new();

for page in self.pages.values() {
permalinks.insert(page.relative_path.clone(), page.permalink.clone());
}

for section in self.sections.values() {
permalinks.insert(section.relative_path.clone(), section.permalink.clone());
}

// TODO: make that parallel
for page in self.pages.values_mut() {
page.render_markdown(&permalinks, &self.tera, &self.config)?;
page.render_markdown(&self.permalinks, &self.tera, &self.config)?;
}
// TODO: make that parallel
for section in self.sections.values_mut() {
section.render_markdown(&permalinks, &self.tera, &self.config)?;
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
}

self.permalinks = permalinks;
self.populate_sections();
self.populate_tags_and_categories();

@@ -175,73 +175,82 @@ impl Site {
Ok(())
}

/// Simple wrapper fn to avoid repeating that code in several places
fn add_page(&mut self, path: &Path) -> Result<()> {
/// Add a page to the site
/// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page
/// Returns the previous page struct if there was one
pub fn add_page(&mut self, path: &Path, render: bool) -> Result<Option<Page>> {
let page = Page::from_file(&path, &self.config)?;
self.pages.insert(page.file_path.clone(), page);
Ok(())
}
self.permalinks.insert(page.relative_path.clone(), page.permalink.clone());
let prev = self.pages.insert(page.file_path.clone(), page);

/// Simple wrapper fn to avoid repeating that code in several places
fn add_section(&mut self, path: &Path) -> Result<()> {
let section = Section::from_file(path, &self.config)?;
self.sections.insert(section.file_path.clone(), section);
Ok(())
}
if render {
let mut page = self.pages.get_mut(path).unwrap();
page.render_markdown(&self.permalinks, &self.tera, &self.config)?;
}

/// Called in serve, add the section and render it
fn add_section_and_render(&mut self, path: &Path) -> Result<()> {
self.add_section(path)?;
let mut section = self.sections.get_mut(path).unwrap();
self.permalinks.insert(section.relative_path.clone(), section.permalink.clone());
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
Ok(())
Ok(prev)
}

/// Called in serve, add a page again updating permalinks and its content
/// The bool in the result is whether the front matter has been updated or not
/// TODO: the above is very confusing, change that
fn add_page_and_render(&mut self, path: &Path) -> Result<(bool, Page)> {
let existing_page = self.pages.get(path).cloned();
self.add_page(path)?;
let mut page = self.pages.get_mut(path).unwrap();
self.permalinks.insert(page.relative_path.clone(), page.permalink.clone());
page.render_markdown(&self.permalinks, &self.tera, &self.config)?;
/// Add a section to the site
/// The `render` parameter is used in the serve command, when rebuilding a page.
/// If `true`, it will also render the markdown for that page
/// Returns the previous page struct if there was one
pub fn add_section(&mut self, path: &Path, render: bool) -> Result<Option<Section>> {
let section = Section::from_file(path, &self.config)?;
self.permalinks.insert(section.relative_path.clone(), section.permalink.clone());
let prev = self.sections.insert(section.file_path.clone(), section);

if let Some(prev_page) = existing_page {
return Ok((prev_page.meta != page.meta, page.clone()));
if render {
let mut section = self.sections.get_mut(path).unwrap();
section.render_markdown(&self.permalinks, &self.tera, &self.config)?;
}
Ok((true, page.clone()))

Ok(prev)
}

/// Find out the direct subsections of each subsection if there are some
/// as well as the pages for each section
pub fn populate_sections(&mut self) {
for page in self.pages.values() {
if self.sections.contains_key(&page.parent_path.join("_index.md")) {
self.sections.get_mut(&page.parent_path.join("_index.md")).unwrap().pages.push(page.clone());
}
}

let mut grandparent_paths = HashMap::new();
for section in self.sections.values() {
for section in self.sections.values_mut() {
if let Some(grand_parent) = section.parent_path.parent() {
grandparent_paths.entry(grand_parent.to_path_buf()).or_insert_with(|| vec![]).push(section.clone());
}
// Make sure the pages of a section are empty since we can call that many times on `serve`
section.pages = vec![];
section.ignored_pages = vec![];
}

for section in self.sections.values_mut() {
// TODO: avoid this clone
let (mut sorted_pages, cannot_be_sorted_pages) = sort_pages(section.pages.clone(), section.meta.sort_by());
sorted_pages = populate_previous_and_next_pages(&sorted_pages);
section.pages = sorted_pages;
section.ignored_pages = cannot_be_sorted_pages;
for page in self.pages.values() {
if self.sections.contains_key(&page.parent_path.join("_index.md")) {
self.sections.get_mut(&page.parent_path.join("_index.md")).unwrap().pages.push(page.clone());
}
}

for section in self.sections.values_mut() {
match grandparent_paths.get(&section.parent_path) {
Some(paths) => section.subsections.extend(paths.clone()),
None => continue,
};
}

self.sort_sections_pages(None);
}

/// Sorts the pages of the section at the given path
/// By default will sort all sections but can be made to only sort a single one by providing a path
pub fn sort_sections_pages(&mut self, only: Option<&Path>) {
for (path, section) in self.sections.iter_mut() {
if let Some(p) = only {
if p != path {
continue;
}
}
let (sorted_pages, cannot_be_sorted_pages) = sort_pages(section.pages.clone(), section.meta.sort_by());
section.pages = populate_previous_and_next_pages(&sorted_pages);
section.ignored_pages = cannot_be_sorted_pages;
}
}

/// Separated from `parse` for easier testing
@@ -277,7 +286,7 @@ impl Site {
html
}

pub fn ensure_public_directory_exists(&self) -> Result<()> {
fn ensure_public_directory_exists(&self) -> Result<()> {
let public = self.output_path.clone();
if !public.exists() {
create_directory(&public)?;
@@ -324,58 +333,6 @@ impl Site {
Ok(())
}

pub fn rebuild_after_content_change(&mut self, path: &Path) -> Result<()> {
let is_section = path.ends_with("_index.md");

if path.exists() {
// file exists, either a new one or updating content
if is_section {
self.add_section_and_render(path)?;
self.render_sections()?;
} else {
// probably just an update so just re-parse that page
let (frontmatter_changed, page) = self.add_page_and_render(path)?;
// TODO: can probably be smarter and check what changed
if frontmatter_changed {
self.populate_sections();
self.populate_tags_and_categories();
self.build()?;
} else {
self.render_page(&page)?;
}
}
} else {
// File doesn't exist -> a deletion so we remove it from everything
let relative_path = if is_section {
self.sections[path].relative_path.clone()
} else {
self.pages[path].relative_path.clone()
};
self.permalinks.remove(&relative_path);

if is_section {
self.sections.remove(path);
} else {
self.pages.remove(path);
}
// TODO: probably no need to do that, we should be able to only re-render a page or a section.
self.populate_sections();
self.populate_tags_and_categories();
self.build()?;
}

Ok(())
}

pub fn rebuild_after_template_change(&mut self, path: &Path) -> Result<()> {
self.tera.full_reload()?;
match path.file_name().unwrap().to_str().unwrap() {
"sitemap.xml" => self.render_sitemap(),
"rss.xml" => self.render_rss_feed(),
_ => self.build() // TODO: change that
}
}

/// Renders a single content page
pub fn render_page(&self, page: &Page) -> Result<()> {
self.ensure_public_directory_exists()?;
@@ -417,18 +374,16 @@ impl Site {
self.render_rss_feed()?;
}
self.render_robots()?;
if self.config.generate_categories_pages.unwrap() {
self.render_categories_and_tags(RenderList::Categories)?;
}
if self.config.generate_tags_pages.unwrap() {
self.render_categories_and_tags(RenderList::Tags)?;
}
// `render_categories` and `render_tags` will check whether the config allows
// them to render or not
self.render_categories()?;
self.render_tags()?;

self.copy_static_directory()
}

/// Renders robots.txt
fn render_robots(&self) -> Result<()> {
pub fn render_robots(&self) -> Result<()> {
self.ensure_public_directory_exists()?;
create_file(
self.output_path.join("robots.txt"),
@@ -436,8 +391,27 @@ impl Site {
)
}

/// Renders all categories if the config allows it
pub fn render_categories(&self) -> Result<()> {
if self.config.generate_categories_pages.unwrap() {
self.render_categories_and_tags(RenderList::Categories)
} else {
Ok(())
}
}

/// Renders all tags if the config allows it
pub fn render_tags(&self) -> Result<()> {
if self.config.generate_tags_pages.unwrap() {
self.render_categories_and_tags(RenderList::Tags)
} else {
Ok(())
}
}

/// Render the /{categories, list} pages and each individual category/tag page
/// They are the same thing fundamentally, a list of pages with something in common
/// TODO: revisit this function, lots of things have changed since then
fn render_categories_and_tags(&self, kind: RenderList) -> Result<()> {
let items = match kind {
RenderList::Categories => &self.categories,
@@ -509,7 +483,8 @@ impl Site {
Ok(())
}

fn render_sitemap(&self) -> Result<()> {
/// What it says on the tin
pub fn render_sitemap(&self) -> Result<()> {
self.ensure_public_directory_exists()?;
let mut context = Context::new();
context.add("pages", &self.pages.values().collect::<Vec<&Page>>());
@@ -544,7 +519,7 @@ impl Site {
Ok(())
}

fn render_rss_feed(&self) -> Result<()> {
pub fn render_rss_feed(&self) -> Result<()> {
self.ensure_public_directory_exists()?;

let mut context = Context::new();
@@ -587,7 +562,7 @@ impl Site {
}

/// Renders a single section
fn render_section(&self, section: &Section) -> Result<()> {
pub fn render_section(&self, section: &Section, render_pages: bool) -> Result<()> {
self.ensure_public_directory_exists()?;
let public = self.output_path.clone();

@@ -600,8 +575,10 @@ impl Site {
}
}

for page in &section.pages {
self.render_page(page)?;
if render_pages {
for page in &section.pages {
self.render_page(page)?;
}
}

if !section.meta.should_render() {
@@ -622,16 +599,20 @@ impl Site {
Ok(())
}

pub fn render_index(&self) -> Result<()> {
self.render_section(&self.sections[&self.base_path.join("content").join("_index.md")], false)
}

/// Renders all sections
fn render_sections(&self) -> Result<()> {
pub fn render_sections(&self) -> Result<()> {
for section in self.sections.values() {
self.render_section(section)?;
self.render_section(section, true)?;
}
Ok(())
}

/// Renders all pages that do not belong to any sections
fn render_orphan_pages(&self) -> Result<()> {
pub fn render_orphan_pages(&self) -> Result<()> {
self.ensure_public_directory_exists()?;

for page in self.get_all_orphan_pages() {


Loading…
Cancel
Save