Browse Source

update syntect to v5.0

update-syntect
Jonathan Strong 5 months ago
parent
commit
86985bf2d2
10 changed files with 1659 additions and 1156 deletions
  1. +1640
    -1144
      Cargo.lock
  2. +3
    -3
      Cargo.toml
  3. +1
    -1
      components/config/Cargo.toml
  4. +4
    -2
      components/config/src/highlighting.rs
  5. BIN
      components/config/sublime/all.themedump
  6. BIN
      components/config/sublime/newlines.packdump
  7. +1
    -1
      components/errors/Cargo.toml
  8. +1
    -1
      components/rendering/Cargo.toml
  9. +1
    -1
      components/rendering/src/markdown.rs
  10. +8
    -3
      components/search/src/lib.rs

+ 1640
- 1144
Cargo.lock
File diff suppressed because it is too large
View File


+ 3
- 3
Cargo.toml View File

@@ -62,6 +62,6 @@ members = [
[features]
tantivy-indexing = ["tantivy", "search/tantivy-indexing"]

[profile.release]
lto = true
codegen-units = 1
#[profile.release]
#lto = true
#codegen-units = 1

+ 1
- 1
components/config/Cargo.toml View File

@@ -11,7 +11,7 @@ serde_derive = "1"
chrono = "0.4"
globset = "0.4"
lazy_static = "1"
syntect = "=3.2.0"
syntect = "5.0"

errors = { path = "../errors" }
utils = { path = "../utils" }

+ 4
- 2
components/config/src/highlighting.rs View File

@@ -9,11 +9,13 @@ use crate::config::Config;
lazy_static! {
pub static ref SYNTAX_SET: SyntaxSet = {
let ss: SyntaxSet =
from_binary(include_bytes!("../../../sublime_syntaxes/newlines.packdump"));
//from_binary(include_bytes!("../../../sublime_syntaxes/newlines.packdump"));
from_binary(include_bytes!("../sublime/newlines.packdump"));
ss
};
pub static ref THEME_SET: ThemeSet =
from_binary(include_bytes!("../../../sublime_themes/all.themedump"));
//from_binary(include_bytes!("../../../sublime_themes/all.themedump"));
from_binary(include_bytes!("../sublime/all.themedump"));
}

/// Returns the highlighter and whether it was found in the extra or not


BIN
components/config/sublime/all.themedump View File


BIN
components/config/sublime/newlines.packdump View File


+ 1
- 1
components/errors/Cargo.toml View File

@@ -8,4 +8,4 @@ edition = "2018"
tera = "1"
toml = "0.5"
image = "0.23"
syntect = "=3.2.0"
syntect = "5.0"

+ 1
- 1
components/rendering/Cargo.toml View File

@@ -6,7 +6,7 @@ edition = "2018"

[dependencies]
tera = { version = "1", features = ["preserve_order"] }
syntect = "=3.2.0"
syntect = "5.0"
pulldown-cmark = "0.7"
serde = "1"
serde_derive = "1"


+ 1
- 1
components/rendering/src/markdown.rs View File

@@ -205,7 +205,7 @@ pub fn markdown_to_html(content: &str, context: &RenderContext) -> Result<Render
highlighter.highlight(&text, &SYNTAX_SET)
};
//let highlighted = &highlighter.highlight(&text, ss);
let html = styled_line_to_highlighted_html(&highlighted, background);
let html = styled_line_to_highlighted_html(&highlighted, background).unwrap();
return Event::Html(html.into());
}



+ 8
- 3
components/search/src/lib.rs View File

@@ -1,9 +1,10 @@
#[allow(unused_imports)]
use std::str::FromStr;
use std::collections::{HashMap, HashSet};
use chrono::{DateTime, Utc, NaiveDateTime, TimeZone};
use elasticlunr::{Index, Language};
use lazy_static::lazy_static;

#[cfg(feature = "tantivy-indexing")]
use chrono::{DateTime, Utc, NaiveDateTime, TimeZone};
#[allow(unused_imports)]
use errors::{bail, Result, Error};
use library::{Library, Section};
@@ -105,7 +106,7 @@ fn parse_language(lang: &str) -> Option<tantivy::tokenizer::Language> {
}),

// apparently not a code, so this is best available option
_ => serde_json::from_str::<tantivy::tokenizer::Language>(lang).ok()
_ => serde_json::from_str::<tantivy::tokenizer::Language>(&format!("{{\"language\":\"{}\"}}", lang)).ok()
}
}

@@ -176,6 +177,7 @@ pub fn build_tantivy_index<P: AsRef<std::path::Path>>(
let empty_taxonomies: HashMap<String, Vec<String>> = Default::default();

for section in library.sections_values() {
//eprintln!("section: {:?}, section.pages: {:?}", section, section.pages);

// reason for macro: Section/Page are different types but have same attributes
macro_rules! extract_content {
@@ -187,6 +189,8 @@ pub fn build_tantivy_index<P: AsRef<std::path::Path>>(

let cleaned_body: String = AMMONIA.clean(&$page.content).to_string();

//eprintln!("indexing {}", $page.permalink.as_str());

Some(IndexContent {
title: $page.meta.title.as_ref().map(|x| x.as_str()).unwrap_or(""),
description: $page.meta.description.as_ref().map(|x| x.as_str()).unwrap_or(""),
@@ -198,6 +202,7 @@ pub fn build_tantivy_index<P: AsRef<std::path::Path>>(
taxonomies: &empty_taxonomies,
})
} else {
//eprintln!("not indexing {}", $page.permalink.as_str());
None
}
}}


Loading…
Cancel
Save