Browse Source

Removed commoncrawl tokenizer

develop
Paul Masurel 6 years ago
parent
commit
e986f2372e
1 changed files with 0 additions and 8 deletions
  1. +0
    -8
      src/commands/search.rs

+ 0
- 8
src/commands/search.rs View File

@@ -19,14 +19,6 @@ pub fn run_search_cli(matches: &ArgMatches) -> Result<(), String> {


fn run_search(directory: &Path, query: &str) -> tantivy::Result<()> { fn run_search(directory: &Path, query: &str) -> tantivy::Result<()> {
let index = Index::open(directory)?; let index = Index::open(directory)?;
index
.tokenizers()
.register("commoncrawl", SimpleTokenizer
.filter(RemoveLongFilter::limit(40))
.filter(LowerCaser)
.filter(AlphaNumOnlyFilter)
.filter(Stemmer::new())
);
let schema = index.schema(); let schema = index.schema();
let default_fields: Vec<Field> = schema let default_fields: Vec<Field> = schema
.fields() .fields()


Loading…
Cancel
Save