From e986f2372ee6c5d1d5c8749eeb2389a558f16f02 Mon Sep 17 00:00:00 2001
From: Paul Masurel <paul.masurel@gmail.com>
Date: Thu, 25 Jan 2018 13:00:02 +0900
Subject: [PATCH] Removed commoncrawl tokenizer

---
 src/commands/search.rs | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/commands/search.rs b/src/commands/search.rs
index 0397e5d..1ac2d02 100644
--- a/src/commands/search.rs
+++ b/src/commands/search.rs
@@ -19,14 +19,6 @@ pub fn run_search_cli(matches: &ArgMatches) -> Result<(), String> {
 
 fn run_search(directory: &Path, query: &str) -> tantivy::Result<()> {     
     let index = Index::open(directory)?;
-    index
-        .tokenizers()
-        .register("commoncrawl", SimpleTokenizer
-            .filter(RemoveLongFilter::limit(40))
-            .filter(LowerCaser)
-            .filter(AlphaNumOnlyFilter)
-            .filter(Stemmer::new())
-        );
     let schema = index.schema();
     let default_fields: Vec<Field> = schema
         .fields()