Browse Source

Compiles with tantivy 0.3

develop
Paul Masurel 7 years ago
parent
commit
f341800952
5 changed files with 393 additions and 198 deletions
  1. +365
    -178
      Cargo.lock
  2. +3
    -7
      Cargo.toml
  3. +6
    -5
      src/commands/index.rs
  4. +18
    -7
      src/commands/merge.rs
  5. +1
    -1
      src/main.rs

+ 365
- 178
Cargo.lock
File diff suppressed because it is too large
View File


+ 3
- 7
Cargo.toml View File

@@ -1,6 +1,6 @@
[package] [package]
name = "tantivy-cli" name = "tantivy-cli"
version = "0.2.0"
version = "0.3.0"
authors = ["Paul Masurel <paul.masurel@gmail.com>"] authors = ["Paul Masurel <paul.masurel@gmail.com>"]


description = """Command line interface for Tantivy, a search engine library.""" description = """Command line interface for Tantivy, a search engine library."""
@@ -26,14 +26,10 @@ chan = "0.1"
bincode = "0.4" bincode = "0.4"
byteorder = "0.5" byteorder = "0.5"
log = "0.3" log = "0.3"
futures = "0.1"
env_logger = "0.3" env_logger = "0.3"
version = "2" version = "2"


[dependencies.tantivy]
version = "0.2.0"
features = []
default-features = false
tantivy = "0.3.0"


[[bin]] [[bin]]
name = "tantivy" name = "tantivy"


+ 6
- 5
src/commands/index.rs View File

@@ -82,9 +82,12 @@ fn run_index(directory: PathBuf, document_source: DocumentSource, buffer_size_pe




let index_result = index_documents(&mut index_writer, doc_receiver); let index_result = index_documents(&mut index_writer, doc_receiver);
try!(match index_result {
match index_result {
Ok(docstamp) => { Ok(docstamp) => {
println!("Commit succeed, docstamp at {}", docstamp); println!("Commit succeed, docstamp at {}", docstamp);
println!("Waiting for merging threads");
index_writer.wait_merging_threads()?;
println!("Terminated successfully!");
Ok(()) Ok(())
} }
Err(e) => { Err(e) => {
@@ -93,9 +96,7 @@ fn run_index(directory: PathBuf, document_source: DocumentSource, buffer_size_pe
println!("Rollback succeeded"); println!("Rollback succeeded");
Err(e) Err(e)
} }
});
index_writer.wait_merging_threads()
}
} }


fn index_documents(index_writer: &mut IndexWriter, doc_receiver: chan::Receiver<Document>) -> tantivy::Result<u64> { fn index_documents(index_writer: &mut IndexWriter, doc_receiver: chan::Receiver<Document>) -> tantivy::Result<u64> {
@@ -103,7 +104,7 @@ fn index_documents(index_writer: &mut IndexWriter, doc_receiver: chan::Receiver<
let mut num_docs = 0; let mut num_docs = 0;
let mut cur = PreciseTime::now(); let mut cur = PreciseTime::now();
for doc in doc_receiver { for doc in doc_receiver {
try!(index_writer.add_document(doc));
index_writer.add_document(doc);
if num_docs > 0 && (num_docs % group_count == 0) { if num_docs > 0 && (num_docs % group_count == 0) {
println!("{} Docs", num_docs); println!("{} Docs", num_docs);
let new = PreciseTime::now(); let new = PreciseTime::now();


+ 18
- 7
src/commands/merge.rs View File

@@ -1,20 +1,31 @@
extern crate tantivy; extern crate tantivy;


use tantivy::Index;
use tantivy::{Index, SegmentMeta};
use std::path::PathBuf; use std::path::PathBuf;
use clap::ArgMatches; use clap::ArgMatches;
use futures::Future;


const HEAP_SIZE: usize = 300_000_000; const HEAP_SIZE: usize = 300_000_000;



fn error_msg(err: tantivy::Error) -> String {
format!("Merge failed : {:?}", err)
}

pub fn run_merge_cli(argmatch: &ArgMatches) -> Result<(), String> { pub fn run_merge_cli(argmatch: &ArgMatches) -> Result<(), String> {
let index_directory = PathBuf::from(argmatch.value_of("index").unwrap()); let index_directory = PathBuf::from(argmatch.value_of("index").unwrap());
run_merge(index_directory).map_err(|e| format!("Indexing failed : {:?}", e))
let segment_meta = run_merge(index_directory).map_err(error_msg)?;
println!("Merge finished with segment meta {:?}", segment_meta);
Ok(())
} }




fn run_merge(path: PathBuf) -> tantivy::Result<()> {
let index = try!(Index::open(&path));
let segments = index.searchable_segments();
let mut index_writer = try!(index.writer(HEAP_SIZE));
index_writer.merge(&segments)
fn run_merge(path: PathBuf) -> tantivy::Result<SegmentMeta> {
let index = Index::open(&path)?;
let segments = index.searchable_segment_ids()?;
index
.writer(HEAP_SIZE)?
.merge(&segments)
.wait()
.map_err(|_| tantivy::Error::ErrorInThread(String::from("Merge got cancelled")))
} }

+ 1
- 1
src/main.rs View File

@@ -2,7 +2,6 @@
extern crate clap; extern crate clap;
#[macro_use] #[macro_use]
extern crate version; extern crate version;
#[macro_use]
extern crate rustc_serialize; extern crate rustc_serialize;
#[macro_use] #[macro_use]
extern crate log; extern crate log;
@@ -10,6 +9,7 @@ extern crate env_logger;
extern crate tantivy; extern crate tantivy;
extern crate time; extern crate time;
extern crate persistent; extern crate persistent;
extern crate futures;
extern crate urlencoded; extern crate urlencoded;
extern crate iron; extern crate iron;
extern crate chan; extern crate chan;


Loading…
Cancel
Save