Trait tantivy::tokenizer::BoxedTokenizer

pub trait BoxedTokenizer: Send + Sync {
    fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a>;
    fn token_stream_texts<'b>(
        &self, 
        texts: &'b [&'b str]
    ) -> Box<TokenStream + 'b>;
    fn boxed_clone(&self) -> Box<BoxedTokenizer>;
}

A boxed tokenizer

Required Methods

Important traits for Box<R>

impl<R> Read for Box<R> where
    R: Read + ?Sized, impl<W> Write for Box<W> where
    W: Write + ?Sized, impl<I> Iterator for Box<I> where
    I: Iterator + ?Sized,  type Item = <I as Iterator>::Item;

fn token_stream<'a>(&self, text: &'a str) -> Box<TokenStream + 'a>

Tokenize a &str

Important traits for Box<R>

impl<R> Read for Box<R> where
    R: Read + ?Sized, impl<W> Write for Box<W> where
    W: Write + ?Sized, impl<I> Iterator for Box<I> where
    I: Iterator + ?Sized,  type Item = <I as Iterator>::Item;

fn token_stream_texts<'b>(&self, texts: &'b [&'b str]) -> Box<TokenStream + 'b>

Tokenize an array&str

The resulting TokenStream is equivalent to what would be obtained if the &str were one concatenated &str, with an artificial position gap of 2 between the different fields to prevent accidental PhraseQuery to match accross two terms.

Important traits for Box<R>

impl<R> Read for Box<R> where
    R: Read + ?Sized, impl<W> Write for Box<W> where
    W: Write + ?Sized, impl<I> Iterator for Box<I> where
    I: Iterator + ?Sized,  type Item = <I as Iterator>::Item;

fn boxed_clone(&self) -> Box<BoxedTokenizer>

Return a boxed clone of the tokenizer

Implementors