Trait tantivy::tokenizer::TokenFilter

pub trait TokenFilter<TailTokenStream: TokenStream>: Clone {
    type ResultTokenStream: TokenStream;
    fn transform(
        &self, 
        token_stream: TailTokenStream
    ) -> Self::ResultTokenStream;
}

Trait for the pluggable components of Tokenizers.

Associated Types

type ResultTokenStream: TokenStream

The resulting TokenStream type.

Required Methods

fn transform(&self, token_stream: TailTokenStream) -> Self::ResultTokenStream

Wraps a token stream and returns the modified one.

Implementors

• impl<TailTokenStream> TokenFilter<TailTokenStream> for LowerCaser where
    TailTokenStream: TokenStream,  type ResultTokenStream = LowerCaserTokenStream<TailTokenStream>;
• impl<TailTokenStream> TokenFilter<TailTokenStream> for RemoveLongFilter where
    TailTokenStream: TokenStream,  type ResultTokenStream = RemoveLongFilterStream<TailTokenStream>;
• impl<TailTokenStream> TokenFilter<TailTokenStream> for Stemmer where
    TailTokenStream: TokenStream,  type ResultTokenStream = StemmerTokenStream<TailTokenStream>;
• impl<TailTokenStream> TokenFilter<TailTokenStream> for AlphaNumOnlyFilter where
    TailTokenStream: TokenStream,  type ResultTokenStream = AlphaNumOnlyFilterStream<TailTokenStream>;