You are an expert in writing code in many different languages. Your goal is to perform code completion for the following code, keeping in mind the rest of the code and the file meta data.
    Metadata details:
    filename: tokenize.rs
    path of file: ./repos/mail-server/crates/nlp/src/bayes
    
    the code of the file until where you have to start completion: use std::borrow::Cow;

use utils::suffixlist::PublicSuffix;

use crate::{
    language::{
        detect::{LanguageDetector, MIN_LANGUAGE_SCORE},
        stemmer::STEMMER_MAP,
        stopwords::STOP_WORDS,
        Language,
    },
    tokenizers::{
        chinese::JIEBA,
        types::{TokenType, TypesTokenizer},
    },
};

pub struct BayesTokenizer<'x, 'y> {
    text: &'x str,
    tokenizer: TypesTokenizer<'x, 'y>,
    stemmer: Stemmer,
    stop_words: Option<&'static phf::Set<&'static str>>,
    tokens: Vec<Cow<'x, str>>,
}

enum Stemmer {
    IndoEuropean(rust_stemmers::Stemmer),
    Mandarin,
    Japanese,
    None,
}

impl<'x, 'y> BayesTokenizer<'x, 'y> {
    pub fn new(text: &'x str, suffixes: &'y PublicSuffix) -> Self {
        // Detect language
        let (mut language, score) =
            LanguageDetector::detect_single(text).unwrap_or((Language::English, 1.0));
        if score < MIN_LANGUAGE_SCORE {
            language = Language::English;
        }

        Self {
            text,
            tokenizer: TypesTokenizer::new(text, suffixes),
            stemmer: match language {
                Language::Mandarin => Stemmer::Mandarin,
                Language::Japanese => Stemmer::Japanese,
                _ => STEMMER_MAP[language as usize]
                    .map(|algo| Stemmer::Ind
    
    fn next(&mut self) -> Option<Self::Item> {
        if let Some(prev_token) = self.tokens.pop() {
            return Some(prev_token);
        }

        loop {
            let token = self.tokenizer.next()?;

            let word: Cow<str> = match token.word {
                TokenType::Alphabetic(word) => {
                    let word = word.to_lowercase();
                    if self
                        .stop_words
                        .map_or(false, |sw| sw.contains(word.as_str()))
                    {
                        continue;
                    }
                    match &self.stemmer {
                        Stemmer::IndoEuropean(stemmer) => match stemmer.stem(&word) {
                            Cow::Borrowed(_) => word.into(),
                            Cow::Owned(stemmed_word) => stemmed_word.into(),
                        },
                        Stemmer::Mandarin => {
                            let mut result = JIEBA.cut(&word, false).into_iter();
                            if let Some(stemmed_word) = result.next() {
                                let stemmed_word = stemmed_word.to_string();
                                self.tokens = result
                                    .rev()
                                    .map(|word| Cow::from(word.to_string()))
                                    .collect::<Vec<_>>();
                                stemmed_word.into()
                            } else {
                                // This shouldn't happen, but just in case
                                continue;
                            }
                        }
                        Stemmer::Japanese => {
                            let mut result = tinysegmenter::tokenize(&word).into_iter();
                            if let Some(stemmed_word) = result.next() {
                                self.tokens = result.rev().map(Cow::from).collect::<Vec<_>>();
                                stemmed_word.into()
                            } else {
                                // This shouldn't happen, but just in case
                                continue;
                            }
                        }
                        Stemmer::None => word.into(),
                    }
                }

                TokenType::Url(word) => {
                    if let Some((_, host)) = word.split_once("://") {
                        host.split_once('/')
                            .map_or(host, |(h, _)| h)
                            .to_lowercase()
                            .into()
                    } else {
                        continue;
                    }
                }
                TokenType::UrlNoScheme(word) => word
                    .split_once('/')
                    .map_or(word, |(h, _)| h)
                    .to_lowercase()
                    .into(),
                TokenType::Alphanumeric(word)
                | TokenType::Email(word)
                | TokenType::UrlNoHost(word) => word.to_lowercase().into(),
                TokenType::Other(ch) => {
                    if SYMBOLS.contains(&ch) {
                        (&self.text[token.from..token.to]).into()
                    } else {
                        continue;
                    }
                }
                TokenType::Integer(word) => number_to_tag("INTEGER", word).into(),
                TokenType::Float(word) => number_to_tag("FLOAT", word).into(),
                TokenType::Punctuation(_) | TokenType::Space => {
                    continue;
                }
            };

            return Some(word);
        }
    }