(
        self,
        vocab_file,
        do_lower_case=True,
        max_len=None,
        do_basic_tokenize=True,
        never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"),
    )
Constructs a BertTokenizer.

        Args:
          vocab_file: Path to a one-wordpiece-per-line vocabulary file
          do_lower_case: Whether to lower case the input
                         Only has an effect when do_wordpiece_only=False
          do_basic_tokenize: Whether to do basic tokenization before wordpiece.
          max_len: An artificial maximum length to truncate tokenized sequences to;
                         Effective maximum length is always the minimum of this
                         value (if specified) and the underlying BERT model's
                         sequence length.
          never_split: List of tokens which will never be split during tokenization.
                         Only has an effect when do_wordpiece_only=False
Constructs a BertTokenizer.
Constructs a BertTokenizer .
def __init__(
        self,
        vocab_file,
        do_lower_case=True,
        max_len=None,
        do_basic_tokenize=True,
        never_split=("[UNK]", "[SEP]", "[PAD]", "[CLS]", "[MASK]"),
    ):
        """Constructs a BertTokenizer.

        Args:
          vocab_file: Path to a one-wordpiece-per-line vocabulary file
          do_lower_case: Whether to lower case the input
                         Only has an effect when do_wordpiece_only=False
          do_basic_tokenize: Whether to do basic tokenization before wordpiece.
          max_len: An artificial maximum length to truncate tokenized sequences to;
                         Effective maximum length is always the minimum of this
                         value (if specified) and the underlying BERT model's
                         sequence length.
          never_split: List of tokens which will never be split during tokenization.
                         Only has an effect when do_wordpiece_only=False
        """
        if not os.path.isfile(vocab_file):
            raise ValueError(
                "Can't find a vocabulary file at path '{}'. "
                "To load the vocabulary from a Google pretrained model use "
                "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(
                    vocab_file
                )
            )
        self.vocab = load_vocab(vocab_file)
        self.ids_to_tokens = collections.OrderedDict(
            [(ids, tok) for tok, ids in self.vocab.items()]
        )
        self.do_basic_tokenize = do_basic_tokenize
        if do_basic_tokenize:
            self.basic_tokenizer = BasicTokenizer(
                do_lower_case=do_lower_case, never_split=never_split
            )
        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
        self.max_len = max_len if max_len is not None else int(1e12)
def __init__ ( self , vocab_file , do_lower_case = True , max_len = None , do_basic_tokenize = True , never_split = ( "[UNK]" , "[SEP]" , "[PAD]" , "[CLS]" , "[MASK]" ) , ) : if not os . path . isfile ( vocab_file ) : raise ValueError ( "Can't find a vocabulary file at path '{}'. " "To load the vocabulary from a Google pretrained model use " "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`" . format ( vocab_file ) ) self . vocab = load_vocab ( vocab_file ) self . ids_to_tokens = collections . OrderedDict ( [ ( ids , tok ) for tok , ids in self . vocab . items ( ) ] ) self . do_basic_tokenize = do_basic_tokenize if do_basic_tokenize : self . basic_tokenizer = BasicTokenizer ( do_lower_case = do_lower_case , never_split = never_split ) self . wordpiece_tokenizer = WordpieceTokenizer ( vocab = self . vocab ) self . max_len = max_len if max_len is not None else int ( 1e12 )