(vocab_path=None)
return tokenizer
Constructs a BERT tokenizer.
Constructs a BERT tokenizer.
Constructs a BERT tokenizer .
def bert_tokenizer(vocab_path=None):
  """Constructs a BERT tokenizer."""
  # This import is from https://github.com/google-research/bert which is not
  # listed as a dependency in trax.
  # TODO(piotrekp1): using SubwordTextEncoder instead after fixing the
  # differences
  from bert.tokenization.bert_tokenization import FullTokenizer  # pylint: disable=g-import-not-at-top
  if vocab_path is None:
    raise ValueError('vocab_path is required to construct the BERT tokenizer.')
  tokenizer = FullTokenizer(vocab_path, do_lower_case=True)
  return tokenizer
def bert_tokenizer ( vocab_path = None ) : # This import is from https://github.com/google-research/bert which is not # listed as a dependency in trax. # TODO(piotrekp1): using SubwordTextEncoder instead after fixing the # differences from bert . tokenization . bert_tokenization import FullTokenizer # pylint: disable=g-import-not-at-top if vocab_path is None : raise ValueError ( 'vocab_path is required to construct the BERT tokenizer.' ) tokenizer = FullTokenizer ( vocab_path , do_lower_case = True ) return tokenizer