code: def tokenize(self, char_input, max_len, by_char=False):
      
      if by_char:
        tokenized_list = [self._vocab_dict[token] for token in char_input]
      else:
        tokenized_list = []
        compound_token = ""
        for token in char_input:
          # Compose alphanumeric inputs into compound tokens.
          add_number = compound_token.isdigit() and not token.isdigit()
          add_word = compound_token.isalpha() and not token.isalpha()
          if add_number or add_word:
            tokenized_list.append(self._vocab_dict[compound_token])
            compound_token = ""
          # Add token or build compound token.
          if token.isdigit():
            compound_token += token
          elif token.isalpha():
            compound_token += token
          else:
            tokenized_list.append(self._vocab_dict[token])
        if compound_token:
          tokenized_list.append(self._vocab_dict[compound_token])
  
      # To ensure uniform batch sequence length pad the sequence.
      seq_size = len(tokenized_list)
      if seq_size < max_len:
        padding = [self._vocab_dict[get_padding()]] * (max_len - seq_size)
        tokenized_list.extend(padding)
      elif seq_size > max_len:
        raise ValueError("Token sequence is too large: {}".format(
            len(tokenized_list)))
  
      return tokenized_list, seq_size
docs: Produces the list of integer indices corresponding to a token list.
  
      Args:
        char_input: The character string to be tokenized.
        max_len: Truncation length.
        by_char: If true each character is a token - otherwise alpha-numeric
                 groupings are tokens.
  
      Returns:
        A padded list of string tokens and the true sequence length.
  
      Raises:
        ValueError: the token sequence is too long.