(token, alphabet)
return u"".join(ret) + "_"
Escape away underscores and OOV characters and append '_'.

  This allows the token to be expressed as the concatenation of a list
  of subtokens from the vocabulary. The underscore acts as a sentinel
  which allows us to invertibly concatenate multiple such lists.

  Args:
    token: A unicode string to be escaped.
    alphabet: A set of all characters in the vocabulary's alphabet.

  Returns:
    escaped_token: An escaped unicode string.

  Raises:
    ValueError: If the provided token is not unicode.
Escape away underscores and OOV characters and append '_'.
Escape away underscores and OOV characters and append _ .
def _escape_token(token, alphabet):
  """Escape away underscores and OOV characters and append '_'.

  This allows the token to be expressed as the concatenation of a list
  of subtokens from the vocabulary. The underscore acts as a sentinel
  which allows us to invertibly concatenate multiple such lists.

  Args:
    token: A unicode string to be escaped.
    alphabet: A set of all characters in the vocabulary's alphabet.

  Returns:
    escaped_token: An escaped unicode string.

  Raises:
    ValueError: If the provided token is not unicode.
  """
  if not isinstance(token, six.text_type):
    raise ValueError("Expected string type for token, got %s" % type(token))

  token = token.replace(u"\\", u"\\\\").replace(u"_", u"\\u")
  ret = [c if c in alphabet and c != u"\n" else r"\%d;" % ord(c) for c in token]
  return u"".join(ret) + "_"
def _escape_token ( token , alphabet ) : if not isinstance ( token , six . text_type ) : raise ValueError ( "Expected string type for token, got %s" % type ( token ) ) token = token . replace ( u"\\" , u"\\\\" ) . replace ( u"_" , u"\\u" ) ret = [ c if c in alphabet and c != u"\n" else r"\%d;" % ord ( c ) for c in token ] return u"" . join ( ret ) + "_"