(tokens)
return "".join(ret)
Decode a list of tokens to a unicode string.

  Args:
    tokens: a list of Unicode strings
  Returns:
    a unicode string
Decode a list of tokens to a unicode string.
Decode a list of tokens to a unicode string .
def decode(tokens):
  """Decode a list of tokens to a unicode string.

  Args:
    tokens: a list of Unicode strings
  Returns:
    a unicode string
  """
  token_is_alnum = [t[0] in _ALPHANUMERIC_CHAR_SET for t in tokens]
  ret = []
  for i, token in enumerate(tokens):
    if i > 0 and token_is_alnum[i - 1] and token_is_alnum[i]:
      ret.append(u" ")
    ret.append(token)
  return "".join(ret)
def decode ( tokens ) : token_is_alnum = [ t [ 0 ] in _ALPHANUMERIC_CHAR_SET for t in tokens ] ret = [ ] for i , token in enumerate ( tokens ) : if i > 0 and token_is_alnum [ i - 1 ] and token_is_alnum [ i ] : ret . append ( u" " ) ret . append ( token ) return "" . join ( ret )