(self, text)
return "".join(output)
Performs invalid character removal and whitespace cleanup on text.
Performs invalid character removal and whitespace cleanup on text.
Performs invalid character removal and whitespace cleanup on text .
def _clean_text(self, text):
    """Performs invalid character removal and whitespace cleanup on text."""
    output = []
    for char in text:
      cp = ord(char)
      if cp == 0 or cp == 0xfffd or _bert_is_control(char):
        continue
      if _bert_is_whitespace(char):
        output.append(" ")
      else:
        output.append(char)
    return "".join(output)
def _clean_text ( self , text ) : output = [ ] for char in text : cp = ord ( char ) if cp == 0 or cp == 0xfffd or _bert_is_control ( char ) : continue if _bert_is_whitespace ( char ) : output . append ( " " ) else : output . append ( char ) return "" . join ( output )