parameters: (buf)
return_statement: return filtered
docstring: We define three types of bytes:
          alphabet: english alphabets [a-zA-Z]
          international: international characters [\x80-\xFF]
          marker: everything else [^a-zA-Z\x80-\xFF]
  
          The input buffer can be thought to contain a series of words delimited
          by markers. This function works to filter all words that contain at
          least one international character. All contiguous sequences of markers
          are replaced by a single space ascii character.
  
          This filter applies to all scripts which do not use English characters.
docstring_summary: We define three types of bytes:
          alphabet: english alphabets [a-zA-Z]
          international: international characters [\x80-\xFF]
          marker: everything else [^a-zA-Z\x80-\xFF]
docstring_tokens: We define three types of bytes : alphabet : english alphabets [ a - zA - Z ] international : international characters [ \ x80 - \ xFF ] marker : everything else [ ^a - zA - Z \ x80 - \ xFF ]
function: def filter_international_words(buf):
          """
          We define three types of bytes:
          alphabet: english alphabets [a-zA-Z]
          international: international characters [\x80-\xFF]
          marker: everything else [^a-zA-Z\x80-\xFF]
  
          The input buffer can be thought to contain a series of words delimited
          by markers. This function works to filter all words that contain at
          least one international character. All contiguous sequences of markers
          are replaced by a single space ascii character.
  
          This filter applies to all scripts which do not use English characters.
          """
          filtered = bytearray()
  
          # This regex expression filters out only words that have at-least one
          # international character. The word may include one marker character at
          # the end.
          words = re.findall(b'[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?',
                             buf)
  
          for word in words:
              filtered.extend(word[:-1])
  
              # If the last character in the word is a marker, replace it with a
              # space as markers shouldn't affect our analysis (they are used
              # similarly across all languages and may thus have similar
              # frequencies).
              last_char = word[-1:]
              if not last_char.isalpha() and last_char < b'\x80':
                  last_char = b' '
              filtered.extend(last_char)
  
          return filtered
function_tokens: def filter_international_words ( buf ) : filtered = bytearray ( ) # This regex expression filters out only words that have at-least one # international character. The word may include one marker character at # the end. words = re . findall ( b'[a-zA-Z]*[\x80-\xFF]+[a-zA-Z]*[^a-zA-Z\x80-\xFF]?' , buf ) for word in words : filtered . extend ( word [ : - 1 ] ) # If the last character in the word is a marker, replace it with a # space as markers shouldn't affect our analysis (they are used # similarly across all languages and may thus have similar # frequencies). last_char = word [ - 1 : ] if not last_char . isalpha ( ) and last_char < b'\x80' : last_char = b' ' filtered . extend ( last_char ) return filtered