docs: def _tokenize(bytes, mimetype, extra_tokens=True):
      """ This function tokenizes the input bytes based on some common characters. It returns
          the tokens in ASCII format. """
  
      # Start with only the ASCII bytes. Limit it to 8+ character strings.
      try:
          ascii_bytes = b' '.join(re.compile(b'[\x00\x09\x0A\x0D\x20-\x7E]{8,}').findall(bytes))
          ascii_bytes = ascii_bytes.replace(b'\x00', b'')
      except:
          return []
  
      tokens = []
  
      # Find anything sandwiched between ( )
      results = re.compile(b'\(([^\)]+)\)').findall(ascii_bytes)
      #results = re.compile(b'\(([^\(\)]+)\)').findall(ascii_bytes)
      tokens += results
  
      # Find anything sandwiched between < >
      results = re.compile(b'\<([^\>]+)\>').findall(ascii_bytes)
      #results = re.compile(b'\<([^\<\>]+)\>').findall(ascii_bytes)
      tokens += results
  
      # Find anything sandwiched between [ ]
      results = re.compile(b'\[([^\]]+)\]').findall(ascii_bytes)
      #results = re.compile(b'\[([^\[\]]+)\]').findall(ascii_bytes)
      tokens += results
  
      # Find anything sandwiched between { }
      results = re.compile(b'\{([^\}]+)\}').findall(ascii_bytes)
      #results = re.compile(b'\{([^\{\}]+)\}').findall(ascii_bytes)
      tokens += results
  
      if not mimetype == 'data':
  
          # Find anything sandwiched between ' '
          results = re.compile(b'\'\s*(.*?)\s*\'').findall(ascii_bytes)
          tokens += results
  
          # Find anything sandwiched between " "
          results = re.compile(b'\"\s*(.*?)\s*\"').findall(ascii_bytes)
          tokens += results
  
      # Find anything sandwiched between spaces
      results = re.compile(b'\ ([^\ ]+)\ ').findall(ascii_bytes)
      tokens += results
  
      # Break the bytes apart if we want to generate even more tokens.
      if extra_tokens:
  
          # Add each newline as a token.
          lines = ascii_bytes.decode('ascii', errors='ignore').splitlines()
          tokens += [l.encode('ascii') for l in lines]
  
          # Now replace these characters in the ASCII bytes with spaces to create more tokens.
          replace_these = [b'(', b')', b'<', b'>', b'[', b']', b'{', b'}', b"'", b'"', b'\n', b'\r', b'\t', b'\\', b'`', b';']
          for char in replace_these:
              ascii_bytes = ascii_bytes.replace(char, b' ')
  
          # Split the bytes on spaces and use the results as tokens.
          tokens += ascii_bytes.split(b' ')
  
      # Decode the tokens as ASCII.
      tokens = [token.decode('ascii', errors='ignore') for token in tokens]
  
      # Since we want this to find full URLs and not just potential domain names, we want
      # to ensure that the tokens have at least "http" or "/" in them.
      tokens = list(set([t for t in tokens if ('http' in t or 'ftp' in t or ('/' in t and '.' in t)) and len(t) >= 8]))
  
      # Remove any tokens that look like they're probably bad based on the regex statements above.
      tokens = [t for t in tokens if not all(x in t for x in ['(', ')'])]
      tokens = [t for t in tokens if not all(x in t for x in ['<', '>'])]
      tokens = [t for t in tokens if not all(x in t for x in ['[', ']'])]
      tokens = [t for t in tokens if not all(x in t for x in ['{', '}'])]
      tokens = [t for t in tokens if not any(x in t for x in ['\t', '\n', '\r'])]
      tokens = [t for t in tokens if t[:1].isalpha() or t[:1].isdigit()]
      tokens = [t for t in tokens if not t.count('\'') > 1]
      tokens = [t for t in tokens if not t.count('\"') > 1]
  
      # Return a list of the tokens in ASCII format.
      return tokens