docs: def make_ngram_corpus(corpus_clean_visibles, num_tokens, filter_punctuation,
                        zoning_rules=False):
      '''takes a list of clean_visible texts, such as from StreamItems or
      FCs, tokenizes all the texts, and constructs n-grams using
      `num_tokens` sized windows.
  
      ``corpus_clean_visibles`` -- list of unicode strings
      ``num_tokens`` --- the n of the n-grams
      ``filter_punctuation`` --- if True, punctuation is filtered
      '''
      ## TODO: generatlize this zoning code, so that it works on many
      ## sites in the HT domain; consider finishing streamcorpus-zoner
      ## to do this.
      if filter_punctuation:
          ## word tokenizer that removes punctuation
          tokenize = RegexpTokenizer(r'\w+').tokenize
          backpage_string = 'backpage'
          end_string = 'Poster'
      else:
          #tokenize = word_tokenize
          tokenize = lambda s: string.split(s)
          backpage_string = 'backpage.com'
          end_string = 'Poster\'s'
  
      corpus = list()
      for clean_vis in corpus_clean_visibles:
          ## crudely skip pages that have "error"
          if re.search(u'error', clean_vis, re.I & re.UNICODE):
              continue
  
          ## make tokens
          tokens = tokenize(clean_vis) ## already a unicode string
  
          if zoning_rules:
              ## filter out non backpage pages
              if backpage_string not in tokens:
                  continue
  
              ## string that signals the beginning of the body
              try:
                  idx0 = tokens.index('Reply')
              except:
                  continue
              ## string that signals the end of the body
              try:
                  idx1 = tokens.index(end_string)
              except:
                  continue
  
              tokens = tokens[idx0:idx1]
  
          ## make ngrams, attach to make strings
          ngrams_strings = list()
  
  
          for ngram_tuple in ngrams(tokens, num_tokens):
              # ## attempt to remove unwanted phrases
              ## score with many_stop_words and drop bad tuples
              # stop_count = sum([int(bool(tok.lower() in stop_words))
              #                   for tok in ngram_tuple])
              # if stop_count > num_tokens / 1.5:
              #     continue
  
              ## remove ones with many repeated words
              if len(set(ngram_tuple)) < len(ngram_tuple) / 2:
                  continue
  
              ## this adds ngrams for the current doc
              ngrams_strings.append(' '.join(ngram_tuple))
  
  
          ## this adds a list of all the ngrams from the current doc
          ## to the corpus list
          corpus.append(ngrams_strings)
      return corpus