(url_file, art_out_file,
                  abs_out_file,
                  makevocab=False,
                  vocab_dir=None)
Reads the tokenized .story files corresponding
  to the urls listed in the url_file and writes them to a out_file.
Reads the tokenized .story files corresponding
  to the urls listed in the url_file and writes them to a out_file.
Reads the tokenized . story files corresponding to the urls listed in the url_file and writes them to a out_file .
def write_to_file(url_file, art_out_file,
                  abs_out_file,
                  makevocab=False,
                  vocab_dir=None):
  """Reads the tokenized .story files corresponding
  to the urls listed in the url_file and writes them to a out_file."""
  logging.info("Making bin file for URLs listed in {}...".format(url_file))
  url_list = read_text_file(url_file)
  url_hashes = get_url_hashes(url_list)
  story_fnames = [s + ".story" for s in url_hashes]
  num_stories = len(story_fnames)

  if makevocab:
    vocab_counter = collections.Counter()

  abs_writer = open(abs_out_file, 'w', encoding='utf8')
  art_writer = open(art_out_file, 'w', encoding='utf8')

  for idx, s in enumerate(story_fnames):
    if idx % 1000 == 0:
      logging.info("Writing story {} of {}; {:.2f} percent done"
            .format(idx, num_stories, float(idx) * 100.0 / float(num_stories)))

    # Look in the tokenized story dirs
    # to find the .story file corresponding to this url
    if os.path.isfile(os.path.join(cnn_tokenized_stories_dir, s)):
      story_file = os.path.join(cnn_tokenized_stories_dir, s)
    else:
      continue

    article, abstract = get_art_abs(story_file)
    if len(article) == 0:
      continue

    art_writer.write(article + '\n')
    abs_writer.write(abstract + '\n')

    # Write the vocab to file, if applicable
    if makevocab:
      art_tokens = article.split(' ')
      abs_tokens = abstract.split(' ')
      abs_tokens = [t for t in abs_tokens if t not in [SENTENCE_START, SENTENCE_END]]
      tokens = art_tokens + abs_tokens
      tokens = [t.strip() for t in tokens]
      tokens = [t for t in tokens if t != ""]
      vocab_counter.update(tokens)

  logging.info("Finished writing file\n")

  # write vocab to file
  if makevocab:
    logging.info("Writing vocab file...")
    with open(os.path.join(vocab_dir, "vocab"), 'w') as writer:
      for word, count in vocab_counter.most_common(VOCAB_SIZE):
        writer.write(word + ' ' + str(count) + '\n')
    logging.info("Finished writing vocab file")
def write_to_file ( url_file , art_out_file , abs_out_file , makevocab = False , vocab_dir = None ) : logging . info ( "Making bin file for URLs listed in {}..." . format ( url_file ) ) url_list = read_text_file ( url_file ) url_hashes = get_url_hashes ( url_list ) story_fnames = [ s + ".story" for s in url_hashes ] num_stories = len ( story_fnames ) if makevocab : vocab_counter = collections . Counter ( ) abs_writer = open ( abs_out_file , 'w' , encoding = 'utf8' ) art_writer = open ( art_out_file , 'w' , encoding = 'utf8' ) for idx , s in enumerate ( story_fnames ) : if idx % 1000 == 0 : logging . info ( "Writing story {} of {}; {:.2f} percent done" . format ( idx , num_stories , float ( idx ) * 100.0 / float ( num_stories ) ) ) # Look in the tokenized story dirs # to find the .story file corresponding to this url if os . path . isfile ( os . path . join ( cnn_tokenized_stories_dir , s ) ) : story_file = os . path . join ( cnn_tokenized_stories_dir , s ) else : continue article , abstract = get_art_abs ( story_file ) if len ( article ) == 0 : continue art_writer . write ( article + '\n' ) abs_writer . write ( abstract + '\n' ) # Write the vocab to file, if applicable if makevocab : art_tokens = article . split ( ' ' ) abs_tokens = abstract . split ( ' ' ) abs_tokens = [ t for t in abs_tokens if t not in [ SENTENCE_START , SENTENCE_END ] ] tokens = art_tokens + abs_tokens tokens = [ t . strip ( ) for t in tokens ] tokens = [ t for t in tokens if t != "" ] vocab_counter . update ( tokens ) logging . info ( "Finished writing file\n" ) # write vocab to file if makevocab : logging . info ( "Writing vocab file..." ) with open ( os . path . join ( vocab_dir , "vocab" ) , 'w' ) as writer : for word , count in vocab_counter . most_common ( VOCAB_SIZE ) : writer . write ( word + ' ' + str ( count ) + '\n' ) logging . info ( "Finished writing vocab file" )