(stories_dir, tokenized_stories_dir)
Maps a whole directory of .story files
   to a tokenized version using Stanford CoreNLP Tokenizer
Maps a whole directory of .story files
   to a tokenized version using Stanford CoreNLP Tokenizer
Maps a whole directory of . story files to a tokenized version using Stanford CoreNLP Tokenizer
def tokenize_stories(stories_dir, tokenized_stories_dir):
  """Maps a whole directory of .story files
   to a tokenized version using Stanford CoreNLP Tokenizer"""
  logging.info("Preparing to tokenize {} to {}...".format(stories_dir, tokenized_stories_dir))
  stories = os.listdir(stories_dir)
  # make IO list file
  logging.info("Making list of files to tokenize...")
  with open("mapping.txt", "w") as f:
    for s in stories:
      f.write("%s \t %s\n" % (os.path.join(stories_dir, s),
                              os.path.join(tokenized_stories_dir, s)))
  command = ['java', 'edu.stanford.nlp.process.PTBTokenizer',
             '-ioFileList', '-preserveLines', 'mapping.txt']
  logging.info("Tokenizing {} files in {} and saving in {}..."
        .format(len(stories), stories_dir, tokenized_stories_dir))
  subprocess.call(command)
  logging.info("Stanford CoreNLP Tokenizer has finished.")
  os.remove("mapping.txt")

  num_orig = len(os.listdir(stories_dir))
  num_tokenized = len(os.listdir(tokenized_stories_dir))
  if num_orig != num_tokenized:
    raise Exception(
      "The tokenized stories directory {} contains {} files, but it "
      "should contain the same number as {} (which has {} files)."
      " Was there an error during tokenization?".format(
      tokenized_stories_dir, num_tokenized, stories_dir, num_orig))
  logging.info("Successfully finished tokenizing {} to {}.\n".format(stories_dir, tokenized_stories_dir))
def tokenize_stories ( stories_dir , tokenized_stories_dir ) : logging . info ( "Preparing to tokenize {} to {}..." . format ( stories_dir , tokenized_stories_dir ) ) stories = os . listdir ( stories_dir ) # make IO list file logging . info ( "Making list of files to tokenize..." ) with open ( "mapping.txt" , "w" ) as f : for s in stories : f . write ( "%s \t %s\n" % ( os . path . join ( stories_dir , s ) , os . path . join ( tokenized_stories_dir , s ) ) ) command = [ 'java' , 'edu.stanford.nlp.process.PTBTokenizer' , '-ioFileList' , '-preserveLines' , 'mapping.txt' ] logging . info ( "Tokenizing {} files in {} and saving in {}..." . format ( len ( stories ) , stories_dir , tokenized_stories_dir ) ) subprocess . call ( command ) logging . info ( "Stanford CoreNLP Tokenizer has finished." ) os . remove ( "mapping.txt" ) num_orig = len ( os . listdir ( stories_dir ) ) num_tokenized = len ( os . listdir ( tokenized_stories_dir ) ) if num_orig != num_tokenized : raise Exception ( "The tokenized stories directory {} contains {} files, but it " "should contain the same number as {} (which has {} files)." " Was there an error during tokenization?" . format ( tokenized_stories_dir , num_tokenized , stories_dir , num_orig ) ) logging . info ( "Successfully finished tokenizing {} to {}.\n" . format ( stories_dir , tokenized_stories_dir ) )