(wiki_dir, save_path, preprocess, num_workers=None, hotpotqa_format=False)
Preprocess and store a corpus of documents in sqlite.

    Args:
        data_path: Root path to directory (or directory of directories) of files
          containing json encoded documents (must have `id` and `text` fields).
        save_path: Path to output sqlite db.
        preprocess: Path to file defining a custom `preprocess` function. Takes
          in and outputs a structured doc.
        num_workers: Number of parallel processes to use when reading docs.
Preprocess and store a corpus of documents in sqlite.
Preprocess and store a corpus of documents in sqlite .
def store_contents(wiki_dir, save_path, preprocess, num_workers=None, hotpotqa_format=False):
    """Preprocess and store a corpus of documents in sqlite.

    Args:
        data_path: Root path to directory (or directory of directories) of files
          containing json encoded documents (must have `id` and `text` fields).
        save_path: Path to output sqlite db.
        preprocess: Path to file defining a custom `preprocess` function. Takes
          in and outputs a structured doc.
        num_workers: Number of parallel processes to use when reading docs.
    """
    filenames = [f for f in glob.glob(
        wiki_dir + "/*/wiki_*", recursive=True) if ".bz2" not in f]
    if os.path.isfile(save_path):
        raise RuntimeError('%s already exists! Not overwriting.' % save_path)

    logger.info('Reading into database...')
    conn = sqlite3.connect(save_path)
    c = conn.cursor()
    c.execute(
        "CREATE TABLE documents (id PRIMARY KEY, text, linked_title, original_title);")

    workers = ProcessPool(num_workers, initializer=init,
                          initargs=(preprocess,))
    count = 0
    # Due to the slight difference of input format between preprocessed HotpotQA wikipedia data and 
    # the ones by Wikiextractor, we call different functions for data process.
    if hotpotqa_format is True:
        content_processing_method = get_contents_hotpotqa
    else:
        content_processing_method = get_contents
        
    with tqdm(total=len(filenames)) as pbar:
        for pairs in tqdm(workers.imap_unordered(content_processing_method, filenames)):
            count += len(pairs)
            c.executemany(
                "INSERT OR REPLACE INTO documents VALUES (?,?,?,?)", pairs)
            pbar.update()
    logger.info('Read %d docs.' % count)
    logger.info('Committing...')
    conn.commit()
    conn.close()
def store_contents ( wiki_dir , save_path , preprocess , num_workers = None , hotpotqa_format = False ) : filenames = [ f for f in glob . glob ( wiki_dir + "/*/wiki_*" , recursive = True ) if ".bz2" not in f ] if os . path . isfile ( save_path ) : raise RuntimeError ( '%s already exists! Not overwriting.' % save_path ) logger . info ( 'Reading into database...' ) conn = sqlite3 . connect ( save_path ) c = conn . cursor ( ) c . execute ( "CREATE TABLE documents (id PRIMARY KEY, text, linked_title, original_title);" ) workers = ProcessPool ( num_workers , initializer = init , initargs = ( preprocess , ) ) count = 0 # Due to the slight difference of input format between preprocessed HotpotQA wikipedia data and  # the ones by Wikiextractor, we call different functions for data process. if hotpotqa_format is True : content_processing_method = get_contents_hotpotqa else : content_processing_method = get_contents with tqdm ( total = len ( filenames ) ) as pbar : for pairs in tqdm ( workers . imap_unordered ( content_processing_method , filenames ) ) : count += len ( pairs ) c . executemany ( "INSERT OR REPLACE INTO documents VALUES (?,?,?,?)" , pairs ) pbar . update ( ) logger . info ( 'Read %d docs.' % count ) logger . info ( 'Committing...' ) conn . commit ( ) conn . close ( )