(data_path=CORPUS_DATA_PATH)
加载数据集数据
加载数据集数据
加载数据集数据
def load(data_path=CORPUS_DATA_PATH):
    """
    加载数据集数据
    """
    if not os.path.exists(data_path):
        # download all pair data
        print("\n [efaqa-corpus-zh] downloading data %s ... \n" % CORPUS_URL)
        wget.download(CORPUS_URL, out = data_path)
    
    with smart_open(data_path) as f:
        for x in f:
            yield json.loads(x)
def load ( data_path = CORPUS_DATA_PATH ) : if not os . path . exists ( data_path ) : # download all pair data print ( "\n [efaqa-corpus-zh] downloading data %s ... \n" % CORPUS_URL ) wget . download ( CORPUS_URL , out = data_path ) with smart_open ( data_path ) as f : for x in f : yield json . loads ( x )