()
Download and unpack dialogs
Download and unpack dialogs
Download and unpack dialogs
def prepare_ubuntu_data():
    """Download and unpack dialogs"""

    tar_filename = 'ubuntu_dialogs.tgz'
    url = 'http://dataset.cs.mcgill.ca/ubuntu-corpus-1.0/ubuntu_dialogs.tgz'
    tarfile_path = ubuntu_dir.joinpath(tar_filename)
    metadata_url = 'https://raw.githubusercontent.com/rkadlec/ubuntu-ranking-dataset-creator/master/src/meta/'

    if not datasets_dir.exists():
        datasets_dir.mkdir()
    if not ubuntu_dir.exists():
        ubuntu_dir.mkdir()
    if not ubuntu_meta_dir.exists():
        ubuntu_meta_dir.mkdir()

    # Prepare Dialog data
    if not dialogs_dir.joinpath("10/1.tst").exists():
        # Download Dialog tarfile
        if not tarfile_path.exists():
            print(f"Downloading {url} to {tarfile_path}")
            urlretrieve(url, tarfile_path)
            print(f"Successfully downloaded {tarfile_path}")

        # Unpack tarfile
        if not dialogs_dir.exists():
            print("Unpacking dialogs ... (This can take 5~10 mins.)")
            with tarfile.open(tarfile_path) as tar:
                tar.extractall(path=ubuntu_dir)
            print("Archive unpacked.")

    # Download metadata
    if not ubuntu_meta_dir.joinpath('trainfiles.csv').exists():
        print('Downloading metadata ... (This can take 5~10 mins.)')
        for filename in ['trainfiles.csv', 'valfiles.csv', 'testfiles.csv']:
            csv_path = ubuntu_meta_dir.joinpath(filename)
            print(f"Downloading {metadata_url+filename} to {csv_path}")
            urlretrieve(metadata_url + filename, csv_path)
            print(f"Successfully downloaded {csv_path}")

    print('Ubuntu Data prepared!')
def prepare_ubuntu_data ( ) : tar_filename = 'ubuntu_dialogs.tgz' url = 'http://dataset.cs.mcgill.ca/ubuntu-corpus-1.0/ubuntu_dialogs.tgz' tarfile_path = ubuntu_dir . joinpath ( tar_filename ) metadata_url = 'https://raw.githubusercontent.com/rkadlec/ubuntu-ranking-dataset-creator/master/src/meta/' if not datasets_dir . exists ( ) : datasets_dir . mkdir ( ) if not ubuntu_dir . exists ( ) : ubuntu_dir . mkdir ( ) if not ubuntu_meta_dir . exists ( ) : ubuntu_meta_dir . mkdir ( ) # Prepare Dialog data if not dialogs_dir . joinpath ( "10/1.tst" ) . exists ( ) : # Download Dialog tarfile if not tarfile_path . exists ( ) : print ( f"Downloading {url} to {tarfile_path}" ) urlretrieve ( url , tarfile_path ) print ( f"Successfully downloaded {tarfile_path}" ) # Unpack tarfile if not dialogs_dir . exists ( ) : print ( "Unpacking dialogs ... (This can take 5~10 mins.)" ) with tarfile . open ( tarfile_path ) as tar : tar . extractall ( path = ubuntu_dir ) print ( "Archive unpacked." ) # Download metadata if not ubuntu_meta_dir . joinpath ( 'trainfiles.csv' ) . exists ( ) : print ( 'Downloading metadata ... (This can take 5~10 mins.)' ) for filename in [ 'trainfiles.csv' , 'valfiles.csv' , 'testfiles.csv' ] : csv_path = ubuntu_meta_dir . joinpath ( filename ) print ( f"Downloading {metadata_url+filename} to {csv_path}" ) urlretrieve ( metadata_url + filename , csv_path ) print ( f"Successfully downloaded {csv_path}" ) print ( 'Ubuntu Data prepared!' )