()
Download bilingual text dataset for Machine translation example.
Download bilingual text dataset for Machine translation example.
Download bilingual text dataset for Machine translation example .
def get_data():
    """
    Download bilingual text dataset for Machine translation example.
    """

    # vocab_size and time_steps are hard coded here
    vocab_size = 16384
    time_steps = 20

    # download dataset
    url = 'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/'
    filename = 'bitexts.tgz'
    size = 1313280000

    parser = NeonArgparser(__doc__)
    args = parser.parse_args(gen_be=False)
    data_dir = os.path.join(args.data_dir, 'nmt')

    _, filepath = Dataset._valid_path_append(data_dir, '', filename)
    if not os.path.exists(filepath):
        Dataset.fetch_dataset(url, filename, filepath, size)

    # extract selected datasets
    datafiles = dict()
    datafiles['un2000'] = ('un2000_pc34.en.gz', 'un2000_pc34.fr.gz')
    datafiles['europarl7'] = ('ep7_pc45.en.gz', 'ep7_pc45.fr.gz')

    extractpath = os.path.join(data_dir, 'bitexts.selected')
    with tarfile.open(filepath, 'r') as tar_ref:
        for dset, files in datafiles.items():
            datasetpath = os.path.join(data_dir, dset)
            # extract the files for dataset, if not already there
            for zipped in files:
                fname = '.'.join(zipped.split('.')[:-1])
                fpath = os.path.join(datasetpath, fname)
                if not os.path.exists(fpath):
                    gzpath = os.path.join(extractpath, zipped)
                    if not os.path.exists(gzpath):
                        select = [ti for ti in tar_ref if os.path.split(ti.name)[1] == zipped]
                        tar_ref.extractall(path=data_dir, members=select)
                    # get contents of gz files
                    if not os.path.exists(datasetpath):
                        os.makedirs(datasetpath)
                    with gzip.open(gzpath, 'r') as fin, open(fpath, 'w') as fout:
                        fout.write(fin.read())
                    os.remove(gzpath)

    if os.path.exists(extractpath):
        os.rmdir(extractpath)

    # process data and save to h5 file
    # loop through all datasets and get train and valid splits
    for dataset in datafiles.keys():

        s_vocab, t_vocab = create_h5py(data_dir, dataset, 'train',
                                       vocab_size=vocab_size, time_steps=time_steps)
        create_h5py(data_dir, dataset, 'valid', s_vocab=s_vocab, t_vocab=t_vocab,
                    time_steps=time_steps)
def get_data ( ) : # vocab_size and time_steps are hard coded here vocab_size = 16384 time_steps = 20 # download dataset url = 'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/' filename = 'bitexts.tgz' size = 1313280000 parser = NeonArgparser ( __doc__ ) args = parser . parse_args ( gen_be = False ) data_dir = os . path . join ( args . data_dir , 'nmt' ) _ , filepath = Dataset . _valid_path_append ( data_dir , '' , filename ) if not os . path . exists ( filepath ) : Dataset . fetch_dataset ( url , filename , filepath , size ) # extract selected datasets datafiles = dict ( ) datafiles [ 'un2000' ] = ( 'un2000_pc34.en.gz' , 'un2000_pc34.fr.gz' ) datafiles [ 'europarl7' ] = ( 'ep7_pc45.en.gz' , 'ep7_pc45.fr.gz' ) extractpath = os . path . join ( data_dir , 'bitexts.selected' ) with tarfile . open ( filepath , 'r' ) as tar_ref : for dset , files in datafiles . items ( ) : datasetpath = os . path . join ( data_dir , dset ) # extract the files for dataset, if not already there for zipped in files : fname = '.' . join ( zipped . split ( '.' ) [ : - 1 ] ) fpath = os . path . join ( datasetpath , fname ) if not os . path . exists ( fpath ) : gzpath = os . path . join ( extractpath , zipped ) if not os . path . exists ( gzpath ) : select = [ ti for ti in tar_ref if os . path . split ( ti . name ) [ 1 ] == zipped ] tar_ref . extractall ( path = data_dir , members = select ) # get contents of gz files if not os . path . exists ( datasetpath ) : os . makedirs ( datasetpath ) with gzip . open ( gzpath , 'r' ) as fin , open ( fpath , 'w' ) as fout : fout . write ( fin . read ( ) ) os . remove ( gzpath ) if os . path . exists ( extractpath ) : os . rmdir ( extractpath ) # process data and save to h5 file # loop through all datasets and get train and valid splits for dataset in datafiles . keys ( ) : s_vocab , t_vocab = create_h5py ( data_dir , dataset , 'train' , vocab_size = vocab_size , time_steps = time_steps ) create_h5py ( data_dir , dataset , 'valid' , s_vocab = s_vocab , t_vocab = t_vocab , time_steps = time_steps )