(corpus_type, fields, opt)
return [pt_file]
Building and saving the dataset
Building and saving the dataset
Building and saving the dataset
def build_save_dataset(corpus_type, fields, opt):
    """ Building and saving the dataset """
    assert corpus_type in ['train', 'valid']

    if corpus_type == 'train':
        src_corpus = opt.train_src
        tgt_corpus = opt.train_tgt
    else:
        src_corpus = opt.valid_src
        tgt_corpus = opt.valid_tgt

    # Currently we only do preprocess sharding for corpus: data_type=='text'.
    if opt.data_type == 'text':
        return build_save_in_shards(
            src_corpus, tgt_corpus, fields,
            corpus_type, opt)

    if (opt.shard_size > 0):
        return build_save_in_shards_using_shards_size(src_corpus,
                                                      tgt_corpus,
                                                      fields,
                                                      corpus_type,
                                                      opt)

    # For data_type == 'img' or 'audio', currently we don't do
    # preprocess sharding. We only build a monolithic dataset.
    # But since the interfaces are uniform, it would be not hard
    # to do this should users need this feature.
    dataset = inputters.build_dataset(
        fields, opt.data_type,
        src_path=src_corpus,
        tgt_path=tgt_corpus,
        src_dir=opt.src_dir,
        src_seq_length=opt.src_seq_length,
        tgt_seq_length=opt.tgt_seq_length,
        src_seq_length_trunc=opt.src_seq_length_trunc,
        tgt_seq_length_trunc=opt.tgt_seq_length_trunc,
        dynamic_dict=opt.dynamic_dict,
        sample_rate=opt.sample_rate,
        window_size=opt.window_size,
        window_stride=opt.window_stride,
        window=opt.window,
        image_channel_size=opt.image_channel_size)

    # We save fields in vocab.pt seperately, so make it empty.
    dataset.fields = []

    pt_file = "{:s}.{:s}.pt".format(opt.save_data, corpus_type)
    logger.info(" * saving %s dataset to %s." % (corpus_type, pt_file))
    torch.save(dataset, pt_file)

    return [pt_file]
def build_save_dataset ( corpus_type , fields , opt ) : assert corpus_type in [ 'train' , 'valid' ] if corpus_type == 'train' : src_corpus = opt . train_src tgt_corpus = opt . train_tgt else : src_corpus = opt . valid_src tgt_corpus = opt . valid_tgt # Currently we only do preprocess sharding for corpus: data_type=='text'. if opt . data_type == 'text' : return build_save_in_shards ( src_corpus , tgt_corpus , fields , corpus_type , opt ) if ( opt . shard_size > 0 ) : return build_save_in_shards_using_shards_size ( src_corpus , tgt_corpus , fields , corpus_type , opt ) # For data_type == 'img' or 'audio', currently we don't do # preprocess sharding. We only build a monolithic dataset. # But since the interfaces are uniform, it would be not hard # to do this should users need this feature. dataset = inputters . build_dataset ( fields , opt . data_type , src_path = src_corpus , tgt_path = tgt_corpus , src_dir = opt . src_dir , src_seq_length = opt . src_seq_length , tgt_seq_length = opt . tgt_seq_length , src_seq_length_trunc = opt . src_seq_length_trunc , tgt_seq_length_trunc = opt . tgt_seq_length_trunc , dynamic_dict = opt . dynamic_dict , sample_rate = opt . sample_rate , window_size = opt . window_size , window_stride = opt . window_stride , window = opt . window , image_channel_size = opt . image_channel_size ) # We save fields in vocab.pt seperately, so make it empty. dataset . fields = [ ] pt_file = "{:s}.{:s}.pt" . format ( opt . save_data , corpus_type ) logger . info ( " * saving %s dataset to %s." % ( corpus_type , pt_file ) ) torch . save ( dataset , pt_file ) return [ pt_file ]