(train_dataset_files, fields, data_type, share_vocab,
                src_vocab_path, src_vocab_size, src_words_min_frequency,
                tgt_vocab_path, tgt_vocab_size, tgt_words_min_frequency)
return fields
Args:
        train_dataset_files: a list of train dataset pt file.
        fields (dict): fields to build vocab for.
        data_type: "text", "img" or "audio"?
        share_vocab(bool): share source and target vocabulary?
        src_vocab_path(string): Path to src vocabulary file.
        src_vocab_size(int): size of the source vocabulary.
        src_words_min_frequency(int): the minimum frequency needed to
                include a source word in the vocabulary.
        tgt_vocab_path(string): Path to tgt vocabulary file.
        tgt_vocab_size(int): size of the target vocabulary.
        tgt_words_min_frequency(int): the minimum frequency needed to
                include a target word in the vocabulary.

    Returns:
        Dict of Fields
Args:
        train_dataset_files: a list of train dataset pt file.
        fields (dict): fields to build vocab for.
        data_type: "text", "img" or "audio"?
        share_vocab(bool): share source and target vocabulary?
        src_vocab_path(string): Path to src vocabulary file.
        src_vocab_size(int): size of the source vocabulary.
        src_words_min_frequency(int): the minimum frequency needed to
                include a source word in the vocabulary.
        tgt_vocab_path(string): Path to tgt vocabulary file.
        tgt_vocab_size(int): size of the target vocabulary.
        tgt_words_min_frequency(int): the minimum frequency needed to
                include a target word in the vocabulary.
Args : train_dataset_files : a list of train dataset pt file . fields ( dict ) : fields to build vocab for . data_type : text img or audio ? share_vocab ( bool ) : share source and target vocabulary? src_vocab_path ( string ) : Path to src vocabulary file . src_vocab_size ( int ) : size of the source vocabulary . src_words_min_frequency ( int ) : the minimum frequency needed to include a source word in the vocabulary . tgt_vocab_path ( string ) : Path to tgt vocabulary file . tgt_vocab_size ( int ) : size of the target vocabulary . tgt_words_min_frequency ( int ) : the minimum frequency needed to include a target word in the vocabulary .
def build_vocab(train_dataset_files, fields, data_type, share_vocab,
                src_vocab_path, src_vocab_size, src_words_min_frequency,
                tgt_vocab_path, tgt_vocab_size, tgt_words_min_frequency):
    """
    Args:
        train_dataset_files: a list of train dataset pt file.
        fields (dict): fields to build vocab for.
        data_type: "text", "img" or "audio"?
        share_vocab(bool): share source and target vocabulary?
        src_vocab_path(string): Path to src vocabulary file.
        src_vocab_size(int): size of the source vocabulary.
        src_words_min_frequency(int): the minimum frequency needed to
                include a source word in the vocabulary.
        tgt_vocab_path(string): Path to tgt vocabulary file.
        tgt_vocab_size(int): size of the target vocabulary.
        tgt_words_min_frequency(int): the minimum frequency needed to
                include a target word in the vocabulary.

    Returns:
        Dict of Fields
    """
    counter = {}

    # Prop src from field to get lower memory using when training with image
    if data_type == 'img':
        fields.pop("src")

    for k in fields:
        counter[k] = Counter()

    # Load vocabulary
    src_vocab = load_vocabulary(src_vocab_path, tag="source")
    tgt_vocab = load_vocabulary(tgt_vocab_path, tag="target")

    for index, path in enumerate(train_dataset_files):
        dataset = torch.load(path)
        logger.info(" * reloading %s." % path)
        for ex in dataset.examples:
            for k in fields:
                val = getattr(ex, k, None)
                if val is not None and not fields[k].sequential:
                    val = [val]
                elif k == 'src' and src_vocab:
                    val = [item for item in val if item in src_vocab]
                elif k == 'tgt' and tgt_vocab:
                    val = [item for item in val if item in tgt_vocab]
                counter[k].update(val)

        # Drop the none-using from memory but keep the last
        if (index < len(train_dataset_files) - 1):
            dataset.examples = None
            gc.collect()
            del dataset.examples
            gc.collect()
            del dataset
            gc.collect()

    _build_field_vocab(fields["tgt"], counter["tgt"],
                       max_size=tgt_vocab_size,
                       min_freq=tgt_words_min_frequency)
    logger.info(" * tgt vocab size: %d." % len(fields["tgt"].vocab))

    # All datasets have same num of n_tgt_features,
    # getting the last one is OK.
    for j in range(dataset.n_tgt_feats):
        key = "tgt_feat_" + str(j)
        _build_field_vocab(fields[key], counter[key])
        logger.info(" * %s vocab size: %d." % (key,
                                               len(fields[key].vocab)))

    if data_type == 'text':
        _build_field_vocab(fields["src"], counter["src"],
                           max_size=src_vocab_size,
                           min_freq=src_words_min_frequency)
        logger.info(" * src vocab size: %d." % len(fields["src"].vocab))

        # All datasets have same num of n_src_features,
        # getting the last one is OK.
        for j in range(dataset.n_src_feats):
            key = "src_feat_" + str(j)
            _build_field_vocab(fields[key], counter[key])
            logger.info(" * %s vocab size: %d." %
                        (key, len(fields[key].vocab)))

        # Merge the input and output vocabularies.
        if share_vocab:
            # `tgt_vocab_size` is ignored when sharing vocabularies
            logger.info(" * merging src and tgt vocab...")
            merged_vocab = merge_vocabs(
                [fields["src"].vocab, fields["tgt"].vocab],
                vocab_size=src_vocab_size)
            fields["src"].vocab = merged_vocab
            fields["tgt"].vocab = merged_vocab

    return fields
def build_vocab ( train_dataset_files , fields , data_type , share_vocab , src_vocab_path , src_vocab_size , src_words_min_frequency , tgt_vocab_path , tgt_vocab_size , tgt_words_min_frequency ) : counter = { } # Prop src from field to get lower memory using when training with image if data_type == 'img' : fields . pop ( "src" ) for k in fields : counter [ k ] = Counter ( ) # Load vocabulary src_vocab = load_vocabulary ( src_vocab_path , tag = "source" ) tgt_vocab = load_vocabulary ( tgt_vocab_path , tag = "target" ) for index , path in enumerate ( train_dataset_files ) : dataset = torch . load ( path ) logger . info ( " * reloading %s." % path ) for ex in dataset . examples : for k in fields : val = getattr ( ex , k , None ) if val is not None and not fields [ k ] . sequential : val = [ val ] elif k == 'src' and src_vocab : val = [ item for item in val if item in src_vocab ] elif k == 'tgt' and tgt_vocab : val = [ item for item in val if item in tgt_vocab ] counter [ k ] . update ( val ) # Drop the none-using from memory but keep the last if ( index < len ( train_dataset_files ) - 1 ) : dataset . examples = None gc . collect ( ) del dataset . examples gc . collect ( ) del dataset gc . collect ( ) _build_field_vocab ( fields [ "tgt" ] , counter [ "tgt" ] , max_size = tgt_vocab_size , min_freq = tgt_words_min_frequency ) logger . info ( " * tgt vocab size: %d." % len ( fields [ "tgt" ] . vocab ) ) # All datasets have same num of n_tgt_features, # getting the last one is OK. for j in range ( dataset . n_tgt_feats ) : key = "tgt_feat_" + str ( j ) _build_field_vocab ( fields [ key ] , counter [ key ] ) logger . info ( " * %s vocab size: %d." % ( key , len ( fields [ key ] . vocab ) ) ) if data_type == 'text' : _build_field_vocab ( fields [ "src" ] , counter [ "src" ] , max_size = src_vocab_size , min_freq = src_words_min_frequency ) logger . info ( " * src vocab size: %d." % len ( fields [ "src" ] . vocab ) ) # All datasets have same num of n_src_features, # getting the last one is OK. for j in range ( dataset . n_src_feats ) : key = "src_feat_" + str ( j ) _build_field_vocab ( fields [ key ] , counter [ key ] ) logger . info ( " * %s vocab size: %d." % ( key , len ( fields [ key ] . vocab ) ) ) # Merge the input and output vocabularies. if share_vocab : # `tgt_vocab_size` is ignored when sharing vocabularies logger . info ( " * merging src and tgt vocab..." ) merged_vocab = merge_vocabs ( [ fields [ "src" ] . vocab , fields [ "tgt" ] . vocab ] , vocab_size = src_vocab_size ) fields [ "src" ] . vocab = merged_vocab fields [ "tgt" ] . vocab = merged_vocab return fields