(path, file_ext=['txt'], valid_split=None, vocab_file_name=None,
              max_vocab_size=None, max_len_w=None, output_path=None, subset_pct=100)
return h5_file_name, vocab_file_name
Given a path where data are saved, look for the ones with the right extensions
    If a split factor is given, it will split all the files into training and valid
    set. Then build vocabulary from the training and validation sets.

    Arguments:
        path: which directory to look for all the documents
        file_ext: what extension of the files to look for
        valid_split: to split the data into train/valid set. If None, no split
        vocab_file_name: optional file name. If None, the script will decide a name
                         given path and split
        max_vocab_size: maximum number of words to use in vocabulary (by most frequent)
        max_len_w: maximum length of sentences in words
        output_path: path used to save preprocessed data and resuts
        subset_pct: subset of dataset to load into H5 file (percentage)

    Returns:
        The function saves 2 files:
        h5 file with preprocessed data
        vocabulary file with: vocab, reverse_vocab, word_count
Given a path where data are saved, look for the ones with the right extensions
    If a split factor is given, it will split all the files into training and valid
    set. Then build vocabulary from the training and validation sets.
Given a path where data are saved look for the ones with the right extensions If a split factor is given it will split all the files into training and valid set . Then build vocabulary from the training and validation sets .
def load_data(path, file_ext=['txt'], valid_split=None, vocab_file_name=None,
              max_vocab_size=None, max_len_w=None, output_path=None, subset_pct=100):
    """
    Given a path where data are saved, look for the ones with the right extensions
    If a split factor is given, it will split all the files into training and valid
    set. Then build vocabulary from the training and validation sets.

    Arguments:
        path: which directory to look for all the documents
        file_ext: what extension of the files to look for
        valid_split: to split the data into train/valid set. If None, no split
        vocab_file_name: optional file name. If None, the script will decide a name
                         given path and split
        max_vocab_size: maximum number of words to use in vocabulary (by most frequent)
        max_len_w: maximum length of sentences in words
        output_path: path used to save preprocessed data and resuts
        subset_pct: subset of dataset to load into H5 file (percentage)

    Returns:
        The function saves 2 files:
        h5 file with preprocessed data
        vocabulary file with: vocab, reverse_vocab, word_count
    """
    file_names = get_file_list(path, file_ext)

    file_str = get_file_str(path, len(file_names), labelled=False,
                            valid_split=valid_split, subset_pct=subset_pct)

    # create output dir if needed
    if not os.path.isdir(output_path):
        os.makedirs(output_path)

    # file name to store the vocabulary
    if vocab_file_name is None:
        vocab_file_name = file_str + '.vocab'
        vocab_file_name = os.path.join(output_path, vocab_file_name)

    # If max sizes arent set, assume no limit
    if not max_len_w:
        max_len_w = sys.maxsize
    if not max_vocab_size:
        max_vocab_size = sys.maxsize

    # file name to store the pre-processed train/valid dataset
    h5_file_name = os.path.join(output_path, file_str + '.h5')

    if os.path.exists(h5_file_name) and os.path.exists(vocab_file_name):
        neon_logger.display("dataset files {} and vocabulary file {} already exist. "
                            "will use cached data. ".format(h5_file_name, vocab_file_name))
        return h5_file_name, vocab_file_name

    # split into training/valid set
    if valid_split is not None:
        if 'json' in file_ext:
            # Split based on number of files
            train_split = int(np.ceil(len(file_names) * (1 - valid_split)))
            train_files = file_names[:train_split]
            valid_files = file_names[train_split:]

            train_sent = load_json_sent(train_files, subset_pct)
            valid_sent = load_json_sent(valid_files, subset_pct)
            all_sent = train_sent + valid_sent
        elif 'txt' in file_ext:
            # Split based on number of lines (since only 2 files)
            all_sent = load_txt_sent(file_names, subset_pct)
            train_split = int(np.ceil(len(all_sent) * (1 - valid_split)))

            train_sent = all_sent[:train_split]
            valid_sent = all_sent[train_split:]
        else:
            neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'."
                                .format(file_ext))
    else:
        train_files = file_names
        if 'json' in file_ext:
            train_sent = load_json_sent(train_files, subset_pct)
        elif 'txt' in file_ext:
            train_sent = load_txt_sent(train_files, subset_pct)
        else:
            neon_logger.display("Unsure how to load file_ext {}, please use 'json' or 'txt'."
                                .format(file_ext))
        all_sent = train_sent

    if os.path.exists(vocab_file_name):
        neon_logger.display("open existing vocab file: {}".format(vocab_file_name))
        vocab, rev_vocab, word_count = load_obj(vocab_file_name)
    else:
        neon_logger.display("Building  vocab file")

        # build vocab
        word_count = defaultdict(int)
        for sent in all_sent:
            sent_words = tokenize(sent)

            if len(sent_words) > max_len_w or len(sent_words) == 0:
                continue

            for word in sent_words:
                word_count[word] += 1

        # sort the word_count , re-assign ids by its frequency. Useful for downstream tasks
        # only done for train vocab
        vocab_sorted = sorted(word_count.items(), key=lambda kv: kv[1], reverse=True)

        vocab = OrderedDict()

        # get word count as array in same ordering as vocab (but with maximum length)
        word_count_ = np.zeros((len(word_count), ), dtype=np.int64)
        for i, t in enumerate(list(zip(*vocab_sorted))[0][:max_vocab_size]):
            word_count_[i] = word_count[t]
            vocab[t] = i
        word_count = word_count_

        # generate the reverse vocab
        rev_vocab = dict((wrd_id, wrd) for wrd, wrd_id in vocab.items())

        neon_logger.display("vocabulary from {} is saved into {}".format(path, vocab_file_name))
        save_obj((vocab, rev_vocab, word_count), vocab_file_name)

    vocab_size = len(vocab)
    neon_logger.display("\nVocab size from the dataset is: {}".format(vocab_size))

    neon_logger.display("\nProcessing and saving training data into {}".format(h5_file_name))

    # now process and save the train/valid data
    h5f = h5py.File(h5_file_name, 'w', libver='latest')
    shape, maxshape = (len(train_sent),), (None)
    dt = np.dtype([('text', h5py.special_dtype(vlen=str)),
                   ('num_words', np.uint16)])
    report_text_train = h5f.create_dataset('report_train', shape=shape,
                                           maxshape=maxshape, dtype=dt,
                                           compression='gzip')
    report_train = h5f.create_dataset('train', shape=shape, maxshape=maxshape,
                                      dtype=h5py.special_dtype(vlen=np.int32),
                                      compression='gzip')

    # map text to integers
    wdata = np.zeros((1, ), dtype=dt)
    ntrain = 0
    for sent in train_sent:
        text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)]

        # enforce maximum sentence length
        if len(text_int) > max_len_w or len(text_int) == 0:
            continue

        report_train[ntrain] = text_int

        wdata['text'] = clean_string(sent)
        wdata['num_words'] = len(text_int)
        report_text_train[ntrain] = wdata
        ntrain += 1

    report_train.attrs['nsample'] = ntrain
    report_train.attrs['vocab_size'] = vocab_size
    report_text_train.attrs['nsample'] = ntrain
    report_text_train.attrs['vocab_size'] = vocab_size

    if valid_split:
        neon_logger.display("\nProcessing and saving validation data into {}".format(h5_file_name))
        shape = (len(valid_sent),)
        report_text_valid = h5f.create_dataset('report_valid', shape=shape,
                                               maxshape=maxshape, dtype=dt,
                                               compression='gzip')
        report_valid = h5f.create_dataset('valid', shape=shape, maxshape=maxshape,
                                          dtype=h5py.special_dtype(vlen=np.int32),
                                          compression='gzip')
        nvalid = 0
        for sent in valid_sent:
            text_int = [-1 if t not in vocab else vocab[t] for t in tokenize(sent)]

            # enforce maximum sentence length
            if len(text_int) > max_len_w or len(text_int) == 0:
                continue

            report_valid[nvalid] = text_int
            wdata['text'] = clean_string(sent)
            wdata['num_words'] = len(text_int)
            report_text_valid[nvalid] = wdata
            nvalid += 1

        report_valid.attrs['nsample'] = nvalid
        report_valid.attrs['vocab_size'] = vocab_size
        report_text_valid.attrs['nsample'] = nvalid
        report_text_valid.attrs['vocab_size'] = vocab_size

    h5f.close()

    return h5_file_name, vocab_file_name
def load_data ( path , file_ext = [ 'txt' ] , valid_split = None , vocab_file_name = None , max_vocab_size = None , max_len_w = None , output_path = None , subset_pct = 100 ) : file_names = get_file_list ( path , file_ext ) file_str = get_file_str ( path , len ( file_names ) , labelled = False , valid_split = valid_split , subset_pct = subset_pct ) # create output dir if needed if not os . path . isdir ( output_path ) : os . makedirs ( output_path ) # file name to store the vocabulary if vocab_file_name is None : vocab_file_name = file_str + '.vocab' vocab_file_name = os . path . join ( output_path , vocab_file_name ) # If max sizes arent set, assume no limit if not max_len_w : max_len_w = sys . maxsize if not max_vocab_size : max_vocab_size = sys . maxsize # file name to store the pre-processed train/valid dataset h5_file_name = os . path . join ( output_path , file_str + '.h5' ) if os . path . exists ( h5_file_name ) and os . path . exists ( vocab_file_name ) : neon_logger . display ( "dataset files {} and vocabulary file {} already exist. " "will use cached data. " . format ( h5_file_name , vocab_file_name ) ) return h5_file_name , vocab_file_name # split into training/valid set if valid_split is not None : if 'json' in file_ext : # Split based on number of files train_split = int ( np . ceil ( len ( file_names ) * ( 1 - valid_split ) ) ) train_files = file_names [ : train_split ] valid_files = file_names [ train_split : ] train_sent = load_json_sent ( train_files , subset_pct ) valid_sent = load_json_sent ( valid_files , subset_pct ) all_sent = train_sent + valid_sent elif 'txt' in file_ext : # Split based on number of lines (since only 2 files) all_sent = load_txt_sent ( file_names , subset_pct ) train_split = int ( np . ceil ( len ( all_sent ) * ( 1 - valid_split ) ) ) train_sent = all_sent [ : train_split ] valid_sent = all_sent [ train_split : ] else : neon_logger . display ( "Unsure how to load file_ext {}, please use 'json' or 'txt'." . format ( file_ext ) ) else : train_files = file_names if 'json' in file_ext : train_sent = load_json_sent ( train_files , subset_pct ) elif 'txt' in file_ext : train_sent = load_txt_sent ( train_files , subset_pct ) else : neon_logger . display ( "Unsure how to load file_ext {}, please use 'json' or 'txt'." . format ( file_ext ) ) all_sent = train_sent if os . path . exists ( vocab_file_name ) : neon_logger . display ( "open existing vocab file: {}" . format ( vocab_file_name ) ) vocab , rev_vocab , word_count = load_obj ( vocab_file_name ) else : neon_logger . display ( "Building  vocab file" ) # build vocab word_count = defaultdict ( int ) for sent in all_sent : sent_words = tokenize ( sent ) if len ( sent_words ) > max_len_w or len ( sent_words ) == 0 : continue for word in sent_words : word_count [ word ] += 1 # sort the word_count , re-assign ids by its frequency. Useful for downstream tasks # only done for train vocab vocab_sorted = sorted ( word_count . items ( ) , key = lambda kv : kv [ 1 ] , reverse = True ) vocab = OrderedDict ( ) # get word count as array in same ordering as vocab (but with maximum length) word_count_ = np . zeros ( ( len ( word_count ) , ) , dtype = np . int64 ) for i , t in enumerate ( list ( zip ( * vocab_sorted ) ) [ 0 ] [ : max_vocab_size ] ) : word_count_ [ i ] = word_count [ t ] vocab [ t ] = i word_count = word_count_ # generate the reverse vocab rev_vocab = dict ( ( wrd_id , wrd ) for wrd , wrd_id in vocab . items ( ) ) neon_logger . display ( "vocabulary from {} is saved into {}" . format ( path , vocab_file_name ) ) save_obj ( ( vocab , rev_vocab , word_count ) , vocab_file_name ) vocab_size = len ( vocab ) neon_logger . display ( "\nVocab size from the dataset is: {}" . format ( vocab_size ) ) neon_logger . display ( "\nProcessing and saving training data into {}" . format ( h5_file_name ) ) # now process and save the train/valid data h5f = h5py . File ( h5_file_name , 'w' , libver = 'latest' ) shape , maxshape = ( len ( train_sent ) , ) , ( None ) dt = np . dtype ( [ ( 'text' , h5py . special_dtype ( vlen = str ) ) , ( 'num_words' , np . uint16 ) ] ) report_text_train = h5f . create_dataset ( 'report_train' , shape = shape , maxshape = maxshape , dtype = dt , compression = 'gzip' ) report_train = h5f . create_dataset ( 'train' , shape = shape , maxshape = maxshape , dtype = h5py . special_dtype ( vlen = np . int32 ) , compression = 'gzip' ) # map text to integers wdata = np . zeros ( ( 1 , ) , dtype = dt ) ntrain = 0 for sent in train_sent : text_int = [ - 1 if t not in vocab else vocab [ t ] for t in tokenize ( sent ) ] # enforce maximum sentence length if len ( text_int ) > max_len_w or len ( text_int ) == 0 : continue report_train [ ntrain ] = text_int wdata [ 'text' ] = clean_string ( sent ) wdata [ 'num_words' ] = len ( text_int ) report_text_train [ ntrain ] = wdata ntrain += 1 report_train . attrs [ 'nsample' ] = ntrain report_train . attrs [ 'vocab_size' ] = vocab_size report_text_train . attrs [ 'nsample' ] = ntrain report_text_train . attrs [ 'vocab_size' ] = vocab_size if valid_split : neon_logger . display ( "\nProcessing and saving validation data into {}" . format ( h5_file_name ) ) shape = ( len ( valid_sent ) , ) report_text_valid = h5f . create_dataset ( 'report_valid' , shape = shape , maxshape = maxshape , dtype = dt , compression = 'gzip' ) report_valid = h5f . create_dataset ( 'valid' , shape = shape , maxshape = maxshape , dtype = h5py . special_dtype ( vlen = np . int32 ) , compression = 'gzip' ) nvalid = 0 for sent in valid_sent : text_int = [ - 1 if t not in vocab else vocab [ t ] for t in tokenize ( sent ) ] # enforce maximum sentence length if len ( text_int ) > max_len_w or len ( text_int ) == 0 : continue report_valid [ nvalid ] = text_int wdata [ 'text' ] = clean_string ( sent ) wdata [ 'num_words' ] = len ( text_int ) report_text_valid [ nvalid ] = wdata nvalid += 1 report_valid . attrs [ 'nsample' ] = nvalid report_valid . attrs [ 'vocab_size' ] = vocab_size report_text_valid . attrs [ 'nsample' ] = nvalid report_text_valid . attrs [ 'vocab_size' ] = vocab_size h5f . close ( ) return h5_file_name , vocab_file_name