(self, path, tokenizer, vocab)
return X, y
Tokenizer and vocab are unused but provided to match superclass method signature
Tokenizer and vocab are unused but provided to match superclass method signature
Tokenizer and vocab are unused but provided to match superclass method signature
def _get_data(self, path, tokenizer, vocab):
        """
        Tokenizer and vocab are unused but provided to match superclass method signature
        """
        def vocab_to_dicts(vocab):
            t2i = dict((t, i) for i, t in enumerate(vocab))
            i2t = dict((i, t) for i, t in enumerate(vocab))
            return t2i, i2t

        # get saved processed data
        logger.debug("Loading parsed data from %s", path)
        with h5py.File(path, 'r') as f:
            self.s_vocab = f['s_vocab'][:].tolist()
            self.t_vocab = f['t_vocab'][:].tolist()
            self.s_token_to_index, self.s_index_to_token = vocab_to_dicts(self.s_vocab)
            self.t_token_to_index, self.t_index_to_token = vocab_to_dicts(self.t_vocab)
            X = f['X'][:]
            y = f['y'][:]
        self.nclass = len(self.t_vocab)

        # Trim subset and patial minibatch
        if self.subset_pct < 100:
            X = X[:int(X.shape[0] * self.subset_pct / 100.), :]
            y = y[:int(y.shape[0] * self.subset_pct / 100.), :]
            logger.debug("subset %d%% of data", self.subset_pct*100)

        extra_sentences = X.shape[0] % self.be.bsz
        if extra_sentences:
            X = X[:-extra_sentences, :]
            y = y[:-extra_sentences, :]
            logger.debug("removing %d extra sentences", extra_sentences)
        self.nbatches = X.shape[0] // self.be.bsz
        self.ndata = self.nbatches * self.be.bsz  # no leftovers

        return X, y
def _get_data ( self , path , tokenizer , vocab ) : def vocab_to_dicts ( vocab ) : t2i = dict ( ( t , i ) for i , t in enumerate ( vocab ) ) i2t = dict ( ( i , t ) for i , t in enumerate ( vocab ) ) return t2i , i2t # get saved processed data logger . debug ( "Loading parsed data from %s" , path ) with h5py . File ( path , 'r' ) as f : self . s_vocab = f [ 's_vocab' ] [ : ] . tolist ( ) self . t_vocab = f [ 't_vocab' ] [ : ] . tolist ( ) self . s_token_to_index , self . s_index_to_token = vocab_to_dicts ( self . s_vocab ) self . t_token_to_index , self . t_index_to_token = vocab_to_dicts ( self . t_vocab ) X = f [ 'X' ] [ : ] y = f [ 'y' ] [ : ] self . nclass = len ( self . t_vocab ) # Trim subset and patial minibatch if self . subset_pct < 100 : X = X [ : int ( X . shape [ 0 ] * self . subset_pct / 100. ) , : ] y = y [ : int ( y . shape [ 0 ] * self . subset_pct / 100. ) , : ] logger . debug ( "subset %d%% of data" , self . subset_pct * 100 ) extra_sentences = X . shape [ 0 ] % self . be . bsz if extra_sentences : X = X [ : - extra_sentences , : ] y = y [ : - extra_sentences , : ] logger . debug ( "removing %d extra sentences" , extra_sentences ) self . nbatches = X . shape [ 0 ] // self . be . bsz self . ndata = self . nbatches * self . be . bsz # no leftovers return X , y