(self, data_file=None, sent_name=None, text_name=None,
                 nwords=None, max_len=30, index_from=2, eos=3)
Construct a sentence dataset object.
        Build the context using skip-thought model

        Aguments:
            data_file (str): path to hdf5 file containing sentences
            sent_name (str): name of tokenized dataset
            text_name (str): name of raw text dataset
            nwords (int): size of vocabulary
            max_len (int): maximum number of words per sentence
            index_from (int): index offset for padding (0) and OOV (1)
            eos (int): index of EOS token
Construct a sentence dataset object.
        Build the context using skip-thought model
Construct a sentence dataset object . Build the context using skip - thought model
def __init__(self, data_file=None, sent_name=None, text_name=None,
                 nwords=None, max_len=30, index_from=2, eos=3):
        """
        Construct a sentence dataset object.
        Build the context using skip-thought model

        Aguments:
            data_file (str): path to hdf5 file containing sentences
            sent_name (str): name of tokenized dataset
            text_name (str): name of raw text dataset
            nwords (int): size of vocabulary
            max_len (int): maximum number of words per sentence
            index_from (int): index offset for padding (0) and OOV (1)
            eos (int): index of EOS token
        """
        super(SentenceHomogenous, self).__init__(name=None)
        self.nwords = nwords
        self.batch_index = 0
        self.nbatches = 0
        self.max_len = max_len
        self.index_from = index_from
        self.eos = eos
        self.data_file = data_file
        self.sent_name = sent_name
        self.text_name = text_name

        h5f = h5py.File(self.data_file, 'r+')

        # Extract sentences array from h5 file and make copy in memory
        sentences = h5f[self.sent_name][:]

        # Load sentence raw text if desired
        # sentence_text = h5f[self.text_name]

        self.nsamples = h5f[self.sent_name].attrs['nsample'] - 2

        # Use shifted view of in-memory copy of sentences to group sentences into triplets
        self.source = sentences[1:-1]
        self.forward = sentences[2:]
        self.backward = sentences[:-2]

        self.lengths = [len(cc) for cc in self.source]
        self.len_unique = np.unique(self.lengths)
        self.len_unique = [ll for ll in self.len_unique if ll <= self.max_len]

        self.len_indicies = dict()
        self.len_counts = dict()
        for ll in self.len_unique:
            self.len_indicies[ll] = np.where(self.lengths == ll)[0]
            self.len_counts[ll] = len(self.len_indicies[ll])

        # Compute number of batches of homogenous lengths
        self.nbatches = 0
        for ll in self.len_unique:
            self.nbatches += int(np.ceil(self.len_counts[ll] / float(self.be.bsz)))

        # compute the total number of samples (including empty samples in minibatches)
        self.ndata = self.nbatches * self.be.bsz

        self.len_curr_counts = copy.copy(self.len_counts)

        # inputs using embeddings
        self.dev_X = self.be.iobuf(self.max_len, dtype=np.int32)
        self.dev_X_p = self.be.iobuf(self.max_len, dtype=np.int32)  # previous sentence as input
        self.dev_X_n = self.be.iobuf(self.max_len, dtype=np.int32)  # next sentence as input

        # the np array to put noncontiguous data in. use for entire minibatch
        self.X_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)
        self.X_p_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)
        self.X_n_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)

        # flat output labels, do one-hot on device
        self.dev_y_p_flat = self.be.iobuf((1, self.max_len), dtype=np.int32)
        self.dev_y_n_flat = self.be.iobuf((1, self.max_len), dtype=np.int32)
        self.dev_y_p = self.be.iobuf((nwords, self.max_len), dtype=np.int32)
        self.dev_y_n = self.be.iobuf((nwords, self.max_len), dtype=np.int32)

        # output labels and masks to deal with variable length sentences
        self.dev_y_p_mask = self.be.iobuf((nwords, self.max_len), dtype=np.int32)
        self.dev_y_n_mask = self.be.iobuf((nwords, self.max_len), dtype=np.int32)
        self.dev_y_p_mask_list = self.get_bsz(self.dev_y_p_mask, self.max_len)
        self.dev_y_n_mask_list = self.get_bsz(self.dev_y_n_mask, self.max_len)

        # for the flat label
        self.y_p_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)
        self.y_n_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)

        self.clear_list = [self.X_np, self.X_p_np, self.X_n_np,
                           self.y_p_np, self.y_n_np,
                           self.dev_y_p_mask,
                           self.dev_y_n_mask]
        self.shape = [(self.max_len, 1), (self.max_len, 1), (self.max_len, 1)]

        h5f.close()
        self.reset()
def __init__ ( self , data_file = None , sent_name = None , text_name = None , nwords = None , max_len = 30 , index_from = 2 , eos = 3 ) : super ( SentenceHomogenous , self ) . __init__ ( name = None ) self . nwords = nwords self . batch_index = 0 self . nbatches = 0 self . max_len = max_len self . index_from = index_from self . eos = eos self . data_file = data_file self . sent_name = sent_name self . text_name = text_name h5f = h5py . File ( self . data_file , 'r+' ) # Extract sentences array from h5 file and make copy in memory sentences = h5f [ self . sent_name ] [ : ] # Load sentence raw text if desired # sentence_text = h5f[self.text_name] self . nsamples = h5f [ self . sent_name ] . attrs [ 'nsample' ] - 2 # Use shifted view of in-memory copy of sentences to group sentences into triplets self . source = sentences [ 1 : - 1 ] self . forward = sentences [ 2 : ] self . backward = sentences [ : - 2 ] self . lengths = [ len ( cc ) for cc in self . source ] self . len_unique = np . unique ( self . lengths ) self . len_unique = [ ll for ll in self . len_unique if ll <= self . max_len ] self . len_indicies = dict ( ) self . len_counts = dict ( ) for ll in self . len_unique : self . len_indicies [ ll ] = np . where ( self . lengths == ll ) [ 0 ] self . len_counts [ ll ] = len ( self . len_indicies [ ll ] ) # Compute number of batches of homogenous lengths self . nbatches = 0 for ll in self . len_unique : self . nbatches += int ( np . ceil ( self . len_counts [ ll ] / float ( self . be . bsz ) ) ) # compute the total number of samples (including empty samples in minibatches) self . ndata = self . nbatches * self . be . bsz self . len_curr_counts = copy . copy ( self . len_counts ) # inputs using embeddings self . dev_X = self . be . iobuf ( self . max_len , dtype = np . int32 ) self . dev_X_p = self . be . iobuf ( self . max_len , dtype = np . int32 ) # previous sentence as input self . dev_X_n = self . be . iobuf ( self . max_len , dtype = np . int32 ) # next sentence as input # the np array to put noncontiguous data in. use for entire minibatch self . X_np = np . empty ( ( self . max_len , self . be . bsz ) , dtype = np . int32 ) self . X_p_np = np . empty ( ( self . max_len , self . be . bsz ) , dtype = np . int32 ) self . X_n_np = np . empty ( ( self . max_len , self . be . bsz ) , dtype = np . int32 ) # flat output labels, do one-hot on device self . dev_y_p_flat = self . be . iobuf ( ( 1 , self . max_len ) , dtype = np . int32 ) self . dev_y_n_flat = self . be . iobuf ( ( 1 , self . max_len ) , dtype = np . int32 ) self . dev_y_p = self . be . iobuf ( ( nwords , self . max_len ) , dtype = np . int32 ) self . dev_y_n = self . be . iobuf ( ( nwords , self . max_len ) , dtype = np . int32 ) # output labels and masks to deal with variable length sentences self . dev_y_p_mask = self . be . iobuf ( ( nwords , self . max_len ) , dtype = np . int32 ) self . dev_y_n_mask = self . be . iobuf ( ( nwords , self . max_len ) , dtype = np . int32 ) self . dev_y_p_mask_list = self . get_bsz ( self . dev_y_p_mask , self . max_len ) self . dev_y_n_mask_list = self . get_bsz ( self . dev_y_n_mask , self . max_len ) # for the flat label self . y_p_np = np . empty ( ( self . max_len , self . be . bsz ) , dtype = np . int32 ) self . y_n_np = np . empty ( ( self . max_len , self . be . bsz ) , dtype = np . int32 ) self . clear_list = [ self . X_np , self . X_p_np , self . X_n_np , self . y_p_np , self . y_n_np , self . dev_y_p_mask , self . dev_y_n_mask ] self . shape = [ ( self . max_len , 1 ) , ( self . max_len , 1 ) , ( self . max_len , 1 ) ] h5f . close ( ) self . reset ( )