(self)
return (self.dev_X, self.dev_X_p, self.dev_X_n), \
            ((self.dev_y_p, self.dev_y_p_mask), (self.dev_y_n, self.dev_y_n_mask))
Method called by iterator to get a new batch of sentence triplets:
        (source, forward, backward). Sentences are returned in order of increasing length,
        and source sentences of each batch all have the same length.
Method called by iterator to get a new batch of sentence triplets:
        (source, forward, backward). Sentences are returned in order of increasing length,
        and source sentences of each batch all have the same length.
Method called by iterator to get a new batch of sentence triplets : ( source forward backward ) . Sentences are returned in order of increasing length and source sentences of each batch all have the same length .
def next(self):
        """
        Method called by iterator to get a new batch of sentence triplets:
        (source, forward, backward). Sentences are returned in order of increasing length,
        and source sentences of each batch all have the same length.
        """
        self.clear_device_buffer()

        # Select the next length which we havent used up yet
        count = 0
        while True:
            self.len_idx = np.mod(self.len_idx+1, len(self.len_unique))
            if self.len_curr_counts[self.len_unique[self.len_idx]] > 0:
                break
            count += 1
            if count >= len(self.len_unique):
                break

        if count >= len(self.len_unique):
            self.reset()
            raise StopIteration()

        curr_len = self.len_unique[self.len_idx]
        # get the batch size
        curr_batch_size = np.minimum(self.be.bsz,
                                     self.len_curr_counts[curr_len])
        curr_pos = self.len_indices_pos[curr_len]

        curr_indices = self.len_indicies[curr_len][curr_pos:curr_pos+curr_batch_size]
        self.len_indices_pos[curr_len] += curr_batch_size
        self.len_curr_counts[curr_len] -= curr_batch_size

        # 'feats' corresponds to the after and before sentences
        source_batch = [self.source[ii] for ii in curr_indices]
        forward_batch = [self.forward[ii] for ii in curr_indices]
        backward_batch = [self.backward[ii] for ii in curr_indices]

        # Loop over the batch and clip by length, add eos, and flip decoder sentences
        for i in range(len(source_batch)):
            l_s = min(len(source_batch[i]), self.max_len)

            if len(source_batch[i][-l_s:]) == 0:
                continue

            # NO FLIPPING of the source sentence
            self.X_np[-l_s:, i] = source_batch[i][-l_s:] + self.index_from

            l_p = min(len(backward_batch[i]), self.max_len)

            # clip a long sentence from the left
            # for decoder input: take the sent_length-1, prepend a <eos>
            # for decoder output: take the sent_length
            self.X_p_np[:l_p, i] = [self.eos] + (backward_batch[i][-l_p:-1] +
                                                 self.index_from).tolist()

            self.y_p_np[:l_p, i] = backward_batch[i][-l_p:] + self.index_from
            self.dev_y_p_mask_list[i][:, :l_p] = 1

            l_n = min(len(forward_batch[i]), self.max_len)

            self.X_n_np[:l_n, i] = [self.eos] + (forward_batch[i][-l_n:-1] +
                                                 self.index_from).tolist()

            self.y_n_np[:l_n, i] = forward_batch[i][-l_n:] + self.index_from
            self.dev_y_n_mask_list[i][:, :l_n] = 1

        self.dev_X.set(self.X_np)
        self.dev_X_p.set(self.X_p_np)
        self.dev_X_n.set(self.X_n_np)

        self.dev_y_p_flat.set(self.y_p_np.reshape(1, -1))
        self.dev_y_n_flat.set(self.y_n_np.reshape(1, -1))

        self.dev_y_p[:] = self.be.onehot(self.dev_y_p_flat, axis=0)
        self.dev_y_n[:] = self.be.onehot(self.dev_y_n_flat, axis=0)

        self.batch_index += 1

        return (self.dev_X, self.dev_X_p, self.dev_X_n), \
            ((self.dev_y_p, self.dev_y_p_mask), (self.dev_y_n, self.dev_y_n_mask))
def next ( self ) : self . clear_device_buffer ( ) # Select the next length which we havent used up yet count = 0 while True : self . len_idx = np . mod ( self . len_idx + 1 , len ( self . len_unique ) ) if self . len_curr_counts [ self . len_unique [ self . len_idx ] ] > 0 : break count += 1 if count >= len ( self . len_unique ) : break if count >= len ( self . len_unique ) : self . reset ( ) raise StopIteration ( ) curr_len = self . len_unique [ self . len_idx ] # get the batch size curr_batch_size = np . minimum ( self . be . bsz , self . len_curr_counts [ curr_len ] ) curr_pos = self . len_indices_pos [ curr_len ] curr_indices = self . len_indicies [ curr_len ] [ curr_pos : curr_pos + curr_batch_size ] self . len_indices_pos [ curr_len ] += curr_batch_size self . len_curr_counts [ curr_len ] -= curr_batch_size # 'feats' corresponds to the after and before sentences source_batch = [ self . source [ ii ] for ii in curr_indices ] forward_batch = [ self . forward [ ii ] for ii in curr_indices ] backward_batch = [ self . backward [ ii ] for ii in curr_indices ] # Loop over the batch and clip by length, add eos, and flip decoder sentences for i in range ( len ( source_batch ) ) : l_s = min ( len ( source_batch [ i ] ) , self . max_len ) if len ( source_batch [ i ] [ - l_s : ] ) == 0 : continue # NO FLIPPING of the source sentence self . X_np [ - l_s : , i ] = source_batch [ i ] [ - l_s : ] + self . index_from l_p = min ( len ( backward_batch [ i ] ) , self . max_len ) # clip a long sentence from the left # for decoder input: take the sent_length-1, prepend a <eos> # for decoder output: take the sent_length self . X_p_np [ : l_p , i ] = [ self . eos ] + ( backward_batch [ i ] [ - l_p : - 1 ] + self . index_from ) . tolist ( ) self . y_p_np [ : l_p , i ] = backward_batch [ i ] [ - l_p : ] + self . index_from self . dev_y_p_mask_list [ i ] [ : , : l_p ] = 1 l_n = min ( len ( forward_batch [ i ] ) , self . max_len ) self . X_n_np [ : l_n , i ] = [ self . eos ] + ( forward_batch [ i ] [ - l_n : - 1 ] + self . index_from ) . tolist ( ) self . y_n_np [ : l_n , i ] = forward_batch [ i ] [ - l_n : ] + self . index_from self . dev_y_n_mask_list [ i ] [ : , : l_n ] = 1 self . dev_X . set ( self . X_np ) self . dev_X_p . set ( self . X_p_np ) self . dev_X_n . set ( self . X_n_np ) self . dev_y_p_flat . set ( self . y_p_np . reshape ( 1 , - 1 ) ) self . dev_y_n_flat . set ( self . y_n_np . reshape ( 1 , - 1 ) ) self . dev_y_p [ : ] = self . be . onehot ( self . dev_y_p_flat , axis = 0 ) self . dev_y_n [ : ] = self . be . onehot ( self . dev_y_n_flat , axis = 0 ) self . batch_index += 1 return ( self . dev_X , self . dev_X_p , self . dev_X_n ) , ( ( self . dev_y_p , self . dev_y_p_mask ) , ( self . dev_y_n , self . dev_y_n_mask ) )