(self, time_steps, path, vocab=None, tokenizer=None,
                 onehot_input=True, reverse_target=False, get_prev_target=False)
Construct a text dataset object.

        Arguments:
            time_steps (int) : Length of a sequence.
            path (str) : Path to text file.
            vocab (python.set) : A set of unique tokens.
            tokenizer (function) : Tokenizer function.
            onehot_input (boolean): One-hot representation of input
            reverse_target (boolean): for sequence to sequence models, set to
                                      True to reverse target sequence. Also
                                      disables shifting target by one.
            get_prev_target (boolean): for sequence to sequence models, set to
                                       True for training data to provide correct
                                       target from previous time step as decoder
                                       input. If condition, shape will be a tuple
                                       of shapes, corresponding to encoder and
                                       decoder inputs.
Construct a text dataset object.
Construct a text dataset object .
def __init__(self, time_steps, path, vocab=None, tokenizer=None,
                 onehot_input=True, reverse_target=False, get_prev_target=False):
        """
        Construct a text dataset object.

        Arguments:
            time_steps (int) : Length of a sequence.
            path (str) : Path to text file.
            vocab (python.set) : A set of unique tokens.
            tokenizer (function) : Tokenizer function.
            onehot_input (boolean): One-hot representation of input
            reverse_target (boolean): for sequence to sequence models, set to
                                      True to reverse target sequence. Also
                                      disables shifting target by one.
            get_prev_target (boolean): for sequence to sequence models, set to
                                       True for training data to provide correct
                                       target from previous time step as decoder
                                       input. If condition, shape will be a tuple
                                       of shapes, corresponding to encoder and
                                       decoder inputs.
        """
        super(Text, self).__init__(name=None)

        self.seq_length = time_steps
        self.onehot_input = onehot_input
        self.batch_index = 0
        self.reverse_target = reverse_target
        self.get_prev_target = get_prev_target

        X, y = self._get_data(path, tokenizer, vocab)

        # reshape to preserve sentence continuity across batches
        self.X = X.reshape(self.be.bsz, self.nbatches, time_steps)
        self.y = y.reshape(self.be.bsz, self.nbatches, time_steps)

        # stuff below this comment needs to be cleaned up and commented
        self.nout = self.nclass
        if self.onehot_input:
            self.shape = (self.nout, time_steps)
            self.dev_X = self.be.iobuf((self.nout, time_steps))
            if self.get_prev_target:
                self.dev_Z = self.be.iobuf((self.nout, time_steps))
        else:
            self.shape = (time_steps, 1)
            self.dev_X = self.be.iobuf(time_steps, dtype=np.int32)
            if self.get_prev_target:
                self.dev_Z = self.be.iobuf(time_steps, dtype=np.int32)
        self.decoder_shape = self.shape

        self.dev_y = self.be.iobuf((self.nout, time_steps))
        self.dev_lbl = self.be.iobuf(time_steps, dtype=np.int32)
        self.dev_lblflat = self.dev_lbl.reshape((1, -1))
def __init__ ( self , time_steps , path , vocab = None , tokenizer = None , onehot_input = True , reverse_target = False , get_prev_target = False ) : super ( Text , self ) . __init__ ( name = None ) self . seq_length = time_steps self . onehot_input = onehot_input self . batch_index = 0 self . reverse_target = reverse_target self . get_prev_target = get_prev_target X , y = self . _get_data ( path , tokenizer , vocab ) # reshape to preserve sentence continuity across batches self . X = X . reshape ( self . be . bsz , self . nbatches , time_steps ) self . y = y . reshape ( self . be . bsz , self . nbatches , time_steps ) # stuff below this comment needs to be cleaned up and commented self . nout = self . nclass if self . onehot_input : self . shape = ( self . nout , time_steps ) self . dev_X = self . be . iobuf ( ( self . nout , time_steps ) ) if self . get_prev_target : self . dev_Z = self . be . iobuf ( ( self . nout , time_steps ) ) else : self . shape = ( time_steps , 1 ) self . dev_X = self . be . iobuf ( time_steps , dtype = np . int32 ) if self . get_prev_target : self . dev_Z = self . be . iobuf ( time_steps , dtype = np . int32 ) self . decoder_shape = self . shape self . dev_y = self . be . iobuf ( ( self . nout , time_steps ) ) self . dev_lbl = self . be . iobuf ( time_steps , dtype = np . int32 ) self . dev_lblflat = self . dev_lbl . reshape ( ( 1 , - 1 ) )