(self,
               source_vocab_size,
               target_vocab_size,
               buckets,
               size,
               num_layers,
               latent_dim,
               max_gradient_norm,
               batch_size,
               learning_rate,
               kl_min=2,
               word_dropout_keep_prob=1.0,
               anneal=False,
               kl_rate_rise_factor=None,
               use_lstm=False,
               num_samples=512,
               optimizer=None,
               activation=tf.nn.relu,
               forward_only=False,
               feed_previous=True,
               bidirectional=False,
               weight_initializer=None,
               bias_initializer=None,
               iaf=False,
               dtype=tf.float32)
Create the model.

    Args:
      source_vocab_size: size of the source vocabulary.
      target_vocab_size: size of the target vocabulary.
      buckets: a list of pairs (I, O), where I specifies maximum input length
        that will be processed in that bucket, and O specifies maximum output
        length. Training instances that have inputs longer than I or outputs
        longer than O will be pushed to the next bucket and padded accordingly.
        We assume that the list is sorted, e.g., [(2, 4), (8, 16)].
      size: number of units in each layer of the model.
      num_layers: number of layers in the model.
      max_gradient_norm: gradients will be clipped to maximally this norm.
      batch_size: the size of the batches used during training;
        the model construction is independent of batch_size, so it can be
        changed after initialization if this is convenient, e.g., for decoding.
      learning_rate: learning rate to start with.
      use_lstm: if true, we use LSTM cells instead of GRU cells.
      num_samples: number of samples for sampled softmax.
      forward_only: if set, we do not construct the backward pass in the model.
      dtype: the data type to use to store internal variables.
Create the model.
Create the model .
def __init__(self,
               source_vocab_size,
               target_vocab_size,
               buckets,
               size,
               num_layers,
               latent_dim,
               max_gradient_norm,
               batch_size,
               learning_rate,
               kl_min=2,
               word_dropout_keep_prob=1.0,
               anneal=False,
               kl_rate_rise_factor=None,
               use_lstm=False,
               num_samples=512,
               optimizer=None,
               activation=tf.nn.relu,
               forward_only=False,
               feed_previous=True,
               bidirectional=False,
               weight_initializer=None,
               bias_initializer=None,
               iaf=False,
               dtype=tf.float32):
    """Create the model.

    Args:
      source_vocab_size: size of the source vocabulary.
      target_vocab_size: size of the target vocabulary.
      buckets: a list of pairs (I, O), where I specifies maximum input length
        that will be processed in that bucket, and O specifies maximum output
        length. Training instances that have inputs longer than I or outputs
        longer than O will be pushed to the next bucket and padded accordingly.
        We assume that the list is sorted, e.g., [(2, 4), (8, 16)].
      size: number of units in each layer of the model.
      num_layers: number of layers in the model.
      max_gradient_norm: gradients will be clipped to maximally this norm.
      batch_size: the size of the batches used during training;
        the model construction is independent of batch_size, so it can be
        changed after initialization if this is convenient, e.g., for decoding.
      learning_rate: learning rate to start with.
      use_lstm: if true, we use LSTM cells instead of GRU cells.
      num_samples: number of samples for sampled softmax.
      forward_only: if set, we do not construct the backward pass in the model.
      dtype: the data type to use to store internal variables.
    """
    self.source_vocab_size = source_vocab_size
    self.target_vocab_size = target_vocab_size
    self.latent_dim = latent_dim
    self.buckets = buckets
    self.batch_size = batch_size
    self.word_dropout_keep_prob = word_dropout_keep_prob
    self.kl_min = kl_min
    feed_previous = feed_previous or forward_only

    self.learning_rate = tf.Variable(
        float(learning_rate), trainable=False, dtype=dtype)

    self.enc_embedding = tf.get_variable("enc_embedding", [source_vocab_size, size], dtype=dtype, initializer=weight_initializer())

    self.dec_embedding = tf.get_variable("dec_embedding", [target_vocab_size, size], dtype=dtype, initializer=weight_initializer())

    self.kl_rate = tf.Variable(
       0.0, trainable=False, dtype=dtype)
    self.new_kl_rate = tf.placeholder(tf.float32, shape=[], name="new_kl_rate")
    self.kl_rate_update = tf.assign(self.kl_rate, self.new_kl_rate)

    self.replace_input = tf.placeholder(tf.int32, shape=[None], name="replace_input")
    replace_input = tf.nn.embedding_lookup(self.dec_embedding, self.replace_input)

    self.global_step = tf.Variable(0, trainable=False)

    # If we use sampled softmax, we need an output projection.
    output_projection = None
    softmax_loss_function = None
    # Sampled softmax only makes sense if we sample less than vocabulary size.
    if num_samples > 0 and num_samples < self.target_vocab_size:
      w_t = tf.get_variable("proj_w", [self.target_vocab_size, size], dtype=dtype, initializer=weight_initializer())
      w = tf.transpose(w_t)
      b = tf.get_variable("proj_b", [self.target_vocab_size], dtype=dtype, initializer=bias_initializer)
      output_projection = (w, b)

      def sampled_loss(inputs, labels):
        labels = tf.reshape(labels, [-1, 1])
        # We need to compute the sampled_softmax_loss using 32bit floats to
        # avoid numerical instabilities.
        local_w_t = tf.cast(w_t, tf.float32)
        local_b = tf.cast(b, tf.float32)
        local_inputs = tf.cast(inputs, tf.float32)
        return tf.cast(
            tf.nn.sampled_softmax_loss(local_w_t, local_b, local_inputs, labels,
                                       num_samples, self.target_vocab_size),
            dtype)
      softmax_loss_function = sampled_loss
    # Create the internal multi-layer cell for our RNN.
    single_cell = tf.nn.rnn_cell.GRUCell(size)
    if use_lstm:
      single_cell = tf.nn.rnn_cell.BasicLSTMCell(size)

    cell = single_cell

    def encoder_f(encoder_inputs):
      return seq2seq.embedding_encoder(
          encoder_inputs,
          cell,
          self.enc_embedding,
          num_symbols=source_vocab_size,
          embedding_size=size,
          bidirectional=bidirectional,
          weight_initializer=weight_initializer,
          dtype=dtype)

    def decoder_f(encoder_state, decoder_inputs):
      return seq2seq.embedding_rnn_decoder(
          decoder_inputs,
          encoder_state,
          cell,
          embedding=self.dec_embedding,
          word_dropout_keep_prob=word_dropout_keep_prob,
          replace_input=replace_input,
          num_symbols=target_vocab_size,
          embedding_size=size,
          output_projection=output_projection,
          feed_previous=feed_previous,
          weight_initializer=weight_initializer)

    def enc_latent_f(encoder_state):
      return seq2seq.encoder_to_latent(
                     encoder_state,
                     embedding_size=size,
                     latent_dim=latent_dim,
                     num_layers=num_layers,
                     activation=activation,
                     use_lstm=use_lstm,
                     enc_state_bidirectional=bidirectional,
                     dtype=dtype)

    def latent_dec_f(latent_vector):
      return seq2seq.latent_to_decoder(latent_vector,
           embedding_size=size,
           latent_dim=latent_dim,
           num_layers=num_layers,
           activation=activation,
           use_lstm=use_lstm,
           dtype=dtype)


    def sample_f(mean, logvar):
      return seq2seq.sample(
           mean,
           logvar,
           latent_dim,
           iaf,
           kl_min,
           anneal,
           self.kl_rate,
           dtype)

    # The seq2seq function: we use embedding for the input and attention.
    def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
      return tf.nn.seq2seq.embedding_attention_seq2seq_f(
          encoder_inputs,
          decoder_inputs,
          cell,
          num_encoder_symbols=source_vocab_size,
          num_decoder_symbols=target_vocab_size,
          embedding_size=size,
          output_projection=output_projection,
          feed_previous=do_decode,
          dtype=dtype)


    # Feeds for inputs.
    self.encoder_inputs = []
    self.decoder_inputs = []
    self.target_weights = []
    for i in xrange(buckets[-1][0]):  # Last bucket is the biggest one.
      self.encoder_inputs.append(tf.placeholder(tf.int32, shape=[None],
                                                name="encoder{0}".format(i)))
    for i in xrange(buckets[-1][1] + 1):
      self.decoder_inputs.append(tf.placeholder(tf.int32, shape=[None],
                                                name="decoder{0}".format(i)))
      self.target_weights.append(tf.placeholder(dtype, shape=[None],
                                                name="weight{0}".format(i)))

    # Our targets are decoder inputs shifted by one.
    targets = [self.decoder_inputs[i + 1]
               for i in xrange(len(self.decoder_inputs) - 1)]


    self.means, self.logvars = seq2seq.variational_encoder_with_buckets(
        self.encoder_inputs, buckets, encoder_f, enc_latent_f,
        softmax_loss_function=softmax_loss_function)
    self.outputs, self.losses, self.KL_objs, self.KL_costs = seq2seq.variational_decoder_with_buckets(
        self.means, self.logvars, self.decoder_inputs, targets,
        self.target_weights, buckets, decoder_f, latent_dec_f,
        sample_f, softmax_loss_function=softmax_loss_function)

    # If we use output projection, we need to project outputs for decoding.
    if output_projection is not None:
      for b in xrange(len(buckets)):
        self.outputs[b] = [
            tf.matmul(output, output_projection[0]) + output_projection[1]
            for output in self.outputs[b]
          ]
    # Gradients and SGD update operation for training the model.
    params = tf.trainable_variables()
    if not forward_only:
      self.gradient_norms = []
      self.updates = []
      for b in xrange(len(buckets)):
        total_loss = self.losses[b] + self.KL_objs[b]
        gradients = tf.gradients(total_loss, params)
        clipped_gradients, norm = tf.clip_by_global_norm(gradients,
                                                         max_gradient_norm)
        self.gradient_norms.append(norm)
        self.updates.append(optimizer.apply_gradients(
            zip(clipped_gradients, params), global_step=self.global_step))

    self.saver = tf.train.Saver(tf.global_variables())
def __init__ ( self , source_vocab_size , target_vocab_size , buckets , size , num_layers , latent_dim , max_gradient_norm , batch_size , learning_rate , kl_min = 2 , word_dropout_keep_prob = 1.0 , anneal = False , kl_rate_rise_factor = None , use_lstm = False , num_samples = 512 , optimizer = None , activation = tf . nn . relu , forward_only = False , feed_previous = True , bidirectional = False , weight_initializer = None , bias_initializer = None , iaf = False , dtype = tf . float32 ) : self . source_vocab_size = source_vocab_size self . target_vocab_size = target_vocab_size self . latent_dim = latent_dim self . buckets = buckets self . batch_size = batch_size self . word_dropout_keep_prob = word_dropout_keep_prob self . kl_min = kl_min feed_previous = feed_previous or forward_only self . learning_rate = tf . Variable ( float ( learning_rate ) , trainable = False , dtype = dtype ) self . enc_embedding = tf . get_variable ( "enc_embedding" , [ source_vocab_size , size ] , dtype = dtype , initializer = weight_initializer ( ) ) self . dec_embedding = tf . get_variable ( "dec_embedding" , [ target_vocab_size , size ] , dtype = dtype , initializer = weight_initializer ( ) ) self . kl_rate = tf . Variable ( 0.0 , trainable = False , dtype = dtype ) self . new_kl_rate = tf . placeholder ( tf . float32 , shape = [ ] , name = "new_kl_rate" ) self . kl_rate_update = tf . assign ( self . kl_rate , self . new_kl_rate ) self . replace_input = tf . placeholder ( tf . int32 , shape = [ None ] , name = "replace_input" ) replace_input = tf . nn . embedding_lookup ( self . dec_embedding , self . replace_input ) self . global_step = tf . Variable ( 0 , trainable = False ) # If we use sampled softmax, we need an output projection. output_projection = None softmax_loss_function = None # Sampled softmax only makes sense if we sample less than vocabulary size. if num_samples > 0 and num_samples < self . target_vocab_size : w_t = tf . get_variable ( "proj_w" , [ self . target_vocab_size , size ] , dtype = dtype , initializer = weight_initializer ( ) ) w = tf . transpose ( w_t ) b = tf . get_variable ( "proj_b" , [ self . target_vocab_size ] , dtype = dtype , initializer = bias_initializer ) output_projection = ( w , b ) def sampled_loss ( inputs , labels ) : labels = tf . reshape ( labels , [ - 1 , 1 ] ) # We need to compute the sampled_softmax_loss using 32bit floats to # avoid numerical instabilities. local_w_t = tf . cast ( w_t , tf . float32 ) local_b = tf . cast ( b , tf . float32 ) local_inputs = tf . cast ( inputs , tf . float32 ) return tf . cast ( tf . nn . sampled_softmax_loss ( local_w_t , local_b , local_inputs , labels , num_samples , self . target_vocab_size ) , dtype ) softmax_loss_function = sampled_loss # Create the internal multi-layer cell for our RNN. single_cell = tf . nn . rnn_cell . GRUCell ( size ) if use_lstm : single_cell = tf . nn . rnn_cell . BasicLSTMCell ( size ) cell = single_cell def encoder_f ( encoder_inputs ) : return seq2seq . embedding_encoder ( encoder_inputs , cell , self . enc_embedding , num_symbols = source_vocab_size , embedding_size = size , bidirectional = bidirectional , weight_initializer = weight_initializer , dtype = dtype ) def decoder_f ( encoder_state , decoder_inputs ) : return seq2seq . embedding_rnn_decoder ( decoder_inputs , encoder_state , cell , embedding = self . dec_embedding , word_dropout_keep_prob = word_dropout_keep_prob , replace_input = replace_input , num_symbols = target_vocab_size , embedding_size = size , output_projection = output_projection , feed_previous = feed_previous , weight_initializer = weight_initializer ) def enc_latent_f ( encoder_state ) : return seq2seq . encoder_to_latent ( encoder_state , embedding_size = size , latent_dim = latent_dim , num_layers = num_layers , activation = activation , use_lstm = use_lstm , enc_state_bidirectional = bidirectional , dtype = dtype ) def latent_dec_f ( latent_vector ) : return seq2seq . latent_to_decoder ( latent_vector , embedding_size = size , latent_dim = latent_dim , num_layers = num_layers , activation = activation , use_lstm = use_lstm , dtype = dtype ) def sample_f ( mean , logvar ) : return seq2seq . sample ( mean , logvar , latent_dim , iaf , kl_min , anneal , self . kl_rate , dtype ) # The seq2seq function: we use embedding for the input and attention. def seq2seq_f ( encoder_inputs , decoder_inputs , do_decode ) : return tf . nn . seq2seq . embedding_attention_seq2seq_f ( encoder_inputs , decoder_inputs , cell , num_encoder_symbols = source_vocab_size , num_decoder_symbols = target_vocab_size , embedding_size = size , output_projection = output_projection , feed_previous = do_decode , dtype = dtype ) # Feeds for inputs. self . encoder_inputs = [ ] self . decoder_inputs = [ ] self . target_weights = [ ] for i in xrange ( buckets [ - 1 ] [ 0 ] ) : # Last bucket is the biggest one. self . encoder_inputs . append ( tf . placeholder ( tf . int32 , shape = [ None ] , name = "encoder{0}" . format ( i ) ) ) for i in xrange ( buckets [ - 1 ] [ 1 ] + 1 ) : self . decoder_inputs . append ( tf . placeholder ( tf . int32 , shape = [ None ] , name = "decoder{0}" . format ( i ) ) ) self . target_weights . append ( tf . placeholder ( dtype , shape = [ None ] , name = "weight{0}" . format ( i ) ) ) # Our targets are decoder inputs shifted by one. targets = [ self . decoder_inputs [ i + 1 ] for i in xrange ( len ( self . decoder_inputs ) - 1 ) ] self . means , self . logvars = seq2seq . variational_encoder_with_buckets ( self . encoder_inputs , buckets , encoder_f , enc_latent_f , softmax_loss_function = softmax_loss_function ) self . outputs , self . losses , self . KL_objs , self . KL_costs = seq2seq . variational_decoder_with_buckets ( self . means , self . logvars , self . decoder_inputs , targets , self . target_weights , buckets , decoder_f , latent_dec_f , sample_f , softmax_loss_function = softmax_loss_function ) # If we use output projection, we need to project outputs for decoding. if output_projection is not None : for b in xrange ( len ( buckets ) ) : self . outputs [ b ] = [ tf . matmul ( output , output_projection [ 0 ] ) + output_projection [ 1 ] for output in self . outputs [ b ] ] # Gradients and SGD update operation for training the model. params = tf . trainable_variables ( ) if not forward_only : self . gradient_norms = [ ] self . updates = [ ] for b in xrange ( len ( buckets ) ) : total_loss = self . losses [ b ] + self . KL_objs [ b ] gradients = tf . gradients ( total_loss , params ) clipped_gradients , norm = tf . clip_by_global_norm ( gradients , max_gradient_norm ) self . gradient_norms . append ( norm ) self . updates . append ( optimizer . apply_gradients ( zip ( clipped_gradients , params ) , global_step = self . global_step ) ) self . saver = tf . train . Saver ( tf . global_variables ( ) )