(self, x)
return np.reshape(res, x_shape)
Executes this layer as part of a forward pass through the model.

    Args:
      x: Tensor of same shape and dtype as the input signature used to
          initialize this layer.

    Returns:
      Tensor of same shape and dtype as the input.
Executes this layer as part of a forward pass through the model.
Executes this layer as part of a forward pass through the model .
def forward(self, x):
    """Executes this layer as part of a forward pass through the model.

    Args:
      x: Tensor of same shape and dtype as the input signature used to
          initialize this layer.

    Returns:
      Tensor of same shape and dtype as the input.
    """
    w1, w2, b2 = self.weights
    x_shape = x.shape
    x = np.reshape(x, [-1, x_shape[-1]])  # Easier to operate on flattened x.

    # Hash x into hash buckets; x_buckets is [n_hashes, joint_batch].
    x_buckets, _ = hash_vecs(x, self._n_buckets, self._n_hashes, self.rng)

    # Hash w1 into hash buckets; w1_buckets is [n_hashes, d_ff].
    # Note that we use the same self.rng - so the same hash vectors as for x.
    w1_buckets, _ = hash_vecs(w1, self._n_buckets, self._n_hashes, self.rng)

    # Create a mask to determine which x's have the same hash as which w1's.
    # First: just subtract the hashes and make them non-negative.
    hash_mask = (x_buckets[:, :, None] - w1_buckets[:, None, :])**2
    hash_mask = fastmath.stop_gradient(hash_mask)  # make sure no gradients here
    # hash_mask is [n_hashes, joint_batch, d_ff], 0 iff hashes were equal
    hash_mask = 1 - np.minimum(hash_mask, 1)  # now 1 if equal, 0 otherwise
    # we now sum over n_hashes and use min, it's 1 iff any of n_hashes was equal
    hash_mask = np.minimum(np.sum(hash_mask, axis=0), 1)
    hash_mask = hash_mask.astype(np.float32)  # convert to float to use mask

    # First dense layer of the block, with hash masking.
    mid = np.dot(x, w1.T) * hash_mask  # [joint_batch, d_ff]

    # Relu and the second dense layer, as in a standard feed-forward block.
    # Note: we merge the second block into this layer because of future plans,
    # not anything implemented yet. The potential gain would be as follows:
    # in predict mode, we would pre-hash (once) both w1 and w2 and only do
    # matmuls (and memory copies) for the parts that correspond to the hash
    # of the input. The hash of w1 determines which parts of Relu are 0, so
    # it also determines which parts of w2 can be skipped.
    relu = np.where(mid <= 0, np.zeros_like(mid), mid)
    res = np.dot(relu, w2) + b2
    return np.reshape(res, x_shape)
def forward ( self , x ) : w1 , w2 , b2 = self . weights x_shape = x . shape x = np . reshape ( x , [ - 1 , x_shape [ - 1 ] ] ) # Easier to operate on flattened x. # Hash x into hash buckets; x_buckets is [n_hashes, joint_batch]. x_buckets , _ = hash_vecs ( x , self . _n_buckets , self . _n_hashes , self . rng ) # Hash w1 into hash buckets; w1_buckets is [n_hashes, d_ff]. # Note that we use the same self.rng - so the same hash vectors as for x. w1_buckets , _ = hash_vecs ( w1 , self . _n_buckets , self . _n_hashes , self . rng ) # Create a mask to determine which x's have the same hash as which w1's. # First: just subtract the hashes and make them non-negative. hash_mask = ( x_buckets [ : , : , None ] - w1_buckets [ : , None , : ] ) ** 2 hash_mask = fastmath . stop_gradient ( hash_mask ) # make sure no gradients here # hash_mask is [n_hashes, joint_batch, d_ff], 0 iff hashes were equal hash_mask = 1 - np . minimum ( hash_mask , 1 ) # now 1 if equal, 0 otherwise # we now sum over n_hashes and use min, it's 1 iff any of n_hashes was equal hash_mask = np . minimum ( np . sum ( hash_mask , axis = 0 ) , 1 ) hash_mask = hash_mask . astype ( np . float32 ) # convert to float to use mask # First dense layer of the block, with hash masking. mid = np . dot ( x , w1 . T ) * hash_mask # [joint_batch, d_ff] # Relu and the second dense layer, as in a standard feed-forward block. # Note: we merge the second block into this layer because of future plans, # not anything implemented yet. The potential gain would be as follows: # in predict mode, we would pre-hash (once) both w1 and w2 and only do # matmuls (and memory copies) for the parts that correspond to the hash # of the input. The hash of w1 determines which parts of Relu are 0, so # it also determines which parts of w2 can be skipped. relu = np . where ( mid <= 0 , np . zeros_like ( mid ) , mid ) res = np . dot ( relu , w2 ) + b2 return np . reshape ( res , x_shape )