我们从Python开源项目中，提取了以下30个代码示例，用于说明如何使用tensorflow.matrix_set_diag()。
def top_K_loss(self,sentence,image,K=50,margin=0.4): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss(self,sentence,image,K=50,margin=0.3): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss_margin(self,sentence,image,K=50,margin=0.3): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) flag =8-7*tf.sign(tf.nn.relu(self.sen_margin-self.sen_similarity)) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d *flag, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d*flag)*tf.sign(tf.nn.relu(self.image_margin-self.im_similarity)), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss(self,sentence,image,K=30,margin=0.3): sim_matrix = tf.matmul(sentence, image,transpose_b=True) bs = tf.shape(sim_matrix)[0] s_square = tf.reduce_sum(tf.square(sentence),axis=1) im_square =tf.reduce_sum(tf.square(image),axis=1) d = tf.reshape(s_square,[-1,1])-2*sim_matrix+tf.reshape(im_square,[1,-1]) positive = tf.stack([tf.matrix_diag_part(d)]*K,1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 100*tf.ones([length])) sen_loss_K ,_= tf.nn.top_k(-d,K,sorted=False) im_loss_K,_=tf.nn.top_k(tf.transpose(-d),K,sorted=False) sentence_center_loss = tf.nn.relu(sen_loss_K + positive +margin) image_center_loss = tf.nn.relu(im_loss_K + positive +margin) self.d_neg =tf.reduce_mean(-sen_loss_K-im_loss_K)/2 self.d_pos = tf.reduce_mean(positive) self.endpoint['debug/sentence_center_loss']=sentence_center_loss self.endpoint['debug/image_center_loss']=image_center_loss self.endpoint['debug/sim_matrix']=sim_matrix self.endpoint['debug/sen_loss_K']=-sen_loss_K self.endpoint['debug/image_loss_K']=-im_loss_K self.endpoint['debug/distance']=d self.endpoint['debug/positive']=positive return tf.reduce_sum(sentence_center_loss),tf.reduce_sum(image_center_loss)
def top_K_loss_margin(self,sentence,image,K=50,margin=0.3): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) flag =8-7*tf.sign(tf.nn.relu(self.sen_margin-self.sen_similarity)) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d *flag, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d*flag), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss(self, sentence, image, K=30, margin=0.5): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = (sen_loss_K + im_loss_K)/-2.0 self.d_pos = positive self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss(self,sentence,image,K=50,margin=0.1): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def top_K_loss_margin(self,sentence,image,K=50,margin=0.2): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = 1-tf.sigmoid(sim_matrix) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] dd = tf.matrix_set_diag(d, 8 * tf.ones([length])) flag =8-7*tf.sign(tf.nn.relu(self.sen_margin-self.sen_similarity)) sen_loss_K ,_ = tf.nn.top_k(-1.0 * dd *flag, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(-tf.transpose(1.0 * dd*flag), K, sorted=False) # note: this is negative value sentence_center_loss = -tf.log(1-positive+1e-12)-tf.log(-sen_loss_K+1e-12) image_center_loss = -tf.log(1-positive+1e-12)-tf.log(-im_loss_K+1e-12) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def softmax_topK_loss(self,sentence,image,K=50,margin=0.2): sim_matrix = [] self.sparse_loss = tf.reduce_sum(2-(tf.reduce_sum(tf.nn.top_k(sentence, k=20, sorted=False)[0],axis=1)+tf.reduce_sum(tf.nn.top_k(image, k=20, sorted=False)[0],axis=1))) with tf.device('cpu:0'): for i in range(self.batch_size): sim_matrix.append(tf.reduce_sum(tf.abs(sentence-image[i,:]),axis=1)) d=tf.stack(sim_matrix,axis=1) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = tf.reduce_mean((sen_loss_K + im_loss_K)/-2.0) self.d_pos =tf.reduce_mean(positive) self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss),self.sparse_loss
def top_K_loss(self, sentence, image, K=50, margin=0.3): # change: K=300, but i choose index 25 to 75 for training. # so, the real 'K' is 50 sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value #sen_loss_K = sen_loss_K[:, 25:75] #im_loss_K = im_loss_K[:, 25:75] sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = (sen_loss_K + im_loss_K)/-2.0 self.d_pos = positive self.endpoint['debug/im_distance_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_distance_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss),tf.reduce_sum(image_center_loss)
def correlation_loss(self, opts, input_): """ Independence test based on Pearson's correlation. Keep in mind that this captures only linear dependancies. However, for multivariate Gaussian independence is equivalent to zero correlation. """ batch_size = self.get_batch_size(opts, input_) dim = int(input_.get_shape()[1]) transposed = tf.transpose(input_, perm=[1, 0]) mean = tf.reshape(tf.reduce_mean(transposed, axis=1), [-1, 1]) centered_transposed = transposed - mean # Broadcasting mean cov = tf.matmul(centered_transposed, centered_transposed, transpose_b=True) cov = cov / (batch_size - 1) #cov = tf.Print(cov, [cov], "cov") sigmas = tf.sqrt(tf.diag_part(cov) + 1e-5) #sigmas = tf.Print(sigmas, [sigmas], "sigmas") sigmas = tf.reshape(sigmas, [1, -1]) sigmas = tf.matmul(sigmas, sigmas, transpose_a=True) #sigmas = tf.Print(sigmas, [sigmas], "sigmas") # Pearson's correlation corr = cov / sigmas triangle = tf.matrix_set_diag(tf.matrix_band_part(corr, 0, -1), tf.zeros(dim)) #triangle = tf.Print(triangle, [triangle], "triangle") loss = tf.reduce_sum(tf.square(triangle)) / ((dim * dim - dim) / 2.0) #loss = tf.Print(loss, [loss], "Correlation loss") return loss
def __knn_bruteforce(X, k=50): # calculate euclidean distances r = tf.reduce_sum(X*X, 1) r = tf.reshape(r, [-1, 1]) D = r - 2*tf.matmul(X, tf.transpose(X)) + tf.transpose(r) D = tf.matrix_set_diag(D, tf.constant(1e32, dtype=X.dtype, shape=(X.shape[0],))) D = tf.sqrt(D) #find kNNs distances, indices = tf.nn.top_k(-D, k) return -distances, indices
def top_K_loss(self, sentence, image, K=50, margin=0.5, img_input_feat=None, text_input_feat=None): sim_matrix = tf.matmul(sentence, image, transpose_b=True) s_square = tf.reduce_sum(tf.square(sentence), axis=1) im_square = tf.reduce_sum(tf.square(image), axis=1) d = tf.reshape(s_square,[-1,1]) - 2 * sim_matrix + tf.reshape(im_square, [1, -1]) positive = tf.stack([tf.matrix_diag_part(d)] * K, axis=1) length = tf.shape(d)[-1] d = tf.matrix_set_diag(d, 8 * tf.ones([length])) if img_input_feat is not None: img_input_norm1 = img_input_feat / tf.norm(img_input_feat, axis=-1, keep_dims=True) S_input_img = tf.matmul(img_input_norm1, img_input_norm1, transpose_b=True) img_coeff = 8 - 7 * tf.sign(tf.nn.relu(0.99 - S_input_img)) sen_loss_K ,_ = tf.nn.top_k(-1.0 * d * img_coeff, K, sorted=False) # note: this is negative value self.endpoint['debug/S_input_img'] = S_input_img self.endpoint['debug/img_coeff'] = img_coeff else: sen_loss_K ,_ = tf.nn.top_k(-1.0 * d, K, sorted=False) # note: this is negative value if text_input_feat is not None: text_input_norm1 = text_input_feat / (tf.norm(text_input_feat, axis=-1, keep_dims=True) + 1e-10) S_input_text = tf.matmul(text_input_norm1, text_input_norm1, transpose_b=True) text_coeff = 8 - 7 * tf.sign(tf.nn.relu(0.98 - S_input_text)) im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d * text_coeff), K, sorted=False) self.endpoint['debug/S_input_text'] = S_input_text self.endpoint['debug/text_coeff'] = text_coeff else: im_loss_K,_ = tf.nn.top_k(tf.transpose(-1.0 * d), K, sorted=False) # note: this is negative value sentence_center_loss = tf.nn.relu(positive + sen_loss_K + margin) image_center_loss = tf.nn.relu(positive + im_loss_K + margin) self.d_neg = (sen_loss_K + im_loss_K)/-2.0 self.d_pos = positive self.endpoint['debug/im_loss_topK'] = -1.0 * im_loss_K self.endpoint['debug/sen_loss_topK'] = -1.0 * sen_loss_K self.endpoint['debug/d_Matrix'] = d self.endpoint['debug/positive'] = positive self.endpoint['debug/s_center_loss'] = sentence_center_loss self.endpoint['debug/i_center_loss'] = image_center_loss self.endpoint['debug/S'] = sim_matrix self.endpoint['debug/sentence_square'] = s_square self.endpoint['debug/image_square'] = im_square return tf.reduce_sum(sentence_center_loss), tf.reduce_sum(image_center_loss)
def _link(self, prev_link, prev_precedence_weights, write_weights): """Calculates the new link graphs. For each write head, the link is a directed graph (represented by a matrix with entries in range [0, 1]) whose vertices are the memory locations, and an edge indicates temporal ordering of writes. Args: prev_link: A tensor of shape `[batch_size, num_writes, memory_size, memory_size]` representing the previous link graphs for each write head. prev_precedence_weights: A tensor of shape `[batch_size, num_writes, memory_size]` which is the previous "aggregated" write weights for each write head. write_weights: A tensor of shape `[batch_size, num_writes, memory_size]` containing the new locations in memory written to. Returns: A tensor of shape `[batch_size, num_writes, memory_size, memory_size]` containing the new link graphs for each write head. """ with tf.name_scope('link'): batch_size = prev_link.get_shape()[0].value write_weights_i = tf.expand_dims(write_weights, 3) write_weights_j = tf.expand_dims(write_weights, 2) prev_precedence_weights_j = tf.expand_dims(prev_precedence_weights, 2) prev_link_scale = 1 - write_weights_i - write_weights_j new_link = write_weights_i * prev_precedence_weights_j link = prev_link_scale * prev_link + new_link # Return the link with the diagonal set to zero, to remove self-looping # edges. return tf.matrix_set_diag( link, tf.zeros( [batch_size, self._num_writes, self._memory_size], dtype=link.dtype))
def block_CG(A_,B_): """ block version of CG. Get solution to matrix equation AX = B, ie X = A^-1 * B. Will be much faster than Cholesky for large-scale problems. """ n = tf.shape(B_)[0] m = tf.shape(B_)[1] X = tf.zeros((n,m)) V_ = tf.zeros((n,m)) R = B_ R_ = tf.matrix_set_diag(tf.zeros((n,m)),tf.ones([m])) #somewhat arbitrary again, may want to check sensitivity CG_EPS = tf.cast(n/1000,"float") MAX_ITER = tf.div(n,250) + 3 def cond(i,X,R_,R,V_): return tf.logical_and(i < MAX_ITER, tf.norm(R) > CG_EPS) def body(i,X,R_,R,V_): S = tf.matrix_solve(tf.matmul(tf.transpose(R_),R_), tf.matmul(tf.transpose(R),R)) V = R + tf.matmul(V_,S) T = tf.matrix_solve(tf.matmul(tf.transpose(V),tf.matmul(A_,V)), tf.matmul(tf.transpose(R),R)) X = X + tf.matmul(V,T) V_ = V R_ = R R = R - tf.matmul(A_,tf.matmul(V,T)) return i+1,X,R_,R,V_ i = tf.constant(0) i,X,_,_,_ = tf.while_loop(cond,body,[i,X,R_,R,V_]) return X
def sample(means, logvars, latent_dim, iaf=True, kl_min=None, anneal=False, kl_rate=None, dtype=None): """Perform sampling and calculate KL divergence. Args: means: tensor of shape (batch_size, latent_dim) logvars: tensor of shape (batch_size, latent_dim) latent_dim: dimension of latent space. iaf: perform linear IAF or not. kl_min: lower bound for KL divergence. anneal: perform KL cost annealing or not. kl_rate: KL divergence is multiplied by kl_rate if anneal is set to True. Returns: latent_vector: latent variable after sampling. A vector of shape (batch_size, latent_dim). kl_obj: objective to be minimized for the KL term. kl_cost: real KL divergence. """ if iaf: with tf.variable_scope('iaf'): prior = DiagonalGaussian(tf.zeros_like(means, dtype=dtype), tf.zeros_like(logvars, dtype=dtype)) posterior = DiagonalGaussian(means, logvars) z = posterior.sample logqs = posterior.logps(z) L = tf.get_variable("inverse_cholesky", [latent_dim, latent_dim], dtype=dtype, initializer=tf.zeros_initializer) diag_one = tf.ones([latent_dim], dtype=dtype) L = tf.matrix_set_diag(L, diag_one) mask = np.tril(np.ones([latent_dim,latent_dim])) L = L * mask latent_vector = tf.matmul(z, L) logps = prior.logps(latent_vector) kl_cost = logqs - logps else: noise = tf.random_normal(tf.shape(mean)) sample = mean + tf.exp(0.5 * logvar) * noise kl_cost = -0.5 * (logvars - tf.square(means) - tf.exp(logvars) + 1.0) kl_ave = tf.reduce_mean(kl_cost, [0]) #mean of kl_cost over batches kl_obj = kl_cost = tf.reduce_sum(kl_ave) if kl_min: kl_obj = tf.reduce_sum(tf.maximum(kl_ave, kl_min)) if anneal: kl_obj = kl_obj * kl_rate return latent_vector, kl_obj, kl_cost #both kl_obj and kl_cost are scalar
