我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.add_n()。
def loss(logits, labels): """Add L2Loss to all the trainable variables. Add summary for for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def loss(c_fuse, s_fuse, labels): """Add L2Loss to all the trainable variables. Add summary for "Loss" and "Loss/avg". Args: c_fuse: Contours output map from inference(). s_fuse: Segments output map from inference(). labels: Labels from distorted_inputs or inputs(). Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. # Split the labels tensor into contours and segments image tensors # Each has shape [FLAGS.batch_size, 696, 520, 1] contours_labels, segments_labels = tf.split(labels, 2, 3) _add_cross_entropy(contours_labels, c_fuse, 'c') _add_cross_entropy(segments_labels, s_fuse, 's') return tf.add_n(tf.get_collection('losses'), name='total_loss')
def _get_loss(self,labels): with tf.name_scope("Loss"): """ with tf.name_scope("logloss"): logit = tf.squeeze(tf.nn.sigmoid(self.logit)) self.loss = tf.reduce_mean(self._logloss(labels, logit)) """ with tf.name_scope("L2_loss"): if self.flags.lambdax: lambdax = self.flags.lambdax else: lambdax = 0 self.l2loss = lambdax*tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) with tf.name_scope("dice_coef"): #yp_label = tf.cast(logit>self.flags.threshold, tf.float32) logit = tf.squeeze(self.logit) self.acc = tf.reduce_mean(self._dice_coef(labels,logit)) self.metric = "dice_coef" self.loss = -self.acc with tf.name_scope("summary"): if self.flags.visualize: tf.summary.scalar(name='dice coef', tensor=self.acc, collections=[tf.GraphKeys.SCALARS])
def loss(self, l2_lambda=0.0001): # 0.001 with tf.name_scope("loss"): # input: `logits`:[batch_size, num_classes], and `labels`:[batch_size] # output: A 1-D `Tensor` of length `batch_size` of the same type as `logits` with the softmax cross entropy loss. losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label,logits=self.logits); # sigmoid_cross_entropy_with_logits.#losses=tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y,logits=self.logits) # print("1.sparse_softmax_cross_entropy_with_logits.losses:",losses) # shape=(?,) loss = tf.reduce_mean(losses) # print("2.loss.loss:", loss) #shape=() l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if ('bias' not in v.name ) and ('alpha' not in v.name)]) * l2_lambda loss = loss + l2_losses return loss #def loss_seq2seq(self): # with tf.variable_scope("loss"): # losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.input_y_label, logits=self.logits);#losses:[batch_size,self.decoder_sent_length] # loss_batch=tf.reduce_sum(losses,axis=1)/self.decoder_sent_length #loss_batch:[batch_size] # loss=tf.reduce_mean(loss_batch) # l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * self.l2_lambda # loss = loss + l2_losses # return loss
def loss_nce(self,l2_lambda=0.0001): #0.0001-->0.001 """calculate loss using (NCE)cross entropy here""" # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. if self.is_training: #training #labels=tf.reshape(self.input_y,[-1]) #[batch_size,1]------>[batch_size,] labels=tf.expand_dims(self.input_y,1) #[batch_size,]----->[batch_size,1] loss = tf.reduce_mean( #inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. tf.nn.nce_loss(weights=tf.transpose(self.W_projection),#[hidden_size*2, num_classes]--->[num_classes,hidden_size*2]. nce_weights:A `Tensor` of shape `[num_classes, dim].O.K. biases=self.b_projection, #[label_size]. nce_biases:A `Tensor` of shape `[num_classes]`. labels=labels, #[batch_size,1]. train_labels, # A `Tensor` of type `int64` and shape `[batch_size,num_true]`. The target classes. inputs=self.output_rnn_last,# [batch_size,hidden_size*2] #A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. num_sampled=self.num_sampled, #scalar. 100 num_classes=self.num_classes,partition_strategy="div")) #scalar. 1999 l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * l2_lambda loss = loss + l2_losses return loss
def loss(logits, label_batch): """ Add L2Loss to all the trainable variables. Add summary for "Loss" and "Loss/avg". Args: logits -> logits from inference() label_batch -> 1D tensor of [batch_size] Rtns: total_loss -> float tensor """ # Calculate the average cross entropy loss across the batch. label_batch = tf.cast(label_batch,tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, label_batch,name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses',cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def tower_loss(name_scope, autoencoder, clips): # calculate reconstruction loss rec_loss = tf.reduce_mean(tf.abs(clips-autoencoder.rec_vid)) weight_decay_loss_list = tf.get_collection('losses', name_scope) weight_decay_loss = 0.0 if len(weight_decay_loss_list) > 0: weight_decay_loss = tf.add_n(weight_decay_loss_list) tf.add_to_collection('losses', rec_loss) losses = tf.get_collection('losses', name_scope) # Calculate the total loss for the current tower. total_loss = tf.add_n(losses, name='total_loss') return total_loss, rec_loss, weight_decay_loss
def tower_loss(name_scope, mfb, use_pretrained_encoder, encoder_gradient_ratio=1.0): # get reconstruction and ground truth ac_loss = mfb.ac_loss weight_decay_loss_list = tf.get_collection('losses', name_scope) if use_pretrained_encoder: if encoder_gradient_ratio == 0.0: weight_decay_loss_list = [var for var in weight_decay_loss_list \ if 'c3d' not in var.name and 'mapping' not in var.name] weight_decay_loss = 0.0 if len(weight_decay_loss_list) > 0: weight_decay_loss = tf.add_n(weight_decay_loss_list) total_loss = weight_decay_loss * 100 + ac_loss return total_loss, ac_loss, weight_decay_loss
def _loss_shared(logits, labels): """Add L2Loss to all the trainable variables. Add summary for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits, labels, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def inference(images, batch_size, train): """Build the ocr model. Args: images: Images returned from distorted_inputs() or inputs(). Returns: Logits. """ features, timesteps = convolutional_layers(images, batch_size, train) logits = get_lstm_layers(features, timesteps, batch_size) return logits, timesteps # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). # return tf.add_n(tf.get_collection('losses'), name='total_loss')
def __init__(self, ob_dim, ac_dim): #pylint: disable=W0613 X = tf.placeholder(tf.float32, shape=[None, ob_dim*2+ac_dim*2+2]) # batch of observations vtarg_n = tf.placeholder(tf.float32, shape=[None], name='vtarg') wd_dict = {} h1 = tf.nn.elu(dense(X, 64, "h1", weight_init=U.normc_initializer(1.0), bias_init=0, weight_loss_dict=wd_dict)) h2 = tf.nn.elu(dense(h1, 64, "h2", weight_init=U.normc_initializer(1.0), bias_init=0, weight_loss_dict=wd_dict)) vpred_n = dense(h2, 1, "hfinal", weight_init=U.normc_initializer(1.0), bias_init=0, weight_loss_dict=wd_dict)[:,0] sample_vpred_n = vpred_n + tf.random_normal(tf.shape(vpred_n)) wd_loss = tf.get_collection("vf_losses", None) loss = U.mean(tf.square(vpred_n - vtarg_n)) + tf.add_n(wd_loss) loss_sampled = U.mean(tf.square(vpred_n - tf.stop_gradient(sample_vpred_n))) self._predict = U.function([X], vpred_n) optim = kfac.KfacOptimizer(learning_rate=0.001, cold_lr=0.001*(1-0.9), momentum=0.9, \ clip_kl=0.3, epsilon=0.1, stats_decay=0.95, \ async=1, kfac_update=2, cold_iter=50, \ weight_decay_dict=wd_dict, max_grad_norm=None) vf_var_list = [] for var in tf.trainable_variables(): if "vf" in var.name: vf_var_list.append(var) update_op, self.q_runner = optim.minimize(loss, loss_sampled, var_list=vf_var_list) self.do_update = U.function([X, vtarg_n], update_op) #pylint: disable=E1101 U.initialize() # Initialize uninitialized TF variables
def loss(self, predictions, real_values): """Return the loss operation between predictions and real_values. Add L2 weight decay term if any. Args: predictions: predicted values real_values: real values Returns: Loss tensor of type float. """ with tf.variable_scope('loss'): # 1/2n \sum^{n}_{i=i}{(x_i - x'_i)^2} mse = tf.divide( tf.reduce_mean( tf.square(tf.subtract(predictions, real_values))), 2., name="mse") tf.add_to_collection(LOSSES, mse) # mse + weight_decay per layer error = tf.add_n(tf.get_collection(LOSSES), name='total_loss') return error
def loss(self, logits, labels): """Add L2Loss to all the trainable variables. Args: logits: Logits from get(). labels: Labels from train_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ with tf.variable_scope('loss'): # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=labels, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean( cross_entropy, name='cross_entropy') tf.add_to_collection(LOSSES, cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). error = tf.add_n(tf.get_collection(LOSSES), name='total_loss') return error
def loss(self, inf_targets, inf_vads, targets, vads, mtl_fac): ''' Loss definition Only speech inference loss is defined and work quite well Add VAD cross entropy loss if you want ''' loss_v1 = tf.nn.l2_loss(inf_targets - targets) / self.batch_size loss_o = loss_v1 reg_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) # ipdb.set_trace() loss_v = loss_o + tf.add_n(reg_loss) tf.scalar_summary('loss', loss_v) # loss_merge = tf.cond( # is_val, lambda: tf.scalar_summary('val_loss_batch', loss_v), # lambda: tf.scalar_summary('loss', loss_v)) return loss_v, loss_o # return tf.reduce_mean(tf.nn.l2_loss(inf_targets - targets))
def loss(logits, labels): """Add L2Loss to all the trainable variables. Add summary for for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits, labels, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def loss(logits, labels): """Add L2Loss to all the trainable variables. Add summary for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def average_gradients(tower_grads): """Calculate the average gradient for each shared variable across all towers. Note that this function provides a synchronization point across all towers. Args: tower_grads: List of lists of (gradient, variable) tuples. The outer list is over individual gradients. The inner list is over the gradient calculation for each tower. Returns: List of pairs of (gradient, variable) where the gradient has been averaged across all towers. """ average_grads = [] for single_grads in zip(*tower_grads): grads = [g for g, _ in single_grads] grad = tf.add_n(grads) grad = tf.multiply(grad, 1.0/len(grads)) v = single_grads[0][1] grad_and_var = (grad, v) average_grads.append(grad_and_var) return average_grads
def loss(logits, labels): """Add L2Loss to all the trainable variables. Add summary for "Loss" and "Loss/avg". Args: logits: Logits from inference(). labels: Labels from distorted_inputs or inputs(). 1-D tensor of shape [batch_size] Returns: Loss tensor of type float. """ # Calculate the average cross entropy loss across the batch. labels = tf.cast(labels, tf.int64) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits, labels, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return tf.add_n(tf.get_collection('losses'), name='total_loss')
def __init__(self, config, models): model = models[0] assert isinstance(model, Model) self.config = config self.model = model self.opt = tf.train.AdadeltaOptimizer(config.init_lr) self.var_list = model.get_var_list() self.global_step = model.get_global_step() self.summary = model.summary self.models = models losses = [] grads_list = [] for gpu_idx, model in enumerate(models): with tf.name_scope("grads_{}".format(gpu_idx)), tf.device("/gpu:{}".format(gpu_idx)): loss = model.get_loss() grads = self.opt.compute_gradients(loss, var_list=self.var_list) losses.append(loss) grads_list.append(grads) self.loss = tf.add_n(losses)/len(losses) self.grads = average_gradients(grads_list) self.train_op = self.opt.apply_gradients(self.grads, global_step=self.global_step)
def _build_loss(self): config = self.config JX = tf.shape(self.x)[2] M = tf.shape(self.x)[1] JQ = tf.shape(self.q)[1] loss_mask = tf.reduce_max(tf.cast(self.q_mask, 'float'), 1) losses = tf.nn.softmax_cross_entropy_with_logits( self.logits, tf.cast(tf.reshape(self.y, [-1, M * JX]), 'float')) ce_loss = tf.reduce_mean(loss_mask * losses) tf.add_to_collection('losses', ce_loss) ce_loss2 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( self.logits2, tf.cast(tf.reshape(self.y2, [-1, M * JX]), 'float'))) tf.add_to_collection("losses", ce_loss2) self.loss = tf.add_n(tf.get_collection('losses', scope=self.scope), name='loss') tf.scalar_summary(self.loss.op.name, self.loss) tf.add_to_collection('ema/scalar', self.loss)
def __init__(self, config, models): model = models[0] assert isinstance(model, Model) self.config = config self.model = model self.opt = tf.train.AdadeltaOptimizer(config.init_lr) self.var_list = model.get_var_list() self.global_step = model.get_global_step() self.summary = model.summary self.models = models losses = [] grads_list = [] for gpu_idx, model in enumerate(models): with tf.name_scope("grads_{}".format(gpu_idx)), tf.device("/{}:{}".format(config.device_type, gpu_idx)): loss = model.get_loss() grads = self.opt.compute_gradients(loss, var_list=self.var_list) losses.append(loss) grads_list.append(grads) self.loss = tf.add_n(losses)/len(losses) self.grads = average_gradients(grads_list) self.train_op = self.opt.apply_gradients(self.grads, global_step=self.global_step)
def grad_variance(self): grad_var_ops = [] tensor_to_avg = [] for t, g in zip(self._tvars, self._grads): if isinstance(g, ops.IndexedSlices): tensor_to_avg.append( tf.reshape(tf.unsorted_segment_sum( g.values, g.indices, g.dense_shape[0]), shape=t.get_shape())) else: tensor_to_avg.append(g) avg_op = self._moving_averager.apply(tensor_to_avg) grad_var_ops.append(avg_op) with tf.control_dependencies([avg_op]): self._grad_avg = [ self._moving_averager.average(val) for val in tensor_to_avg] self._grad_avg_squared = [tf.square(val) for val in self._grad_avg] self._grad_var = tf.maximum( tf.constant(EPS, dtype=self._grad_norm_squared_avg.dtype), self._grad_norm_squared_avg - tf.add_n([tf.reduce_sum(val) for val in self._grad_avg_squared] ) ) if self._sparsity_debias: self._grad_var *= self._sparsity_avg return grad_var_ops
def loss(infer, count_diff_infer, label): l2_loss = tf.reduce_mean(tf.reduce_sum(tf.square(infer - label), [1,2,3]), name = 'l2_loss') #l2_loss = mf.huber_loss(tf.reduce_sum(infer, [1,2,3]), tf.reduce_sum(label, [1,2,3]), huber_epsilon, 'density_loss') huber_epsilon = 5.0 c_lambda = 0.1 count_infer = tf.add(tf.squeeze(count_diff_infer), tf.reduce_sum(infer, [1,2,3]), name = "count_infer") count_loss = tf.mul(c_lambda, mf.huber_loss(count_infer, tf.reduce_sum(label, [1,2,3]), huber_epsilon, 'huber_loss'), name = 'count_loss') #count_loss = tf.mul(c_lambda, tf.reduce_mean(tf.square(count_infer - tf.reduce_sum(label, [1,2,3]))), #name = 'count_loss') tf.add_to_collection('losses', count_loss) tf.add_to_collection('losses', l2_loss) return tf.add_n(tf.get_collection('losses'), name = 'total_loss'), count_infer
def _setup_classification_predictions_and_loss(self): self.inputs, self.target = self.input_queue.dequeue() self.num_batch_elems = tf.size(self.target) self.inputs = tf.reshape(self.inputs, self.input_shape) self.training_end_points = self.model(is_training=True, reuse=None, inputs=self.inputs) training_logits, self.training_predictions = self.training_end_points['logits'], self.training_end_points[ 'predictions'] self.validation_end_points = self.model(is_training=False, reuse=True, inputs=self.inputs) validation_logits, self.validation_predictions = self.validation_end_points['logits'], \ self.validation_end_points[ 'predictions'] with tf.name_scope('loss'): training_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=training_logits, labels=self.target)) self.validation_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=validation_logits, labels=self.target)) l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) self.regularized_training_loss = training_loss + l2_loss * self.cnf.get('l2_reg', 0.0)
def _setup_regression_predictions_and_loss(self): self.inputs, self.target = self.input_queue.dequeue() self.num_batch_elems = tf.size(self.target) self.inputs = tf.reshape(self.inputs, self.input_shape) self.training_end_points = self.model(is_training=True, reuse=None, inputs=self.inputs) self.training_predictions = self.training_end_points['predictions'] self.validation_end_points = self.model(is_training=False, reuse=True, inputs=self.inputs) self.validation_predictions = self.validation_end_points['predictions'] with tf.name_scope('loss'): training_loss = tf.reduce_mean( tf.square(tf.subtract(self.training_predictions, self.target))) self.validation_loss = tf.reduce_mean( tf.square(tf.subtract(self.validation_predictions, self.target))) l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) self.regularized_training_loss = training_loss + l2_loss * self.cnf.get('l2_reg', 0.0)
def _setup_classification_predictions_and_loss(self): self.training_end_points = self.model(is_training=True, reuse=None) self.inputs = self.training_end_points['inputs'] training_logits, self.training_predictions = self.training_end_points['logits'], self.training_end_points[ 'predictions'] self.validation_end_points = self.model(is_training=False, reuse=True) self.validation_inputs = self.validation_end_points['inputs'] validation_logits, self.validation_predictions = self.validation_end_points['logits'], \ self.validation_end_points[ 'predictions'] with tf.name_scope('predictions'): self.target = tf.placeholder(tf.int32, shape=(None,), name='target') with tf.name_scope('loss'): training_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=training_logits, labels=self.target)) self.validation_loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=validation_logits, labels=self.target)) l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) self.regularized_training_loss = training_loss + l2_loss * self.cnf.get('l2_reg', 0.0)
def _setup_regression_predictions_and_loss(self): self.training_end_points = self.model(is_training=True, reuse=None) self.inputs = self.training_end_points['inputs'] self.training_predictions = self.training_end_points['predictions'] self.validation_end_points = self.model(is_training=False, reuse=True) self.validation_inputs = self.validation_end_points['inputs'] self.validation_predictions = self.validation_end_points['predictions'] with tf.name_scope('predictions'): self.target = tf.placeholder(tf.float32, shape=(None, 1), name='target') with tf.name_scope('loss'): training_loss = tf.reduce_mean( tf.square(tf.subtract(self.training_predictions, self.target))) self.validation_loss = tf.reduce_mean( tf.square(tf.subtract(self.validation_predictions, self.target))) l2_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) self.regularized_training_loss = training_loss + l2_loss * self.cnf.get('l2_reg', 0.0)
def fisher_vec_bk(ys, xs, vs): """Implements Fisher vector product using backward AD. Args: ys: Loss function, scalar. xs: Weights, list of tensors. vs: List of tensors to multiply, for each weight tensor. Returns: J'Jv: Fisher vector product. """ # Validate the input if type(xs) == list: if len(vs) != len(xs): raise ValueError("xs and vs must have the same length.") grads = tf.gradients(ys, xs, gate_gradients=True) gradsv = list(map(lambda x: tf.reduce_sum(x[0] * x[1]), zip(grads, vs))) jv = tf.add_n(gradsv) jjv = list(map(lambda x: x * jv, grads)) return jjv
def total_loss_sum(losses): ''' Adds L2 regularization loss to the given list of losses Parameters ---------- losses : list List of losses Returns ------- total_loss: float L2 regularized loss ''' # Assemble all of the losses for the current tower only. # Calculate the total loss for the current tower. regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) total_loss = tf.add_n(losses + regularization_losses, name='total_loss') return total_loss
def average_precision_voc07(precision, recall, name=None): """Compute (interpolated) average precision from precision and recall Tensors. The implementation follows Pascal 2007 guidelines. See also: https://sanchom.wordpress.com/tag/average-precision/ """ with tf.name_scope(name, 'average_precision_voc07', [precision, recall]): # Convert to float64 to decrease error on cumulated sums. precision = tf.cast(precision, dtype=tf.float64) recall = tf.cast(recall, dtype=tf.float64) # Add zero-limit value to avoid any boundary problem... precision = tf.concat([precision, [0.]], axis=0) recall = tf.concat([recall, [np.inf]], axis=0) # Split the integral into 10 bins. l_aps = [] for t in np.arange(0., 1.1, 0.1): mask = tf.greater_equal(recall, t) v = tf.reduce_max(tf.boolean_mask(precision, mask)) l_aps.append(v / 11.) ap = tf.add_n(l_aps) return ap
def _attention_iter(self, inputs, lrnSize, itersize, name = 'attention_iter'): with tf.name_scope(name): numIn = inputs.get_shape().as_list()[3] padding = np.floor(lrnSize/2) pad = tf.pad(inputs, np.array([[0,0],[1,1],[1,1],[0,0]])) U = self._conv(pad, filters=1, kernel_size=3, strides=1) pad_2 = tf.pad(U, np.array([[0,0],[padding,padding],[padding,padding],[0,0]])) sharedK = tf.Variable(tf.contrib.layers.xavier_initializer(uniform=False)([lrnSize,lrnSize, 1, 1]), name= 'shared_weights') Q = [] C = [] for i in range(itersize): if i ==0: conv = tf.nn.conv2d(pad_2, sharedK, [1,1,1,1], padding='VALID', data_format='NHWC') else: conv = tf.nn.conv2d(Q[i-1], sharedK, [1,1,1,1], padding='SAME', data_format='NHWC') C.append(conv) Q_tmp = tf.nn.sigmoid(tf.add_n([C[i], U])) Q.append(Q_tmp) stacks = [] for i in range(numIn): stacks.append(Q[-1]) pfeat = tf.multiply(inputs,tf.concat(stacks, axis = 3) ) return pfeat
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_layers = FLAGS.multiscale_cnn_lstm_layers lstm_size = int(FLAGS.lstm_cells) pool_size=2 num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) sub_predictions = [] cnn_input = model_input cnn_max_frames = model_input.get_shape().as_list()[1] for layer in range(num_layers): cnn_output = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output_relu = tf.nn.relu(cnn_output) lstm_memory = self.rnn(cnn_output_relu, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1)) sub_prediction = self.moe(lstm_memory, vocab_size, scopename="moe%d"%(layer+1)) sub_predictions.append(sub_prediction) cnn_max_frames /= pool_size max_pooled_cnn_output = tf.reduce_max( tf.reshape( cnn_output_relu[:, :cnn_max_frames*2, :], [-1, cnn_max_frames, pool_size, features_size] ), axis=2) # for the next cnn layer cnn_input = max_pooled_cnn_output num_frames = tf.maximum(num_frames/pool_size, 1) support_predictions = tf.concat(sub_predictions, axis=1) predictions = tf.add_n(sub_predictions) / len(sub_predictions) return {"predictions": predictions, "support_predictions": support_predictions}
def l2loss(params): if len(params) == 0: return tf.constant(0.0) else: return tf.add_n([sum(tf.square(p)) for p in params])
def neglogp(self, x): return tf.add_n([p.neglogp(px) for p, px in zip(self.categoricals, tf.unstack(x - self.low, axis=len(x.get_shape()) - 1))])
def kl(self, other): return tf.add_n([ p.kl(q) for p, q in zip(self.categoricals, other.categoricals) ])
def entropy(self): return tf.add_n([p.entropy() for p in self.categoricals])
