我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.Print()。
def omniglot(): sess = tf.InteractiveSession() """ def wrapper(v): return tf.Print(v, [v], message="Printing v") v = tf.Variable(initial_value=np.arange(0, 36).reshape((6, 6)), dtype=tf.float32, name='Matrix') sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) temp = tf.Variable(initial_value=np.arange(0, 36).reshape((6, 6)), dtype=tf.float32, name='temp') temp = wrapper(v) #with tf.control_dependencies([temp]): temp.eval() print 'Hello'""" def update_tensor(V, dim2, val): # Update tensor V, with index(:,dim2[:]) by val[:] val = tf.cast(val, V.dtype) def body(_, (v, d2, chg)): d2_int = tf.cast(d2, tf.int32) return tf.slice(tf.concat_v2([v[:d2_int],[chg] ,v[d2_int+1:]], axis=0), [0], [v.get_shape().as_list()[0]]) Z = tf.scan(body, elems=(V, dim2, val), initializer=tf.constant(1, shape=V.get_shape().as_list()[1:], dtype=tf.float32), name="Scan_Update") return Z
def decode_jpeg(image_buffer, scope=None): # , dtype=tf.float32): """Decode a JPEG string into one 3-D float image Tensor. Args: image_buffer: scalar string Tensor. scope: Optional scope for op_scope. Returns: 3-D float Tensor with values ranging from [0, 1). """ # with tf.op_scope([image_buffer], scope, 'decode_jpeg'): # with tf.name_scope(scope, 'decode_jpeg', [image_buffer]): with tf.name_scope(scope or 'decode_jpeg'): # Decode the string as an RGB JPEG. # Note that the resulting image contains an unknown height and width # that is set dynamically by decode_jpeg. In other words, the height # and width of image is unknown at compile-time. image = tf.image.decode_jpeg(image_buffer, channels=3, fancy_upscaling=False, dct_method='INTEGER_FAST') # image = tf.Print(image, [tf.shape(image)], 'Image shape: ') return image
def init_var(self): self.rand_h = tf.random_uniform([1], 1.0 - float(self.rnd_hflip), 1.0) self.rand_v = tf.random_uniform([1], 1.0 - float(self.rnd_vflip), 1.0) self.rand_t = tf.random_uniform( [1], 1.0 - float(self.rnd_transpose), 1.0) self.offset = tf.random_uniform( [2], dtype='int32', maxval=self.padding * 2 + self.shrink) if self._debug: self.offset = tf.Print(self.offset, ['Forward RND module', self.offset]) if self.rnd_size: self.space = 2 * self.padding - self.offset self.offset20 = tf.random_uniform( [], dtype='int32', maxval=self.space[0] * 2) - self.space[0] self.offset21 = tf.random_uniform( [], dtype='int32', maxval=self.space[1] * 2) - self.space[1] self.offset2 = tf.pack([self.offset20, self.offset21]) else: self.offset2 = tf.zeros([2], dtype='int32') pass
def test_vgg(): vgg = Vgg16() image_tensor = tf.placeholder(tf.float32) with tf.Session() as sess: vgg.build(image_tensor) init = tf.initialize_all_variables() sess.run(init) load_feature_layer_params('/Users/dtong/code/data/tf-image-interpreter/pretrain/vgg16_weights.npz', sess) for v in tf.get_collection(tf.GraphKeys.VARIABLES): print_op = tf.Print(v, [v], message=v.name, first_n=10) sess.run(print_op) roidb = RoiDb('val.txt', 2007) batch_gen = BatchGenerator(roidb) for i in range(10): image, scale, bboxes = batch_gen.next_batch() print(sess.run(vgg.conv5_3, feed_dict={image_tensor: image}))
def main(): roidb = RoiDb('val.txt', 2007) batch_gen = BatchGenerator(roidb) image_tensor = tf.placeholder(dtype=tf.float32) scale_tensor = tf.placeholder(dtype=tf.float32) bboxes_tensor = tf.placeholder(dtype=tf.float32) p_op = tf.Print(image_tensor, [tf.shape(image_tensor), scale_tensor, bboxes_tensor]) sess = tf.Session() init = tf.initialize_all_variables() sess.run(init) coord = tf.train.Coordinator() queue_threads = queue_runner.start_queue_runners(sess, coord=coord) for i in range(10): if coord.should_stop(): break image, scale, bboxes = batch_gen.next_batch() sess.run([p_op], feed_dict={image_tensor: image, scale_tensor: scale, bboxes_tensor:bboxes}) coord.request_stop() coord.join(queue_threads)
def generate(self, image, scale, bboxes): shape = tf.shape(image) # TODO: NotImplementedError: Negative start indices are not currently supported # height, width = shape[-2:] # height, width = shape[-2:] height = shape[1] width = shape[2] if self._debug: height = tf.Print(height, [height], message='image height: ') width = tf.Print(width, [width], message='image width: ') anchors = self._generate_valid_anchors(width, height) overlaps = self._calculate_overlaps(tf.cast(anchors, dtype=tf.float32), tf.cast(bboxes, dtype=tf.float32)) labels = self._generate_labels(overlaps) labels = self._subsample_positive(labels) labels = self._subsample_negative(labels) return labels
def _generate_labels(self, overlaps): labels = tf.Variable(tf.ones(shape=(tf.shape(overlaps)[0],), dtype=tf.float32) * -1, trainable=False, validate_shape=False) gt_max_overlaps = tf.arg_max(overlaps, dimension=0) anchor_max_overlaps = tf.arg_max(overlaps, dimension=1) mask = tf.one_hot(anchor_max_overlaps, tf.shape(overlaps)[1], on_value=True, off_value=False) max_overlaps = tf.boolean_mask(overlaps, mask) if self._debug: max_overlaps = tf.Print(max_overlaps, [max_overlaps]) labels = tf.scatter_update(labels, gt_max_overlaps, tf.ones((tf.shape(gt_max_overlaps)[0],))) # TODO: extract config object over_threshold_mask = tf.reshape(tf.where(max_overlaps > 0.5), (-1,)) if self._debug: over_threshold_mask = tf.Print(over_threshold_mask, [over_threshold_mask], message='over threshold index : ') labels = tf.scatter_update(labels, over_threshold_mask, tf.ones((tf.shape(over_threshold_mask)[0],))) # TODO: support clobber positive in the origin implement below_threshold_mask = tf.reshape(tf.where(max_overlaps < 0.3), (-1,)) if self._debug: below_threshold_mask = tf.Print(below_threshold_mask, [below_threshold_mask], message='below threshold index : ') labels = tf.scatter_update(labels, below_threshold_mask, tf.zeros((tf.shape(below_threshold_mask)[0],))) return labels
def bag_hinge_loss(config, preds, sent_mask, flip_sent_mask, hete_mask, sent_trgt, sent_num): """ HINGE LOSS: DEFINED AS: MAX(0, M - MIN(SENT+) - MAX(SENT-)) THIS ONLY APPLIES TO HETE BAGS. """ flip_sent_trgt = \ tf.constant(1, shape=[config.batch_size,sent_num], dtype=config.data_type) - \ sent_trgt pos_preds = preds + flip_sent_trgt + flip_sent_mask # [batch_size, sent_num] neg_preds = preds * flip_sent_trgt * sent_mask # [batch_size, sent_num] min_pos_pred = tf.reduce_min(pos_preds, 1) # min_pos_pred = tf.Print(min_pos_pred, [min_pos_pred], message='min_pos_pred') max_neg_pred = tf.reduce_max(neg_preds, 1) # max_neg_pred = tf.Print(max_neg_pred, [max_neg_pred], message='max_neg_pred') hinge_loss = hete_mask * tf.reduce_max(tf.pack( [tf.constant(0, shape=[config.batch_size], dtype=config.data_type), (0.20 - min_pos_pred + max_neg_pred)], axis=1), 1) # [batch_size] # hinge_loss = tf.Print(hinge_loss, [hinge_loss], message='hinge_loss', summarize=20) avg_hinge_loss = tf.reduce_sum(hinge_loss) / (tf.reduce_sum(hete_mask) + 1e-12) return avg_hinge_loss
def distorted_inputs(): """Construct distorted input for CIFAR training using the Reader ops. Returns: images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size. labels: Labels. 1D tensor of [batch_size] size. Raises: ValueError: If no data_dir """ with tf.variable_scope('distorted_inputs'): if not FLAGS.train_data_dir: raise ValueError('Please supply a data_dir') data_dir = FLAGS.train_data_dir images, labels, seq_lengths = ocr_input.distorted_inputs(data_dir=data_dir, batch_size=FLAGS.batch_size) # seq_lengths = tf.Print(seq_lengths, [seq_lengths], "seq_lengths") return images, labels, seq_lengths
def create_ctc_loss(logits, labels, timesteps, label_seq_lengths): with tf.variable_scope('CTC_Loss'): print() print("Labels shape") print(labels) print() print("Logits shape") print(logits) print() print("Labels len shape") print(label_seq_lengths) # logits = tf.Print(logits, [logits], "Logits") ctc_loss = tf.nn.ctc_loss(labels, logits, timesteps) cost = tf.reduce_mean(ctc_loss, name='ctc') # The total loss is defined as the cross entropy loss plus all of the weight # decay terms (L2 loss). return cost
def main(unused_argv): if FLAGS.log_dir is None or FLAGS.log_dir == "": raise ValueError("Must specify an explicit `log_dir`") if FLAGS.data_dir is None or FLAGS.data_dir == "": raise ValueError("Must specify an explicit `data_dir`") device, target = device_and_target() with tf.device(device): images = tf.placeholder(tf.float32, [None, 784], name='image_input') labels = tf.placeholder(tf.float32, [None], name='label_input') data = read_data_sets(FLAGS.data_dir, one_hot=False, fake_data=False) logits = mnist.inference(images, FLAGS.hidden1, FLAGS.hidden2) loss = mnist.loss(logits, labels) loss = tf.Print(loss, [loss], message="Loss = ") train_op = mnist.training(loss, FLAGS.learning_rate) with tf.train.MonitoredTrainingSession( master=target, is_chief=(FLAGS.task_index == 0), checkpoint_dir=FLAGS.log_dir) as sess: while not sess.should_stop(): xs, ys = data.train.next_batch(FLAGS.batch_size, fake_data=False) sess.run(train_op, feed_dict={images:xs, labels:ys})
def tf_debug_gradient(x, y, verbose=True): """ Print the theoretical and numeric gradients, and the absolute difference between the two Args: x (tf.Variable): input variable y (tf.Variable): output variable verbose: switch display of information Returns: the theoretical and numeric gradient """ with tf.Session() as sess: sess.run(tf.initialize_all_variables()) if verbose: print(y.eval()) gt, gn = tf.test.compute_gradient( x, [d.value for d in x.get_shape()], y, [d.value for d in y.get_shape()], delta=1e-2) if verbose: print(np.concatenate((gt, gn, np.round(np.abs(gt-gn),2)), len(gt.shape) - 1)) print(y.eval()) return gt, gn
def __match_with_labels(self,gt_anchor_labels,gt_anchor_bboxes,gt_anchor_scores,jaccard,matching_threshold,gt_labels,gt_bboxes,num_anchors): #debugging info #jaccard = tf.Print(jaccard, [gt_labels], "gt_labels") #match default boxes to any ground truth with jaccard overlap higher than a threshold (0.5). mask = tf.reduce_max (jaccard, axis = 0) > matching_threshold mask_inds = tf.argmax(jaccard, axis = 0) matched_labels = tf.gather(gt_labels, mask_inds) gt_anchor_labels = tf.where(mask, matched_labels, gt_anchor_labels) gt_anchor_bboxes = tf.where(mask, tf.gather(gt_bboxes, mask_inds),gt_anchor_bboxes) gt_anchor_scores = tf.reduce_max(jaccard, axis= 0) #matching each ground truth box to the default box with the best jaccard overlap use_no_miss = True if use_no_miss: gt_anchor_labels,gt_anchor_bboxes,gt_anchor_scores = self.__match_no_miss(gt_anchor_labels, \ gt_anchor_bboxes, gt_anchor_scores, jaccard, \ gt_labels, gt_bboxes, num_anchors) return gt_anchor_labels,gt_anchor_bboxes,gt_anchor_scores
def mixture_loss(pred, y, n_mixtures, batch_size): pred = tf.verify_tensor_all_finite(pred, "Pred not finite!") out_pi, out_sigma, out_mu, out_rho = splitMix(pred, n_mixtures, batch_size) result_binorm, result_delta = tf_bivariate_normal(y, out_mu, out_sigma, out_rho, n_mixtures, batch_size) result_binorm = tf.verify_tensor_all_finite(result_binorm, "Result not finite1!") result_weighted = tf.mul(result_binorm, out_pi) result_weighted = tf.verify_tensor_all_finite(result_weighted, "Result not finite2!") result_raw = tf.reduce_sum(result_weighted + epsilon, 1, keep_dims=True) result_raw = tf.Print(result_raw, [tf.reduce_sum(result_raw)], "Sum of weighted density. If zero, sigma is too small: ") result_raw = tf.Print(result_raw, [tf.reduce_max(result_raw)], "Max of weighted density. If zero, sigma is too small: ") result_raw = tf.verify_tensor_all_finite(result_raw, "Result not finite3!") result = -tf.log(result_raw + e) result = tf.verify_tensor_all_finite(result, "Result not finite4!") result = tf.reduce_sum(result) result = tf.verify_tensor_all_finite(result, "Result not finite5!") return result # Returns the LSTM stack created based on the parameters. # Processes several batches at once. # Input shape is: (parameters['batch_size'], parameters['n_steps'], parameters['n_input'])
def test_tabular_UCB(self): nb_states = 3 nb_actions = 2 with tf.Graph().as_default(): tf.set_random_seed(1) inputs_t = tf.random_uniform(shape=[1], minval=0, maxval=3, dtype=tf.int32) # inputs_t = tf.Print(inputs_t, data=[inputs_t], message='inputs_t') Qs = tf.ones([nb_states, nb_actions], dtype=tf.float32) # Qs = tf.Print(Qs, data=[Qs], message='Qs', summarize=12) actions_t, probs = capacities.tabular_UCB(Qs, inputs_t) # actions_t = tf.Print(actions_t, data=[timestep, actions_t], message='actions_t') with tf.Session() as sess: sess.run(tf.global_variables_initializer()) inputs, actions = sess.run([inputs_t, actions_t]) inputs, actions = sess.run([inputs_t, actions_t]) inputs, actions = sess.run([inputs_t, actions_t]) self.assertEqual(np.array_equal(inputs, [ 0 ]), True) self.assertEqual(np.array_equal(actions, [ 0 ]), True)
def _normalize(self, x, mean, mean_sq, message): # make sure this is called with a variable scope shape = x.get_shape().as_list() assert len(shape) == 4 self.gamma_driver = tf.get_variable("gamma_driver", [shape[-1]], initializer=tf.random_normal_initializer(0., 0.02)) gamma = tf.exp(self.gamma_driver) gamma = tf.reshape(gamma, [1, 1, 1, -1]) self.beta = tf.get_variable("beta", [shape[-1]], initializer=tf.constant_initializer(0.)) beta = tf.reshape(self.beta, [1, 1, 1, -1]) assert self.epsilon is not None assert mean_sq is not None assert mean is not None std = tf.sqrt(self.epsilon + mean_sq - tf.square(mean)) out = x - mean out = out / std # out = tf.Print(out, [tf.reduce_mean(out, [0, 1, 2]), # tf.reduce_mean(tf.square(out - tf.reduce_mean(out, [0, 1, 2], keep_dims=True)), [0, 1, 2])], # message, first_n=-1) out = out * gamma out = out + beta return out
def _normalize(self, x, mean, mean_sq, message): # make sure this is called with a variable scope shape = x.get_shape().as_list() assert len(shape) == 4 self.gamma_driver = tf.get_variable("gamma_driver", shape[1:], initializer=tf.random_normal_initializer(0., 0.02)) gamma = tf.exp(self.gamma_driver) gamma = tf.expand_dims(gamma, 0) self.beta = tf.get_variable("beta", shape[1:], initializer=tf.constant_initializer(0.)) beta = tf.expand_dims(self.beta, 0) assert self.epsilon is not None assert mean_sq is not None assert mean is not None std = tf.sqrt(self.epsilon + mean_sq - tf.square(mean)) out = x - mean out = out / std # out = tf.Print(out, [tf.reduce_mean(out, [0, 1, 2]), # tf.reduce_mean(tf.square(out - tf.reduce_mean(out, [0, 1, 2], keep_dims=True)), [0, 1, 2])], # message, first_n=-1) out = out * gamma out = out + beta return out
def _normalize(self, x, mean, mean_sq, message): # make sure this is called with a variable scope shape = x.get_shape().as_list() assert len(shape) == 4 self.gamma = tf.get_variable("gamma", [shape[-1]], initializer=tf.random_normal_initializer(1., 0.02)) gamma = tf.reshape(self.gamma, [1, 1, 1, -1]) self.beta = tf.get_variable("beta", [shape[-1]], initializer=tf.constant_initializer(0.)) beta = tf.reshape(self.beta, [1, 1, 1, -1]) assert self.epsilon is not None assert mean_sq is not None assert mean is not None std = tf.sqrt(self.epsilon + mean_sq - tf.square(mean)) out = x - mean out = out / std # out = tf.Print(out, [tf.reduce_mean(out, [0, 1, 2]), # tf.reduce_mean(tf.square(out - tf.reduce_mean(out, [0, 1, 2], keep_dims=True)), [0, 1, 2])], # message, first_n=-1) out = out * gamma out = out + beta return out
def _get_weight_vector(self, M, w_tm1, k, beta, g, s, gamma): # M = tf.Print(M, [M, w_tm1, k], message='get weights beg1: ') # M = tf.Print(M, [beta, g, s, gamma], message='get weights beg2: ') # Content adressing, see Chapter 3.3.1: num = beta * _cosine_distance(M, k) w_c = K.softmax(num) # It turns out that equation (5) is just softmax. # Location adressing, see Chapter 3.3.2: # Equation 7: w_g = (g * w_c) + (1-g)*w_tm1 # C_s is the circular convolution #C_w = K.sum((self.C[None, :, :, :] * w_g[:, None, None, :]),axis=3) # Equation 8: # TODO: Explain C_s = K.sum(K.repeat_elements(self.C[None, :, :, :], self.batch_size, axis=0) * s[:,:,None,None], axis=1) w_tilda = K.batch_dot(C_s, w_g) # Equation 9: w_out = _renorm(w_tilda ** gamma) return w_out
def _fc_relu_layers(self, bottom, dim, name = None): with tf.name_scope(name) as scope: shape = int(np.prod(bottom.get_shape()[1:])) weights = tf.Variable(tf.truncated_normal([shape, dim], dtype=tf.float32, stddev=0.005), name='weights') bias = tf.Variable(tf.constant(1.0, shape=[dim], dtype=tf.float32), name='biases') bottom_flat = tf.reshape(bottom, [-1, shape]) fc_weights = tf.nn.bias_add(tf.matmul(bottom_flat, weights), bias) self.parameters[name] = [weights, bias] if not tf.get_variable_scope().reuse: weight_decay = tf.multiply(tf.nn.l2_loss(weights), self.wd, name='fc_relu_weight_loss') tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, weight_decay) top = tf.nn.relu(fc_weights, name=scope) _activation_summary(top) top = tf.Print(top, [tf.shape(top)], message='Shape of %s' % name, first_n = 1, summarize=4) return top
def _fc_layers(self, bottom, dim, name = None): with tf.name_scope(name) as scope: shape = int(np.prod(bottom.get_shape()[1:])) weights = tf.Variable(tf.truncated_normal([shape, dim], dtype=tf.float32, stddev=0.005), name='weights') bias = tf.Variable(tf.constant(1.0, shape=[dim], dtype=tf.float32), name='biases') bottom_flat = tf.reshape(bottom, [-1, shape]) top = tf.nn.bias_add(tf.matmul(bottom_flat, weights), bias, name=scope) self.parameters[name] = [weights, bias] if not tf.get_variable_scope().reuse: weight_decay = tf.multiply(tf.nn.l2_loss(weights), self.wd, name='fc_weight_loss') tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, weight_decay) _activation_summary(top) top = tf.Print(top, [tf.shape(top)], message='Shape of %s' % name, first_n = 1, summarize=4) return top
def print_(var, name: str, first_n=10, summarize=5): """Util for debugging, by printing values of tf.Variable `var` during training""" # name = (next(k for k, v in globals().items() if v == var) # get name automagically # if name is None else name) # TODO make work for list ? # name = (next(k for k, v in globals().items() if id(v) == id(var)) # if name is None else name) # print(name) # return ([k for k, v in globals().items() if id(v) == id(var)] # if name is None else name) try: return tf.Print(var, [var], '{}: '.format(name), first_n=first_n, summarize=summarize) except(TypeError): # variables are already in a list return tf.Print(var, var, '{}: '.format(name), first_n=first_n, summarize=summarize)
def apply_gradients(self, grads): coldOptim = tf.train.MomentumOptimizer( self._cold_lr * (1. - self._momentum), self._momentum) def coldSGDstart(): sgd_step_op = tf.assign_add(self.sgd_step, 1) coldOptim_op = coldOptim.apply_gradients(grads) if KFAC_DEBUG: with tf.control_dependencies([sgd_step_op, coldOptim_op]): sgd_step_op = tf.Print( sgd_step_op, [self.sgd_step, tf.convert_to_tensor('doing cold sgd step')]) return tf.group(*[sgd_step_op, coldOptim_op]) kfacOptim_op, qr = self.apply_gradients_kfac(grads) def warmKFACstart(): return kfacOptim_op return tf.cond(tf.greater(self.sgd_step, self._cold_iter), warmKFACstart, coldSGDstart), qr
def add_training_op(self): loss=self.total_loss opt1=tf.train.AdagradOptimizer(self.config.lr) opt2=tf.train.AdagradOptimizer(self.config.emb_lr) ts=tf.trainable_variables() gs=tf.gradients(loss,ts) gs_ts=zip(gs,ts) gt_emb,gt_nn=[],[] for g,t in gs_ts: #print t.name,g.name if "Embed/embedding:0" in t.name: #g=tf.Print(g,[g.get_shape(),t.get_shape()]) gt_emb.append((g,t)) #print t.name else: gt_nn.append((g,t)) #print t.name train_op1=opt1.apply_gradients(gt_nn) train_op2=opt2.apply_gradients(gt_emb) train_op=[train_op1,train_op2] return train_op
def bits_err_per_seq(out, expected, nsteps): rel_pred = predict(out, nsteps) rel_pred = tf.Print( rel_pred, [tf.slice(rel_pred, [0, 0, 0], [1, -1, 1])], "predicted", summarize=20, ) expected = tf.Print( expected, [tf.slice(expected, [0, 0, 0], [1, -1, 1])], "expected", summarize=20, ) diff = rel_pred - expected return tf.reduce_mean(tf.reduce_sum(tf.abs(diff), [1, 2]))
def _get_optimizer(self, training_iters, global_step): if self.optimizer == "momentum": learning_rate = self.opt_kwargs.pop("learning_rate", 0.2) decay_rate = self.opt_kwargs.pop("decay_rate", 0.95) self.learning_rate_node = tf.train.exponential_decay(learning_rate=learning_rate, global_step=global_step, decay_steps=training_iters, decay_rate=decay_rate, staircase=True) optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate_node, momentum=0.9, **self.opt_kwargs).minimize(self.net.cost, global_step=global_step) # optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate_node, momentum=0.9, # **self.opt_kwargs) # gvs = optimizer.compute_gradients(self.net.cost) # # [print(grad) for grad, var in gvs] # tf.Print(self.net.cost,self.net.cost) # capped_gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs] # train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) elif self.optimizer == "adam": learning_rate = self.opt_kwargs.pop("learning_rate", 0.001) self.learning_rate_node = tf.Variable(learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate_node, **self.opt_kwargs).minimize(self.net.cost, global_step=global_step) return optimizer
def _filter_inside_anchors(self, all_anchors, height, width): # filter anchors inds_inside = tf.where( (all_anchors[:, 0] > 0) & (all_anchors[:, 1] > 0) & (all_anchors[:, 2] < width) & (all_anchors[:, 3] < height) ) if self._debug: inds_inside = tf.Print(inds_inside, [tf.shape(inds_inside)], message='inside anchors: ') anchors = tf.gather(all_anchors, inds_inside) return anchors
def _generate_all_anchors(self, shifts): num_anchors = self._anchors.shape[0] num_shifts = tf.shape(shifts)[0] all_anchors = (self._anchors.reshape(1, num_anchors, 4) + tf.transpose(tf.reshape(shifts, (1, num_shifts, 4)), perm=(1, 0, 2))) all_anchors = tf.reshape(all_anchors, (num_shifts * num_anchors, 4)) if self._debug: num_all_anchors = num_shifts * num_anchors tf.Print(num_all_anchors, [num_all_anchors], message='all anchor: ') return all_anchors
def rpn_rois(self, gt_boxes, labels): self.proposals = tf.Print(self.proposals, [tf.shape(self.proposals)], message='proposal shape') filled_gt_boxes = tf.concat(1, [tf.zeros([tf.shape(gt_boxes)[0], 1], dtype=tf.float32), gt_boxes]) all_rois = tf.concat(0, [self.proposals, filled_gt_boxes]) overlaps = self._calculate_overlaps(all_rois[:, 1:5], gt_boxes) # because faster-rcnn process one image per batch, leave the num_images here to keep consistency. num_images = 1 rois_per_image = tf.constant(cfg.TRAIN.BATCH_SIZE / num_images, dtype=tf.float32) fg_rois_per_image = tf.cast(tf.round(cfg.TRAIN.FG_FRACTION * rois_per_image), dtype=tf.int32) gt_assignment = tf.arg_max(overlaps, dimension=1) max_overlaps = tf.reduce_max(overlaps, reduction_indices=1) labels = tf.gather(labels, gt_assignment) fg_inds = tf.reshape(tf.cast(tf.where(max_overlaps >= cfg.TRAIN.FG_THRESH), dtype=tf.int32), [-1, ]) fg_rois_this_image = tf.minimum(fg_rois_per_image, tf.shape(fg_inds)[0]) # TODO: Check if fg_inds.size > 0: fg_inds = tf.random_crop(fg_inds, size=[fg_rois_this_image]) bg_inds = tf.reshape(tf.cast(tf.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO)), dtype=tf.int32), [-1, ]) bg_rois_this_image = tf.minimum(tf.cast(rois_per_image, dtype=tf.int32) - fg_rois_this_image, tf.shape(bg_inds)[0]) # TODO: Check if bg_inds.size > 0: bg_inds = tf.random_crop(bg_inds, size=[bg_rois_this_image]) keep_inds = tf.concat(0, [fg_inds, bg_inds]) self.train_labels = tf.concat(0, (tf.gather(labels, fg_inds), tf.zeros((tf.shape(bg_inds)[0],), dtype=tf.int32))) self.train_rois = tf.gather(all_rois, keep_inds) bbox_target_data = self._compute_targets( self.train_rois[:, 1:5], tf.gather(gt_boxes, tf.gather(gt_assignment, keep_inds)), self.train_labels) return self.train_rois, self.train_labels, bbox_target_data # TODO: implement this # self.bbox_targets, self.bbox_inside_weights = \ # self._get_bbox_regression_labels(bbox_target_data, num_classes)
def _filter_top_n(self, proposals, scores): order = tf.py_func(arg_sort_op, [tf.reshape(scores, (-1,))], [tf.int64])[0] if self._debug: order = tf.Print(order, [tf.shape(order), tf.shape(proposals), tf.shape(scores)], message='order shape: ') if self._pre_nms_top_n > 0: order = tf.gather(order, tf.range(0, self._pre_nms_top_n)) proposals = tf.gather(proposals, order) scores = tf.gather(scores, order) return proposals, scores
def meanShift(n_updates=-1): X1 = tf.expand_dims(tf.transpose(input_X), 0) X2 = tf.expand_dims(input_X, 0) C = init_C sbs_C = tf.TensorArray(dtype=tf.float32, size=10000, infer_shape=False) sbs_C = sbs_C.write(0, init_C) def _mean_shift_step(C): C = tf.expand_dims(C, 2) Y = tf.reduce_sum(tf.pow((C - X1) / window_radius, 2), axis=1) gY = tf.exp(-Y) num = tf.reduce_sum(tf.expand_dims(gY, 2) * X2, axis=1) denom = tf.reduce_sum(gY, axis=1, keep_dims=True) C = num / denom return C if n_updates > 0: for i in range(n_updates): C = _mean_shift_step(C) sbs_C = sbs_C.write(i + 1, C) else: def _mean_shift(i, C, sbs_C, max_diff): new_C = _mean_shift_step(C) max_diff = tf.reshape(tf.reduce_max(tf.sqrt(tf.reduce_sum(tf.pow(new_C - C, 2), axis=1))), []) sbs_C = sbs_C.write(i + 1, new_C) return i + 1, new_C, sbs_C, max_diff def _cond(i, C, sbs_C, max_diff): return max_diff > 1e-5 n_updates, C, sbs_C, _ = tf.while_loop(cond=_cond, body=_mean_shift, loop_vars=(tf.constant(0), C, sbs_C, tf.constant(1e10))) n_updates = tf.Print(n_updates, [n_updates]) return C, sbs_C.gather(tf.range(n_updates + 1))
def _max_pool(bottom, name, debug): pool = tf.nn.max_pool(bottom, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name=name) print('Layer name: %s' % name) print('Layer shape:%s' % str(pool.get_shape())) if debug: pool = tf.Print(pool, [pool.get_shape()], message='Shape of %s' % name, summarize=4, first_n=1) _activation_summary(pool) return pool
def _upscore_layer(bottom, shape, num_classes, name, debug, ksize=4, stride=2): strides = [1, stride, stride, 1] with tf.variable_scope(name) : in_features = bottom.get_shape()[3].value if shape is None: # Compute shape out of Bottom in_shape = bottom.get_shape() h = ((in_shape[1] - 1) * stride) + 1 w = ((in_shape[2] - 1) * stride) + 1 new_shape = [in_shape[0], h, w, num_classes] else: new_shape = [shape[0], shape[1], shape[2], num_classes] output_shape = tf.pack(new_shape) logging.debug("Layer: %s, Fan-in: %d" % (name, in_features)) f_shape = [ksize, ksize, num_classes, in_features] weights = get_deconv_filter(f_shape) deconv = tf.nn.conv2d_transpose(bottom, weights, output_shape, strides=strides, padding='SAME') deconv.set_shape(new_shape) print('Layer name: %s' % name) print('Layer shape: %s' % str(deconv.get_shape())) if debug: deconv = tf.Print(deconv, [deconv.get_shape()], message='Shape of %s' % name, summarize=4, first_n=1) _activation_summary(deconv) return deconv
def get_lstm_layers(features, timesteps, batch_size): with tf.variable_scope('RNN'): # Has size [batch_size, max_stepsize, num_features], but the # batch_size and max_stepsize can vary along each step #tf.placeholder(tf.float32, [None, None, ocr_input.IMAGE_HEIGHT]) inputs = features shape = tf.shape(features) batch_size, max_timesteps = shape[0], shape[1] # Defining the cell # Can be: # tf.nn.rnn_cell.RNNCell # tf.nn.rnn_cell.GRUCell cell = tf.contrib.rnn.core_rnn_cell.LSTMCell(LSTM_HIDDEN_SIZE, state_is_tuple=True) # Stacking rnn cells stack = tf.contrib.rnn.core_rnn_cell.MultiRNNCell([cell] * NUM_LSTM_LAYERS, state_is_tuple=True) # The second output is the last state and we will no use that outputs, _ = tf.nn.dynamic_rnn(stack, inputs, timesteps, dtype=tf.float32) # Reshaping to apply the same weights over the timesteps outputs = tf.reshape(outputs, [-1, LSTM_HIDDEN_SIZE]) # outputs = tf.Print(outputs, [outputs], "Outputs") with tf.variable_scope('logits'): w = tf.Variable(tf.truncated_normal([LSTM_HIDDEN_SIZE, NUM_CLASSES], stddev=0.1), name="w") b = tf.Variable(tf.constant(0., shape=[NUM_CLASSES]), name="b") # Doing the affine projection logits = tf.matmul(outputs, w) + b # Reshaping back to the original shape logits = tf.reshape(logits, [batch_size, -1, NUM_CLASSES]) logits = tf.transpose(logits, [1, 0, 2], name="out_logits") return logits
def check_decoder(logits, labels, timesteps): with tf.variable_scope('check_decoder'): decoded, log_prob = tf.nn.ctc_greedy_decoder(logits, timesteps) decoded = tf.cast(decoded[0], tf.int32) decoded = tf.sparse_tensor_to_dense(decoded) # decoded = tf.Print(decoded, [decoded], "Decoded") return decoded
def print_tensor_summary(tensor, tag=None, n_print=21): tensor_min = tf.reduce_min(tensor) tensor_max = tf.reduce_max(tensor) tensor_avg = tf.reduce_mean(tensor) tensor_zero_fraction = tf.nn.zero_fraction(tensor) tensor_shape = tf.shape(tensor) tag = tag or tensor.name tensor = tf.Print(tensor, [tensor_min, tensor_max, tensor_avg, tensor_zero_fraction, tensor_shape, tensor], message=(tag + ' Min, max, mean, sparsity, shape, value:'), summarize=n_print) return tensor
def print_tensor(x, message=''): '''Print the message and the tensor when evaluated and return the same tensor. ''' return tf.Print(x, [x], message) # GRAPH MANIPULATION
def _apply_stats(self, statsUpdates, accumulate=False, accumulateCoeff=0.): updateOps = [] # obtain the stats var list for stats_var in statsUpdates: stats_new = statsUpdates[stats_var] if accumulate: # simple superbatch averaging update_op = tf.assign_add( stats_var, accumulateCoeff * stats_new, use_locking=True) else: # exponential running averaging update_op = tf.assign( stats_var, stats_var * self._stats_decay, use_locking=True) update_op = tf.assign_add( update_op, (1. - self._stats_decay) * stats_new, use_locking=True) updateOps.append(update_op) with tf.control_dependencies(updateOps): stats_step_op = tf.assign_add(self.stats_step, 1) if KFAC_DEBUG: stats_step_op = (tf.Print(stats_step_op, [tf.convert_to_tensor('step:'), self.global_step, tf.convert_to_tensor('fac step:'), self.factor_step, tf.convert_to_tensor('sgd step:'), self.sgd_step, tf.convert_to_tensor('Accum:'), tf.convert_to_tensor(accumulate), tf.convert_to_tensor('Accum coeff:'), tf.convert_to_tensor(accumulateCoeff), tf.convert_to_tensor('stat step:'), self.stats_step, updateOps[0], updateOps[1]])) return [stats_step_op, ]
def applyStatsEigen(self, eigen_list): updateOps = [] print(('updating %d eigenvalue/vectors' % len(eigen_list))) for i, (tensor, mark) in enumerate(zip(eigen_list, self.eigen_update_list)): stats_eigen_var = self.eigen_reverse_lookup[mark] updateOps.append( tf.assign(stats_eigen_var, tensor, use_locking=True)) with tf.control_dependencies(updateOps): factor_step_op = tf.assign_add(self.factor_step, 1) updateOps.append(factor_step_op) if KFAC_DEBUG: updateOps.append(tf.Print(tf.constant( 0.), [tf.convert_to_tensor('updated kfac factors')])) return updateOps
def apply_gradients(self, grads): coldOptim = tf.train.MomentumOptimizer( self._cold_lr, self._momentum) def coldSGDstart(): sgd_grads, sgd_var = zip(*grads) if self.max_grad_norm != None: sgd_grads, sgd_grad_norm = tf.clip_by_global_norm(sgd_grads,self.max_grad_norm) sgd_grads = list(zip(sgd_grads,sgd_var)) sgd_step_op = tf.assign_add(self.sgd_step, 1) coldOptim_op = coldOptim.apply_gradients(sgd_grads) if KFAC_DEBUG: with tf.control_dependencies([sgd_step_op, coldOptim_op]): sgd_step_op = tf.Print( sgd_step_op, [self.sgd_step, tf.convert_to_tensor('doing cold sgd step')]) return tf.group(*[sgd_step_op, coldOptim_op]) kfacOptim_op, qr = self.apply_gradients_kfac(grads) def warmKFACstart(): return kfacOptim_op return tf.cond(tf.greater(self.sgd_step, self._cold_iter), warmKFACstart, coldSGDstart), qr
def detectMinVal(input_mat, var, threshold=1e-6, name='', debug=False): eigen_min = tf.reduce_min(input_mat) eigen_max = tf.reduce_max(input_mat) eigen_ratio = eigen_max / eigen_min input_mat_clipped = clipoutNeg(input_mat, threshold) if debug: input_mat_clipped = tf.cond(tf.logical_or(tf.greater(eigen_ratio, 0.), tf.less(eigen_ratio, -500)), lambda: input_mat_clipped, lambda: tf.Print( input_mat_clipped, [tf.convert_to_tensor('screwed ratio ' + name + ' eigen values!!!'), tf.convert_to_tensor(var.name), eigen_min, eigen_max, eigen_ratio])) return input_mat_clipped
def __call__(self, enc_input, dec_input_indices, valid_indices, left_indices, right_indices, values, valid_masks=None): batch_size = tf.shape(enc_input)[0] # forward computation graph with tf.variable_scope(self.scope): # encoder output enc_memory, enc_final_state_fw, _ = self.encoder(enc_input) # decoder dec_hiddens, dec_actions, dec_act_logps = self.decoder( enc_memory, dec_input_indices, valid_indices, left_indices, right_indices, valid_masks, init_state=enc_final_state_fw) # cost costs = [] update_ops = [] for step_idx, (act_logp, value, baseline) in enumerate(zip(dec_act_logps, values, self.baselines)): # costs.append(-tf.reduce_mean(act_logp * (value - baseline))) new_baseline = self.bl_ratio * baseline + (1-self.bl_ratio) * tf.reduce_mean(value) costs.append(-tf.reduce_mean(act_logp * value)) update_ops.append(tf.assign(baseline, new_baseline)) # gradient computation graph self.params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.scope) train_ops = [] for limit in self.buckets: print '0 ~ %d' % (limit-1) grad_params = tf.gradients(tf.reduce_sum(tf.pack(costs[:limit])), self.params) if self.max_grad_norm is not None: clipped_gradients, norm = tf.clip_by_global_norm(grad_params, self.max_grad_norm) else: clipped_gradients = grad_params train_op = self.optimizer.apply_gradients( zip(clipped_gradients, self.params)) with tf.control_dependencies([train_op] + update_ops[:limit]): # train_ops.append(tf.Print(tf.constant(1.), [norm])) train_ops.append(tf.constant(1.)) return dec_hiddens, dec_actions, train_ops #### test script
def __call__(self, inputs, state, scope=None): with tf.variable_scope(scope or type(self).__name__, initializer=self._initializer): # Split the hidden state into blocks (each U, V, W are shared across blocks). # U = tf.get_variable('U', [self._num_units_per_block, self._num_units_per_block], # initializer= tf.constant_initializer(np.identity(self._num_units_per_block)), # trainable = False) # W = tf.get_variable('W', [self._num_units_per_block, self._num_units_per_block], # initializer = tf.constant_initializer(np.zeros(self._num_units_per_block, self._num_units_per_block)), # trainable = False) # V = tf.get_variable('V', [self._num_units_per_block, self._num_units_per_block], # initializer = tf.constant_initializer(np.zeros(self._num_units_per_block, self._num_units_per_block)), # trainable = False) # b = tf.get_variable('biasU',[self._num_units_per_block]) # self._q = tf.Print(self._q, [self._q],summarize=10) # TODO: layer norm? state = tf.split(state, self._num_blocks, 1) next_states = [] for j, state_j in enumerate(state): # Hidden State (j) key_j = self._keys[j] gate_j = self.get_gate(state_j, key_j, inputs) candidate_j = inputs # Equation 4: h_j <- h_j + g_j * h_j^~ # Perform an update of the hidden state (memory). state_j_next = state_j + tf.expand_dims(gate_j, -1) * candidate_j # # Forget previous memories by normalization. # state_j_next = tf.nn.l2_normalize(state_j_next, -1) # TODO: Is epsilon necessary? next_states.append(state_j_next) state_next = tf.concat(next_states, 1) return state_next, state_next
def ornstein_uhlenbeck_noise(a, t_decay=100000): noise_var = get_local_variable("nm", initializer=tf.zeros(a.get_shape()[1:])) ou_theta = get_local_variable("ou_theta", initializer=0.2) ou_sigma = get_local_variable("ou_sigma", initializer=0.15) # ou_theta = tf.Print(ou_theta, [noise_var], 'noise: ', first_n=2000) ou_sigma = tf.train.exponential_decay(ou_sigma, tt.function.step(), t_decay, 1e-6) n = noise_var.assign_sub(ou_theta * noise_var - tf.random_normal(a.get_shape()[1:], stddev=ou_sigma)) return a + n
def build(self, predictions, targets, inputs=None): """ Prints the number of each kind of prediction """ self.built = True pshape = predictions.get_shape() self.inner_metric.build(predictions, targets, inputs) with tf.name_scope(self.name): if len(pshape) == 1 or (len(pshape) == 2 and int(pshape[1]) == 1): self.name = self.name or "binary_prediction_counts" y, idx, count = tf.unique_with_counts(tf.argmax(predictions)) self.tensor = tf.Print(self.inner_metric, [y, count], name=self.inner_metric.name) else: self.name = self.name or "categorical_prediction_counts" y, idx, count = tf.unique_with_counts(tf.argmax(predictions, dimension=1)) self.tensor = tf.Print(self.inner_metric.tensor, [y, count], name=self.inner_metric.name)
def test_basic(self): with tf.Graph().as_default(), self.test_session() as sess: rnd = np.random.RandomState(0) x = self.get_random_tensor([18, 12], rnd=rnd) y = tf.Print(x, [x]) self.assert_bw_fw(sess, x, y, rnd=rnd)
def init_ops_for_training(self, target_critic): # update critic using bellman equation; Q(s1, a) = reward + discount * Q(s2, A(s2)) # left hand side of bellman is just q_value, but let's be explicit about it... bellman_lhs = self.q_value # right hand side is ... # = reward + discounted q value from target actor & critic in the non terminal case # = reward # in the terminal case self.reward = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="critic_reward") self.terminal_mask = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="critic_terminal_mask") self.input_state_2 = target_critic.input_state bellman_rhs = self.reward + (self.terminal_mask * opts.discount * target_critic.q_value) # note: since we are NOT training target networks we stop gradients flowing to them bellman_rhs = tf.stop_gradient(bellman_rhs) # the value we are trying to mimimise is the difference between these two; the # temporal difference we use a squared loss for optimisation and, as for actor, we # wrap optimiser in a namespace so it's not picked up by target network variable # handling. self.temporal_difference = bellman_lhs - bellman_rhs self.temporal_difference_loss = tf.reduce_mean(tf.pow(self.temporal_difference, 2)) # self.temporal_difference_loss = tf.Print(self.temporal_difference_loss, [self.temporal_difference_loss], 'temporal_difference_loss') with tf.variable_scope("optimiser"): # calc gradients optimiser = tf.train.GradientDescentOptimizer(opts.critic_learning_rate) gradients = optimiser.compute_gradients(self.temporal_difference_loss) # potentially clip and wrap with debugging tf.Print gradients = util.clip_and_debug_gradients(gradients, opts) # apply self.train_op = optimiser.apply_gradients(gradients)
def clip_and_debug_gradients(gradients, opts): # extract just the gradients temporarily for global clipping and then rezip if opts.gradient_clip is not None: just_gradients, variables = zip(*gradients) just_gradients, _ = tf.clip_by_global_norm(just_gradients, opts.gradient_clip) gradients = zip(just_gradients, variables) # verbose debugging if opts.print_gradients: for i, (gradient, variable) in enumerate(gradients): if gradient is not None: gradients[i] = (tf.Print(gradient, [l2_norm(gradient)], "gradient %s l2_norm " % variable.name), variable) # done return gradients
