我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.clip_by_value()。
def calculate_loss_distill_boost(self, predictions, labels_distill, labels, **unused_params): with tf.name_scope("loss_distill_boost"): print("loss_distill_boost") epsilon = 10e-6 float_labels = tf.cast(labels, tf.float32) batch_size = tf.shape(float_labels)[0] float_labels_distill = tf.cast(labels_distill, tf.float32) error = tf.negative(float_labels * tf.log(float_labels_distill + epsilon) + ( 1 - float_labels) * tf.log(1 - float_labels_distill + epsilon)) error = tf.reduce_sum(error,axis=1,keep_dims=True) alpha = error / tf.reduce_sum(error) * tf.cast(batch_size,dtype=tf.float32) alpha = tf.clip_by_value(alpha, 0.5, 5) alpha = alpha / tf.reduce_sum(alpha) * tf.cast(batch_size,dtype=tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss * alpha) return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def get_optimizer(self, learning_rate = 0.001): with tf.name_scope('loss'): input_shape = tf.shape(self.inputs) ones = tf.ones([input_shape[0], input_shape[1]]) loss = tf.contrib.seq2seq.sequence_loss(self.logits, self.targets, ones) #----------------------------------------------------------------------- # Build the optimizer #----------------------------------------------------------------------- with tf.name_scope('optimizer'): optimizer = tf.train.AdamOptimizer(learning_rate) gradients = optimizer.compute_gradients(loss) capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) \ for grad, var in gradients if grad is not None] optimizer_op = optimizer.apply_gradients(capped_gradients) return optimizer_op, loss
def _fc(self, x, fan_in, fan_out, layer_name, activation=None, L2=1, use_bias=True, wmin=None,wmax=None,analysis=False): show_weight = self.flags.visualize and 'weight' in self.flags.visualize if wmin is not None or wmax is not None: use_bias = False assert wmin is not None and wmax is not None with tf.variable_scope(layer_name.split('/')[-1]): w,b = self._get_fc_weights(fan_in, fan_out, layer_name) if wmin is not None: wr = wmax-wmin w = self._activate(w,'sigmoid')*wr+wmin #w = tf.clip_by_value(w,wmin,wmax) net = tf.matmul(x,w) if use_bias: net = tf.nn.bias_add(net, b) net = self._activate(net, activation) if show_weight: tf.summary.histogram(name='W', values=w, collections=[tf.GraphKeys.WEIGHTS]) if use_bias: tf.summary.histogram(name='bias', values=b, collections=[tf.GraphKeys.WEIGHTS]) if analysis: net1 = tf.expand_dims(x,2)*tf.expand_dims(w,0) #net1 = tf.reshape(net1,[tf.shape(x)[0],fan_in*fan_out]) return net,net1 return net
def get_trainer(cost, learning_rate=.001, grad_clips=(-1., 1.), logger=logger, **kwargs): """Return opertation that trains parameters, given cost tensor.""" opt = tf.train.AdamOptimizer(learning_rate) if grad_clips is None: return opt.minimize(cost, **kwargs) grads_vars = [] for grad_var in opt.compute_gradients(cost, **kwargs): if grad_var[0] is None: if logger is not None: logger.info('No gradient for variable {}', grad_var[1].name) continue grads_vars.append((tf.clip_by_value(grad_var[0], -1., 1.), grad_var[1])) return opt.apply_gradients(grads_vars)
def recode_cost(self, inputs, variation, eps=1e-5, **kwargs): """ Cost for given input batch of samples, under current params. """ h = self.get_h_inputs(inputs) z_mu = tf.matmul(h, self.params['Mhz']) + self.params['bMhz'] z_sig = tf.matmul(h, self.params['Shz']) + self.params['bShz'] # KL divergence between latent space induced by encoder and ... lat_loss = -tf.reduce_sum(1 + z_sig - z_mu**2 - tf.exp(z_sig), 1) z = z_mu + tf.sqrt(tf.exp(z_sig)) * variation h = self.get_h_latents(z) x_mu = self.decoding(tf.matmul(h, self.params['Mhx']) + self.params['bMhx']) x_sig = self.decoding(tf.matmul(h, self.params['Shx']) + self.params['bShx']) # x_sig = tf.clip_by_value(x_mu * (1 - x_mu), .05, 1) # decoding likelihood term like_loss = tf.reduce_sum(tf.log(x_sig + eps) + (inputs - x_mu)**2 / x_sig, 1) # # Mean cross entropy between input and encode-decoded input. # like_loss = 2 * tf.reduce_sum(functions.cross_entropy(inputs, x_mu), 1) return .5 * tf.reduce_mean(like_loss + lat_loss)
def compute_gradients(self, loss, *args, **kwargs): train_vars = None if self.trainable_names is not None: log.info('All trainable vars:\n'+str([var.name for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])) train_vars = [] for scope_name in self.trainable_names: new_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope_name) if len(new_vars) == 0: raise ValueError('The scope name, {}, you specified does not contain any trainable variables.'.format(scope_name)) train_vars.extend(new_vars) log.info('Variables to be trained:\n'+str([var.name for var in train_vars])) if train_vars is not None: self.var_list = train_vars gvs = self._optimizer.compute_gradients(loss, var_list=train_vars, *args, **kwargs) if self.clip: # gradient clipping. Some gradients returned are 'None' because # no relation between the variable and loss; so we skip those. gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs if grad is not None] return gvs
def adjust_saturation(image, saturation_factor, name=None): with ops.op_scope([image], name, 'adjust_saturation') as name: # Remember original dtype to so we can convert back if needed orig_dtype = image.dtype flt_image = tf.image.convert_image_dtype(image, tf.float32) hsv = gen_image_ops.rgb_to_hsv(flt_image) hue = tf.slice(hsv, [0, 0, 0, 0], [-1, -1, -1, 1]) saturation = tf.slice(hsv, [0, 0, 0, 1], [-1, -1, -1, 1]) value = tf.slice(hsv, [0, 0, 0, 2], [-1, -1, -1, 1]) saturation *= saturation_factor saturation = clip_ops.clip_by_value(saturation, 0.0, 1.0) hsv_altered = tf.concat(3, [hue, saturation, value]) rgb_altered = gen_image_ops.hsv_to_rgb(hsv_altered) return tf.image.convert_image_dtype(rgb_altered, orig_dtype)
def buildmodel(X, paras): hconv1 = convlayer(X, paras['wconv1'], paras['bconv1'], flag='maxpool') hconv2 = convlayer(hconv1, paras['wconv2'], paras['bconv2'], flag='maxpool') hconv3 = tf.nn.conv2d(hconv2, paras['wconv3'], strides=[1,1,1,1], padding='VALID') hconv3bias = tf.nn.bias_add(hconv3, paras['bconv3']) hconv3tan = tf.nn.tanh(hconv3bias) hconv4 = tf.nn.conv2d_transpose(hconv3tan, paras['wconv4'], [batchsize,boxheight,boxwidth,2], strides=[1,1,1,1], padding='VALID') hconv4 = tf.reshape(hconv4, [-1,boxheight*boxwidth*2]) hconv4bias = tf.nn.bias_add(hconv4, paras['bconv4']) hconv4bias = tf.reshape(hconv4bias, [-1, boxheight, boxwidth, 2]) hconv4bias = tf.reshape(hconv4bias, [-1,2]) hconv4soft = tf.nn.softmax(hconv4bias) hconv4clip = tf.clip_by_value(hconv4soft, 1e-6, 1.) hconv4clip = (tf.reshape(hconv4clip, [-1, boxheight, boxwidth, 2])) return hconv4clip
def forward(self,z): if not self.ar: mu,log_sigma = self._get_mu_and_sigma(z) else: # permute z z = tf.reshape(z,[-1]+[1]*self.hps.z_size) perm = np.random.permutation(self.hps.z_size)+1 z = tf.transpose(z,np.append([0],perm)) z = tf.reshape(z,[-1,self.hps.z_size]) mu,log_sigma = ar_layer(z,self.hps,n_hidden=self.n_hidden) log_sigma = tf.clip_by_value(log_sigma,-5,5) if not self.hps.ignore_sigma_flow: y = z * tf.exp(log_sigma) + mu log_det = -1 * log_sigma else: y = z + mu log_det = 0.0 return y,log_det
def pwlin_grid(r_,rvar_,theta_,dtheta = .75): """piecewise linear with noise-adaptive grid spacing. returns xhat,dxdr where q = r/dtheta/sqrt(rvar) xhat = r * interp(q,theta) all but the last dimensions of theta must broadcast to r_ e.g. r.shape = (500,1000) is compatible with theta.shape=(500,1,7) """ ntheta = int(theta_.get_shape()[-1]) scale_ = dtheta / tf.sqrt(rvar_) ars_ = tf.clip_by_value( tf.expand_dims( tf.abs(r_)*scale_,-1),0.0, ntheta-1.0 ) centers_ = tf.constant( np.arange(ntheta),dtype=tf.float32 ) outer_distance_ = tf.maximum(0., 1.0-tf.abs(ars_ - centers_) ) # new dimension for distance to closest bin centers (or center) gain_ = tf.reduce_sum( theta_ * outer_distance_,axis=-1) # apply the gain (learnable) xhat_ = gain_ * r_ dxdr_ = tf.gradients(xhat_,r_)[0] return (xhat_,dxdr_)
def interp1d_(xin_,xp,yp_): """ Interpolate a uniformly sampled piecewise linear function. Mapping elements from xin_ to the result. Input values will be clipped to range of xp. xin_ : input tensor (real) xp : x grid (constant -- must be a 1d numpy array, uniformly spaced) yp_ : tensor of the result values at the gridpoints xp """ import tensorflow as tf x_ = tf.clip_by_value(xin_,xp.min(),xp.max()) dx = xp[1]-xp[0] assert len(xp.shape)==1,'only 1d interpolation' assert xp.shape[0]==int(yp_.get_shape()[0]) assert abs(np.diff(xp)/dx - 1.0).max() < 1e-6,'must be uniformly sampled' newshape = [ ] x1_ = tf.expand_dims(x_,-1) dt = yp_.dtype wt_ = tf.maximum(tf.constant(0.,dtype=dt), 1-abs(x1_ - tf.constant(xp,dtype=dt))/dx ) y_ = tf.reduce_sum(wt_ * yp_,axis=-1) return y_
def attack_single_step(self, x, eta, y): """ Given the original image and the perturbation computed so far, computes a new perturbation. :param x: A tensor with the original input. :param eta: A tensor the same shape as x that holds the perturbation. :param y: A tensor with the target labels or ground-truth labels. """ import tensorflow as tf from cleverhans.utils_tf import model_loss, clip_eta adv_x = x + eta preds = self.model.get_probs(adv_x) loss = model_loss(y, preds) if self.targeted: loss = -loss grad, = tf.gradients(loss, adv_x) scaled_signed_grad = self.eps_iter * tf.sign(grad) adv_x = adv_x + scaled_signed_grad if self.clip_min is not None and self.clip_max is not None: adv_x = tf.clip_by_value(adv_x, self.clip_min, self.clip_max) eta = adv_x - x eta = clip_eta(eta, self.ord, self.eps) return x, eta
def attack(self, x, y): """ This method creates a symbolic graph that given an input image, first randomly perturbs the image. The perturbation is bounded to an epsilon ball. Then multiple steps of gradient descent is performed to increase the probability of a target label or decrease the probability of the ground-truth label. :param x: A tensor with the input image. """ import tensorflow as tf from cleverhans.utils_tf import clip_eta eta = tf.random_uniform(tf.shape(x), -self.eps, self.eps) eta = clip_eta(eta, self.ord, self.eps) for i in range(self.nb_iter): x, eta = self.attack_single_step(x, eta, y) adv_x = x + eta if self.clip_min is not None and self.clip_max is not None: adv_x = tf.clip_by_value(adv_x, self.clip_min, self.clip_max) return adv_x
def expand_bboxes(xmin, xmax, ymin, ymax, cfg): """ Expand the bboxes. """ w = xmax - xmin h = ymax - ymin w = w * cfg.WIDTH_EXPANSION_FACTOR h = h * cfg.HEIGHT_EXPANSION_FACTOR half_w = w / 2. half_h = h / 2. xmin = tf.clip_by_value(xmin - half_w, 0, 1) xmax = tf.clip_by_value(xmax + half_w, 0, 1) ymin = tf.clip_by_value(ymin - half_h, 0, 1) ymax = tf.clip_by_value(ymax + half_h, 0, 1) return tf.tuple([xmin, xmax, ymin, ymax])
def demo(lr_image, hr_image): model_sr = LapSRN(mode = 'demo') hr_images_fake, residuals = model_sr.construct_net(lr_image, hr_image) ckpt_path = tf.train.latest_checkpoint('checkpoint') print(ckpt_path) restorer = tf.train.Saver(tf.global_variables()) with tf.Session() as sess: restorer.restore(sess, ckpt_path) hr_image_fake_level_2 = hr_images_fake['hr_image_fake_level_1']+residuals['residual_level_1'] hr_image_fake_level_2 = tf.clip_by_value(hr_image_fake_level_2, 0, 1) hr_image_fake_level_2 = sess.run(hr_image_fake_level_2) hr_image_fake_level_2 = hr_image_fake_level_2.squeeze() lr_image = sess.run(lr_image) lr_image = lr_image.squeeze() hr_image = sess.run(hr_image) psnr_value = psnr(hr_image.squeeze(), hr_image_fake_level_2.squeeze()) print(psnr_value) imshow(hr_image.squeeze()) imshow(hr_image_fake_level_2)
def demo(img_path): lr_img, hr_img = imgread(img_path) model = pix2pix_model(cfg) model.test_model(lr_img, hr_img) ckpt_path = tf.train.latest_checkpoint('checkpoint') restorer = tf.train.Saver(tf.global_variables()) with tf.Session() as sess: restorer.restore(sess, ckpt_path) hr_image_fake = model.fake_hr_image hr_image_fake = tf.clip_by_value(hr_image_fake, 0, 1) hr_image_fake = sess.run(hr_image_fake) hr_image_fake = hr_image_fake.squeeze() hr_image = sess.run(hr_img) psnr_value = psnr(hr_image.squeeze(), hr_image_fake.squeeze()) print(psnr_value) imshow(hr_image_fake) imshow(hr_image.squeeze())
def _create_optimizer(self): lr = self.optimizer_args['lr'] if self.optimizer_args else 1e-4 with tf.variable_scope('optimize', reuse=not self.is_train) as scope: # Setup global_step, optimizer self.global_step = tf.get_variable('global_step', shape=(), initializer=tf.constant_initializer(0.0), trainable=False) self.learning_rate = tf.train.exponential_decay(lr, self.global_step, 1e5, 0.9, staircase=True) self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, name='optimizer') # According to original paper code, learning rate of bias is 2x of base learning rate grads_vars = self.optimizer.compute_gradients(self.loss) bias_pattern = re.compile('.*/b') grads_vars_mult = [] for grad, var in grads_vars: if bias_pattern.match(var.op.name): grads_vars_mult.append((grad * 2.0, var)) else: grads_vars_mult.append((grad, var)) # According to original paper, gradient should be clipped with [-0.1, 0.1] grads_clip = [(tf.clip_by_value(grad, -0.1, 0.1), var) for grad, var in grads_vars_mult] self.train = self.optimizer.apply_gradients(grads_clip, global_step=self.global_step)
def train(self, optimizer, training_set: Iterable[Tuple[QASetting, List[Answer]]], batch_size: int, max_epochs=10, hooks=tuple(), l2=0.0, clip=None, clip_op=tf.clip_by_value, summary_writer=None, **kwargs): """ This method trains the reader (and changes its state). Args: optimizer: TF optimizer training_set: the training instances. batch_size: size of training batches max_epochs: maximum number of epochs hooks: TrainingHook implementations that are called after epochs and batches l2: whether to use l2 regularization clip: whether to apply gradient clipping and at which value clip_op: operation to perform for clipping """ batches, loss, min_op, summaries = self._setup_training( batch_size, clip, optimizer, training_set, summary_writer, l2, clip_op, **kwargs) self._train_loop(min_op, loss, batches, hooks, max_epochs, summaries, summary_writer, **kwargs)
def distance_biases(time_steps, window_size=10, reuse=False): """ Return a 2-d tensor with the values of the distance biases to be applied on the intra-attention matrix of size sentence_size Args: time_steps: tensor scalar window_size: window size reuse: reuse variables Returns: 2-d tensor (time_steps, time_steps) """ with tf.variable_scope('distance-bias', reuse=reuse): # this is d_{i-j} distance_bias = tf.get_variable('dist_bias', [window_size], initializer=tf.zeros_initializer()) r = tf.range(0, time_steps) r_matrix = tf.tile(tf.reshape(r, [1, -1]), tf.stack([time_steps, 1])) raw_idxs = r_matrix - tf.reshape(r, [-1, 1]) clipped_idxs = tf.clip_by_value(raw_idxs, 0, window_size - 1) values = tf.nn.embedding_lookup(distance_bias, clipped_idxs) return values
def relu(x, alpha=0., max_value=None): '''Rectified linear unit # Arguments alpha: slope of negative section. max_value: saturation threshold. ''' if alpha != 0.: negative_part = tf.nn.relu(-x) x = tf.nn.relu(x) if max_value is not None: max_value = _to_tensor(max_value, x.dtype.base_dtype) zero = _to_tensor(0., x.dtype.base_dtype) x = tf.clip_by_value(x, zero, max_value) if alpha != 0.: alpha = _to_tensor(alpha, x.dtype.base_dtype) x -= alpha * negative_part return x
def categorical_crossentropy(output, target, from_logits=False): '''Categorical crossentropy between an output tensor and a target tensor, where the target is a tensor of the same shape as the output. ''' # Note: tf.nn.softmax_cross_entropy_with_logits # expects logits, Keras expects probabilities. if not from_logits: # scale preds so that the class probas of each sample sum to 1 output /= tf.reduce_sum(output, reduction_indices=len(output.get_shape()) - 1, keep_dims=True) # manual computation of crossentropy epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype) output = tf.clip_by_value(output, epsilon, 1. - epsilon) return - tf.reduce_sum(target * tf.log(output), reduction_indices=len(output.get_shape()) - 1) else: return tf.nn.softmax_cross_entropy_with_logits(output, target)
def sparse_categorical_crossentropy(output, target, from_logits=False): '''Categorical crossentropy between an output tensor and a target tensor, where the target is an integer tensor. ''' # Note: tf.nn.softmax_cross_entropy_with_logits # expects logits, Keras expects probabilities. if not from_logits: epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype) output = tf.clip_by_value(output, epsilon, 1 - epsilon) output = tf.log(output) output_shape = output.get_shape() res = tf.nn.sparse_softmax_cross_entropy_with_logits( tf.reshape(output, [-1, int(output_shape[-1])]), cast(flatten(target), 'int64')) if len(output_shape) == 3: # if our output includes timesteps we need to reshape return tf.reshape(res, [-1, int(output_shape[-2])]) else: return res
def extract_batch(dataset, config): with tf.device("/cpu:0"): bboxer = PriorBoxGrid(config) data_provider = slim.dataset_data_provider.DatasetDataProvider( dataset, num_readers=2, common_queue_capacity=512, common_queue_min=32) if args.segment: im, bbox, gt, seg = data_provider.get(['image', 'object/bbox', 'object/label', 'image/segmentation']) else: im, bbox, gt = data_provider.get(['image', 'object/bbox', 'object/label']) seg = tf.expand_dims(tf.zeros(tf.shape(im)[:2]), 2) im = tf.to_float(im)/255 bbox = yxyx_to_xywh(tf.clip_by_value(bbox, 0.0, 1.0)) im, bbox, gt, seg = data_augmentation(im, bbox, gt, seg, config) inds, cats, refine = bboxer.encode_gt_tf(bbox, gt) return tf.train.shuffle_batch([im, inds, refine, cats, seg], args.batch_size, 2048, 64, num_threads=4)
def _create(self): d_loss = gan.graph.d_loss g_loss = gan.graph.g_loss g_lr = np.float32(config.g_learn_rate) d_lr = np.float32(config.d_learn_rate) gan.graph.d_vars = d_vars g_defk = {k[2:]: v for k, v in config.items() if k[2:] in inspect.getargspec(config.g_trainer).args and k.startswith("d_")} d_defk = {k[2:]: v for k, v in config.items() if k[2:] in inspect.getargspec(config.d_trainer).args and k.startswith("g_")} g_optimizer = config.g_trainer(g_lr, **g_defk) d_optimizer = config.d_trainer(d_lr, **d_defk) if(config.clipped_gradients): g_optimizer = capped_optimizer(g_optimizer, config.clipped_gradients, g_loss, g_vars) d_optimizer = capped_optimizer(d_optimizer, config.clipped_gradients, d_loss, d_vars) else: g_optimizer = g_optimizer.minimize(g_loss, var_list=g_vars) d_optimizer = d_optimizer.minimize(d_loss, var_list=d_vars) gan.graph.clip = [tf.assign(d,tf.clip_by_value(d, -config.d_clipped_weights, config.d_clipped_weights)) for d in d_vars] return g_optimizer, d_optimizer
def setup_critic_optimizer(self): logger.info('setting up critic optimizer') normalized_critic_target_tf = tf.clip_by_value(normalize(self.critic_target, self.ret_rms), self.return_range[0], self.return_range[1]) self.critic_loss = tf.reduce_mean(tf.square(self.normalized_critic_tf - normalized_critic_target_tf)) if self.critic_l2_reg > 0.: critic_reg_vars = [var for var in self.critic.trainable_vars if 'kernel' in var.name and 'output' not in var.name] for var in critic_reg_vars: logger.info(' regularizing: {}'.format(var.name)) logger.info(' applying l2 regularization with {}'.format(self.critic_l2_reg)) critic_reg = tc.layers.apply_regularization( tc.layers.l2_regularizer(self.critic_l2_reg), weights_list=critic_reg_vars ) self.critic_loss += critic_reg critic_shapes = [var.get_shape().as_list() for var in self.critic.trainable_vars] critic_nb_params = sum([reduce(lambda x, y: x * y, shape) for shape in critic_shapes]) logger.info(' critic shapes: {}'.format(critic_shapes)) logger.info(' critic params: {}'.format(critic_nb_params)) self.critic_grads = U.flatgrad(self.critic_loss, self.critic.trainable_vars, clip_norm=self.clip_norm) self.critic_optimizer = MpiAdam(var_list=self.critic.trainable_vars, beta1=0.9, beta2=0.999, epsilon=1e-08)
def clip(x, min_value, max_value): """Element-wise value clipping. If min_value > max_value, clipping range is [min_value,min_value]. # Arguments x: Tensor or variable. min_value: Tensor, float, int, or None. If min_value is None, defaults to -infinity. max_value: Tensor, float, int, or None. If max_value is None, defaults to infinity. # Returns A tensor. """ if max_value is None: max_value = np.inf if min_value is None: min_value = -np.inf min_value = _to_tensor(min_value, x.dtype.base_dtype) max_value = _to_tensor(max_value, x.dtype.base_dtype) max_value = tf.maximum(min_value, max_value) return tf.clip_by_value(x, min_value, max_value)
def create_network(self,state_dim,action_dim,scope): with tf.variable_scope(scope): layer1_size = LAYER1_SIZE layer2_size = LAYER2_SIZE state_input = tf.placeholder("float",[None,state_dim]) W1 = self.variable([state_dim,layer1_size],state_dim) b1 = self.variable([layer1_size],state_dim) W2 = self.variable([layer1_size,layer2_size],layer1_size) b2 = self.variable([layer2_size],layer1_size) W3 = tf.Variable(tf.random_uniform([layer2_size,action_dim],-3e-3,3e-3)) b3 = tf.Variable(tf.random_uniform([action_dim],-3e-3,3e-3)) layer1 = tf.nn.relu(tf.matmul(state_input,W1) + b1) layer2 = tf.nn.relu(tf.matmul(layer1,W2) + b2) action_output = tf.clip_by_value(tf.nn.relu(tf.matmul(layer2,W3) + b3),0.0,1.0) return state_input,action_output,[W1,b1,W2,b2,W3,b3]
def _deepfool2(model, x, epochs, eta, clip_min, clip_max, min_prob): y0 = tf.stop_gradient(tf.reshape(model(x), [-1])[0]) y0 = tf.to_int32(tf.greater(y0, 0.5)) def _cond(i, z): xadv = tf.clip_by_value(x + z*(1+eta), clip_min, clip_max) y = tf.stop_gradient(tf.reshape(model(xadv), [-1])[0]) y = tf.to_int32(tf.greater(y, 0.5)) return tf.logical_and(tf.less(i, epochs), tf.equal(y0, y)) def _body(i, z): xadv = tf.clip_by_value(x + z*(1+eta), clip_min, clip_max) y = tf.reshape(model(xadv), [-1])[0] g = tf.gradients(y, xadv)[0] dx = - y * g / tf.norm(g) return i+1, z+dx _, noise = tf.while_loop(_cond, _body, [0, tf.zeros_like(x)], name='_deepfool2_impl', back_prop=False) return noise
def dice_accuracy(decoded_predictions, annotations, class_nums): DiceRatio = tf.constant(0,tf.float32) misclassnum = tf.constant(0,tf.float32) class_num = tf.constant(class_nums,tf.float32) sublist = [] for index in range(1,class_nums-2): current_annotation = tf.cast(tf.equal(tf.ones_like(annotations)*index,\ annotations),tf.float32) cureent_prediction = tf.cast(tf.equal(tf.ones_like(decoded_predictions)*index,\ decoded_predictions),tf.float32) Overlap = tf.add(current_annotation,cureent_prediction) Common = tf.reduce_sum(tf.cast(tf.equal(tf.ones_like(Overlap)*2,Overlap),\ tf.float32),[0,1,2,3]) annotation_num = tf.reduce_sum(current_annotation,[0,1,2,3]) predict_num = tf.reduce_sum(cureent_prediction,[0,1,2,3]) all_num = tf.add(annotation_num,predict_num) Sub_DiceRatio = Common*2/tf.clip_by_value(all_num, 1e-10, 1e+10) misclassnum = tf.cond(tf.equal(Sub_DiceRatio,0.0), lambda: misclassnum + 1, lambda: misclassnum) sublist.append(Sub_DiceRatio) DiceRatio = DiceRatio + Sub_DiceRatio DiceRatio = DiceRatio/tf.clip_by_value(tf.cast((class_num-misclassnum-3),tf.float32),1e-10,1e+1000) return DiceRatio, sublist
def _meshgrid(self): with tf.variable_scope('_meshgrid'): x_t = tf.matmul(tf.ones(shape=tf.stack([self.out_height, 1])), tf.transpose(tf.expand_dims(tf.linspace(-1.0, 1.0, self.out_width), 1), [1, 0])) y_t = tf.matmul(tf.expand_dims(tf.linspace(-1.0, 1.0, self.out_height), 1), tf.ones(shape=tf.stack([1, self.out_width]))) x_t_flat = tf.reshape(x_t, (1, -1)) y_t_flat = tf.reshape(y_t, (1, -1)) px,py = tf.stack([x_t_flat],axis=2),tf.stack([y_t_flat],axis=2) #source control points x,y = tf.linspace(-1.,1.,self.Column_controlP_number),tf.linspace(-1.,1.,self.Row_controlP_number) x,y = tf.meshgrid(x,y) xs,ys = tf.transpose(tf.reshape(x,(-1,1))),tf.transpose(tf.reshape(y,(-1,1))) cpx,cpy = tf.transpose(tf.stack([xs],axis=2),perm=[1,0,2]),tf.transpose(tf.stack([ys],axis=2),perm=[1,0,2]) px, cpx = tf.meshgrid(px,cpx);py, cpy = tf.meshgrid(py,cpy) #Compute distance R Rx,Ry = tf.square(tf.subtract(px,cpx)),tf.square(tf.subtract(py,cpy)) R = tf.add(Rx,Ry) R = tf.multiply(R,tf.log(tf.clip_by_value(R,1e-10,1e+10))) #Source coordinates ones = tf.ones_like(x_t_flat) grid = tf.concat([ones, x_t_flat, y_t_flat,R],0) grid = tf.reshape(grid,[-1]) grid = tf.reshape(grid,[self.Column_controlP_number*self.Row_controlP_number+3,self.out_height*self.out_width]) return grid
def Train(self, loss, learning_rate, clip_value_min, clip_value_max, name='training'): tf.scalar_summary(':'.join([name, loss.op.name]), loss) optimizer = tf.train.AdagradOptimizer(learning_rate) grads_and_vars = optimizer.compute_gradients(loss) clipped_grads_and_vars = [ (tf.clip_by_value(g, clip_value_min, clip_value_max), v) for g, v in grads_and_vars ] for g, v in clipped_grads_and_vars: _ = tf.histogram_summary(':'.join([name, v.name]), v) _ = tf.histogram_summary('%s: gradient for %s' % (name, v.name), g) train_op = optimizer.apply_gradients(clipped_grads_and_vars) return train_op
def augment_image_pair(self, left_image, right_image): # randomly shift gamma random_gamma = tf.random_uniform([], 0.8, 1.2) left_image_aug = left_image ** random_gamma right_image_aug = right_image ** random_gamma # randomly shift brightness random_brightness = tf.random_uniform([], 0.5, 2.0) left_image_aug = left_image_aug * random_brightness right_image_aug = right_image_aug * random_brightness # randomly shift color random_colors = tf.random_uniform([3], 0.8, 1.2) white = tf.ones([tf.shape(left_image)[0], tf.shape(left_image)[1]]) color_image = tf.stack([white * random_colors[i] for i in range(3)], axis=2) left_image_aug *= color_image right_image_aug *= color_image # saturate left_image_aug = tf.clip_by_value(left_image_aug, 0, 1) right_image_aug = tf.clip_by_value(right_image_aug, 0, 1) return left_image_aug, right_image_aug
def SSIM(self, x, y): C1 = 0.01 ** 2 C2 = 0.03 ** 2 mu_x = slim.avg_pool2d(x, 3, 1, 'VALID') mu_y = slim.avg_pool2d(y, 3, 1, 'VALID') sigma_x = slim.avg_pool2d(x ** 2, 3, 1, 'VALID') - mu_x ** 2 sigma_y = slim.avg_pool2d(y ** 2, 3, 1, 'VALID') - mu_y ** 2 sigma_xy = slim.avg_pool2d(x * y , 3, 1, 'VALID') - mu_x * mu_y SSIM_n = (2 * mu_x * mu_y + C1) * (2 * sigma_xy + C2) SSIM_d = (mu_x ** 2 + mu_y ** 2 + C1) * (sigma_x + sigma_y + C2) SSIM = SSIM_n / SSIM_d return tf.clip_by_value((1 - SSIM) / 2, 0, 1)
def clip_gradients_by_stddev(grads_and_vars, clip_factor = 2.5): """ Clip gradients to [-clip_factor*stddev, clip_factor*stddev].""" gradients, variables = zip(*grads_and_vars) clipped_gradients = [] for gradient in gradients: if gradient is None: clipped_gradients.append(None) continue mean_gradient = tf.reduce_mean(gradient) stddev_gradient = tf.sqrt(tf.reduce_mean(tf.square(gradient - mean_gradient))) #clipped_gradient = tf.clip_by_value(gradient, -clip_factor * stddev_gradient, clip_factor * stddev_gradient) clipped_gradient = tf.cond(tf.size(gradient) < FLAGS.size_to_binarize, lambda: gradient, lambda: tf.clip_by_value(gradient, -clip_factor * stddev_gradient, clip_factor * stddev_gradient)) clipped_gradients.append(clipped_gradient) return list(zip(clipped_gradients, variables))
def create_networks(self): self.mean = tf.get_variable("means", shape=(1, int(self.state_input.get_shape()[1])), initializer=tf.constant_initializer(0), trainable=False) self.std = tf.get_variable("stds", shape=(1, int(self.state_input.get_shape()[1])), initializer=tf.constant_initializer(1), trainable=False) mean_ph = tf.placeholder(tf.float32, shape=self.mean.get_shape()) std_ph = tf.placeholder(tf.float32, shape=self.std.get_shape()) self.norm_set_op = [self.mean.assign(mean_ph), self.std.assign(std_ph)] self.norm_phs = [mean_ph, std_ph] self.good_input = tf.clip_by_value((self.state_input - self.mean) / (self.std + 1e-5), -50, 50) self.good_next_input = tf.clip_by_value((self.next_state_input - self.mean) / (self.std + 1e-5), -50, 50) self.atom_probs, self.weights, self.weights_phs = self.create_network("network", self.good_input) self.target_atom_probs, self.target_weights, self.target_weights_phs = self.create_network("target", self.good_next_input)
def relu(x, alpha=0., max_value=None): """Rectified linear unit. With default values, it returns element-wise `max(x, 0)`. # Arguments x: A tensor or variable. alpha: A scalar, slope of negative section (default=`0.`). max_value: Saturation threshold. # Returns A tensor. """ if alpha != 0.: negative_part = tf.nn.relu(-x) x = tf.nn.relu(x) if max_value is not None: max_value = _to_tensor(max_value, x.dtype.base_dtype) zero = _to_tensor(0., x.dtype.base_dtype) x = tf.clip_by_value(x, zero, max_value) if alpha != 0.: alpha = _to_tensor(alpha, x.dtype.base_dtype) x -= alpha * negative_part return x
def categorical_crossentropy(output, target, from_logits=False): """Categorical crossentropy between an output tensor and a target tensor, where the target is a tensor of the same shape as the output. """ # Note: tf.nn.softmax_cross_entropy_with_logits # expects logits, Keras expects probabilities. if not from_logits: # scale preds so that the class probas of each sample sum to 1 output /= tf.reduce_sum(output, reduction_indices=len(output.get_shape()) - 1, keep_dims=True) # manual computation of crossentropy epsilon = _to_tensor(_EPSILON, output.dtype.base_dtype) output = tf.clip_by_value(output, epsilon, 1. - epsilon) return - tf.reduce_sum(target * tf.log(output), reduction_indices=len(output.get_shape()) - 1) else: try: return tf.nn.softmax_cross_entropy_with_logits(labels=target, logits=output) except TypeError: return tf.nn.softmax_cross_entropy_with_logits(output, target)
def hard_sigmoid(x): """Segment-wise linear approximation of sigmoid. Faster than sigmoid. Returns `0.` if `x < -2.5`, `1.` if `x > 2.5`. In `-2.5 <= x <= 2.5`, returns `0.2 * x + 0.5`. # Arguments x: A tensor or variable. # Returns A tensor. """ x = (0.2 * x) + 0.5 zero = _to_tensor(0., x.dtype.base_dtype) one = _to_tensor(1., x.dtype.base_dtype) x = tf.clip_by_value(x, zero, one) return x
def build_network_rnn(self): self.states = tf.placeholder(tf.float32, [None] + list(self.env.observation_space.shape), name="states") # Observation # self.n_states = tf.placeholder(tf.float32, shape=[None], name="n_states") # Observation self.a_n = tf.placeholder(tf.float32, name="a_n") # Discrete action self.adv_n = tf.placeholder(tf.float32, name="adv_n") # Advantage n_states = tf.shape(self.states)[:1] states = tf.expand_dims(flatten(self.states), [0]) enc_cell = tf.contrib.rnn.GRUCell(self.config["n_hidden_units"]) L1, _ = tf.nn.dynamic_rnn(cell=enc_cell, inputs=states, sequence_length=n_states, dtype=tf.float32) L1 = L1[0] mu, sigma = mu_sigma_layer(L1, 1) self.normal_dist = tf.contrib.distributions.Normal(mu, sigma) self.action = self.normal_dist.sample(1) self.action = tf.clip_by_value(self.action, self.env.action_space.low[0], self.env.action_space.high[0])
def create_dis_model(self): # Set up discriminator model parameters self.get_dis_params() # Set up discriminator graph inputs self.person_board_1 = tf.placeholder(tf.float32, [None, self.n_input]) self.person_board_2 = tf.placeholder(tf.float32, [None, self.n_input]) self.gen_board = tf.placeholder(tf.float32, [None, self.n_input]) # Get discriminator outputs self.d_pred_real = self.d_predict(tf.concat(1, [self.person_board_1, self.person_board_2]), self.p_keep) self.d_pred_fake = self.d_predict(tf.concat(1, [self.person_board_1, self.gen_board]), self.p_keep) # Clamp weights self.weight_clamps = [tf.clip_by_value(self.d_weights[layer], -0.01, 0.01) for layer in self.d_weights] self.bias_clamps = [tf.clip_by_value(self.d_biases[layer], -0.01, 0.01) for layer in self.d_biases]
def get_train_op(loss, var_list=None, grad_clip=None, learning_rate=0.001, beta1=0.9, beta2=0.999): optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate, beta1=beta1, beta2=beta2) if grad_clip is None: return optimizer.minimize(loss, var_list=var_list) else: gvs = optimizer.compute_gradients(loss, var_list=var_list) def clip(grad): if grad is None: return grad else: return tf.clip_by_value(grad, -grad_clip, grad_clip) capped_gvs = [(clip(grad), var) for grad, var in gvs] return optimizer.apply_gradients(capped_gvs)
