我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.Optimizer()。
def _optimize_clone(optimizer, clone, num_clones, regularization_losses, **kwargs): """Compute losses and gradients for a single clone. Args: optimizer: A tf.Optimizer object. clone: A Clone namedtuple. num_clones: The number of clones being deployed. regularization_losses: Possibly empty list of regularization_losses to add to the clone losses. **kwargs: Dict of kwarg to pass to compute_gradients(). Returns: A tuple (clone_loss, clone_grads_and_vars). - clone_loss: A tensor for the total loss for the clone. Can be None. - clone_grads_and_vars: List of (gradient, variable) for the clone. Can be empty. """ sum_loss = _gather_clone_loss(clone, num_clones, regularization_losses) clone_grad = None if sum_loss is not None: with tf.device(clone.device): clone_grad = optimizer.compute_gradients(sum_loss, **kwargs) return sum_loss, clone_grad
def __init__(self, opt): """Constructor. Args: opt: an instance of a class that implements tf.train.Optimizer. """ if not isinstance(opt, optimizer.Optimizer): raise TypeError( 'Supplied optimizer must be an instance of tf.train.Optimizer') self._opt = opt overridden_methods = ('_apply_dense', '_resource_apply_dense', '_apply_sparse', '_resource_apply_sparse') for name in overridden_methods: fn = getattr(self._opt, name) wrapper = _get_wrapper(fn, self._opt) setattr(self._opt, name, wrapper)
def __init__(self, opt, average_decay=0.9999, num_updates=None, sequential_update=True): """Construct a new MovingAverageOptimizer. Args: opt: A tf.Optimizer that will be used to compute and apply gradients. average_decay: Float. Decay to use to maintain the moving averages of trained variables. See tf.train.ExponentialMovingAverage for details. num_updates: Optional count of number of updates applied to variables. See tf.train.ExponentialMovingAverage for details. sequential_update: Bool. If False, will compute the moving average at the same time as the model is updated, potentially doing benign data races. If True, will update the moving average after gradient updates. """ self._optimizer = opt self._ema = tf.train.ExponentialMovingAverage( average_decay, num_updates=num_updates) self._variable_map = None self._sequential_update = sequential_update
def init_train_args(self, mode='recode'): # To be used for training by tf.Optimizer objects. self.train_args = [tf.placeholder(self.dtype, shape=[None, self.n_visible])] if mode == 'target': self.train_args.append(tf.placeholder(self.dtype, shape=[None, self.n_hidden])) elif mode == 'label': self.train_args.append(tf.placeholder(tf.int32, shape=[None])) return self.train_args
def init_train_args(self, mode='recode'): # To be used for training by tf.Optimizer objects. self.train_args = [tf.placeholder(self.dtype, shape=[None] + self.shapes[0])] if mode == 'target': h_shape = self.output_shape(**kwargs) self.train_args.append(tf.placeholder(self.dtype, shape=[None] + h_shape[1:])) elif mode == 'label': self.train_args.append(tf.placeholder(tf.int32, shape=[None])) return self.train_args
def init_train_args(self, mode='recode'): # To be used for training by tf.Optimizer objects. self.train_args = [tf.placeholder(self.dtype, name='train', shape=[None, self.n_visible, self.seq_length])] if mode == 'target': self.train_args.append(tf.placeholder(self.dtype, shape=[None, self.n_output])) elif mode == 'label': self.train_args.append(tf.placeholder(tf.int32, shape=[None])) return self.train_args
def build_optimizer(args, steps, global_step): """Build the specified optimizer, log the learning rate and enalble learning rate decay is specified. Args: args: the optimization argument dict global_step: integer tensor, the current training step Returns: optimizer: tf.Optimizer object initialized """ # Extract the initial learning rate initial_lr = float(args["gd"]["args"]['learning_rate']) if args["lr_decay"]["enabled"]: # Decay the learning rate exponentially based on the number of steps. learning_rate = tf.train.exponential_decay( initial_lr, global_step, steps["decay"], args["lr_decay"]["factor"], staircase=True) # Update the learning rate parameter of the optimizer args["gd"]["args"]['learning_rate'] = learning_rate # Log the learning rate tf_log(tf.summary.scalar('learning_rate', learning_rate)) else: learning_rate = tf.constant(initial_lr) # Instantiate the optimizer optimizer = args["gd"]["optimizer"](**args["gd"]["args"]) return optimizer
def build(self, step_tensor=None): """ build optimizer tensor. This method creates the optimizer with specified parameters. It must be implemented for every `Optimizer`. Arguments: step_tensor: `tf.Tensor`. A variable holding the training step. Only necessary when optimizer has a learning rate decay. """ raise NotImplementedError
def get_tensor(self): """ get_tensor. A method to retrieve the optimizer tensor. Returns: The `Optimizer`. """ if not self.built: self.build() return self.tensor
def __call__(self): """ __call__ A shortcut for `get_tensor`. Retrieve the optimizer tensor. Returns: The `Optimizer`. """ return self.get_tensor()
def __init__(self, opt, vars_to_clip_dims, max_norm, use_locking=False, colocate_clip_ops_with_vars=False, name="VariableClipping"): """Construct a new clip-norm optimizer. Args: opt: The actual optimizer that will be used to compute and apply the gradients. Must be one of the Optimizer classes. vars_to_clip_dims: A dict with keys as Variables and values as lists of dimensions along which to compute the L2-norm. See `tf.clip_by_norm` for more details. max_norm: The L2-norm to clip to, for all variables specified. use_locking: If `True` use locks for clip update operations. colocate_clip_ops_with_vars: If `True`, try colocating the clip norm ops with the corresponding variable. name: Optional name prefix for the operations created when applying gradients. Defaults to "VariableClipping". """ super(VariableClippingOptimizer, self).__init__(use_locking, name) self._opt = opt # Defensive copy of input dict self._vars_to_clip_dims = { var: clip_dims[:] for var, clip_dims in vars_to_clip_dims.items()} self._max_norm = max_norm self._colocate_clip_ops_with_vars = colocate_clip_ops_with_vars
def optimize_clones(clones, optimizer, regularization_losses=None, **kwargs): """Compute clone losses and gradients for the given list of `Clones`. Note: The regularization_losses are added to the first clone losses. Args: clones: List of `Clones` created by `create_clones()`. optimizer: An `Optimizer` object. regularization_losses: Optional list of regularization losses. If None it will gather them from tf.GraphKeys.REGULARIZATION_LOSSES. Pass `[]` to exclude them. **kwargs: Optional list of keyword arguments to pass to `compute_gradients`. Returns: A tuple (total_loss, grads_and_vars). - total_loss: A Tensor containing the average of the clone losses including the regularization loss. - grads_and_vars: A List of tuples (gradient, variable) containing the sum of the gradients for each variable. """ grads_and_vars = [] clones_losses = [] num_clones = len(clones) if regularization_losses is None: regularization_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) for clone in clones: with tf.name_scope(clone.scope): clone_loss, clone_grad = _optimize_clone( optimizer, clone, num_clones, regularization_losses, **kwargs) if clone_loss is not None: clones_losses.append(clone_loss) grads_and_vars.append(clone_grad) # Only use regularization_losses for the first clone regularization_losses = None # Compute the total_loss summing all the clones_losses. total_loss = tf.add_n(clones_losses, name='total_loss') # Sum the gradients accross clones. grads_and_vars = _sum_clones_gradients(grads_and_vars) return total_loss, grads_and_vars
def optimize_clones(clones, optimizer, regularization_losses=None, **kwargs): """Compute clone losses and gradients for the given list of `Clones`. Note: The regularization_losses are added to the first clone losses. Args: clones: List of `Clones` created by `create_clones()`. optimizer: An `Optimizer` object. regularization_losses: Optional list of regularization losses. If None it will gather them from tf.GraphKeys.REGULARIZATION_LOSSES. Pass `[]` to exclude them. **kwargs: Optional list of keyword arguments to pass to `compute_gradients`. Returns: A tuple (total_loss, grads_and_vars). - total_loss: A Tensor containing the average of the clone losses including the regularization loss. - grads_and_vars: A List of tuples (gradient, variable) containing the sum of the gradients for each variable. """ grads_and_vars = [] clones_losses = [] num_clones = len(clones) if regularization_losses is None: regularization_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) for clone in clones: with tf.name_scope(clone.scope): clone_loss, clone_grad = _optimize_clone( optimizer, clone, num_clones, regularization_losses, **kwargs) if clone_loss is not None: clones_losses.append(clone_loss) grads_and_vars.append(clone_grad) # Only use regularization_losses for the first clone regularization_losses = None # Compute the total_loss summing all the clones_losses. total_loss = tf.add_n(clones_losses, name='total_loss') # Sum the gradients across clones. grads_and_vars = _sum_clones_gradients(grads_and_vars) return total_loss, grads_and_vars
def swapping_saver(self, var_list=None, name='swapping_saver', **kwargs): """Create a saver swapping moving averages and variables. You should use this saver during training. It will save the moving averages of the trained parameters under the original parameter names. For evaluations or inference you should use a regular saver and it will automatically use the moving averages for the trained variable. You must call this function after all variables have been created and after you have called Optimizer.minimize(). Args: var_list: List of variables to save, as per `Saver()`. If set to None, will save all the variables that have been created before this call. name: The name of the saver. **kwargs: Keyword arguments of `Saver()`. Returns: A `tf.train.Saver` object. Raises: RuntimeError: If apply_gradients or minimize has not been called before. """ if self._variable_map is None: raise RuntimeError('Must call apply_gradients or minimize before ' 'creating the swapping_saver') if var_list is None: var_list = tf.global_variables() if not isinstance(var_list, dict): var_list = saver.BaseSaverBuilder.OpListToDict(var_list) # Now swap variables and moving averages swapped_var_list = {} for k, v in six.iteritems(var_list): v_swap = self._variable_map.get(v.op.name, None) if v_swap: swapped_var_list[k] = v_swap else: swapped_var_list[k] = v # Build the swapping saver. return saver.Saver(swapped_var_list, name=name, **kwargs)
def optimize_clones(clones, optimizer, regularization_losses=None, **kwargs): """Compute clone losses and gradients for the given list of `Clones`. Note: The regularization_losses are added to the first clone losses. Args: clones: List of `Clones` created by `create_clones()`. optimizer: An `Optimizer` object. regularization_losses: Optional list of regularization losses. If None it will gather them from tf.GraphKeys.REGULARIZATION_LOSSES. Pass `[]` to exclude them. **kwargs: Optional list of keyword arguments to pass to `compute_gradients`. Returns: A tuple (total_loss, grads_and_vars). - total_loss: A Tensor containing the average of the clone losses including the regularization loss. - grads_and_vars: A List of tuples (gradient, variable) containing the sum of the gradients for each variable. """ grads_and_vars = [] clones_losses = [] num_clones = len(clones) if regularization_losses is None: regularization_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) for clone in clones: with tf.name_scope(clone.scope): clone_loss, clone_grad = _optimize_clone(optimizer, clone, num_clones, regularization_losses, **kwargs) if clone_loss is not None: clones_losses.append(clone_loss) grads_and_vars.append(clone_grad) # Only use regularization_losses for the first clone regularization_losses = None # Compute the total_loss summing all the clones_losses. total_loss = tf.add_n(clones_losses, name='total_loss') # Sum the gradients accross clones. grads_and_vars = _sum_clones_gradients(grads_and_vars) return total_loss, grads_and_vars
