我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.truncated_normal_initializer()。
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = _variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev)) if wd: # weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ dtype = tf.float16 if FLAGS.use_fp16 else tf.float32 var = _variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) if wd is not None and not tf.get_variable_scope().reuse: weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def conv3d(input_, output_dim, f_size, is_training, scope='conv3d'): with tf.variable_scope(scope) as scope: # VGG network uses two 3*3 conv layers to effectively increase receptive field w1 = tf.get_variable('w1', [f_size, f_size, f_size, input_.get_shape()[-1], output_dim], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv1 = tf.nn.conv3d(input_, w1, strides=[1, 1, 1, 1, 1], padding='SAME') b1 = tf.get_variable('b1', [output_dim], initializer=tf.constant_initializer(0.0)) conv1 = tf.nn.bias_add(conv1, b1) bn1 = tf.contrib.layers.batch_norm(conv1, is_training=is_training, scope='bn1', variables_collections=['bn_collections']) r1 = tf.nn.relu(bn1) w2 = tf.get_variable('w2', [f_size, f_size, f_size, output_dim, output_dim], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv2 = tf.nn.conv3d(r1, w2, strides=[1, 1, 1, 1, 1], padding='SAME') b2 = tf.get_variable('b2', [output_dim], initializer=tf.constant_initializer(0.0)) conv2 = tf.nn.bias_add(conv2, b2) bn2 = tf.contrib.layers.batch_norm(conv2, is_training=is_training, scope='bn2', variables_collections=['bn_collections']) r2 = tf.nn.relu(bn2) return r2
def arg_scope(self): """Configure the neural network's layers.""" batch_norm_params = { "is_training" : self.is_training, "decay" : 0.9997, "epsilon" : 0.001, "variables_collections" : { "beta" : None, "gamma" : None, "moving_mean" : ["moving_vars"], "moving_variance" : ["moving_vars"] } } with slim.arg_scope([slim.conv2d, slim.fully_connected], weights_initializer=tf.truncated_normal_initializer( stddev=self._hparams.init_stddev), weights_regularizer=slim.l2_regularizer( self._hparams.regularize_constant), activation_fn=tf.nn.relu, normalizer_fn=slim.batch_norm, normalizer_params=batch_norm_params) as sc: return sc
def _variable_with_weight_decay(self, name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ dtype = tf.float32 #if FLAGS.use_fp16 else tf.float32 var = self._variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) if wd is not None: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def __init__(self,input,name='disc'): with tf.variable_scope(name): conv1=conv_layer(input,[3,3,3,64],1) lrelu1=leaky_relu(conv1) ochannels=[64,128,128,256,256,512,512] stride=[2,1] block=[lrelu1] for i in xrange(7): block.append(self.get_block(block[-1],ochannels[i],stride[i%2])) dense1=tf.layers.dense(block[-1],1024, kernel_initializer=tf.truncated_normal_initializer() ) lrelu2=leaky_relu(dense1) self.dense2=tf.layers.dense(lrelu2,1, kernel_initializer=tf.truncated_normal_initializer(), activation=tf.sigmoid)
def build_network(name_scope, env): w_init_dense = tf.truncated_normal_initializer() #contrib.layers.xavier_initializer() b_init = tf.constant_initializer(value=0.0) with tf.variable_scope(name_scope): input_tensor = tf.placeholder(tf.float32, shape=tf_utils.get_input_tensor_shape(env), name='policy_input_'+name_scope) net = tf.contrib.layers.fully_connected(input_tensor, 32, #env.action_space.n, #32, activation_fn=tf.nn.tanh, #sigmoid, weights_initializer=w_init_dense, biases_initializer=b_init, scope='dense1_'+name_scope) net = tf.contrib.layers.fully_connected(net, env.action_space.n, weights_initializer=w_init_dense, biases_initializer=b_init, scope='dense2_'+name_scope) net = tf.contrib.layers.softmax(net) return [input_tensor], [net]
def variable_with_weight_decay(name, shape, stddev, wd): """ Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name -> name of the variable shape -> list of ints stddev -> standard deviation of a truncated Gaussian wd -> add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Rtns: var -> variable tensor """ dtype = tf.float16 if FLAGS.use_fp16 else tf.float32 var = variable_on_cpu(name,shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) if wd is not None: weight_decay = tf.mul(tf.nn.l2_loss(var),wd,name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def fc(self, input, num_out, name, relu=True): with tf.variable_scope(name) as scope: input_shape = input.get_shape() if input_shape.ndims == 4: # The input is spatial. Vectorize it first. dim = 1 for d in input_shape[1:].as_list(): dim *= d feed_in = tf.reshape(input, [-1, dim]) else: feed_in, dim = (input, input_shape[-1].value) weights = self.make_var('weights', shape=[dim, num_out], init_func=tf.truncated_normal_initializer(stddev = 0.1)) biases = self.make_var('biases', [num_out], init_func=tf.constant_initializer(0.1)) op = tf.nn.relu_layer if relu else tf.nn.xw_plus_b fc = op(feed_in, weights, biases, name=scope.name) return fc
def conv3d(input_, output_dim, f_size, is_training, scope='conv3d'): with tf.variable_scope(scope) as scope: # VGG network uses two 3*3 conv layers to effectively increase receptive field w1 = tf.get_variable('w1', [f_size, f_size, f_size, input_.get_shape()[-1], output_dim], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv1 = tf.nn.conv3d(input_, w1, strides=[1, 1, 1, 1, 1], padding='SAME') b1 = tf.get_variable('b1', [output_dim], initializer=tf.constant_initializer(0.0)) conv1 = tf.nn.bias_add(conv1, b1) bn1 = tf.contrib.layers.batch_norm(conv1, is_training=is_training, scope='bn1', decay=0.9, zero_debias_moving_mean=True, variables_collections=['bn_collections']) r1 = tf.nn.relu(bn1) w2 = tf.get_variable('w2', [f_size, f_size, f_size, output_dim, output_dim], initializer=tf.truncated_normal_initializer(stddev=0.1)) conv2 = tf.nn.conv3d(r1, w2, strides=[1, 1, 1, 1, 1], padding='SAME') b2 = tf.get_variable('b2', [output_dim], initializer=tf.constant_initializer(0.0)) conv2 = tf.nn.bias_add(conv2, b2) bn2 = tf.contrib.layers.batch_norm(conv2, is_training=is_training, scope='bn2', decay=0.9, zero_debias_moving_mean=True, variables_collections=['bn_collections']) r2 = tf.nn.relu(bn2) return r2
def _action_norm_dist(inpt, num_actions, w_init, activation_fn_v, activation_fn_a): mu = layers.fully_connected(inpt, num_outputs=num_actions, weights_initializer=w_init, activation_fn=activation_fn_v) sigma = layers.fully_connected(inpt, num_outputs=num_actions, weights_initializer=w_init, activation_fn=activation_fn_a) return mu, sigma # # cnn network frame # def cnn_frame_continu(hiddens, kerners, strides, inpt, num_actions, scope=None, activation_fn=tf.nn.relu, activation_fn_mu=tf.nn.relu, activation_fn_sigma=tf.nn.relu, reuse=None): # with tf.variable_scope(scope, reuse=reuse): # out = inpt # for kerner, stride in kerners, strides: # out = tf.nn.conv2d(input=out, filter=kerner, stride=stride) # out = layers.flatten(out) # with tf.name_scope("out"): # mu = layers.fully_connected(out, num_outputs=num_actions, weights_initializer=tf.truncated_normal_initializer(0 , 0.3), activation_fn=None) # sigma = layers.fully_connected(out, num_outputs=num_actions, weights_initializer=tf.truncated_normal_initializer(0 , 0.3), activation_fn=tf.nn.softplus) # return mu, sigma
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = _variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=tf.float32)) if wd is not None: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = _variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=tf.float32)) if wd is not None: weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev)) if wd: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def _extra_conv_arg_scope(weight_decay=0.00001, activation_fn=None, normalizer_fn=None): with slim.arg_scope( [slim.conv2d, slim.conv2d_transpose], padding='SAME', weights_regularizer=slim.l2_regularizer(weight_decay), weights_initializer=tf.truncated_normal_initializer(stddev=0.001), activation_fn=activation_fn, normalizer_fn=normalizer_fn,) as arg_sc: with slim.arg_scope( [slim.fully_connected], weights_regularizer=slim.l2_regularizer(weight_decay), weights_initializer=tf.truncated_normal_initializer(stddev=0.001), activation_fn=activation_fn, normalizer_fn=normalizer_fn) as arg_sc: return arg_sc
def conv2d(input_, output_shape, is_train, k=5, s=2, stddev=0.01, name='conv2d', with_w=False): k_h = k_w = k s_h = s_w = s with tf.variable_scope(name): weights = tf.get_variable('weights', [k_h, k_w, input_.get_shape()[-1], output_shape[-1]], initializer=tf.truncated_normal_initializer(stddev=stddev)) conv = tf.nn.conv2d(input_, weights, strides=[1, s_h, s_w, 1], padding='SAME') biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) bn = tf.contrib.layers.batch_norm(conv, center=True, scale=True, decay=0.9, is_training=is_train, updates_collections=None) out = lrelu(bn, name=lrelu) if with_w: return out, weights, biases else: return out
def conv3d(input_, output_shape, is_train, k=4, s=2, stddev=0.01, name='conv3d', with_w=False): k_d = k_h = k_w = k s_d = s_h = s_w = s with tf.variable_scope(name): weights = tf.get_variable('weights', [k_d, k_h, k_w, input_.get_shape()[-1], output_shape[-1]], initializer=tf.truncated_normal_initializer(stddev=stddev)) conv = tf.nn.conv3d(input_, weights, strides=[1, s_d, s_h, s_w, 1], padding='SAME') biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0)) conv = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) bn = tf.contrib.layers.batch_norm(conv, center=True, scale=True, decay=0.9, is_training=is_train, updates_collections=None) out = lrelu(bn, name='lrelu') if with_w: return out, weights, biases else: return out
def conv(inputs, out_num, kernel_size, scope, data_type='2D', norm=True): if data_type == '2D': outs = tf.layers.conv2d( inputs, out_num, kernel_size, padding='same', name=scope+'/conv', kernel_initializer=tf.truncated_normal_initializer) else: shape = list(kernel_size) + [inputs.shape[-1].value, out_num] weights = tf.get_variable( scope+'/conv/weights', shape, initializer=tf.truncated_normal_initializer()) outs = tf.nn.conv3d( inputs, weights, (1, 1, 1, 1, 1), padding='SAME', name=scope+'/conv') if norm: return tf.contrib.layers.batch_norm( outs, decay=0.9, epsilon=1e-5, activation_fn=tf.nn.relu, updates_collections=None, scope=scope+'/batch_norm') else: return tf.contrib.layers.batch_norm( outs, decay=0.9, epsilon=1e-5, activation_fn=None, updates_collections=None, scope=scope+'/batch_norm')
def deconv(inputs, out_num, kernel_size, scope, data_type='2D', **kws): if data_type == '2D': outs = tf.layers.conv2d_transpose( inputs, out_num, kernel_size, (2, 2), padding='same', name=scope, kernel_initializer=tf.truncated_normal_initializer) else: shape = list(kernel_size) + [out_num, out_num] input_shape = inputs.shape.as_list() out_shape = [input_shape[0]] + \ list(map(lambda x: x*2, input_shape[1:-1])) + [out_num] weights = tf.get_variable( scope+'/deconv/weights', shape, initializer=tf.truncated_normal_initializer()) outs = tf.nn.conv3d_transpose( inputs, weights, out_shape, (1, 2, 2, 2, 1), name=scope+'/deconv') return tf.contrib.layers.batch_norm( outs, decay=0.9, epsilon=1e-5, activation_fn=tf.nn.relu, updates_collections=None, scope=scope+'/batch_norm')
def constrained_conv2d(input_, output_dim, k_h=6, k_w=6, d_h=2, d_w=2, stddev=0.02, name="conv2d"): assert k_h % d_h == 0 assert k_w % d_w == 0 # constrained to have stride be a factor of kernel width # this is intended to reduce convolution artifacts with tf.variable_scope(name): w = tf.get_variable('w', [k_h, k_w, input_.get_shape()[-1], output_dim], initializer=tf.truncated_normal_initializer(stddev=stddev)) # This is meant to reduce boundary artifacts padded = tf.pad(input_, [[0, 0], [k_h-1, 0], [k_w-1, 0], [0, 0]]) conv = tf.nn.conv2d(input_, w, strides=[1, d_h, d_w, 1], padding='SAME') biases = tf.get_variable('biases', [output_dim], initializer=tf.constant_initializer(0.0)) conv = tf.nn.bias_add(conv, biases) return conv
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = _variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev)) if wd: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def deconv(inputs, out_num, kernel_size, scope, data_type='2D'): if data_type == '2D': outputs = tf.layers.conv2d_transpose( inputs, out_num, kernel_size, (2, 2), padding='same', name=scope, kernel_initializer=tf.truncated_normal_initializer) else: shape = list(kernel_size) + [out_num, out_num] input_shape = inputs.shape.as_list() out_shape = [input_shape[0]] + \ list(map(lambda x: x*2, input_shape[1:-1])) + [out_num] weights = tf.get_variable( scope+'/deconv/weights', shape, initializer=tf.truncated_normal_initializer()) outputs = tf.nn.conv3d_transpose( inputs, weights, out_shape, (1, 2, 2, 2, 1), name=scope+'/deconv') return tf.contrib.layers.batch_norm( outputs, decay=0.9, epsilon=1e-5, activation_fn=tf.nn.relu, updates_collections=None, scope=scope+'/batch_norm')
def initilizae_layer(self, name_scope, row_size, col_size, activation_function, last_hidden): # Bulid layer of the network with weights and biases weights = get_scope_variable(name_scope=name_scope, var="weights", shape=[row_size, col_size], initializer=tf.truncated_normal_initializer(mean=0.0, stddev=1.0 / np.sqrt( float(row_size)))) biases = get_scope_variable(name_scope=name_scope, var='biases', shape=[col_size], initializer=tf.constant_initializer(0.0)) self.weights_all.append(weights) self.biases_all.append(biases) variable_summaries(weights) variable_summaries(biases) with tf.variable_scope(name_scope) as scope: input = tf.matmul(last_hidden, weights) + biases if activation_function == None: output = input else: output = activation_function(input, name='output') self.inputs.append(input) self.hidden.append(output) return output
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ dtype = tf.float16 if FLAGS.use_fp16 else tf.float32 var = _variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype)) if wd is not None: weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def setUp(self): super(MLPTest, self).setUp() self.output_sizes = [11, 13, 17] self.batch_size = 5 self.input_size = 7 self.module_name = "mlp" self.initializers = { "w": tf.truncated_normal_initializer(stddev=1.0), } self.regularizers = { "w": tf.contrib.layers.l1_regularizer(scale=0.1), } self.partitioners = { "w": tf.fixed_size_partitioner(num_shards=2), }
def _build(self): """Connects the TrainableTensor module into the graph. Returns: A Tensor of shape as determined in the constructor. """ if "w" not in self._initializers: stddev = 1 / math.sqrt(np.prod(self._shape)) self._initializers["w"] = tf.truncated_normal_initializer(stddev=stddev) self._w = tf.get_variable("w", shape=self._shape, dtype=self._dtype, initializer=self._initializers["w"], partitioner=self._partitioners.get("w", None), regularizer=self._regularizers.get("w", None)) return self._w
def testInvalidInitializationParameters(self): variable_name = "trainable_variable" with self.assertRaisesRegexp(KeyError, "Invalid initializer keys.*"): snt.TrainableVariable( name=variable_name, shape=[1], initializers={"w": tf.truncated_normal_initializer(stddev=1.0), "extra": tf.truncated_normal_initializer(stddev=1.0)}) with self.assertRaisesRegexp(KeyError, "Invalid initializer keys.*"): snt.TrainableVariable( name=variable_name, shape=[1], initializers={"not_w": tf.truncated_normal_initializer(stddev=1.0)}) err = "Initializer for 'w' is not a callable function" with self.assertRaisesRegexp(TypeError, err): snt.TrainableVariable(name=variable_name, shape=[1], initializers={"w": tf.zeros([1, 2, 3])})
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType): global conv_counter name = 'conv' + str(conv_counter) conv_counter += 1 with tf.variable_scope(name): kernel_initializer = tf.truncated_normal_initializer(stddev=1e-2) conv = tf.layers.conv2d(inpOp, nOut, [kH, kW], strides=[dH, dW], padding=padType, data_format=data_format_c, kernel_initializer=kernel_initializer, use_bias=False) biases = tf.get_variable( 'biases', [nOut], tf.float32, tf.constant_initializer(0.0)) bias = tf.reshape(tf.nn.bias_add(conv, biases, data_format=data_format), conv.get_shape()) return conv
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType): global conv_counter name = 'conv' + str(conv_counter) conv_counter += 1 with tf.variable_scope(name): kernel_initializer = tf.truncated_normal_initializer(stddev=1e-2) conv = tf.layers.conv2d(inpOp, nOut, [kH, kW], strides=[dH, dW], padding=padType, data_format=data_format_c, kernel_initializer=kernel_initializer, use_bias=False) biases = tf.get_variable( 'biases', [nOut], tf.float32, tf.constant_initializer(0.0)) bias = tf.reshape(tf.nn.bias_add(conv, biases, data_format=data_format), conv.get_shape()) return bias
def _conv(inpOp, nIn, nOut, kH, kW, dH, dW, padType): global conv_counter global parameters name = 'conv' + str(conv_counter) conv_counter += 1 with tf.variable_scope(name) as scope: #kernel = tf.get_variable(name='weights', initializer=tf.random_normal([kH, kW, nIn, nOut], dtype=tf.float32, stddev=1e-2)) kernel = tf.get_variable(name='weights', shape=[kH, kW, nIn, nOut], initializer=tf.truncated_normal_initializer(dtype=tf.float32, stddev=1e-2)) strides = [1, dH, dW, 1] conv = tf.nn.conv2d(inpOp, kernel, strides, padding=padType) #biases = tf.Variable(tf.constant(0.0, shape=[nOut], dtype=tf.float32), # trainable=True, name='biases') biases = tf.get_variable(name='biases', initializer=tf.constant(0.0, shape=[nOut], dtype=tf.float32), dtype=tf.float32) bias = tf.reshape(tf.nn.bias_add(conv, biases), conv.get_shape()) parameters += [kernel, biases] return bias
def _variable_with_weight_decay(name, shape, stddev, wd): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = _variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev)) if wd is not None: weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) return var
def _variable_with_weight_decay(self, name, shape, stddev, wd=None): """Helper to create an initialized Variable with weight decay. Note that the Variable is initialized with a truncated normal distribution. A weight decay is added only if one is specified. Args: name: name of the variable shape: list of ints stddev: standard deviation of a truncated Gaussian wd: add L2Loss weight decay multiplied by this float. If None, weight decay is not added for this Variable. Returns: Variable Tensor """ var = self._variable_on_cpu( name, shape, tf.truncated_normal_initializer(stddev=stddev)) if wd is not None: # weight_decay = tf.mul(tf.constant(0.1), wd, name='weight_loss') weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss') tf.add_to_collection('losses', weight_decay) # tf.add_to_collection('losses', wd) return var
def cifarnet_arg_scope(weight_decay=0.004): """Defines the default cifarnet argument scope. Args: weight_decay: The weight decay to use for regularizing the model. Returns: An `arg_scope` to use for the inception v3 model. """ with slim.arg_scope( [slim.conv2d], weights_initializer=tf.truncated_normal_initializer(stddev=5e-2), activation_fn=tf.nn.relu): with slim.arg_scope( [slim.fully_connected], biases_initializer=tf.constant_initializer(0.1), weights_initializer=trunc_normal(0.04), weights_regularizer=slim.l2_regularizer(weight_decay), activation_fn=tf.nn.relu) as sc: return sc
def spoofnet_y_arg_scope(weight_decay=0.0004): """Defines the default cifarnet argument scope. Args: weight_decay: The weight decay to use for regularizing the model. Returns: An `arg_scope` to use for the inception v3 model. """ with slim.arg_scope( [slim.conv2d], weights_initializer=tf.truncated_normal_initializer(stddev=5e-2), #TODO: or: weights_initializer=slim.variance_scaling_initializer(), as inception/vgg/resnet # weights_regularizer=slim.l2_regularizer(weight_decay), activation_fn=tf.nn.relu ): # with slim.arg_scope( # [slim.fully_connected], # biases_initializer=tf.constant_initializer(0.1), # weights_initializer=trunc_normal(0.04), # weights_regularizer=slim.l2_regularizer(weight_decay), # activation_fn=tf.nn.relu): with slim.arg_scope([slim.max_pool2d], padding='SAME') as sc: return sc
def conv(x, c): ksize = c['ksize'] stride = c['stride'] filters_out = c['conv_filters_out'] filters_in = x.get_shape()[-1] shape = [ksize, ksize, filters_in, filters_out] # initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV) initializer = tf.contrib.layers.xavier_initializer() weights = _get_variable('weights', shape=shape, #dtype='float', initializer=initializer, weight_decay=CONV_WEIGHT_DECAY) bias = tf.get_variable('bias', [filters_out], 'float', tf.constant_initializer(0.05, dtype='float')) x = tf.nn.conv2d(x, weights, [1, stride, stride, 1], padding='SAME') return tf.nn.bias_add(x, bias)
def conv_3d(x, c): ksize = c['ksize'] stride = c['stride'] filters_out = c['conv_filters_out'] filters_in = x.get_shape()[-1] shape = [ksize, ksize, ksize, filters_in, filters_out] # initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV) initializer = tf.contrib.layers.xavier_initializer() weights = _get_variable('weights', shape=shape, #dtype='float', initializer=initializer, weight_decay=CONV_WEIGHT_DECAY) bias = tf.get_variable('bias', [filters_out], 'float', tf.constant_initializer(0.05, dtype='float')) x = tf.nn.conv3d(x, weights, [1, stride, stride, stride, 1], padding='SAME') return tf.nn.bias_add(x, bias)
def deconv_3d(x, c): ksize = c['ksize'] stride = c['stride'] filters_out = c['conv_filters_out'] filters_in = x.get_shape()[-1] # must have as_list to get a python list!!!!!!!!!!!!!! x_shape = x.get_shape().as_list() d = x_shape[1] * stride height = x_shape[2] * stride width = x_shape[3] * stride output_shape = [1, d, height, width, filters_out] strides = [1, stride, stride, stride, 1] shape = [ksize, ksize, ksize, filters_out, filters_in] # initializer = tf.truncated_normal_initializer(stddev=CONV_WEIGHT_STDDEV) initializer = tf.contrib.layers.xavier_initializer() weights = _get_variable('weights', shape=shape, dtype='float32', initializer=initializer, weight_decay=CONV_WEIGHT_DECAY) bias = tf.get_variable('bias', [filters_out], 'float32', tf.constant_initializer(0.05, dtype='float32')) x = tf.nn.conv3d_transpose(x, weights, output_shape=output_shape, strides=strides, padding='SAME') return tf.nn.bias_add(x, bias) # wrapper for batch-norm op
def conv2d(self, x, w_shape, strides, padding, name, reuse=False, initializer_w=tf.truncated_normal_initializer(mean=0.0, stddev=1e-2), initializer_b=tf.truncated_normal_initializer(mean=0.0, stddev=1e-2) ): ''' convolution layer: Input - x:input tensor - w_shape:weight shape for convolution kernel - strides - padding:'SAME' or 'VALID' - name:variable name scope - initializer_w/b:initializer of weight and bias ''' _, _, _, num_out = w_shape with tf.variable_scope(name, reuse=reuse) as scope: weights = tf.get_variable('weights', w_shape, initializer=initializer_w) biases = tf.get_variable('biases', [num_out], initializer=initializer_b) #conv conv = tf.nn.conv2d(x, weights, strides, padding) #relu relu = tf.nn.relu(conv + biases, name=scope.name) return relu
def weightVariable(shape,std=1.0,name=None): # Create a set of weights initialized with truncated normal random values name = 'weights' if name is None else name return tf.get_variable(name,shape,initializer=tf.truncated_normal_initializer(stddev=std/math.sqrt(shape[0])))
