我们从Python开源项目中，提取了以下23个代码示例，用于说明如何使用tensorflow.orthogonal_initializer()。
def __init__(self, config={}, device="/gpu:0"): config = hc.Config(config) dtype = config.dtype or "float32" initializer = config.initializer or 'orthogonal' orthogonal_gain = config.orthogonal_gain or 1.0 random_stddev = config.random_stddev or 0.02 self.dtype = self.parse_dtype(dtype) self.scope_count = 0 self.description = '' self.weights = [] self.biases = [] self.device = config.device self.initialized = False self._reuse = False if initializer == 'orthogonal': self.initializer = self.orthogonal_initializer(orthogonal_gain) else: self.initializer = self.random_initializer(random_stddev)
def recurrent_layer(tensor, cell=None, hidden_dims=128, sequence_length=None, decoder_fn=None, activation=tf.nn.tanh, initializer=tf.orthogonal_initializer(), initial_state=None, keep_prob=1.0, return_final_state=False, return_next_cell_input=True, **opts): if cell is None: cell = tf.contrib.rnn.BasicRNNCell(hidden_dims, activation=activation) # cell = tf.contrib.rnn.LSTMCell(hidden_dims, activation=activation) if keep_prob < 1.0: keep_prob = _global_keep_prob(keep_prob) cell = tf.contrib.rnn.DropoutWrapper(cell, keep_prob, keep_prob) if opts.get("name"): tf.add_to_collection(opts.get("name"), cell) if decoder_fn is None: outputs, final_state = tf.nn.dynamic_rnn(cell, tensor, sequence_length=sequence_length, initial_state=initial_state, dtype=tf.float32) final_context_state = None else: # TODO: turn off sequence_length? outputs, final_state, final_context_state = seq2seq.dynamic_rnn_decoder( cell, decoder_fn, inputs=None, sequence_length=sequence_length) if return_final_state: return final_state else: return outputs
def __init__(self, n_documents, n_topics, n_dim, temperature=1.0, W_in=None, factors_in=None): self.n_documents = n_documents # self.n_topics = n_topics # self.n_dim = n_dim self.temperature = temperature self.dropout = tf.placeholder_with_default(1., shape=[], name="dropout") scalar = 1 / np.sqrt(n_documents + n_topics) self.W = (tf.Variable( # unnormalized embedding weights tf.random_normal([n_documents, n_topics], mean=0, stddev=50*scalar), name="doc_embeddings") if W_in is None else W_in) # factors = (tf.Variable( # topic vectors # _orthogonal_matrix((n_topics, n_dim)).astype("float32") * scalar, # name="topics") if factors_in is None else factors_in) # tf 0.12.0 only factors = (tf.get_variable("topics", shape=(n_topics, n_dim), dtype=tf.float32, initializer= tf.orthogonal_initializer(gain=scalar)) if factors_in is None else factors_in) self.factors = tf.nn.dropout(factors, self.dropout)
def embed_inputs(raw_inputs, vocab_size, embed_size, reserve_zero=True, name='embed', embeddings=None): with tf.variable_scope(name): if embeddings is None: shape = (vocab_size, embed_size) embeddings = tf.get_variable( 'embeddings', shape=(vocab_size, embed_size), initializer=tf.orthogonal_initializer(), dtype=tf.float32) # If reserve_zero, make sure first row is always zeros if reserve_zero: zeros = tf.zeros((1, embed_size), dtype=tf.float32) embeddings = tf.concat([zeros, embeddings], axis=0) inputs = tf.nn.embedding_lookup(embeddings, raw_inputs) return inputs
def head(endpoints, embedding_dim, is_training): endpoints['head_output'] = slim.fully_connected( endpoints['model_output'], 1024, normalizer_fn=slim.batch_norm, normalizer_params={ 'decay': 0.9, 'epsilon': 1e-5, 'scale': True, 'is_training': is_training, 'updates_collections': tf.GraphKeys.UPDATE_OPS, }) endpoints['emb'] = endpoints['emb_raw'] = slim.fully_connected( endpoints['head_output'], embedding_dim, activation_fn=None, weights_initializer=tf.orthogonal_initializer(), scope='emb') return endpoints
def head(endpoints, embedding_dim, is_training): endpoints['head_output'] = slim.fully_connected( endpoints['model_output'], 1024, normalizer_fn=slim.batch_norm, normalizer_params={ 'decay': 0.9, 'epsilon': 1e-5, 'scale': True, 'is_training': is_training, 'updates_collections': tf.GraphKeys.UPDATE_OPS, }) endpoints['emb_raw'] = slim.fully_connected( endpoints['head_output'], embedding_dim, activation_fn=None, weights_initializer=tf.orthogonal_initializer(), scope='emb') endpoints['emb'] = tf.nn.l2_normalize(endpoints['emb_raw'], -1) return endpoints
def random_orthogonal_initializer(use_gpu=True): return tf.orthogonal_initializer()
def __init__(self, cell_size): self.cell_size = cell_size self.default_initializer = tf.get_variable_scope().initializer or init_ops.glorot_uniform_initializer() self.initializer = tf.orthogonal_initializer()
def orthogonal_initializer(self, gain): def _build(): return tf.orthogonal_initializer(gain) return _build
def Weight_variable(self, shape, weight_decay=0.000004): with tf.name_scope('Weight') as scope: #weights = tf.get_variable(name='Weight', initializer=tf.truncated_normal(shape, stddev=0.1), trainable=True, regularizer=self.Regloss_l2) #initi = tf.contrib.layers.xavier_initializer() #initi = tf.orthogonal_initializer() initi = tf.random_uniform_initializer(minval=-0.08, maxval=0.08) weights = tf.Variable(initi(shape)) tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, tf.nn.l2_loss(weights)* weight_decay) return weights
def get_initializer(name): if name == 'identity': return identity_initializer() elif name == 'orthogonal': return tf.orthogonal_initializer(gain=1.0) elif name == 'truncated_normal': return tf.truncated_normal_initializer()
def orthogonal_initializer(): """Return an orthogonal initializer. Random orthogonal matrix is byproduct of singular value decomposition applied on a matrix initialized with normal distribution. The initializer works with 2D square matrices and matrices that can be splitted along axis 1 to several 2D matrices. In the latter case, each submatrix is initialized independently and the resulting orthogonal matrices are concatenated along axis 1. Note this is a higher order function in order to mimic the tensorflow initializer API. """ # pylint: disable=unused-argument def func(shape, dtype, partition_info=None): if len(shape) != 2: raise ValueError( "Orthogonal initializer only works with 2D matrices.") if shape[1] % shape[0] != 0: raise ValueError("Shape {} is not compatible with orthogonal " "initializer.".format(str(shape))) mult = int(shape[1] / shape[0]) dim = shape[0] orthogonals = [] for _ in range(mult): matrix = tf.random_normal([dim, dim], dtype=dtype) orthogonals.append(tf.svd(matrix)[1]) return tf.concat(orthogonals, 1) # pylint: enable=unused-argument return func # pylint: disable=too-few-public-methods
def __init__(self, num_units, activation=None, reuse=None): tf.contrib.rnn.GRUCell.__init__( self, num_units, activation, reuse, kernel_initializer=tf.orthogonal_initializer())
def call(self, inputs, state): """Gated recurrent unit (GRU) with nunits cells.""" with tf.variable_scope("gates"): input_to_gates = tf.layers.dense( inputs, 2 * self._num_units, name="input_proj", kernel_initializer=tf.glorot_normal_initializer(), use_bias=self.use_input_bias) # Nematus does the orthogonal initialization probably differently state_to_gates = tf.layers.dense( state, 2 * self._num_units, use_bias=self.use_state_bias, kernel_initializer=orthogonal_initializer(), name="state_proj") gates_input = state_to_gates + input_to_gates reset, update = tf.split( tf.sigmoid(gates_input), num_or_size_splits=2, axis=1) with tf.variable_scope("candidate"): input_to_candidate = tf.layers.dense( inputs, self._num_units, use_bias=self.use_input_bias, kernel_initializer=tf.glorot_normal_initializer(), name="input_proj") state_to_candidate = tf.layers.dense( state, self._num_units, use_bias=self.use_state_bias, kernel_initializer=orthogonal_initializer(), name="state_proj") candidate = self._activation( state_to_candidate * reset + input_to_candidate) new_state = update * state + (1 - update) * candidate return new_state, new_state
def get_variable_initializer(hparams): """Get variable initializer from hparams.""" if hparams.initializer == "orthogonal": return tf.orthogonal_initializer(gain=hparams.initializer_gain) elif hparams.initializer == "uniform": max_val = 0.1 * hparams.initializer_gain return tf.random_uniform_initializer(-max_val, max_val) elif hparams.initializer == "normal_unit_scaling": return tf.variance_scaling_initializer( hparams.initializer_gain, mode="fan_avg", distribution="normal") elif hparams.initializer == "uniform_unit_scaling": return tf.variance_scaling_initializer( hparams.initializer_gain, mode="fan_avg", distribution="uniform") else: raise ValueError("Unrecognized initializer: %s" % hparams.initializer)
def head(endpoints, embedding_dim, is_training): endpoints['emb'] = endpoints['emb_raw'] = slim.fully_connected( endpoints['model_output'], embedding_dim, activation_fn=None, weights_initializer=tf.orthogonal_initializer(), scope='emb') return endpoints
def head(endpoints, embedding_dim, is_training): endpoints['emb_raw'] = slim.fully_connected( endpoints['model_output'], embedding_dim, activation_fn=None, weights_initializer=tf.orthogonal_initializer(), scope='emb') endpoints['emb'] = tf.nn.l2_normalize(endpoints['emb_raw'], -1) return endpoints
def create_model(session, config, forward_only): """Create translation model and initialize or load parameters in session.""" dtype = tf.float32 optimizer = None if not forward_only: optimizer = tf.train.AdamOptimizer(config.learning_rate) if config.activation == "elu": activation = tf.nn.elu elif config.activation == "prelu": activation = prelu else: activation = tf.identity weight_initializer = tf.orthogonal_initializer if config.orthogonal_initializer else tf.uniform_unit_scaling_initializer bias_initializer = tf.zeros_initializer model = seq2seq_model.Seq2SeqModel( config.en_vocab_size, config.fr_vocab_size, config.buckets, config.size, config.num_layers, config.latent_dim, config.max_gradient_norm, config.batch_size, config.learning_rate, config.kl_min, config.word_dropout_keep_prob, config.anneal, config.use_lstm, optimizer=optimizer, activation=activation, forward_only=forward_only, feed_previous=config.feed_previous, bidirectional=config.bidirectional, weight_initializer=weight_initializer, bias_initializer=bias_initializer, iaf=config.iaf, dtype=dtype) ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir) if not FLAGS.new and ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) model.saver.restore(session, ckpt.model_checkpoint_path) else: print("Created model with fresh parameters.") session.run(tf.global_variables_initializer()) return model
