我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.expand_dims()。
def one_hot_encoding(labels, num_classes, scope=None): """Transform numeric labels into onehot_labels. Args: labels: [batch_size] target labels. num_classes: total number of classes. scope: Optional scope for op_scope. Returns: one hot encoding of the labels. """ with tf.op_scope([labels], scope, 'OneHotEncoding'): batch_size = labels.get_shape()[0] indices = tf.expand_dims(tf.range(0, batch_size), 1) labels = tf.cast(tf.expand_dims(labels, 1), indices.dtype) concated = tf.concat(1, [indices, labels]) onehot_labels = tf.sparse_to_dense( concated, tf.pack([batch_size, num_classes]), 1.0, 0.0) onehot_labels.set_shape([batch_size, num_classes]) return onehot_labels
def SampleRandomFrames(model_input, num_frames, num_samples): """Samples a random set of frames of size num_samples. Args: model_input: A tensor of size batch_size x max_frames x feature_size num_frames: A tensor of size batch_size x 1 num_samples: A scalar Returns: `model_input`: A tensor of size batch_size x num_samples x feature_size """ batch_size = tf.shape(model_input)[0] frame_index = tf.cast( tf.multiply( tf.random_uniform([batch_size, num_samples]), tf.tile(tf.cast(num_frames, tf.float32), [1, num_samples])), tf.int32) batch_index = tf.tile( tf.expand_dims(tf.range(batch_size), 1), [1, num_samples]) index = tf.stack([batch_index, frame_index], 2) return tf.gather_nd(model_input, index)
def model(self, features, labels): x = features["observation"] x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) actions = tf.one_hot(tf.reshape(features["action"],[-1]), depth=6, on_value=1.0, off_value=0.0, axis=1) x = tf.concat(1, [tf.contrib.layers.flatten(x), actions]) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None) prediction = tf.sigmoid(logits, name="prediction") loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1)),name="loss") train_op = tf.contrib.layers.optimize_loss( loss, tf.contrib.framework.get_global_step(), optimizer='Adam', learning_rate=self.learning_rate) tf.add_to_collection('prediction', prediction) tf.add_to_collection('loss', loss) return prediction, loss, train_op
def parse_example(serialized_example): features = tf.parse_single_example( serialized_example, # Defaults are not specified since both keys are required. features={ 'shape': tf.FixedLenFeature([], tf.string), 'img_raw': tf.FixedLenFeature([], tf.string), 'gt_raw': tf.FixedLenFeature([], tf.string), 'example_name': tf.FixedLenFeature([], tf.string) }) with tf.variable_scope('decoder'): shape = tf.decode_raw(features['shape'], tf.int32) image = tf.decode_raw(features['img_raw'], tf.float32) ground_truth = tf.decode_raw(features['gt_raw'], tf.uint8) example_name = features['example_name'] with tf.variable_scope('image'): # reshape and add 0 dimension (would be batch dimension) image = tf.expand_dims(tf.reshape(image, shape), 0) with tf.variable_scope('ground_truth'): # reshape ground_truth = tf.cast(tf.reshape(ground_truth, shape[:-1]), tf.float32) return image, ground_truth, example_name
def convolve_me(self, hyp, pd): network = input_data(shape=[None, pd.max_sequence], name='input') network = tflearn.embedding(network, input_dim=pd.vocab_size, output_dim=pd.emb_size, name="embedding") branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2") branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2") branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2") network = merge([branch1, branch2, branch3], mode='concat', axis=1) network = tf.expand_dims(network, 2) network = global_max_pool(network) network = dropout(network, 0.5) network = fully_connected(network, 2, activation='softmax') network = regression(network, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy', name='target') return network
def decode(self, cell_dec, enc_final_state, output_size, output_embed_matrix, training, grammar_helper=None): if self.config.use_dot_product_output: output_layer = DotProductLayer(output_embed_matrix) else: output_layer = tf.layers.Dense(output_size, use_bias=False) go_vector = tf.ones((self.batch_size,), dtype=tf.int32) * self.config.grammar.start if training: output_ids_with_go = tf.concat([tf.expand_dims(go_vector, axis=1), self.output_placeholder], axis=1) outputs = tf.nn.embedding_lookup([output_embed_matrix], output_ids_with_go) helper = TrainingHelper(outputs, self.output_length_placeholder+1) else: helper = GreedyEmbeddingHelper(output_embed_matrix, go_vector, self.config.grammar.end) if self.config.use_grammar_constraints: decoder = GrammarBasicDecoder(self.config.grammar, cell_dec, helper, enc_final_state, output_layer=output_layer, training_output = self.output_placeholder if training else None, grammar_helper=grammar_helper) else: decoder = BasicDecoder(cell_dec, helper, enc_final_state, output_layer=output_layer) final_outputs, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder, impute_finished=True, maximum_iterations=self.max_length) return final_outputs
def bag_of_tokens(config, labels, label_lengths): if config.train_output_embeddings: with tf.variable_scope('embed', reuse=True): output_embeddings = tf.get_variable('output_embedding') else: output_embeddings = tf.constant(config.output_embedding_matrix) #everything_label_placeholder = tf.placeholder(shape=(None, config.max_length,), dtype=tf.int32) #everything_label_length_placeholder = tf.placeholder(shape=(None,), dtype=tf.int32) labels = tf.constant(np.array(labels)) embedded_output = tf.gather(output_embeddings, labels) print('embedded_output before', embedded_output) #mask = tf.sequence_mask(label_lengths, maxlen=config.max_length, dtype=tf.float32) # note: this multiplication will broadcast the mask along all elements of the depth dimension # (which is why we run the expand_dims to choose how to broadcast) #embedded_output = embedded_output * tf.expand_dims(mask, axis=2) #print('embedded_output after', embedded_output) return tf.reduce_sum(embedded_output, axis=1)
def build_encoder(self): """Inference Network. q(h|X)""" with tf.variable_scope("encoder"): q_cell = tf.nn.rnn_cell.LSTMCell(self.embed_dim, self.vocab_size) a_cell = tf.nn.rnn_cell.LSTMCell(self.embed_dim, self.vocab_size) l1 = tf.nn.relu(tf.nn.rnn_cell.linear(tf.expand_dims(self.x, 0), self.embed_dim, bias=True, scope="l1")) l2 = tf.nn.relu(tf.nn.rnn_cell.linear(l1, self.embed_dim, bias=True, scope="l2")) self.mu = tf.nn.rnn_cell.linear(l2, self.h_dim, bias=True, scope="mu") self.log_sigma_sq = tf.nn.rnn_cell.linear(l2, self.h_dim, bias=True, scope="log_sigma_sq") eps = tf.random_normal((1, self.h_dim), 0, 1, dtype=tf.float32) sigma = tf.sqrt(tf.exp(self.log_sigma_sq)) _ = tf.histogram_summary("mu", self.mu) _ = tf.histogram_summary("sigma", sigma) self.h = self.mu + sigma * eps
def build_encoder(self): """Inference Network. q(h|X)""" with tf.variable_scope("encoder"): self.l1_lin = linear(tf.expand_dims(self.x, 0), self.embed_dim, bias=True, scope="l1") self.l1 = tf.nn.relu(self.l1_lin) self.l2_lin = linear(self.l1, self.embed_dim, bias=True, scope="l2") self.l2 = tf.nn.relu(self.l2_lin) self.mu = linear(self.l2, self.h_dim, bias=True, scope="mu") self.log_sigma_sq = linear(self.l2, self.h_dim, bias=True, scope="log_sigma_sq") self.eps = tf.random_normal((1, self.h_dim), 0, 1, dtype=tf.float32) self.sigma = tf.sqrt(tf.exp(self.log_sigma_sq)) self.h = tf.add(self.mu, tf.mul(self.sigma, self.eps)) _ = tf.histogram_summary("mu", self.mu) _ = tf.histogram_summary("sigma", self.sigma) _ = tf.histogram_summary("h", self.h) _ = tf.histogram_summary("mu + sigma", self.mu + self.sigma)
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): """ A super model that combine one or more models """ models = FLAGS.wide_and_deep_models outputs = [] for model_name in map(lambda x: x.strip(), models.split(",")): model = getattr(frame_level_models, model_name, None)() output = model.create_model(model_input, vocab_size, num_frames, l2_penalty=l2_penalty, **unused_params)["predictions"] outputs.append(tf.expand_dims(output, axis=2)) num_models = len(outputs) model_outputs = tf.concat(outputs, axis=2) # linear_combination = tf.get_variable("combine", shape=[vocab_size,num_models], # dtype=tf.float32, initializer=tf.zeros_initializer(), # regularizer=slim.l2_regularizer(l2_penalty)) # combination = tf.nn.softmax(linear_combination) combination = tf.fill(dims=[vocab_size,num_models], value=1.0/num_models) output_sum = tf.einsum("ijk,jk->ij", model_outputs, combination) return {"predictions": output_sum}
def __init__(self, ob_space, ac_space, size=256, **kwargs): self.x = x = tf.placeholder(tf.float32, [None] + list(ob_space)) for i in range(4): x = tf.nn.elu(conv2d(x, 32, "l{}".format(i + 1), [3, 3], [2, 2])) # introduce a "fake" batch dimension of 1 after flatten so that we can do GRU over time dim x = tf.expand_dims(flatten(x), 1) gru = rnn.GRUCell(size) h_init = np.zeros((1, size), np.float32) self.state_init = [h_init] h_in = tf.placeholder(tf.float32, [1, size]) self.state_in = [h_in] gru_outputs, gru_state = tf.nn.dynamic_rnn( gru, x, initial_state=h_in, sequence_length=[size], time_major=True) x = tf.reshape(gru_outputs, [-1, size]) self.logits = linear(x, ac_space, "action", normalized_columns_initializer(0.01)) self.vf = tf.reshape(linear(x, 1, "value", normalized_columns_initializer(1.0)), [-1]) self.state_out = [gru_state[:1]] self.sample = categorical_sample(self.logits, ac_space)[0, :] self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name)
def model(self, features, labels): x = features["observation"] x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.convolution2d(x, 2, kernel_size=[3, 3], stride=[2, 2], activation_fn=tf.nn.elu) x = tf.contrib.layers.flatten(x) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) x = tf.contrib.layers.fully_connected(x, 100, activation_fn=tf.nn.elu) logits = tf.contrib.layers.fully_connected(x, 1, activation_fn=None) prediction = tf.sigmoid(logits) loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, tf.expand_dims(labels, axis=1))) train_op = tf.contrib.layers.optimize_loss( loss, tf.contrib.framework.get_global_step(), optimizer='Adam', learning_rate=0.01) tf.add_to_collection('prediction', prediction) tf.add_to_collection('loss', loss) return prediction, loss, train_op
def lengths_to_mask(lengths_b, max_length): """ Turns a vector of lengths into a boolean mask Args: lengths_b: an integer vector of lengths max_length: maximum length to fill the mask Returns: a boolean array of shape (batch_size, max_length) row[i] consists of True repeated lengths_b[i] times, followed by False """ lengths_b = tf.convert_to_tensor(lengths_b) assert lengths_b.get_shape().ndims == 1 mask_bt = tf.expand_dims(tf.range(max_length), 0) < tf.expand_dims(lengths_b, 1) return mask_bt
def _distort_image(self, image): """Distort one image for training a network. Adopted the standard data augmentation scheme that is widely used for this dataset: the images are first zero-padded with 4 pixels on each side, then randomly cropped to again produce distorted images; half of the images are then horizontally mirrored. Args: image: input image. Returns: distored image. """ image = tf.image.resize_image_with_crop_or_pad( image, self.height + 8, self.width + 8) distorted_image = tf.random_crop(image, [self.height, self.width, self.depth]) # Randomly flip the image horizontally. distorted_image = tf.image.random_flip_left_right(distorted_image) if self.summary_verbosity >= 3: tf.summary.image('distorted_image', tf.expand_dims(distorted_image, 0)) return distorted_image
def _fc(self, x, fan_in, fan_out, layer_name, activation=None, L2=1, use_bias=True, wmin=None,wmax=None,analysis=False): show_weight = self.flags.visualize and 'weight' in self.flags.visualize if wmin is not None or wmax is not None: use_bias = False assert wmin is not None and wmax is not None with tf.variable_scope(layer_name.split('/')[-1]): w,b = self._get_fc_weights(fan_in, fan_out, layer_name) if wmin is not None: wr = wmax-wmin w = self._activate(w,'sigmoid')*wr+wmin #w = tf.clip_by_value(w,wmin,wmax) net = tf.matmul(x,w) if use_bias: net = tf.nn.bias_add(net, b) net = self._activate(net, activation) if show_weight: tf.summary.histogram(name='W', values=w, collections=[tf.GraphKeys.WEIGHTS]) if use_bias: tf.summary.histogram(name='bias', values=b, collections=[tf.GraphKeys.WEIGHTS]) if analysis: net1 = tf.expand_dims(x,2)*tf.expand_dims(w,0) #net1 = tf.reshape(net1,[tf.shape(x)[0],fan_in*fan_out]) return net,net1 return net
def Minibatch_Discriminator(input, num_kernels=100, dim_per_kernel=5, init=False, name='MD'): num_inputs=df_dim*4 theta = tf.get_variable(name+"/theta",[num_inputs, num_kernels, dim_per_kernel], initializer=tf.random_normal_initializer(stddev=0.05)) log_weight_scale = tf.get_variable(name+"/lws",[num_kernels, dim_per_kernel], initializer=tf.constant_initializer(0.0)) W = tf.mul(theta, tf.expand_dims(tf.exp(log_weight_scale)/tf.sqrt(tf.reduce_sum(tf.square(theta),0)),0)) W = tf.reshape(W,[-1,num_kernels*dim_per_kernel]) x = input x=tf.reshape(x, [batchsize,num_inputs]) activation = tf.matmul(x, W) activation = tf.reshape(activation,[-1,num_kernels,dim_per_kernel]) abs_dif = tf.mul(tf.reduce_sum(tf.abs(tf.sub(tf.expand_dims(activation,3),tf.expand_dims(tf.transpose(activation,[1,2,0]),0))),2), 1-tf.expand_dims(tf.constant(np.eye(batchsize),dtype=np.float32),1)) f = tf.reduce_sum(tf.exp(-abs_dif),2)/tf.reduce_sum(tf.exp(-abs_dif)) print(f.get_shape()) print(input.get_shape()) return tf.concat(1,[x, f])
def output_module(self): """ 1.use attention mechanism between query and hidden states, to get weighted sum of hidden state. 2.non-linearity of query and hidden state to get label. input: query_embedding:[batch_size,embed_size], hidden state:[batch_size,block_size,hidden_size] of memory :return:y: predicted label.[] """ # 1.use attention mechanism between query and hidden states, to get weighted sum of hidden state. # 1.1 get possibility distribution (of similiarity) p=tf.nn.softmax(tf.multiply(tf.expand_dims(self.query_embedding,axis=1),self.hidden_state)) #shape:[batch_size,block_size,hidden_size]<---query_embedding_expand:[batch_size,1,hidden_size]; hidden_state:[batch_size,block_size,hidden_size] # 1.2 get weighted sum of hidden state u=tf.reduce_sum(tf.multiply(p,self.hidden_state),axis=1) #shape:[batch_size,hidden_size]<----------([batch_size,block_size,hidden_size],[batch_size,block_size,hidden_size]) # 2.non-linearity of query and hidden state to get label H_u_matmul=tf.matmul(u,self.H)+self.h_u_bias #shape:[batch_size,hidden_size]<----([batch_size,hidden_size],[hidden_size,hidden_size]) activation=self.activation(self.query_embedding + H_u_matmul,scope="query_add_hidden") #shape:[batch_size,hidden_size] activation = tf.nn.dropout(activation,keep_prob=self.dropout_keep_prob) #shape:[batch_size,hidden_size] y=tf.matmul(activation,self.R)+self.y_bias #shape:[batch_size,vocab_size]<-----([batch_size,hidden_size],[hidden_size,vocab_size]) return y #shape:[batch_size,vocab_size]
def rnn_story(self): """ run rnn for story to get last hidden state input is: story: [batch_size,story_length,embed_size] :return: last hidden state. [batch_size,embed_size] """ # 1.split input to get lists. input_split=tf.split(self.story_embedding,self.story_length,axis=1) #a list.length is:story_length.each element is:[batch_size,1,embed_size] input_list=[tf.squeeze(x,axis=1) for x in input_split] #a list.length is:story_length.each element is:[batch_size,embed_size] # 2.init keys(w_all) and values(h_all) of memory h_all=tf.get_variable("hidden_states",shape=[self.block_size,self.dimension],initializer=self.initializer)# [block_size,hidden_size] w_all=tf.get_variable("keys", shape=[self.block_size,self.dimension],initializer=self.initializer)# [block_size,hidden_size] # 3.expand keys and values to prepare operation of rnn w_all_expand=tf.tile(tf.expand_dims(w_all,axis=0),[self.batch_size,1,1]) #[batch_size,block_size,hidden_size] h_all_expand=tf.tile(tf.expand_dims(h_all,axis=0),[self.batch_size,1,1]) #[batch_size,block_size,hidden_size] # 4. run rnn using input with cell. for i,input in enumerate(input_list): h_all_expand=self.cell(input,h_all_expand,w_all_expand,i) #w_all:[batch_size,block_size,hidden_size]; h_all:[batch_size,block_size,hidden_size] return h_all_expand #[batch_size,block_size,hidden_size]
def position_wise_feed_forward_fn(self): """ x: [batch,sequence_length,d_model] :return: [batch,sequence_length,d_model] """ output=None #1.conv1 input=tf.expand_dims(self.x,axis=3) #[batch,sequence_length,d_model,1] # conv2d.input: [None,sentence_length,embed_size,1]. filter=[filter_size,self.embed_size,1,self.num_filters] # output with padding:[None,sentence_length,1,1] filter1 = tf.get_variable("filter1"+str(self.layer_index) , shape=[1, self.d_model, 1, 1],initializer=self.initializer) ouput_conv1=tf.nn.conv2d(input,filter1,strides=[1,1,1,1],padding="VALID",name="conv1") #[batch,sequence_length,1,1] print("output_conv1:",ouput_conv1) #2.conv2 filter2 = tf.get_variable("filter2"+str(self.layer_index), [1, 1, 1, self.d_model], initializer=self.initializer) output_conv2=tf.nn.conv2d(ouput_conv1,filter2,strides=[1,1,1,1],padding="VALID",name="conv2") #[batch,sequence_length,1,d_model] output=tf.squeeze(output_conv2) #[batch,sequence_length,d_model] return output #[batch,sequence_length,d_model] #test function of position_wise_feed_forward_fn #time spent:OLD VERSION: length=8000,time spent:35.6s; NEW VERSION:0.03s
def inference(self): """main computation graph here: 1. embeddding layers, 2.convolutional layer, 3.max-pooling, 4.softmax layer.""" # 1.=====>get emebedding of words in the sentence self.embedded_words1 = tf.nn.embedding_lookup(self.Embedding,self.input_x)#[None,sentence_length,embed_size] self.sentence_embeddings_expanded1=tf.expand_dims(self.embedded_words1,-1) #[None,sentence_length,embed_size,1). expand dimension so meet input requirement of 2d-conv self.embedded_words2 = tf.nn.embedding_lookup(self.Embedding,self.input_x2)#[None,sentence_length,embed_size] self.sentence_embeddings_expanded2=tf.expand_dims(self.embedded_words2,-1) #[None,sentence_length,embed_size,1). expand dimension so meet input requirement of 2d-conv #2.1 get features of sentence1 h1=self.conv_relu_pool_dropout(self.sentence_embeddings_expanded1,name_scope_prefix="s1") #[None,num_filters_total] #2.2 get features of sentence2 h2 =self.conv_relu_pool_dropout(self.sentence_embeddings_expanded2,name_scope_prefix="s2") # [None,num_filters_total] #3. concat features h=tf.concat([h1,h2],axis=1) #[None,num_filters_total*2] #4. logits(use linear layer)and predictions(argmax) with tf.name_scope("output"): logits = tf.matmul(h,self.W_projection) + self.b_projection #shape:[None, self.num_classes]==tf.matmul([None,self.num_filters_total*2],[self.num_filters_total*2,self.num_classes]) return logits
def loss_nce(self,l2_lambda=0.0001): #0.0001-->0.001 """calculate loss using (NCE)cross entropy here""" # Compute the average NCE loss for the batch. # tf.nce_loss automatically draws a new sample of the negative labels each # time we evaluate the loss. if self.is_training: #training #labels=tf.reshape(self.input_y,[-1]) #[batch_size,1]------>[batch_size,] labels=tf.expand_dims(self.input_y,1) #[batch_size,]----->[batch_size,1] loss = tf.reduce_mean( #inputs: A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. tf.nn.nce_loss(weights=tf.transpose(self.W_projection),#[hidden_size*2, num_classes]--->[num_classes,hidden_size*2]. nce_weights:A `Tensor` of shape `[num_classes, dim].O.K. biases=self.b_projection, #[label_size]. nce_biases:A `Tensor` of shape `[num_classes]`. labels=labels, #[batch_size,1]. train_labels, # A `Tensor` of type `int64` and shape `[batch_size,num_true]`. The target classes. inputs=self.output_rnn_last,# [batch_size,hidden_size*2] #A `Tensor` of shape `[batch_size, dim]`. The forward activations of the input network. num_sampled=self.num_sampled, #scalar. 100 num_classes=self.num_classes,partition_strategy="div")) #scalar. 1999 l2_losses = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables() if 'bias' not in v.name]) * l2_lambda loss = loss + l2_losses return loss
def average_gradients(cls, tower_grads): """Average a list of (grads, vars) produced by `compute_gradients`.""" average_grads = [] for grads_and_vars in zip(*tower_grads): # print(grads_and_vars) grads = [] for g, _ in grads_and_vars: # print(g.get_shape().as_list(), g) grads.append(tf.expand_dims(g, axis=0)) grad = tf.concat(grads, axis=0) grad = tf.reduce_mean(grad, axis=0) # all variables are the same so we just use the first gpu variables var = grads_and_vars[0][1] grad_and_var = (grad, var) average_grads.append(grad_and_var) return average_grads
def custom_loss(y_true, y_pred): # Get prediction pred_box_xy = tf.sigmoid(y_pred[..., :2]) pred_box_wh = y_pred[..., 2:4] pred_box_conf = tf.sigmoid(y_pred[..., 4]) # Get ground truth true_box_xy = y_true[..., :2] true_box_wh = y_true[..., 2:4] true_box_conf = y_true[..., 4] # Determine the mask: simply the position of the ground truth boxes (the predictors) true_mask = tf.expand_dims(y_true[..., 4], axis=-1) # Calculate the loss. A scale can be associated with each loss, indicating how important # the loss is. The bigger the scale, more important the loss is. loss_xy = tf.reduce_sum(tf.square(true_box_xy - pred_box_xy) * true_mask) * 1.0 loss_wh = tf.reduce_sum(tf.square(true_box_wh - pred_box_wh) * true_mask) * 1.0 loss_conf = tf.reduce_sum(tf.square(true_box_conf - pred_box_conf)) * 1.2 loss = loss_xy + loss_wh + loss_conf return loss
def read_tensor_from_image_file(file_name='test.jpg', input_height=128, input_width=128, input_mean=0, input_std=255): input_name = "file_reader" output_name = "normalized" file_reader = tf.read_file(file_name, input_name) image_reader = tf.image.decode_jpeg(file_reader, channels = 3, name='jpeg_reader') float_caster = tf.cast(image_reader, tf.float32) dims_expander = tf.expand_dims(float_caster, 0); resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width]) normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std]) sess = tf.Session() result = sess.run(normalized) return result
def masked_softmax(tensor, mask, expand=2, axis=1): """Masked soft-max using Lambda and merge-multiplication. Args: tensor: tensor containing scores mask: mask for tensor where 1 - means values at this position and 0 - means void, padded, etc.. expand: axis along which to repeat mask axis: axis along which to compute soft-max Returns: masked soft-max values """ mask = tf.expand_dims(mask, axis=expand) exponentiate = Lambda(lambda x: K.exp(x - K.max(x, axis=axis, keepdims=True)))(tensor) masked = tf.multiply(exponentiate, mask) div = tf.expand_dims(tf.reduce_sum(masked, axis=axis), axis=axis) predicted = tf.divide(masked, div) return predicted
def _meshgrid(self, height, width): with tf.variable_scope('_meshgrid'): # This should be equivalent to: # x_t, y_t = np.meshgrid(np.linspace(-1, 1, width), # np.linspace(-1, 1, height)) # ones = np.ones(np.prod(x_t.shape)) # grid = np.vstack([x_t.flatten(), y_t.flatten(), ones]) x_t = tf.matmul(tf.ones(shape=tf.pack([height, 1])), tf.transpose(tf.expand_dims(tf.linspace(-1.0, 1.0, width), 1), [1, 0])) y_t = tf.matmul(tf.expand_dims(tf.linspace(-1.0, 1.0, height), 1), tf.ones(shape=tf.pack([1, width]))) x_t_flat = tf.reshape(x_t, (1, -1)) y_t_flat = tf.reshape(y_t, (1, -1)) ones = tf.ones_like(x_t_flat) grid = tf.concat(0, [x_t_flat, y_t_flat, ones]) return grid
def kSparse(self, x, topk): print 'run regular k-sparse' dim = int(x.get_shape()[1]) if topk > dim: warnings.warn('Warning: topk should not be larger than dim: %s, found: %s, using %s' % (dim, topk, dim)) topk = dim k = dim - topk values, indices = tf.nn.top_k(-x, k) # indices will be [[0, 1], [2, 1]], values will be [[6., 2.], [5., 4.]] # We need to create full indices like [[0, 0], [0, 1], [1, 2], [1, 1]] my_range = tf.expand_dims(tf.range(0, tf.shape(indices)[0]), 1) # will be [[0], [1]] my_range_repeated = tf.tile(my_range, [1, k]) # will be [[0, 0], [1, 1]] full_indices = tf.stack([my_range_repeated, indices], axis=2) # change shapes to [N, k, 1] and [N, k, 1], to concatenate into [N, k, 2] full_indices = tf.reshape(full_indices, [-1, 2]) to_reset = tf.sparse_to_dense(full_indices, tf.shape(x), tf.reshape(values, [-1]), default_value=0., validate_indices=False) res = tf.add(x, to_reset) return res
def minibatch_discrimination(input_layer, num_kernels, dim_per_kernel=5, name='minibatch_discrim'): # batch_size = input_layer.shape[0] # num_features = input_layer.shape[1] batch_size = input_layer.get_shape().as_list()[0] num_features = input_layer.get_shape().as_list()[1] W = tf.get_variable('W', [num_features, num_kernels * dim_per_kernel], initializer=tf.contrib.layers.xavier_initializer()) b = tf.get_variable('b', [num_kernels], initializer=tf.constant_initializer(0.0)) activation = tf.matmul(input_layer, W) activation = tf.reshape(activation, [batch_size, num_kernels, dim_per_kernel]) tmp1 = tf.expand_dims(activation, 3) tmp2 = tf.transpose(activation, perm=[1, 2, 0]) tmp2 = tf.expand_dims(tmp2, 0) abs_diff = tf.reduce_sum(tf.abs(tmp1 - tmp2), reduction_indices=[2]) f = tf.reduce_sum(tf.exp(-abs_diff), reduction_indices=[2]) f = f + b return f
def yolo_loss(labels, predictions, mask): masked_labels = tf.boolean_mask(labels, mask) masked_predictions = tf.boolean_mask(predictions, mask) # ious = tensor_iou(masked_predictions[..., 1:5], masked_labels[..., 1:5]) # ious = tf.expand_dims(ious, axis=-1) xy_loss = tf.reduce_sum((masked_labels[..., :2] - masked_predictions[..., 1:3]) ** 2) wh_loss = tf.reduce_sum((tf.sqrt(masked_predictions[..., 3:5]) - tf.sqrt(masked_labels[..., 2:4])) ** 2) # conf_loss = tf.reduce_sum((masked_predictions[..., 0] - ious) ** 2) conf_loss = tf.reduce_sum((1 - masked_predictions[..., 0]) ** 2) no_obj_loss = tf.reduce_sum((tf.boolean_mask(predictions, ~mask)[..., 0] ** 2)) class_loss = tf.reduce_sum((masked_predictions[..., 5:] - masked_labels[..., 4:]) ** 2) loss = 5 * (xy_loss + wh_loss) + conf_loss + no_obj_loss + class_loss return loss
def random_access_problem(which=1): import raputil as ru if which == 1: opts = ru.Problem.scenario1() else: opts = ru.Problem.scenario2() p = ru.Problem(**opts) x1 = p.genX(1) y1 = p.fwd(x1) A = p.S M,N = A.shape nbatches = int(math.ceil(1000 /x1.shape[1])) prob = NumpyGenerator(p=p,nbatches=nbatches,A=A,opts=opts,iid=(which==1)) if which==2: prob.maskX_ = tf.expand_dims( tf.constant( (np.arange(N) % (N//2) < opts['Nu']).astype(np.float32) ) , 1) _,prob.noise_var = p.add_noise(y1) unused = p.genYX(nbatches) # for legacy reasons -- want to compare against a previous run (prob.yval, prob.xval) = p.genYX(nbatches) (prob.yinit, prob.xinit) = p.genYX(nbatches) import multiprocessing as mp prob.nsubprocs = mp.cpu_count() return prob
def pwlin_grid(r_,rvar_,theta_,dtheta = .75): """piecewise linear with noise-adaptive grid spacing. returns xhat,dxdr where q = r/dtheta/sqrt(rvar) xhat = r * interp(q,theta) all but the last dimensions of theta must broadcast to r_ e.g. r.shape = (500,1000) is compatible with theta.shape=(500,1,7) """ ntheta = int(theta_.get_shape()[-1]) scale_ = dtheta / tf.sqrt(rvar_) ars_ = tf.clip_by_value( tf.expand_dims( tf.abs(r_)*scale_,-1),0.0, ntheta-1.0 ) centers_ = tf.constant( np.arange(ntheta),dtype=tf.float32 ) outer_distance_ = tf.maximum(0., 1.0-tf.abs(ars_ - centers_) ) # new dimension for distance to closest bin centers (or center) gain_ = tf.reduce_sum( theta_ * outer_distance_,axis=-1) # apply the gain (learnable) xhat_ = gain_ * r_ dxdr_ = tf.gradients(xhat_,r_)[0] return (xhat_,dxdr_)
def interp1d_(xin_,xp,yp_): """ Interpolate a uniformly sampled piecewise linear function. Mapping elements from xin_ to the result. Input values will be clipped to range of xp. xin_ : input tensor (real) xp : x grid (constant -- must be a 1d numpy array, uniformly spaced) yp_ : tensor of the result values at the gridpoints xp """ import tensorflow as tf x_ = tf.clip_by_value(xin_,xp.min(),xp.max()) dx = xp[1]-xp[0] assert len(xp.shape)==1,'only 1d interpolation' assert xp.shape[0]==int(yp_.get_shape()[0]) assert abs(np.diff(xp)/dx - 1.0).max() < 1e-6,'must be uniformly sampled' newshape = [ ] x1_ = tf.expand_dims(x_,-1) dt = yp_.dtype wt_ = tf.maximum(tf.constant(0.,dtype=dt), 1-abs(x1_ - tf.constant(xp,dtype=dt))/dx ) y_ = tf.reduce_sum(wt_ * yp_,axis=-1) return y_
def calculate_cosine_similarity_matrix(v1, v2): """ Calculate the cosine similarity matrix between two sentences. Parameters ---------- v1: Tensor Tensor of shape (batch_size, num_sentence_words, context_rnn_hidden_size), representing the output of running a sentence through a BiLSTM. v2: Tensor Tensor of shape (batch_size, num_sentence_words, context_rnn_hidden_size), representing the output of running another sentences through a BiLSTM. """ # Shape: (batch_size, 1, num_sentence_words, rnn_hidden_size) expanded_v1 = tf.expand_dims(v1, 1) # Shape: (batch_size, num_sentence_words, 1, rnn_hidden_size) expanded_v2 = tf.expand_dims(v2, 2) # Shape: (batch_size, num_sentence_words, num_sentence_words) cosine_relevancy_matrix = cosine_distance(expanded_v1, expanded_v2) return cosine_relevancy_matrix
def mask_similarity_matrix(similarity_matrix, mask_a, mask_b): """ Given the mask of the two sentences, apply the mask to the similarity matrix. Parameters ---------- similarity_matrix: Tensor Tensor of shape (batch_size, num_sentence_words, num_sentence_words). mask_a: Tensor Tensor of shape (batch_size, num_sentence_words). This mask should correspond to the first vector (v1) used to calculate the similarity matrix. mask_b: Tensor Tensor of shape (batch_size, num_sentence_words). This mask should correspond to the second vector (v2) used to calculate the similarity matrix. """ similarity_matrix = tf.multiply(similarity_matrix, tf.expand_dims(tf.cast(mask_a, "float"), 1)) similarity_matrix = tf.multiply(similarity_matrix, tf.expand_dims(tf.cast(mask_b, "float"), 2)) return similarity_matrix
def multi_perspective_expand_for_1D(in_tensor, weights): """ Given a 1D input tensor and weights of the appropriate shape, weight the input tensor by the weights by multiplying them together. Parameters ---------- in_tensor: Tensor of shape (x,) to be weighted. weights: Tensor of shape (y, x) to multiply the input tensor by. In this case, y is the number of perspectives. Returns ------- weighted_input: Tensor of shape (y, x), representing the weighted input across multiple perspectives. """ # Shape: (1, rnn_hidden_dim) in_tensor_expanded = tf.expand_dims(in_tensor, axis=0) # Shape: (multiperspective_dims, rnn_hidden_dim) return tf.multiply(in_tensor_expanded, weights)
def dueling_model(img_in, num_actions, scope, reuse=False): """As described in https://arxiv.org/abs/1511.06581""" with tf.variable_scope(scope, reuse=reuse): out = img_in with tf.variable_scope("convnet"): # original architecture out = layers.convolution2d(out, num_outputs=32, kernel_size=8, stride=4, activation_fn=tf.nn.relu) out = layers.convolution2d(out, num_outputs=64, kernel_size=4, stride=2, activation_fn=tf.nn.relu) out = layers.convolution2d(out, num_outputs=64, kernel_size=3, stride=1, activation_fn=tf.nn.relu) out = layers.flatten(out) with tf.variable_scope("state_value"): state_hidden = layers.fully_connected(out, num_outputs=512, activation_fn=tf.nn.relu) state_score = layers.fully_connected(state_hidden, num_outputs=1, activation_fn=None) with tf.variable_scope("action_value"): actions_hidden = layers.fully_connected(out, num_outputs=512, activation_fn=tf.nn.relu) action_scores = layers.fully_connected(actions_hidden, num_outputs=num_actions, activation_fn=None) action_scores_mean = tf.reduce_mean(action_scores, 1) action_scores = action_scores - tf.expand_dims(action_scores_mean, 1) return state_score + action_scores
def preprocess_for_eval(image, height, width, central_fraction=0.875, scope=None): """Prepare one image for evaluation. If height and width are specified it would output an image with that size by applying resize_bilinear. If central_fraction is specified it would cropt the central fraction of the input image. Args: image: 3-D Tensor of image. If dtype is tf.float32 then the range should be [0, 1], otherwise it would converted to tf.float32 assuming that the range is [0, MAX], where MAX is largest positive representable number for int(8/16/32) data type (see `tf.image.convert_image_dtype` for details) height: integer width: integer central_fraction: Optional Float, fraction of the image to crop. scope: Optional scope for name_scope. Returns: 3-D float Tensor of prepared image. """ with tf.name_scope(scope, 'eval_image', [image, height, width]): if image.dtype != tf.float32: image = tf.image.convert_image_dtype(image, dtype=tf.float32) # Crop the central region of the image with an area containing 87.5% of # the original image. if central_fraction: image = tf.image.central_crop(image, central_fraction=central_fraction) if height and width: # Resize the image to the specified height and width. image = tf.expand_dims(image, 0) image = tf.image.resize_bilinear(image, [height, width], align_corners=False) image = tf.squeeze(image, [0]) image = tf.subtract(image, 0.5) image = tf.multiply(image, 2.0) return image
def load_batch(dataset, batch_size, height=image_size, width=image_size, is_training=True): ''' Loads a batch for training. INPUTS: - dataset(Dataset): a Dataset class object that is created from the get_split function - batch_size(int): determines how big of a batch to train - height(int): the height of the image to resize to during preprocessing - width(int): the width of the image to resize to during preprocessing - is_training(bool): to determine whether to perform a training or evaluation preprocessing OUTPUTS: - images(Tensor): a Tensor of the shape (batch_size, height, width, channels) that contain one batch of images - labels(Tensor): the batch's labels with the shape (batch_size,) (requires one_hot_encoding). ''' #First create the data_provider object data_provider = slim.dataset_data_provider.DatasetDataProvider( dataset, common_queue_capacity = 24 + 3 * batch_size, common_queue_min = 24) #Obtain the raw image using the get method raw_image, label = data_provider.get(['image', 'label']) #Perform the correct preprocessing for this image depending if it is training or evaluating image = inception_preprocessing.preprocess_image(raw_image, height, width, is_training) #As for the raw images, we just do a simple reshape to batch it up raw_image = tf.expand_dims(raw_image, 0) raw_image = tf.image.resize_nearest_neighbor(raw_image, [height, width]) raw_image = tf.squeeze(raw_image) #Batch up the image by enqueing the tensors internally in a FIFO queue and dequeueing many elements with tf.train.batch. images, raw_images, labels = tf.train.batch( [image, raw_image, label], batch_size = batch_size, num_threads = 4, capacity = 4 * batch_size, allow_smaller_final_batch = True) return images, raw_images, labels
def generate_on_topic(self, sess, topic_id, start_word_id, temperature=1.0, max_length=30, stop_word_id=None): if topic_id != -1: topic_emb = sess.run(tf.expand_dims(tf.nn.embedding_lookup(self.topic_output_embedding, topic_id), 0)) else: topic_emb = None return self.generate(sess, topic_emb, start_word_id, temperature, max_length, stop_word_id) #generate a sequence of words, given a document
def finalize_predictions(self, preds): # add a dimension of 1 between the batch size and the sequence length to emulate a beam width of 1 return tf.expand_dims(preds.sequence, axis=1)
def _tile_batch(self, t): if t.shape.ndims is None or t.shape.ndims < 1: raise ValueError("t must have statically known rank") tiling = [1] * (t.shape.ndims + 1) tiling[1] = self._beam_width tiled = tf.tile(tf.expand_dims(t, 1), tiling) return tiled
def _make_beam_mask(self, num_available_beams): mask = tf.sequence_mask(num_available_beams, self._beam_width) return tf.tile(tf.expand_dims(mask, axis=2), multiples=[1, 1, self._output_size])
def _tensor_gather_helper(gather_indices, gather_from, batch_size, range_size, gather_shape): """Helper for gathering the right indices from the tensor. This works by reshaping gather_from to gather_shape (e.g. [-1]) and then gathering from that according to the gather_indices, which are offset by the right amounts in order to preserve the batch order. Args: gather_indices: The tensor indices that we use to gather. gather_from: The tensor that we are gathering from. batch_size: The input batch size. range_size: The number of values in each range. Likely equal to beam_width. gather_shape: What we should reshape gather_from to in order to preserve the correct values. An example is when gather_from is the attention from an AttentionWrapperState with shape [batch_size, beam_width, attention_size]. There, we want to preserve the attention_size elements, so gather_shape is [batch_size * beam_width, -1]. Then, upon reshape, we still have the attention_size as desired. Returns: output: Gathered tensor of shape tf.shape(gather_from)[:1+len(gather_shape)] """ range_ = tf.expand_dims(tf.range(batch_size) * range_size, 1) gather_indices = tf.reshape(gather_indices + range_, [-1]) output = tf.gather(tf.reshape(gather_from, gather_shape), gather_indices) final_shape = tf.shape(gather_from)[:1 + len(gather_shape)] final_static_shape = (tf.TensorShape([None]).concatenate(gather_from.shape[1:1 + len(gather_shape)])) output = tf.reshape(output, final_shape) output.set_shape(final_static_shape) return output
def decov_loss(xs): """Decov loss as described in https://arxiv.org/pdf/1511.06068.pdf 'Reducing Overfitting In Deep Networks by Decorrelating Representation' """ x = tf.reshape(xs, [int(xs.get_shape()[0]), -1]) m = tf.reduce_mean(x, 0, True) z = tf.expand_dims(x-m, 2) corr = tf.reduce_mean(tf.matmul(z, tf.transpose(z, perm=[0,2,1])), 0) corr_frob_sqr = tf.reduce_sum(tf.square(corr)) corr_diag_sqr = tf.reduce_sum(tf.square(tf.diag_part(corr))) loss = 0.5*(corr_frob_sqr - corr_diag_sqr) return loss
def average_gradients(tower_gradients): r''' A routine for computing each variable's average of the gradients obtained from the GPUs. Note also that this code acts as a syncronization point as it requires all GPUs to be finished with their mini-batch before it can run to completion. ''' # List of average gradients to return to the caller average_grads = [] # Loop over gradient/variable pairs from all towers for grad_and_vars in zip(*tower_gradients): # Introduce grads to store the gradients for the current variable grads = [] # Loop over the gradients for the current variable for g, _ in grad_and_vars: # Add 0 dimension to the gradients to represent the tower. expanded_g = tf.expand_dims(g, 0) # Append on a 'tower' dimension which we will average over below. grads.append(expanded_g) # Average over the 'tower' dimension grad = tf.concat(grads, 0) grad = tf.reduce_mean(grad, 0) # Create a gradient/variable tuple for the current variable with its average gradient grad_and_var = (grad, grad_and_vars[0][1]) # Add the current tuple to average_grads average_grads.append(grad_and_var) # Return result to caller return average_grads # Logging # =======
def ctc_label_dense_to_sparse(labels, label_lengths, batch_size): # The second dimension of labels must be equal to the longest label length in the batch correct_shape_assert = tf.assert_equal(tf.shape(labels)[1], tf.reduce_max(label_lengths)) with tf.control_dependencies([correct_shape_assert]): labels = tf.identity(labels) label_shape = tf.shape(labels) num_batches_tns = tf.stack([label_shape[0]]) max_num_labels_tns = tf.stack([label_shape[1]]) def range_less_than(previous_state, current_input): return tf.expand_dims(tf.range(label_shape[1]), 0) < current_input init = tf.cast(tf.fill(max_num_labels_tns, 0), tf.bool) init = tf.expand_dims(init, 0) dense_mask = tf.scan(range_less_than, label_lengths, initializer=init, parallel_iterations=1) dense_mask = dense_mask[:, 0, :] label_array = tf.reshape(tf.tile(tf.range(0, label_shape[1]), num_batches_tns), label_shape) label_ind = tf.boolean_mask(label_array, dense_mask) batch_array = tf.transpose(tf.reshape(tf.tile(tf.range(0, label_shape[0]), max_num_labels_tns), tf.reverse(label_shape, [0]))) batch_ind = tf.boolean_mask(batch_array, dense_mask) indices = tf.transpose(tf.reshape(tf.concat([batch_ind, label_ind], 0), [2, -1])) shape = [batch_size, tf.reduce_max(label_lengths)] vals_sparse = gather_nd(labels, indices, shape) return tf.SparseTensor(tf.to_int64(indices), vals_sparse, tf.to_int64(label_shape)) # Validate and normalize transcriptions. Returns a cleaned version of the label # or None if it's invalid.
