我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.unpack()。
def define_model(x, keep_prob, number_of_classes, number_of_filters, number_of_fc_features): splitted = tf.unpack(x, axis=4) branches = [] with tf.variable_scope('branches') as scope: for index, tensor_slice in enumerate(splitted): branches.append(single_branch(splitted[index], number_of_filters, number_of_fc_features)) if (index == 0): scope.reuse_variables() concatenated = tf.pack(branches, axis=2) ti_pooled = tf.reduce_max(concatenated, reduction_indices=[2]) drop = tf.nn.dropout(ti_pooled, keep_prob) with tf.variable_scope('fc2'): logits = fc(drop, [number_of_fc_features, number_of_classes], [number_of_classes]) return logits
def sequence_loss(self, y_pred, y_true): ''' Loss function for the seq2seq RNN. Reshape predicted and true (label) tensors, generate dummy weights, then use seq2seq.sequence_loss to actually compute the loss function. ''' if self.verbose > 2: print ("my_sequence_loss y_pred=%s, y_true=%s" % (y_pred, y_true)) logits = tf.unpack(y_pred, axis=1) # list of [-1, num_decoder_synbols] elements targets = tf.unpack(y_true, axis=1) # y_true has shape [-1, self.out_seq_len]; unpack to list of self.out_seq_len [-1] elements if self.verbose > 2: print ("my_sequence_loss logits=%s" % (logits,)) print ("my_sequence_loss targets=%s" % (targets,)) weights = [tf.ones_like(yp, dtype=tf.float32) for yp in targets] if self.verbose > 4: print ("my_sequence_loss weights=%s" % (weights,)) sl = seq2seq.sequence_loss(logits, targets, weights) if self.verbose > 2: print ("my_sequence_loss return = %s" % sl) return sl
def distort_image(image): """Perform random distortions to the given 4D image and return result""" # Switch to 3D as that's what these operations require slices = tf.unpack(image) output = [] # Perform pixel-wise distortions for image in slices: image = tf.image.random_flip_left_right(image) image = tf.image.random_saturation(image, .2, 2.) image += tf.truncated_normal(image.get_shape(), stddev=.05) image = tf.image.random_contrast(image, .85, 1.15) image = tf.image.random_brightness(image, .3) output.append(image) # Go back to 4D image = tf.pack(output) return image
def Unroll(axis, num=None): """ defines an _OperationalLayer that unpacks a tensor along a given axis Parameters: ---------- axis: int num: int the numeber if tensors to unpack form the gievn tensor Returns: _OperationalLayer """ def unroll_op(obj, X): return tf.unpack(X, obj.params[0], 1) return _OperationalLayer(unroll_op, [num, axis])
def _tile_along_beam(cls, beam_size, state): if nest.is_sequence(state): return nest_map( lambda val: cls._tile_along_beam(beam_size, val), state ) if not isinstance(state, tf.Tensor): raise ValueError("State should be a sequence or tensor") tensor = state tensor_shape = tensor.get_shape().with_rank_at_least(1) try: new_first_dim = tensor_shape[0] * beam_size except: new_first_dim = None dynamic_tensor_shape = tf.unpack(tf.shape(tensor)) res = tf.expand_dims(tensor, 1) res = tf.tile(res, [1, beam_size] + [1] * (tensor_shape.ndims-1)) res = tf.reshape(res, [-1] + list(dynamic_tensor_shape[1:])) res.set_shape([new_first_dim] + list(tensor_shape[1:])) return res
def extract_patches(inputs, size, offsets): batch_size = inputs.get_shape()[0] padded = tf.pad(inputs, [[0,0],[2,2],[2,2],[0,0]]) unpacked = tf.unpack(tf.squeeze(padded)) extra_margins = tf.constant([1,1,2,2]) sliced_list = [] for i in xrange(batch_size.value): margins = tf.random_shuffle(extra_margins) margins = margins[:2] start_pts = tf.sub(offsets[i,:],margins) sliced = tf.slice(unpacked[i],start_pts,size) sliced_list.append(sliced) patches = tf.pack(sliced_list) patches = tf.expand_dims(patches,3) return patches
def reverse_seq(input_seq, lengths): """Reverse a list of Tensors up to specified lengths. Args: input_seq: Sequence of seq_len tensors of dimension (batch_size, depth) lengths: A tensor of dimension batch_size, containing lengths for each sequence in the batch. If "None" is specified, simply reverses the list. Returns: time-reversed sequence """ for input_ in input_seq: input_.set_shape(input_.get_shape().with_rank(2)) # Join into (time, batch_size, depth) s_joined = tf.pack(input_seq) # Reverse along dimension 0 s_reversed = tf.reverse_sequence(s_joined, lengths, 0, 1) # Split again into list result = tf.unpack(s_reversed) return result
def compute_states(self,emb): def unpack_sequence(tensor): return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2])) with tf.variable_scope("Composition",initializer= tf.contrib.layers.xavier_initializer(),regularizer= tf.contrib.layers.l2_regularizer(self.reg)): cell = rnn_cell.LSTMCell(self.hidden_dim) #tf.cond(tf.less(self.dropout #if tf.less(self.dropout, tf.constant(1.0)): cell = rnn_cell.DropoutWrapper(cell, output_keep_prob=self.dropout,input_keep_prob=self.dropout) #output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32) outputs,_=rnn.rnn(cell,unpack_sequence(emb),sequence_length=self.lngths,dtype=tf.float32) #output = pack_sequence(outputs) sum_out=tf.reduce_sum(tf.pack(outputs),[0]) sent_rep = tf.div(sum_out,tf.expand_dims(tf.to_float(self.lngths),1)) final_state=sent_rep return final_state
def _build_global_context( net, is_training=False, bayesian=False, dropout_keep_prob=0.8): with tf.variable_scope('GlobalContext'): # Reduce feature dimension before LSTM to reduce param count net = slim.conv2d(net, 1024, 1, padding='VALID', scope='conv_reduce_1x1') #net = slim.dropout(net, dropout_keep_prob, is_training=bayesian or is_training, scope='Dropout') rows = tf.unpack(net, axis=1) net = tf.pack( [lstm.bidir_lstm(r, 512, scope='row%d' % i) for i, r in enumerate(rows)], axis=1) print('Horizontal LSTM', net.get_shape()) cols = tf.unpack(net, axis=2) net = tf.pack( [lstm.bidir_lstm(r, 512, scope='col%d' % i) for i, r in enumerate(cols)], axis=2) print('Vertical LSTM', net.get_shape()) return net
def loss_op(self, outputs, labels, weights, len_vocab): """ Calculate the loss from the predicted outputs and the labels Args: outputs : A list of tensors of size [batch_size * num_symbols] labels : A list of tensors of size [sequence_length * batch_size] Returns: loss: loss of type float """ _labels = tf.unpack(labels) all_ones = [tf.ones(shape=tf.shape(_labels[0])) for _ in range(len(_labels))] weights = tf.to_float(weights) _weights = tf.unpack(weights) #print(_weights[0].get_shape()) loss_per_batch = sequence_loss(outputs, _labels, _weights) self.calculated_loss = loss_per_batch return loss_per_batch
def loss_op(self, outputs, labels, weights): """ Calculate the loss from the predicted outputs and the labels Args: outputs : A list of tensors of size [batch_size * num_symbols] labels : A list of tensors of size [sequence_length * batch_size] Returns: loss: loss of type float """ _labels = tf.unpack(labels) all_ones = [tf.ones(shape=tf.shape(_labels[0])) for _ in range(len(_labels))] weights = tf.to_float(weights) _weights = tf.unpack(weights) #print(_weights[0].get_shape()) loss_per_batch = sequence_loss(outputs, _labels, _weights) self.calculated_loss = loss_per_batch return loss_per_batch
def lstm_cell1(i, o, state): """Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf Note that in this formulation, we omit the various connections between the previous state and the gates.""" m_input2 = tf.pack([i for _ in range(m_rows)]) m_saved_output2 = tf.pack([o for _ in range(m_rows)]) # m_input2 = tf.nn.dropout(m_input2, keep_prob) m_all = tf.batch_matmul(m_input2, m_input_w2) + tf.batch_matmul(m_saved_output2, m_middle_w2) + m_biases m_all = tf.unpack(m_all) input_gate = tf.sigmoid(m_all[m_input_index]) forget_gate = tf.sigmoid(m_all[m_forget_index]) update = m_all[m_update_index] state = forget_gate * state + input_gate * tf.tanh(update) output_gate = tf.sigmoid(m_all[m_output_index]) return output_gate * tf.tanh(state), state # Input data.
def decoderFn(num_samples=1, modality='rgb'): class decoder_func(slim.data_decoder.DataDecoder): @staticmethod def list_items(): return ['image', 'label'] @staticmethod def decode(data, items): with tf.name_scope('decode_video'): if modality == 'rgb': data.set_shape((num_samples,)) elif modality.startswith('flow'): optical_flow_frames = int(modality[4:]) data.set_shape((num_samples, 2 * optical_flow_frames)) elif modality.startswith('rgb+flow'): optical_flow_frames = int(modality[-2:]) data.set_shape((num_samples, 1 + 2 * optical_flow_frames)) else: logging.error('Unknown modality %s\n' % modality) image_buffer = [_decode_from_string(el, modality) for el in tf.unpack(data)] # image_buffer = tf.pack(image_buffer) return image_buffer return decoder_func
def unpack_sequence(tensor): """Split the single tensor of a sequence into a list of frames.""" return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2]))
def flatten(output): sh = tf.unpack(tf.shape(output)) batch, output_shape = sh[0], sh[1:] flat_shape = 1 for d in output_shape: flat_shape *= d return tf.reshape(output, tf.pack([batch, flat_shape]))
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10): correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1)) correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1]) # indices of correctly recognised images incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images everything_incorrect_first = tf.concat(0, [incorrectly_recognised_indices, correctly_recognised_indices]) # images reordered with indeces of unrecognised images first everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway # compute n=100 digits to display only Xs = tf.gather(X, everything_incorrect_first) Ys = tf.gather(Y, everything_incorrect_first) Ys_ = tf.gather(Y_, everything_incorrect_first) correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first) digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left()) correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right()) computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images #superimposed_digits = correct_tags+computed_tags superimposed_digits = tf.select(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images correct_bkg = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background recognised_bkg = tf.gather(tf.concat(0, [incorrect_bkg, correct_bkg]), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status I = tf.image.grayscale_to_rgb(Xs) I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True) Islices = [] # 100 images => 10x10 image block for imslice in range(lines): Islices.append(tf.concat(1, tf.unpack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3])))) I = tf.concat(0, Islices) return I # n = HISTOGRAM_BUCKETS (global) # Buckets the data into n buckets so that there are an equal number of data points in # each bucket. Returns n+1 bucket boundaries. Spreads the reaminder data.size % n more # or less evenly among the central buckets. # data: 1-D ndarray containing float data, MUST BE SORTED in ascending order # n: integer, the number of desired output buckets # return value: ndarray, 1-D vector of size n+1 containing the bucket boundaries # the first value is the min of the data, the last value is the max
def loss_graph(logits, batch_size, num_unroll_steps): with tf.variable_scope('Loss'): targets = tf.placeholder(tf.int64, [batch_size, num_unroll_steps], name='targets') # target_list = [tf.squeeze(x, [1]) for x in tf.split(1, num_unroll_steps, targets)] target_list=tf.unpack(targets, axis=1)#hjq loss= tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, target_list), name='loss') # reduce_mean to reduce_sum,hjq return adict( targets=targets, loss=loss )
def create_output_unit(self, params): self.Wo = tf.Variable(self.params[13]) self.bo = tf.Variable(self.params[14]) params.extend([self.Wo, self.bo]) def unit(hidden_memory_tuple): hidden_state, c_prev = tf.unpack(hidden_memory_tuple) # hidden_state : batch x hidden_dim logits = tf.matmul(hidden_state, self.Wo) + self.bo # output = tf.nn.softmax(logits) return logits return unit
def create_output_unit(self): self.Wo = tf.identity(self.lstm.Wo) self.bo = tf.identity(self.lstm.bo) def unit(hidden_memory_tuple): hidden_state, c_prev = tf.unpack(hidden_memory_tuple) # hidden_state : batch x hidden_dim logits = tf.matmul(hidden_state, self.Wo) + self.bo # output = tf.nn.softmax(logits) return logits return unit
def create_output_unit(self, params): self.Wo = tf.Variable(self.init_matrix([self.hidden_dim, self.num_emb])) self.bo = tf.Variable(self.init_matrix([self.num_emb])) params.extend([self.Wo, self.bo]) def unit(hidden_memory_tuple): hidden_state, c_prev = tf.unpack(hidden_memory_tuple) # hidden_state : batch x hidden_dim logits = tf.matmul(hidden_state, self.Wo) + self.bo # output = tf.nn.softmax(logits) return logits return unit
def RNN(inputs, lens, name, reuse): print ("Building network " + name) # Define weights inputs = tf.gather(one_hots, inputs) weights = tf.Variable(tf.random_normal([__n_hidden, n_output]), name=name+"_weights") biases = tf.Variable(tf.random_normal([n_output]), name=name+"_biases") # Define a lstm cell with tensorflow outputs, states = rnn.dynamic_rnn( __cell_kind(__n_hidden), inputs, sequence_length=lens, dtype=tf.float32, scope=name, time_major=False) # Prepare data shape to match `rnn` function requirements # Current data input shape: (__batch_size, __n_steps, n_input) # Required shape: '__n_steps' tensors list of shape (__batch_size, n_input) '''outputs, states = rnn.rnn( __cell_kind(__n_hidden), tf.unpack(tf.transpose(inputs, [1, 0, 2])), sequence_length=lens, dtype=tf.float32, scope=name) outputs = tf.transpose(tf.pack(outputs), [1, 0, 2])''' print ("Done building network " + name) # Asserts are actually documentation: they can't be out of date assert outputs.get_shape() == (__batch_size, __n_steps, __n_hidden) # Linear activation, using rnn output for each char # Reshaping here for a `batch` matrix multiply # It's faster than `batch_matmul` probably because it can guarantee a # static shape outputs = tf.reshape(outputs, [__batch_size * __n_steps, __n_hidden]) finals = tf.matmul(outputs, weights) return tf.reshape(finals, [__batch_size, __n_steps, n_output]) + biases # tf Graph input
def __preprocess_particle(self, batch_data): # scale the image to the model input size #batch_data = tf.image.resize_images(batch_data, self.num_col, self.num_row) # get the scale tensor shape batch_data_shape = batch_data.get_shape().as_list() # uppack the tensor into sub-tensor batch_data_list = tf.unpack(batch_data) for i in xrange(batch_data_shape[0]): # Pass image tensor object to a PIL image image = Image.fromarray(batch_data_list[i].eval()) # Use PIL or other library of the sort to rotate random_degree = random.randint(0, 359) rotated = Image.Image.rotate(image, random_degree) # Convert rotated image back to tensor rotated_tensor = tf.convert_to_tensor(np.array(rotated)) #slice_image = tf.slice(batch_data, [i, 0, 0, 0], [1, -1, -1, -1]) #slice_image_reshape = tf.reshape(slice_image, [batch_data_shape[1], batch_data_shape[2], batch_data_shape[3]]) #distorted_image = tf.image.random_flip_up_down(batch_data_list[i], seed = 1234) #distorted_image = tf.image.random_flip_left_right(distorted_image, seed = 1234) #distorted_image = tf.image.random_brightness(distorted_image, max_delta=63) #distorted_image = tf.image.random_contrast(distorted_image, lower=0.2, upper=1.8) # Subtract off the mean and divide by the variance of the pixels. distorted_image = tf.image.per_image_whitening(rotated_tensor) batch_data_list[i] = distorted_image # pack the list of tensor into one tensor batch_data = tf.pack(batch_data_list) return batch_data
def __init__(self,params): config = tf.ConfigProto(allow_soft_placement=True) self.sess = tf.Session(config = config) K.set_session(self.sess) # Pull out all of the parameters self.batch_size = params['batch_size'] self.seq_len = params['seq_len'] self.vocab_size = params['vocab_size'] self.embed_size = params['embed_size'] self.hidden_dim = params['hidden_dim'] self.num_layers = params['num_layers'] with tf.device('/gpu:0'): # Set up the input placeholder self.input_seq = tf.placeholder(tf.float32, shape=[None, self.seq_len]) # Build the RNN self.rnn = Embedding(self.vocab_size + 1, self.embed_size, input_length=self.seq_len)(self.input_seq) with tf.device('/gpu:1'): for l in range(self.num_layers): self.rnn = LSTM(output_dim=self.hidden_dim, return_sequences=True, name='rnn_1')(self.rnn) rnn_output = tf.unpack(self.rnn, axis=1) self.w_proj = tf.Variable(tf.zeros([self.vocab_size, self.hidden_dim])) self.b_proj = tf.Variable(tf.zeros([self.vocab_size])) self.output_seq = tf.placeholder(tf.int64, shape=([None, self.seq_len])) losses = [] outputs = [] for t in range(self.seq_len): rnn_t = rnn_output[t] y_t = tf.reshape(self.output_seq[:, t],[-1,1]) step_loss = tf.nn.sampled_softmax_loss(weights=self.w_proj, biases=self.b_proj, inputs=rnn_t, labels=y_t, num_sampled=512, num_classes=self.vocab_size) losses.append(step_loss) outputs.append(tf.matmul(rnn_t, tf.transpose(self.w_proj)) + self.b_proj) self.step_losses = losses self.output = outputs self.loss = tf.reduce_mean(self.step_losses) self.softmax = tf.nn.softmax(self.output)
def __init__(self, classifier, input_dim, max_length): ''' NnetDecoder constructor, creates the decoding graph Args: classifier: the classifier that will be used for decoding input_dim: the input dimension to the nnnetgraph ''' self.graph = tf.Graph() self.max_length = max_length with self.graph.as_default(): #create the inputs placeholder self.inputs = tf.placeholder( tf.float32, shape=[max_length, input_dim], name='inputs') #create the sequence length placeholder self.seq_length = tf.placeholder( tf.int32, shape=[1], name='seq_length') split_inputs = tf.unpack(tf.expand_dims(self.inputs, 1)) #create the decoding graph logits, _, self.saver, _ = classifier(split_inputs, self.seq_length, is_training=False, reuse=False, scope='Classifier') #convert logits to non sequence for the softmax computation logits = seq_convertors.seq2nonseq(logits, self.seq_length) #compute the outputs self.outputs = tf.nn.softmax(logits) #specify that the graph can no longer be modified after this point self.graph.finalize()
def seq2nonseq(tensorlist, seq_length, name=None): ''' Convert sequential data to non sequential data Args: tensorlist: the sequential data, wich is a list containing an N x F tensor for each time step where N is the batch size and F is the input dimension seq_length: a vector containing the sequence lengths name: [optional] the name of the operation Returns: non sequential data, which is a TxF tensor where T is the sum of all sequence lengths ''' with tf.name_scope(name or 'seq2nonseq'): #convert the list for each time step to a list for each sequence sequences = tf.unpack(tf.pack(tensorlist), axis=1) #remove the padding from sequences sequences = [tf.gather(sequences[s], tf.range(seq_length[s])) for s in range(len(sequences))] #concatenate the sequences tensor = tf.concat(0, sequences) return tensor
def nonseq2seq(tensor, seq_length, length, name=None): ''' Convert non sequential data to sequential data Args: tensor: non sequential data, which is a TxF tensor where T is the sum of all sequence lengths seq_length: a vector containing the sequence lengths length: the constant length of the output sequences name: [optional] the name of the operation Returns: sequential data, wich is a list containing an N x F tensor for each time step where N is the batch size and F is the input dimension ''' with tf.name_scope(name or'nonseq2seq'): #get the cumulated sequence lengths to specify the positions in tensor cum_seq_length = tf.concat(0, [tf.constant([0]), tf.cumsum(seq_length)]) #get the indices in the tensor for each sequence indices = [tf.range(cum_seq_length[l], cum_seq_length[l+1]) for l in range(int(seq_length.get_shape()[0]))] #create the non-padded sequences sequences = [tf.gather(tensor, i) for i in indices] #pad the sequences with zeros sequences = [tf.pad(sequences[s], [[0, length-seq_length[s]], [0, 0]]) for s in range(len(sequences))] #specify that the sequences have been padded to the constant length for seq in sequences: seq.set_shape([length, int(tensor.get_shape()[1])]) #convert the list for eqch sequence to a list for eqch time step tensorlist = tf.unpack(tf.pack(sequences), axis=1) return tensorlist
def compute_states(self,emb): def unpack_sequence(tensor): return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2])) with tf.variable_scope("Composition",initializer= tf.contrib.layers.xavier_initializer(),regularizer= tf.contrib.layers.l2_regularizer(self.reg)): cell_fw = rnn_cell.LSTMCell(self.hidden_dim) cell_bw = rnn_cell.LSTMCell(self.hidden_dim) #tf.cond(tf.less(self.dropout #if tf.less(self.dropout, tf.constant(1.0)): cell_fw = rnn_cell.DropoutWrapper(cell_fw, output_keep_prob=self.dropout,input_keep_prob=self.dropout) cell_bw=rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=self.dropout,input_keep_prob=self.dropout) #output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32) outputs,_,_=rnn.bidirectional_rnn(cell_fw,cell_bw,unpack_sequence(emb),sequence_length=self.lngths,dtype=tf.float32) #output = pack_sequence(outputs) sum_out=tf.reduce_sum(tf.pack(outputs),[0]) sent_rep = tf.div(sum_out,tf.expand_dims(tf.to_float(self.lngths),1)) final_state=sent_rep return final_state
def process_example(self, tensors, mode='eval', thread_id=0): train = (mode == 'train') image, image_timestamp, camera_id = tensors[:3] #FIXME push single/multi image handling into image_process_sdc if we want to share random augmentations if self.num_input_images > 1: assert(len(image.get_shape()) > 0) print('Multi image', image.get_shape()) split_image = tf.unpack(image) split_processed = [] for i, x in enumerate(split_image): suffix = '%d' % i xp, _ = image_preprocess_sdc( x, camera_id, height=self.height, width=self.width, image_fmt=self.image_fmt, normalize=self.standardize_input, train=train, summary_suffix=suffix, thread_id=thread_id) split_processed.append(xp) processed_image = tf.pack(split_processed) #FIXME need to sort out flip across mult-images flip_coeff = tf.constant(1.0, dtype=tf.float32) else: print('Single image') processed_image, flip_coeff = image_preprocess_sdc( image, camera_id, height=self.height, width=self.width, image_fmt=self.image_fmt, normalize=self.standardize_input, train=train, thread_id=thread_id) if mode != 'pred': steering_angle, gps_coord = tensors[-2:] if steering_angle is not None: steering_angle = tf.mul(steering_angle, flip_coeff) if self.standardize_labels: steering_angle /= STEERING_STD elif self.mu_law_steering: print("Encode mu-law angles") steering_angle = mu_law_steering_enc(steering_angle) if gps_coord is not None and self.standardize_labels: gps_coord = (gps_coord - GPS_MEAN) / GPS_STD return processed_image, image_timestamp, steering_angle, gps_coord else: return processed_image, image_timestamp, tf.zeros((1,)), tf.zeros((2,))
def lstm( inputs, num_units, num_layers=1, initializer_fn=tf.truncated_normal, initializer_params=_default_initializer_params, dtype=tf.float32, scope=None ): print('input shape', inputs.get_shape()) shape = inputs.get_shape().as_list() batch_size = shape[0] inputs_unpacked = tf.unpack(inputs, axis=1) cell = tf.contrib.rnn.python.ops.lstm_ops.LSTMBlockCell(num_units=num_units) print('cell state size', cell.state_size) if num_layers > 1: cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers) initializer_params = initializer_params or {} initializer_params['dtype'] = dtype if isinstance(cell.state_size, tuple): initial_state = tuple(initializer_fn([batch_size, s]) for s in cell.state_size) else: initial_state = initializer_fn(shape=[batch_size, cell.state_size], **initializer_params) outputs, _, _ = tf.nn.rnn( cell, inputs_unpacked, initial_state=initial_state, dtype=dtype, scope=scope) outputs = tf.pack(outputs, axis=1) print('output shape', outputs.get_shape()) return outputs
def RNN(X, weights, biases): # hidden layer for input to cell, we have to start by reshaping, we have to reshape the 3 dimensional x into a 2 dimensional X # in the hidden layer Wx + b, the W is a 2d dimension and so is the x so everything must be reshaped into a 2d dimension # X -> (128 batch * 28 steps, 28 inputs) is now it should be reshaped, 128 batch * 28 steps is one dimension and 28 inputs is the other dimension. X = tf.reshape(X, [-1, n_inputs]) # after this, we can multiply by the weights metric, the X is now 2d and the weight['in'] is also 2d so they can be multiplied. We calculate the x_in which is the result in the input hidden layer # X_in -> (128 batch * 28 steps, 128 hidden) X_in = tf.matmul(X, weights['in']) + biases['in'] # once we've done that, we then reshape it back to 3 dimensions # X_in -> (128 batch, 28 steps, 128 hidden) - this 128 hidden is the hidden units in the cell (seen below) X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units]) # cell LSTM, 128 hidden is the hidden units in the cell and we input the X_in result into the cell - this is 3d and the cell can accept 3d, these next two steps are used to define the cells. LSTM has a forget gate and the forget_bias is the initial bias for the forget gate and it's set to 1 which is to say that the first few steps we want to open our forget gate, we don't want to forget anything in the beginning since the beginning is very important in the learning procedure. So open the gate and don't forget anything, later on we can figure out how much we want to forget or don't want to forget. lstm cell is divided into two states (c_state, h_state). c_state is what is happening in the main story (going back to our storyline example)or overall state, the h_state is the sub storyline/instance moment or latest state. state_is_tuple just means that we pass a tuple of states, 2 states, for LSTM. The basic RNN cell doesn't have 2 states, it doesn't have a c_state, just a h_state. Tensorflow recommends state_is_tuple = True over False, because the calculation is faster in the true state. lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units, forget_bias=1.0, state_is_tuple=True) # lstm cell is divided into two parts (c_state, h_state) _init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32) # lstm_cell.zero_state means we define the initial state as all zero. So the state takes 1 input as the batch_size so we just pass batch_size in. Once we define the initial state we can now use dynamic_rnn outputs, final_state = tf.nn.dynamic_rnn(lstm_cell, X_in, initial_state = _init_state, time_major = False) # dynamic_rnn is the main loop/structure in the RNN, you can use just rnn but this one is considered better. lstm_cell is just passed in from above. X_in is from above as well, just reshaped. initial_state is just the _init_state from above. time_major = False, if you look at the what we have, (128 batch, 28 steps, 128 hidden), the time is 28 because we have 28 time steps and that is in the 2nd dimension which is not a major dimension so we pass the false in here. If we have the time step in the 1st dimension then the time_major would be True. From this loop, we output all outputs from every step, in other words, this output is the collection of all outputs for every step. And the final_state is just one final state in the last time step # hidden layer for output as the final results, basically we want to calculate the outputs from the cell. We have 2 options for this. results = tf.matmul(final_state[1], weights['out']) + biases['out'] # option 1, here we only care about the last output in the last step because the rest of them are meaningless we only care once we have reached through all the rows in the image and what are conclusion is at that point. And the outputs at the last time step is also this state. Thus we use final_state[1] with the 1 representing the h of the final state (h_state). So use the h_state * the weights + biases give you the final result # # option 2 unpack to list [(batch, outputs)...] * steps, use the outputs itself and use the last output as the results once we've reach through all the rows. Remember for each of the rows we have one output and the collection of outputs is stored in outputs so we want to unpack and transpose the outputs in order to get out the last output from the outputs # outputs = tf.unpack(tf.transpose(outputs, [1,0,2])) # states is the last output # results = tf.matmul(outputs[-1], weights['out']) + biases['out'] # outputs[-1] is the last output in outputs. These two steps are bascially the same as the one step we had in option 1 but there are times where that may not be the case and it's important to know both. You can tell the options are the same when you run both and the first values are identical return results
def lstm_cell(i, o, state): m_input = tf.pack([i for _ in range(m_rows)]) m_saved_output = tf.pack([o for _ in range(m_rows)]) m_all = tf.batch_matmul(m_input, m_input_w) + tf.batch_matmul(m_saved_output, m_middle) + m_biases m_all = tf.unpack(m_all) input_gate = tf.sigmoid(m_all[m_input_index]) forget_gate = tf.sigmoid(m_all[m_forget_index]) update = m_all[m_update_index] state = forget_gate * state + input_gate * tf.tanh(update) output_gate = tf.sigmoid(m_all[m_output_index]) return output_gate * tf.tanh(state), state
def decode_flow(img_str): # IMPORTANT NOTE: I am now resizing the flow frames before running through # the preprocessing. I was not doing that earlier (in the master). This leads # to the 66 number to drop to 63 on HMDB. But it should be fixable by # re-training with this setup with tf.variable_scope('decode_flow_frame'): img = tf.concat(2, [tf.image.decode_jpeg(el, channels=1) for el in tf.unpack(img_str)]) # Always convert before resize, this is a bug in TF # https://github.com/tensorflow/tensorflow/issues/1763 img = tf.image.convert_image_dtype(img, dtype=tf.float32) img = tf.image.resize_images(img, [IM_HT, IM_WD]) return [img]
def get_preprocessing(name, is_training=False): """Returns preprocessing_fn(image, height, width, **kwargs). Args: name: The name of the preprocessing function. is_training: `True` if the model is being used for training and `False` otherwise. Returns: preprocessing_fn: A function that preprocessing a single image (pre-batch). It has the following signature: image = preprocessing_fn(image, output_height, output_width, ...). Raises: ValueError: If Preprocessing `name` is not recognized. """ preprocessing_fn_map = { 'vgg_ucf': vgg_ucf_preprocessing, } if name not in preprocessing_fn_map: raise ValueError('Preprocessing name [%s] was not recognized' % name) def preprocessing_fn(image, output_height, output_width, **kwargs): with tf.variable_scope('preprocess_image'): if len(image.get_shape()) == 3: return preprocessing_fn_map[name].preprocess_image( image, output_height, output_width, is_training=is_training, **kwargs) elif len(image.get_shape()) == 4: # preprocess all the images in one set in the same way by concat-ing # them in channels nImgs = image.get_shape().as_list()[0] final_img_concat = preprocessing_fn_map[name].preprocess_image( tf.concat(2, tf.unpack(image)), output_height, output_width, is_training=is_training, **kwargs) return tf.concat(0, tf.split(3, nImgs, final_img_concat)) else: print('Incorrect dims image!') return preprocessing_fn
def rnn(input_data, weights, biases): # hidden layer for input to cell ######################################## # transpose the inputs shape from # X?128 batch ,28 steps, 18 inputs? # ==> (128 batch * 28 steps, 28 inputs) input_data = tf.reshape(input_data, [-1, n_inputs]) # into hidden # data_in = (128 batch * 28 steps, 128 hidden) data_in = tf.matmul(input_data, weights['in']) + biases['in'] # data_in ==> (128 batch, 28 steps, 128 hidden_units) data_in = tf.reshape(data_in, [-1, n_steps, n_hidden_units]) # cell ########################################## # basic LSTM Cell. lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(n_hidden_units, forget_bias=1.0, state_is_tuple=True) # lstm cell is divided into two parts (c_state, h_state) _init_state = lstm_cell.zero_state(batch_size, dtype=tf.float32) # You have 2 options for following step. # 1: tf.nn.rnn(cell, inputs); # 2: tf.nn.dynamic_rnn(cell, inputs). # If use option 1, you have to modified the shape of data_in, go and check out this: # https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/recurrent_network.py # In here, we go for option 2. # dynamic_rnn receive Tensor (batch, steps, inputs) or (steps, batch, inputs) as data_in. # Make sure the time_major is changed accordingly. outputs, final_state = tf.nn.dynamic_rnn(lstm_cell, data_in, initial_state=_init_state, time_major=False) # hidden layer for output as the final results ############################################# # results = tf.matmul(final_state[1], weights_1['out']) + biases_1['out'] # # or # unpack to list [(batch, outputs)..] * steps outputs = tf.unpack(tf.transpose(outputs, [1, 0, 2])) # states is the last outputs results = tf.matmul(outputs[-1], weights['out']) + biases['out'] return results
def standard_nextstep_inference(cell, inputs, output_size, scope=None, return_states=False): """Gets the forward pass of a standard model for next step prediction. Args: cell (tf.nn.rnn_cell.RNNCell): the cell to use at each step. For multiple layers, wrap them up in tf.nn.rnn_cell.MultiRNNCell. inputs: [sequence_length, batch_size, num_features] float input tensor. If the inputs are discrete, we expect this to already be embedded. output_size (int): the size we project the output to (ie. the number of symbols in the vocabulary). scope (Optional): variable scope, defaults to "rnn". return_states (Optional[bool]): whether or not to return the states as well as the projected logits. Returns: tuple: (initial_state, logits, final_state) - *initial_state* are float tensors representing the first state of the rnn, to pass states across batches. - *logits* is a list of [batch_size, vocab_size] float tensors representing the output of the network at each timestep. - *final_state* are float tensors containing the final state of the network, evaluate these to get a state to feed into initial_state next batch. """ with tf.variable_scope(scope or "rnn") as scope: inputs = tf.unpack(inputs) batch_size = inputs[0].get_shape()[0].value initial_state = cell.zero_state(batch_size, tf.float32) states, final_state = tf.nn.rnn( cell, inputs, initial_state=initial_state, dtype=tf.float32, scope=scope) # we want to project them all at once, because it's faster logits = tf.concat(0, states) with tf.variable_scope('output_layer'): # output layer needs weights weights = tf.get_variable('weights', [cell.output_size, output_size]) biases = tf.get_variable('bias', [output_size]) logits = tf.nn.bias_add(tf.matmul(logits, weights), biases) # and split them back up to what we expect logits = tf.split(0, len(inputs), logits) if return_states: logits = (states, logits) return [initial_state], logits, [final_state]
def standard_nextstep_sample(cell, inputs, output_size, embedding, scope=None, argmax=False, softmax_temperature=1): """Generate samples from the standard next step prediction model. Assumes that we are modelling sequence of discrete symbols. Args: cell (tf.nn.rnn_cell.RNNCell): a cell to reproduce the model. inputs: input variable, all but the first is ignored. output_size (int): the size of the vocabulary. embedding: the embedding matrix used. scope (Optional): variable scope, needs to line up with what was used to make the model for inference the first time around. argmax (Optional[bool]): if True, then instead of sampling we simply take the argmax of the logits, if False we put a softmax on first. Defaults to False. softmax_temperature (Optional[bool]): the temparature for softmax. The logits are divided by this before feeding into the softmax: a high value means higher entropy. Default 1. Returns: tuple: (initial_state, sequence, final_state) where - *initial_state* is a variable for the starting state of the net. - *sequence* is a list of len(inputs) length containing the sampled symbols. - *final_state* the finishing state of the network, to pass along. """ # have to be quite careful to ensure the scopes line up with tf.variable_scope(scope or 'rnn') as scope: inputs = tf.unpack(inputs) batch_size = inputs[0].get_shape()[0].value initial_state = cell.zero_state(batch_size, tf.float32) # get the output weights with tf.variable_scope('output_layer'): weights = tf.get_variable('weights', [cell.output_size, output_size]) biases = tf.get_variable('bias', [output_size]) # choose an appropriate loop function sequence = [] if argmax: loop_fn = argmax_and_embed(embedding, output_list=sequence, output_projection=(weights, biases)) else: loop_fn = sample_and_embed(embedding, softmax_temperature, output_list=sequence, output_projection=(weights, biases)) all_states, fstate = tf.nn.seq2seq.rnn_decoder( inputs, initial_state, cell, loop_function=loop_fn, scope=scope) return [initial_state], sequence, [fstate]
def __init__(self, input_size, target_size, num_layers, hidden_layer_size, init_scale, dropout_keep_prob): """ A base class for LSTM models that includes predictions and loss. Args: input_size: An integer. The number of inputs per time step. target_size: An integer. The dimensionality of the one-hot encoded targets. num_layers: An integer. The number of hidden layers. hidden_layer_size: An integer. The number of hidden units per layer. init_scale: A float. All weights will be initialized using a uniform distribution over `[-init_scale, init_scale]`. dropout_keep_prob: A float. The fraction of inputs to keep whenever dropout is applied. Dropout is applied to the inputs/outputs of each time step, and is never applied across time steps. """ self.input_size = input_size self.target_size = target_size self.num_layers = num_layers self.hidden_layer_size = hidden_layer_size self.init_scale = init_scale self.dropout_keep_prob = dropout_keep_prob self._inputs = tf.placeholder( tf.float32, shape=[None, None, input_size], name='inputs') self._resets = tf.placeholder( tf.bool, shape=[None, None, 1], name='resets') self._targets = tf.placeholder( tf.float32, shape=[None, None, target_size], name='targets') self._training = tf.placeholder(tf.bool, shape=[], name='training') outputs = self._compute_rnn_outputs() output_size = self._compute_rnn_output_size() initializer = tf.random_uniform_initializer(-self.init_scale, self.init_scale) with tf.variable_scope('logits', initializer=initializer): W = tf.get_variable('W', shape=[output_size, self.target_size]) b = tf.get_variable('b', shape=[self.target_size]) outputs_matrix = tf.reshape(outputs, [-1, output_size]) logits = tf.nn.xw_plus_b(outputs_matrix, W, b) batch_size, duration, _ = tf.unpack(tf.shape(self.inputs)) logits_shape = tf.pack([batch_size, duration, self.target_size]) self._logits = tf.reshape(logits, logits_shape, name='logits') with tf.variable_scope('loss'): logits = tf.reshape(self.logits, [-1, self.target_size]) targets = tf.reshape(self.targets, [-1, self.target_size]) cross_entropies = tf.nn.softmax_cross_entropy_with_logits(logits, targets) self._loss = tf.reduce_mean(cross_entropies, name='loss')
def create_graph(g): initer = tf.random_uniform_initializer(0.0,INIT_SCALE) with tf.variable_scope("graph", reuse=None, initializer=initer): g['x'] = list() g['y'] = list() g['s'] = list() g['seq_lengths'] = tf.placeholder(tf.int64,shape=[BATCH_SIZE]); for _ in range(UNROLLS): g['x'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE,INPUT_SIZE]) ) g['y'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE,INPUT_SIZE]) ) g['s'].append( tf.placeholder(tf.float32,shape=[BATCH_SIZE]) ) num_inputs = INPUT_SIZE * UNROLLS # num_outputs = OUTPUT_SIZE * UNROLLS g['w'] = tf.get_variable("softmax_w", [num_inputs,OUTPUT_SIZE]) g['b'] = tf.get_variable("softmax_b", [OUTPUT_SIZE]) g['cat_x'] = tf.concat(1, g['x'] ) g['logits'] = tf.nn.xw_plus_b(g['cat_x'], g['w'], g['b'] ) g['cat_y'] = tf.unpack(tf.reverse_sequence(tf.reshape( tf.concat(1, g['y'] ), [BATCH_SIZE,UNROLLS,OUTPUT_SIZE] ),g['seq_lengths'],1,0),axis=1)[0] g['loss'] = tf.nn.softmax_cross_entropy_with_logits(g['logits'], g['cat_y']) g['r_s'] = tf.unpack(tf.reverse_sequence(tf.transpose( tf.reshape( tf.concat(0, g['s'] ), [UNROLLS, BATCH_SIZE] ) ), g['seq_lengths'],1,0),axis=1)[0] g['train_loss'] = tf.mul( g['loss'], g['r_s'] ) g['preds'] = tf.nn.softmax(g['logits']) g['class_preds'] = tf.floor( g['preds'] + 0.5 ) g['accy'] = tf.mul( g['class_preds'], g['cat_y'] ) g['w_accy'] = tf.mul(g['accy'], tf.reshape(g['r_s'],shape=[BATCH_SIZE,1]))
def build_rnn_layer(self, inputs_, train_phase): """ Build the computation graph from inputs to outputs of the RNN layer. :param inputs_: [batch, t, emb], float :param train_phase: bool :return: rnn_outputs_: [batch, t, hid_dim], float """ config = self.config def unrolled_rnn(cell, emb_inputs_, initial_state_, seq_len_): if not config.fix_seq_len: raise Exception("`config.fix_seq_len` should be set to `True` if using unrolled_rnn()") outputs = [] state = initial_state_ with tf.variable_scope("unrolled_rnn"): for t in range(config.max_seq_len): if t > 0: tf.get_variable_scope().reuse_variables() output, state = cell(emb_inputs_[:, t], state) # [batch, hid_dim] outputs.append(output) rnn_outputs_ = tf.pack(outputs, axis=1) # [batch, t, hid_dim] return rnn_outputs_ def dynamic_rnn(cell, emb_inputs_, initial_state_, seq_len_): rnn_outputs_, last_states_ = tf.nn.dynamic_rnn(cell, emb_inputs_, initial_state=initial_state_, sequence_length=seq_len_, dtype=config.float_type) # you should define dtype if initial_state is not provided return rnn_outputs_ def bidirectional_rnn(cell, emb_inputs_, initial_state_, seq_len_): rnn_outputs_, output_states = tf.nn.bidirectional_dynamic_rnn(cell, cell, emb_inputs_, seq_len_, initial_state_, initial_state_, config.float_type) return tf.concat(2, rnn_outputs_) def rnn(cell, emb_inputs_, initial_state_, seq_len_): if not config.fix_seq_len: raise Exception("`config.fix_seq_len` should be set to `True` if using rnn()") inputs_ = tf.unpack(emb_inputs_, axis=1) outputs_, states_ = tf.nn.rnn(cell, inputs_, initial_state_, dtype=config.float_type, sequence_length=seq_len_) return outputs_ if config.rnn == 'rnn': cell = tf.nn.rnn_cell.BasicRNNCell(config.hidden_dim) elif config.rnn == 'lstm': cell = tf.nn.rnn_cell.BasicLSTMCell(config.hidden_dim) elif config.rnn == 'gru': cell = tf.nn.rnn_cell.GRUCell(config.hidden_dim) else: raise Exception("`config.rnn` should be correctly defined.") if train_phase and config.keep_prob < 1: cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=config.keep_prob) if config.num_layers is not None and config.num_layers > 1: cell = tf.nn.rnn_cell.MultiRNNCell([cell] * config.num_layers) initial_state_ = cell.zero_state(config.batch_size, dtype=config.float_type) if config.use_seq_len_in_rnn: seq_len_ = self.seq_len_ else: seq_len_ = None rnn_outputs_ = dynamic_rnn(cell, inputs_, initial_state_, seq_len_) # [batch, time, hid_dim] return rnn_outputs_
def create_recurrent_unit(self, params): # Weights and Bias for input and hidden tensor self.Wi = tf.Variable(self.params[1]) self.Ui = tf.Variable(self.params[2]) self.bi = tf.Variable(self.params[3]) self.Wf = tf.Variable(self.params[4]) self.Uf = tf.Variable(self.params[5]) self.bf = tf.Variable(self.params[6]) self.Wog = tf.Variable(self.params[7]) self.Uog = tf.Variable(self.params[8]) self.bog = tf.Variable(self.params[9]) self.Wc = tf.Variable(self.params[10]) self.Uc = tf.Variable(self.params[11]) self.bc = tf.Variable(self.params[12]) params.extend([ self.Wi, self.Ui, self.bi, self.Wf, self.Uf, self.bf, self.Wog, self.Uog, self.bog, self.Wc, self.Uc, self.bc]) def unit(x, hidden_memory_tm1): previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1) # Input Gate i = tf.sigmoid( tf.matmul(x, self.Wi) + tf.matmul(previous_hidden_state, self.Ui) + self.bi ) # Forget Gate f = tf.sigmoid( tf.matmul(x, self.Wf) + tf.matmul(previous_hidden_state, self.Uf) + self.bf ) # Output Gate o = tf.sigmoid( tf.matmul(x, self.Wog) + tf.matmul(previous_hidden_state, self.Uog) + self.bog ) # New Memory Cell c_ = tf.nn.tanh( tf.matmul(x, self.Wc) + tf.matmul(previous_hidden_state, self.Uc) + self.bc ) # Final Memory cell c = f * c_prev + i * c_ # Current Hidden state current_hidden_state = o * tf.nn.tanh(c) return tf.pack([current_hidden_state, c]) return unit
def create_recurrent_unit(self): # Weights and Bias for input and hidden tensor self.Wi = tf.identity(self.lstm.Wi) self.Ui = tf.identity(self.lstm.Ui) self.bi = tf.identity(self.lstm.bi) self.Wf = tf.identity(self.lstm.Wf) self.Uf = tf.identity(self.lstm.Uf) self.bf = tf.identity(self.lstm.bf) self.Wog = tf.identity(self.lstm.Wog) self.Uog = tf.identity(self.lstm.Uog) self.bog = tf.identity(self.lstm.bog) self.Wc = tf.identity(self.lstm.Wc) self.Uc = tf.identity(self.lstm.Uc) self.bc = tf.identity(self.lstm.bc) def unit(x, hidden_memory_tm1): previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1) # Input Gate i = tf.sigmoid( tf.matmul(x, self.Wi) + tf.matmul(previous_hidden_state, self.Ui) + self.bi ) # Forget Gate f = tf.sigmoid( tf.matmul(x, self.Wf) + tf.matmul(previous_hidden_state, self.Uf) + self.bf ) # Output Gate o = tf.sigmoid( tf.matmul(x, self.Wog) + tf.matmul(previous_hidden_state, self.Uog) + self.bog ) # New Memory Cell c_ = tf.nn.tanh( tf.matmul(x, self.Wc) + tf.matmul(previous_hidden_state, self.Uc) + self.bc ) # Final Memory cell c = f * c_prev + i * c_ # Current Hidden state current_hidden_state = o * tf.nn.tanh(c) return tf.pack([current_hidden_state, c]) return unit
def update_recurrent_unit(self): # Weights and Bias for input and hidden tensor self.Wi = self.update_rate * self.Wi + (1 - self.update_rate) * tf.identity(self.lstm.Wi) self.Ui = self.update_rate * self.Ui + (1 - self.update_rate) * tf.identity(self.lstm.Ui) self.bi = self.update_rate * self.bi + (1 - self.update_rate) * tf.identity(self.lstm.bi) self.Wf = self.update_rate * self.Wf + (1 - self.update_rate) * tf.identity(self.lstm.Wf) self.Uf = self.update_rate * self.Uf + (1 - self.update_rate) * tf.identity(self.lstm.Uf) self.bf = self.update_rate * self.bf + (1 - self.update_rate) * tf.identity(self.lstm.bf) self.Wog = self.update_rate * self.Wog + (1 - self.update_rate) * tf.identity(self.lstm.Wog) self.Uog = self.update_rate * self.Uog + (1 - self.update_rate) * tf.identity(self.lstm.Uog) self.bog = self.update_rate * self.bog + (1 - self.update_rate) * tf.identity(self.lstm.bog) self.Wc = self.update_rate * self.Wc + (1 - self.update_rate) * tf.identity(self.lstm.Wc) self.Uc = self.update_rate * self.Uc + (1 - self.update_rate) * tf.identity(self.lstm.Uc) self.bc = self.update_rate * self.bc + (1 - self.update_rate) * tf.identity(self.lstm.bc) def unit(x, hidden_memory_tm1): previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1) # Input Gate i = tf.sigmoid( tf.matmul(x, self.Wi) + tf.matmul(previous_hidden_state, self.Ui) + self.bi ) # Forget Gate f = tf.sigmoid( tf.matmul(x, self.Wf) + tf.matmul(previous_hidden_state, self.Uf) + self.bf ) # Output Gate o = tf.sigmoid( tf.matmul(x, self.Wog) + tf.matmul(previous_hidden_state, self.Uog) + self.bog ) # New Memory Cell c_ = tf.nn.tanh( tf.matmul(x, self.Wc) + tf.matmul(previous_hidden_state, self.Uc) + self.bc ) # Final Memory cell c = f * c_prev + i * c_ # Current Hidden state current_hidden_state = o * tf.nn.tanh(c) return tf.pack([current_hidden_state, c]) return unit
def create_recurrent_unit(self, params): # Weights and Bias for input and hidden tensor self.Wi = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim])) self.Ui = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim])) self.bi = tf.Variable(self.init_matrix([self.hidden_dim])) self.Wf = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim])) self.Uf = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim])) self.bf = tf.Variable(self.init_matrix([self.hidden_dim])) self.Wog = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim])) self.Uog = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim])) self.bog = tf.Variable(self.init_matrix([self.hidden_dim])) self.Wc = tf.Variable(self.init_matrix([self.emb_dim, self.hidden_dim])) self.Uc = tf.Variable(self.init_matrix([self.hidden_dim, self.hidden_dim])) self.bc = tf.Variable(self.init_matrix([self.hidden_dim])) params.extend([ self.Wi, self.Ui, self.bi, self.Wf, self.Uf, self.bf, self.Wog, self.Uog, self.bog, self.Wc, self.Uc, self.bc]) def unit(x, hidden_memory_tm1): previous_hidden_state, c_prev = tf.unpack(hidden_memory_tm1) # Input Gate i = tf.sigmoid( tf.matmul(x, self.Wi) + tf.matmul(previous_hidden_state, self.Ui) + self.bi ) # Forget Gate f = tf.sigmoid( tf.matmul(x, self.Wf) + tf.matmul(previous_hidden_state, self.Uf) + self.bf ) # Output Gate o = tf.sigmoid( tf.matmul(x, self.Wog) + tf.matmul(previous_hidden_state, self.Uog) + self.bog ) # New Memory Cell c_ = tf.nn.tanh( tf.matmul(x, self.Wc) + tf.matmul(previous_hidden_state, self.Uc) + self.bc ) # Final Memory cell c = f * c_prev + i * c_ # Current Hidden state current_hidden_state = o * tf.nn.tanh(c) return tf.pack([current_hidden_state, c]) return unit
def build_model(self): with tf.variable_scope("encoder"): source_xs = tf.nn.embedding_lookup(self.s_emb, self.source) source_xs = tf.split(1, self.max_size, source_xs) with tf.variable_scope("decoder"): target_xs = tf.nn.embedding_lookup(self.t_emb, self.target) target_xs = tf.split(1, self.max_size, target_xs) s = self.encoder.zero_state(self.batch_size, tf.float32) encoder_hs = [] with tf.variable_scope("encoder"): for t in xrange(self.max_size): if t > 0: tf.get_variable_scope().reuse_variables() x = tf.squeeze(source_xs[t], [1]) x = tf.matmul(x, self.s_proj_W) + self.s_proj_b h, s = self.encoder(x, s) encoder_hs.append(h) encoder_hs = tf.pack(encoder_hs) s = self.decoder.zero_state(self.batch_size, tf.float32) logits = [] probs = [] with tf.variable_scope("decoder"): for t in xrange(self.max_size): if t > 0: tf.get_variable_scope().reuse_variables() if not self.is_test or t == 0: x = tf.squeeze(target_xs[t], [1]) x = tf.matmul(x, self.t_proj_W) + self.t_proj_b h_t, s = self.decoder(x, s) h_tld = self.attention(h_t, encoder_hs) oemb = tf.matmul(h_tld, self.proj_W) + self.proj_b logit = tf.matmul(oemb, self.proj_Wo) + self.proj_bo prob = tf.nn.softmax(logit) logits.append(logit) probs.append(prob) if self.is_test: x = tf.cast(tf.argmax(prob, 1), tf.int32) x = tf.nn.embedding_lookup(self.t_emb, x) logits = logits[:-1] targets = tf.split(1, self.max_size, self.target)[1:] weights = tf.unpack(tf.sequence_mask(self.target_len - 1, self.max_size - 1, dtype=tf.float32), None, 1) self.loss = tf.nn.seq2seq.sequence_loss(logits, targets, weights) self.probs = tf.transpose(tf.pack(probs), [1, 0, 2]) self.optim = tf.contrib.layers.optimize_loss(self.loss, None, self.lr_init, "SGD", clip_gradients=5., summaries=["learning_rate", "loss", "gradient_norm"]) tf.initialize_all_variables().run() self.saver = tf.train.Saver()
def _reverse_seq(input_seq, lengths): """Reverse a list of Tensors up to specified lengths. Args: input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features) or nested tuples of tensors. lengths: A `Tensor` of dimension batch_size, containing lengths for each sequence in the batch. If "None" is specified, simply reverses the list. Returns: time-reversed sequence """ if lengths is None: return list(reversed(input_seq)) flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq) flat_results = [[] for _ in range(len(input_seq))] for sequence in zip(*flat_input_seq): input_shape = tensor_shape.unknown_shape( ndims=sequence[0].get_shape().ndims) for input_ in sequence: input_shape.merge_with(input_.get_shape()) input_.set_shape(input_shape) # Join into (time, batch_size, depth) s_joined = array_ops.pack(sequence) # TODO(schuster, ebrevdo): Remove cast when reverse_sequence takes int32 if lengths is not None: lengths = math_ops.to_int64(lengths) # Reverse along dimension 0 s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1) # Split again into list result = array_ops.unpack(s_reversed) for r, flat_result in zip(result, flat_results): r.set_shape(input_shape) flat_result.append(r) results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result) for input_, flat_result in zip(input_seq, flat_results)] return results
