我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.TensorArray()。
def _call_helper(self): time = tf.constant(0, dtype=tf.int32) inp = self._decoder.init_input() state = self._decoder.init_state() finished = tf.tile([False], [utils.get_dimension(inp, 0)]) output_ta = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True) loop_vars = [time, inp, state, finished, output_ta] results = tf.while_loop( cond=self.cond, body=self.body, loop_vars=loop_vars, parallel_iterations=self._parallel_iterations, swap_memory=self._swap_memory) output_ta = results[-1] output = output_ta.stack() output = tf.transpose(output, [1, 0, 2]) state = results[2] return output, state
def empty_attention_loop_state() -> AttentionLoopStateTA: """Create an empty attention loop state. The attention loop state is a technical object for storing the attention distributions and the context vectors in time. It is used with the ``tf.while_loop`` dynamic implementation of the decoder. This function returns an empty attention loop state which means there are two empty arrays, one for attention distributions in time, and one for the attention context vectors in time. """ return AttentionLoopStateTA( contexts=tf.TensorArray( dtype=tf.float32, size=0, dynamic_size=True, name="contexts"), weights=tf.TensorArray( dtype=tf.float32, size=0, dynamic_size=True, name="distributions", clear_after_read=False))
def meta_loss(make_loss): x, constants = _get_variables(make_loss) print("Optimizee variables") print([op.name for op in x]) print("Problem variables") print([op.name for op in constants]) fx = _make_with_custom_variables(make_loss, x) log.info(type(fx)) print fx is None fx_array = tf.TensorArray(tf.float32, 1, clear_after_read=False) fx_array = fx_array.write(0, fx) loss = tf.reduce_sum(fx_array.stack(), name="loss") # problem = simple() # meta_minimize(problem) # log.info(type(fx)) # sess = tf.Session() # sess.run(tf.global_variables_initializer()) # print sess.run(loss)
def pack_into_tensor(array, axis): """ packs a given TensorArray into a tensor along a given axis Parameters: ---------- array: TensorArray the tensor array to pack axis: int the axis to pack the array along Returns: Tensor the packed tensor """ packed_tensor = array.pack() shape = packed_tensor.get_shape() rank = len(shape) dim_permutation = [axis] + range(1, axis) + [0] + range(axis + 1, rank) correct_shape_tensor = tf.transpose(packed_tensor, dim_permutation) return correct_shape_tensor
def __init__(self, training, cell, embedding, start_tokens, end_token, initial_state, beam_width, output_layer=None, gold_sequence=None, gold_sequence_length=None): self._training = training self._cell = cell self._output_layer = output_layer self._embedding_fn = lambda ids: tf.nn.embedding_lookup(embedding, ids) self._output_size = output_layer.units if output_layer is not None else self._output.output_size self._batch_size = tf.size(start_tokens) self._beam_width = beam_width self._tiled_initial_cell_state = nest.map_structure(self._maybe_split_batch_beams, initial_state, self._cell.state_size) self._start_tokens = start_tokens self._tiled_start_tokens = self._maybe_tile_batch(start_tokens) self._end_token = end_token self._original_gold_sequence = gold_sequence self._gold_sequence = gold_sequence self._gold_sequence_length = gold_sequence_length if training: assert self._gold_sequence is not None assert self._gold_sequence_length is not None self._max_time = int(self._gold_sequence.shape[1]) # transpose gold sequence to be time major and make it into a TensorArray self._gold_sequence = tf.TensorArray(dtype=tf.int32, size=self._max_time) self._gold_sequence = self._gold_sequence.unstack(tf.transpose(gold_sequence, [1, 0]))
def _maybe_split_batch_beams(self, t, s): """Maybe splits the tensor from a batch by beams into a batch of beams. We do this so that we can use nest and not run into problems with shapes. Args: t: Tensor of dimension [batch_size*beam_width, s] s: Tensor, Python int, or TensorShape. Returns: Either a reshaped version of t with dimension [batch_size, beam_width, s] if t's first dimension is of size batch_size*beam_width or t if not. Raises: TypeError: If t is an instance of TensorArray. ValueError: If the rank of t is not statically known. """ return self._split_batch_beams(t, s) if t.shape.ndims >= 1 else t
def _maybe_merge_batch_beams(self, t, s): """Splits the tensor from a batch by beams into a batch of beams. More exactly, t is a tensor of dimension [batch_size*beam_width, s]. We reshape this into [batch_size, beam_width, s] Args: t: Tensor of dimension [batch_size*beam_width, s] s: Tensor, Python int, or TensorShape. Returns: A reshaped version of t with dimension [batch_size, beam_width, s]. Raises: TypeError: If t is an instance of TensorArray. ValueError: If the rank of t is not statically known. """ return self._merge_batch_beams(t, s) if t.shape.ndims >= 2 else t
def __init__(self, grammar : AbstractGrammar, *args, training_output=None, grammar_helper : GrammarHelper = None, **kw): super().__init__(*args, **kw) self._grammar = grammar self._grammar_helper = grammar_helper if grammar_helper is not None else GrammarHelper(grammar) self._fixed_outputs = training_output if training_output is not None: self._fixed_outputs = tf.TensorArray(dtype=tf.int32, size=training_output.get_shape()[1]) self._fixed_outputs = self._fixed_outputs.unstack(tf.transpose(training_output, [1, 0]))
def meanShift(n_updates=-1): X1 = tf.expand_dims(tf.transpose(input_X), 0) X2 = tf.expand_dims(input_X, 0) C = init_C sbs_C = tf.TensorArray(dtype=tf.float32, size=10000, infer_shape=False) sbs_C = sbs_C.write(0, init_C) def _mean_shift_step(C): C = tf.expand_dims(C, 2) Y = tf.reduce_sum(tf.pow((C - X1) / window_radius, 2), axis=1) gY = tf.exp(-Y) num = tf.reduce_sum(tf.expand_dims(gY, 2) * X2, axis=1) denom = tf.reduce_sum(gY, axis=1, keep_dims=True) C = num / denom return C if n_updates > 0: for i in range(n_updates): C = _mean_shift_step(C) sbs_C = sbs_C.write(i + 1, C) else: def _mean_shift(i, C, sbs_C, max_diff): new_C = _mean_shift_step(C) max_diff = tf.reshape(tf.reduce_max(tf.sqrt(tf.reduce_sum(tf.pow(new_C - C, 2), axis=1))), []) sbs_C = sbs_C.write(i + 1, new_C) return i + 1, new_C, sbs_C, max_diff def _cond(i, C, sbs_C, max_diff): return max_diff > 1e-5 n_updates, C, sbs_C, _ = tf.while_loop(cond=_cond, body=_mean_shift, loop_vars=(tf.constant(0), C, sbs_C, tf.constant(1e10))) n_updates = tf.Print(n_updates, [n_updates]) return C, sbs_C.gather(tf.range(n_updates + 1))
def get_distorted_inputs(original_image, bboxes, cfg, add_summaries): distorter = DistortedInputs(cfg, add_summaries) num_bboxes = tf.shape(bboxes)[0] distorted_inputs = tf.TensorArray( dtype=tf.float32, size=num_bboxes, element_shape=tf.TensorShape([1, cfg.INPUT_SIZE, cfg.INPUT_SIZE, 3]) ) if add_summaries: image_summaries = tf.TensorArray( dtype=tf.float32, size=4, element_shape=tf.TensorShape([1, cfg.INPUT_SIZE, cfg.INPUT_SIZE, 3]) ) else: image_summaries = tf.constant([]) current_index = tf.constant(0, dtype=tf.int32) loop_vars = [original_image, bboxes, distorted_inputs, image_summaries, current_index] original_image, bboxes, distorted_inputs, image_summaries, current_index = tf.while_loop( cond=bbox_crop_loop_cond, body=distorter.apply, loop_vars=loop_vars, parallel_iterations=10, back_prop=False, swap_memory=False ) distorted_inputs = distorted_inputs.concat() if add_summaries: tf.summary.image('0.original_image', image_summaries.read(0)) tf.summary.image('1.image_with_random_crop', image_summaries.read(1)) tf.summary.image('2.cropped_resized_image', image_summaries.read(2)) tf.summary.image('3.final_distorted_image', image_summaries.read(3)) return distorted_inputs
def __init__(self, cell, location_softmax, pointing_output, input_size, decoder_inputs=None, trainable=True, name=None, **kwargs): """Initializes a new PointingSoftmaxDecoder instance. See the class documentation for the escription of all the arguments. """ super(PointingSoftmaxDecoder, self).__init__( trainable=trainable, name=name, **kwargs) self._cell = cell self._loc = location_softmax self._out = pointing_output self._inp_size = input_size if decoder_inputs is not None: tensors = tf.transpose(decoder_inputs, [1, 0, 2]) dtype = tensors.dtype size = tf.shape(tensors)[0] element_shape = tensors.get_shape()[1:] tensor_array = tf.TensorArray(dtype=dtype, size=size, element_shape=element_shape) decoder_inputs = tensor_array.unstack(tensors) self._inputs_ta = decoder_inputs # infer the batch/location size from the `states` tensor # of the attention layer of the injected location softmax. states = self._loc.attention.states self._batch_size = utils.get_dimension(states, 0) self._loc_size = utils.get_dimension(states, 1)
def _array_to_tuple(inputs, size, shape=None): """ Convert tf.TensorArray to tf.Tuple. """ with tf.variable_scope('array_to_tuple'): if shape is None: output = tf.tuple([inputs.read(i) for i in range(size)]) else: output = tf.tuple([tf.reshape(inputs.read(i), shape) for i in range(size)]) return output
def _unstack_ta(inp): return tf.TensorArray( dtype=inp.dtype, size=tf.shape(inp)[0], element_shape=inp.get_shape()[1:]).unstack(inp)
def get_initial_loop_state(self) -> BeamSearchLoopState: # TODO make these feedable output_ta = SearchStepOutputTA( scores=tf.TensorArray(dtype=tf.float32, dynamic_size=True, size=0, name="beam_scores"), parent_ids=tf.TensorArray(dtype=tf.int32, dynamic_size=True, size=0, name="beam_parents"), token_ids=tf.TensorArray(dtype=tf.int32, dynamic_size=True, size=0, name="beam_tokens")) # We run the decoder once to get logits for ensembling dec_ls = self.parent_decoder.get_initial_loop_state() decoder_body = self.parent_decoder.get_body(False) dec_ls = decoder_body(*dec_ls) # We want to feed these values in ensembles self._search_state = SearchState( logprob_sum=tf.placeholder_with_default([0.0], [None]), prev_logprobs=tf.nn.log_softmax(dec_ls.feedables.prev_logits), lengths=tf.placeholder_with_default([1], [None]), finished=tf.placeholder_with_default([False], [None])) self._decoder_state = dec_ls.feedables # TODO make TensorArrays also feedable return BeamSearchLoopState( bs_state=self._search_state, bs_output=output_ta, decoder_loop_state=dec_ls)
def get_energies(self, y: tf.Tensor, weights_in_time: tf.TensorArray): weight_sum = tf.cond( tf.greater(weights_in_time.size(), 0), lambda: tf.reduce_sum(weights_in_time.stack(), axis=0), lambda: 0.0) coverage = weight_sum / self.fertility * self.attention_mask logits = tf.reduce_sum( self.similarity_bias_vector * tf.tanh( self.hidden_features + y + self.coverage_weights * tf.expand_dims(tf.expand_dims(coverage, -1), -1)), [2, 3]) return logits
def initial_loop_state(self) -> MultiHeadLoopStateTA: return MultiHeadLoopStateTA( contexts=tf.TensorArray( dtype=tf.float32, size=0, dynamic_size=True, name="contexts"), head_weights=[tf.TensorArray( dtype=tf.float32, size=0, dynamic_size=True, name="distributions_head{}".format(i), clear_after_read=False) for i in range(self.n_heads)])
def _compute_states(self): """ Compute hidden states. Returns: A tuple, (outputs, states). """ _inputs = tf.transpose(self.inputs, [1, 0, 2]) x_ta = tf.TensorArray(tf.float32, size=self.length).unstack(_inputs) h_ta = tf.TensorArray(tf.float32, size=self.length) def cond(t, h, h_ta): return tf.less(t, self.length) def body(t, h, h_ta): x = x_ta.read(t) num_units, input_size = self.num_hidden_units, self.input_size with tf.variable_scope('simple_rnn'): h_new = self.activation(self._linear(h, x, num_units, scope='simple_rnn')) h_ta_new = h_ta.write(t, h_new) return t + 1, h_new, h_ta_new t = tf.constant(0) h = tf.squeeze(self.initial_states, [1]) _, _, h_ta = tf.while_loop(cond, body, [t, h, h_ta]) states = tf.transpose(h_ta.stack(), [1, 0, 2], name='states') outputs = tf.identity(states, name='outputs') return outputs, states
def _compute_states(self): _inputs = tf.transpose(self.inputs, [1, 0, 2]) x_ta = tf.TensorArray(tf.float32, size=self.length).unstack(_inputs) h_ta = tf.TensorArray(tf.float32, size=self.length) c_ta = tf.TensorArray(tf.float32, size=self.length) def cond(t, c, h, c_ta, h_ta): return tf.less(t, self.length) def body(t, c, h, c_ta, h_ta): x = x_ta.read(t) num_units, input_size = self.num_hidden_units, self.input_size with tf.variable_scope('lstm'): c_tilde = self.activation(self._linear(h, x, num_units, scope='c')) i = tf.nn.sigmoid(self._linear(h, x, num_units, scope='i')) f = tf.nn.sigmoid(self._linear(h, x, num_units, shift=self.optional_bias_shift, scope='f')) o = tf.nn.sigmoid(self._linear(h, x, num_units, scope='o')) c_new = i * c_tilde + f * c h_new = o * self.activation(c_new) c_ta_new = c_ta.write(t, c_new) h_ta_new = h_ta.write(t, h_new) return t + 1, c_new, h_new, c_ta_new, h_ta_new t = tf.constant(0) c, h = tf.split(tf.squeeze(self.initial_states, [1]), 2, axis=1) _, _, _, c_ta, h_ta = tf.while_loop(cond, body, [t, c, h, c_ta, h_ta]) outputs = tf.transpose(h_ta.stack(), [1, 0, 2], name='outputs') cells = tf.transpose(c_ta.stack(), [1, 0, 2]) states = tf.concat([cells, outputs], axis=2, name='states') return outputs, states
def _compute_states(self): _inputs = tf.transpose(self.inputs, [1, 0, 2]) x_ta = tf.TensorArray(tf.float32, size=self.length).unstack(_inputs) h_ta = tf.TensorArray(tf.float32, size=self.length) def cond(t, h, h_ta): return tf.less(t, self.length) def body(t, h, h_ta): x = x_ta.read(t) num_units, input_size = self.num_hidden_units, self.input_size with tf.variable_scope('gru'): r = tf.nn.sigmoid(self._linear(h, x, num_units, scope='r')) h_pre_act = r * h h_tilde = self.activation(self._linear(h_pre_act, x, num_units, scope='h')) z = tf.nn.sigmoid(self._linear(h, x, num_units, shift=self.optional_bias_shift, scope='z')) h_new = z * h + (1 - z) * h_tilde h_ta_new = h_ta.write(t, h_new) return t + 1, h_new, h_ta_new t = tf.constant(0) h = tf.squeeze(self.initial_states, [1]) _, _, h_ta = tf.while_loop(cond, body, [t, h, h_ta]) states = tf.transpose(h_ta.stack(), [1, 0, 2], name='states') outputs = tf.identity(states, name='outputs') return outputs, states
def source_distance(x,y): y = tf.cast(tf.argmax(y,axis=1),tf.float32) y1,_,_ = tf.unique_with_counts(y) TensorArr = tf.TensorArray(tf.float32,size=1, dynamic_size=True,clear_after_read=False) x_array = TensorArr.unstack(y1) size = x_array.size() initial_outputs = tf.TensorArray(dtype=tf.float32,size=size) i = tf.constant(0) def should_continue(i, *args): return i < size def loop(i,output): y_class = x_array.read(i) idx_i = tf.where(tf.equal(y,y_class)) xi = tf.gather_nd(x,idx_i) initial_outputs1 = tf.TensorArray(dtype=tf.float32,size=size) j = tf.constant(0) def should_continue1(j,*args): return j<size def loop1(j,output1): y2=x_array.read(j) idx_j = tf.where(tf.equal(y,y2)) xj = tf.gather_nd(x,idx_j) dis = tf.reduce_mean (tf.square(tf.reduce_mean(xi,0) -tf.reduce_mean(xj,0))) output1 = output1.write(j,dis) return j+1,output1 j,r1=tf.while_loop(should_continue1,loop1,[j,initial_outputs1]) output = output.write(i,r1.stack()) return i+1,output i,r = tf.while_loop(should_continue,loop,[i,initial_outputs]) out = r.stack() return out
def __init__(self, size, dtype): # A TensorArray is required as the sequences don't have the same # length. Alternatively a FIFO query can be used. # Because the data is read more than once by the queue, # clear_after_read is set to False (but I can't confirm an effect). # Because the items has diffrent sequence lengths the infer_shape # is set to False. The shape is then restored in the .read method. self.data = tf.TensorArray(size=size, dtype=dtype, dynamic_size=False, clear_after_read=False, infer_shape=False)
def unpack_to_array(tensor): return tf.TensorArray(tensor.dtype, tf.shape(tensor)[0]).unpack(tensor)
def unpack_into_tensorarray(value, axis, size=None): """ unpacks a given tensor along a given axis into a TensorArray Parameters: ---------- value: Tensor the tensor to be unpacked axis: int the axis to unpack the tensor along size: int the size of the array to be used if shape inference resulted in None Returns: TensorArray the unpacked TensorArray """ shape = value.get_shape().as_list() rank = len(shape) dtype = value.dtype array_size = shape[axis] if not shape[axis] is None else size if array_size is None: raise ValueError("Can't create TensorArray with size None") array = tf.TensorArray(dtype=dtype, size=array_size) dim_permutation = [axis] + range(1, axis) + [0] + range(axis + 1, rank) unpack_axis_major_value = tf.transpose(value, dim_permutation) full_array = array.unpack(unpack_axis_major_value) return full_array
def add_model_variable(self): with tf.variable_scope('embedding'): self.embedding = tf.get_variable(name='wVector', shape=[len(self.lexicon), self.wordSize]) with tf.variable_scope('weights'): self.tensorV = tf.get_variable(name='tensorV', shape=[2 * self.wordSize, 2 * self.wordSize, self.wordSize]) self.linearW = tf.get_variable(name='linearW', shape=[2 * self.wordSize, self.wordSize]) self.softW = tf.get_variable(name='softW', shape=[self.wordSize, self.labelNum]) with tf.variable_scope('bias'): self.linearB = tf.get_variable(name='linearB', shape=[1, self.wordSize]) self.softB = tf.get_variable(name='softB', shape=[1, self.labelNum]) self.modelArray = tf.TensorArray(tf.float32, size=0, dynamic_size=True, clear_after_read=False, infer_shape=False) # word vector indice
def _custom_rnn_loop_fn(self, cell_size, training_wheels): def loop_fn(time, cell_output, cell_state, loop_state): if cell_output is None: # time == 0 context_vectors_array = tf.TensorArray(tf.float32, size=tf.shape(self.references_placeholder)[1] + 1) attention_logits_array = tf.TensorArray(tf.float32, size=tf.shape(self.references_placeholder)[1] + 1) pointer_probability_array = tf.TensorArray(tf.float32, size=tf.shape(self.references_placeholder)[1] + 1) next_cell_state = self.final_encoder_state go_id = self.summary_vocabulary.word_to_id('<GO>') last_output_embedding = tf.nn.embedding_lookup(self.embeddings, tf.tile([go_id], [self.batch_size])) else: context_vectors_array, attention_logits_array, pointer_probability_array = loop_state next_cell_state = cell_state if training_wheels: voc_indices = self.references_placeholder[:, time - 1] pointer_indices = self.pointer_reference_placeholder[:, time - 1] pointer_switch = tf.cast(self.pointer_switch_placeholder[:, time - 1], tf.bool) batch_range = tf.range(self.batch_size) pointer_indexer = tf.stack([batch_range, pointer_indices], axis=1) attention_vocabulary_indices = tf.gather_nd(self.documents_placeholder, pointer_indexer) mixed_indices = tf.where(pointer_switch, attention_vocabulary_indices, voc_indices) last_output_embedding = tf.nn.embedding_lookup(self.embeddings, mixed_indices) else: last_output_embedding = self._extract_argmax_and_embed(cell_output, cell_size, tf.shape(self.documents_placeholder)[0]) context_vector, attention_logits = self._attention(next_cell_state, last_output_embedding) pointer_probabilities = self._pointer_probabilities(context_vector, next_cell_state, last_output_embedding) context_vectors_array = context_vectors_array.write(time, context_vector) attention_logits_array = attention_logits_array.write(time, attention_logits) pointer_probability_array = pointer_probability_array.write(time, pointer_probabilities) next_input = tf.concat([last_output_embedding, context_vector, self.query_last], axis=1) elements_finished = (time >= self.reference_lengths_placeholder) emit_output = cell_output next_loop_state = (context_vectors_array, attention_logits_array, pointer_probability_array) return elements_finished, next_input, next_cell_state, emit_output, next_loop_state return loop_fn
def _gru_encoder(cell, inputs, sequence_length, initial_state, dtype=None): # Assume that the underlying cell is GRUCell-like output_size = cell.output_size dtype = dtype or inputs.dtype batch = tf.shape(inputs)[0] time_steps = tf.shape(inputs)[1] zero_output = tf.zeros([batch, output_size], dtype) if initial_state is None: initial_state = cell.zero_state(batch, dtype) input_ta = tf.TensorArray(dtype, time_steps, tensor_array_name="input_array") output_ta = tf.TensorArray(dtype, time_steps, tensor_array_name="output_array") input_ta = input_ta.unstack(tf.transpose(inputs, [1, 0, 2])) def loop_func(t, out_ta, state): inp_t = input_ta.read(t) cell_output, new_state = cell(inp_t, state) cell_output = _copy_through(t, sequence_length, zero_output, cell_output) new_state = _copy_through(t, sequence_length, state, new_state) out_ta = out_ta.write(t, cell_output) return t + 1, out_ta, new_state time = tf.constant(0, dtype=tf.int32, name="time") loop_vars = (time, output_ta, initial_state) outputs = tf.while_loop(lambda t, *_: t < time_steps, loop_func, loop_vars, parallel_iterations=32, swap_memory=True) output_final_ta = outputs[1] final_state = outputs[2] all_output = output_final_ta.stack() all_output.set_shape([None, None, output_size]) all_output = tf.transpose(all_output, [1, 0, 2]) return all_output, final_state
def non_max_suppression(inputs, scores, batch_size, max_output_size, score_threshold=0.7, iou_threshold=0.7, nonempty=False, name='nms'): """ Perform NMS on batch of images. Parameters ---------- inputs: tf.Tuple each components is a set of bboxes for corresponding image scores: tf.Tuple scores of inputs batch_size: size of batch of inputs max_output_size: maximal size of bboxes per image score_threshold: float bboxes with score less the score_threshold will be dropped iou_threshold: float bboxes with iou which is greater then iou_threshold will be merged nonempty: bool if True at least one bbox per image will be returned name: str scope name Returns ------- tf.Tuple indices of selected bboxes for each image """ with tf.variable_scope(name): ix = tf.constant(0) filtered_rois = tf.TensorArray(dtype=tf.int32, size=batch_size, infer_shape=False) loop_cond = lambda ix, filtered_rois: tf.less(ix, batch_size) def _loop_body(ix, filtered_rois): indices, score, roi = _filter_tensor(scores[ix], score_threshold, inputs[ix]) # pylint: disable=unbalanced-tuple-unpacking roi_corners = tf.concat([roi[:, :2], roi[:, :2]+roi[:, 2:]], axis=-1) roi_after_nms = tf.image.non_max_suppression(roi_corners, score, max_output_size, iou_threshold) if nonempty: is_not_empty = lambda: filtered_rois.write(ix, tf.cast(tf.gather(indices, roi_after_nms), dtype=tf.int32)) is_empty = lambda: filtered_rois.write(ix, tf.constant([[0]])) filtered_rois = tf.cond(tf.not_equal(tf.shape(indices)[0], 0), is_not_empty, is_empty) else: filtered_rois = filtered_rois.write(ix, tf.cast(tf.gather(indices, roi_after_nms), dtype=tf.int32)) return [ix+1, filtered_rois] _, res = tf.while_loop(loop_cond, _loop_body, [ix, filtered_rois]) res = _array_to_tuple(res, batch_size, [-1, 1]) return res
def _resize_except_axis(inputs, size, axis, **kwargs): """ Resize 3D input tensor to size except just one axis. """ perm = np.arange(5) reverse_perm = np.arange(5) if axis == 0: spatial_perm = [2, 3, 1] reverse_spatial_perm = [3, 1, 2] elif axis == 1: spatial_perm = [1, 3, 2] reverse_spatial_perm = [1, 3, 2] else: spatial_perm = [1, 2, 3] reverse_spatial_perm = [1, 2, 3] perm[1:4] = spatial_perm reverse_perm[1:4] = reverse_spatial_perm x = tf.transpose(inputs, perm) if isinstance(size, tf.Tensor): size = tf.unstack(size) size = [size[i-1] for i in spatial_perm] size = tf.stack(size) else: size = [size[i-1] for i in spatial_perm] real_size, static_shape = _calc_size_after_resize(x, size, [0, 1]) real_size = size[:-1] array = tf.TensorArray(tf.float32, size=tf.shape(x)[-2]) partial_sl = [slice(None)] * 5 def _loop(idx, array): partial_sl[-2] = idx tensor = x[partial_sl] tensor = tf.image.resize_bilinear(tensor, size=real_size, name='resize_2d', **kwargs) array = array.write(idx, tensor) return [idx+1, array] i = 0 _, array = tf.while_loop(lambda i, array: i < tf.shape(x)[-2], _loop, [i, array]) array = array.stack() array = tf.transpose(array, [1, 2, 3, 0, 4]) array.set_shape(static_shape) array = tf.transpose(array, reverse_perm) return array
def do_inference_steps(self, initial_state, premise, hypothesis): self.one_minus_eps = tf.constant(1.0 - self.config.eps, tf.float32,[self.batch_size]) self.N = tf.constant(self.config.max_computation, tf.float32,[self.batch_size]) prob = tf.constant(0.0,tf.float32,[self.batch_size], name="prob") prob_compare = tf.constant(0.0,tf.float32,[self.batch_size], name="prob_compare") counter = tf.constant(0.0, tf.float32,[self.batch_size], name="counter") i = tf.constant(0, tf.int32, name="index") acc_states = tf.zeros_like(initial_state, tf.float32, name="state_accumulator") batch_mask = tf.constant(True, tf.bool,[self.batch_size]) # Tensor arrays to collect information about the run: array_probs = tf.TensorArray(tf.float32,0, dynamic_size=True) premise_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) hypothesis_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) incremental_states = tf.TensorArray(tf.float32,0, dynamic_size=True) # While loop stops when this predicate is FALSE. # Ie all (probability < 1-eps AND counter < N) are false. pred = lambda i ,incremental_states, array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob,\ counter,state,premise, hypothesis ,acc_state:\ tf.reduce_any( tf.logical_and( tf.less(prob_compare,self.one_minus_eps), tf.less(counter,self.N))) # only stop if all of the batch have passed either threshold # Do while loop iterations until predicate above is false. i,incremental_states, array_probs,premise_attention,hypothesis_attention,_,_,remainders,iterations,_,_,_,state = \ tf.while_loop(pred,self.inference_step, [i,incremental_states, array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob, counter,initial_state,premise, hypothesis, acc_states]) self.ACTPROB = array_probs.pack() self.ACTPREMISEATTN = premise_attention.pack() self.ACTHYPOTHESISATTN = hypothesis_attention.pack() self.incremental_states = incremental_states.pack() return state, remainders, iterations
def do_inference_steps(self, initial_state, premise, hypothesis): self.one_minus_eps = tf.constant(1.0 - self.config.eps, tf.float32,[self.batch_size]) self.N = tf.constant(self.config.max_computation, tf.float32,[self.batch_size]) prob = tf.constant(0.0,tf.float32,[self.batch_size], name="prob") prob_compare = tf.constant(0.0,tf.float32,[self.batch_size], name="prob_compare") counter = tf.constant(0.0, tf.float32,[self.batch_size], name="counter") i = tf.constant(0, tf.int32, name="index") acc_states = tf.zeros_like(initial_state, tf.float32, name="state_accumulator") batch_mask = tf.constant(True, tf.bool,[self.batch_size]) # Tensor arrays to collect information about the run: array_probs = tf.TensorArray(tf.float32,0, dynamic_size=True) premise_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) hypothesis_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) # While loop stops when this predicate is FALSE. # Ie all (probability < 1-eps AND counter < N) are false. pred = lambda i ,array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob,\ counter,state,premise, hypothesis ,acc_state:\ tf.reduce_any( tf.logical_and( tf.less(prob_compare,self.one_minus_eps), tf.less(counter,self.N))) # only stop if all of the batch have passed either threshold # Do while loop iterations until predicate above is false. i,array_probs,premise_attention,hypothesis_attention,_,_,remainders,iterations,_,_,_,state = \ tf.while_loop(pred,self.inference_step, [i,array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob, counter,initial_state,premise, hypothesis, acc_states]) self.ACTPROB = array_probs.pack() self.ACTPREMISEATTN = premise_attention.pack() self.ACTHYPOTHESISATTN = hypothesis_attention.pack() return state, remainders, iterations
def do_act_steps(self, premise, hypothesis): self.rep_size = premise.get_shape()[-1].value self.one_minus_eps = tf.constant(1.0 - self.config.eps, tf.float32,[self.batch_size]) self.N = tf.constant(self.config.max_computation, tf.float32,[self.batch_size]) prob = tf.constant(0.0,tf.float32,[self.batch_size], name="prob") prob_compare = tf.constant(0.0,tf.float32,[self.batch_size], name="prob_compare") counter = tf.constant(0.0, tf.float32,[self.batch_size], name="counter") initial_state = tf.zeros([self.batch_size, 2*self.rep_size], tf.float32, name="state") i = tf.constant(0, tf.int32, name="index") acc_states = tf.zeros_like(initial_state, tf.float32, name="state_accumulator") batch_mask = tf.constant(True, tf.bool,[self.batch_size]) # Tensor arrays to collect information about the run: array_probs = tf.TensorArray(tf.float32,0, dynamic_size=True) premise_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) hypothesis_attention = tf.TensorArray(tf.float32,0, dynamic_size=True) # While loop stops when this predicate is FALSE. # Ie all (probability < 1-eps AND counter < N) are false. pred = lambda i ,array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob,\ counter,state,premise, hypothesis ,acc_state:\ tf.reduce_any( tf.logical_and( tf.less(prob_compare,self.one_minus_eps), tf.less(counter,self.N))) # only stop if all of the batch have passed either threshold # Do while loop iterations until predicate above is false. i,array_probs,premise_attention,hypothesis_attention,_,_,remainders,iterations,_,_,_,state = \ tf.while_loop(pred,self.inference_step, [i,array_probs, premise_attention, hypothesis_attention, batch_mask,prob_compare,prob, counter,initial_state,premise, hypothesis, acc_states]) self.ACTPROB = array_probs.pack() self.ACTPREMISEATTN = premise_attention.pack() self.ACTHYPOTHESISATTN = hypothesis_attention.pack() return state, remainders, iterations
def __call__(self, X): """ Performs the LSTM's forget, input and output operations according to: http://arxiv.org/pdf/1402.1128v1.pdf without peepholes Parameters: ---------- X: list[Tensor] The input list to process by the LSTM """ outputs = tf.TensorArray(tf.float32, len(X)) inputs = tf.TensorArray(tf.float32, len(X)) t = tf.constant(0, dtype=tf.int32) for i, step_input in enumerate(X): inputs = inputs.write(i, step_input) def step_op(time, prev_state, prev_output, inputs_list, outputs_list): time_step = inputs_list.read(time) gates = tf.matmul(time_step, self.input_weights) + tf.matmul(prev_output, self.output_weights) + self.bias gates = tf.reshape(gates, [-1, self.num_hidden, 4]) input_gate = tf.sigmoid(gates[:, :, 0]) forget_gate = tf.sigmoid(gates[:, :, 1]) candidate_state = tf.tanh(gates[:, :, 2]) output_gate = tf.sigmoid(gates[:, :, 3]) state = forget_gate * prev_state + input_gate * candidate_state output = output_gate * tf.tanh(state) new_outputs = outputs_list.write(time, output) return time + 1, state, output, inputs_list, new_outputs _, state, output, _, final_outputs = tf.while_loop( cond=lambda time, *_: time < len(X), body= step_op, loop_vars=(t, self.prev_state, self.prev_output, inputs, outputs), parallel_iterations=32, swap_memory=True ) self.prev_state.assign(state) self.prev_output.assign(output) return [final_outputs.read(t) for t in range(len(X))]
def get_initial_loop_state(self) -> LoopState: rnn_output_ta = tf.TensorArray(dtype=tf.float32, dynamic_size=True, size=0, name="decoder_outputs") rnn_output_ta = rnn_output_ta.write(0, self.initial_state) logit_ta = tf.TensorArray(dtype=tf.float32, dynamic_size=True, size=0, name="logits") outputs_ta = tf.TensorArray(dtype=tf.int32, dynamic_size=True, size=0, name="outputs") contexts = [tf.zeros([self.batch_size, a.context_vector_size]) for a in self.attentions] mask_ta = tf.TensorArray(dtype=tf.bool, dynamic_size=True, size=0, name="mask") attn_loop_states = [a.initial_loop_state() for a in self.attentions if a is not None] # pylint: disable=not-callable rnn_feedables = RNNFeedables( # general: step=0, finished=tf.zeros([self.batch_size], dtype=tf.bool), input_symbol=self.go_symbols, prev_logits=tf.zeros([self.batch_size, len(self.vocabulary)]), # rnn-specific: prev_rnn_state=self.initial_state, prev_rnn_output=self.initial_state, prev_contexts=contexts) rnn_histories = RNNHistories( attention_histories=attn_loop_states, # general: logits=logit_ta, decoder_outputs=rnn_output_ta, outputs=outputs_ta, mask=mask_ta) # pylint: enable=not-callable loop_constants = DecoderConstants(train_inputs=self.train_inputs) return LoopState( histories=rnn_histories, constants=loop_constants, feedables=rnn_feedables)
def _compute_states(self): _inputs = tf.transpose(self.inputs, [1, 0, 2]) x_ta = tf.TensorArray(tf.float32, size=self.length).unstack(_inputs) h_ta_size = self.num_initial_states + self.length initial_states = tf.transpose(self.initial_states, [1, 0, 2]) # infer_shapes=True is buggy and says that shape (?, num_hidden_units) is incompatible with # shape (?, num_hidden_units). I've verified that they both have shape # (batch_size, num_hidden_units). To avoid this, we'll set infer_shape=False and # skip the consistency check entirely. h_ta = tf.TensorArray(tf.float32, size=h_ta_size, clear_after_read=False, infer_shape=False) h_ta = h_ta.unstack(initial_states) def cond(t, h_ta): return tf.less(t, self.length) def body(t, h_ta): h = h_ta.read(self.num_initial_states + t - 1) x = x_ta.read(t) num_units, input_size = self.num_hidden_units, self.input_size with tf.variable_scope('pre_act'): # Shape [batch_size, pre_act_mixture_delays.size, num_units] h_history = tf.transpose(h_ta.gather(self.num_initial_states + t - self.pre_act_mixture_delays), [1, 0, 2]) # Shape [batch_size, pre_act_mixture_delays.size, 1] coefs = tf.expand_dims(self._linear(h, x, self.pre_act_mixture_delays.size, scope='coefs'), 2) coefs = tf.nn.softmax(coefs, dim=1) # Shape [batch_size, num_units] h_pre_act = tf.reduce_sum(coefs * h_history, axis=[1]) r = tf.nn.sigmoid(self._linear(h, x, num_units, scope='r')) h_pre_act = r * h_pre_act h_tilde = self.activation(self._linear(h_pre_act, x, num_units, scope='mist')) h_ta_new = h_ta.write(self.num_initial_states + t, h_tilde) return t + 1, h_ta_new t = tf.constant(0) _, h_ta = tf.while_loop(cond, body, [t, h_ta]) all_states = h_ta.stack() states = tf.transpose(all_states[self.num_initial_states:], [1, 0, 2], name='states') outputs = tf.identity(states, name='outputs') return outputs, states
def decode(self, enc_outputs, enc_final_state): with tf.variable_scope(self.decoder.scope): def condition(time, all_outputs: tf.TensorArray, inputs, states): def check_outputs_ends(): def has_end_word(t): return tf.reduce_any(tf.equal(t, ANSWER_MAX)) output_label = tf.arg_max(all_outputs.stack(), 2) output_label = tf.Print(output_label, [output_label], "Output Labels: ") # The outputs are time-major, which means time is the first # dimension. Here I need to check whether all the generated # answers are ends with "</s>", so we need to transpose it # to batch-major. Because `map_fn` only map function by the # first dimension. batch_major_outputs = tf.transpose(output_label, (1, 0)) all_outputs_ends = tf.reduce_all(tf.map_fn(has_end_word, batch_major_outputs, dtype=tf.bool)) return all_outputs_ends # If the TensorArray has 0 size, stack() will trigger error, # so I have to use condition function to check whether the # size is 0. all_ends = tf.cond(tf.equal(all_outputs.size(), 0), lambda: tf.constant(False, tf.bool), check_outputs_ends) condition_result = tf.logical_and(tf.logical_not(all_ends), tf.less(time, ANSWER_MAX)) return condition_result def body(time, all_outputs, inputs, state): dec_outputs, dec_state, output_logits, next_input = self.decoder.step(inputs, state) all_outputs = all_outputs.write(time, output_logits) return time + 1, all_outputs, next_input, dec_state output_ta = tensor_array_ops.TensorArray(dtype=tf.float32, size=0, dynamic_size=True, element_shape=(None, config.DEC_VOCAB), clear_after_read=False) # with time-major data input, the batch size is the second dimension batch_size = tf.shape(enc_outputs)[1] zero_input = tf.ones(tf.expand_dims(batch_size, axis=0), dtype=tf.int32) * ANSWER_START res = control_flow_ops.while_loop( condition, body, loop_vars=[0, output_ta, self.decoder.zero_input(zero_input), enc_final_state], ) final_outputs = res[1].stack() final_outputs = tf.Print(final_outputs, [final_outputs], "Final Output: ") final_state = res[3] return final_outputs, final_state
def match(qstates, pstates, d, dropout=None): # infer batch_size, passage length and question length qlen, batch_size, _ = tf.unstack(tf.shape(qstates)) plen = tf.shape(pstates)[0] # ouput projection params # Wo = tf.get_variable('Wo', shape=[2*d, d], dtype=tf.float32) # define rnn cell # TODO : replace with LSTM cell = rcell('lstm', num_units=2*d, dropout=dropout) states = tf.TensorArray(dtype=tf.float32, size=plen+1, name='states', clear_after_read=False) outputs = tf.TensorArray(dtype=tf.float32, size=plen, name='outputs', clear_after_read=False) # set init state #init_state = tf.zeros(dtype=tf.float32, shape=[batch_size, 2*d]) init_state = cell.zero_state(batch_size, tf.float32) states = states.write(0, init_state) def mlstm_step(i, states, outputs): # get previous state prev_state = states.read(i) prev_state = tf.unstack(prev_state) prev_state_tuple = tf.contrib.rnn.LSTMStateTuple(prev_state[0], prev_state[1]) prev_state_c = prev_state_tuple.c # get attention weighted representation ci = attention(qstates, pstates[i], prev_state_c, d) # combine ci and input(i) input_ = tf.concat([pstates[i], ci], axis=-1) output, state = cell(input_, prev_state_tuple) # save output, state states = states.write(i+1, state) outputs = outputs.write(i, output) return (i+1, states, outputs) # execute loop #i = tf.constant(0) c = lambda x, y, z : tf.less(x, plen) b = lambda x, y, z : mlstm_step(x, y, z) _, fstates, foutputs = tf.while_loop(c,b, [0, states, outputs]) return foutputs.stack(), project_lstm_states(fstates.stack()[1:], 4*d, d)
def naive_decoder(cell, enc_states, targets, start_token, end_token, feed_previous=True, training=True, scope='naive_decoder.0'): init_state = enc_states[-1] timesteps = tf.shape(enc_states)[0] # targets time major targets_tm = tf.transpose(targets, [1,0,2]) states = tf.TensorArray(dtype=tf.float32, size=timesteps+1, name='states', clear_after_read=False) outputs = tf.TensorArray(dtype=tf.float32, size=timesteps+1, name='outputs', clear_after_read=False) def step(i, states, outputs): # run one step # read from TensorArray (states) state_prev = states.read(i) if is_lstm(cell): # previous state <tensor> -> <LSTMStateTuple> c, h = tf.unstack(state_prev) state_prev = rnn.LSTMStateTuple(c,h) if feed_previous: input_ = outputs.read(i) else: input_ = targets_tm[i] output, state = cell(input_, state_prev) # add state, output to list states = states.write(i+1, state) outputs = outputs.write(i+1, output) i = tf.add(i,1) return i, states, outputs with tf.variable_scope(scope): # initial state states = states.write(0, init_state) # initial input outputs = outputs.write(0, start_token) i = tf.constant(0) # Stop loop condition if training: c = lambda x, y, z : tf.less(x, timesteps) else: c = lambda x, y, z : tf.reduce_all(tf.not_equal(tf.argmax(z.read(x), axis=-1), end_token)) # body b = lambda x, y, z : step(x, y, z) # execution _, fstates, foutputs = tf.while_loop(c,b, [i, states, outputs]) return foutputs.stack()[1:] # add states; but why?
def uni_net_dynamic(cell, inputs, proj_dim=None, init_state=None, scope='uni_net_d0'): # transpose to time major inputs_tm = tf.transpose(inputs, [1,0,2], name='inputs_tm') # infer timesteps and batch_size timesteps, batch_size, _ = tf.unstack(tf.shape(inputs_tm)) # check if init_state is provided # TODO : fix and add this # init_state = init_state if init_state else cell.zero_state(batch_size,tf.float32) if init_state is None: init_state = cell.zero_state(batch_size, tf.float32) states = tf.TensorArray(dtype=tf.float32, size=timesteps+1, name='states', clear_after_read=False) outputs = tf.TensorArray(dtype=tf.float32, size=timesteps, name='outputs', clear_after_read=False) def step(i, states, outputs): # run one step # read from TensorArray (states) state_prev = states.read(i) if is_lstm(cell): # previous state <tensor> -> <LSTMStateTuple> c, h = tf.unstack(state_prev) state_prev = rnn.LSTMStateTuple(c,h) output, state = cell(inputs_tm[i], state_prev) # add state, output to list states = states.write(i+1, state) outputs = outputs.write(i, output) i = tf.add(i,1) return i, states, outputs with tf.variable_scope(scope): # initial state states = states.write(0, init_state) i = tf.constant(0) # stopping condition c = lambda x, y, z : tf.less(x, timesteps) # body b = lambda x, y, z : step(x, y, z) # execution _, fstates, foutputs = tf.while_loop(c,b, [i, states, outputs]) # if LSTM, project states if is_lstm(cell): d1 = 2*cell.state_size.c d2 = proj_dim if proj_dim else d1//2 return foutputs.stack(), project_lstm_states(fstates.stack()[1:], d1, d2) return foutputs.stack(), fstates.stack()[1:]
def build_graph(self): """ builds the computational graph that performs a step-by-step evaluation of the input data batches """ self.unpacked_input_data = utility.unpack_into_tensorarray(self.input_data, 1, self.sequence_length) outputs = tf.TensorArray(tf.float32, self.sequence_length) read_weightings = tf.TensorArray(tf.float32, self.sequence_length) write_weightings = tf.TensorArray(tf.float32, self.sequence_length) write_vectors = tf.TensorArray(tf.float32, self.sequence_length) key_vectors = tf.TensorArray(tf.float32, self.sequence_length) beta_vectors = tf.TensorArray(tf.float32, self.sequence_length) shift_vectors = tf.TensorArray(tf.float32, self.sequence_length) gamma_vectors = tf.TensorArray(tf.float32, self.sequence_length) gates_vectors = tf.TensorArray(tf.float32, self.sequence_length) memory_vectors = tf.TensorArray(tf.float32, self.sequence_length) controller_state = self.controller.get_state() if self.controller.has_recurrent_nn else (tf.zeros(1), tf.zeros(1)) if not isinstance(controller_state, LSTMStateTuple): controller_state = LSTMStateTuple(controller_state[0], controller_state[1]) memory_state = self.memory.init_memory() final_results = None with tf.variable_scope("Sequence_Loop") as scope: time = tf.constant(0, dtype=tf.int32) final_results = tf.while_loop( cond=lambda time, *_: time < self.sequence_length, body=self._loop_body, loop_vars=( time, memory_state, outputs, read_weightings, write_weightings, controller_state, write_vectors, key_vectors, beta_vectors, shift_vectors, gamma_vectors, gates_vectors, memory_vectors ), parallel_iterations=32, swap_memory=True ) dependencies = [] if self.controller.has_recurrent_nn: dependencies.append(self.controller.update_state(final_results[5])) with tf.control_dependencies(dependencies): self.packed_output = utility.pack_into_tensor(final_results[2], axis=1) # packed_memory_view and its content is just for debugging purposes. self.packed_memory_view = { 'read_weightings': utility.pack_into_tensor(final_results[3], axis=1), 'write_weightings': utility.pack_into_tensor(final_results[4], axis=1), 'write_vectors': utility.pack_into_tensor(final_results[6], axis=1), 'key_vectors': utility.pack_into_tensor(final_results[7], axis=1), 'beta_vectors': utility.pack_into_tensor(final_results[8], axis=1), 'shift_vectors': utility.pack_into_tensor(final_results[9], axis=1), 'gamma_vectors': utility.pack_into_tensor(final_results[10], axis=1), 'gates_vectors': utility.pack_into_tensor(final_results[11], axis=1), 'memory_vectors': utility.pack_into_tensor(final_results[12], axis=1) }
