我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用tensorflow.einsum()。
def calculate_loss(self, predictions, labels, weights=None, **unused_params): with tf.name_scope("loss_xent"): epsilon = 10e-6 if FLAGS.label_smoothing: float_labels = smoothing(labels) else: float_labels = tf.cast(labels, tf.float32) cross_entropy_loss = float_labels * tf.log(predictions + epsilon) + ( 1 - float_labels) * tf.log(1 - predictions + epsilon) cross_entropy_loss = tf.negative(cross_entropy_loss) if weights is not None: print cross_entropy_loss, weights weighted_loss = tf.einsum("ij,i->ij", cross_entropy_loss, weights) print "create weighted_loss", weighted_loss return tf.reduce_mean(tf.reduce_sum(weighted_loss, 1)) else: return tf.reduce_mean(tf.reduce_sum(cross_entropy_loss, 1))
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] original_input = tf.nn.l2_normalize(original_input, dim=1) gate_activations = slim.fully_connected( original_input, num_methods, activation_fn=tf.nn.softmax, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates"+sub_scope) output = tf.einsum("ijk,ik->ij", model_input, gate_activations) return {"predictions": output}
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params): """Creates a matrix regression model. Args: model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" num_features = model_input.get_shape().as_list()[-2] num_methods = model_input.get_shape().as_list()[-1] weight1d = tf.get_variable("ensemble_weight1d", shape=[num_methods], regularizer=slim.l2_regularizer(l2_penalty)) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_features, num_methods], regularizer=slim.l2_regularizer(10 * l2_penalty)) weight = tf.nn.softmax(tf.einsum("ij,j->ij", weight2d, weight1d), dim=-1) output = tf.einsum("ijk,jk->ij", model_input, weight) return {"predictions": output}
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, **unused_params): """Creates a linear regression model. Args: model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" num_methods = model_input.get_shape().as_list()[-1] weight = tf.get_variable("ensemble_weight", shape=[num_methods], regularizer=slim.l2_regularizer(l2_penalty)) weight = tf.nn.softmax(weight) output = tf.einsum("ijk,k->ij", model_input, weight) return {"predictions": output}
def create_model(self, model_input, vocab_size, l2_penalty=1e-8, original_input=None, epsilon=1e-5, **unused_params): """Creates a non-unified matrix regression model. Args: model_input: 'batch' x 'num_features' x 'num_methods' matrix of input features. vocab_size: The number of classes in the dataset. Returns: A dictionary with a tensor containing the probability predictions of the model in the 'predictions' key. The dimensions of the tensor are batch_size x num_classes.""" num_features = model_input.get_shape().as_list()[-2] num_methods = model_input.get_shape().as_list()[-1] log_model_input = tf.stop_gradient(tf.log((epsilon + model_input) / (1.0 + epsilon - model_input))) weight = tf.get_variable("ensemble_weight", shape=[num_features, num_methods], regularizer=slim.l2_regularizer(l2_penalty)) weight = tf.nn.softmax(weight) output = tf.nn.sigmoid(tf.einsum("ijk,jk->ij", log_model_input, weight)) return {"predictions": output}
def update_link_matrix(self, link_matrix_old, precedence_weighting_old, write_weighting): """ Updating the link matrix takes some effort (in order to vectorize the implementation) Instead of the original index-by-index operation, it's all done at once. :param link_matrix_old: from previous time step, shape [batch_size, memory_size, memory_size] :param precedence_weighting_old: from previous time step, shape [batch_size, memory_size] :param write_weighting: from current time step, shape [batch_size, memory_size] :return: updated link matrix """ expanded = tf.expand_dims(write_weighting, axis=2) # vectorizing the paper's original implementation w = tf.tile(expanded, [1, 1, self.memory_size]) # shape [batch_size, memory_size, memory_size] # shape of w_transpose is the same: [batch_size, memory_size, memory_size] w_transp = tf.tile(tf.transpose(expanded, [0, 2, 1]), [1, self.memory_size, 1]) # in einsum, m and n are the same dimension because tensorflow doesn't support duplicated subscripts. Why? lm = (1 - w - w_transp) * link_matrix_old + tf.einsum("bn,bm->bmn", precedence_weighting_old, write_weighting) lm *= (1 - tf.eye(self.memory_size, batch_shape=[self.batch_size])) # making sure self links are off return tf.identity(lm, name="Link_matrix")
def bilinear_answer_layer(size, encoded_question, question_length, encoded_support, support_length, support2question, answer2support, is_eval, beam_size=1, max_span_size=10000): """Answer layer for multiple paragraph QA.""" # computing single time attention over question size = encoded_support.get_shape()[-1].value question_state = compute_question_state(encoded_question, question_length) # compute logits hidden = tf.gather(tf.layers.dense(question_state, 2 * size, name="hidden"), support2question) hidden_start, hidden_end = tf.split(hidden, 2, 1) support_mask = misc.mask_for_lengths(support_length) start_scores = tf.einsum('ik,ijk->ij', hidden_start, encoded_support) start_scores = start_scores + support_mask end_scores = tf.einsum('ik,ijk->ij', hidden_end, encoded_support) end_scores = end_scores + support_mask return compute_spans(start_scores, end_scores, answer2support, is_eval, support2question, beam_size, max_span_size)
def compute_energy(hidden, state, attn_size, attn_keep_prob=None, pervasive_dropout=False, layer_norm=False, mult_attn=False, **kwargs): if attn_keep_prob is not None: state_noise_shape = [1, tf.shape(state)[1]] if pervasive_dropout else None state = tf.nn.dropout(state, keep_prob=attn_keep_prob, noise_shape=state_noise_shape) hidden_noise_shape = [1, 1, tf.shape(hidden)[2]] if pervasive_dropout else None hidden = tf.nn.dropout(hidden, keep_prob=attn_keep_prob, noise_shape=hidden_noise_shape) if mult_attn: state = dense(state, attn_size, use_bias=False, name='state') hidden = dense(hidden, attn_size, use_bias=False, name='hidden') return tf.einsum('ijk,ik->ij', hidden, state) else: y = dense(state, attn_size, use_bias=not layer_norm, name='W_a') y = tf.expand_dims(y, axis=1) if layer_norm: y = tf.contrib.layers.layer_norm(y, scope='layer_norm_state') hidden = tf.contrib.layers.layer_norm(hidden, center=False, scope='layer_norm_hidden') f = dense(hidden, attn_size, use_bias=False, name='U_a') v = get_variable('v_a', [attn_size]) s = f + y return tf.reduce_sum(v * tf.tanh(s), axis=2)
def _linear(t_in, n_out): v_w = tf.get_variable( "w", shape=(t_in.get_shape()[-1], n_out), initializer=tf.uniform_unit_scaling_initializer( factor=INIT_SCALE)) v_b = tf.get_variable( "b", shape=n_out, initializer=tf.constant_initializer(0)) if len(t_in.get_shape()) == 2: return tf.einsum("ij,jk->ik", t_in, v_w) + v_b elif len(t_in.get_shape()) == 3: return tf.einsum("ijk,kl->ijl", t_in, v_w) + v_b else: assert False
def apply(self, is_train, x, mask=None): if self.key_mapper is not None: with tf.variable_scope("map_keys"): keys = self.key_mapper.apply(is_train, x, mask) else: keys = x weights = tf.get_variable("weights", keys.shape.as_list()[-1], dtype=tf.float32, initializer=get_keras_initialization(self.init)) dist = tf.tensordot(keys, weights, axes=[[2], [0]]) # (batch, x_words) dist = exp_mask(dist, mask) dist = tf.nn.softmax(dist) out = tf.einsum("ajk,aj->ak", x, dist) # (batch, x_dim) if self.post_process is not None: with tf.variable_scope("post_process"): out = self.post_process.apply(is_train, out) return out
def apply(self, is_train, x, mask=None): if self.key_mapper is not None: with tf.variable_scope("map_keys"): keys = self.key_mapper.apply(is_train, x, mask) else: keys = x weights = tf.get_variable("weights", (keys.shape.as_list()[-1], self.n_encodings), dtype=tf.float32, initializer=get_keras_initialization(self.init)) dist = tf.tensordot(keys, weights, axes=[[2], [0]]) # (batch, x_words, n_encoding) if self.bias: dist += tf.get_variable("bias", (1, 1, self.n_encodings), dtype=tf.float32, initializer=tf.zeros_initializer()) if mask is not None: bool_mask = tf.expand_dims(tf.cast(tf.sequence_mask(mask, tf.shape(x)[1]), tf.float32), 2) dist = bool_mask * bool_mask + (1 - bool_mask) * VERY_NEGATIVE_NUMBER dist = tf.nn.softmax(dist, dim=1) out = tf.einsum("ajk,ajn->ank", x, dist) # (batch, n_encoding, feature) if self.post_process is not None: with tf.variable_scope("post_process"): out = self.post_process.apply(is_train, out) return out
def ntn(name, lhs, rhs, nr_output_channels, use_bias=True, nonlin=__default_nonlin__, W=None, b=None, param_dtype=__default_dtype__): lhs, rhs= map(O.flatten2, [lhs, rhs]) assert lhs.static_shape[1] is not None and rhs.static_shape[1] is not None W_shape = (lhs.static_shape[1], nr_output_channels, rhs.static_shape[1]) b_shape = (nr_output_channels, ) if W is None: W = tf.contrib.layers.xavier_initializer() W = O.ensure_variable('W', W, shape=W_shape, dtype=param_dtype) if use_bias: if b is None: b = tf.constant_initializer() b = O.ensure_variable('b', b, shape=b_shape, dtype=param_dtype) out = tf.einsum('ia,abc,ic->ib', lhs.tft, W.tft, rhs.tft) if use_bias: out = tf.identity(out + b.add_axis(0), name='bias') out = nonlin(out, name='nonlin') return tf.identity(out, name='out')
def testFlatInnerTTTensbyTTTensBroadcasting(self): # Inner product between two batch TT-tensors with broadcasting. tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=1) tt_2 = initializers.random_tensor_batch((2, 3, 4), batch_size=3) res_actual_1 = ops.flat_inner(tt_1, tt_2) res_actual_2 = ops.flat_inner(tt_2, tt_1) res_desired = tf.einsum('ijk,oijk->o', ops.full(tt_1[0]), ops.full(tt_2)) with self.test_session() as sess: res = sess.run([res_actual_1, res_actual_2, res_desired]) res_actual_1_val, res_actual_2_val, res_desired_val = res self.assertAllClose(res_actual_1_val, res_desired_val) self.assertAllClose(res_actual_2_val, res_desired_val) tt_1 = initializers.random_tensor_batch((2, 3, 4), batch_size=2) with self.assertRaises(ValueError): # The batch_sizes are different. ops.flat_inner(tt_1, tt_2)
def testFullMatrix3d(self): np.random.seed(1) for rank in [1, 2]: a = np.random.rand(3, 2, 3, rank).astype(np.float32) b = np.random.rand(3, rank, 4, 5, rank).astype(np.float32) c = np.random.rand(3, rank, 2, 2).astype(np.float32) tt_cores = (a.reshape(3, 1, 2, 3, rank), b.reshape(3, rank, 4, 5, rank), c.reshape(3, rank, 2, 2, 1)) # Basically do full by hand. desired = np.einsum('oija,oaklb,obpq->oijklpq', a, b, c) desired = desired.reshape((3, 2, 3, 4, 5, 2, 2)) desired = desired.transpose((0, 1, 3, 5, 2, 4, 6)) desired = desired.reshape((3, 2 * 4 * 2, 3 * 5 * 2)) with self.test_session(): tf_mat = TensorTrainBatch(tt_cores) actual = ops.full(tf_mat) self.assertAllClose(desired, actual.eval())
def create_model(self, model_input, vocab_size, num_frames, **unused_params): shape = model_input.get_shape().as_list() frames_sum = tf.reduce_sum(tf.abs(model_input),axis=2) frames_true = tf.ones(tf.shape(frames_sum)) frames_false = tf.zeros(tf.shape(frames_sum)) frames_bool = tf.reshape(tf.where(tf.greater(frames_sum, frames_false), frames_true, frames_false),[-1,shape[1],1]) activation_1 = tf.reduce_max(model_input, axis=1) activation_2 = tf.reduce_sum(model_input*frames_bool, axis=1)/(tf.reduce_sum(frames_bool, axis=1)+1e-6) activation_3 = tf.reduce_min(model_input, axis=1) model_input_1, final_probilities_1 = self.sub_moe(activation_1,vocab_size,scopename="_max") model_input_2, final_probilities_2 = self.sub_moe(activation_2,vocab_size,scopename="_mean") model_input_3, final_probilities_3 = self.sub_moe(activation_3,vocab_size,scopename="_min") final_probilities = tf.stack((final_probilities_1,final_probilities_2,final_probilities_3),axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[shape[2], 3, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) activations = tf.stack((model_input_1, model_input_2, model_input_3), axis=2) weight = tf.nn.softmax(tf.einsum("aij,ijk->ajk", activations, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def cnn(self, model_input, l2_penalty=1e-8, num_filters = [1024, 1024, 1024], filter_sizes = [1,2,3], sub_scope="", **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in range(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) cnn_output = slim.batch_norm( cnn_output, center=True, scale=True, is_training=FLAGS.train, scope=sub_scope+"cluster_bn") return cnn_output, max_frames
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, distill_labels=None, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input cnn_size = FLAGS.cnn_cells num_filters = [cnn_size, cnn_size, cnn_size*2] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale,vocab_size,distill_labels=distill_labels, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, lstm_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input num_filters = [256, 256, 512] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn_gate(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities, axis=1) moe_inputs = tf.stack(moe_inputs, axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size]) result["predictions"] = tf.reduce_mean(final_probilities, axis=1) return result
def cnn(self, model_input, l2_penalty=1e-8, num_filters=[1024,1024,1024], filter_sizes=[1,2,3], sub_scope="", **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in range(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) cnn_output = slim.batch_norm( cnn_output, center=True, scale=True, is_training=FLAGS.train, scope=sub_scope+"cluster_bn") return cnn_output, max_frames
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size = 2 cnn_input = model_input num_filters = [256, 256, 512] filter_sizes = [1, 2, 3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = self.rnn_glu(cnn_output, lstm_size, num_frames, sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale, vocab_size, scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities, axis=1) moe_inputs = tf.stack(moe_inputs, axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1, vocab_size]) result["predictions"] = tf.reduce_mean(final_probilities, axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend lstm_size = FLAGS.lstm_cells pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = LstmMultiscaleModel().cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = LstmMultiscaleModel().rnn(cnn_output,lstm_size, num_frames,sub_scope="rnn%d"%(layer+1)) moe_inputs.append(cnn_multiscale) final_probility = LstmMultiscaleModel().sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) num_frames = tf.maximum(num_frames//pool_size,1) final_probilities = tf.stack(final_probilities, axis=1) moe_inputs = tf.stack(moe_inputs, axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", tf.stop_gradient(moe_inputs), weight2d), dim=1) result = {} result["predictions"] = tf.reduce_sum(tf.stop_gradient(final_probilities)*weight, axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = 10 pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) if layer < 3: num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) else: cnn_input = cnn_output cnn_output, num_t = self.kmax(cnn_input, num_filters=features_size, filter_sizes=num_extend, sub_scope="kmax") cnn_input = tf.reshape(cnn_output,[-1,features_size]) final_probilities = self.sub_moe(cnn_input,vocab_size) final_probilities = tf.reshape(final_probilities,[-1,num_extend,vocab_size]) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", cnn_output, weight2d), dim=1) result = {} result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = self.cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1)) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = tf.reduce_max(cnn_output,axis=1) moe_inputs.append(cnn_multiscale) final_probility = self.sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1)) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", moe_inputs, weight2d), dim=1) result = {} result["prediction_frames"] = tf.reshape(final_probilities,[-1,vocab_size]) result["predictions"] = tf.reduce_sum(final_probilities*weight,axis=1) return result
def create_model(self, model_input, vocab_size, num_frames, l2_penalty=1e-8, **unused_params): num_extend = FLAGS.moe_num_extend num_layers = num_extend pool_size=2 cnn_input = model_input num_filters=[256,256,512] filter_sizes=[1,2,3] features_size = sum(num_filters) final_probilities = [] moe_inputs = [] for layer in range(num_layers): cnn_output, num_t = CnnKmaxModel().cnn(cnn_input, num_filters=num_filters, filter_sizes=filter_sizes, sub_scope="cnn%d"%(layer+1), l2_penalty=0.0) cnn_output = tf.nn.relu(cnn_output) cnn_multiscale = tf.reduce_max(cnn_output,axis=1) moe_inputs.append(cnn_multiscale) final_probility = CnnKmaxModel().sub_moe(cnn_multiscale,vocab_size,scopename="moe%d"%(layer+1), l2_penalty=0.0) final_probilities.append(final_probility) num_t = pool_size*(num_t//pool_size) cnn_output = tf.reshape(cnn_output[:,:num_t,:],[-1,num_t//pool_size,pool_size,features_size]) cnn_input = tf.reduce_max(cnn_output, axis=2) final_probilities = tf.stack(final_probilities,axis=1) moe_inputs = tf.stack(moe_inputs,axis=1) weight2d = tf.get_variable("ensemble_weight2d", shape=[num_extend, features_size, vocab_size], regularizer=slim.l2_regularizer(1.0e-8)) weight = tf.nn.softmax(tf.einsum("aij,ijk->aik", tf.stop_gradient(moe_inputs), weight2d), dim=1) result = {} result["predictions"] = tf.reduce_sum(tf.stop_gradient(final_probilities)*weight, axis=1) return result
def cnn(self, model_input, l2_penalty=1e-8, num_filters=[1024, 1024, 1024], filter_sizes=[1,2,3], sub_scope="", **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in range(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) return cnn_output, max_frames
def cnn(self, model_input, l2_penalty=1e-8, num_filters = [1024, 1024, 1024], filter_sizes = [1,2,3], sub_scope="", **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in xrange(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) return cnn_output
def add_positional_embedding(self, model_input, num_frames, l2_penalty=1e-8): batch_size, max_frames, num_features = model_input.get_shape().as_list() positional_embedding = tf.get_variable("positional_embedding", dtype=tf.float32, shape=[1, max_frames, num_features], initializer=tf.zeros_initializer(), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32) model_input_with_positional_embedding = tf.einsum("ijk,ij->ijk", model_input + positional_embedding, mask) return model_input_with_positional_embedding
def cnn(self, model_input, l2_penalty=1e-8, num_filters = [1024, 1024, 1024], filter_sizes = [1,2,3], sub_scope="", **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in xrange(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable(sub_scope+"cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) cnn_output = slim.batch_norm( cnn_output, center=True, scale=True, is_training=FLAGS.is_training, scope=sub_scope+"cluster_bn") return cnn_output
def matching_matrix(self, model_input, vocab_size, l2_penalty=1e-8, **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] embedding_size = FLAGS.mm_label_embedding model_input = tf.reshape(model_input, [-1, num_features]) frame_relu = slim.fully_connected( model_input, embedding_size, activation_fn=tf.nn.relu, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="mm_relu") frame_activation = slim.fully_connected( frame_relu, embedding_size, activation_fn=tf.nn.tanh, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="mm_activation") label_embedding = tf.get_variable("label_embedding", shape=[vocab_size,embedding_size], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.5), regularizer=slim.l2_regularizer(l2_penalty), trainable=True) mm_matrix = tf.einsum("ik,jk->ij", frame_activation, label_embedding) mm_output = tf.reshape(mm_matrix, [-1,max_frames,vocab_size]) return mm_output
def get_mean_input(self, model_input, num_frames): batch_size, max_frames, num_features = model_input.get_shape().as_list() mask = tf.sequence_mask(lengths=num_frames, maxlen=max_frames, dtype=tf.float32) mean_input = tf.einsum("ijk,ij->ik", model_input, mask) / tf.expand_dims(tf.cast(num_frames, dtype=tf.float32), dim=1) tiled_mean_input = tf.tile(tf.expand_dims(mean_input, dim=1), multiples=[1,max_frames,1]) return tiled_mean_input
def cnn(self, model_input, l2_penalty=1e-8, num_filters = [1024, 1024, 1024], filter_sizes = [1,2,3], **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] shift_inputs = [] for i in xrange(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable("cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) return cnn_output
def cnn(self, model_input, l2_penalty=1e-8, num_filters = [1024, 1024, 1024], filter_sizes = [1,2,3], **unused_params): max_frames = model_input.get_shape().as_list()[1] num_features = model_input.get_shape().as_list()[2] normalize_class = getattr(self, FLAGS.lstm_normalization, self.identical) shift_inputs = [] for i in xrange(max(filter_sizes)): if i == 0: shift_inputs.append(model_input) else: shift_inputs.append(tf.pad(model_input, paddings=[[0,0],[i,0],[0,0]])[:,:max_frames,:]) cnn_outputs = [] for nf, fs in zip(num_filters, filter_sizes): sub_input = tf.concat(shift_inputs[:fs], axis=2) sub_filter = tf.get_variable("cnn-filter-len%d"%fs, shape=[num_features*fs, nf], dtype=tf.float32, initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1), regularizer=tf.contrib.layers.l2_regularizer(l2_penalty)) cnn_outputs.append(tf.einsum("ijk,kl->ijl", sub_input, sub_filter)) cnn_output = tf.concat(cnn_outputs, axis=2) return cnn_output
def frame_mean(self, model_input, frame_start, frame_end, **unused_params): max_frames = model_input.shape.as_list()[-2] frame_start = tf.cast(frame_start, tf.int32) frame_end = tf.cast(frame_end, tf.int32) frame_length = tf.expand_dims(tf.cast(frame_end - frame_start, tf.float32), axis=1) frame_mask = tf.sequence_mask(frame_end, maxlen=max_frames, dtype=tf.float32) \ - tf.sequence_mask(frame_start, maxlen=max_frames, dtype=tf.float32) mean_frame = tf.einsum("ijk,ij->ik", model_input, frame_mask) / (0.1 + frame_length) return mean_frame
def avg(self, model_input_raw, num_frames, mask): max_frames = model_input_raw.get_shape().as_list()[1] num_frames_matrix = tf.maximum(tf.cast( tf.expand_dims(num_frames, axis=1), dtype=tf.float32), 1.0) mean_matrix = mask / num_frames_matrix mean_input = tf.einsum("ijk,ij->ik", model_input_raw, mean_matrix) mean_input_tile = tf.tile(tf.expand_dims(mean_input, axis=1), multiples=[1,max_frames,1]) return mean_input_tile
def std(self, model_input_raw, num_frames, mask): mean_input = self.avg(model_input_raw, num_frames, mask) error = tf.einsum("ijk,ij->ijk", model_input_raw - mean_input, mask) return error
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_relu = FLAGS.attention_relu_cells num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] original_input = tf.nn.l2_normalize(original_input, dim=1) model_input_list = tf.unstack(model_input, axis=2) relu_units = [self.relu(original_input, num_relu, sub_scope="input")] i = 0 for mi in model_input_list: relu_units.append(self.relu(mi, num_relu, sub_scope="sub"+str(i))) i += 1 gate_activations = slim.fully_connected( tf.concat(relu_units, axis=1), num_methods, activation_fn=None, biases_initializer=None, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gate") gate = tf.nn.softmax(gate_activations) output = tf.einsum("ijk,ik->ij", model_input, gate) return {"predictions": output}
def create_model(self, model_input, vocab_size, num_mixtures=None, l2_penalty=1e-8, sub_scope="", original_input=None, **unused_params): num_methods = model_input.get_shape().as_list()[-1] num_features = model_input.get_shape().as_list()[-2] num_mixtures = FLAGS.moe_num_mixtures # gating coefficients original_input = tf.nn.l2_normalize(original_input, dim=1) mean_output = tf.reduce_mean(model_input, axis=2) ## batch_size x moe_num_mixtures gate_activations = slim.fully_connected( tf.concat([original_input, mean_output], axis=1), num_mixtures, activation_fn=tf.nn.softmax, weights_regularizer=slim.l2_regularizer(l2_penalty), scope="gates"+sub_scope) # matrix weight_var = tf.get_variable("ensemble_weight", shape=[num_mixtures, num_methods], regularizer=slim.l2_regularizer(l2_penalty)) # weight gated_weight = tf.einsum("ij,jk->ik", gate_activations, weight_var) rl_gated_weight = tf.nn.relu(gated_weight) + 1e-9 sum_gated_weight = tf.reduce_sum(rl_gated_weight, axis=1, keep_dims=True) weight = rel_gated_weight / sum_gated_weight # weighted output output = tf.einsum("ik,ijk->ij", weight, model_input) return {"predictions": output}
