我们从Python开源项目中，提取了以下18个代码示例，用于说明如何使用tensorflow.python.ops.rnn_cell.MultiRNNCell()。
def createRNN(self): with self.sess.graph.as_default(): self.prob = tf.placeholder("float", name="keep_prob") # input layer # with tf.name_scope("input"): self.s = tf.placeholder("float", [None, DAYS_RANGE, INPUT_DIM], name='input_state') s_tran = tf.transpose(self.s, [1, 0, 2]) s_re = tf.reshape(s_tran, [-1, INPUT_DIM]) s_list = tf.split(0, DAYS_RANGE, s_re) ## split s to DAYS_RANGE tensor of shape [BATCH, INPUT_DIM] lstm_cell = rnn_cell.LSTMCell(1024, use_peepholes=True, forget_bias=1.0, state_is_tuple=True) lstm_drop = rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=self.prob) lstm_stack = rnn_cell.MultiRNNCell([lstm_cell]*3, state_is_tuple=True) lstm_output, hidden_states = rnn.rnn(lstm_stack, s_list, dtype='float', scope='LSTMStack') # out: [timestep, batch, hidden], state: [cell, c+h, batch, hidden] h_fc1 = self.FC_layer(lstm_output[-1], [1024, 1024], name='h_fc1', activate=True) h_fc1_d = tf.nn.dropout(h_fc1, keep_prob=self.prob, name='h_fc1_drop') h_fc2 = self.FC_layer(h_fc1_d, [1024, ACTIONS], name='h_fc2', activate=False) # output layer # self.pred_action = tf.nn.softmax(h_fc2)
def build_encoder_cell (self): return MultiRNNCell([self.build_single_cell() for i in range(self.depth)]) # Building decoder cell and attention. Also returns decoder_initial_state
def _build_pre(self): self.dimH = 20 self.cellH = MultiRNNCell([LSTMCell(self.dimH)] * 2) self.lr = 0.1
def _build_pre(self): self.dimA = 20 self.cellA = MultiRNNCell([LSTMCell(self.dimA)] * 2) self.b1 = 0.95 self.b2 = 0.95 self.lr = 0.1 self.eps = 1e-8
def save_variables_list(self): # Return a list of the trainable variables created within the rnnlm model. # This consists of the two projection softmax variables (softmax_w and softmax_b), # embedding, and all of the weights and biases in the MultiRNNCell model. # Save only the trainable variables and the placeholders needed to resume training; # discard the rest, including optimizer state. save_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='rnnlm') save_vars.append(self.lr) save_vars.append(self.global_epoch_fraction) save_vars.append(self.global_seconds_elapsed) return save_vars
def build_graph(self): with tf.variable_scope('lstm'): lstm_cell = LSTMCell(self.layer_size) rnn_cell = MultiRNNCell([lstm_cell] * self.layers) cell_output, self.init_state = rnn_cell(self.model_input, self.init_state) print("%i layers created" % self.layers) self.output_layer = self.__add_output_layer("fc_out", cell_output, self.layer_size, self.output_dim) self.output_layer = tf.Print(self.output_layer, [self.output_layer, tf.convert_to_tensor(self.ground_truth)], 'Value of output layer and ground truth:', summarize=6) tf.histogram_summary('lstm_output', self.output_layer) return self.output_layer
def _get_rnn_cell(cell_type, num_units, num_layers): """Constructs and return an `RNNCell`. Args: cell_type: either a string identifying the `RNNCell` type, or a subclass of `RNNCell`. num_units: the number of units in the `RNNCell`. num_layers: the number of layers in the RNN. Returns: An initialized `RNNCell`. Raises: ValueError: `cell_type` is an invalid `RNNCell` name. TypeError: `cell_type` is not a string or a subclass of `RNNCell`. """ if isinstance(cell_type, str): cell_type = _CELL_TYPES.get(cell_type) if cell_type is None: raise ValueError('The supported cell types are {}; got {}'.format( list(_CELL_TYPES.keys()), cell_type)) if not issubclass(cell_type, rnn_cell.RNNCell): raise TypeError( 'cell_type must be a subclass of RNNCell or one of {}.'.format( list(_CELL_TYPES.keys()))) cell = cell_type(num_units=num_units) if num_layers > 1: cell = rnn_cell.MultiRNNCell( [cell] * num_layers, state_is_tuple=True) return cell
def _to_rnn_cell(cell_or_type, num_units, num_layers): """Constructs and return an `RNNCell`. Args: cell_or_type: Either a string identifying the `RNNCell` type, a subclass of `RNNCell` or an instance of an `RNNCell`. num_units: The number of units in the `RNNCell`. num_layers: The number of layers in the RNN. Returns: An initialized `RNNCell`. Raises: ValueError: `cell_or_type` is an invalid `RNNCell` name. TypeError: `cell_or_type` is not a string or a subclass of `RNNCell`. """ if isinstance(cell_or_type, rnn_cell.RNNCell): return cell_or_type if isinstance(cell_or_type, str): cell_or_type = _CELL_TYPES.get(cell_or_type) if cell_or_type is None: raise ValueError('The supported cell types are {}; got {}'.format( list(_CELL_TYPES.keys()), cell_or_type)) if not issubclass(cell_or_type, rnn_cell.RNNCell): raise TypeError( 'cell_or_type must be a subclass of RNNCell or one of {}.'.format( list(_CELL_TYPES.keys()))) cell = cell_or_type(num_units=num_units) if num_layers > 1: cell = rnn_cell.MultiRNNCell( [cell] * num_layers, state_is_tuple=True) return cell
def __init__(self, vocabularySize, config_param): self.vocabularySize = vocabularySize self.config = config_param self._inputX = tf.placeholder(tf.int32, [self.config.batch_size, self.config.sequence_size], "InputsX") self._inputTargetsY = tf.placeholder(tf.int32, [self.config.batch_size, self.config.sequence_size], "InputTargetsY") #Converting Input in an Embedded form with tf.device("/cpu:0"): #Tells Tensorflow what GPU to use specifically embedding = tf.get_variable("embedding", [self.vocabularySize, self.config.embeddingSize]) embeddingLookedUp = tf.nn.embedding_lookup(embedding, self._inputX) inputs = tf.split(1, self.config.sequence_size, embeddingLookedUp) inputTensorsAsList = [tf.squeeze(input_, [1]) for input_ in inputs] #Define Tensor RNN singleRNNCell = rnn_cell.BasicRNNCell(self.config.hidden_size) self.multilayerRNN = rnn_cell.MultiRNNCell([singleRNNCell] * self.config.num_layers) self._initial_state = self.multilayerRNN.zero_state(self.config.batch_size, tf.float32) #Defining Logits hidden_layer_output, last_state = rnn.rnn(self.multilayerRNN, inputTensorsAsList, initial_state=self._initial_state) hidden_layer_output = tf.reshape(tf.concat(1, hidden_layer_output), [-1, self.config.hidden_size]) self._logits = tf.nn.xw_plus_b(hidden_layer_output, tf.get_variable("softmax_w", [self.config.hidden_size, self.vocabularySize]), tf.get_variable("softmax_b", [self.vocabularySize])) self._predictionSoftmax = tf.nn.softmax(self._logits) #Define the loss loss = seq2seq.sequence_loss_by_example([self._logits], [tf.reshape(self._inputTargetsY, [-1])], [tf.ones([self.config.batch_size * self.config.sequence_size])], self.vocabularySize) self._cost = tf.div(tf.reduce_sum(loss), self.config.batch_size) self._final_state = last_state
def __init__(self, args): self.args = args if args.disc_model == 'rnn': cell_fn = rnn_cell.BasicRNNCell elif args.disc_model == 'gru': cell_fn = rnn_cell.GRUCell elif args.disc_model == 'lstm': cell_fn = rnn_cell.BasicLSTMCell else: raise Exception("model type not supported: {}".format(args.model)) self.embedding = tf.Variable(tf.random_uniform([self.args.vocab_size, self.args.rnn_size], minval=-.05, maxval=.05, dtype=tf.float32), name='embedding') with tf.variable_scope('DISC') as scope: cell = cell_fn(args.rnn_size) self.cell = cell = rnn_cell.MultiRNNCell([cell] * args.num_layers) # If the input data is given as word tokens, feed this value self.input_data_text = tf.placeholder(tf.int32, [args.batch_size, args.seq_length], name='input_data_text') #self.input_data_text = tf.Variable(tf.zeros((args.batch_size, args.seq_length), dtype=tf.int32), name='input_data_text') self.initial_state = cell.zero_state(args.batch_size, tf.float32) # Fully connected layer is applied to the final state to determine the output class self.fc_layer = tf.Variable(tf.random_normal([args.rnn_size, 1], stddev=0.35, dtype=tf.float32), name='disc_fc_layer') self.lr = tf.Variable(0.0, trainable=False, name='learning_rate') self.has_init_seq2seq = False
def __init__(self, args): self.args = args if args.gen_model == 'rnn': cell_fn = rnn_cell.BasicRNNCell elif args.gen_model == 'gru': cell_fn = rnn_cell.GRUCell elif args.gen_model == 'lstm': cell_fn = rnn_cell.BasicLSTMCell else: raise Exception("model type not supported: {}".format(args.model)) with tf.variable_scope('GEN') as scope: cell = cell_fn(args.rnn_size) self.cell = cell = rnn_cell.MultiRNNCell([cell] * args.num_layers) # sequence of word tokens taken as input self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length], name='input_data') self.latent_state = tf.placeholder(tf.float32, [args.batch_size, args.latent_size]) # weights to map the latent state into the (usually) bigger initial state # right now this only works for rnn (other more complex models have more than # one initial state which needs to be given a value) # Right now we support up to two layers (state1 and state2) self.latent_to_initial_state1 = tf.Variable(tf.random_normal([args.latent_size, args.rnn_size], stddev=0.35, dtype=tf.float32), name='latent_to_intial_state1') self.latent_to_initial_state2 = tf.Variable(tf.random_normal([args.latent_size, args.rnn_size], stddev=0.35, dtype=tf.float32), name='latent_to_intial_state2') self.initial_state1 = tf.matmul(self.latent_state, self.latent_to_initial_state1) self.initial_state2 = tf.matmul(self.latent_state, self.latent_to_initial_state2) # these are the actual approximate word vectors generated by the model self.outputs = tf.placeholder(tf.float32, [args.seq_length, args.batch_size, args.rnn_size]) self.lr = tf.Variable(0.0, trainable=False, name='learning_rate') self.has_init_seq2seq = False
def __init__(self, args, data, infer=False): if infer: args.batch_size = 1 args.seq_length = 1 with tf.name_scope('inputs'): self.input_data = tf.placeholder( tf.int32, [args.batch_size, args.seq_length]) self.target_data = tf.placeholder( tf.int32, [args.batch_size, args.seq_length]) with tf.name_scope('model'): self.cell = rnn_cell.BasicLSTMCell(args.state_size) self.cell = rnn_cell.MultiRNNCell([self.cell] * args.num_layers) self.initial_state = self.cell.zero_state( args.batch_size, tf.float32) with tf.variable_scope('rnnlm'): w = tf.get_variable( 'softmax_w', [args.state_size, data.vocab_size]) b = tf.get_variable('softmax_b', [data.vocab_size]) with tf.device("/cpu:0"): embedding = tf.get_variable( 'embedding', [data.vocab_size, args.state_size]) inputs = tf.nn.embedding_lookup(embedding, self.input_data) outputs, last_state = tf.nn.dynamic_rnn( self.cell, inputs, initial_state=self.initial_state) with tf.name_scope('loss'): output = tf.reshape(outputs, [-1, args.state_size]) self.logits = tf.matmul(output, w) + b self.probs = tf.nn.softmax(self.logits) self.last_state = last_state targets = tf.reshape(self.target_data, [-1]) loss = seq2seq.sequence_loss_by_example([self.logits], [targets], [tf.ones_like(targets, dtype=tf.float32)]) self.cost = tf.reduce_sum(loss) / args.batch_size tf.summary.scalar('loss', self.cost) with tf.name_scope('optimize'): self.lr = tf.placeholder(tf.float32, []) tf.summary.scalar('learning_rate', self.lr) optimizer = tf.train.AdamOptimizer(self.lr) tvars = tf.trainable_variables() grads = tf.gradients(self.cost, tvars) for g in grads: tf.summary.histogram(g.name, g) grads, _ = tf.clip_by_global_norm(grads, args.grad_clip) self.train_op = optimizer.apply_gradients(zip(grads, tvars)) self.merged_op = tf.summary.merge_all()
def __init__(self, args, infer=False): self.args = args if infer: args.batch_size = 1 args.seq_length = 1 if args.model == 'rnn': cell_fn = rnn_cell.BasicRNNCell elif args.model == 'gru': cell_fn = rnn_cell.GRUCell elif args.model == 'lstm': cell_fn = rnn_cell.BasicLSTMCell else: raise Exception("model type not supported: {}".format(args.model)) cell = cell_fn(args.rnn_size) self.cell = cell = rnn_cell.MultiRNNCell([cell] * args.num_layers) self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.targets = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.initial_state = cell.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnnlm'): softmax_w = tf.get_variable("softmax_w", [args.rnn_size, args.vocab_size]) softmax_b = tf.get_variable("softmax_b", [args.vocab_size]) with tf.device("/cpu:0"): embedding = tf.get_variable("embedding", [args.vocab_size, args.rnn_size]) inputs = tf.split(1, args.seq_length, tf.nn.embedding_lookup(embedding, self.input_data)) inputs = [tf.squeeze(input_, [1]) for input_ in inputs] def loop(prev, _): prev = tf.matmul(prev, softmax_w) + softmax_b prev_symbol = tf.stop_gradient(tf.argmax(prev, 1)) return tf.nn.embedding_lookup(embedding, prev_symbol) outputs, last_state = seq2seq.rnn_decoder(inputs, self.initial_state, cell, loop_function=loop if infer else None, scope='rnnlm') output = tf.reshape(tf.concat(1, outputs), [-1, args.rnn_size]) self.logits = tf.matmul(output, softmax_w) + softmax_b self.probs = tf.nn.softmax(self.logits) loss = seq2seq.sequence_loss_by_example([self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([args.batch_size * args.seq_length])], args.vocab_size) self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length self.final_state = last_state self.lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), args.grad_clip) optimizer = tf.train.AdamOptimizer(self.lr) self.train_op = optimizer.apply_gradients(zip(grads, tvars))
def __init__(self, args, infer=False): self.args = args if infer: args.batch_size = 1 args.seq_length = 1 if args.rnncell == 'rnn': cell_fn = rnn_cell.BasicRNNCell elif args.rnncell == 'gru': cell_fn = rnn_cell.GRUCell elif args.rnncell == 'lstm': cell_fn = rnn_cell.BasicLSTMCell else: raise Exception("rnncell type not supported: {}".format(args.rnncell)) cell = cell_fn(args.rnn_size) self.cell = rnn_cell.MultiRNNCell([cell] * args.num_layers) self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.targets = tf.placeholder(tf.int32, [args.batch_size, args.seq_length]) self.initial_state = self.cell.zero_state(args.batch_size, tf.float32) with tf.variable_scope('rnnlm'): softmax_w = build_weight([args.rnn_size, args.vocab_size],name='soft_w') softmax_b = build_weight([args.vocab_size],name='soft_b') word_embedding = build_weight([args.vocab_size, args.embedding_size],name='word_embedding') inputs_list = tf.split(1, args.seq_length, tf.nn.embedding_lookup(word_embedding, self.input_data)) inputs_list = [tf.squeeze(input_, [1]) for input_ in inputs_list] def loop(prev, _): prev = tf.matmul(prev, softmax_w) + softmax_b prev_symbol = tf.stop_gradient(tf.argmax(prev, 1)) return tf.nn.embedding_lookup(embedding, prev_symbol) if not args.attention: outputs, last_state = seq2seq.rnn_decoder(inputs_list, self.initial_state, self.cell, loop_function=loop if infer else None, scope='rnnlm') else: self.attn_length = 5 self.attn_size = 32 self.attention_states = build_weight([args.batch_size, self.attn_length, self.attn_size]) outputs, last_state = seq2seq.attention_decoder(inputs_list, self.initial_state, self.attention_states, self.cell, loop_function=loop if infer else None, scope='rnnlm') self.final_state = last_state output = tf.reshape(tf.concat(1, outputs), [-1, args.rnn_size]) self.logits = tf.matmul(output, softmax_w) + softmax_b self.probs = tf.nn.softmax(self.logits) loss = seq2seq.sequence_loss_by_example([self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([args.batch_size * args.seq_length])], args.vocab_size) # average loss for each word of each timestep self.cost = tf.reduce_sum(loss) / args.batch_size / args.seq_length self.lr = tf.Variable(0.0, trainable=False) self.var_trainable_op = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, self.var_trainable_op), args.grad_clip) optimizer = tf.train.AdamOptimizer(self.lr) self.train_op = optimizer.apply_gradients(zip(grads, self.var_trainable_op)) self.initial_op = tf.initialize_all_variables() self.saver = tf.train.Saver(tf.all_variables(),max_to_keep=5,keep_checkpoint_every_n_hours=1) self.logfile = args.log_dir+str(datetime.datetime.strftime(datetime.datetime.now(),'%Y-%m-%d %H:%M:%S')+'.txt').replace(' ','').replace('/','') self.var_op = tf.all_variables()
def createMultiRNN(self, n_layer, n_hidden): with self.sess.graph.as_default(): self.prob = tf.placeholder("float", name="keep_prob") # input # with tf.name_scope('input'): self.s = tf.placeholder('float', shape=[None, INPUT_DIM, DAYS_RANGE], name='input_state') input_trans = tf.transpose(self.s, [2, 0, 1]) # [DAYS_RANGE, None, INPUT_DIM] input_reshape = tf.reshape(input_trans, [-1, INPUT_DIM]) input_list = tf.split(0, DAYS_RANGE, input_reshape) # split to DAY_RANGE element with tf.name_scope('tg_input'): self.target_s = tf.placeholder('float', shape=[None, INPUT_DIM, DAYS_RANGE], name='input_state') tg_input_trans = tf.transpose(self.target_s, [2, 0, 1]) # [DAYS_RANGE, None, INPUT_DIM] tg_input_reshape = tf.reshape(tg_input_trans, [-1, INPUT_DIM]) tg_input_list = tf.split(0, DAYS_RANGE, tg_input_reshape) # split to DAY_RANGE element # multi LSTM # lstm_cell = rnn_cell.LSTMCell(n_hidden, use_peepholes=True, forget_bias=1.0, state_is_tuple=True) lstm_drop = rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=self.prob) lstm_stack = rnn_cell.MultiRNNCell([lstm_drop] * n_layer, state_is_tuple=True) tg_lstm_cell = rnn_cell.LSTMCell(n_hidden, use_peepholes=True, forget_bias=1.0, state_is_tuple=True) tg_lstm_drop = rnn_cell.DropoutWrapper(tg_lstm_cell, output_keep_prob=self.prob) tg_lstm_stack = rnn_cell.MultiRNNCell([tg_lstm_drop] * n_layer, state_is_tuple=True) lstm_output, hidden_states = rnn.rnn(lstm_stack, input_list, dtype='float', scope='LSTMStack') # out: [timestep, batch, hidden], state: [cell, 2(for c, h), batch, hidden] tg_lstm_output, tg_hidden_states = rnn.rnn(tg_lstm_stack, tg_input_list, dtype='float', scope='tg_LSTMStack') for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope="LSTMStack"): tf.add_to_collection("L2_VARIABLES", var) h_fc1 = self.FC_layer(lstm_output[-1], tg_lstm_output[-1], [n_hidden, 1024], name='h_fc1', activate=True) h_fc2 = self.FC_layer(h_fc1[0], h_fc1[1], [1024, ACTIONS], name='h_fc2', activate=False) key = tf.GraphKeys.TRAINABLE_VARIABLES update_pair = zip(tf.get_collection(key, scope="LSTMStack"), tf.get_collection(key, scope="tg_LSTMStack")) for var, tg_var in update_pair: self.update_list.append(tg_var.assign(var)) # readout layer self.readout = h_fc2[0] self.target_readout = h_fc2[1]
def __init__(self, args, embedding): self.args = args if args.model == 'rnn': cell_fn = rnn_cell.BasicRNNCell elif args.model == 'gru': cell_fn = rnn_cell.GRUCell elif args.model == 'lstm': cell_fn = rnn_cell.BasicLSTMCell else: raise Exception("model type not supported: {}".format(args.model)) cell = cell_fn(args.rnn_size) self.cell = cell = rnn_cell.MultiRNNCell([cell] * args.num_layers) self.input_data = tf.placeholder(tf.int32, [args.batch_size, args.seq_length], name='STAND_input') self.targets = tf.placeholder(tf.int32, [args.batch_size, args.seq_length], name='STAND_targets') self.initial_state = cell.zero_state(args.batch_size, tf.float32) self.embedding = embedding with tf.variable_scope('STAND'): softmax_w = tf.get_variable("softmax_w", [args.rnn_size, args.vocab_size]) softmax_b = tf.get_variable("softmax_b", [args.vocab_size]) inputs = tf.split(1, args.seq_length, tf.nn.embedding_lookup(self.embedding, self.input_data)) inputs = map(lambda i: tf.nn.l2_normalize(i, 1), [tf.squeeze(input_, [1]) for input_ in inputs]) def loop(prev, i): prev = tf.matmul(prev, softmax_w) + softmax_b prev_symbol = tf.stop_gradient(tf.argmax(prev, 1)) return tf.nn.l2_normalize(tf.nn.embedding_lookup(embedding, prev_symbol), 1) o, _ = seq2seq.rnn_decoder(inputs, self.initial_state, cell, loop_function=None, scope='STAND') with tf.variable_scope('STAND', reuse=True) as scope: sf_o, _ = seq2seq.rnn_decoder(inputs, self.initial_state, cell, loop_function=loop, scope=scope) output = tf.reshape(tf.concat(1, o), [-1, args.rnn_size]) self.logits = tf.matmul(output, softmax_w) + softmax_b self.probs = tf.nn.softmax(self.logits) sf_output = tf.reshape(tf.concat(1, sf_o), [-1, args.rnn_size]) self_feed_logits = tf.matmul(sf_output, softmax_w) + softmax_b self.self_feed_probs = tf.nn.softmax(self_feed_logits) loss = seq2seq.sequence_loss_by_example([self.logits], [tf.reshape(self.targets, [-1])], [tf.ones([args.batch_size * args.seq_length])], args.vocab_size) self.loss = tf.reduce_sum(loss) / args.batch_size / args.seq_length self.lr = tf.Variable(0.0, trainable=False) tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars), args.grad_clip) for g, v in zip(grads, tvars): print v.name optimizer = tf.train.AdamOptimizer(self.lr) self.train_op = optimizer.apply_gradients(zip(grads, tvars))
