我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用lasagne.layers.DenseLayer()。
def _set_inverse_parameters(self, patterns=None): self.trainable_layers = [self.inverse_map[l] for l in L.get_all_layers(self.output_layer) if type(l) in [L.Conv2DLayer, L.DenseLayer]] if patterns is not None: if type(patterns) is list: patterns = patterns[0] for i,layer in enumerate(self.trainable_layers): pattern = patterns['A'][i] if pattern.ndim == 4: pattern = pattern.transpose(1,0,2,3) elif pattern.ndim == 2: pattern = pattern.T layer.W.set_value(pattern) else: print("Patterns not given, explanation is random.")
def exe_rnn(use_embedd, length, num_units, position, binominal): batch_size = BATCH_SIZE input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) target_var = T.ivector(name='targets') layer_input = lasagne.layers.InputLayer(shape=(None, length, 1), input_var=input_var, name='input') if use_embedd: layer_position = construct_position_input(batch_size, length, num_units) layer_input = lasagne.layers.concat([layer_input, layer_position], axis=2) layer_rnn = RecurrentLayer(layer_input, num_units, nonlinearity=nonlinearities.tanh, only_return_final=True, W_in_to_hid=lasagne.init.GlorotUniform(), W_hid_to_hid=lasagne.init.GlorotUniform(), b=lasagne.init.Constant(0.), name='RNN') # W = layer_rnn.W_hid_to_hid.sum() # U = layer_rnn.W_in_to_hid.sum() # b = layer_rnn.b.sum() layer_output = DenseLayer(layer_rnn, num_units=1, nonlinearity=nonlinearities.sigmoid, name='output') return train(layer_output, layer_rnn, input_var, target_var, batch_size, length, position, binominal)
def build_critic(input_var=None): from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer, DenseLayer) try: from lasagne.layers.dnn import batch_norm_dnn as batch_norm except ImportError: from lasagne.layers import batch_norm from lasagne.nonlinearities import LeakyRectify lrelu = LeakyRectify(0.2) # input: (None, 1, 28, 28) layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var) # two convolutions layer = batch_norm(Conv2DLayer(layer, 64, 5, stride=2, pad='same', nonlinearity=lrelu)) layer = batch_norm(Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu)) # fully-connected layer layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu)) # output layer (linear) layer = DenseLayer(layer, 1, nonlinearity=None) print ("critic output:", layer.output_shape) return layer
def build_critic(input_var=None, verbose=False): from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer, DenseLayer) try: from lasagne.layers.dnn import batch_norm_dnn as batch_norm except ImportError: from lasagne.layers import batch_norm from lasagne.nonlinearities import LeakyRectify, sigmoid lrelu = LeakyRectify(0.2) # input: (None, 1, 28, 28) layer = InputLayer(shape=(None, 3, 32, 32), input_var=input_var) # two convolutions layer = batch_norm(Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu)) layer = batch_norm(Conv2DLayer(layer, 256, 5, stride=2, pad='same', nonlinearity=lrelu)) layer = batch_norm(Conv2DLayer(layer, 512, 5, stride=2, pad='same', nonlinearity=lrelu)) # # fully-connected layer # layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu)) # output layer (linear) layer = DenseLayer(layer, 1, nonlinearity=None) if verbose: print ("critic output:", layer.output_shape) return layer
def build_critic(input_var=None): from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer, DenseLayer) try: from lasagne.layers.dnn import batch_norm_dnn as batch_norm except ImportError: from lasagne.layers import batch_norm from lasagne.nonlinearities import LeakyRectify lrelu = LeakyRectify(0.2) # input: (None, 1, 28, 28) layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_var) # two convolutions layer = batch_norm(Conv2DLayer(layer, 64, 5, stride=2, pad='same', nonlinearity=lrelu)) layer = batch_norm(Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=lrelu)) # fully-connected layer layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu)) # output layer (linear and without bias) layer = DenseLayer(layer, 1, nonlinearity=None, b=None) print ("critic output:", layer.output_shape) return layer
def build_mlp(input_var=None): l_in = InputLayer(shape=(None, 1, 28, 28), input_var=input_var) l_hid1 = DenseLayer( l_in, num_units=500, nonlinearity=rectify, W=lasagne.init.GlorotUniform()) l_hid1_drop = DropoutLayer(l_hid1, p=0.4) l_hid2 = DenseLayer( l_hid1_drop, num_units=300, nonlinearity=rectify) l_hid2_drop = DropoutLayer(l_hid2, p=0.4) l_out = DenseLayer( l_hid2_drop, num_units=10, nonlinearity=softmax) return l_out # generator giving the batches
def build_model(input_var=None): layers = [1, 5, 10, 1] l_in = InputLayer((None, None, layers[0]), input_var=input_var) l_lstm1 = LSTMLayer(l_in, layers[1]) l_lstm1_dropout = DropoutLayer(l_lstm1, p=0.2) l_lstm2 = LSTMLayer(l_lstm1_dropout, layers[2]) l_lstm2_dropout = DropoutLayer(l_lstm2, p=0.2) # The objective of this task depends only on the final value produced by # the network. So, we'll use SliceLayers to extract the LSTM layer's # output after processing the entire input sequence. For the forward # layer, this corresponds to the last value of the second (sequence length) # dimension. l_slice = SliceLayer(l_lstm2_dropout, -1, 1) l_out = DenseLayer(l_slice, 1, nonlinearity=lasagne.nonlinearities.linear) return l_out
def __build_48_net__(self): network = layers.InputLayer((None, 3, 48, 48), input_var=self.__input_var__) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu) network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2) network = layers.batch_norm(network) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu) network = layers.batch_norm(network) network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(3,3),stride=1,nonlinearity=relu) network = layers.batch_norm(network) network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2) network = layers.DenseLayer(network,num_units = 256,nonlinearity = relu) network = layers.DenseLayer(network,num_units = 2, nonlinearity = softmax) return network
def test_lnlstm_nparams_hid_init_layer(): # test that you can see layers through hid_init l_inp = InputLayer((2, 2, 3)) l_inp_h = InputLayer((2, 5)) l_inp_h_de = DenseLayer(l_inp_h, 7) l_inp_cell = InputLayer((2, 5)) l_inp_cell_de = DenseLayer(l_inp_cell, 7) l_lstm = LNLSTMLayer(l_inp, 7, hid_init=l_inp_h_de, cell_init=l_inp_cell_de) # directly check the layers can be seen through hid_init layers_to_find = [l_inp, l_inp_h, l_inp_h_de, l_inp_cell, l_inp_cell_de, l_lstm] assert lasagne.layers.get_all_layers(l_lstm) == layers_to_find # 7*n_gates + 3*n_peepholes + 4 # the 7 is because we have hid_to_gate, in_to_gate and bias and # layer normalization for each gate # 4 is for the W and b parameters in the two DenseLayer layers print lasagne.layers.get_all_params(l_lstm, trainable=True) assert len(lasagne.layers.get_all_params(l_lstm, trainable=True)) == 37 # LSTM bias params(4) + LN betas(2*#gate) (+ Dense bias params(1) * 2 assert len(lasagne.layers.get_all_params(l_lstm, regularizable=False)) == 15
def test_lstm_nparams_hid_init_layer(): # test that you can see layers through hid_init l_inp = InputLayer((2, 2, 3)) l_inp_h = InputLayer((2, 5)) l_inp_h_de = DenseLayer(l_inp_h, 7) l_inp_cell = InputLayer((2, 5)) l_inp_cell_de = DenseLayer(l_inp_cell, 7) l_lstm = LSTMLayer(l_inp, 7, hid_init=l_inp_h_de, cell_init=l_inp_cell_de) # directly check the layers can be seen through hid_init layers_to_find = [l_inp, l_inp_h, l_inp_h_de, l_inp_cell, l_inp_cell_de, l_lstm] assert lasagne.layers.get_all_layers(l_lstm) == layers_to_find # 3*n_gates + 4 # the 3 is because we have hid_to_gate, in_to_gate and bias for each gate # 4 is for the W and b parameters in the two DenseLayer layers assert len(lasagne.layers.get_all_params(l_lstm, trainable=True)) == 19 # GRU bias params(3) + Dense bias params(1) * 2 assert len(lasagne.layers.get_all_params(l_lstm, regularizable=False)) == 6
def network_classifier(self, input_var): network = {} network['classifier/input'] = InputLayer(shape=(None, 3, 64, 64), input_var=input_var, name='classifier/input') network['classifier/conv1'] = Conv2DLayer(network['classifier/input'], num_filters=32, filter_size=3, stride=1, pad='valid', nonlinearity=rectify, name='classifier/conv1') network['classifier/pool1'] = MaxPool2DLayer(network['classifier/conv1'], pool_size=2, stride=2, pad=0, name='classifier/pool1') network['classifier/conv2'] = Conv2DLayer(network['classifier/pool1'], num_filters=32, filter_size=3, stride=1, pad='valid', nonlinearity=rectify, name='classifier/conv2') network['classifier/pool2'] = MaxPool2DLayer(network['classifier/conv2'], pool_size=2, stride=2, pad=0, name='classifier/pool2') network['classifier/conv3'] = Conv2DLayer(network['classifier/pool2'], num_filters=32, filter_size=3, stride=1, pad='valid', nonlinearity=rectify, name='classifier/conv3') network['classifier/pool3'] = MaxPool2DLayer(network['classifier/conv3'], pool_size=2, stride=2, pad=0, name='classifier/pool3') network['classifier/conv4'] = Conv2DLayer(network['classifier/pool3'], num_filters=32, filter_size=3, stride=1, pad='valid', nonlinearity=rectify, name='classifier/conv4') network['classifier/pool4'] = MaxPool2DLayer(network['classifier/conv4'], pool_size=2, stride=2, pad=0, name='classifier/pool4') network['classifier/dense1'] = DenseLayer(network['classifier/pool4'], num_units=64, nonlinearity=rectify, name='classifier/dense1') network['classifier/output'] = DenseLayer(network['classifier/dense1'], num_units=10, nonlinearity=softmax, name='classifier/output') return network
def discriminator(input_var): network = lasagne.layers.InputLayer(shape=(None, 1, 28, 28), input_var=input_var) network = ll.DropoutLayer(network, p=0.5) network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 64, (4,4), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu)) network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), stride=2, pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu)) network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu)) network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 32, (5,5), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu)) network = nn.weight_norm(dnn.Conv2DDNNLayer(network, 16, (3,3), pad='valid', W=Normal(0.05), nonlinearity=nn.lrelu)) network =nn.weight_norm(ll.DenseLayer(network, num_units=1, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1) return network
def generator(input_var): network = lasagne.layers.InputLayer(shape=(None, NLAT,1,1), input_var=input_var) network = ll.DenseLayer(network, num_units=4*4*64, W=Normal(0.05), nonlinearity=nn.relu) #print(input_var.shape[0]) network = ll.ReshapeLayer(network, (batch_size,64,4,4)) network = nn.Deconv2DLayer(network, (batch_size,32,7,7), (4,4), stride=(1,1), pad='valid', W=Normal(0.05), nonlinearity=nn.relu) network = nn.Deconv2DLayer(network, (batch_size,32,11,11), (5,5), stride=(1,1), pad='valid', W=Normal(0.05), nonlinearity=nn.relu) network = nn.Deconv2DLayer(network, (batch_size,32,25,25), (5,5), stride=(2,2), pad='valid', W=Normal(0.05), nonlinearity=nn.relu) network = nn.Deconv2DLayer(network, (batch_size,1,28,28), (4,4), stride=(1,1), pad='valid', W=Normal(0.05), nonlinearity=sigmoid) #network =lasagne.layers.Conv2DLayer(network, num_filters=1, filter_size=1, stride=1, nonlinearity=sigmoid) return network # In[23]:
def build_net(nz=10): # nz = size of latent code #N.B. using batch_norm applies bn before non-linearity! F=32 enc = InputLayer(shape=(None,1,28,28)) enc = Conv2DLayer(incoming=enc, num_filters=F*2, filter_size=5,stride=2, nonlinearity=lrelu(0.2),pad=2) enc = Conv2DLayer(incoming=enc, num_filters=F*4, filter_size=5,stride=2, nonlinearity=lrelu(0.2),pad=2) enc = Conv2DLayer(incoming=enc, num_filters=F*4, filter_size=5,stride=1, nonlinearity=lrelu(0.2),pad=2) enc = reshape(incoming=enc, shape=(-1,F*4*7*7)) enc = DenseLayer(incoming=enc, num_units=nz, nonlinearity=sigmoid) #Generator networks dec = InputLayer(shape=(None,nz)) dec = DenseLayer(incoming=dec, num_units=F*4*7*7) dec = reshape(incoming=dec, shape=(-1,F*4,7,7)) dec = Deconv2DLayer(incoming=dec, num_filters=F*4, filter_size=4, stride=2, nonlinearity=relu, crop=1) dec = Deconv2DLayer(incoming=dec, num_filters=F*4, filter_size=4, stride=2, nonlinearity=relu, crop=1) dec = Deconv2DLayer(incoming=dec, num_filters=1, filter_size=3, stride=1, nonlinearity=sigmoid, crop=1) return enc, dec
def build_rnn(conv_input_var, seq_input_var, conv_shape, word_dims, n_hid, lstm_layers): ret = {} ret['seq_input'] = seq_layer = InputLayer((None, None, word_dims), input_var=seq_input_var) batchsize, seqlen, _ = seq_layer.input_var.shape ret['seq_resh'] = seq_layer = ReshapeLayer(seq_layer, shape=(-1, word_dims)) ret['seq_proj'] = seq_layer = DenseLayer(seq_layer, num_units=n_hid) ret['seq_resh2'] = seq_layer = ReshapeLayer(seq_layer, shape=(batchsize, seqlen, n_hid)) ret['conv_input'] = conv_layer = InputLayer(conv_shape, input_var = conv_input_var) ret['conv_proj'] = conv_layer = DenseLayer(conv_layer, num_units=n_hid) ret['conv_resh'] = conv_layer = ReshapeLayer(conv_layer, shape=([0], 1, -1)) ret['input_concat'] = layer = ConcatLayer([conv_layer, seq_layer], axis=1) for lstm_layer_idx in xrange(lstm_layers): ret['lstm_{}'.format(lstm_layer_idx)] = layer = LSTMLayer(layer, n_hid) ret['out_resh'] = layer = ReshapeLayer(layer, shape=(-1, n_hid)) ret['output_proj'] = layer = DenseLayer(layer, num_units=word_dims, nonlinearity=log_softmax) ret['output'] = layer = ReshapeLayer(layer, shape=(batchsize, seqlen+1, word_dims)) ret['output'] = layer = SliceLayer(layer, indices=slice(None, -1), axis=1) return ret # originally from # https://github.com/Lasagne/Recipes/blob/master/examples/styletransfer/Art%20Style%20Transfer.ipynb
def extract_encoder(dbn): dbn_layers = dbn.get_all_layers() encoder = NeuralNet( layers=[ (InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}), (DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[1].W, 'b': dbn_layers[1].b}), (DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[2].W, 'b': dbn_layers[2].b}), (DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[3].W, 'b': dbn_layers[3].b}), (DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear, 'W': dbn_layers[4].W, 'b': dbn_layers[4].b}), ], update=adadelta, update_learning_rate=0.01, objective_l2=0.005, verbose=1, regression=True ) encoder.initialize() return encoder
def extract_encoder(dbn): dbn_layers = dbn.get_all_layers() encoder = NeuralNet( layers=[ (InputLayer, {'name': 'input', 'shape': dbn_layers[0].shape}), (DenseLayer, {'name': 'l1', 'num_units': dbn_layers[1].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[1].W, 'b': dbn_layers[1].b}), (DenseLayer, {'name': 'l2', 'num_units': dbn_layers[2].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[2].W, 'b': dbn_layers[2].b}), (DenseLayer, {'name': 'l3', 'num_units': dbn_layers[3].num_units, 'nonlinearity': sigmoid, 'W': dbn_layers[3].W, 'b': dbn_layers[3].b}), (DenseLayer, {'name': 'l4', 'num_units': dbn_layers[4].num_units, 'nonlinearity': linear, 'W': dbn_layers[4].W, 'b': dbn_layers[4].b}), ], update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.5, objective_l2=0.005, verbose=1, regression=True ) encoder.initialize() return encoder
def build_encoder_layers(input_size, encode_size, sigma=0.5): """ builds an autoencoder with gaussian noise layer :param input_size: input size :param encode_size: encoded size :param sigma: gaussian noise standard deviation :return: Weights of encoder layer, denoising autoencoder layer """ W = theano.shared(GlorotUniform().sample(shape=(input_size, encode_size))) layers = [ (InputLayer, {'shape': (None, input_size)}), (GaussianNoiseLayer, {'name': 'corrupt', 'sigma': sigma}), (DenseLayer, {'name': 'encoder', 'num_units': encode_size, 'nonlinearity': sigmoid, 'W': W}), (DenseLayer, {'name': 'decoder', 'num_units': input_size, 'nonlinearity': linear, 'W': W.T}), ] return W, layers
def build_model(self): ''' Build Acoustic Event Net model :return: ''' # A architecture 41 classes nonlin = lasagne.nonlinearities.rectify net = {} net['input'] = InputLayer((None, feat_shape[0], feat_shape[1], feat_shape[2])) # channel, time. frequency # ----------- 1st layer group --------------- net['conv1a'] = ConvLayer(net['input'], num_filters=64, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['conv1b'] = ConvLayer(net['conv1a'], num_filters=64, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['pool1'] = MaxPool2DLayer(net['conv1b'], pool_size=(1, 2)) # (time, freq) # ----------- 2nd layer group --------------- net['conv2a'] = ConvLayer(net['pool1'], num_filters=128, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['conv2b'] = ConvLayer(net['conv2a'], num_filters=128, filter_size=(3, 3), stride=1, nonlinearity=nonlin) net['pool2'] = MaxPool2DLayer(net['conv2b'], pool_size=(2, 2)) # (time, freq) # ----------- fully connected layer group --------------- net['fc5'] = DenseLayer(net['pool2'], num_units=1024, nonlinearity=nonlin) net['fc6'] = DenseLayer(net['fc5'], num_units=1024, nonlinearity=nonlin) net['prob'] = DenseLayer(net['fc6'], num_units=41, nonlinearity=lasagne.nonlinearities.softmax) return net
def build_cnn(self): # Building the network layer_in = InputLayer(shape=(None, 784), input_var=self.input_var) # Hidden layer layer = DenseLayer( layer_in, num_units=self.hidden_size, W=lasagne.init.Uniform( range=(-np.sqrt(6. / (784 + self.hidden_size)), np.sqrt(6. / (784 + self.hidden_size)))), nonlinearity=tanh, ) # LR layer layer = DenseLayer( layer, num_units=self.output_size, W=lasagne.init.Constant(0.), nonlinearity=softmax, ) return layer
def define(self, n_units = 1): self.sample_weights = T.fvector(name='weights') self.labels = T.fvector(name='labels') self.input = T.fmatrix(name='input') input_layer = layers.InputLayer(shape=(None , 1), input_var=self.input) dense1 = layers.DenseLayer( input_layer, num_units=n_units, nonlinearity=nonlinearities.sigmoid ) self.net = layers.DenseLayer( dense1, num_units=1, nonlinearity=nonlinearities.sigmoid )
def build_multi_dssm(input_var=None, num_samples=None, num_entries=6, num_ngrams=42**3, num_hid1=300, num_hid2=300, num_out=128): """Builds a DSSM structure in a Lasagne/Theano way. The built DSSM is the neural network that computes the projection of only one paper. The input ``input_var`` should have two dimensions: (``num_samples * num_entries``, ``num_ngrams``). The output is then computed in a batch way: one paper at a time, but all papers from the same sample in the dataset are grouped (cited papers, citing papers and ``num_entries - 2`` irrelevant papers). Args: input_var (:class:`theano.tensor.TensorType` or None): symbolic input variable of the DSSM num_samples (int): the number of samples in the batch input dataset (number of rows) num_entries (int): the number of compared papers in the DSSM structure num_ngrams (int): the size of the vocabulary num_hid1 (int): the number of units in the first hidden layer num_hid2 (int): the number of units in the second hidden layer num_out (int): the number of units in the output layer Returns: :class:`lasagne.layers.Layer`: the output layer of the DSSM """ assert (num_entries > 2) # Initialise input layer if num_samples is None: num_rows = None else: num_rows = num_samples * num_entries l_in = layers.InputLayer(shape=(num_rows, num_ngrams), input_var=input_var) # Initialise the hidden and output layers or the DSSM l_hid1 = layers.DenseLayer(l_in, num_units=num_hid1, nonlinearity=nonlinearities.tanh, W=init.GlorotUniform()) l_hid2 = layers.DenseLayer(l_hid1, num_units=num_hid2, nonlinearity=nonlinearities.tanh, W=init.GlorotUniform()) l_out = layers.DenseLayer(l_hid2, num_units=num_out, nonlinearity=nonlinearities.tanh, W=init.GlorotUniform()) l_out = layers.ExpressionLayer(l_out, lambda X: X / X.norm(2), output_shape='auto') return l_out
def CNN(n_epochs): net1 = NeuralNet( layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), # Convolutional layer. Params defined below ('pool1', layers.MaxPool2DLayer), # Like downsampling, for execution speed ('conv2', layers.Conv2DLayer), ('hidden3', layers.DenseLayer), ('output', layers.DenseLayer), ], input_shape=(None, 1, 6, 5), conv1_num_filters=8, conv1_filter_size=(3, 3), conv1_nonlinearity=lasagne.nonlinearities.rectify, pool1_pool_size=(2, 2), conv2_num_filters=12, conv2_filter_size=(1, 1), conv2_nonlinearity=lasagne.nonlinearities.rectify, hidden3_num_units=1000, output_num_units=2, output_nonlinearity=lasagne.nonlinearities.softmax, update_learning_rate=0.0001, update_momentum=0.9, max_epochs=n_epochs, verbose=0, ) return net1
def _set_inverse_parameters(self, patterns=None): for l in L.get_all_layers(self.output_layer): if type(l) is L.Conv2DLayer: W = l.W.get_value() if l.flip_filters: W = W[:,:,::-1,::-1] W = W.transpose(1,0,2,3) self.inverse_map[l].W.set_value(W) elif type(l) is L.DenseLayer: self.inverse_map[l].W.set_value(l.W.get_value().T)
def _get_normalised_relevance_layer(self, layer, feeder): def add_epsilon(Zs): tmp = (T.cast(Zs >= 0, theano.config.floatX)*2.0 - 1.0) return Zs + self.epsilon * tmp if isinstance(layer, L.DenseLayer): forward_layer = L.DenseLayer(layer.input_layer, layer.num_units, W=layer.W, b=layer.b, nonlinearity=None) elif isinstance(layer, L.Conv2DLayer): forward_layer = L.Conv2DLayer(layer.input_layer, num_filters=layer.num_filters, W=layer.W, b=layer.b, stride=layer.stride, filter_size=layer.filter_size, flip_filters=layer.flip_filters, untie_biases=layer.untie_biases, pad=layer.pad, nonlinearity=None) else: raise NotImplementedError() forward_layer = L.ExpressionLayer(forward_layer, lambda x: 1.0 / add_epsilon(x)) feeder = L.ElemwiseMergeLayer([forward_layer, feeder], merge_function=T.mul) return feeder
def _invert_DenseLayer(self,layer,feeder): # Warning they are swapped here feeder = self._put_rectifiers(feeder, layer) feeder = self._get_normalised_relevance_layer(layer, feeder) output_units = np.prod(L.get_output_shape(layer.input_layer)[1:]) output_layer = L.DenseLayer(feeder, num_units=output_units) W = output_layer.W tmp_shape = np.asarray((-1,)+L.get_output_shape(output_layer)[1:]) x_layer = L.ReshapeLayer(layer.input_layer, tmp_shape.tolist()) output_layer = L.ElemwiseMergeLayer(incomings=[x_layer, output_layer], merge_function=T.mul) output_layer.W = W return output_layer
def _invert_DenseLayer(self, layer, feeder): # Warning they are swapped here feeder = self._put_rectifiers(feeder, layer) output_units = np.prod(L.get_output_shape(layer.input_layer)[1:]) output_layer = L.DenseLayer(feeder, num_units=output_units, nonlinearity=None, b=None) return output_layer
def _invert_layer(self, layer, feeder): layer_type = type(layer) if L.get_output_shape(feeder) != L.get_output_shape(layer): feeder = L.ReshapeLayer(feeder, (-1,)+L.get_output_shape(layer)[1:]) if layer_type is L.InputLayer: return self._invert_InputLayer(layer, feeder) elif layer_type is L.FlattenLayer: return self._invert_FlattenLayer(layer, feeder) elif layer_type is L.DenseLayer: return self._invert_DenseLayer(layer, feeder) elif layer_type is L.Conv2DLayer: return self._invert_Conv2DLayer(layer, feeder) elif layer_type is L.DropoutLayer: return self._invert_DropoutLayer(layer, feeder) elif layer_type in [L.MaxPool2DLayer, L.MaxPool1DLayer]: return self._invert_MaxPoolingLayer(layer, feeder) elif layer_type is L.PadLayer: return self._invert_PadLayer(layer, feeder) elif layer_type is L.SliceLayer: return self._invert_SliceLayer(layer, feeder) elif layer_type is L.LocalResponseNormalization2DLayer: return self._invert_LocalResponseNormalisation2DLayer(layer, feeder) elif layer_type is L.GlobalPoolLayer: return self._invert_GlobalPoolLayer(layer, feeder) else: return self._invert_UnknownLayer(layer, feeder)
def _collect_layers(self): self.all_layers = L.get_all_layers(self.output_layer) ret = [l for l in self.all_layers if type(l) in [L.DenseLayer, L.Conv2DLayer]] return ret
def _get_split(self, layer, deterministic=True, conv_all_patches=True, **kwargs): # Get the patches and the outputs without the non-linearities. if type(layer) is L.DenseLayer: x, y = putils.get_dense_xy(layer, deterministic) elif type(layer) is L.Conv2DLayer: if conv_all_patches is True: x, y = putils.get_conv_xy_all(layer, deterministic) else: x, y = putils.get_conv_xy(layer, deterministic) else: raise ValueError("Unknown layer as input") # Create an output dictionary outputs = dict() for name, fun in subtypes: outputs[name] = dict() mrk_y = 1.0* T.cast(fun(y), dtype=theano.config.floatX) # (N,O) y_current = y*mrk_y # This has a binary mask cnt_y = T.shape_padaxis(T.sum(mrk_y, axis=0), axis=0) # (1,O) norm = T.maximum(cnt_y, 1.) # Count how many datapoints are considered outputs[name]['cnt'] = cnt_y # The mean of the current batch outputs[name]['m_y'] = T.shape_padaxis(y_current.sum(axis=0), axis=0) / norm # (1,O) mean output for batch outputs[name]['m_x'] = T.dot(x.T, mrk_y) / norm # (D,O) mean input for batch # The mean of the current batch outputs[name]['yty'] = T.shape_padaxis(T.sum(y_current ** 2., axis=0), axis=0) / norm # (1,O) outputs[name]['xty'] = T.dot(x.T, y_current) / norm # D,O return dict_to_list(outputs)
def get_split(self, layer, deterministic=True, conv_all_patches=True, **kwargs): # Get the patches and the outputs without the non-linearities. if type(layer) is L.DenseLayer: x, y = get_dense_xy(layer, deterministic) elif type(layer) is L.Conv2DLayer: if conv_all_patches is True: x, y = get_conv_xy_all(layer, deterministic) else: x, y = get_conv_xy(layer, deterministic) else: raise ValueError("Unknown layer as input") # Create an output dictionary outputs = dict() for name, fun in subtypes: outputs[name] = dict() mrk_y = 1.0* T.cast(fun(y), dtype=theano.config.floatX) # (N,O) y_current = y*mrk_y # This has a binary mask cnt_y = T.shape_padaxis(T.sum(mrk_y, axis=0), axis=0) # (1,O) norm = T.maximum(cnt_y, 1.) # Count how many datapoints are considered outputs[name]['cnt'] = cnt_y # The mean of the current batch outputs[name]['m_y'] = T.shape_padaxis(y_current.sum(axis=0), axis=0) / norm # (1,O) mean output for batch outputs[name]['m_x'] = T.dot(x.T, mrk_y) / norm # (D,O) mean input for batch # The mean of the current batch outputs[name]['yty'] = T.shape_padaxis(T.sum(y_current ** 2., axis=0), axis=0) / norm # (1,O) outputs[name]['xty'] = T.dot(x.T, y_current) / norm # D,O return dict_to_list(outputs)
def exe_maxru(length, num_units, position, binominal): batch_size = BATCH_SIZE input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) target_var = T.ivector(name='targets') layer_input = lasagne.layers.InputLayer(shape=(None, length, 1), input_var=input_var, name='input') time_updategate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None) time_update = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None, b=lasagne.init.Constant(0.), nonlinearity=nonlinearities.tanh) resetgate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=lasagne.init.GlorotUniform()) updategate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=lasagne.init.GlorotUniform()) hiden_update = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None, b=lasagne.init.Constant(0.), nonlinearity=nonlinearities.tanh) layer_taru = MAXRULayer(layer_input, num_units, max_length=length, P_time=lasagne.init.GlorotUniform(), nonlinearity=nonlinearities.tanh, resetgate=resetgate, updategate=updategate, hidden_update=hiden_update, time_updategate=time_updategate, time_update=time_update, only_return_final=True, name='MAXRU', p=0.) # W = layer_taru.W_hid_to_hidden_update.sum() # U = layer_taru.W_in_to_hidden_update.sum() # b = layer_taru.b_hidden_update.sum() layer_output = DenseLayer(layer_taru, num_units=1, nonlinearity=nonlinearities.sigmoid, name='output') return train(layer_output, input_var, target_var, batch_size, length, position, binominal)
def exe_lstm(use_embedd, length, num_units, position, binominal): batch_size = BATCH_SIZE input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) target_var = T.ivector(name='targets') layer_input = lasagne.layers.InputLayer(shape=(None, length, 1), input_var=input_var, name='input') if use_embedd: layer_position = construct_position_input(batch_size, length, num_units) layer_input = lasagne.layers.concat([layer_input, layer_position], axis=2) ingate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=lasagne.init.Uniform(range=0.1)) outgate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=lasagne.init.Uniform(range=0.1)) # according to Jozefowicz et al.(2015), init bias of forget gate to 1. forgetgate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=lasagne.init.Uniform(range=0.1), b=lasagne.init.Constant(1.)) # now use tanh for nonlinear function of cell, need to try pure linear cell cell = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None, b=lasagne.init.Constant(0.), nonlinearity=nonlinearities.tanh) layer_lstm = LSTMLayer(layer_input, num_units, ingate=ingate, forgetgate=forgetgate, cell=cell, outgate=outgate, peepholes=False, nonlinearity=nonlinearities.tanh, only_return_final=True, name='LSTM') # W = layer_lstm.W_hid_to_cell.sum() # U = layer_lstm.W_in_to_cell.sum() # b = layer_lstm.b_cell.sum() layer_output = DenseLayer(layer_lstm, num_units=1, nonlinearity=nonlinearities.sigmoid, name='output') return train(layer_output, layer_lstm, input_var, target_var, batch_size, length, position, binominal)
def exe_gru(use_embedd, length, num_units, position, binominal, reset_input): batch_size = BATCH_SIZE input_var = T.tensor3(name='inputs', dtype=theano.config.floatX) target_var = T.ivector(name='targets') layer_input = lasagne.layers.InputLayer(shape=(batch_size, length, 1), input_var=input_var, name='input') if use_embedd: layer_position = construct_position_input(batch_size, length, num_units) layer_input = lasagne.layers.concat([layer_input, layer_position], axis=2) resetgate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None) updategate = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None) hiden_update = Gate(W_in=lasagne.init.GlorotUniform(), W_hid=lasagne.init.GlorotUniform(), W_cell=None, b=lasagne.init.Constant(0.), nonlinearity=nonlinearities.tanh) layer_gru = GRULayer_ANA(layer_input, num_units, resetgate=resetgate, updategate=updategate, hidden_update=hiden_update, reset_input=reset_input, only_return_final=True, name='GRU') # W = layer_gru.W_hid_to_hidden_update.sum() # U = layer_gru.W_in_to_hidden_update.sum() # b = layer_gru.b_hidden_update.sum() layer_output = DenseLayer(layer_gru, num_units=1, nonlinearity=nonlinearities.sigmoid, name='output') return train(layer_output, layer_gru, input_var, target_var, batch_size, length, position, binominal)
def dense_layer(input, n_units, name, network_weights, nonlinearity=None, bn=False): layer = DenseLayer(input, num_units=n_units, nonlinearity=nonlinearity, name=name, W=get_W(network_weights, name), b=get_b(network_weights, name)) if bn: layer = batch_norm(layer) return layer
def create_network(): l = 1000 pool_size = 5 test_size1 = 13 test_size2 = 7 test_size3 = 5 kernel1 = 128 kernel2 = 128 kernel3 = 128 layer1 = InputLayer(shape=(None, 1, 4, l+1024)) layer2_1 = SliceLayer(layer1, indices=slice(0, l), axis = -1) layer2_2 = SliceLayer(layer1, indices=slice(l, None), axis = -1) layer2_3 = SliceLayer(layer2_2, indices = slice(0,4), axis = -2) layer2_f = FlattenLayer(layer2_3) layer3 = Conv2DLayer(layer2_1,num_filters = kernel1, filter_size = (4,test_size1)) layer4 = Conv2DLayer(layer3,num_filters = kernel1, filter_size = (1,test_size1)) layer5 = Conv2DLayer(layer4,num_filters = kernel1, filter_size = (1,test_size1)) layer6 = MaxPool2DLayer(layer5, pool_size = (1,pool_size)) layer7 = Conv2DLayer(layer6,num_filters = kernel2, filter_size = (1,test_size2)) layer8 = Conv2DLayer(layer7,num_filters = kernel2, filter_size = (1,test_size2)) layer9 = Conv2DLayer(layer8,num_filters = kernel2, filter_size = (1,test_size2)) layer10 = MaxPool2DLayer(layer9, pool_size = (1,pool_size)) layer11 = Conv2DLayer(layer10,num_filters = kernel3, filter_size = (1,test_size3)) layer12 = Conv2DLayer(layer11,num_filters = kernel3, filter_size = (1,test_size3)) layer13 = Conv2DLayer(layer12,num_filters = kernel3, filter_size = (1,test_size3)) layer14 = MaxPool2DLayer(layer13, pool_size = (1,pool_size)) layer14_d = DenseLayer(layer14, num_units= 256) layer3_2 = DenseLayer(layer2_f, num_units = 128) layer15 = ConcatLayer([layer14_d,layer3_2]) layer16 = DropoutLayer(layer15,p=0.5) layer17 = DenseLayer(layer16, num_units=256) network = DenseLayer(layer17, num_units= 2, nonlinearity=softmax) return network #random search to initialize the weights
def create_network(): l = 1000 pool_size = 5 test_size1 = 13 test_size2 = 7 test_size3 = 5 kernel1 = 128 kernel2 = 128 kernel3 = 128 layer1 = InputLayer(shape=(None, 1, 4, l+1024)) layer2_1 = SliceLayer(layer1, indices=slice(0, l), axis = -1) layer2_2 = SliceLayer(layer1, indices=slice(l, None), axis = -1) layer2_3 = SliceLayer(layer2_2, indices = slice(0,4), axis = -2) layer2_f = FlattenLayer(layer2_3) layer3 = Conv2DLayer(layer2_1,num_filters = kernel1, filter_size = (4,test_size1)) layer4 = Conv2DLayer(layer3,num_filters = kernel1, filter_size = (1,test_size1)) layer5 = Conv2DLayer(layer4,num_filters = kernel1, filter_size = (1,test_size1)) layer6 = MaxPool2DLayer(layer5, pool_size = (1,pool_size)) layer7 = Conv2DLayer(layer6,num_filters = kernel2, filter_size = (1,test_size2)) layer8 = Conv2DLayer(layer7,num_filters = kernel2, filter_size = (1,test_size2)) layer9 = Conv2DLayer(layer8,num_filters = kernel2, filter_size = (1,test_size2)) layer10 = MaxPool2DLayer(layer9, pool_size = (1,pool_size)) layer11 = Conv2DLayer(layer10,num_filters = kernel3, filter_size = (1,test_size3)) layer12 = Conv2DLayer(layer11,num_filters = kernel3, filter_size = (1,test_size3)) layer13 = Conv2DLayer(layer12,num_filters = kernel3, filter_size = (1,test_size3)) layer14 = MaxPool2DLayer(layer13, pool_size = (1,pool_size)) layer14_d = DenseLayer(layer14, num_units= 256) layer3_2 = DenseLayer(layer2_f, num_units = 128) layer15 = ConcatLayer([layer14_d,layer3_2]) #layer16 = DropoutLayer(layer15,p=0.5) layer17 = DenseLayer(layer15, num_units=256) network = DenseLayer(layer17, num_units= 1, nonlinearity=None) return network #random search to initialize the weights
def build_tempral_model(): net={} net['input']=InputLayer((None,24,2048)) net['lstm1']=LSTMLayer(net['input'],256) net['fc']=DenseLayer(net['lstm1'],num_units=12,nonlinearity=sigmoid) return net
def build_model(): net = {} net['input'] = InputLayer((None, 512*20, 3, 3)) au_fc_layers=[] for i in range(20): net['roi_AU_N_'+str(i)]=SliceLayer(net['input'],indices=slice(i*512,(i+1)*512),axis=1) #try to adding upsampling here for more conv net['Roi_upsample_'+str(i)]=Upscale2DLayer(net['roi_AU_N_'+str(i)],scale_factor=2) net['conv_roi_'+str(i)]=ConvLayer(net['Roi_upsample_'+str(i)],512,3) net['au_fc_'+str(i)]=DenseLayer(net['conv_roi_'+str(i)],num_units=150) au_fc_layers+=[net['au_fc_'+str(i)]] # net['local_fc']=concat(au_fc_layers) net['local_fc2']=DenseLayer(net['local_fc'],num_units=2048) net['local_fc_dp']=DropoutLayer(net['local_fc2'],p=0.5) # net['fc_comb']=concat([net['au_fc_layer'],net['local_fc_dp']]) # net['fc_dense']=DenseLayer(net['fc_comb'],num_units=1024) # net['fc_dense_dp']=DropoutLayer(net['fc_dense'],p=0.3) net['real_out']=DenseLayer(net['local_fc_dp'],num_units=12,nonlinearity=sigmoid) # net['final']=concat([net['pred_pos_layer'],net['output_layer']]) return net
def getTrainedRNN(): ''' Read from file and set the params (To Do: Refactor so as to do this only once) ''' input_size = 39 hidden_size = 50 num_output_classes = 29 learning_rate = 0.001 output_size = num_output_classes+1 batch_size = None input_seq_length = None gradient_clipping = 5 l_in = InputLayer(shape=(batch_size, input_seq_length, input_size)) n_batch, n_time_steps, n_features = l_in.input_var.shape #Unnecessary in this version. Just collecting the info so that we can reshape the output back to the original shape # h_1 = DenseLayer(l_in, num_units=hidden_size, nonlinearity=clipped_relu) l_rec_forward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu) l_rec_backward = RecurrentLayer(l_in, num_units=hidden_size, grad_clipping=gradient_clipping, nonlinearity=clipped_relu, backwards=True) l_rec_accumulation = ElemwiseSumLayer([l_rec_forward,l_rec_backward]) l_rec_reshaped = ReshapeLayer(l_rec_accumulation, (-1,hidden_size)) l_h2 = DenseLayer(l_rec_reshaped, num_units=hidden_size, nonlinearity=clipped_relu) l_out = DenseLayer(l_h2, num_units=output_size, nonlinearity=lasagne.nonlinearities.linear) l_out_reshaped = ReshapeLayer(l_out, (n_batch, n_time_steps, output_size))#Reshaping back l_out_softmax = NonlinearityLayer(l_out, nonlinearity=lasagne.nonlinearities.softmax) l_out_softmax_reshaped = ReshapeLayer(l_out_softmax, (n_batch, n_time_steps, output_size)) with np.load('CTC_model.npz') as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(l_out_softmax_reshaped, param_values, trainable = True) output = lasagne.layers.get_output( l_out_softmax_reshaped ) return l_in, output
def getTrainedCLM(): ''' Read CLM from file ''' #Some parameters for the CLM INPUT_SIZE = 29 #Hidden layer hyper-parameters N_HIDDEN = 100 HIDDEN_NONLINEARITY = 'rectify' #Gradient clipping GRAD_CLIP = 100 l_in = lasagne.layers.InputLayer(shape = (None, None, INPUT_SIZE)) #One-hot represenntation of character indices l_mask = lasagne.layers.InputLayer(shape = (None, None)) l_recurrent = lasagne.layers.RecurrentLayer(incoming = l_in, num_units=N_HIDDEN, mask_input = l_mask, learn_init=True, grad_clipping=GRAD_CLIP) Recurrent_output=lasagne.layers.get_output(l_recurrent) n_batch, n_time_steps, n_features = l_in.input_var.shape l_reshape = lasagne.layers.ReshapeLayer(l_recurrent, (-1, N_HIDDEN)) Reshape_output = lasagne.layers.get_output(l_reshape) l_h1 = lasagne.layers.DenseLayer(l_reshape, num_units=N_HIDDEN) l_h2 = lasagne.layers.DenseLayer(l_h1, num_units=N_HIDDEN) l_dense = lasagne.layers.DenseLayer(l_h2, num_units=INPUT_SIZE, nonlinearity = lasagne.nonlinearities.softmax) with np.load('CLM_model.npz') as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(l_dense, param_values,trainable = True) output = lasagne.layers.get_output( l_dense ) return l_in,l_mask,output #def getCLMOneHot( sequence ):
