我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用keras.layers.Dropout()。
def build_encoder(self,input_shape): return [Reshape((*input_shape,1)), GaussianNoise(self.parameters['noise']), BN(), *[Convolution2D(self.parameters['clayer'],(3,3), activation=self.parameters['activation'],padding='same', use_bias=False), Dropout(self.parameters['dropout']), BN(), MaxPooling2D((2,2)),], *[Convolution2D(self.parameters['clayer'],(3,3), activation=self.parameters['activation'],padding='same', use_bias=False), Dropout(self.parameters['dropout']), BN(), MaxPooling2D((2,2)),], flatten, Sequential([ Dense(self.parameters['layer'], activation=self.parameters['activation'], use_bias=False), BN(), Dropout(self.parameters['dropout']), Dense(self.parameters['N']*self.parameters['M']), ])]
def build_encoder(self,input_shape): return [Reshape((*input_shape,1)), GaussianNoise(0.1), BN(), Convolution2D(self.parameters['clayer'],(3,3), activation=self.parameters['activation'],padding='same', use_bias=False), Dropout(self.parameters['dropout']), BN(), MaxPooling2D((2,2)), Convolution2D(self.parameters['clayer'],(3,3), activation=self.parameters['activation'],padding='same', use_bias=False), Dropout(self.parameters['dropout']), BN(), MaxPooling2D((2,2)), Convolution2D(self.parameters['clayer'],(3,3), activation=self.parameters['activation'],padding='same', use_bias=False), Dropout(self.parameters['dropout']), BN(), MaxPooling2D((2,2)), flatten,]
def __init__(self, input_shape, lr=0.01, n_layers=2, n_hidden=8, rate_dropout=0.2, loss='risk_estimation'): print("initializing..., learing rate %s, n_layers %s, n_hidden %s, dropout rate %s." %(lr, n_layers, n_hidden, rate_dropout)) self.model = Sequential() self.model.add(Dropout(rate=rate_dropout, input_shape=(input_shape[0], input_shape[1]))) for i in range(0, n_layers - 1): self.model.add(LSTM(n_hidden * 4, return_sequences=True, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', dropout=rate_dropout, recurrent_dropout=rate_dropout)) self.model.add(LSTM(n_hidden, return_sequences=False, activation='tanh', recurrent_activation='hard_sigmoid', kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', dropout=rate_dropout, recurrent_dropout=rate_dropout)) self.model.add(Dense(1, kernel_initializer=initializers.glorot_uniform())) # self.model.add(BatchNormalization(axis=-1, moving_mean_initializer=Constant(value=0.5), # moving_variance_initializer=Constant(value=0.25))) self.model.add(BatchRenormalization(axis=-1, beta_init=Constant(value=0.5))) self.model.add(Activation('relu_limited')) opt = RMSprop(lr=lr) self.model.compile(loss=loss, optimizer=opt, metrics=['accuracy'])
def first_block(tensor_input,filters,kernel_size=3,pooling_size=1,dropout=0.5): k1,k2 = filters out = Conv1D(k1,1,padding='same')(tensor_input) out = BatchNormalization()(out) out = Activation('relu')(out) out = Dropout(dropout)(out) out = Conv1D(k2,kernel_size,padding='same')(out) pooling = MaxPooling1D(pooling_size,padding='same')(tensor_input) # out = merge([out,pooling],mode='sum') out = add([out,pooling]) return out
def get_encoded_phrase(self, phrase_input_layer, dropout={}, embedding=None): ''' Takes a Keras input layer, dropout and returns the output of the encoder as a Keras object. Arguments: phrase_input_layer (Input): Keras Input layer of the appropriate shape. dropout (dict [str -> float]): Dict containing dropout values applied after `embedding` and `encoder`. embedding_file (str): Optional gzipped embedding file to use as initialization for embedding layer. ''' embedding_layer = self._get_embedding_layer(embedding) embedded_phrase = embedding_layer(phrase_input_layer) embedding_dropout = dropout.pop("embedding", 0.0) if embedding_dropout > 0: embedded_phrase = Dropout(embedding_dropout)(embedded_phrase) encoder = self._get_encoder_layer() encoded_phrase = encoder(embedded_phrase) encoder_dropout = dropout.pop("encoder", 0.0) if encoder_dropout > 0: encoded_phrase = Dropout(encoder_dropout)(encoded_phrase) return encoded_phrase
def model_discriminator(input_shape=(1, 28, 28), dropout_rate=0.5): d_input = dim_ordering_input(input_shape, name="input_x") nch = 512 # nch = 128 H = Convolution2D(int(nch / 2), 5, 5, subsample=(2, 2), border_mode='same', activation='relu')(d_input) H = LeakyReLU(0.2)(H) H = Dropout(dropout_rate)(H) H = Convolution2D(nch, 5, 5, subsample=(2, 2), border_mode='same', activation='relu')(H) H = LeakyReLU(0.2)(H) H = Dropout(dropout_rate)(H) H = Flatten()(H) H = Dense(int(nch / 2))(H) H = LeakyReLU(0.2)(H) H = Dropout(dropout_rate)(H) d_V = Dense(1, activation='sigmoid')(H) return Model(d_input, d_V)
def largeann(input_shape, n_classes, layers=3, neurons=2000, dropout=0.35 ): """ for working with extracted features """ # gpu = switch_gpu() # with K.tf.device('/gpu:{}'.format(gpu)): # K.set_session(K.tf.Session(config=K.tf.ConfigProto(allow_soft_placement=True, log_device_placement=False))) model = Sequential(name='ann') # model.gpu = gpu for l in range(layers): model.add(Dense (neurons, input_shape=input_shape, activation='elu', kernel_initializer='he_normal')) model.add(BatchNormalization()) model.add(Dropout(dropout)) model.add(Dense(n_classes, activation = 'softmax')) model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=[keras.metrics.categorical_accuracy]) return model #%% everyhing recurrent for ANN
def HAN1(MAX_NB_WORDS, MAX_WORDS, MAX_SENTS, EMBEDDING_DIM, WORDGRU, embedding_matrix, DROPOUTPER): #model = Sequential() wordInputs = Input(shape=(MAX_WORDS,), name='word1', dtype='float32') wordEmbedding = Embedding(MAX_NB_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], mask_zero=True, trainable=True, name='emb1')(wordInputs) #Assuming all the sentences have same number of words. Check for input_length again. hij = Bidirectional(GRU(WORDGRU, name='gru1', return_sequences=True))(wordEmbedding) wordDrop = Dropout(DROPOUTPER, name='drop1')(hij) alpha_its, Si = AttentionLayer(name='att1')(wordDrop) v6 = Dense(1, activation="sigmoid", name="dense")(Si) #model.add(Dense(1, activation="sigmoid", name="documentOut3")) model = Model(inputs=[wordInputs] , outputs=[v6]) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def answer_start_pred(context_encoding, question_attention_vector, context_mask, W, dropout_rate): """Answer start prediction layer.""" answer_start = Lambda(lambda arg: concatenate([arg[0], arg[1], arg[2]]))([ context_encoding, question_attention_vector, multiply([context_encoding, question_attention_vector])]) answer_start = TimeDistributed(Dense(W, activation='relu'))(answer_start) answer_start = Dropout(rate=dropout_rate)(answer_start) answer_start = TimeDistributed(Dense(1))(answer_start) # apply masking answer_start = Lambda(lambda q: masked_softmax(q[0], q[1]))([answer_start, context_mask]) answer_start = Lambda(lambda q: flatten(q))(answer_start) return answer_start
def create_lstm_layer(self, input_dim): """Create a LSTM layer of a model.""" inp = Input(shape=(input_dim, self.embedding_dim,)) inp_dropout = Dropout(self.ldrop_val)(inp) ker_in = glorot_uniform(seed=self.seed) rec_in = Orthogonal(seed=self.seed) outp = LSTM(self.hidden_dim, input_shape=(input_dim, self.embedding_dim,), kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, recurrent_dropout=self.recdrop_val, dropout=self.inpdrop_val, kernel_initializer=ker_in, recurrent_initializer=rec_in, return_sequences=True)(inp_dropout) outp_dropout = Dropout(self.dropout_val)(outp) model = Model(inputs=inp, outputs=outp_dropout, name="LSTM_encoder") return model
def create_lstm_layer_2(self, input_dim): """Create a LSTM layer of a model.""" inp = Input(shape=(input_dim, 2*self.perspective_num,)) inp_drop = Dropout(self.ldrop_val)(inp) ker_in = glorot_uniform(seed=self.seed) rec_in = Orthogonal(seed=self.seed) bioutp = Bidirectional(LSTM(self.aggregation_dim, input_shape=(input_dim, 2*self.perspective_num,), kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, recurrent_dropout=self.recdrop_val, dropout=self.inpdrop_val, kernel_initializer=ker_in, recurrent_initializer=rec_in, return_sequences=True), merge_mode=None)(inp_drop) dropout_forw = Dropout(self.dropout_val)(bioutp[0]) dropout_back = Dropout(self.dropout_val)(bioutp[1]) model = Model(inputs=inp, outputs=[dropout_forw, dropout_back], name="biLSTM_enc_persp") return model
def create_lstm_layer_last(self, input_dim): """Create a LSTM layer of a model.""" inp = Input(shape=(input_dim, self.embedding_dim,)) inp_drop = Dropout(self.ldrop_val)(inp) ker_in = glorot_uniform(seed=self.seed) rec_in = Orthogonal(seed=self.seed) bioutp = Bidirectional(LSTM(self.hidden_dim, input_shape=(input_dim, self.embedding_dim,), kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, recurrent_dropout=self.recdrop_val, dropout=self.inpdrop_val, kernel_initializer=ker_in, recurrent_initializer=rec_in, return_sequences=False), merge_mode='concat')(inp_drop) dropout = Dropout(self.dropout_val)(bioutp) model = Model(inputs=inp, outputs=dropout, name="biLSTM_encoder_last") return model
def classifier(base_layers, input_rois, batch_size, nb_classes = 3, trainable=False): # compile times tend to be very high, so we use smaller ROI pooling regions to workaround if K.backend() == 'tensorflow': pooling_regions = 14 input_shape = (batch_size,14,14,2048) elif K.backend() == 'theano': pooling_regions = 7 input_shape = (batch_size,2048,7,7) out_roi_pool = RoiPoolingConv(pooling_regions, batch_size)([base_layers, input_rois]) out = TimeDistributed(Flatten())(out_roi_pool) # out = TimeDistributed(Dropout(0.4))(out) # out = TimeDistributed(Dense(2048,activation='relu'))(out) out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out) # note: no regression target for bg class out_regr = TimeDistributed(Dense(4 * nb_classes, activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out) return [out_class, out_regr]
def classifier(base_layers, input_rois, batch_size, nb_classes = 3, trainable=False): # compile times tend to be very high, so we use smaller ROI pooling regions to workaround if K.backend() == 'tensorflow': pooling_regions = 14 input_shape = (batch_size,14,14,512) elif K.backend() == 'theano': pooling_regions = 7 input_shape = (batch_size,512,7,7) out_roi_pool = RoiPoolingConv(pooling_regions, batch_size)([base_layers, input_rois]) out = TimeDistributed(Flatten())(out_roi_pool) out = TimeDistributed(Dense(4096,activation='relu'))(out) out = TimeDistributed(Dropout(0.5))(out) out = TimeDistributed(Dense(4096,activation='relu'))(out) out = TimeDistributed(Dropout(0.5))(out) out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out) # note: no regression target for bg class out_regr = TimeDistributed(Dense(4 * nb_classes, activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out) return [out_class, out_regr]
def classifier(base_layers, input_rois, batch_size, nb_classes = 3, trainable=False): # compile times tend to be very high, so we use smaller ROI pooling regions to workaround if K.backend() == 'tensorflow': pooling_regions = 14 input_shape = (batch_size,14,14,1024) elif K.backend() == 'theano': pooling_regions = 7 input_shape = (batch_size,1024,7,7) out_roi_pool = RoiPoolingConv(pooling_regions, batch_size)([base_layers, input_rois]) out = TimeDistributed(Flatten())(out_roi_pool) out = TimeDistributed(Dense(4096,activation='relu'))(out) out = TimeDistributed(Dropout(0.5))(out) out = TimeDistributed(Dense(4096,activation='relu'))(out) out = TimeDistributed(Dropout(0.5))(out) out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out) # note: no regression target for bg class out_regr = TimeDistributed(Dense(4 * nb_classes, activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out) return [out_class, out_regr]
def classifier(base_layers, input_rois, num_rois, nb_classes = 21, trainable=False): # compile times on theano tend to be very high, so we use smaller ROI pooling regions to workaround if K.backend() == 'tensorflow': pooling_regions = 7 input_shape = (num_rois,7,7,512) elif K.backend() == 'theano': pooling_regions = 7 input_shape = (num_rois,512,7,7) out_roi_pool = RoiPoolingConv(pooling_regions, num_rois)([base_layers, input_rois]) out = TimeDistributed(Flatten(name='flatten'))(out_roi_pool) out = TimeDistributed(Dense(4096, activation='relu', name='fc1'))(out) out = TimeDistributed(Dropout(0.5))(out) out = TimeDistributed(Dense(4096, activation='relu', name='fc2'))(out) out = TimeDistributed(Dropout(0.5))(out) out_class = TimeDistributed(Dense(nb_classes, activation='softmax', kernel_initializer='zero'), name='dense_class_{}'.format(nb_classes))(out) # note: no regression target for bg class out_regr = TimeDistributed(Dense(4 * (nb_classes-1), activation='linear', kernel_initializer='zero'), name='dense_regress_{}'.format(nb_classes))(out) return [out_class, out_regr]
def build_encoder(self,input_shape): return [GaussianNoise(self.parameters['noise']), BN(), Dense(self.parameters['layer'], activation='relu', use_bias=False), BN(), Dropout(self.parameters['dropout']), Dense(self.parameters['layer'], activation='relu', use_bias=False), BN(), Dropout(self.parameters['dropout']), Dense(self.parameters['layer'], activation='relu', use_bias=False), BN(), Dropout(self.parameters['dropout']), Dense(self.parameters['N']*self.parameters['M']),]
def _build(self,input_shape): x = Input(shape=input_shape) N = input_shape[0] // 2 y = Sequential([ flatten, *[Sequential([BN(), Dense(self.parameters['layer'],activation=self.parameters['activation']), Dropout(self.parameters['dropout']),]) for i in range(self.parameters['num_layers']) ], Dense(1,activation="sigmoid") ])(x) self.loss = bce self.net = Model(x, y) # self.callbacks.append(self.linear_schedule([0.2,0.5], 0.1)) self.callbacks.append(GradientEarlyStopping(verbose=1,epoch=50,min_grad=self.parameters['min_grad'])) # self.custom_log_functions['lr'] = lambda: K.get_value(self.net.optimizer.lr)
def __call__(self, x): """Build the actual model here. Args: x: The encoded or embedded input sequence. Returns: The model output tensor. """ # Avoid mask propagation when dynamic mini-batches are not supported. if not self.allows_dynamic_length(): x = ConsumeMask()(x) x = self.build_model(x) if self.dropout_rate > 0: x = Dropout(self.dropout_rate)(x) return x
def create_BiLSTM(wordvecs, lstm_dim=300, output_dim=2, dropout=.5, weights=None, train=True): model = Sequential() if weights != None: model.add(Embedding(len(wordvecs)+1, len(wordvecs['the']), weights=[weights], trainable=train)) else: model.add(Embedding(len(wordvecs)+1, len(wordvecs['the']), trainable=train)) model.add(Dropout(dropout)) model.add(Bidirectional(LSTM(lstm_dim))) model.add(Dropout(dropout)) model.add(Dense(output_dim, activation='softmax')) if output_dim == 2: model.compile('adam', 'binary_crossentropy', metrics=['accuracy']) else: model.compile('adam', 'categorical_crossentropy', metrics=['accuracy']) return model
def make_model(dense_layer_sizes, filters, kernel_size, pool_size): '''Creates model comprised of 2 convolutional layers followed by dense layers dense_layer_sizes: List of layer sizes. This list has one number for each layer filters: Number of convolutional filters in each convolutional layer kernel_size: Convolutional kernel size pool_size: Size of pooling area for max pooling ''' model = Sequential() model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape)) model.add(Activation('relu')) model.add(Conv2D(filters, kernel_size)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=pool_size)) model.add(Dropout(0.25)) model.add(Flatten()) for layer_size in dense_layer_sizes: model.add(Dense(layer_size)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) return model
def load_model(input_shape, num_classes): model = Sequential() model.add(Convolution2D(6, kernel_size=(3, 3), activation='relu', input_shape=input_shape, padding="same")) model.add(Convolution2D(32, kernel_size=(3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Convolution2D(64, kernel_size=(3, 3), border_mode='same', activation='relu')) model.add(Convolution2D(64, kernel_size=(3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) return model
def repeated_block(x,filters,kernel_size=3,pooling_size=1,dropout=0.5): k1,k2 = filters out = BatchNormalization()(x) out = Activation('relu')(out) out = Conv1D(k1,kernel_size,strides=2,padding='same')(out) out = BatchNormalization()(out) out = Activation('relu')(out) out = Dropout(dropout)(out) out = Conv1D(k2,kernel_size,strides=2,padding='same')(out) pooling = MaxPooling1D(pooling_size,strides=4,padding='same')(x) out = add([out, pooling]) #out = merge([out,pooling]) return out
def build_simple_rnn_model(timestep,input_dim,output_dim,dropout=0.4,lr=0.001): input = Input((timestep,input_dim)) # LSTM, Single output = LSTM(50,return_sequences=False)(input) # for _ in range(1): # output = LSTM(32,return_sequences=True)(output) # output = LSTM(50,return_sequences=False)(output) output = Dropout(dropout)(output) output = Dense(output_dim)(output) model = Model(inputs=input,outputs=output) optimizer = Adam(lr=lr) model.compile(loss='mae',optimizer=optimizer,metrics=['mse']) return model
def first_block(tensor_input,filters,kernel_size=3,pooling_size=1,dropout=0.5): k1,k2 = filters out = Conv1D(k1,1,padding='same')(tensor_input) out = BatchNormalization()(out) out = Activation('relu')(out) out = Dropout(dropout)(out) out = Conv1D(k2,kernel_size,strides=2,padding='same')(out) pooling = MaxPooling1D(pooling_size,strides=2,padding='same')(tensor_input) # out = merge([out,pooling],mode='sum') out = add([out,pooling]) return out
def repeated_block(x,filters,kernel_size=3,pooling_size=1,dropout=0.5): k1,k2 = filters out = BatchNormalization()(x) out = Activation('relu')(out) out = Conv1D(k1,kernel_size,padding='same')(out) out = BatchNormalization()(out) out = Activation('relu')(out) out = Dropout(dropout)(out) out = Conv1D(k2,kernel_size,strides=2,padding='same')(out) pooling = MaxPooling1D(pooling_size,strides=2,padding='same')(x) out = add([out, pooling]) #out = merge([out,pooling]) return out
def repeated_2d_block(x,filters,kernel_size=3,pooling_size=1,dropout=0.5): k1,k2 = filters out = BatchNormalization()(x) out = Activation('relu')(out) out = Conv2D(k1,kernel_size,2,padding='same',data_format='channels_last')(out) out = BatchNormalization()(out) out = Activation('relu')(out) out = Dropout(dropout)(out) out = Conv2D(k2,kernel_size,2,padding='same',data_format='channels_last')(out) pooling = MaxPooling2D(pooling_size,padding='same',data_format='channels_last')(x) out = add([out, pooling]) #out = merge([out,pooling]) return out
def build(self) -> None: """ ?????????? ??????. """ inp = Input(shape=(None,)) emb = Embedding(len(self.grapheme_alphabet), self.emb_dimension)(inp) encoded = Bidirectional(self.rnn(self.units1, return_sequences=True, recurrent_dropout=self.dropout))(emb) encoded = Dropout(self.dropout)(encoded) decoded = TimeDistributed(Dense(self.units2, activation="relu"))(encoded) predictions = TimeDistributed(Dense(len(self.phonetic_alphabet), activation="softmax"))(decoded) model = Model(inputs=inp, outputs=predictions) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) self.model = model
def build(self) -> None: """ ?????????? ??????. """ inp = Input(shape=(None,)) emb = Embedding(len(self.grapheme_set), self.emb_dimension)(inp) encoded = Bidirectional(self.rnn(self.units, return_sequences=True, recurrent_dropout=self.dropout))(emb) encoded = Dropout(self.dropout)(encoded) decoded = TimeDistributed(Dense(self.units, activation="relu"))(encoded) predictions = TimeDistributed(Dense(3, activation="softmax"))(decoded) model = Model(inputs=inp, outputs=predictions) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) self.model = model
def build(self) -> None: """ ?????????? ??????. """ inp = Input(shape=(None,)) emb = Embedding(len(self.phonetic_alphabet), self.emb_dimension)(inp) encoded = Bidirectional(self.rnn(self.units, return_sequences=True, recurrent_dropout=self.dropout))(emb) encoded = Dropout(self.dropout)(encoded) decoded = Bidirectional(self.rnn(self.units, return_sequences=True, recurrent_dropout=self.dropout))(encoded) decoded = Dropout(self.dropout)(decoded) predictions = TimeDistributed(Dense(3, activation="softmax"))(decoded) model = Model(inputs=inp, outputs=predictions) model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) self.model = model
def train(img_shape): classes = ['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT'] # Model model = Sequential() model.add(Convolution2D( 32, 3, 3, input_shape=img_shape, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.2)) model.add(Convolution2D(32, 3, 3, activation='relu', W_constraint=maxnorm(3))) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(512, activation='relu', W_constraint=maxnorm(3))) model.add(Dropout(0.5)) model.add(Dense(len(classes), activation='softmax')) features, labels = get_featurs_labels(img_shape) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit(features, labels, nb_epoch=10, batch_size=32, validation_split=0.2, verbose=1) return model
def assemble_mlp(input_shape, output_shape, batch_size, nb_train_samples): """Assemble a simple MLP model. """ inputs = Input(shape=input_shape) hidden = Dense(1024, activation='relu', name='dense1')(inputs) hidden = Dropout(0.5)(hidden) hidden = Dense(512, activation='relu', name='dense2')(hidden) hidden = Dropout(0.5)(hidden) hidden = Dense(64, activation='relu', name='dense3')(hidden) hidden = Dropout(0.25)(hidden) hidden = Dense(2, activation='relu', name='dense4')(hidden) gp = GP(hyp={ 'lik': np.log(0.3), 'mean': [], 'cov': [[0.5], [1.0]], }, inf='infGrid', dlik='dlikGrid', opt={'cg_maxit': 2000, 'cg_tol': 1e-6}, mean='meanZero', cov='covSEiso', update_grid=1, grid_kwargs={'eq': 1, 'k': 70.}, batch_size=batch_size, nb_train_samples=nb_train_samples) outputs = [gp(hidden)] return Model(inputs=inputs, outputs=outputs)
def define_network(vector_size, loss): base_model = InceptionV3(weights='imagenet', include_top=True) for layer in base_model.layers: # Freeze layers in pretrained model layer.trainable = False # fully-connected layer to predict x = Dense(4096, activation='relu', name='fc1')(base_model.layers[-2].output) x = Dense(8096, activation='relu', name='fc2')(x) x = Dropout(0.5)(x) x = Dense(2048,activation='relu', name='fc3')(x) predictions = Dense(vector_size, activation='relu')(x) l2 = Lambda(lambda x: K.l2_normalize(x, axis=1))(predictions) model = Model(inputs=base_model.inputs, outputs=l2) optimizer = 'adam' if loss == 'euclidean': model.compile(optimizer = optimizer, loss = euclidean_distance) else: model.compile(optimizer = optimizer, loss = loss) return model
def _create_layers(self, input_shape, n_output): """ Create the network layers :param input_shape: :param n_output: :return: self """ # Hidden layers for i, l in enumerate(self.layers): self._model.add(Dense(units=l, input_shape=[input_shape[-1] if i == 0 else None], activation=self.activation[i], kernel_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]), bias_regularizer=l1_l2(self.l1_reg[i], self.l2_reg[i]))) if self.dropout[i] > 0: self._model.add(Dropout(rate=self.dropout[i])) # Output layer self._model.add(Dense(units=n_output, activation=self.out_activation))
def _create_layers(self, input_shape, n_output): """ Create the finetuning model :param input_shape: :param n_output: :return: self """ # Hidden layers for i, l in enumerate(self.layers): self._model.add(Dense(input_shape=[input_shape[1] if i == 0 else None], units=l.n_hidden, weights=l.get_model_parameters()['enc'], activation=l.enc_activation, kernel_regularizer=l1_l2(l.l1_reg, l.l2_reg), bias_regularizer=l1_l2(l.l1_reg, l.l2_reg))) if self.dropout[i] > 0: self._model.add(Dropout(rate=self.dropout[i])) # Output layer self._model.add(Dense(units=n_output, activation=self.out_activation))
def create_network(): from keras.models import Sequential from keras.layers import Dense, Dropout, Flatten from keras.layers import Conv2D, MaxPooling2D model = Sequential() model.add(Conv2D(32, (3, 3), activation='relu', input_shape=INPUT_SHAPE)) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(NUM_CLASSES, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return KerasNetwork(model, 'cnn_weights.hd5')
def BiDi(input_shape,vocabSize,veclen,wordWeights,nLayers,nHidden,lr): assert len(nHidden) == nLayers, '#Neurons for each layer does not match #Layers' r_flag = True _Input = Input(shape = (input_shape,),dtype = 'int32') E = keras.layers.embeddings.Embedding(vocabSize,veclen,weights=(wordWeights,),mask_zero = True)(_Input) for ind in range(nLayers): if ind == (nLayers-1): r_flag = False fwd_layer = keras.layers.recurrent.GRU(nHidden[ind],init='glorot_uniform',inner_init='orthogonal',activation='tanh',inner_activation='hard_sigmoid',return_sequences = r_flag)(E) bkwd_layer = keras.layers.recurrent.GRU(nHidden[ind],init='glorot_uniform',inner_init='orthogonal',activation='tanh',inner_activation='hard_sigmoid',return_sequences = r_flag,go_backwards = True)(E) E = merge([fwd_layer,bkwd_layer],mode = 'ave') #nHidden/= 2 Output = Dense(1,activation = 'sigmoid')(Dropout(0.5)(E)) model = Model(input = _Input, output = Output) opt = keras.optimizers.Adam(lr) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def prep_model(inputs, N, s0pad, s1pad, c): # Word-level projection before averaging inputs[0] = TimeDistributed(Dense(N, activation='relu'))(inputs[0]) inputs[0] = Lambda(lambda x: K.max(x, axis=1), output_shape=(N, ))(inputs[0]) inputs[1] = TimeDistributed(Dense(N, activation='relu'))(inputs[1]) inputs[1] = Lambda(lambda x: K.max(x, axis=1), output_shape=(N, ))(inputs[1]) merged = concatenate([inputs[0], inputs[1]]) # Deep for i in range(c['deep']): merged = Dense(c['nndim'], activation=c['nnact'])(merged) merged = Dropout(c['nndropout'])(merged) merged = BatchNormalization()(merged) is_duplicate = Dense(1, activation='sigmoid')(merged) return [is_duplicate], N
def prep_model(inputs, N, s0pad, s1pad, c): Nc, outputs = B.cnnsum_input(inputs, N, s0pad, siamese=c['cnnsiamese'], dropout=c['dropout'], l2reg=c['l2reg'], cnninit=c['cnninit'], cnnact=c['cnnact'], cdim=c['cdim']) # Projection if c['project']: outputs = Dense(int(N*c['pdim']), kernal_regularizer=l2(c['l2reg']), activation=c['pact'])(outputs) # model.add_shared_node(name='proj', inputs=['e0s_', 'e1s_'], outputs=['e0p', 'e1p'], # layer=Dense(input_dim=Nc, output_dim=int(N*c['pdim']), # W_regularizer=l2(c['l2reg']), activation=c['pact'])) # This dropout is controversial; it might be harmful to apply, # or at least isn't a clear win. # model.add_shared_node(name='projdrop', inputs=['e0p', 'e1p'], outputs=['e0p_', 'e1p_'], # layer=Dropout(c['dropout'], input_shape=(N,))) # return ('e0p_', 'e1p_') return outputs, N
def addLayer(previousLayer, nChannels, nOutChannels, dropRate, blockNum): bn = BatchNormalization(name = 'denseb_BatchNorm_{}'.format(blockNum) , axis = 1)(previousLayer) relu = Activation('relu', name ='denseb_relu_{}'.format(blockNum))(bn) conv = Convolution2D(nOutChannels, 3, 3, border_mode='same', name='denseb_conv_{}'.format(blockNum))(relu) if dropRate is not None: dp = Dropout(dropRate, name='denseb_dropout_{}'.format)(conv) return merge([dp, previousLayer], mode='concat', concat_axis=1) else: return merge([conv, previousLayer], mode='concat', concat_axis=1)
def addTransition(previousLayer, nChannels, nOutChannels, dropRate, blockNum): bn = BatchNormalization(name = 'tr_BatchNorm_{}'.format(blockNum), axis = 1)(previousLayer) relu = Activation('relu', name ='tr_relu_{}'.format(blockNum))(bn) conv = Convolution2D(nOutChannels, 1, 1, border_mode='same', name='tr_conv_{}'.format(blockNum))(relu) if dropRate is not None: dp = Dropout(dropRate, name='tr_dropout_{}'.format)(conv) avgPool = AveragePooling2D(pool_size=(2, 2))(dp) else: avgPool = AveragePooling2D(pool_size=(2, 2))(conv) return avgPool
def model_cnn(net_layers, input_shape): inp = Input(shape=input_shape) model = inp for cl in net_layers['conv_layers']: model = Conv2D(filters=cl[0], kernel_size=cl[1], activation='relu')(model) if cl[4]: model = MaxPooling2D()(model) if cl[2]: model = BatchNormalization()(model) if cl[3]: model = Dropout(0.2)(model) model = Flatten()(model) for dl in net_layers['dense_layers']: model = Dense(dl[0])(model) model = Activation('relu')(model) if dl[1]: model = BatchNormalization()(model) if dl[2]: model = Dropout(0.2)(model) model = Dense(1)(model) model = Activation('sigmoid')(model) model = Model(inp, model) return model # %% # LSTM architecture # conv_layers -> [(filters, kernel_size, BatchNormaliztion, Dropout, MaxPooling)] # dense_layers -> [(num_neurons, BatchNormaliztion, Dropout)]
def model_lstm(input_shape): inp = Input(shape=input_shape) model = inp if input_shape[0] > 2: model = Conv1D(filters=24, kernel_size=(3), activation='relu')(model) # if input_shape[0] > 0: model = TimeDistributed(Conv1D(filters=24, kernel_size=3, activation='relu'))(model) model = LSTM(16)(model) model = Activation('relu')(model) model = Dropout(0.2)(model) model = Dense(16)(model) model = Activation('relu')(model) model = BatchNormalization()(model) model = Dense(1)(model) model = Activation('sigmoid')(model) model = Model(inp, model) return model # %% # Conv-1D architecture. Just one sample as input
def _get_encoded_sentence_variables(self, sent1_input_layer, sent2_input_layer, dropout, embedding_file, tune_embedding): if embedding_file is None: if not tune_embedding: print >>sys.stderr, "Pretrained embedding is not given. Setting tune_embedding to True." tune_embedding = True embedding = None else: # Put the embedding in a list for Keras to treat it as initiali weights of the embeddign layer. embedding = [self.data_processor.get_embedding_matrix(embedding_file, onto_aware=False)] vocab_size = self.data_processor.get_vocab_size(onto_aware=False) embedding_layer = Embedding(input_dim=vocab_size, output_dim=self.embed_dim, weights=embedding, trainable=tune_embedding, mask_zero=True, name="embedding") embedded_sent1 = embedding_layer(sent1_input_layer) embedded_sent2 = embedding_layer(sent2_input_layer) if "embedding" in dropout: embedded_sent1 = Dropout(dropout["embedding"])(embedded_sent1) embedded_sent2 = Dropout(dropout["embedding"])(embedded_sent2) if self.shared_memory: encoder = MultipleMemoryAccessNSE(output_dim=self.embed_dim, return_mode="output_and_memory", name="encoder") mmanse_sent1_input = InputMemoryMerger(name="merge_sent1_input")([embedded_sent1, embedded_sent2]) encoded_sent1_and_memory = encoder(mmanse_sent1_input) encoded_sent1 = OutputSplitter("output", name="get_sent1_output")(encoded_sent1_and_memory) shared_memory = OutputSplitter("memory", name="get_shared_memory")(encoded_sent1_and_memory) mmanse_sent2_input = InputMemoryMerger(name="merge_sent2_input")([embedded_sent2, shared_memory]) encoded_sent2_and_memory = encoder(mmanse_sent2_input) encoded_sent2 = OutputSplitter("output", name="get_sent2_output")(encoded_sent2_and_memory) else: encoder = NSE(output_dim=self.embed_dim, name="encoder") encoded_sent1 = encoder(embedded_sent1) encoded_sent2 = encoder(embedded_sent2) if "encoder" in dropout: encoded_sent1 = Dropout(dropout["encoder"])(encoded_sent1) encoded_sent2 = Dropout(dropout["encoder"])(encoded_sent2) return encoded_sent1, encoded_sent2
def train(self, S_ind, C_ind, use_onto_lstm=True, use_attention=True, num_epochs=20, hierarchical=False, base=2): # Predict next word from current synsets X = C_ind[:,:-1] if use_onto_lstm else S_ind[:,:-1] # remove the last words' hyps in all sentences Y_inds = S_ind[:,1:] # remove the first words in all sentences if hierarchical: train_targets = self._factor_target_indices(Y_inds, base=base) else: train_targets = [self._make_one_hot(Y_inds, Y_inds.max() + 1)] length = Y_inds.shape[1] lstm_outdim = self.word_dim num_words = len(self.dp.word_index) num_syns = len(self.dp.synset_index) input = Input(shape=X.shape[1:], dtype='int32') embed_input_dim = num_syns if use_onto_lstm else num_words embed_layer = HigherOrderEmbedding(name='embedding', input_dim=embed_input_dim, output_dim=self.word_dim, input_shape=X.shape[1:], mask_zero=True) sent_rep = embed_layer(input) reg_sent_rep = Dropout(0.5)(sent_rep) if use_onto_lstm: lstm_out = OntoAttentionLSTM(name='sent_lstm', input_dim=self.word_dim, output_dim=lstm_outdim, input_length=length, num_senses=self.num_senses, num_hyps=self.num_hyps, return_sequences=True, use_attention=use_attention)(reg_sent_rep) else: lstm_out = LSTM(name='sent_lstm', input_dim=self.word_dim, output_dim=lstm_outdim, input_length=length, return_sequences=True)(reg_sent_rep) output_nodes = [] # Make one node for each factored target for target in train_targets: node = TimeDistributed(Dense(input_dim=lstm_outdim, output_dim=target.shape[-1], activation='softmax'))(lstm_out) output_nodes.append(node) model = Model(input=input, output=output_nodes) print >>sys.stderr, model.summary() early_stopping = EarlyStopping() precompile_time = time.time() model.compile(loss='categorical_crossentropy', optimizer='adam') postcompile_time = time.time() print >>sys.stderr, "Model compilation took %d s"%(postcompile_time - precompile_time) model.fit(X, train_targets, nb_epoch=num_epochs, validation_split=0.1, callbacks=[early_stopping]) posttrain_time = time.time() print >>sys.stderr, "Training took %d s"%(posttrain_time - postcompile_time) concept_reps = model.layers[1].get_weights() self.model = model return concept_reps
def image_caption_model(vocab_size=2500, embedding_matrix=None, lang_dim=100, max_caplen=28, img_dim=2048, clipnorm=1): print('generating vocab_history model v5') # text: current word lang_input = Input(shape=(1,)) img_input = Input(shape=(img_dim,)) seq_input = Input(shape=(max_caplen,)) vhist_input = Input(shape=(vocab_size,)) if embedding_matrix is not None: x = Embedding(output_dim=lang_dim, input_dim=vocab_size, init='glorot_uniform', input_length=1, weights=[embedding_matrix])(lang_input) else: x = Embedding(output_dim=lang_dim, input_dim=vocab_size, init='glorot_uniform', input_length=1)(lang_input) lang_embed = Reshape((lang_dim,))(x) lang_embed = merge([lang_embed, seq_input], mode='concat', concat_axis=-1) lang_embed = Dense(lang_dim)(lang_embed) lang_embed = Dropout(0.25)(lang_embed) merge_layer = merge([img_input, lang_embed, vhist_input], mode='concat', concat_axis=-1) merge_layer = Reshape((1, lang_dim+img_dim+vocab_size))(merge_layer) gru_1 = GRU(img_dim)(merge_layer) gru_1 = Dropout(0.25)(gru_1) gru_1 = Dense(img_dim)(gru_1) gru_1 = BatchNormalization()(gru_1) gru_1 = Activation('softmax')(gru_1) attention_1 = merge([img_input, gru_1], mode='mul', concat_axis=-1) attention_1 = merge([attention_1, lang_embed, vhist_input], mode='concat', concat_axis=-1) attention_1 = Reshape((1, lang_dim + img_dim + vocab_size))(attention_1) gru_2 = GRU(1024)(attention_1) gru_2 = Dropout(0.25)(gru_2) gru_2 = Dense(vocab_size)(gru_2) gru_2 = BatchNormalization()(gru_2) out = Activation('softmax')(gru_2) model = Model(input=[img_input, lang_input, seq_input, vhist_input], output=out) model.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=0.0001, clipnorm=1.)) return model
def run_benchmark(self, gpus=0): num_classes = 10 # Generate random input data input_shape = (self.num_samples, 28, 28) x_train, y_train = generate_img_input_data(input_shape) x_train = x_train.reshape(self.num_samples, 784) x_train = x_train.astype('float32') x_train /= 255 # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) model = Sequential() model.add(Dense(512, activation='relu', input_shape=(784,))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(num_classes, activation='softmax')) if keras.backend.backend() is "tensorflow" and gpus > 1: model = multi_gpu_model(model, gpus=gpus) model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) # create a distributed trainer for cntk if keras.backend.backend() is "cntk" and gpus > 1: start, end = cntk_gpu_mode_config(model, x_train.shape[0]) x_train = x_train[start: end] y_train = y_train[start: end] time_callback = timehistory.TimeHistory() model.fit(x_train, y_train, batch_size=self.batch_size, epochs=self.epochs, verbose=1, callbacks=[time_callback]) self.total_time = 0 for i in range(1, self.epochs): self.total_time += time_callback.times[i]
def cnn3adam_slim(input_shape, n_classes): """ Input size should be [batch, 1d, 2d, ch] = (None, 3000, 3) """ model = Sequential(name='cnn3adam') model.add(Conv1D (kernel_size = (50), filters = 32, strides=5, input_shape=input_shape, kernel_initializer='he_normal', activation='elu')) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(Conv1D (kernel_size = (5), filters = 64, strides=1, kernel_initializer='he_normal', activation='elu')) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(MaxPooling1D()) model.add(Conv1D (kernel_size = (5), filters = 64, strides=2, kernel_initializer='he_normal', activation='elu')) model.add(BatchNormalization()) model.add(Dropout(0.2)) model.add(MaxPooling1D()) model.add(Flatten()) model.add(Dense (250, activation='elu', kernel_initializer='he_normal')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense (250, activation='elu', kernel_initializer='he_normal')) model.add(BatchNormalization()) model.add(Dropout(0.5)) model.add(Dense(n_classes, activation = 'softmax')) model.compile(loss='categorical_crossentropy', optimizer=Adam()) return model
