我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用keras.regularizers.l2()。
def cnn_word_model(self): embed_input = Input(shape=(self.opt['max_sequence_length'], self.opt['embedding_dim'],)) outputs = [] for i in range(len(self.kernel_sizes)): output_i = Conv1D(self.opt['filters_cnn'], kernel_size=self.kernel_sizes[i], activation=None, kernel_regularizer=l2(self.opt['regul_coef_conv']), padding='same')(embed_input) output_i = BatchNormalization()(output_i) output_i = Activation('relu')(output_i) output_i = GlobalMaxPooling1D()(output_i) outputs.append(output_i) output = concatenate(outputs, axis=1) output = Dropout(rate=self.opt['dropout_rate'])(output) output = Dense(self.opt['dense_dim'], activation=None, kernel_regularizer=l2(self.opt['regul_coef_dense']))(output) output = BatchNormalization()(output) output = Activation('relu')(output) output = Dropout(rate=self.opt['dropout_rate'])(output) output = Dense(1, activation=None, kernel_regularizer=l2(self.opt['regul_coef_dense']))(output) output = BatchNormalization()(output) act_output = Activation('sigmoid')(output) model = Model(inputs=embed_input, outputs=act_output) return model
def test_activity_regularization(): from keras.engine import Input, Model layer = core.ActivityRegularization(l1=0.01, l2=0.01) # test in functional API x = Input(shape=(3,)) z = core.Dense(2)(x) y = layer(z) model = Model(input=x, output=y) model.compile('rmsprop', 'mse', mode='FAST_COMPILE') model.predict(np.random.random((2, 3))) # test serialization model_config = model.get_config() model = Model.from_config(model_config) model.compile('rmsprop', 'mse')
def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4): ''' SubpixelConvolutional Upscaling (factor = 2) Args: ip: keras tensor nb_filters: number of layers type: can be 'upsampling', 'subpixel', 'deconv'. Determines type of upsampling performed weight_decay: weight decay factor Returns: keras tensor, after applying upsampling operation. ''' if type == 'upsampling': x = UpSampling2D()(ip) elif type == 'subpixel': x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay), use_bias=False, kernel_initializer='he_normal')(ip) x = SubPixelUpscaling(scale_factor=2)(x) x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay), use_bias=False, kernel_initializer='he_normal')(x) else: x = Conv2DTranspose(nb_filters, (3, 3), activation='relu', padding='same', strides=(2, 2), kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(ip) return x
def _conv_bn_relu(**conv_params): # ???????BN """Helper to build a conv -> BN -> relu block """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(input): conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer)(input) return _bn_relu(conv) return f
def _bn_relu_conv(**conv_params): # ????????BN """Helper to build a BN -> relu -> conv block. This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(input): activation = _bn_relu(input) return Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer)(activation) return f
def _shortcut(input, residual): # ??????????????????y=F(x)+x """Adds a shortcut between input and residual block and merges them with "sum" """ # Expand channels of shortcut to match residual. # Stride appropriately to match residual (width, height) # Should be int if network architecture is correctly configured. input_shape = K.int_shape(input) residual_shape = K.int_shape(residual) stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS])) stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS])) equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS] shortcut = input # 1 X 1 conv if shape is different. Else identity. if stride_width > 1 or stride_height > 1 or not equal_channels: shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS], kernel_size=(1, 1), strides=(stride_width, stride_height), padding="valid", kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(input) return add([shortcut, residual])
def basic_block(filters, init_strides=(1, 1), is_first_block_of_first_layer=False): # ???????????? """Basic 3 X 3 convolution blocks for use on resnets with layers <= 34. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv1 = Conv2D(filters=filters, kernel_size=(3, 3), strides=init_strides, padding="same", kernel_initializer="he_normal", kernel_regularizer=l2(1e-4))(input) else: conv1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3), strides=init_strides)(input) residual = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv1) return _shortcut(input, residual) return f
def bottleneck(filters, init_strides=(1, 1), is_first_block_of_first_layer=False): """Bottleneck architecture for > 34 layer resnet. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf Returns: A final conv layer of filters * 4 """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv_1_1 = Conv2D(filters=filters, kernel_size=(1, 1), strides=init_strides, padding="same", kernel_initializer="he_normal", kernel_regularizer=l2(1e-4))(input) else: conv_1_1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3), strides=init_strides)(input) conv_3_3 = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv_1_1) residual = _bn_relu_conv(filters=filters * 4, kernel_size=(1, 1))(conv_3_3) return _shortcut(input, residual) return f
def lstm_word_model(self): embed_input = Input(shape=(self.opt['max_sequence_length'], self.opt['embedding_dim'],)) output = Bidirectional(LSTM(self.opt['units_lstm'], activation='tanh', kernel_regularizer=l2(self.opt['regul_coef_lstm']), dropout=self.opt['dropout_rate']))(embed_input) output = Dropout(rate=self.opt['dropout_rate'])(output) output = Dense(self.opt['dense_dim'], activation=None, kernel_regularizer=l2(self.opt['regul_coef_dense']))(output) output = BatchNormalization()(output) output = Activation('relu')(output) output = Dropout(rate=self.opt['dropout_rate'])(output) output = Dense(1, activation=None, kernel_regularizer=l2(self.opt['regul_coef_dense']))(output) output = BatchNormalization()(output) act_output = Activation('sigmoid')(output) model = Model(inputs=embed_input, outputs=act_output) return model
def create_actor_network(self, state_size, action_dim): """Create actor network.""" print ("[MESSAGE] Build actor network.""") S = Input(shape=state_size) h_0 = Conv2D(32, (3, 3), padding="same", kernel_regularizer=l2(0.0001), activation="relu")(S) h_1 = Conv2D(32, (3, 3), padding="same", kernel_regularizer=l2(0.0001), activation="relu")(h_0) h_1 = AveragePooling2D(2, 2)(h_1) h_1 = Flatten()(h_1) h_1 = Dense(600, activation="relu")(h_1) A = Dense(action_dim, activation="softmax")(h_1) model = Model(inputs=S, outputs=A) return model, model.trainable_weights, S
def test_dense(): from keras import regularizers from keras import constraints layer_test(core.Dense, kwargs={'output_dim': 3}, input_shape=(3, 2)) layer_test(core.Dense, kwargs={'output_dim': 3, 'W_regularizer': regularizers.l2(0.01), 'b_regularizer': regularizers.l1(0.01), 'activity_regularizer': regularizers.activity_l2(0.01), 'W_constraint': constraints.MaxNorm(1), 'b_constraint': constraints.MaxNorm(1)}, input_shape=(3, 2))
def test_maxout_dense(): from keras import regularizers from keras import constraints layer_test(core.MaxoutDense, kwargs={'output_dim': 3}, input_shape=(3, 2)) layer_test(core.MaxoutDense, kwargs={'output_dim': 3, 'W_regularizer': regularizers.l2(0.01), 'b_regularizer': regularizers.l1(0.01), 'activity_regularizer': regularizers.activity_l2(0.01), 'W_constraint': constraints.MaxNorm(1), 'b_constraint': constraints.MaxNorm(1)}, input_shape=(3, 2))
def test_timedistributeddense(): from keras import regularizers from keras import constraints layer_test(core.TimeDistributedDense, kwargs={'output_dim': 2, 'input_length': 2}, input_shape=(3, 2, 3)) layer_test(core.TimeDistributedDense, kwargs={'output_dim': 3, 'W_regularizer': regularizers.l2(0.01), 'b_regularizer': regularizers.l1(0.01), 'activity_regularizer': regularizers.activity_l2(0.01), 'W_constraint': constraints.MaxNorm(1), 'b_constraint': constraints.MaxNorm(1)}, input_shape=(3, 2, 3))
def _conv_bn_relu(**conv_params): """Helper to build a conv -> BN -> relu residual unit activation function. This is the original ResNet v1 scheme in https://arxiv.org/abs/1512.03385 """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) dilation_rate = conv_params.setdefault("dilation_rate", (1, 1)) conv_name = conv_params.setdefault("conv_name", None) bn_name = conv_params.setdefault("bn_name", None) relu_name = conv_params.setdefault("relu_name", None) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(x): x = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, dilation_rate=dilation_rate, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer, name=conv_name)(x) return _bn_relu(x, bn_name=bn_name, relu_name=relu_name) return f
def _bn_relu_conv(**conv_params): """Helper to build a BN -> relu -> conv residual unit with full pre-activation function. This is the ResNet v2 scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) dilation_rate = conv_params.setdefault("dilation_rate", (1, 1)) conv_name = conv_params.setdefault("conv_name", None) bn_name = conv_params.setdefault("bn_name", None) relu_name = conv_params.setdefault("relu_name", None) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(x): activation = _bn_relu(x, bn_name=bn_name, relu_name=relu_name) return Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, dilation_rate=dilation_rate, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer, name=conv_name)(activation) return f
def _conv_bn_relu(**conv_params): """Helper to build a conv -> BN -> relu block """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(input): conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer)(input) return _bn_relu(conv) return f
def _bn_relu_conv(**conv_params): """Helper to build a BN -> relu -> conv block. This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf """ filters = conv_params["filters"] kernel_size = conv_params["kernel_size"] strides = conv_params.setdefault("strides", (1, 1)) kernel_initializer = conv_params.setdefault("kernel_initializer", "he_normal") padding = conv_params.setdefault("padding", "same") kernel_regularizer = conv_params.setdefault("kernel_regularizer", l2(1.e-4)) def f(input): activation = _bn_relu(input) return Conv2D(filters=filters, kernel_size=kernel_size, strides=strides, padding=padding, kernel_initializer=kernel_initializer, kernel_regularizer=kernel_regularizer)(activation) return f
def _shortcut(input, residual): """Adds a shortcut between input and residual block and merges them with "sum" """ # Expand channels of shortcut to match residual. # Stride appropriately to match residual (width, height) # Should be int if network architecture is correctly configured. input_shape = K.int_shape(input) residual_shape = K.int_shape(residual) stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS])) stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS])) equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS] shortcut = input # 1 X 1 conv if shape is different. Else identity. if stride_width > 1 or stride_height > 1 or not equal_channels: shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS], kernel_size=(1, 1), strides=(stride_width, stride_height), padding="valid", kernel_initializer="he_normal", kernel_regularizer=l2(0.0001))(input) return add([shortcut, residual])
def basic_block(filters, init_strides=(1, 1), is_first_block_of_first_layer=False): """Basic 3 X 3 convolution blocks for use on resnets with layers <= 34. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv1 = Conv2D(filters=filters, kernel_size=(3, 3), strides=init_strides, padding="same", kernel_initializer="he_normal", kernel_regularizer=l2(1e-4))(input) else: conv1 = _bn_relu_conv(filters=filters, kernel_size=(3, 3), strides=init_strides)(input) residual = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv1) return _shortcut(input, residual) return f
def bottleneck(filters, init_strides=(1, 1), is_first_block_of_first_layer=False): """Bottleneck architecture for > 34 layer resnet. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf Returns: A final conv layer of filters * 4 """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv_1_1 = Conv2D(filters=filters, kernel_size=(1, 1), strides=init_strides, padding="same", kernel_initializer="he_normal", kernel_regularizer=l2(1e-4))(input) else: conv_1_1 = _bn_relu_conv(filters=filters, kernel_size=(1, 1), strides=init_strides)(input) conv_3_3 = _bn_relu_conv(filters=filters, kernel_size=(3, 3))(conv_1_1) residual = _bn_relu_conv(filters=filters * 4, kernel_size=(1, 1))(conv_3_3) return _shortcut(input, residual) return f
def gen_Model(num_units, actfn='linear', reg_coeff=0.0, last_act='softmax'): ''' Generate a neural network model of approporiate architecture Args: num_units: architecture of network in the format [n1, n2, ... , nL] actfn: activation function for hidden layers ('relu'/'sigmoid'/'linear'/'softmax') reg_coeff: L2-regularization coefficient last_act: activation function for final layer ('relu'/'sigmoid'/'linear'/'softmax') Output: model: Keras sequential model with appropriate fully-connected architecture ''' model = Sequential() for i in range(1, len(num_units)): if i == 1 and i < len(num_units) - 1: model.add(Dense(input_dim=num_units[0], output_dim=num_units[i], activation=actfn, W_regularizer=Reg.l2(l=reg_coeff), init='glorot_normal')) elif i == 1 and i == len(num_units) - 1: model.add(Dense(input_dim=num_units[0], output_dim=num_units[i], activation=last_act, W_regularizer=Reg.l2(l=reg_coeff), init='glorot_normal')) elif i < len(num_units) - 1: model.add(Dense(output_dim=num_units[i], activation=actfn, W_regularizer=Reg.l2(l=reg_coeff), init='glorot_normal')) elif i == len(num_units) - 1: model.add(Dense(output_dim=num_units[i], activation=last_act, W_regularizer=Reg.l2(l=reg_coeff), init='glorot_normal')) return model
def cnn(height, width): question_input = Input(shape=(height, width, 1), name='question_input') conv1_Q = Conv2D(512, (2, 320), activation='sigmoid', padding='valid', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.random_normal(mean=0.0, stddev=0.02))(question_input) Max1_Q = MaxPooling2D((29, 1), strides=(1, 1), padding='valid')(conv1_Q) F1_Q = Flatten()(Max1_Q) Drop1_Q = Dropout(0.25)(F1_Q) predictQ = Dense(32, activation='relu', kernel_regularizer=regularizers.l2(0.01), kernel_initializer=initializers.random_normal(mean=0.0, stddev=0.02))(Drop1_Q) prediction2 = Dropout(0.25)(predictQ) predictions = Dense(1, activation='relu')(prediction2) model = Model(inputs=[question_input], outputs=predictions) model.compile(loss='mean_squared_error', optimizer=Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)) # model.compile(loss='mean_squared_error', # optimizer='nadam') return model
def test_keras_import(self): # Conv 1D model = Sequential() model.add(LocallyConnected1D(32, 3, kernel_regularizer=regularizers.l2(0.01), bias_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l2(0.01), kernel_constraint='max_norm', bias_constraint='max_norm', activation='relu', input_shape=(10, 16))) model.build() self.keras_param_test(model, 1, 12) # Conv 2D model = Sequential() model.add(LocallyConnected2D(32, (3, 3), kernel_regularizer=regularizers.l2(0.01), bias_regularizer=regularizers.l2(0.01), activity_regularizer=regularizers.l2(0.01), kernel_constraint='max_norm', bias_constraint='max_norm', activation='relu', input_shape=(10, 16, 16))) model.build() self.keras_param_test(model, 1, 14) # ********** Recurrent Layers **********
def test_keras_import(self): model = Sequential() model.add(BatchNormalization(center=True, scale=True, beta_regularizer=regularizers.l2(0.01), gamma_regularizer=regularizers.l2(0.01), beta_constraint='max_norm', gamma_constraint='max_norm', input_shape=(10, 16))) model.build() json_string = Model.to_json(model) with open(os.path.join(settings.BASE_DIR, 'media', 'test.json'), 'w') as out: json.dump(json.loads(json_string), out, indent=4) sample_file = open(os.path.join(settings.BASE_DIR, 'media', 'test.json'), 'r') response = self.client.post(reverse('keras-import'), {'file': sample_file}) response = json.loads(response.content) layerId = sorted(response['net'].keys()) self.assertEqual(response['result'], 'success') self.assertEqual(response['net'][layerId[0]]['info']['type'], 'Scale') self.assertEqual(response['net'][layerId[1]]['info']['type'], 'BatchNorm') # ********** Noise Layers **********
def feature_extractor(FLAGS, suffix = ""): weights = FLAGS.weights if FLAGS.weights != "random" else None if FLAGS.model == "vgg16": from keras.applications.vgg16 import VGG16 feature_extractor = VGG16(weights = weights) remove_last_layer(feature_extractor) elif FLAGS.model == "vgg19": from keras.applications.vgg19 import VGG19 feature_extractor = VGG19(weights = weights) remove_last_layer(feature_extractor) elif FLAGS.model == "resnet50": from keras.applications.resnet50 import ResNet50 feature_extractor = ResNet50(weights = weights) remove_last_layer(feature_extractor) else: raise NotImplementedError feature_extractor.name = FLAGS.model + suffix if FLAGS.regularizer == "l2": add_regularizer(feature_extractor) elif FLAGS.regularizer != "none": raise NotImplementedError return feature_extractor
def keepsize_256(nx, ny, noise, depth, activation='relu', n_filters=64, l2_reg=1e-7): """ Deep residual network that keeps the size of the input throughout the whole network """ def residual(inputs, n_filters): x = ReflectionPadding2D()(inputs) x = Conv2D(n_filters, (3, 3), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) x = BatchNormalization()(x) x = Activation(activation)(x) x = ReflectionPadding2D()(x) x = Conv2D(n_filters, (3, 3), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) x = BatchNormalization()(x) x = add([x, inputs]) return x inputs = Input(shape=(nx, ny, 1)) x = GaussianNoise(noise)(inputs) x = ReflectionPadding2D()(x) x = Conv2D(n_filters, (3, 3), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) x0 = Activation(activation)(x) x = residual(x0, n_filters) for i in range(depth-1): x = residual(x, n_filters) x = ReflectionPadding2D()(x) x = Conv2D(n_filters, (3, 3), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) x = BatchNormalization()(x) x = add([x, x0]) # Upsampling for superresolution x = UpSampling2D()(x) x = ReflectionPadding2D()(x) x = Conv2D(4*n_filters, (3, 3), padding='valid', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) x = Activation(activation)(x) final = Conv2D(1, (1, 1), padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(l2_reg))(x) return Model(inputs=inputs, outputs=final)
def prep_model(inputs, N, s0pad, s1pad, c, granlevels=1): # LSTM lstm = LSTM(N, return_sequences=True, implementation=2, kernel_regularizer=l2(c['l2reg']), recurrent_regularizer=l2(c['l2reg']), bias_regularizer=l2(c['l2reg'])) x1 = inputs[0] x2 = inputs[1] h1 = lstm(x1) h2 = lstm(x2) W_x = Dense(N, kernel_initializer='glorot_uniform', use_bias=True, kernel_regularizer=l2(c['l2reg'])) W_h = Dense(N, kernel_initializer='orthogonal', use_bias=True, kernel_regularizer=l2(c['l2reg'])) sigmoid = Activation('sigmoid') a1 = multiply([x1, sigmoid( add([W_x(x1), W_h(h1)]) )]) a2 = multiply([x2, sigmoid( add([W_x(x2), W_h(h2)]) )]) # Averaging avg = Lambda(function=lambda x: K.mean(x, axis=1), output_shape=lambda shape: (shape[0], ) + shape[2:]) gran1 = avg(a1) gran2 = avg(a2) return [gran1, gran2], N
def prep_model(inputs, N, s0pad, s1pad, c): Nc, outputs = B.cnnsum_input(inputs, N, s0pad, siamese=c['cnnsiamese'], dropout=c['dropout'], l2reg=c['l2reg'], cnninit=c['cnninit'], cnnact=c['cnnact'], cdim=c['cdim']) # Projection if c['project']: outputs = Dense(int(N*c['pdim']), kernal_regularizer=l2(c['l2reg']), activation=c['pact'])(outputs) # model.add_shared_node(name='proj', inputs=['e0s_', 'e1s_'], outputs=['e0p', 'e1p'], # layer=Dense(input_dim=Nc, output_dim=int(N*c['pdim']), # W_regularizer=l2(c['l2reg']), activation=c['pact'])) # This dropout is controversial; it might be harmful to apply, # or at least isn't a clear win. # model.add_shared_node(name='projdrop', inputs=['e0p', 'e1p'], outputs=['e0p_', 'e1p_'], # layer=Dropout(c['dropout'], input_shape=(N,))) # return ('e0p_', 'e1p_') return outputs, N
def prep_model(inputs, N, s0pad, s1pad, c): outputs = B.rnn_input(inputs, N, s0pad, dropout=c['dropout'], dropoutfix_inp=c['dropoutfix_inp'], dropoutfix_rec=c['dropoutfix_rec'], sdim=c['sdim'], rnnbidi=c['rnnbidi'], rnn=c['rnn'], rnnact=c['rnnact'], rnninit=c['rnninit'], rnnbidi_mode=c['rnnbidi_mode'], rnnlevels=c['rnnlevels']) # Projection if c['project']: proj = Dense(int(N*c['pdim']), activation=c['pact'], kernel_regularizer=l2(c['l2reg']), name='proj') e0p = proj(outputs[0]) e1p = proj(outputs[1]) N = N*c['pdim'] return [e0p, e1p], N else: return [outputs[0], outputs[1]], N #input_dim=int(N*c['sdim'])
def _conv_bn_relu(**conv_params): """Helper to build a conv -> BN -> relu block """ nb_filter = conv_params["nb_filter"] nb_row = conv_params["nb_row"] nb_col = conv_params["nb_col"] subsample = conv_params.setdefault("subsample", (1, 1)) init = conv_params.setdefault("init", "he_normal") border_mode = conv_params.setdefault("border_mode", "same") W_regularizer = conv_params.setdefault("W_regularizer", l2(1.e-4)) def f(input): conv = Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample, init=init, border_mode=border_mode, W_regularizer=W_regularizer)(input) return _bn_relu(conv) return f
def _bn_relu_conv(**conv_params): """Helper to build a BN -> relu -> conv block. This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf """ nb_filter = conv_params["nb_filter"] nb_row = conv_params["nb_row"] nb_col = conv_params["nb_col"] subsample = conv_params.setdefault("subsample", (1, 1)) init = conv_params.setdefault("init", "he_normal") border_mode = conv_params.setdefault("border_mode", "same") W_regularizer = conv_params.setdefault("W_regularizer", l2(1.e-4)) def f(input): activation = _bn_relu(input) return Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample, init=init, border_mode=border_mode, W_regularizer=W_regularizer)(activation) return f
def _shortcut(input, residual): """Adds a shortcut between input and residual block and merges them with "sum" """ # Expand channels of shortcut to match residual. # Stride appropriately to match residual (width, height) # Should be int if network architecture is correctly configured. input_shape = K.int_shape(input) residual_shape = K.int_shape(residual) stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS])) stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS])) equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS] shortcut = input # 1 X 1 conv if shape is different. Else identity. if stride_width > 1 or stride_height > 1 or not equal_channels: shortcut = Convolution2D(nb_filter=residual_shape[CHANNEL_AXIS], nb_row=1, nb_col=1, subsample=(stride_width, stride_height), init="he_normal", border_mode="valid", W_regularizer=l2(0.0001))(input) return merge([shortcut, residual], mode="sum")
def ResidualBlock1D_helper(layers, kernel_size, filters, final_stride=1): def f(_input): basic = _input for ln in range(layers): #basic = BatchNormalization()( basic ) # triggers known keras bug w/ TimeDistributed: https://github.com/fchollet/keras/issues/5221 basic = ELU()(basic) basic = Conv1D(filters, kernel_size, kernel_initializer='he_normal', kernel_regularizer=l2(1.e-4), padding='same')(basic) # note that this strides without averaging return AveragePooling1D(pool_size=1, strides=final_stride)(Add()([_input, basic])) return f
def createModel(w=None,h=None): # Input placeholder original = Input(shape=(w, h, 4), name='icon_goes_here') # Model layer stack x = original x = Convolution2D(64, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) x = Convolution2D(64, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) x = Convolution2D(64, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) x = Convolution2D(64, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) x = AveragePooling2D((2, 2), border_mode='valid')(x) x = Convolution2D(16, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) x = Convolution2D(4, 4, 4, activation='relu', border_mode='same', b_regularizer=l2(0.1))(x) downscaled = x # Compile model hintbot = Model(input=original, output=downscaled) hintbot.compile(optimizer='adam', loss='mean_squared_error') # Train if (os.path.isfile(load_weights_filepath)): hintbot.load_weights(load_weights_filepath) return hintbot
def _model(self, input_shape): self.model.add(Dense(self.hidden[0], input_shape=(input_shape[1],), kernel_regularizer=l2(self.wd), kernel_initializer=self.ki)) if self.bn: self.model.add(BatchNormalization(axis=1)) self.model.add(Activation(self.activation)) for i in self.hidden[1:]: self.model.add(Dense(i, kernel_regularizer=l2(self.wd), kernel_initializer=self.ki)) if self.bn: self.model.add(BatchNormalization(axis=1)) self.model.add(Activation(self.activation)) self.model.add(Dense(self.N, activation='softmax', kernel_regularizer=l2(self.wd), kernel_initializer=self.ki))
def _shortcut(input, residual, weight_decay=.0001, dropout=.0): # Expand channels of shortcut to match residual. # Stride appropriately to match residual (width, height) # Should be int if network architecture is correctly configured. # !!! The dropout argument is just a place holder. # !!! It shall not be applied to identity mapping. stride_width = input._keras_shape[ROW_AXIS] // residual._keras_shape[ROW_AXIS] stride_height = input._keras_shape[COL_AXIS] // residual._keras_shape[COL_AXIS] equal_channels = residual._keras_shape[CHANNEL_AXIS] == input._keras_shape[CHANNEL_AXIS] shortcut = input # 1 X 1 conv if shape is different. Else identity. if stride_width > 1 or stride_height > 1 or not equal_channels: shortcut = Convolution2D(nb_filter=residual._keras_shape[CHANNEL_AXIS], nb_row=1, nb_col=1, subsample=(stride_width, stride_height), init="he_normal", border_mode="valid", W_regularizer=l2(weight_decay))(input) return merge([shortcut, residual], mode="sum") # Builds a residual block with repeating bottleneck blocks.
def _bn_relu_conv(**conv_params): """Helper to build a BN -> relu -> conv block. This is an improved scheme proposed in http://arxiv.org/pdf/1603.05027v2.pdf """ nb_filter = conv_params["nb_filter"] nb_row = conv_params["nb_row"] nb_col = conv_params["nb_col"] subsample = conv_params.setdefault("subsample", (1,1)) init = conv_params.setdefault("init", "he_normal") border_mode = conv_params.setdefault("border_mode", "same") W_regularizer = conv_params.setdefault("W_regularizer", l2(1.e-4)) def f(input): activation = _bn_relu(input) return Convolution2D(nb_filter=nb_filter, nb_row=nb_row, nb_col=nb_col, subsample=subsample, init=init, border_mode=border_mode, W_regularizer=W_regularizer)(activation) return f
def basic_block(nb_filter, init_subsample=(1, 1), is_first_block_of_first_layer=False): """Basic 3 X 3 convolution blocks for use on resnets with layers <= 34. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv1 = Convolution2D(nb_filter=nb_filter, nb_row=3, nb_col=3, subsample=init_subsample, init="he_normal", border_mode="same", W_regularizer=l2(0.0001))(input) else: conv1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3, subsample=init_subsample)(input) residual = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3)(conv1) return _shortcut(input, residual) return f
def bottleneck(nb_filter, init_subsample=(1, 1), is_first_block_of_first_layer=False): """Bottleneck architecture for > 34 layer resnet. Follows improved proposed scheme in http://arxiv.org/pdf/1603.05027v2.pdf Returns: A final conv layer of nb_filter * 4 """ def f(input): if is_first_block_of_first_layer: # don't repeat bn->relu since we just did bn->relu->maxpool conv_1_1 = Convolution2D(nb_filter=nb_filter, nb_row=1, nb_col=1, subsample=init_subsample, init="he_normal", border_mode="same", W_regularizer=l2(0.0001))(input) else: conv_1_1 = _bn_relu_conv(nb_filter=nb_filter, nb_row=1, nb_col=1, subsample=init_subsample)(input) conv_3_3 = _bn_relu_conv(nb_filter=nb_filter, nb_row=3, nb_col=3)(conv_1_1) residual = _bn_relu_conv(nb_filter=nb_filter * 4, nb_row=1, nb_col=1)(conv_3_3) return _shortcut(input, residual) return f
def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4): ''' Apply BatchNorm, Relu 1x1, Conv2D, optional compression, dropout and Maxpooling2D Args: ip: keras tensor nb_filter: number of filters compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. dropout_rate: dropout rate weight_decay: weight decay factor Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool ''' concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(ip) x = Activation('relu')(x) x = Conv2D(int(nb_filter * compression), (1, 1), kernel_initializer='he_normal', padding='same', use_bias=False, kernel_regularizer=l2(weight_decay))(x) x = AveragePooling2D((2, 2), strides=(2, 2))(x) return x
def conv_block(ip, nb_filter, dropout_rate=None, weight_decay=1E-4): ''' Apply BatchNorm, Relu 3x3, Conv2D, optional dropout Args: ip: Input keras tensor nb_filter: number of filters dropout_rate: dropout rate weight_decay: weight decay factor Returns: keras tensor with batch_norm, relu and convolution2d added ''' x = Activation('relu')(ip) x = Convolution2D(nb_filter, 3, 3, init="he_uniform", border_mode="same", bias=False, W_regularizer=l2(weight_decay))(x) if dropout_rate: x = Dropout(dropout_rate)(x) return x
def transition_block(ip, nb_filter, dropout_rate=None, weight_decay=1E-4): ''' Apply BatchNorm, Relu 1x1, Conv2D, optional dropout and Maxpooling2D Args: ip: keras tensor nb_filter: number of filters dropout_rate: dropout rate weight_decay: weight decay factor Returns: keras tensor, after applying batch_norm, relu-conv, dropout, maxpool ''' concat_axis = 1 if K.image_dim_ordering() == "th" else -1 x = Convolution2D(nb_filter, 1, 1, init="he_uniform", border_mode="same", bias=False, W_regularizer=l2(weight_decay))(ip) if dropout_rate: x = Dropout(dropout_rate)(x) x = AveragePooling2D((2, 2), strides=(2, 2))(x) x = BatchNormalization(mode=0, axis=concat_axis, gamma_regularizer=l2(weight_decay), beta_regularizer=l2(weight_decay))(x) return x
def build(self): dim_data = self.size_of_input_data_dim nb_time_step = self.size_of_input_timesteps financial_time_series_input = Input(shape=(nb_time_step, dim_data)) lstm_layer_1 = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout, inner_activation='sigmoid', W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha), activation='tanh', return_sequences=True) lstm_layer_2 = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout, inner_activation='sigmoid', W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha), activation='tanh', return_sequences=True) h1 = lstm_layer_1(financial_time_series_input) h2 = lstm_layer_2(h1) time_series_predictions = TimeDistributedDense(1)(h2) self.model = Model(financial_time_series_input, time_series_predictions, name="deep rnn for financial time series forecasting")
