Python keras.layers 模块，GlobalAveragePooling2D() 实例源码

def build_model(nb_classes):
    base_model = InceptionV3(weights='imagenet', include_top=False)

    # add a global spatial average pooling layer
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    # let's add a fully-connected layer
    x = Dense(1024, activation='relu')(x)
    # and a logistic layer
    predictions = Dense(nb_classes, activation='softmax')(x)

    # this is the model we will train
    model = Model(input=base_model.input, output=predictions)

    # first: train only the top layers (which were randomly initialized)
    # i.e. freeze all convolutional InceptionV3 layers
    for layer in base_model.layers:
        layer.trainable = False

    # compile the model (should be done *after* setting layers to non-trainable)
    print "starting model compile"
    compile(model)
    print "model compile done"
    return model

def get_model():
    input_shape = (image_size, image_size, 3)

    model = Sequential()

    model.add(Conv2D(32, kernel_size=(3, 3), padding='same',
                     input_shape=input_shape))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(64, kernel_size=(3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(128, kernel_size=(3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(n_classes, kernel_size=(3, 3), padding='same'))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(GlobalAveragePooling2D())

    print (model.summary())
    #sys.exit(0) #

    model.compile(loss=keras.losses.mean_squared_error,
            optimizer= keras.optimizers.Adadelta())

    return model

def build_image_model():
  base_model = InceptionV3(weights='imagenet', include_top=False)

   # Freeze Inception's weights - we don't want to train these
  for layer in base_model.layers:
     layer.trainable = False

  # add a fully connected layer after Inception - we do want to train these
  x = base_model.output
  x = GlobalAveragePooling2D()(x)
  x = Dense(2048, activation='relu')(x)

  return x, base_model.input


# Build the two models.

def add_new_last_layer(base_model, nb_classes):
  """Add last layer to the convnet

  Args:
    base_model: keras model excluding top
    nb_classes: # of classes

  Returns:
    new keras model with last layer
  """
  x = base_model.output
  x = GlobalAveragePooling2D()(x)
  x = Dense(FC_SIZE, activation='relu')(x) #new FC layer, random init
  predictions = Dense(nb_classes, activation='softmax')(x) #new softmax layer
  model = Model(input=base_model.input, output=predictions)
  return model

def mnist_discriminator(input_shape=(28, 28, 1), scale=1/4):
    x0 = Input(input_shape)
    x = Conv2D(int(128*scale), (3, 3), strides=(2, 2), padding='same')(x0)
    x = InstanceNormalization()(x)
    x = LeakyReLU()(x)
    x = Conv2D(int(64*scale), (3, 3), strides=(2, 2), padding='same')(x)
    x = InstanceNormalization()(x)
    x = LeakyReLU()(x)
    x = residual_block(x, scale, num_id=2)
    x = residual_block(x, scale*2, num_id=3)
    x = Conv2D(int(128*scale), (3, 3), strides=(2, 2), padding='same')(x)
    x = InstanceNormalization()(x)
    x = LeakyReLU()(x)
    x = Conv2D(int(128*scale), (3, 3), strides=(2, 2), padding='same')(x)
    x = InstanceNormalization()(x)
    x = LeakyReLU()(x)
    x = Conv2D(1, (3, 3), strides=(2, 2), padding='same')(x)
    x = GlobalAveragePooling2D()(x) # Flatten
    x = Activation('sigmoid')(x)
    return Model(x0, x)

def topmodel(model, h):
        '''
        Define topmodel
        if BN_Flag is True, BN is used instead of Dropout
        '''
        BN_flag = model.BN_flag

        n_dense = model.n_dense
        p_dropout = model.p_dropout

        h = GlobalAveragePooling2D()(h)
        h = Dense(n_dense, activation='relu')(h)
        if BN_flag:
            h = BatchNormalization()(h)
        else:
            h = Dropout(p_dropout)(h)
        return h

def get_model(weights='imagenet'):
    # create the base pre-trained model
    base_model = InceptionV3(weights=weights, include_top=False)

    # add a global spatial average pooling layer
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    # let's add a fully-connected layer
    x = Dense(1024, activation='relu')(x)
    # and a logistic layer -- let's say we have 2 classes
    predictions = Dense(len(data.classes), activation='softmax')(x)

    # this is the model we will train
    model = Model(inputs=base_model.input, outputs=predictions)

    for layer in base_model.layers:
        layer.trainable = False

    # compile the model (should be done *after* setting layers to non-trainable)
    model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

    return model

def build_model(nb_classes):
    base_model = InceptionV3(weights='imagenet', include_top=False)

    # add a global spatial average pooling layer
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    # let's add a fully-connected layer
    x = Dense(1024, activation='relu')(x)
    # and a logistic layer
    predictions = Dense(nb_classes, activation='softmax')(x)

    # this is the model we will train
    model = Model(input=base_model.input, output=predictions)

    # first: train only the top layers (which were randomly initialized)
    # i.e. freeze all convolutional InceptionV3 layers
    for layer in base_model.layers:
        layer.trainable = False

    # compile the model (should be done *after* setting layers to non-trainable)
    print "starting model compile"
    compile(model)
    print "model compile done"
    return model

def make_model(self):
        # create the base pre-trained model
        if self.model_architecture == 'vgg16':
            from keras.applications.vgg16 import VGG16
            self.base_model = VGG16(weights='imagenet', include_top=False)
        elif self.model_architecture == 'resnet':
            from keras.applications.resnet50 import ResNet50
            self.base_model = ResNet50(weights='imagenet', include_top=False)
        elif self.model_architecture == 'inception':
            from keras.applications.inception_v3 import InceptionV3
            self.base_model = InceptionV3(weights='imagenet', include_top=False)
        else:
            print 'Model architecture parameter unknown. Options are: vgg16, resnet, and inception'
            rospy.signal_shutdown("Model architecture unknown.")

        # now we add a new dense layer to the end of the network inplace of the old layers
        x = self.base_model.output
        x = GlobalAveragePooling2D()(x)
        x = Dense(1024, activation='relu')(x)
        # add the outplut layer
        predictions = Dense(len(self.categories.keys()), activation='softmax')(x)

        # create new model composed of pre-trained network and new final layers
        # if you want to change the input size, you can do this with the input parameter below
        self.model = Model(input=self.base_model.input, output=predictions)

        # now we go through and freeze all of the layers that were pretrained
        for layer in self.base_model.layers:
            layer.trainable = False

        if self.verbose:
            print 'compiling model ... '
            start_time = time.time()

        # in finetuning, these parameters can matter a lot, it is wise to observe 
        # how well your model is learning for this to work well
        self.model.compile(optimizer=self.optimizer, loss='categorical_crossentropy', metrics=['accuracy'])

        if self.verbose:
            end_time = time.time()
            self.print_time(start_time,end_time,'compiling model')

def _add_auxiliary_head(x, classes, weight_decay):
    '''Adds an auxiliary head for training the model

    From section A.7 "Training of ImageNet models" of the paper, all NASNet models are
    trained using an auxiliary classifier around 2/3 of the depth of the network, with
    a loss weight of 0.4

    # Arguments
        x: input tensor
        classes: number of output classes
        weight_decay: l2 regularization weight

    # Returns
        a keras Tensor
    '''
    img_height = 1 if K.image_data_format() == 'channels_last' else 2
    img_width = 2 if K.image_data_format() == 'channels_last' else 3
    channel_axis = 1 if K.image_data_format() == 'channels_first' else -1

    with K.name_scope('auxiliary_branch'):
        auxiliary_x = Activation('relu')(x)
        auxiliary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxiliary_x)
        auxiliary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection',
                             kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(auxiliary_x)
        auxiliary_x = BatchNormalization(axis=channel_axis, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
                                         name='aux_bn_projection')(auxiliary_x)
        auxiliary_x = Activation('relu')(auxiliary_x)

        auxiliary_x = Conv2D(768, (auxiliary_x._keras_shape[img_height], auxiliary_x._keras_shape[img_width]),
                             padding='valid', use_bias=False, kernel_initializer='he_normal',
                             kernel_regularizer=l2(weight_decay), name='aux_conv_reduction')(auxiliary_x)
        auxiliary_x = BatchNormalization(axis=channel_axis, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
                                         name='aux_bn_reduction')(auxiliary_x)
        auxiliary_x = Activation('relu')(auxiliary_x)

        auxiliary_x = GlobalAveragePooling2D()(auxiliary_x)
        auxiliary_x = Dense(classes, activation='softmax', kernel_regularizer=l2(weight_decay),
                            name='aux_predictions')(auxiliary_x)
    return auxiliary_x

def get_model(weights='imagenet'):
    # create the base pre-trained model
    base_model = InceptionV3(weights=weights, include_top=False)

    # add a global spatial average pooling layer
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    # let's add a fully-connected layer
    x = Dense(1024, activation='relu')(x)
    # and a logistic layer
    predictions = Dense(len(data.classes), activation='softmax')(x)

    # this is the model we will train
    model = Model(inputs=base_model.input, outputs=predictions)
    return model

def test_delete_channels_globalaveragepooling2d(channel_index, data_format):
    layer = GlobalAveragePooling2D(data_format=data_format)
    layer_test_helper_2d_global(layer, channel_index, data_format)

def squared_root_normalization(x):
    """
    Squared root normalization for convolution layers` output
    first apply global average pooling followed by squared root on all elements
    then l2 normalize the vector

    :param x: input tensor, output of convolution layer
    :return:
    """
    x = GlobalAveragePooling2D()(x)
    #output shape = (None, nc)
   # x = K.sqrt(x)
    #x = K.l2_normalize(x, axis=0)
    return x

def model1(weights_path=None):
    '''
    Basic ResNet-FT for baseline comparisions.
    Creates a model by for all aesthetic attributes along
    with overall aesthetic score, by finetuning resnet50
    :param weights_path: path of the weight file
    :return: Keras model instance
    '''
    _input = Input(shape=(299, 299, 3))
    resnet = ResNet50(include_top=False, weights='imagenet', input_tensor=_input)

    last_layer_output = GlobalAveragePooling2D()(resnet.get_layer('activation_49').output)

    # output of model
    outputs = []
    attrs = ['BalacingElements', 'ColorHarmony', 'Content', 'DoF',
             'Light', 'MotionBlur', 'Object', 'RuleOfThirds', 'VividColor']
    for attribute in attrs:
        outputs.append(Dense(1, init='glorot_uniform', activation='tanh', name=attribute)(last_layer_output))

    non_negative_attrs = ['Repetition', 'Symmetry', 'score']
    for attribute in non_negative_attrs:
        outputs.append(Dense(1, init='glorot_uniform', activation='sigmoid', name=attribute)(last_layer_output))

    model = Model(input=_input, output=outputs)
    if weights_path:
        model.load_weights(weights_path)
    return model

def test_keras_model(num_classes):

    bounds = (0, 255)
    channels = num_classes

    model = Sequential()
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        model.add(GlobalAveragePooling2D(
            data_format='channels_last', input_shape=(5, 5, channels)))

        model = KerasModel(
            model,
            bounds=bounds,
            predicts='logits')

    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)
    test_label = 7

    assert model.batch_predictions(test_images).shape \
        == (2, num_classes)

    test_logits = model.predictions(test_images[0])
    assert test_logits.shape == (num_classes,)

    test_gradient = model.gradient(test_images[0], test_label)
    assert test_gradient.shape == test_images[0].shape

    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[0],
        test_logits)
    np.testing.assert_almost_equal(
        model.predictions_and_gradient(test_images[0], test_label)[1],
        test_gradient)

    assert model.num_classes() == num_classes

def test_keras_model_gradients():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        inputs = Input(shape=(5, 5, channels))
        logits = GlobalAveragePooling2D(
            data_format='channels_last')(inputs)

        preprocessing = (np.arange(num_classes)[None, None],
                         np.random.uniform(size=(5, 5, channels)) + 1)

        model = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits',
            preprocessing=preprocessing)

    eps = 1e-3

    np.random.seed(22)
    test_image = np.random.rand(5, 5, channels).astype(np.float32)
    test_label = 7

    _, g1 = model.predictions_and_gradient(test_image, test_label)

    test_label_array = np.array([test_label])
    l1 = model._loss_fn([test_image[None] - eps / 2 * g1, test_label_array])[0]
    l2 = model._loss_fn([test_image[None] + eps / 2 * g1, test_label_array])[0]

    assert 1e5 * (l2 - l1) > 1

    # make sure that gradient is numerically correct
    np.testing.assert_array_almost_equal(
        1e5 * (l2 - l1),
        1e5 * eps * np.linalg.norm(g1)**2,
        decimal=1)

def test_keras_backward(num_classes):

    bounds = (0, 255)
    channels = num_classes

    model = Sequential()
    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        model.add(GlobalAveragePooling2D(
            data_format='channels_last', input_shape=(5, 5, channels)))

        model = KerasModel(
            model,
            bounds=bounds,
            predicts='logits')

    test_image = np.random.rand(5, 5, channels).astype(np.float32)
    test_grad_pre = np.random.rand(num_classes).astype(np.float32)

    test_grad = model.backward(test_grad_pre, test_image)
    assert test_grad.shape == test_image.shape

    manual_grad = np.repeat(np.repeat(
        (test_grad_pre / 25.).reshape((1, 1, -1)),
        5, axis=0), 5, axis=1)

    np.testing.assert_almost_equal(
        test_grad,
        manual_grad)

def test_global_average_pooling(self):
        model = Sequential()
        model.add(GlobalAveragePooling2D(input_shape=(16,16,3)))
        self._test_keras_model(model)

def maps_pred_fun(checkpoint):
    # Load model
    model = load_model(checkpoint)
    x = model.input
    # Get feature maps before GAP
    o = [l for l in model.layers if type(l) == GlobalAveragePooling2D][-1].input

    # Setup CAM
    dense_list = [l for l in model.layers if type(l) == Dense]
    num_dense = len(dense_list)
    if num_dense > 1:
        raise ValueError('Expected only one dense layer, found %d' %num_dense)
    # If there is no dense layer after (NiN), the maps are already class maps
    if num_dense: # Apply CAM if there is a dense layer
        dense_layer = dense_list[0]
        # Get dense layer weights
        W = K.get_value(dense_layer.W)[None, None] # (1, 1, ?, ?)
        b = K.get_value(dense_layer.b)

        # Transform it into a 1x1 conv
        # This convolution will map the feature maps into class 'heatmaps'
        o = Convolution2D(W.shape[-1], 1, 1, border_mode='valid', weights=[W, b])(o)

    # Resize with bilinear method
    maps = tf.image.resize_images(o, K.shape(x)[1:3], method=tf.image.ResizeMethod.BILINEAR)
    return K.function([x, K.learning_phase()], [maps, model.output])

def inceptionV3(input_dim):
    base_model = InceptionV3(weights=None, include_top=False, input_shape = (input_dim,input_dim,3))
    # Classification block
    x = GlobalAveragePooling2D(name='avg_pool')(base_model.output)
    x = Dense(17, activation='softmax', name='predictions')(x)
    model = Model(inputs=base_model.input, outputs=x)
    return model

def train(self):
        if self.training_generator is None or self.validation_generator is None:
            self.prepare_generators()

        base_model = InceptionV3(
            weights="imagenet",
            include_top=False,
            input_shape=self.input_shape
        )

        output = base_model.output
        output = GlobalAveragePooling2D()(output)
        output = Dense(1024, activation='relu')(output)

        predictions = Dense(len(self.training_generator.class_indices), activation='softmax')(output)
        self.classifier = Model(inputs=base_model.input, outputs=predictions)

        for layer in base_model.layers:
            layer.trainable = False

        self.classifier.compile(
            optimizer="rmsprop",
            loss="categorical_crossentropy",
            metrics=["accuracy"]
        )

        self.classifier.fit_generator(
            self.training_generator,
            samples_per_epoch=self.training_sample_count,
            nb_epoch=3,
            validation_data=self.validation_generator,
            nb_val_samples=self.validation_sample_count,
            class_weight="auto"
        )

def build_model(model):
        nb_classes = model.nb_classes
        input_shape = model.in_shape
        # print(nb_classes)

        # base_model = VGG16(weights='imagenet', include_top=False)

        base_model = VGG16(weights='imagenet', include_top=False,
                           input_shape=input_shape)

        x = base_model.input
        h = base_model.output
        z_cl = h  # Saving for cl output monitoring.

        h = GlobalAveragePooling2D()(h)
        h = Dense(10, activation='relu')(h)
        z_fl = h  # Saving for fl output monitoring.

        y = Dense(nb_classes, activation='softmax', name='preds')(h)
        # y = Dense(4, activation='softmax')(h)

        for layer in base_model.layers:
            layer.trainable = False

        model.cl_part = Model(x, z_cl)
        model.fl_part = Model(x, z_fl)

        model.x = x
        model.y = y

def topmodel(model, h):
        """
        n_dense and p_dropout are used
        """
        n_dense = model.n_dense
        p_dropout = model.p_dropout

        h = GlobalAveragePooling2D()(h)
        h = Dense(n_dense)(h)
        h = BatchNormalization()(h)
        h = Activation('relu')(h)
        h = Dropout(p_dropout)(h)
        return h

def __init__(self, input_shape, nb_classes, weights='imagenet'):
        base_model = ResNet50(weights=weights, include_top=False,
                              input_shape=input_shape)

        x = base_model.input
        h = base_model.output
        z_cl = h  # Saving for cl output monitoring.

        h = GlobalAveragePooling2D()(h)
        h = Dense(128, activation='relu')(h)
        h = Dropout(0.5)(h)
        z_fl = h  # Saving for fl output monitoring.

        y = Dense(nb_classes, activation='softmax', name='preds')(h)
        # y = Dense(4, activation='softmax')(h)

        for layer in base_model.layers:
            layer.trainable = False

        model = Model(x, y)
        model.compile(loss='categorical_crossentropy', 
                      optimizer='adadelta', metrics=['accuracy'])

        self.model = model
        self.cl_part = Model(x, z_cl)
        self.fl_part = Model(x, z_fl)

def _create(self):
        base_model = KerasInceptionV3(weights='imagenet', include_top=False, input_tensor=self.get_input_tensor())
        self.make_net_layers_non_trainable(base_model)

        x = base_model.output
        x = GlobalAveragePooling2D()(x)
        x = Dense(self.noveltyDetectionLayerSize, activation='elu', name=self.noveltyDetectionLayerName)(x)
        predictions = Dense(len(config.classes), activation='softmax')(x)

        self.model = Model(input=base_model.input, output=predictions)

def densenet(img_input,classes_num):

    def bn_relu(x):
        x = BatchNormalization()(x)
        x = Activation('relu')(x)
        return x

    def bottleneck(x):
        channels = growth_rate * 4
        x = bn_relu(x)
        x = Conv2D(channels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        x = bn_relu(x)
        x = Conv2D(growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        return x

    def single(x):
        x = bn_relu(x)
        x = Conv2D(growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        return x

    def transition(x, inchannels):
        outchannels = int(inchannels * compression)
        x = bn_relu(x)
        x = Conv2D(outchannels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        x = AveragePooling2D((2,2), strides=(2, 2))(x)
        return x, outchannels

    def dense_block(x,blocks,nchannels):
        concat = x
        for i in range(blocks):
            x = bottleneck(concat)
            concat = concatenate([x,concat], axis=-1)
            nchannels += growth_rate
        return concat, nchannels

    def dense_layer(x):
        return Dense(classes_num,activation='softmax',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(x)


    nblocks = (depth - 4) // 6 
    nchannels = growth_rate * 2

    x = Conv2D(nchannels,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(img_input)

    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x = bn_relu(x)
    x = GlobalAveragePooling2D()(x)
    x = dense_layer(x)
    return x

def densenet(img_input,classes_num):

    def bn_relu(x):
        x = BatchNormalization()(x)
        x = Activation('relu')(x)
        return x

    def bottleneck(x):
        channels = growth_rate * 4
        x = bn_relu(x)
        x = Conv2D(channels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        x = bn_relu(x)
        x = Conv2D(growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        return x

    def single(x):
        x = bn_relu(x)
        x = Conv2D(growth_rate,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        return x

    def transition(x, inchannels):
        outchannels = int(inchannels * compression)
        x = bn_relu(x)
        x = Conv2D(outchannels,kernel_size=(1,1),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(x)
        x = AveragePooling2D((2,2), strides=(2, 2))(x)
        return x, outchannels

    def dense_block(x,blocks,nchannels):
        concat = x
        for i in range(blocks):
            x = bottleneck(concat)
            concat = concatenate([x,concat], axis=-1)
            nchannels += growth_rate
        return concat, nchannels

    def dense_layer(x):
        return Dense(classes_num,activation='softmax',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(x)


    nblocks = (depth - 4) // 6 
    nchannels = growth_rate * 2

    x = Conv2D(nchannels,kernel_size=(3,3),strides=(1,1),padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay),use_bias=False)(img_input)

    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x, nchannels = dense_block(x,nblocks,nchannels)
    x, nchannels = transition(x,nchannels)
    x = bn_relu(x)
    x = GlobalAveragePooling2D()(x)
    x = dense_layer(x)
    return x

def wide_residual_network(img_input,classes_num,depth,k):

    print('Wide-Resnet %dx%d' %(depth, k))
    n_filters  = [16, 16*k, 32*k, 64*k]
    n_stack    = (depth - 4) / 6
    in_filters = 16

    def conv3x3(x,filters):
        return Conv2D(filters=filters, kernel_size=(3,3), strides=(1,1), padding='same',
        kernel_initializer=he_normal(),
        kernel_regularizer=regularizers.l2(weight_decay))(x)

    def residual_block(x,out_filters,increase_filter=False):
        if increase_filter:
            first_stride = (2,2)
        else:
            first_stride = (1,1)
        pre_bn   = BatchNormalization()(x)
        pre_relu = Activation('relu')(pre_bn)
        conv_1 = Conv2D(out_filters,kernel_size=(3,3),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(pre_relu)
        bn_1   = BatchNormalization()(conv_1)
        relu1  = Activation('relu')(bn_1)
        conv_2 = Conv2D(out_filters, kernel_size=(3,3), strides=(1,1), padding='same', kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(relu1)
        if increase_filter or in_filters != out_filters:
            projection = Conv2D(out_filters,kernel_size=(1,1),strides=first_stride,padding='same',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(x)
            block = add([conv_2, projection])
        else:
            block = add([conv_2,x])
        return block

    def wide_residual_layer(x,out_filters,increase_filter=False):
        x = residual_block(x,out_filters,increase_filter)
        in_filters = out_filters
        for _ in range(1,int(n_stack)):
            x = residual_block(x,out_filters)
        return x


    x = conv3x3(img_input,n_filters[0])
    x = wide_residual_layer(x,n_filters[1])
    x = wide_residual_layer(x,n_filters[2],increase_filter=True)
    x = wide_residual_layer(x,n_filters[3],increase_filter=True)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)
    x = Dense(classes_num,activation='softmax',kernel_initializer=he_normal(),kernel_regularizer=regularizers.l2(weight_decay))(x)
    return x

def residual_network(img_input,classes_num=10,stack_n=5):
    def residual_block(intput,out_channel,increase=False):
        if increase:
            stride = (2,2)
        else:
            stride = (1,1)

        pre_bn   = BatchNormalization()(intput)
        pre_relu = Activation('relu')(pre_bn)

        conv_1 = Conv2D(out_channel,kernel_size=(3,3),strides=stride,padding='same',
                        kernel_initializer="he_normal",
                        kernel_regularizer=regularizers.l2(weight_decay))(pre_relu)
        bn_1   = BatchNormalization()(conv_1)
        relu1  = Activation('relu')(bn_1)
        conv_2 = Conv2D(out_channel,kernel_size=(3,3),strides=(1,1),padding='same',
                        kernel_initializer="he_normal",
                        kernel_regularizer=regularizers.l2(weight_decay))(relu1)
        if increase:
            projection = Conv2D(out_channel,
                                kernel_size=(1,1),
                                strides=(2,2),
                                padding='same',
                                kernel_initializer="he_normal",
                                kernel_regularizer=regularizers.l2(weight_decay))(intput)
            block = add([conv_2, projection])
        else:
            block = add([intput,conv_2])
        return block

    # build model
    # total layers = stack_n * 3 * 2 + 2
    # stack_n = 5 by default, total layers = 32
    # input: 32x32x3 output: 32x32x16
    x = Conv2D(filters=16,kernel_size=(3,3),strides=(1,1),padding='same',
               kernel_initializer="he_normal",
               kernel_regularizer=regularizers.l2(weight_decay))(img_input)

    # input: 32x32x16 output: 32x32x16
    for _ in range(stack_n):
        x = residual_block(x,16,False)

    # input: 32x32x16 output: 16x16x32
    x = residual_block(x,32,True)
    for _ in range(1,stack_n):
        x = residual_block(x,32,False)

    # input: 16x16x32 output: 8x8x64
    x = residual_block(x,64,True)
    for _ in range(1,stack_n):
        x = residual_block(x,64,False)

    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = GlobalAveragePooling2D()(x)

    # input: 64 output: 10
    x = Dense(classes_num,activation='softmax',
              kernel_initializer="he_normal",
              kernel_regularizer=regularizers.l2(weight_decay))(x)
    return x

def build_model():
  model = Sequential()

  model.add(Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal", input_shape=x_train.shape[1:]))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(160, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(96, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(3, 3),strides=(2,2),padding = 'same'))

  model.add(Dropout(dropout))

  model.add(Conv2D(192, (5, 5), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(192, (1, 1),padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(192, (1, 1),padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(MaxPooling2D(pool_size=(3, 3),strides=(2,2),padding = 'same'))

  model.add(Dropout(dropout))

  model.add(Conv2D(192, (3, 3), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(192, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))
  model.add(Conv2D(10, (1, 1), padding='same', kernel_regularizer=keras.regularizers.l2(weight_decay), kernel_initializer="he_normal"))
  model.add(BatchNormalization())
  model.add(Activation('relu'))

  model.add(GlobalAveragePooling2D())
  model.add(Activation('softmax'))

  sgd = optimizers.SGD(lr=.1, momentum=0.9, nesterov=True)
  model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
  return model

def model2(weights_path=None):
    '''
    Creates a model by concatenating the features from lower layers
    with high level convolution features for all aesthetic attributes along
    with overall aesthetic score
    :param weights_path: path of the weight file
    :return: Keras model instance
    This is the model used in the paper
    '''
    _input = Input(shape=(299, 299, 3))
    resnet = ResNet50(include_top=False, weights='imagenet', input_tensor=_input)
    activation_layers = []
    layers = resnet.layers
    for layer in layers:
        #  print layer.name, layer.input_shape, layer.output_shape
        if 'activation' in layer.name:
            activation_layers.append(layer)

    activations = 0
    activation_plus_squared_outputs = []
    # Remove last activation layer so
    # it can be used with spatial pooling layer if required
    nlayers = len(activation_layers) - 1
    for i in range(1, nlayers):
        layer = activation_layers[i]
        if layer.output_shape[-1] > activation_layers[i - 1].output_shape[-1]:
            #         print layer.name, layer.input_shape, layer.output_shape
            activations += layer.output_shape[-1]
            _out = Lambda(squared_root_normalization,
                          output_shape=squared_root_normalization_output_shape, name=layer.name + '_normalized')(layer.output)
            activation_plus_squared_outputs.append(_out)

            #  print "sum of all activations should be {}".format(activations)

    last_layer_output = GlobalAveragePooling2D()(activation_layers[-1].output)

   # last_layer_output = Lambda(K.sqrt, output_shape=squared_root_normalization_output_shape)(last_layer_output)
    last_layer_output = Lambda(l2_normalize, output_shape=l2_normalize_output_shape,
                               name=activation_layers[-1].name+'_normalized')(last_layer_output)

    activation_plus_squared_outputs.append(last_layer_output)

    merged = merge(activation_plus_squared_outputs, mode='concat', concat_axis=1)
    merged = Lambda(l2_normalize, output_shape=l2_normalize_output_shape, name='merge')(merged)

    # output of model
    outputs = []
    attrs = ['BalacingElements', 'ColorHarmony', 'Content', 'DoF',
             'Light', 'MotionBlur', 'Object', 'RuleOfThirds', 'VividColor']
    for attribute in attrs:

        outputs.append(Dense(1, init='glorot_uniform', activation='tanh', name=attribute)(merged))

    non_negative_attrs = ['Repetition', 'Symmetry', 'score']
    for attribute in non_negative_attrs:
        outputs.append(Dense(1, init='glorot_uniform', activation='sigmoid', name=attribute)(merged))

    model = Model(input=_input, output=outputs)
    if weights_path:
        model.load_weights(weights_path)
    return model

def test_keras_model_probs(num_classes):
    bounds = (0, 255)
    channels = num_classes

    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        inputs = Input(shape=(5, 5, channels))
        logits = GlobalAveragePooling2D(
            data_format='channels_last')(inputs)
        probs = Activation(softmax)(logits)

        model1 = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits')

        model2 = KerasModel(
            Model(inputs=inputs, outputs=probs),
            bounds=bounds,
            predicts='probabilities')

        model3 = KerasModel(
            Model(inputs=inputs, outputs=probs),
            bounds=bounds,
            predicts='probs')

    np.random.seed(22)
    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)

    p1 = model1.batch_predictions(test_images)
    p2 = model2.batch_predictions(test_images)
    p3 = model3.batch_predictions(test_images)

    assert p1.shape == p2.shape == p3.shape == (2, num_classes)

    np.testing.assert_array_almost_equal(
        p1 - p1.max(),
        p2 - p2.max(),
        decimal=1)

    np.testing.assert_array_almost_equal(
        p2 - p2.max(),
        p3 - p3.max(),
        decimal=5)

def test_keras_model_preprocess():
    num_classes = 1000
    bounds = (0, 255)
    channels = num_classes

    with warnings.catch_warnings():
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        inputs = Input(shape=(5, 5, channels))
        logits = GlobalAveragePooling2D(
            data_format='channels_last')(inputs)

        preprocessing = (np.arange(num_classes)[None, None],
                         np.random.uniform(size=(5, 5, channels)) + 1)

        model1 = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits')

        model2 = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits',
            preprocessing=preprocessing)

        model3 = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits')

        preprocessing = (0, np.random.uniform(size=(5, 5, channels)) + 1)

        model4 = KerasModel(
            Model(inputs=inputs, outputs=logits),
            bounds=bounds,
            predicts='logits',
            preprocessing=preprocessing)

    np.random.seed(22)
    test_images = np.random.rand(2, 5, 5, channels).astype(np.float32)
    test_images_copy = test_images.copy()

    p1 = model1.batch_predictions(test_images)
    p2 = model2.batch_predictions(test_images)

    # make sure the images have not been changed by
    # the in-place preprocessing
    assert np.all(test_images == test_images_copy)

    p3 = model3.batch_predictions(test_images)

    assert p1.shape == p2.shape == p3.shape == (2, num_classes)

    np.testing.assert_array_almost_equal(
        p1 - p1.max(),
        p3 - p3.max(),
        decimal=5)

    model4.batch_predictions(test_images)

def run(epochs, PP=None, rgb_mean=None):
    # Load Data
    ig = image.ImageDataGenerator()  # rescale=1/255.0)
    it = ig.flow_from_directory(
        'tiny-imagenet-200/train/', target_size=(64, 64), batch_size=1000)
    ystr2y = it.class_indices
    Xt, Yt, yt = img.load_validation_data(
        it.class_indices, 'tiny-imagenet-200/val/images/', 'tiny-imagenet-200/val/val_annotations.txt')

    Num_classes = 200
    Input_shape = (64, 64, 3)

    # Build a Model
    vgg_model = VGG16(include_top=False)
    x = Input(shape=Input_shape)
    if PP:
        h = PP(8, 0.5, rgb_mean)(x)
        base_model = get_new_base_model_x_h(vgg_model, x, h)
    else:
        base_model = get_new_base_model_x(vgg_model, x)

    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    predictions = Dense(Num_classes, activation='softmax')(x)
    model = Model(inputs=base_model.input, outputs=predictions)

    for layer in base_model.layers:
        layer.trainable = False
    model.compile(optimizer='rmsprop',
                  loss='categorical_crossentropy', metrics=['accuracy'])

    # Training
    model.fit_generator(it, 1000, epochs=epochs, validation_data=(Xt, Yt))

    # Class Activiation Map
    conv_model = base_model
    Ft = conv_model.predict(Xt[:20])

    Wb_all = model.get_weights()
    L_Wb = len(Wb_all)

    W_dense = Wb_all[L_Wb - 2]
    b_dense = Wb_all[L_Wb - 1]
    W_dense.shape, b_dense.shape

    CAM = CAM_process(Ft, yt, W_dense)

    from kakao import bbox
    for i in range(20):
        plt.figure(figsize=(12, 4))
        plt.subplot(1, 3, 1)
        plt.imshow(Xt[i])
        plt.subplot(1, 3, 2)
        CAM4 = img.cam_intp_reshape(CAM[i], Xt[i].shape[:2])
        plt.imshow(CAM4, interpolation='bicubic', cmap='jet_r')
        plt.subplot(1, 3, 3)
        plt.imshow(Xt[i])
        plt.imshow(CAM4, interpolation='bicubic', cmap='jet_r', alpha=0.5)
        plt.show()

def inception_finetune_UCF():
    base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=IMSIZE)
    print('Inception_v3 loaded')

    # freeze the top layers
    # for layer in base_model.layers[:172]:
    #     layer.trainable = False
    for layer in base_model.layers:
        layer.trainable = False

    x = base_model.output
    # x = Flatten()(x)
    x = GlobalAveragePooling2D()(x)
    # x = Dense(256, activation='relu')(x)
    # x = Dropout(0.5)(x)
    # x = Dense(256, activation='relu')(x)
    predictions = Dense(N_CLASSES, activation='softmax')(x)

    model = Model(inputs=base_model.input, outputs=predictions)

    sgd = SGD(lr=0.001, decay=1e-6, momentum=0.5)
    model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])

    print(model.summary())

    data_dir = '/home/changan/ActionRocognition_rnn/data'
    list_dir = os.path.join(data_dir, 'ucfTrainTestlist')
    video_dir = os.path.join(data_dir, 'UCF-Preprocessed')

    train_data, test_data, class_index = get_data_list(list_dir, video_dir)
    print('Train data size: ', len(train_data))
    print('Test data size: ', len(test_data))

    train_generator = video_image_generator(train_data, batch_size, seq_len=SequenceLength, img_size=IMSIZE,
                                            num_classes=101)
    test_generator = video_image_generator(test_data, batch_size, seq_len=SequenceLength, img_size=IMSIZE,
                                           num_classes=101)
    weights_dir = 'inception_finetune.h5'
    if os.path.exists(weights_dir):
        model.load_weights(weights_dir)
        print('weights loaded')
    checkpointer = ModelCheckpoint(weights_dir, save_weights_only=True)
    model.fit_generator(
        train_generator,
        steps_per_epoch=30,
        epochs=200,
        validation_data=test_generator,
        validation_steps=100,
        verbose=2,
        callbacks=[checkpointer]
    )