Python keras 模块，callbacks() 实例源码

def start_train(self):
    """ Starts to Train the entire Model Based on set Parameters """
    # 1, Prep
    callback = [EarlyStopping(patience=self.Patience),
                ReduceLROnPlateau(patience=5, verbose=1),
                CSVLogger(filename=self.rnn_type+'log.csv'),
                ModelCheckpoint(self.rnn_type + '_' + self.dataset + '.check',
                                save_best_only=True,
                                save_weights_only=True)]

    # 2, Train
    self.model.fit(x = [self.train[0],self.train[1]],
                   y = self.train[2],
                   batch_size = self.BatchSize,
                   epochs = self.MaxEpoch,
                   validation_data=([self.test[0], self.test[1]], self.test[2]),
                   callbacks = callback)

    # 3, Evaluate
    self.model.load_weights(self.rnn_type + '_' + self.dataset + '.check') # revert to the best model
    self.evaluate_on_test()

def lengthy_test(model, testrange=[5,10,20,40,80], epochs=100, verboose=True):
    ts = datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
    log_path = LOG_PATH_BASE + ts + "_-_" + model.name 
    tensorboard = TensorBoard(log_dir=log_path,
                                write_graph=False, #This eats a lot of space. Enable with caution!
                                #histogram_freq = 1,
                                write_images=True,
                                batch_size = model.batch_size,
                                write_grads=True)
    model_saver =  ModelCheckpoint(log_path + "/model.ckpt.{epoch:04d}.hdf5", monitor='loss', period=1)
    callbacks = [tensorboard, TerminateOnNaN(), model_saver]

    for i in testrange:
        acc = test_model(model, sequence_length=i, verboose=verboose)
        print("the accuracy for length {0} was: {1}%".format(i,acc))

    train_model(model, epochs=epochs, callbacks=callbacks, verboose=verboose)

    for i in testrange:
        acc = test_model(model, sequence_length=i, verboose=verboose)
        print("the accuracy for length {0} was: {1}%".format(i,acc))
    return

def train_model(model, epochs=10, min_size=5, max_size=20, callbacks=None, verboose=False):
    input_dim = model.input_dim
    output_dim = model.output_dim
    batch_size = model.batch_size

    sample_generator = get_sample(batch_size=batch_size, in_bits=input_dim, out_bits=output_dim,
                                                max_size=max_size, min_size=min_size)
    if verboose:
        for j in range(epochs):
            model.fit_generator(sample_generator, steps_per_epoch=10, epochs=j+1, callbacks=callbacks, initial_epoch=j)
            print("currently at epoch {0}".format(j+1))
            for i in [5,10,20,40]:
                test_model(model, sequence_length=i, verboose=True)
    else:
        model.fit_generator(sample_generator, steps_per_epoch=10, epochs=epochs, callbacks=callbacks)

    print("done training")

def train(self, train_data, validation_data, folder):
        context_data, question_data, answer_data, y_train = train_data
        context_data_v, question_data_v, answer_data_v, y_val = validation_data
        print("Model Fitting")
        filepath = folder + "structures/cos-lstm-nn" + VERSION + "-final-{epoch:02d}-{val_acc:.2f}.hdf5"

        checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
        model_json = self.model.to_json()
        with open(folder + "/structures/cos-lstm-model" + VERSION + ".json", "w") as json_file:
            json_file.write(model_json)
        self.model.summary()
        import numpy as np
        context_data = np.array(list(map(lambda x: x[:MAX_SEQUENCE_LENGTH_C], context_data)))
        context_data_v = np.array(list(map(lambda x: x[:MAX_SEQUENCE_LENGTH_C], context_data_v)))
        self.model.fit({'context': context_data, 'question': question_data, 'answer': answer_data}, y_train,
                  validation_data=({'context': context_data_v, 'question': question_data_v, 'answer': answer_data_v}, y_val),
                  epochs=50, batch_size=256, callbacks=[checkpoint], verbose=2)

def train_with_data_augmentation(self, batch_size, num_epoch, lr_schedule):
        datagen = ImageDataGenerator(
            width_shift_range=0.125, # randomly shift images horizontally, fraction
            height_shift_range=0.125, # randomly shift images vertically, fraction
            horizontal_flip=True)

        opt = keras.optimizers.SGD(lr=lr_schedule(0), momentum=0.9, nesterov=True)
        callback_list = [LearningRateScheduler(lr_schedule)]
        self.ae.compile(optimizer=opt, loss='mse')
        assert False, 'seems that y is not augmented.'
        # history = self.ae.fit_generator(
        #     datagen.flow(
        #         self.dataset.train_xs,
        #         self.dataset.train_xs,
        #     nb_epoch=num_epoch,
        #     batch_size=batch_size,
        #     validation_data=(self.dataset.test_xs, self.dataset.test_xs),
        #     shuffle=True, callbacks=callback_list)
        self.history = history.history

def train_on_memory(model, memory_container, episode):
    # container contains [(observation0, rewards_vec0), (observation1, rewards_vec1),...]
    # x contains model fits X which is the states, y contains model fits y which is the rewards_vec in every state
    x = np.array(memory_container['observations'])
    y = np.array(memory_container['rewards'])

    early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, verbose=1, patience=2)
    model.fit(x, y, epochs=3000, validation_split=0.3, callbacks=[early_stopping])
    return model

def main(_):
    pp.pprint(flags.FLAGS.__flags)
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    if not os.path.isdir(FLAGS.checkpoint):
        os.mkdir(FLAGS.checkpoint)
    if not os.path.isdir(FLAGS.log):
        os.mkdir(FLAGS.log)
    model = genChipModel()
    model.summary()

    opt = keras.optimizers.rmsprop(lr=0.001, decay=1e-6)
    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])#'categorical_crossentropy', metrics=['accuracy'])

    filename = '../../data/finalData.txt'   
    x, y = readData(filename)
    x_train, y_train, x_test, y_test = init(x, y)

    y_train_labels = to_categorical(y_train, num_classes=79)
    y_test_labels = to_categorical(y_test, num_classes=79)  
    model_path = os.path.join(FLAGS.checkpoint, "weights.hdf5")
    callbacks = [
        ModelCheckpoint(filepath=model_path, monitor="val_acc", save_best_only=True, save_weights_only=True),
        TensorBoard(log_dir=FLAGS.log),
        ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2)
    ]
    hist = model.fit(x_train, y_train_labels, epochs=FLAGS.epoch, batch_size=100, validation_data=(x_test, y_test_labels), callbacks=callbacks)

    loss, accuracy = model.evaluate(x_test, y_test_labels, batch_size=100, verbose=1)

def main(_):
    pp.pprint(flags.FLAGS.__flags)
    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
    if not os.path.isdir(FLAGS.checkpoint):
        os.mkdir(FLAGS.checkpoint)
    if not os.path.isdir(FLAGS.log):
        os.mkdir(FLAGS.log)
    model = genChipModel()
    model.summary()

    opt = keras.optimizers.rmsprop(lr=0.001, decay=1e-6)
    model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])#'categorical_crossentropy', metrics=['accuracy'])

    filename = '../../data/finalData.txt'   
    x, y = readData(filename)
    x_train, y_train, x_test, y_test = init(x, y)

    y_train_labels = to_categorical(y_train, num_classes=79)
    y_test_labels = to_categorical(y_test, num_classes=79)  
    model_path = os.path.join(FLAGS.checkpoint, "weights.hdf5")
    callbacks = [
        ModelCheckpoint(filepath=model_path, monitor="val_acc", save_best_only=True, save_weights_only=True),
        TensorBoard(log_dir=FLAGS.log),
        ReduceLROnPlateau(monitor='val_acc', factor=0.5, patience=2)
    ]
    hist = model.fit(x_train, y_train_labels, epochs=FLAGS.epoch, batch_size=100, validation_data=(x_test, y_test_labels), callbacks=callbacks)

    loss, accuracy = model.evaluate(x_test, y_test_labels, batch_size=100, verbose=1)

def fit(self):
        import time
        tic = time.time()
        logger.info("Start train model {}\n".format(self.config.name))
        hist = self.model.fit(self.config.dataset['train_x'],
                              self.config.dataset['train_y'],
                              # validation_split=0.2,
                              validation_data=(self.config.dataset['test_x'], self.config.dataset['test_y']),
                              verbose=self.config.verbose,
                              batch_size=self.config.batch_size,
                              epochs=self.config.epochs,
                              callbacks=[self.config.lr_reducer,
                                         self.config.csv_logger,
                                         self.config.early_stopper,
                                         TensorBoard(log_dir=self.config.tf_log_path,
                                                     # histogram_freq=20,
                                                     # batch_size=32,
                                                     # write_graph=True,
                                                     # write_grads=True,
                                                     # write_images=True,
                                                     # embeddings_freq=0
                                                     )]
                              )
        # todo do we need earlystop?
        logger.info("Fit model {} Consume {}:".format(self.config.name, time.time() - tic))
        return hist

def main():
    RUN_TIME = sys.argv[1]


    if RUN_TIME == "TRAIN":
        image_features = Input(shape=(4096,))
        model = build_model(image_features)
        print model.summary()

        # number of training images 
        _num_train = get_num_train_images()

        # Callbacks 
        # remote_cb = RemoteMonitor(root='http://localhost:9000')
        tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
        epoch_cb    = EpochCheckpoint(folder="./snapshots/")
        valid_cb    = ValidCallBack()

        # fit generator
        steps_per_epoch = math.ceil(_num_train/float(BATCH))
        print "Steps per epoch i.e number of iterations: ",steps_per_epoch

        train_datagen = data_generator(batch_size=INCORRECT_BATCH, image_class_ranges=TRAINING_CLASS_RANGES)
        history = model.fit_generator(
                train_datagen,
                steps_per_epoch=steps_per_epoch,
                epochs=250,
                callbacks=[tensorboard, valid_cb]
            )
        print history.history.keys()


    elif RUN_TIME == "TEST":
        from keras.models import load_model 
        model = load_model("snapshots/epoch_49.hdf5", custom_objects={"hinge_rank_loss":hinge_rank_loss})

    K.clear_session()

def train(self):
        checkpointer = ModelCheckpoint(filepath='./best.model',
                                       verbose=1,
                                       monitor='val_loss',
                                       save_best_only=True)
        earlystop = EarlyStopping(monitor='val_loss',
                                  patience=10,
                                  verbose=1,
                                  mode='auto')

        self.model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
        json_model = self.model.to_json()
        fjson = open('model.json', 'w')
        fjson.write(json_model)
        fjson.close()
        print 'model_json_saved!'
        train_x, train_y = pre_x_y(path='kgs-19-2017-01-new/')
        valid_x, valid_y = pre_x_y(path='kgs-19-2017-01-new/')
        print 'train_data_len: ', len(train_x)
        print 'valid_data_len: ', len(valid_x)
        self.model.fit({'x': train_x}, {'out': train_y},
                  batch_size=32,
                  nb_epoch=1,
                  shuffle=True,
                  verbose=1,
                  callbacks=[checkpointer, earlystop],
                  validation_data=({'x': valid_x}, {'out': valid_y}))

def fit(self, data_stream, 
                nvis=20, 
                nbatch=128,
                niter=1000,
                opt=None,
                save_dir='./'):
        if opt == None: opt = Adam(lr=0.0001)
        if not os.path.exists(save_dir): os.makedirs(save_dir)

        ae = self.autoencoder
        ae.compile(optimizer=opt, loss='mse')

        vis_grid(data_stream().next(), (1, 20), '{}/sample.png'.format(save_dir))

        sampleX = transform(data_stream().next()[:nvis])
        vis_grid(inverse_transform(np.concatenate([sampleX, ae.predict(sampleX)], axis=0)), (2, 20), '{}/sample_generate.png'.format(save_dir))

        def vis_grid_f(epoch, logs):
            vis_grid(inverse_transform(np.concatenate([sampleX, ae.predict(sampleX)], axis=0)), (2, 20), '{}/{}.png'.format(save_dir, epoch))
            if epoch % 50 == 0:
                ae.save_weights('{}/{}_ae_params.h5'.format(save_dir, epoch), overwrite=True)

        def transform_wrapper():
            for data in data_stream():
                yield transform(data), transform(data) 

        ae.fit_generator(transform_wrapper(),
                            samples_per_epoch=nbatch, 
                            nb_epoch=niter, 
                            verbose=1, 
                            callbacks=[LambdaCallback(on_epoch_end=vis_grid_f)],
                        )

def make_lr_scheduler(base_lr, decay_rate, epoch_rate):

    def lr_schedule(epoch):
        if epoch + 1 < epoch_rate:
            lr = base_lr
        else:
            lr = base_lr / (decay_rate * np.floor(epoch + 1 / rate_epochs))

        return lr

    return keras.callbacks.LearningRateScheduler(lr_schedule)

def train(self, train_data, validation_data, folder):
        context_data, question_data, answer_data, y_train = train_data
        context_data_v, question_data_v, answer_data_v, y_val = validation_data
        print("Model Fitting")
        filepath = folder + "structures/lstm-nn" + VERSION + "-final-{epoch:02d}-{val_acc:.2f}.hdf5"

        checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
        model_json = self.model.to_json()
        with open(folder + "/structures/lstm-model" + VERSION + ".json", "w") as json_file:
            json_file.write(model_json)
        self.model.summary()
        self.model.fit({'context': context_data, 'question': question_data, 'answer': answer_data}, y_train,
                  validation_data=({'context': context_data_v, 'question': question_data_v, 'answer': answer_data_v}, y_val),
                  epochs=50, batch_size=256, callbacks=[checkpoint], verbose=2)

def train(self, train_data, validation_data, folder):
        context_data, question_data, answer_data, y_train = train_data
        context_data_v, question_data_v, answer_data_v, y_val = validation_data
        print("Model Fitting")
        filepath = folder + "structures/layered-lstm-nn" + VERSION + "-final-{epoch:02d}-{val_acc:.2f}.hdf5"

        checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
        model_json = self.model.to_json()
        with open(folder + "/structures/layered-lstm-model" + VERSION + ".json", "w") as json_file:
            json_file.write(model_json)
        self.model.summary()
        self.model.fit({'context': context_data, 'question': question_data, 'answer': answer_data}, y_train,
                  validation_data=({'context': context_data_v, 'question': question_data_v, 'answer': answer_data_v}, y_val),
                  epochs=50, batch_size=256, callbacks=[checkpoint], verbose=2)

def train(self, train_data, validation_data, folder):
        context_data, question_data, answer_data, y_train = train_data
        context_data_v, question_data_v, answer_data_v, y_val = validation_data
        print("Model Fitting")
        filepath = folder + "structures/no-context-lstm-nn" + VERSION + "-final-{epoch:02d}-{val_acc:.2f}.hdf5"

        checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
        model_json = self.model.to_json()
        with open(folder + "/structures/no-context-lstm-model" + VERSION + ".json", "w") as json_file:
            json_file.write(model_json)
        self.model.summary()
        self.model.fit({'question': question_data, 'answer': answer_data}, y_train,
                  validation_data=({'question': question_data_v, 'answer': answer_data_v}, y_val),
                  epochs=50, batch_size=256, callbacks=[checkpoint], verbose=2)

def train(self, train_data, validation_data, folder):
        context_data, question_data, answer_data, y_train = train_data
        context_data_v, question_data_v, answer_data_v, y_val = validation_data
        print("Model Fitting")
        filepath = folder + "structures/lstm-cnn-nn" + VERSION + "-final-{epoch:02d}-{val_acc:.2f}.hdf5"

        checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=0, save_best_only=True, mode='max')
        model_json = self.model.to_json()
        with open(folder + "/structures/lstm-cnn-model" + VERSION + ".json", "w") as json_file:
            json_file.write(model_json)
        self.model.summary()
        self.model.fit({'context': context_data, 'question': question_data, 'answer': answer_data}, y_train,
                  validation_data=({'context': context_data_v, 'question': question_data_v, 'answer': answer_data_v}, y_val),
                  epochs=50, batch_size=256, callbacks=[checkpoint], verbose=2)

def train(self, batch_size, num_epoch, lr_schedule):
        opt = keras.optimizers.SGD(lr=lr_schedule(0), momentum=0.9, nesterov=True)
        callback_list = [LearningRateScheduler(lr_schedule)]
        self.ae.compile(optimizer=opt, loss='mse')
        history = self.ae.fit(
            self.dataset.train_xs, self.dataset.train_xs,
            nb_epoch=num_epoch,
            batch_size=batch_size,
            validation_data=(self.dataset.test_xs, self.dataset.test_xs),
            shuffle=True, callbacks=callback_list)
        self.history = history.history

def get_callback():
    def return_callback():
        from keras.callbacks import ReduceLROnPlateau
        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2,
                                      patience=5, min_lr=0.001)
        return reduce_lr
    return return_callback

def test_experiment_fit(self, get_model, get_loss_metric,
                            get_custom_l, get_callback_fix):
        new_session()
        data, data_val = make_data(train_samples, test_samples)
        model, metrics, cust_objects = prepare_model(get_model(get_custom_l),
                                                     get_loss_metric,
                                                     get_custom_l)

        expe = Experiment(model)

        for mod in [None, model]:
            for data_val_loc in [None, data_val]:
                expe.fit([data], [data_val_loc], model=mod, nb_epoch=2,
                         batch_size=batch_size, metrics=metrics,
                         custom_objects=cust_objects, overwrite=True,
                         callbacks=get_callback_fix)

        expe.backend_name = 'another_backend'
        expe.load_model()
        expe.load_model(expe.mod_id, expe.data_id)

        assert expe.data_id is not None
        assert expe.mod_id is not None
        assert expe.params_dump is not None

        if K.backend() == 'tensorflow':
            K.clear_session()

        print(self)

def test_experiment_fit_gen(self, get_model, get_loss_metric,
                                get_custom_l, get_callback_fix):
        new_session()
        model, metrics, cust_objects = prepare_model(get_model(get_custom_l),
                                                     get_loss_metric,
                                                     get_custom_l)

        model_name = model.__class__.__name__
        _, data_val_use = make_data(train_samples, test_samples)
        expe = Experiment(model)

        for val in [1, data_val_use]:
            gen, data, data_stream = make_gen(batch_size)
            if val == 1:
                val, data_2, data_stream_2 = make_gen(batch_size)
            expe.fit_gen([gen], [val], nb_epoch=2,
                         model=model,
                         metrics=metrics,
                         custom_objects=cust_objects,
                         samples_per_epoch=64,
                         nb_val_samples=128,
                         verbose=2, overwrite=True,
                         callbacks=get_callback_fix)

            close_gens(gen, data, data_stream)
            if val == 1:
                close_gens(val, data_2, data_stream_2)

        if K.backend() == 'tensorflow':
            K.clear_session()

        print(self)

def train(model, nnInput, labels, validation, makePlot = True,
        labelIndexes = RP['label_idxs']):
    print('  Training model...')


    # needed format is orthogonal to ours
    '''
    formattedLabels = np.zeros((len(labels[0]), len(labelIndexes)))
    formattedValid = np.zeros((len(validation[1][labelIndexes[0]]),
        len(labelIndexes)))
    for i in range(len(labelIndexes)):
        for j in range(len(labels[0])):
            formattedLabels[j][i] = labels[labelIndexes[i]][j]
        for j in range(len(validation[1][labelIndexes[i]])):
            formattedValid[j][i] = validation[1][labelIndexes[i]][j]
    '''
    early = keras.callbacks.EarlyStopping(monitor = 'val_loss',
            patience = RP['early_stop'])

    learningRateScheduler = keras.callbacks.LearningRateScheduler(learningRateDecayer)

    modelLogger = visualization.ModelLogger()

    history = model.fit(nnInput, labels, nb_epoch = RP['epochs'],
            batch_size = RP['batch'], callbacks = [early],
            validation_data = (validation[0], validation[1]))

    if makePlot:
        values = np.zeros((len(history.history['loss']), 2))
        for i in range(len(history.history['loss'])):
            values[i][0] = history.history['loss'][i]
            values[i][1] = history.history['val_loss'][i]
        utility.plotLoss(values)

    visualization.histograms(modelLogger)

    print('    Model weights:')
    print(model.summary())
    # print(model.get_weights())
    print('  ...done')
    return len(history.history['loss'])

def train_unet(self):

        img_size = self.flag.image_size
        batch_size = self.flag.batch_size
        epochs = self.flag.total_epoch

        datagen_args = dict(featurewise_center=False,  # set input mean to 0 over the dataset
                samplewise_center=False,  # set each sample mean to 0
                featurewise_std_normalization=False,  # divide inputs by std of the dataset
                samplewise_std_normalization=False,  # divide each input by its std
                zca_whitening=False,  # apply ZCA whitening
                rotation_range=5,  # randomly rotate images in the range (degrees, 0 to 180)
                width_shift_range=0.05,  # randomly shift images horizontally (fraction of total width)
                height_shift_range=0.05,  # randomly shift images vertically (fraction of total height)
                # fill_mode='constant',
                # cval=0.,
                horizontal_flip=False,  # randomly flip images
                vertical_flip=False)  # randomly flip images

        image_datagen = ImageDataGenerator(**datagen_args)
        mask_datagen = ImageDataGenerator(**datagen_args)

        seed = random.randrange(1, 1000)
        image_generator = image_datagen.flow_from_directory(
                    os.path.join(self.flag.data_path, 'train/IMAGE'),
                    class_mode=None, seed=seed, batch_size=batch_size, color_mode='grayscale')
        mask_generator = mask_datagen.flow_from_directory(
                    os.path.join(self.flag.data_path, 'train/GT'),
                    class_mode=None, seed=seed, batch_size=batch_size, color_mode='grayscale')
        config = tf.ConfigProto()
        # config.gpu_options.per_process_gpu_memory_fraction = 0.9
        config.gpu_options.allow_growth = True
        set_session(tf.Session(config=config))

        model = get_unet(self.flag)
        if self.flag.pretrained_weight_path != None:
            model.load_weights(self.flag.pretrained_weight_path)

        if not os.path.exists(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name)):
            mkdir_p(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name))
        model_json = model.to_json()
        with open(os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name, 'model.json'), 'w') as json_file:
            json_file.write(model_json)
        vis = callbacks.trainCheck(self.flag)
        model_checkpoint = ModelCheckpoint(
                    os.path.join(self.flag.ckpt_dir, self.flag.ckpt_name,'weights.{epoch:03d}.h5'), 
                    period=self.flag.total_epoch//10+1)
        learning_rate = LearningRateScheduler(self.lr_step_decay)
        model.fit_generator(
            self.train_generator(image_generator, mask_generator),
            steps_per_epoch= image_generator.n // batch_size,
            epochs=epochs,
            callbacks=[model_checkpoint, learning_rate, vis]
        )

def train(self, run_name, start_epoch, stop_epoch, img_w):

        words_per_epoch = 16000
        val_split = 0.2 #0.2
        #val_words = len(val_crop_iter)
        val_words = int(words_per_epoch * (val_split))


        #fdir ='/Users/sofwath/Desktop/dhivehiocr/tmp/'
        fdir = DATA_DIR
        self.img_gen = TextImageGenerator(monogram_file=os.path.join(fdir, 'wordlist_mono_dhivehi.txt'),
                                     bigram_file=os.path.join(fdir, 'wordlist_bi_dhivehi.txt'),
                                     minibatch_size=32,
                                     img_w=img_w,
                                     img_h=self.img_h,
                                     downsample_factor=(self.pool_size ** 2),
                                     val_split=self.words_per_epoch - self.val_words
                                     )

        adam = keras.optimizers.Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

        output_dir = os.path.join(OUTPUT_DIR, run_name)


        labels = Input(name='the_labels', shape=[self.img_gen.absolute_max_string_len], dtype='float32')
        input_length = Input(name='input_length', shape=[1], dtype='int64')
        label_length = Input(name='label_length', shape=[1], dtype='int64')

        y_pred = Activation('softmax', name='softmax')(self.inner)

        loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])

        self.model = Model(inputs=[self.input_data, labels, input_length, label_length], outputs=loss_out)

        self.model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=adam)

        if start_epoch > 0:
            weight_file = os.path.join(OUTPUT_DIR, os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
            self.model.load_weights(weight_file)


        viz_cb = VizCallback(run_name, self.test_func, self.img_gen.next_val(),self.model)


        # use this (uncomment below) for test runs 

        #self.model.fit_generator(generator=self.img_gen.next_train(), steps_per_epoch=(self.words_per_epoch - self.val_words)/self.minibatch_size,
        #                    epochs=stop_epoch, validation_data=self.img_gen.next_val(), validation_steps=self.val_words/self.minibatch_size,
        #                    callbacks=[viz_cb, self.img_gen], initial_epoch=start_epoch, verbose=1)



        self.model.fit_generator(generator=self.img_gen.next_train(), steps_per_epoch=(self.words_per_epoch - self.val_words),
                        epochs=stop_epoch, validation_data=self.img_gen.next_val(), validation_steps=self.val_words,
                        callbacks=[viz_cb, self.img_gen], initial_epoch=start_epoch, verbose=1)

def train(trainset, valset, path_data, path_session, hyper_param):
    """Execute a single training task.


    Returns:
        model: /path/to/best_model as measured by validation's loss
        loss: the loss computed on the validation set
        acc: the accuracy computed on the validation set
    """

    session_id = os.path.basename(path_session)
    model_cp = keras.callbacks.ModelCheckpoint(
        os.path.join(path_session, "{}_model.hdf5".format(session_id)),
        monitor="val_loss",
        save_best_only=True)


    # train
    model = _get_model(hyper_param["optimizer"],
                       hyper_param["batch_norm"],
                       pool_type=hyper_param["pool_type"],
                       dropout_rate=hyper_param["dropout_rate"])
    history = model.fit_generator(
        _sample_generator(trainset, path_data, hyper_param["batch_sz"]),
        steps_per_epoch=int(len(trainset) / hyper_param["batch_sz"]),
        epochs=hyper_param["epochs"],
        validation_data=_sample_generator(valset, path_data, 2),
        validation_steps=int(len(valset) / 2),
        callbacks=[model_cp, hyper_param["lr_schedule"]],
        verbose=1,
        workers=4)

    # plot training curves
    def plot_history(metric):
        plt.ioff()
        str_metric = "accuracy" if metric == "acc" else "loss"
        plt.plot(history.history[metric])
        plt.plot(history.history["val_{}".format(metric)])
        plt.title("model {}".format(str_metric))
        plt.ylabel(str_metric)
        plt.xlabel("epoch")
        plt.legend(["train", "test"], loc="upper left")
        plt.savefig(os.path.join(path_session, 
                                 "{}_{}.png".format(session_id, str_metric)))

    plot_history("loss")
    plt.cla()
    plot_history("acc")   
    with open(os.path.join(path_session,
                           "{}_history.pkl".format(session_id)),
              'wb') as output:
        pickle.dump(history.history, output, pickle.HIGHEST_PROTOCOL)

    # output model and performance measures
    ind_min_loss = np.argmin(history.history["val_loss"])
    return (os.path.join(path_session, "{}.hdf5".format(session_id)),
            history.history["val_loss"][ind_min_loss],
            history.history["val_acc"][ind_min_loss])

def test_experiment_fit_async(self, get_model, get_loss_metric,
                                  get_custom_l, get_callback_fix):
        new_session()
        data, data_val = make_data(train_samples, test_samples)
        model, metrics, cust_objects = prepare_model(get_model(get_custom_l),
                                                     get_loss_metric,
                                                     get_custom_l)

        cust_objects['test_list'] = [1, 2]
        expe = Experiment(model)

        expected_value = 2
        for mod in [None, model]:
            for data_val_loc in [None, data_val]:
                _, thread = expe.fit_async([data], [data_val_loc],
                                           model=mod, nb_epoch=2,
                                           batch_size=batch_size,
                                           metrics=metrics,
                                           custom_objects=cust_objects,
                                           overwrite=True,
                                           verbose=2,
                                           callbacks=get_callback_fix)

                thread.join()

                for k in expe.full_res['metrics']:
                    if 'iter' not in k:
                        assert len(
                            expe.full_res['metrics'][k]) == expected_value

                if data_val_loc is not None:
                    for k in expe.full_res['metrics']:
                        if 'val' in k and 'iter' not in k:
                            assert None not in expe.full_res['metrics'][k]
                else:
                    for k in expe.full_res['metrics']:
                        if 'val' in k and 'iter' not in k:
                            assert all([np.isnan(v)
                                        for v in expe.full_res['metrics'][k]])

        if K.backend() == 'tensorflow':
            K.clear_session()

        print(self)