我们从Python开源项目中，提取了以下19个代码示例，用于说明如何使用hyperopt.fmin()。
def hyperopt_search(args, data, model, param_grid, max_evals): def objective(param_grid): args.num_hidden = param_grid['num_hidden'] args.dropout_output = param_grid['dropout_output'] args.dropout_input = param_grid['dropout_input'] args.clip_norm = param_grid['clip_norm'] args.batch_size = param_grid['batch_size'] # args.learning_rate = param_grid['learning_rate'] print(args) print() scores = run_network(args, data, model, tuning=args.tune) test_score, eval_score = scores tf.reset_default_graph() eval_score = -eval_score[0] return {'loss': eval_score, 'params': args, 'status': STATUS_OK} trials = Trials() results = fmin( objective, param_grid, algo=tpe.suggest, trials=trials, max_evals=max_evals) return results, trials.results
def run_all_dl(csvfile = saving_fp, space = [hp.quniform('h1', 100, 550, 1), hp.quniform('h2', 100, 550, 1), hp.quniform('h3', 100, 550, 1), #hp.choice('activation', ["RectifierWithDropout", "TanhWithDropout"]), hp.uniform('hdr1', 0.001, 0.3), hp.uniform('hdr2', 0.001, 0.3), hp.uniform('hdr3', 0.001, 0.3), hp.uniform('rho', 0.9, 0.999), hp.uniform('epsilon', 1e-10, 1e-4)]): # maxout works well with dropout (Goodfellow et al 2013), and rectifier has worked well with image recognition (LeCun et al 1998) start_save(csvfile = csvfile) trials = Trials() print "Deep learning..." best = fmin(objective, space = space, algo=tpe.suggest, max_evals=evals, trials=trials) print best print trials.losses() with open('output/dlbest.pkl', 'w') as output: pickle.dump(best, output, -1) with open('output/dltrials.pkl', 'w') as output: pickle.dump(trials, output, -1)
def optimize(self, trials, max_evals=250): self.space = { "objective": "multi:softprob", "eval_metric": "mlogloss", #Control complexity of model "eta" : hp.quniform("eta", 0.1, 0.3, 0.025), "max_depth" : hp.quniform("max_depth", 5, 10, 1), "min_child_weight" : hp.quniform('min_child_weight', 5, 10, 1), 'gamma' : hp.quniform('gamma', 0, 1, 0.05), 'learning_rate': hp.quniform('learning_rate', 0., 0.1, 0.01), 'n_estimators': hp.quniform('n_estimators', 500, 800, 10), #Improve noise robustness "subsample" : hp.quniform('subsample', 1.0, 2, 0.01), "colsample_bytree" : hp.quniform('colsample_bytree', 0.3, 0.6, 0.025), 'num_class' : 2, 'silent' : 1} best = fmin(self.score, self.space, algo=tpe.suggest, trials=trials, max_evals=max_evals) print "best parameters", best return best
def run_a_trial(): """Run one TPE meta optimisation step and save its results.""" max_evals = nb_evals = 1 print("Attempt to resume a past training if it exists:") try: # https://github.com/hyperopt/hyperopt/issues/267 trials = pickle.load(open("results.pkl", "rb")) print("Found saved Trials! Loading...") max_evals = len(trials.trials) + nb_evals print("Rerunning from {} trials to add another one.".format( len(trials.trials))) except: trials = Trials() print("Starting from scratch: new trials.") best = fmin( build_and_optimize_cnn, space, algo=tpe.suggest, trials=trials, max_evals=max_evals ) pickle.dump(trials, open("results.pkl", "wb")) print("\nOPTIMIZATION STEP COMPLETE.\n") print("Best results yet (note that this is NOT calculated on the 'loss' " "metric despite the key is 'loss' - we rather take the negative " "best accuracy throughout learning as a metric to minimize):") print(best)
def fit(self, X_train, y_train, X_test=None, y_test=None, n_iters=10, start_vals=None): """ """ if (X_test is None) and (y_test is None): X_test = X_train y_test = y_train elif (X_test is None) or (y_test is None): raise MissingValueException("Need to provide 'X_test' and 'y_test'") def objective(params): model_params = self.model.get_params() model_params.update(params) self.model = self.build_new_model(model_params) self.model.fit(X_train, y_train) y_pred = self.model.predict(X_test) y_true = y_test score = -self.eval_func(y_true, y_pred) return score self.trials = Trials() best_params = fmin(objective, self.param_space, algo=tpe.suggest, max_evals=n_iters, trials=self.trials) self.hyperparam_history = [] for i, loss in enumerate(self.trials.losses()): param_vals = {k:v[i] for k,v in self.trials.vals.items()} self.hyperparam_history.append((-loss, param_vals)) model_params = self.model.get_params() model_params.update(best_params) best_model = self.build_new_model(model_params) return best_params, best_model
def start_optimization(self, max_evals): logger.info("Started optimization for task %s", self.task) space = self.space.hyperopt() best = fmin(self.score, space, algo=tpe.suggest, max_evals=max_evals) self.best = space_eval(space, best) return self.best
def optimize(self): trials = Trials() print('Tuning Parameters') best = fmin(self.score, self.h_param_grid, algo=tpe.suggest, trials=trials, max_evals=200) print('\n\nBest Scoring Value') print(best) self.change_to_int(best, self.to_int_params) self.level0.set_params(**best) self.level0.fit(self.trainX, self.trainY) joblib.dump(self.level0,'model_best.pkl', compress=True)
def best_model(self): algo = partial(tpe.suggest, n_startup_jobs=1) best = fmin(self.GBM, space=self.paras.hyper_opt, algo=algo, max_evals=20) print("best", best) return best
def best_model(self, X_train, y_train, X_test, y_test): self.train_x = X_train self.train_y = y_train self.test_x = X_test self.test_y = y_test algo = partial(tpe.suggest, n_startup_jobs=1) best = fmin(self.LSTM, space=self.paras.hyper_opt, algo=algo, max_evals=20) print("best", best) return best
def start_search_server(self, mongo_uri, data_filename, layers_num, max_evals=100, nb_epoch = 10, patience = 5): space = self.conv_model_space(layers_num, data_filename) exp_key, trials = self.create_mongo_trials(mongo_uri) keras_model = KerasModel(exp_key, nb_epoch = nb_epoch, patience = patience, mongo_uri = mongo_uri) best = fmin(keras_model.run_job, space, trials=trials, algo=tpe.suggest, max_evals=max_evals) print "Done!" print best
def run_all_gbm(csvfile = saving_fp, space = [hp.quniform('ntrees', 200, 750, 1), hp.quniform('max_depth', 5, 15, 1), hp.uniform('learn_rate', 0.03, 0.35)]): # Search space is a stochastic argument-sampling program: start_save(csvfile = csvfile) trials = Trials() best = fmin(objective, space = space, algo=tpe.suggest, max_evals=evals, trials=trials) print best # from hyperopt import space_eval # print space_eval(space, best) # trials.trials # list of dictionaries representing everything about the search # trials.results # list of dictionaries returned by 'objective' during the search print trials.losses() # list of losses (float for each 'ok' trial) # trials.statuses() # list of status strings with open('output/gbmbest.pkl', 'w') as output: pickle.dump(best, output, -1) with open('output/gbmtrials.pkl', 'w') as output: pickle.dump(trials, output, -1) # with open('output/gbmtrials.pkl', 'rb') as input: # trials = pickle.load(input) # with open('output/gbmbest.pkl', 'rb') as input: # best = pickle.load(input)
def run(self): start = time.time() trials = Trials() best = fmin(self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_int_param(best_params) trial_rmses = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_rmses) best_rmse_mean = trial_rmses[best_ind] best_rmse_std = trials.trial_attachments(trials.trials[best_ind])["std"] self.logger.info("-"*50) self.logger.info("Best RMSE") self.logger.info(" Mean: %.6f"%best_rmse_mean) self.logger.info(" std: %.6f"%best_rmse_std) self.logger.info("Best param") self.task._print_param_dict(best_params) end = time.time() _sec = end - start _min = int(_sec/60.) self.logger.info("Time") if _min > 0: self.logger.info(" %d mins"%_min) else: self.logger.info(" %d secs"%_sec) self.logger.info("-"*50) #------------------------ Main -------------------------
def xgb2(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name N_splits = 9 N_seeds = 4 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train2, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=100, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0.5, 10, 0.05), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = load_state(cname + '_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) for n in range(5): best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) save_state(cname + '_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) xgb_common(train2, y, test2, v, z, N_seeds, N_splits, cname, xgb_params)
def xgb2(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name N_splits = 9 N_seeds = 4 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train2, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=100, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0.5, 10, 0.05), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = load_state(cname + '_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) for n in range(15): best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) save_state(cname + '_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) xgb_common(train2, y, test2, v, z, N_seeds, N_splits, cname, xgb_params)
def xgb2(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name N_splits = 9 N_seeds = 4 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train2, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=100, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0.5, 10, 0.05), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = load_state(cname + '_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) for n in range(25): best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) save_state(cname + '_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) xgb_common(train2, y, test2, v, z, N_seeds, N_splits, cname, xgb_params)
def run(self): trials = Trials() Writer.create_empty(self.fpath) Writer.append_line_list(self.fpath, [c for c in self.columns] + self.output_items) best = fmin(self.score, self.space, algo=tpe.suggest, trials=trials, max_evals=self.max_evals) print best
def optimize(self, hyper_param_iterations, old_trials=None): hp_info = self.model_runner.hyper_parameter_info() if old_trials is None: trials = Trials() else: trials = old_trials if hp_info.get('fixed') is True: del hp_info['fixed'] result = self.model_runner.train_and_cv_error(self.features, hp_info) trials.trials.append({ 'result': result, 'misc': {'tid': 0, 'vals': hp_info}, }) return trials, hp_info best_hps = fmin(self.to_minimize, hp_info, algo=tpe.suggest, max_evals=hyper_param_iterations, trials=trials) filtered_hps = {} for hp in self.model_runner.hyper_parameter_info(): try: filtered_hps[hp] = best_hps[hp] except KeyError: filtered_hps[hp] = None return trials, filtered_hps
def optimize( # trials, random_state=SEED): """ This is the optimization function that given a space (space here) of hyperparameters and a scoring function (score here), finds the best hyperparameters. """ # space = { # 'n_estimators': hp.choice('n_estimators', [1000, 1100]), # 'eta': hp.quniform('eta', 0.01, 0.1, 0.025), # 'max_depth': hp.choice('max_depth', [4, 5, 7, 9, 17]), # 'min_child_weight': hp.choice('min_child_weight', [3, 5, 7]), # 'subsample': hp.choice('subsample', [0.4, 0.6, 0.8]), # 'gamma': hp.choice('gamma', [0.3, 0.4]), # 'colsample_bytree': hp.quniform('colsample_bytree', 0.4, 0.7, 0.1), # 'lambda': hp.choice('lambda', [0.01, 0.1, 0.9, 1.0]), # 'alpha': hp.choice('alpha', [0, 0.1, 0.5, 1.0]), # 'eval_metric': 'auc', # 'objective': 'binary:logistic', # # Increase this number if you have more cores. # # Otherwise, remove it and it will default # # to the maxium number. # 'nthread': 4, # 'booster': 'gbtree', # 'tree_method': 'exact', # 'silent': 1, # 'seed': random_state # } space = { 'n_estimators': hp.choice('n_estimators', [1000]), 'eta': hp.choice('eta', [0.01]), 'max_depth': hp.choice('max_depth', [4]), 'min_child_weight': hp.choice('min_child_weight', [5]), 'subsample': hp.choice('subsample', [0.4]), 'gamma': hp.choice('gamma', [0.4, 0.8]), 'colsample_bytree': hp.choice('colsample_bytree', [0.4]), 'lambda': hp.choice('lambda', [0.9, 0.93]), 'alpha': hp.choice('alpha', [0.5]), 'eval_metric': 'auc', 'objective': 'binary:logistic', # Increase this number if you have more cores. # Otherwise, remove it and it will default # to the maxium number. 'nthread': 4, 'booster': 'gbtree', 'tree_method': 'exact', 'silent': 1, 'seed': random_state } # Use the fmin function from Hyperopt to find the best hyperparameters best = fmin(score, space, algo=tpe.suggest, # trials=trials, max_evals=4) return best
def xgb3(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name N_splits = 9 N_seeds = 4 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train2, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=100, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0, 10, 0.05), alpha = hp.quniform('alpha', 0.0, 1, 0.0001), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = load_state(cname + '_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) for n in range(25): best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) save_state(cname + '_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) xgb_common(train2, y, test2, v, z, N_seeds, N_splits, cname, xgb_params)
