# 超参数寻优

## 一、集成学习类

### 1.网格搜索

• 调用的sklearn中的库
``````from sklearn.model_selection import GridSearchCV
``````
• 完整代码
``````import time

from xgboost import XGBRegressor
from sklearn import preprocessing
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split,KFold
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from utils import *

from sklearn.model_selection import GridSearchCV

x_train, x_test, y_train, y_test=load_dataset('./datasets/xxxxxx.csv')

def main():
start = time.clock()

# 构建模型
estimator = XGBRegressor(learning_rate=0.05,max_depth=6,n_estimators=700)

'''
参数空间
'''
learning_rate = [0.01,0.05,0.1]
n_estimators = [700,900,1100,1300]
max_depth = [6,10,15,20]
param_grid = dict(
learning_rate = learning_rate,
n_estimators = n_estimators,
max_depth=max_depth) # 以字典形式给参数

kflod=KFold(n_splits=5,shuffle=True) #五折交叉验证
print("Begin Train")
grid_search = GridSearchCV(estimator,param_grid,scoring = 'neg_mean_squared_error',n_jobs = -1,cv = kflod)
#scoring是损失函数类型
grid_result = grid_search.fit(x_train, y_train)
print("Best: %f using %s" % (grid_result.best_score_,grid_search.best_params_))

means = grid_result.cv_results_['mean_test_score']
params = grid_result.cv_results_['params']
for mean,param in zip(means,params):
print("%f  with:   %r" % (mean,param))
``````

### 2.随机搜索

• 调用的sklearn中的库
``````from sklearn.model_selection import RandomizedSearchCV
``````
• 部分代码
``````#设定搜索的xgboost参数搜索范围
param_dist = {
'n_estimators':range(1,1500,50),
'max_depth':range(1,16,1),
'learning_rate':np.linspace(0.001,0.9),
}

model=XGBRegressor()
grid = RandomizedSearchCV(model,param_dist,cv = 3,scoring = 'neg_mean_squared_error',n_iter=100,n_jobs = -1)
#在训练集上训练
grid.fit(x_train,y_train)
#返回最优的训练器
best_estimator = grid.best_estimator_
print(best_estimator)
#输出最优训练器的精度
print(grid.best_score_)
``````

### 3.贝叶斯优化

• 引用库
``````from bayes_opt import BayesianOptimization
``````
• 部分代码
``````def xgb_cv(max_depth, learning_rate, n_estimators):
val = cross_val_score(estimator=XGBRegressor(max_depth=int(max_depth),
learning_rate=learning_rate,
n_estimators=int(n_estimators),
objective='reg:squarederror',
booster='gbtree',
seed=0), X=x_train, y=y_train, scoring='neg_mean_squared_error',cv=10).mean()
return val

xgb_bo = BayesianOptimization(xgb_cv, pbounds={'max_depth': (1, 16),
'learning_rate': (0.001, 0.9),
'n_estimators': (1, 1500),
})
xgb_bo.maximize(n_iter=100, init_points=10)

print(xgb_bo.max)
``````

### 4.optuna

• 引用库
``````import optuna
``````
• 部分代码
``````def objective(trial):
# 自定义的参数空间
max_depth = trial.suggest_int('max_depth', 1, 16)
learning_rate = trial.suggest_uniform('learning_rate', 0.001, 0.9)
n_estimators  = trial.suggest_int('n_estimators', 50, 1500,step=50)
estimator = XGBRegressor(
objective= 'reg:squarederror',
learning_rate = learning_rate,
n_estimators =n_estimators,
max_depth = max_depth,
seed = 0)
estimator.fit(x_train, y_train)
val_pred = estimator.predict(x_test)
rmse = np.sqrt(mean_squared_error(y_test, val_pred))
return rmse

study = optuna.create_study(sampler=optuna.samplers.TPESampler(), direction='minimize')
start_time = time.time()
study.optimize(objective, n_trials=100)
end_time = time.time()
elapsed_mins, elapsed_secs = epoch_time(start_time, end_time)
print('elapsed_secs:', elapsed_secs)
print('Best value:', study.best_trial.value)
``````

### 5.hyperopt

• 引用库
``````from hyperopt import fmin, hp, partial, Trials, tpe,rand
``````
• 部分代码
``````# 自动化调参并训练
def cat_factory(argsDict):
estimator = XGBRegressor(
objective='reg:squarederror',
learning_rate=argsDict['learning_rate'],
n_estimators=int(argsDict['n_estimators']),
max_depth=int(argsDict['max_depth']),
seed=0)
estimator.fit(x_train, y_train)
val_pred = estimator.predict(x_test)
rmse = np.sqrt(mean_squared_error(y_test, val_pred))
return rmse

# 算法选择 tpe
algo = partial(tpe.suggest)
# 随机搜索
# algo = partial(rand.suggest)
# 初始化每次尝试
trials = Trials()
# 开始自动参数寻优
start_time = time.time()
best = fmin(cat_factory, space, algo=algo, max_evals=100, trials=trials)
end_time = time.time()
elapsed_mins, elapsed_secs = epoch_time(start_time, end_time)
print('elapsed_secs:', elapsed_secs)
all_opts = []
# 遍历每一次的寻参结果
for one in trials:
str_re = str(one)
argsDict = one['misc']['vals']
value = one['result']['loss']
learning_rate = argsDict["learning_rate"][0]
n_estimators = argsDict["n_estimators"][0]
max_depth = argsDict["max_depth"][0]
finish = [value, learning_rate, n_estimators, max_depth]
all_opts.append(finish)

parameters = pd.DataFrame(all_opts, columns=['value', 'learning_rate', 'n_estimators', 'max_depth'])
best = parameters.loc[abs(parameters['value']).idxmin()]  # 得到rmse最小的
print("best: {}".format(best))
``````

## 二、神经网络类

### 1.ray-tune

• 完整代码
``````import torch
import torch.nn as nn
import numpy as np
from ray import tune
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

from sklearn.datasets import load_boston

def normalize(x_data, mean, deviation):
std = (x_data - mean) / deviation
return std

def draw(y1_axis,y2_axis):
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
y1_axis=y1_axis.detach().numpy()
y2_axis = y2_axis.detach().numpy()

rmse=np.sqrt(mean_squared_error(y1_axis,y2_axis))
mae=mean_absolute_error(y1_axis,y2_axis)
r2=r2_score(y1_axis,y2_axis)
print("rmse:  ",rmse)
print("mae:  ", mae)
print("r2:  ", r2)

# 画y_test和y_test_pred折线图
plt.rcParams["font.sans-serif"] = ["SimHei"]  # 设置字体
plt.rcParams["axes.unicode_minus"] = False  # 该语句解决图像中的“-”负号的乱码问题
plt.plot(np.arange(len(y1_axis)),y1_axis, 'r', marker='.', markersize=5)
plt.plot(np.arange(len(y2_axis)),y2_axis, 'b', marker='*', markersize=5)

# 绘制图例
plt.legend(['实际值', '预测值'])
# 显示图像
plt.show()

class LinearRegressionModel(nn.Module):
def __init__(self, input_shape, linear1, linear2, output_shape):
super(LinearRegressionModel, self).__init__()
self.linear1 = nn.Linear(input_shape, linear1)
self.linear2 = nn.Linear(linear1, linear2)
self.linear3 = nn.Linear(linear2, output_shape)

def forward(self, x):
l1 = self.linear1(x)
l2 = self.linear2(l1)
l3 = self.linear3(l2)
return l3

def train_model(config):
# 指定参数与损失函数
model = LinearRegressionModel(x_train.shape[1], config['linear1'], config['linear2'], 1)
epochs = 1000
learning_rate = 0.01
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
criterion = nn.MSELoss()

loss_list = []
for epoch in range(epochs):
epoch += 1
outputs = model(x_train)
loss = criterion(outputs, y_train)
loss.backward()
loss_list.append(loss.detach().numpy())
optimizer.step()
mean_loss = np.mean(loss_list)
tune.report(my_loss=mean_loss)

def my_train(linear1,linear2):
# 指定参数与损失函数
model = LinearRegressionModel(x_train.shape[1], linear1, linear2, 1)
epochs = 1000
learning_rate = 0.01
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
criterion = nn.MSELoss()

for epoch in range(epochs):
epoch += 1
outputs = model(x_train)
loss = criterion(outputs, y_train)
loss.backward()
optimizer.step()

preds=model(x_test)
draw(y_test,preds)

from sklearn import preprocessing

mean = data.values.mean()
deviation = data.values.std()
X = data.iloc[:, :-1]
y = data.iloc[:, [-1]]
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

x_train = normalize(x_train, mean, deviation)
x_test = normalize(x_test, mean, deviation)
return x_train, x_test, y_train, y_test

if __name__ == '__main__':

x_train, x_test, y_train, y_test=load_dataset('boston.csv')
y_train=torch.from_numpy(y_train.values)   #######  解决bug，RuntimeError: mat1 and mat2 must have the same dtype
y_train=y_train.to(torch.float32)
x_train=torch.from_numpy(x_train.values)   #######  解决bug，RuntimeError: mat1 and mat2 must have the same dtype
x_train=x_train.to(torch.float32)
y_test=torch.from_numpy(y_test.values)   #######  解决bug，RuntimeError: mat1 and mat2 must have the same dtype
y_test=y_test.to(torch.float32)
x_test=torch.from_numpy(x_test.values)   #######  解决bug，RuntimeError: mat1 and mat2 must have the same dtype
x_test=x_test.to(torch.float32)

config = {
"linear1": tune.sample_from(lambda _: np.random.randint(2, 64)),  # 自定义采样
"linear2": tune.choice([2, 4, 8, 16,18,20,22,24,26,28,30,32]),  # 从给定值中随机选择
}
result = tune.run(
train_model,
resources_per_trial={"cpu": 8, },
config=config,
num_samples=100,
)
print("======================== Result =========================")
print(result.results_df)

# my_train(64,16)
``````

### 2.optuna

• 部分代码
``````def objective(trial):
model = ConvNet(trial).to(DEVICE)
lr = trial.suggest_float("lr", 1e-5, 1e-1,log=True)
optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)
batch_size=trial.suggest_int("batch_size", 64, 256,step=64)
criterion=nn.CrossEntropyLoss()
for epoch in range(EPOCHS):
model.train()
for batch_idx, (images, labels) in enumerate(train_loader):
images, labels = images.to(DEVICE), labels.to(DEVICE)
output = model(images)
loss = criterion(output, labels)
loss.backward()
optimizer.step()
model.eval()
correct = 0
for batch_idx, (images, labels) in enumerate(valid_loader):
images, labels = images.to(DEVICE), labels.to(DEVICE)
output = model(images)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(labels.view_as(pred)).sum().item()
accuracy = correct / len(valid_loader.dataset)
trial.report(accuracy, epoch)
if trial.should_prune():
raise optuna.exceptions.TrialPruned()
return accuracy

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=20)
trial = study.best_trial
print('Accuracy: {}'.format(trial.value))
print("Best hyperparameters: {}".format(trial.params))

df = study.trials_dataframe().drop(['state','datetime_start','datetime_complete','duration','number'], axis=1)
``````
• optuna寻优过程中的部分图
• 因为绘图用到了plotly库，在jupyter上可以直接绘制出来
• 示例
``````optuna.visualization.plot_optimization_history(study) # ImportError: Tried to import 'plotly' but failed. Please make sure that the package is installed correctly to use this feature. Actual error: No module named 'plotly'.     pip install plotly
``````

``````optuna.visualization.plot_contour(study, params=['batch_size', 'lr'])
``````

``````optuna.visualization.plot_parallel_coordinate(study)
``````

``````optuna.visualization.plot_param_importances(study)
``````

THE END