我尝试将贝叶斯优化应用于 MNIST 手写数字数据集的简单 CNN，但几乎没有迹象表明它有效。我已经尝试进行 k 折验证以消除噪声，但似乎优化似乎没有在向最优参数收敛方面取得任何进展。一般来说，贝叶斯优化可能失败的主要原因是什么？在我的特殊情况下？
剩下的只是上下文和代码片段。
型号定义：
def define_model(learning_rate, momentum):
model = Sequential()
model.add(Conv2D(32, (3,3), activation = 'relu', kernel_initializer = 'he_uniform', input_shape=(28,28,1)))
model.add(MaxPooling2D((2,2)))
model.add(Flatten())
model.add(Dense(100, activation='relu', kernel_initializer='he_uniform'))
model.add(Dense(10, activation='softmax'))
opt = SGD(lr=learning_rate, momentum=momentum)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
return model
使用超参数运行一次训练：batch_size = 32，学习率 = 1e-2，动量 = 0.9，10 个 epoch。（蓝色 = 训练，黄色 = 验证）。
用于 5 倍交叉验证准确性的箱须图，具有与上述相同的超参数（以了解传播）
网格搜索将 batch_size 保持在 32，并保持 10 个 epoch。我是在单次评估而不是 5 倍的情况下这样做的，因为价差不足以破坏结果。
贝叶斯优化。如上，batch_size=32 和 10 个 epoch。搜索相同的范围。但这次使用 5 折交叉验证来消除噪音。它应该进行 100 次迭代，但这还需要 20 个小时。
space = {'lr': hp.loguniform('lr', np.log(np.sqrt(10)*1e-4), np.log(1e-1)), 'momentum': 1 - hp.loguniform('momentum', np.log(np.sqrt(10)*1e-3), np.log(np.sqrt(10)*1e-1))}
tpe_best = fmin(fn=objective, space=space, algo=tpe.suggest, trials=Trials(), max_evals=100)
试验学习率
经受考验的势头
从第 27 次迭代到第 49 次迭代看起来不错，但随后又失去了理智。
编辑
询问的人更详细。
进口
# basic utility libraries
import numpy as np
import pandas as pd
import time
import datetime
import pickle
from matplotlib import pyplot as plt
%matplotlib notebook
# keras
from keras.datasets import mnist
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Input, BatchNormalization
from keras.optimizers import SGD
from keras.callbacks import Callback
from keras.models import load_model
# learning and optimisation helper libraries
from sklearn.model_selection import KFold
from hyperopt import fmin, tpe, Trials, hp, rand
from hyperopt.pyll.stochastic import sample
单一评价
def evaluate_model(trainX, trainY, testX, testY, max_epochs, learning_rate, momentum, batch_size, model=None, callbacks=[]):
if model == None:
model = define_model(learning_rate, momentum)
history = model.fit(trainX, trainY, epochs=max_epochs, batch_size=batch_size, validation_data=(testX, testY), verbose=0, callbacks = callbacks)
return model, history
交叉验证
def evaluate_model_cross_validation(trainX, trainY, max_epochs, learning_rate, momentum, batch_size, n_folds=5):
scores, histories = list(), list()
# prepare cross validation
kfold = KFold(n_folds, shuffle=True, random_state=1)
# enumerate splits
for trainFold_ix, testFold_ix in kfold.split(trainX):
# select rows for train and test
trainFoldsX, trainFoldsY, testFoldX, testFoldY = trainX[trainFold_ix], trainY[trainFold_ix], trainX[testFold_ix], trainY[testFold_ix]
# fit model
model = define_model(learning_rate, momentum)
history = model.fit(trainFoldsX, trainFoldsY, epochs=max_epochs, batch_size=batch_size, validation_data=(testFoldX, testFoldY), verbose=0)
# evaluate model
_, acc = model.evaluate(testFoldX, testFoldY, verbose=0)
# stores scores
scores.append(acc)
histories.append(history)
return scores, histories
我如何设置贝叶斯优化（或随机搜索）
def selective_search(kind, space, max_evals, batch_size=32):
trainX, trainY, testX, testY = prep_data()
histories = list()
hyperparameter_sets = list()
scores = list()
def objective(params):
lr, momentum = params['lr'], params['momentum']
accuracies, _ = evaluate_model_cross_validation(trainX, trainY, max_epochs=10, learning_rate=lr, momentum=momentum, batch_size=batch_size, n_folds=5)
score = np.log10(1 - np.mean(accuracies))
scores.append(score)
with open('{}_scores.pickle'.format(kind), 'wb') as file:
pickle.dump(scores, file)
hyperparameter_sets.append({'learning_rate': lr, 'momentum': momentum, 'batch_size': batch_size})
with open('{}_hpsets.pickle'.format(kind), 'wb') as file:
pickle.dump(hyperparameter_sets, file)
return score
if kind == 'bayesian':
tpe_best = fmin(fn=objective, space=space, algo=tpe.suggest, trials=Trials(), max_evals=max_evals)
elif kind == 'random':
tpe_best = fmin(fn=objective, space=space, algo=rand.suggest, trials=Trials(), max_evals=max_evals)
else:
raise BaseError('First parameter "kind" must be either "bayesian" or "random"')
return histories, hyperparameter_sets, scores
然后我如何实际运行贝叶斯优化。
space = {'lr': hp.loguniform('lr', np.log(np.sqrt(10)*1e-4), np.log(1e-1)), 'momentum': 1 - hp.loguniform('momentum', np.log(np.sqrt(10)*1e-3), np.log(np.sqrt(10)*1e-1))}
histories, hyperparameter_sets, scores = selective_search(kind='bayesian', space=space, max_evals=100, batch_size=32)
