I am using the awesome scikit-optimize toolbox for hyperparameter optimization. My goal is to compare keras and scikit-learn models.
According to the example https://scikit-optimize.github.io/stable/auto_examples/sklearn-gridsearchcv-replacement.html#sphx-glr-auto-examples-sklearn-gridsearchcv-replacement-py only scikit learn models had been used. Trying something like the following code does not allow to integrate the keras mode in the BayesSearchCV.
# Function to create model, required for KerasRegressor
def create_model(optimizer='rmsprop', init='glorot_uniform'):
# create model
model = Sequential()
model.add(Dense(12, input_dim=8, kernel_initializer=init, activation='relu'))
model.add(Dense(8, kernel_initializer=init, activation='relu'))
model.add(Dense(1, kernel_initializer=init, activation='linear'))
# Compile model
model.compile(loss='mse', optimizer=optimizer, metrics=['r2'])
return model
model = KerasRegressor(build_fn=create_model, verbose=0)
NN_search = {
'model': [model()],
'model__optimizers': optimizers,
'model__epochs' : epochs,
'model__batch_size' : batches,
'model__init' : init
}
Has anyone managed to merge a KerasClassifier/Regressor into BayesSearch CV?
Well, I found an option to define a model that is build up based on global parameters. So inside of the scikit-opt minimize function the objective function is called, here the global parameters are set, and used in the create_model_NN function, which is built on the keras scikit-learn KerasRegressor Wrapper.
def create_model_NN():
#start the model making process and create our first layer
model = Sequential()
model.add(Dense(num_input_nodes, input_shape=(40,), activation=activation
))
#create a loop making a new dense layer for the amount passed to this model.
#naming the layers helps avoid tensorflow error deep in the stack trace.
for i in range(num_dense_layers):
name = 'layer_dense_{0}'.format(i+1)
model.add(Dense(num_dense_nodes,
activation=activation,
name=name
))
#add our classification layer.
model.add(Dense(1,activation='linear'))
#setup our optimizer and compile
adam = Adam(lr=learning_rate)
model.compile(optimizer=adam, loss='mean_squared_error',
metrics=['mse'])
return model
def objective_NN(**params):
print(params)
global learning_rate
learning_rate=params["learning_rate"]
global num_dense_layers
num_dense_layers=params["num_dense_layers"]
global num_input_nodes
num_input_nodes=params["num_input_nodes"]
global num_dense_nodes
num_dense_nodes=params["num_dense_nodes"]
global activation
activation=params["activation"]
model = KerasRegressor(build_fn=create_model, epochs=100, batch_size=1000, verbose=0)
X_train, X_test, y_train, y_test = train_test_split(X_time, y_time, test_size=0.33, random_state=42)
model.fit(X_train, y_train)
y_pr = model.predict(X_test)
res = metrics.r2_score(y_test, y_pr)
return -res
And to call it:
res_gp = gp_minimize(objective_NN, space_NN, n_calls=10, random_state=0)