如何在神经网络中执行 K 折交叉验证？

Question

我正在为二进制图像分类问题（猫/狗）开发 CNN。 我的目标是使用 K-Fold CV（在这种情况下我将应用 5 折）来找到最佳参数（批量大小、时期）。

到目前为止我的代码是这样的

# Defining the Loss
loss = binary_crossentropy

# Creating the grid of parameters
batches = [32, 64, 128, 256]
epochs = [20, 30, 40, 50]
params_grid = dict(batch_size = batches, epochs = epochs)

# Creating the model
def model_cnn_three_layer(optimizer='adam'):
    model = tf.keras.Sequential([
        tf.keras.layers.Conv2D(32, (3, 3), padding = "same", use_bias=False, input_shape = (64, 64, 1), activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Conv2D(32, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Conv2D(64, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Conv2D(64, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(128, use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(2, activation = 'softmax')
    ])

    # Compiling the model
    model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])

    model.summary()

    return model

# Create the sklearn CV model for the network

model_cnn_three_layer_CV = KerasClassifier(build_fn=model_cnn_three_layer, verbose=1)

grid = GridSearchCV(estimator=model_cnn_three_layer_CV, 
                    param_grid=params_grid,
                    cv=5)

grid_result = grid.fit(X_train, y_train)

# Print results
print(f'Best Accuracy for {grid_result.best_score_:.4} using {grid_result.best_params_}')
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print(f'mean={mean:.4}, std={stdev:.4} using {param}')

这种方法正确吗？

如果我想“手动”计算 CV（不使用 sklearn），我将如何更改代码？ 我找到了一个类似问题的答案，它做这样的事情

# parameters
epochs = 20
batch_size = 64

# Defining callback(s)
early_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)

# Defining plots
legend_size = 14

# Define the K-fold Cross Validator
num_folds = 5
kfold = KFold(n_splits=num_folds, shuffle=True)

loss_cnn_three_layer = []
acc_cnn_three_layer = []

fold_no = 1
for train, test in kfold.split(X, y):
    model = tf.keras.Sequential([
        tf.keras.layers.Conv2D(32, (3, 3), padding = "same", use_bias=False, input_shape = (64, 64, 1), activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Conv2D(32, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Conv2D(64, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Activation('relu'),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Conv2D(64, (3, 3), padding = "same", use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.MaxPooling2D(pool_size = (2, 2)),
        tf.keras.layers.Dropout(0.25),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(128, use_bias=False, activation = 'relu'),
        tf.keras.layers.BatchNormalization(),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(2, activation = 'softmax')
    ])

    # compiling the model
    model.compile(optimizer='adam', loss=loss, metrics=['accuracy'])

    net_name = "CNN_three_layers_batch_and_dropout"

    model.summary()

    # log dir for saving TensorBoard logs
    logdir = os.path.join("CNN_nets", net_name)

    # callback to run TensorBoard
    tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)
    callbacks = [tensorboard_callback, early_callback]

    history = model.fit(X_train, y_train, epochs=epochs, validation_data=(X_test, y_test),
                        batch_size=batch_size, callbacks=callbacks, verbose=1)

    scores = model.evaluate(X_test, y_test)
    print(
        f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1] * 100}%')
    acc_cnn_three_layer.append(scores[1] * 100)
    loss_cnn_three_layer.append(scores[0])

    # Increase fold number
    fold_no = fold_no + 1

# == Provide average scores ==
print('------------------------------------------------------------------------')
print('Score per fold')
for i in range(0, len(loss_cnn_three_layer)):
    print('------------------------------------------------------------------------')
    print(f'> Fold {i + 1} - Loss: {loss_cnn_three_layer[i]} - Accuracy: {acc_cnn_two_layer[i]}%')
print('------------------------------------------------------------------------')
print('Average scores for all folds:')
print(f'> Accuracy: {np.mean(acc_cnn_three_layer)} (+- {np.std(acc_cnn_three_layer)})')
print(f'> Loss: {np.mean(loss_cnn_three_layer)}')
print('------------------------------------------------------------------------')

但我不相信这种方法，因为它只是在相同数据上运行模型 5 次，而不是在训练数据的不同拆分上运行。这将如何更改以有效地在训练数据的拆分部分上运行 CV，然后对测试数据进行评估？此外，我将如何在网格参数的值上循环最后一个网络？

Answer 1

from sklearn.model_selection import StratifiedKFold as kfold

x = # features
y = # labels

batches = [32, 64, 128, 256]
epochs = [20, 30, 40, 50]

splits = 5
kf = kfold(splits, shuffle=True)
indices = kf.split(x, y)
loss_cnn_three_layer = []
acc_cnn_three_layer = []
preds = []
for train, test in indices:
    x_train, x_test, y_train, y_test = x[train], x[test], y[train], y[test]

    # do model stuff

    history = model.fit(x_train, y_train, shuffle=True, epochs=10, verbose=1)
    prediction = model.predict(x_test)
    loss_cnn_three_layer.append(history.history["loss"])
    acc_cnn_three_layer.append(history.history["accuracy"])
    preds.append(prediction)

编辑以包含参数的可迭代：

from sklearn.model_selection import StratifiedKFold as kfold

x = # features
y = # labels

splits = 5
kf = kfold(splits, shuffle=True)
indices = kf.split(x, y)
loss_cnn_three_layer = []
acc_cnn_three_layer = []
preds = []
for batch, epochs in zip(batches, epochs):
    for train, test in indices:
        x_train, x_test, y_train, y_test = x[train], x[test], y[train], y[test]

        # do model stuff
        
        history = model.fit(x_train, y_train, shuffle=True, batch_size=batch epochs=epochs, verbose=1)
        prediction = model.predict(x_test)
        loss_cnn_three_layer.append(history.history["loss"])
        acc_cnn_three_layer.append(history.history["accuracy"])
        preds.append(prediction)

如果您想根据 kfold 迭代不同的批次和时期，只需交换两个 for 展示位置，但将其他所有内容保留在其中。

如果您想拥有字典，请执行以下操作：

for i, j in zip([*params_grid.values()]):  # assuming batch and epoch lists have the same length
    # where i is batch, and j is epochs
    # do stuff

如果您想根据每个批次大小的 epoch 数来训练模型（反之亦然*），请执行以下操作：

for k, l in [(i, j) for j in epochs for i in batches]:  # swap batches and epochs for vice versa*
    # where k is batch, and l is epochs
    # do stuff

Answer 2

我尝试了以下解决方案

loss_cnn_three_layer = []
acc_cnn_three_layer = []

# create the first loop for batches and epochs
for batch, epoch in zip(batches, epochs):   
# second loop for training the model on each split
    for train, test in indices:
        X_train, X_test, y_train, y_test = X[train], X[test], y[train], y[test]

        # model = tf.keras.Sequential([ ... ])
    
        # compiling the model
        model.compile(optimizer = optimizer, loss=loss, metrics=['accuracy'])

        net_name = "CNN_three_layers_batch_and_dropout"

        model.summary()


        # log dir for saving TensorBoard logs
        logdir = os.path.join("CNN_nets", net_name)

        # callback to run TensorBoard
        tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq = 1)
        callbacks = [tensorboard_callback, early_callback]
        
        # fitting the network
        history = model.fit(X_train, y_train, epochs = epoch,
                            batch_size = batch, callbacks = callbacks, verbose = 1)

        # evaluating the performance
        scores = model.evaluate(X_test, y_test)
        
        # printing accuracy and loss
        print(f'Score per batch {batch} and epochs {epoch}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')
        acc_cnn_three_layer.append(scores[1] * 100)
        loss_cnn_three_layer.append(scores[0])

然而，通过这样做，它仅在批次和时期（32、20）的第一个组合上运行模型和交叉验证，然后停止。