Keras 保存和加载导致准确性差？

Question

我的keras版本是2.2.4-tf

训练模型后，我使用 model.save() 保存了它

然后我从保存文件夹加载模型并在测试集上进行评估，不知何故准确率下降了约 30%。

使用训练集的均值和标准对测试集进行缩放，与我训练模型时相同。

这里是整个代码：

    import os
    import pdb
    import time
    import argparse
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
    import numpy as np
    import pandas as pd
    import tensorflow as tf
    from tensorflow import keras

    keras.backend.clear_session()
    np.random.seed(42)
    tf.random.set_seed(42)

    output_models = "models"

    def handle_data(DATA_DIR, DATA_FILE, TAGS_DIR, train_per, valid_per, train_st = True):

        tags_to_scale = open(os.path.join(TAGS_DIR, 'tags_to_scale.txt'),'r', encoding='utf-8').read().split('\n')    # features tags for scaling
        data = pd.read_csv(os.path.join(DATA_DIR, DATA_FILE), index_col=False, sep=",", dtype=np.float32, nrows=None) # read data

        features  = data.iloc[:,:-200]              # features
        Tem_curve = data.iloc[:,-200:]              # Thermal History Curves

        tags_to_onehot = list(set(features.columns.tolist()) - set(tags_to_scale))
        features = features.copy()

        onehot_features = features[tags_to_onehot]
        scale_features  = features[tags_to_scale ]

        m = np.shape(features)[0]

        # split the features
        train_s_feat = scale_features[:int(m*(train_per - valid_per))]
        valid_s_feat = scale_features[int(m*(train_per - valid_per)):int(m*train_per)]
        test_s_feat  = scale_features[int(m*train_per):]
        if train_st:
            train_stats  = train_s_feat.describe().transpose()
            train_stats.to_csv("{}/stats.data".format(output_models))
        else:
            train_stats = pd.read_csv("{}/stats.data".format(output_models), index_col=0)
        # z-score applied
        train_s_feat = (train_s_feat-train_stats["mean"]) / (train_stats["std"]+1e-8)
        valid_s_feat = (valid_s_feat-train_stats["mean"]) / (train_stats["std"]+1e-8)
        test_s_feat  = (test_s_feat -train_stats["mean"]) / (train_stats["std"]+1e-8)

        X_train = pd.concat([train_s_feat, onehot_features[:int(m*(train_per - valid_per))]], axis=1)
        X_valid = pd.concat([valid_s_feat, onehot_features[int(m*(train_per - valid_per)):int(m*train_per)]], axis=1)
        X_test  = pd.concat([test_s_feat , onehot_features[int(m*train_per):]], axis=1)

        y_train = Tem_curve[:int(m*(train_per - valid_per))]
        y_valid = Tem_curve[int(m*(train_per - valid_per)):int(m*train_per)]
        y_test  = Tem_curve[int(m*train_per):]

        return X_train, y_train, X_valid, y_valid, X_test, y_test

    def build_model(input_size=28):
        model = keras.Sequential([keras.layers.Dense(192, activation="selu", kernel_initializer="lecun_normal", input_shape=[input_size]),
                                  keras.layers.Dense(256, activation="selu", kernel_initializer="lecun_normal"),
                                  keras.layers.Dense(256, activation="selu", kernel_initializer="lecun_normal"),
                                  keras.layers.Dense(160, activation="selu", kernel_initializer="lecun_normal"),
                                  keras.layers.Dense(200)])
        
        model.compile(loss=keras.losses.Huber(), optimizer=keras.optimizers.Nadam(learning_rate=0.01), metrics=["mae"])
        return model

    if __name__ == '__main__':
        parser = argparse.ArgumentParser()
        parser.add_argument("--mode", type=int, default=0)          # 1 for trainning, others for testing
        parser.add_argument("--train_per", type=float, default=0.8)
        parser.add_argument("--valid_per", type=float, default=0.1)
        parser.add_argument("--data_file", type=str, default="Complex2_reset_data.csv")
        parser.add_argument("--data_dir", type=str, default="./data/")
        parser.add_argument("--tags_dir", type=str, default="./tags/")
        args = parser.parse_args()

        assert args.train_per > 0.
        assert args.train_per < 1.
        assert args.valid_per > 0.
        assert args.valid_per < 1.
        assert args.train_per > args.valid_per

        if not os.path.exists(output_models):
            os.makedirs(output_models)

        if args.mode == 1:
            X_train, y_train, X_valid, y_valid, X_test, y_test = handle_data(DATA_DIR=args.data_dir, DATA_FILE=args.data_file, TAGS_DIR=args.tags_dir, train_per=args.train_per, valid_per=args.valid_per)
            model = build_model(np.shape(X_train)[1])
            early_stopping_cb = keras.callbacks.EarlyStopping(patience=20, monitor="val_loss")
            # checkpoint_cb = keras.callbacks.ModelCheckpoint(os.path.join(output_models, "Thermal.h5"))
            model.fit(X_train, y_train, batch_size=1024, epochs=10, validation_data=(X_valid, y_valid), verbose=1, callbacks=[early_stopping_cb])
            model.save_weights(os.path.join(output_models, "Thermal.h5"))
        else:
            X_train, y_train, X_valid, y_valid, X_test, y_test = handle_data(DATA_DIR=args.data_dir, DATA_FILE=args.data_file, TAGS_DIR=args.tags_dir, train_per=args.train_per, valid_per=args.valid_per, train_st=False)
            model = build_model(np.shape(X_train)[1])
            model.load_weights(os.path.join(output_models, "Thermal.h5"))               # load saved model

        X_test = np.vstack((X_train, X_valid, X_test))
        y_test = np.vstack((y_train, y_valid, y_test))
        # np.testing.assert_allclose(model.predict(X_test), re_model.predict(X_test))
        test_num = np.shape(y_test)[0]
        print("test set size:" + str(test_num))                 # test set size
        loss, mae = model.evaluate(X_test, y_test, batch_size=1024, verbose=1)      # evaluate on the test set
        print("test_loss: "+str(loss)+' '+"test_mae: "+str(mae))

        Y_reg = model.predict(X_test)

        Predicted_Curve   = Y_reg
        Groundtruth_Curve = np.array(y_test) 

        statistics = [0, 0, 0, 0]                               # store the accuracy

        for i in range(test_num):
            delta = np.average(np.abs(Predicted_Curve[i] - Groundtruth_Curve[i]))   # Mean Absolute Error per Thermal History Curve
            if delta <= 5:                      # average abs differences smaller than 5
                statistics[0] += 1
            if delta <= 10:                     # average abs differences smaller than 10
                statistics[1] += 1
            if delta <= 15:                     # average abs differences smaller than 15
                statistics[2] += 1
            if delta <= 20:                     # average abs differences smaller than 20
                statistics[3] += 1
        print(statistics)
        for j in range(len(statistics)):
            statistics[j] = (statistics[j] / test_num) * 100    # convert to percentage
        print("{}% data with MAE <= 5 ".format(statistics[0]))
        print("{}% data with MAE <= 10".format(statistics[1]))
        print("{}% data with MAE <= 15".format(statistics[2]))
        print("{}% data with MAE <= 20".format(statistics[3]))


After training I test

test_loss: 3.0357406968625633 test_mae: 3.4672117

test set size:680497

[532391, 636503, 666152, 675335]

78.23561308866903% data with MAE <= 5

93.53501925798349% data with MAE <= 10

97.89198188970708% data with MAE <= 15

99.24143677341708% data with MAE <= 20


After loading the model

test_loss: 6.598824016843591 test_mae: 7.0739527

test set size:680497

[338312, 541556, 620373, 648462]

49.715428576466905% data with MAE <= 5

79.58242284683107% data with MAE <= 10

91.16469286418602% data with MAE <= 15

95.29241128175437% data with MAE <= 20

不知道为什么

Answer 1

事实证明，当我拆分数据时，一部分用于 z-score，另一部分用于 one-hot，pandas 并没有保持特征的原始顺序，所以在连接特征时，我需要保留顺序如下：

features = pd.concat([X_train, X_valid, X_test], axis=0)

features = features.sort_index(axis=1)