Tensorflow 2.0 相同的模型结构和超参数导致不同调用方法的性能不同答案

【问题标题】：Tensorflow 2.0 Identical model structure and hyper parameters result in different performance in different calling approachesTensorflow 2.0 相同的模型结构和超参数导致不同调用方法的性能不同
【发布时间】：2019-11-25 17:45:50
【问题描述】：

那里。我是初学者，正在学习 Tensorflow 2.0。我有一个用 3 种不同方法调用的模型。而且表现也不一样。谁能告诉我为什么会这样？

模型的构建和调用方法：

import os, sys

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, models, layers, regularizers, optimizers


def prepare_mnist_features_and_labels(x, y):
    x = tf.cast(x, tf.float32)/255.0
    y = tf.cast(y, tf.int64)
    return x, y


def mninst_dataset():
    (x_train, y_train), (x_eval, y_eval) = datasets.mnist.load_data()
    print('x_train/y_train shape:', x_train.shape, y_train.shape)
    y_train = tf.one_hot(y_train, depth=10)
    y_eval = tf.one_hot(y_eval, depth=10)

    ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    ds_train = ds_train.map(prepare_mnist_features_and_labels)
    ds_train = ds_train.shuffle(x_train.shape[0]).batch(128)

    ds_eval = tf.data.Dataset.from_tensor_slices((x_eval, y_eval))
    ds_eval = ds_eval.map(prepare_mnist_features_and_labels)
    ds_eval = ds_eval.shuffle(x_eval.shape[0]).batch(128)

    sample = next(iter(ds_train))
    print('sample: ', sample[0].shape, sample[1].shape)

    return ds_train, ds_eval


def main():
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    trainset, evalset = mninst_dataset()

    model = keras.Sequential()
    model.add(layers.Reshape(target_shape=[28, 28, 1], input_shape=[28, 28]))
    model.add(layers.Conv2D(filters=32, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME"))
    model.add(layers.MaxPool2D(pool_size=(2,2), strides=[1,1], padding="SAME"))
    model.add(layers.Conv2D(filters=64, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME"))
    model.add(layers.MaxPool2D(pool_size=(2,2), strides=[2,2], padding="SAME"))
    model.add(layers.Flatten())
    model.add(layers.Dense(units=512, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01)))
    model.add(layers.Dense(units=10, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01)))

    model.compile(optimizer=optimizers.Adam(lr=0.01), loss=tf.losses.CategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
    model.fit(trainset.repeat(), epochs=30, steps_per_epoch=500,
              validation_data=evalset.repeat(), validation_steps=10)

if __name__=='__main__':
    main()

构建模型并运行它的第二种方法如下：

    import os, sys

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, models, layers, regularizers, optimizers

from tqdm import tqdm

def prepare_mnist_features_and_labels(x, y):
    x = tf.cast(x, tf.float32)/255.0
    y = tf.cast(y, tf.int64)
    return x, y


def mnist_dataset():
    (x_train, y_train), (x_eval, y_eval) = datasets.mnist.load_data()
     # y_train = tf.one_hot(y_train, depth=10)
    # y_eval = tf.one_hot(y_eval, depth=10)

    ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    ds_train = ds_train.map(prepare_mnist_features_and_labels)
    # Test: replace x_train.shape[0] by the number of the training samples, which is 60000
    ds_train = ds_train.shuffle(x_train.shape[0]).batch(128)

    ds_eval = tf.data.Dataset.from_tensor_slices((x_eval, y_eval))
    ds_eval = ds_eval.map(prepare_mnist_features_and_labels)
    ds_eval = ds_eval.shuffle(x_eval.shape[0]).batch(128)

    # sample = next(iter(ds_train))
    # print('sample: ', sample[0].shape, sample[1].shape)

    return ds_train, ds_eval

# tf.nn.sparse_softmax_cross_entropy_with_logits(labels, logits, name=None):
# labels: Tensof of shape [d_0, d_1, ..., d_{r-1}]. Each label must be an index in [0, num_classes]
# logits: Unscaled of log probabilities of shape [d_0, d_1, ..., d_{r-1}, num_classes]
# A common use is to have logits of shape [batch_size, num_classes] and have labels of shape [batch_size]
def compute_loss(logits, labels):
    # print(logits.numpy())
    # print(labels.numpy())
    return tf.reduce_mean(
        tf.nn.sparse_softmax_cross_entropy_with_logits(
             labels=labels, logits=logits
                )
            )


def compute_accuracy(logits, labels):
    predictions = tf.argmax(logits, axis=1)
    # print(predictions)
    # print(labels)
    # print(list(zip(predictions.numpy(), labels.numpy())))
    return tf.reduce_mean(tf.cast(tf.equal(predictions, labels), tf.float32))


def train_one_step(model, optimizer, x, y):
    # At each train step, first calculate the forward loss
    with tf.GradientTape() as tape:
        logits = model(x)
        loss = compute_loss(logits, y)

    # Then calculate the backward gradients over each trainable variables
    grads = tape.gradient(loss, model.trainable_variables)
    # Optimize and update variables throught backpropagation
    optimizer.apply_gradients(zip(grads, model.trainable_variables))

    # Compute current model accuracy
    accuracy = compute_accuracy(logits, y)

    return loss, accuracy


def train(epoch, model, optimizer, trainset):
#def train(epoch, model, optimizer):
    # trainset = mnist_dataset()[0]
    loss = 0.0
    accuracy = 0.0

    #for step, (x, y) in enumerate(tqdm(trainset)):
    for step, (x, y) in enumerate(tqdm(trainset)):
        loss, accuracy = train_one_step(model, optimizer, x, y)

        if step % 110 == 0:
            print('epoch', epoch, ': loss', loss.numpy(), '; accuracy', accuracy.numpy())

    return loss, accuracy


class MyModel(keras.Model):


    def __init__(self):
        super(MyModel, self).__init__()
        self.layer1 = layers.Conv2D(filters=32, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME", input_shape=(-1, 28, 28, 1))
        self.layer2 = layers.MaxPool2D(pool_size=(2,2), strides=[1,1], padding="SAME")
        self.layer3 = layers.Conv2D(filters=64, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME")
        self.layer4 = layers.MaxPool2D(pool_size=(2,2), strides=[2,2], padding="SAME")
        self.layer5 = layers.Flatten()
        self.layer6 = layers.Dense(units=512, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01))
        self.layer7 = layers.Dense(units=10, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01))


    def call(self, x, training=False):
        x = tf.reshape(x, (-1, 28, 28, 1))
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.layer5(x)
        x = self.layer6(x)
        x = self.layer7(x)
        return x


def main():

    # set random seed
    tf.random.set_seed(22)    

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    trainset, evalset = mnist_dataset()

    model = MyModel()

    optimizer = optimizers.Adam(lr=0.001)

    # Save checkpoints with keras api as the first approach
    # Save checkpoints manually as a second approach.
    # find a way to implement early-stopping strategy in the programming style

    # for epoch in tqdm(range(30)):
    for epoch in range(50):
        loss, accuracy = train(epoch, model, optimizer, trainset)

    print('Final epoch', epoch, ': loss', loss.numpy(), '; accuracy', accuracy.numpy())


if __name__ == '__main__':
    main()

最后一种方法如下：

import os, sys

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, models, layers, regularizers, optimizers

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt


def prepare_mnist_features_and_labels(x, y):
    x = tf.cast(x, tf.float32)/255.0
    y = tf.cast(y, tf.int64)
    return x, y


def mnist_dataset():
    (x_train, y_train), (x_eval, y_eval) = datasets.mnist.load_data()
    print('x_train/y_train shape:', x_train.shape, y_train.shape)
    y_train = tf.one_hot(y_train, depth=10)
    y_eval = tf.one_hot(y_eval, depth=10)

    ds_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
    ds_train = ds_train.map(prepare_mnist_features_and_labels)
    ds_train = ds_train.shuffle(x_train.shape[0]).batch(128)

    ds_eval = tf.data.Dataset.from_tensor_slices((x_eval, y_eval))
    ds_eval = ds_eval.map(prepare_mnist_features_and_labels)
    ds_eval = ds_eval.shuffle(x_eval.shape[0]).batch(128)

    sample = next(iter(ds_train))
    print('sample: ', sample[0].shape, sample[1].shape)

    return ds_train, ds_eval


class MyModel(keras.Model):

    # self.model = keras.Sequential([
    #     layers.Reshape(target_shape=(28*28, ), input_shape=(28, 28)),
    #     layers.Dense(100, activation=tf.nn.relu),
    #     layers.Dense(100, activation=tf.nn.relu),
    #     layers.Desnse(10)
    # ])
    def __init__(self):
        super(MyModel, self).__init__()
        self.layer1 = layers.Conv2D(filters=32, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME", input_shape=(-1, 28, 28, 1))
        self.layer2 = layers.MaxPool2D(pool_size=(2,2), strides=[1,1], padding="SAME")
        self.layer3 = layers.Conv2D(filters=64, kernel_size=(5, 5), activation=tf.nn.relu, strides=[1,1], padding="SAME")
        self.layer4 = layers.MaxPool2D(pool_size=(2,2), strides=[2,2], padding="SAME")
        self.layer5 = layers.Flatten()
        self.layer6 = layers.Dense(units=512, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01))
        self.layer7 = layers.Dense(units=10, activation=tf.nn.relu, kernel_regularizer=regularizers.l2(0.01))


    def call(self, x, training=False):
        x = tf.reshape(x, (-1, 28, 28, 1))
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        x = self.layer5(x)
        x = self.layer6(x)
        x = self.layer7(x)
        return x


def main():
    tf.random.set_seed(22)

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

    trainset, evalset = mnist_dataset()

    model = MyModel()
    model.compile(optimizer=optimizers.Adam(lr=0.001), loss=tf.losses.CategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
    model.fit(trainset.repeat(), epochs=30, steps_per_epoch=500, verbose=1,
              validation_data=evalset.repeat(), validation_steps=10)


if __name__ == '__main__':
    main()

他们每个人都需要一段时间来训练。谁能告诉我为什么表演不同？以后怎么自己调试呢？

非常感谢您的帮助。

【问题讨论】：

标签： keras deep-learning tensorflow2.0

【解决方案1】：

仔细检查网络后问题解决。事实证明，模型中的最后一个全连接层是用 relu 函数激活的，这是不合适的。并且损失函数 tf.losses.categoricalCrossentropy 和 tf.nn.sparse_softmax_cross_entropy_with_logits 的选择也有很大的不同。无论选择什么，都要确保损失函数和网络的最终输出匹配。

【讨论】：