使用 tf.data.Dataset 时在 keras.sequential 上的 model.fit 会引发 ValueError

Question

我正在尝试使用 tf.data.dataset 作为 Keras.sequential 的输入在 tensorflow 1.10 上构建我的第一个分类器，但 fit 方法返回以下错误：

ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)

首先我用我的数据集的文件名初始化了 2 tf.data.Dataset

 #Initialize dataset directories location and parameters
image_size=50
batch_size=10
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'

#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)

然后我使用 map 方法来准备我的数据集

#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))


def load_resize_label(filename, label):
    image_string = tf.read_file(filename)
    image_decoded = tf.image.decode_jpeg(image_string)
    image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
    image_resized=image_resized/255.0
    return image_resized, tf.convert_to_tensor(label)

然后，我将数据集连接成一个最终数据集并初始化批量大小

#Merge the datasets


dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()

最后使用模型对象的compile and fit方法

model.compile(loss='binary_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

model.fit(dataset, epochs=10, steps_per_epoch=30)

（完整代码如下）

我正在使用：

Windows 10 64 位

cudnn-9.0-windows10-x64-v7.2.1.38

cuda_9.0.176_win10

张量流-GPU 1.10.0

  import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
    image_string = tf.read_file(filename)
    image_decoded = tf.image.decode_jpeg(image_string)
    image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
    image_resized=image_resized/255.0
    return image_resized, tf.convert_to_tensor(label)

#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'

#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)

#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))

#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()

#the CNN architecture
model = keras.Sequential([
    keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size,3)),
    keras.layers.MaxPool2D(pool_size=2),
    keras.layers.BatchNormalization(),
    keras.layers.Flatten(),
    keras.layers.Dense(128, activation=tf.nn.relu),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(2, activation=tf.nn.softmax)
])


model.compile(loss='binary_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

model.fit(dataset, epochs=10, steps_per_epoch=30)

追溯：

    Traceback (most recent call last):
  File "C:/Users/Jonas/PycharmProjects/learning/lesson2.py", line 47, in <module>
    model.fit(dataset, epochs=10, steps_per_epoch=30)
  File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 1278, in fit
    validation_split=validation_split)
  File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training.py", line 917, in _standardize_user_data
    exception_prefix='target')
  File "C:\Users\Jonas\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow\python\keras\engine\training_utils.py", line 182, in standardize_input_data
    'with shape ' + str(data_shape))
ValueError: Error when checking target: expected dense_1 to have 2 dimensions, but got array with shape (None,)

Answer 1

您的代码中缺少一些“=”。

每个数据集操作应该是这样的：

dataset = dataset.some_ops(...)

你的代码应该是这样的：

import tensorflow as tf
from tensorflow import keras
image_size=50
batch_size=10
# Reads an image from a file, decodes it into a dense tensor, resizes it
# to a fixed shape.
def load_resize_label(filename, label):
    image_string = tf.read_file(filename)
    image_decoded = tf.image.decode_jpeg(image_string)
    image_resized = tf.image.resize_images(image_decoded, [image_size, image_size])
    image_resized=image_resized/255.0
    if label == 'morty':
         label = [0, 1]
    elif label == 'rick':
         label = [1, 0]
    else:
         raise ValueError(label)
    return image_resized, tf.convert_to_tensor(label)

#Initialize dataset directories location
mortys_file_pattern = r'C:\Users\Jonas\Downloads\mortys\*'
ricks_file_pattern = r'C:\Users\Jonas\Downloads\ricks\*'

#Each tensor in those dataset will be a filename for a specific image
mortys_dataset = tf.data.Dataset.list_files(mortys_file_pattern)
ricks_dataset = tf.data.Dataset.list_files(ricks_file_pattern)

#Now, each dataset entry will contain 2 tensors: image,label
mortys_dataset = mortys_dataset.map(lambda filename: load_resize_label(filename, "morty"))
ricks_dataset = ricks_dataset.map(lambda filename: load_resize_label(filename, "rick"))

#Merge the datasets
dataset = mortys_dataset.concatenate(ricks_dataset)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()

#the CNN architecture
model = keras.Sequential([
    keras.layers.Convolution2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(image_size, image_size, 3)),
    keras.layers.MaxPool2D(pool_size=2),
    keras.layers.BatchNormalization(),
    keras.layers.Flatten(),
    keras.layers.Dense(128, activation=tf.nn.relu),
    keras.layers.Dropout(0.3),
    keras.layers.Dense(2, activation=tf.nn.softmax)
])


model.compile(loss='categorical_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

model.fit(dataset, epochs=10, steps_per_epoch=30)

另外，我建议您使用 dataset.prefetch(None) 并在 map 函数中使用 num_parallel_calls 参数。 Here is why。 TLDR：更快。