免分割车牌识别模型问题

Question

我们的团队旨在创建一个免分割车牌识别模型，其架构如下：

我们已经成功实现了大部分架构，但我们正在努力连接 8 个全连接层分支（每个分支对应一个车牌字符）。

有没有办法为 8 个分支中的每一个使用相同的输入（25x5x128 张量），让它们独立工作，并将它们的输出与车牌的相应地面真值字母独立比较，并基于惩罚（损失函数）哪些字母不正确？

我们使用 keras 模型（包括 Sequential 和 Model 类 API）尝试了多种方法，但没有任何运气。以下是我们当前版本的模型。我们将不胜感激。

model = models.Sequential()

model.add(layers.Conv2D(32, (3, 3), use_bias=False, input_shape=(32, 32, 3),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(32, (3, 3), use_bias=False, input_shape=(32, 32, 3),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(32, (3, 3), use_bias=False, input_shape=(32, 32, 3),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.MaxPooling2D((2, 2)))

model.add(layers.Conv2D(64, (3, 3), use_bias=False, input_shape=(100, 20, 32),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(64, (3, 3), use_bias=False, input_shape=(100, 20, 32),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(64, (3, 3), use_bias=False, input_shape=(100, 20, 32),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.MaxPooling2D((2, 2)))

model.add(layers.Conv2D(128, (3, 3), use_bias=False, input_shape=(50, 10, 64),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(128, (3, 3), use_bias=False, input_shape=(50, 10, 64),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.Conv2D(128, (3, 3), use_bias=False, input_shape=(50, 10, 64),padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.Activation("relu"))
model.add(layers.MaxPooling2D((2, 2)))


model.add(layers.Flatten())


branch1 = models.Sequential()

branch1 .add(layers.Dense(128, input_shape=(16000,)))
branch1.add(layers.Dense(36, input_shape=(128,)))
branch1.add(layers.Activation("softmax"))

# Another 7 branches follows with exact same definition

final_model = keras.Model(inputs=[model, model, model, model, model, model, model, model],
                          outputs=[branch1, branch2, branch3, branch4, branch5, branch6, branch7, branch8])


final_model.compile(tf.keras.optimizers.Adam(learning_rate=0.001),
        loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
        metrics=['accuracy'])

history = final_model.fit(train_images, train_labels, epochs=80,
                          validation_data=(test_images, test_labels))

Answer 1

这是一个虚拟的例子

inp_dim = (32, 32, 3)
x_inp = Input(shape=inp_dim)
x = Conv2D(32, (3, 3), use_bias=False, padding='same')(x_inp)
x = BatchNormalization()(x)
x = Activation("relu")(x)
x = MaxPooling2D((2, 2))(x)
x_out = Flatten()(x)
model = Model(x_inp, x_out)

inp = Input(shape=inp_dim)

branches = []

for _ in range(8):

    init_branch = model(inp)
    x = Dense(128)(init_branch)
    x = Dense(36)(x)
    x = Activation("softmax")(x)

    branches.append(x)

final_model = Model(inputs = inp, outputs = branches)


final_model.compile(tf.keras.optimizers.Adam(learning_rate=0.001),
                loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                metrics=['accuracy'])

输入是所有 8 个最终分支的 SINGLE 模型（我使用了简化版本）。检查输入维度（(32, 32, 3) 不知道对不对）

---

这是我正确匹配的解决方案（这是我的意见，我不测试）

import string

# dummy target in the format I expected (7 digits)
train_labels = np.array([['7C24698'], ['7F84698']])
test_label = np.array([['8C24698'], ['8F84698']])

# create dict to encode digits to numbers
y_map = {}
for i,d in enumerate(string.digits+string.ascii_uppercase):
    y_map[d] = i

# utility function for split string in char and encode them
def split_encode(x):
    x = list(x[0])
    x = [y_map[d] for d in x]
    return x

# transform target
train_labels = np.apply_along_axis(split_encode, 1, train_labels)
test_label = np.apply_along_axis(split_encode, 1, test_label)

......

# fit model (this works with 7 output branches, 36 digits output prob, and sparse_catcrossent
final_model.fit(train_images, [train_labels[:,i] for i in range(7)], epochs=80,
            validation_data=(test_images, [test_label[:,i] for i in range(7)]))