如何在此代码中实现没有 Visualize_cam 的 CAM？

Question

我想做类激活图，所以我写了代码

from keras.datasets import mnist
from keras.layers import Conv2D, Dense, GlobalAveragePooling2D
from keras.models import Model, Input
from keras.utils import to_categorical

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train_resized = x_train.reshape((60000, 28, 28, 1))
x_test_resized = x_test.reshape((10000, 28, 28, 1))
y_train_hot_encoded = to_categorical(y_train)
y_test_hot_encoded = to_categorical(y_test)

inputs = Input(shape=(28,28, 1))

x = Conv2D(64, (3,3), activation='relu')(inputs)
x = Conv2D(64, (3,3), activation='relu')(x)
x = GlobalAveragePooling2D()(x)
predictions = Dense(10, activation='softmax')(x)

model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.fit(x_train_resized, y_train_hot_encoded, epochs=30, batch_size=256, shuffle=True, validation_split=0.3)

工作正常，所以我已经导入了 Visualize_cam 模块

from vis.visualization import visualize_cam
import matplotlib.pyplot as plt
import numpy as np

for i in range(10):
    ind = np.where(y_test == i)[0][0]
    plt.subplot(141)
    plt.imshow(x_test_resized[ind].reshape((28,28)))
    for j,modifier in enumerate([None, 'guided', 'relu']):
        heat_map = visualize_cam(model, 4, y_test[ind], x_test_resized[ind], backprop_modifier=modifier)
        plt.subplot(1,4,j+2)
        plt.imshow(heat_map)

    plt.show()

但visualize_cam 运行不佳

我尝试了很多次来修复模块，但它并不顺利 （这取决于 scipy 哪个版本低于 1.3。但是）

所以我必须在没有那个模块的情况下实现 cam

是否有任何解决方案可以将 Visualize_cam 替换为其他选项来实现 CAM？

Answer 1

这是一个独立于 scipy 库的实现。

from keras.datasets import mnist
from keras.layers import Conv2D, Dense, GlobalAveragePooling2D
from keras.models import Model, Input
from keras.utils import to_categorical

(x_train, y_train), (x_test, y_test) = mnist.load_data()

x_train_resized = x_train.reshape((60000, 28, 28, 1))
x_test_resized = x_test.reshape((10000, 28, 28, 1))
y_train_hot_encoded = to_categorical(y_train)
y_test_hot_encoded = to_categorical(y_test)

inputs = Input(shape=(28,28, 1))

x = Conv2D(64, (3,3), activation='relu')(inputs)
x = Conv2D(64, (3,3), activation='relu', name='final_conv')(x)
x = GlobalAveragePooling2D()(x)
predictions = Dense(10, activation='softmax')(x)

model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
model.fit(x_train_resized, y_train_hot_encoded, epochs=1, batch_size=256, shuffle=True, validation_split=0.3)

import numpy as np
import cv2
import io
import requests
from PIL import Image
import matplotlib.pyplot as plt
# Using Keras implementation from tensorflow
from tensorflow.python.keras import applications
from tensorflow.python.keras.preprocessing.image import load_img, img_to_array 
from tensorflow.python.keras import backend as K


# Get the layer of the last conv layer
fianlconv = model.get_layer('final_conv')
# Get the weights matrix of the last layer
weight_softmax = model.layers[-1].get_weights()[0]

# Function to generate Class Activation Mapping
HEIGHT = 28
WIDTH = 28

def returnCAM(feature_conv, weight_softmax, class_idx):
    size_upsample = (WIDTH, HEIGHT)

    # Keras default is channels last, hence nc is in last
    bz, h, w, nc = feature_conv.shape

    output_cam = []
    for idx in class_idx:
        cam = np.dot(weight_softmax[:, idx], np.transpose(feature_conv.reshape(h*w, nc)))
        cam = cam.reshape(h, w)
        cam = cam - np.min(cam)
        cam_img = cam / np.max(cam)
        cam_img = np.uint8(255 * cam_img)

        output_cam.append(cv2.resize(cam_img, size_upsample))

    return output_cam

x = x_test_resized[0,:,:,0]

plt.imshow(x)
plt.show()

classes = {1:'1', 2: '2', 3: '3', 4:'4', 5:'5', 6:'6', 7:'7', 8:'8', 9:'9', 0:'0'}

probs_extractor = K.function([model.input], [model.output])
features_conv_extractor = K.function([model.input], [fianlconv.output])

probs = probs_extractor([np.expand_dims(x, 0).reshape(1,28,28,1)])[0]

features_blob = features_conv_extractor([np.expand_dims(x, 0).reshape(1,28,28,1)])[0]

features_blobs = []
features_blobs.append(features_blob)

idx = np.argsort(probs)
probs = np.sort(probs)

for i in range(-1, -6, -1):
    print('{:.3f} -> {}'.format(probs[0, i], classes[idx[0, i]]))


CAMs = returnCAM(features_blobs[0], weight_softmax, [idx[0, -1]])

heatmap = cv2.applyColorMap(cv2.resize(CAMs[0], (28, 28)), cv2.COLORMAP_JET)

result = heatmap[:,:,0] * 0.3 + x * 0.5

print(result.shape)

plt.imshow(result)
plt.show()

注意：我正在绘制标准化图像，所以结果不是很好，我也只训练了 1 个 epoch。为了获得更好的结果，您可以尝试更多地训练，更改为适当的颜色空间。