如何查看哪些测试数据分类错误

Question

输出的准确率不是 100%，因此存在被网络错误分类的文本。网络后怎么看这些文字？

import numpy as np
import keras
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense, Activation,Dropout
from keras.preprocessing.text import Tokenizer
import matplotlib.pyplot as plt
from keras import optimizers
from keras.layers import Conv1D, GlobalMaxPooling1D

np.random.seed(42)

max_features = 10000
maxlen = 400
batch_size = 64
embedding_dims = 200
filters = 150
kernel_size = 5
hidden_dims = 50
epochs =5

(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=1000)

print(x_train.shape)
print(x_test.shape)
print(x_train[0])
print(y_train[0])
tokenizer = Tokenizer(num_words=1000)
x_train = tokenizer.sequences_to_matrix(x_train, mode='binary')
x_test = tokenizer.sequences_to_matrix(x_test, mode='binary')
print(x_train[0])

num_classes = 2
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
print(y_train.shape)
print(y_test.shape)

model = Sequential()

model.add(Dense(512,input_dim = 1000,activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_classes,activation='sigmoid'))
model.summary()



opt = optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=['accuracy'])

clf = model.fit(x_train, y_train, batch_size=128, epochs=5, validation_data=(x_test, y_test))

score = model.evaluate(x_test, y_test, verbose=1)
print("Accuracy: ", score[1])

我尝试了这段代码，但出现错误

y_pred = model.predict(x_test)

# bolean mask
mask = y_pred != y_test

# print rows that was classified incorrectly    
print(x_test[mask])

print(x_test[mask]) IndexError: boolean index does not match indexed 沿维度 1 的数组；维度为 1000 但对应的布尔值 维度是 2

Answer 1

我更改了您的完整代码，使其只运行一个类（因为我们正在研究二元问题），您可以研究错误分类的样本。结果证明您使用的模型完全不适合您的任务。

import numpy as np
import keras
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense, Activation,Dropout
from keras.preprocessing.text import Tokenizer
import matplotlib.pyplot as plt
from keras import optimizers
from keras.layers import Conv1D, GlobalMaxPooling1D
import pandas as pd

np.random.seed(42)

max_features = 10000
maxlen = 400
batch_size = 64
embedding_dims = 200
filters = 150
kernel_size = 5
hidden_dims = 50
epochs =5

(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=1000)

print(x_train.shape)
print(x_test.shape)
print(x_train[0])
print(y_train[0])
tokenizer = Tokenizer(num_words=1000)
x_train = tokenizer.sequences_to_matrix(x_train, mode='binary')
x_test = tokenizer.sequences_to_matrix(x_test, mode='binary')
print(x_train[0])

num_classes = 1
#y_train = keras.utils.to_categorical(y_train, num_classes)
#y_test = keras.utils.to_categorical(y_test, num_classes)
print(y_train.shape)
print(y_test.shape)

model = Sequential()

model.add(Dense(512,input_dim = 1000,activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(128,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64,activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_classes,activation='sigmoid'))
model.summary()



opt = optimizers.Adam()
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=['accuracy'])

clf = model.fit(x_train, y_train, batch_size=128, epochs=5, validation_data=(x_test, y_test))

score = model.evaluate(x_test, y_test, verbose=1)
print("Accuracy: ", score[1])

y_pred = model.predict(x_test)
df_test_pred = pd.concat([pd.DataFrame(x_test), pd.DataFrame(y_test, columns= ['test']), pd.DataFrame(y_pred, columns= ['pred'])], axis=1)

df_wrong= df_test_pred[df_test_pred['test'] != df_test_pred['pred']]