【发布时间】：2021-07-13 07:23:33
【问题描述】：
我从头开始使用 NumPy 在 python 中实现了一个多层感知器。我已经多次完成网络实现，但无法找出网络中导致损失几乎保持不变的错误。坦白说，不知道是网络bug还是别的什么。
数据集是 CIFAR-10 数据集。
这是我的神经网络代码：
from typing import Sequence
import numpy as np
class NeuralNetwork:
def __init__(
self,
input_size: int,
hidden_sizes: Sequence[int],
output_size: int,
num_layers: int,
reg: int,
optim: str
):
self.input_size = input_size
self.hidden_sizes = hidden_sizes
self.output_size = output_size
self.num_layers = num_layers
self.reg_const = reg
if (optim != 'sgd' and optim != 'adam'):
print("Invalid choice of optimizer. Please choose from 'sgd' or 'adam'")
return
self.optim = optim
# adam related vars
if self.optim == 'adam':
self.eta = 0.01
self.beta1 = 0.9
self.beta2 = 0.999
self.epsilon = 1e-8
self.fm = {}
self.sm = {}
assert len(hidden_sizes) == (num_layers - 1)
sizes = [input_size] + hidden_sizes + [output_size]
self.params = {}
for i in range(1, num_layers + 1):
self.params["W" + str(i)] = np.random.randn(
sizes[i - 1], sizes[i]
) * np.sqrt(2 / sizes[i - 1])
self.params["b" + str(i)] = np.zeros((sizes[i], 1))
# moment initialization for the layers
if self.optim == 'adam':
for i in range(1, num_layers + 1):
self.fm['f' + str(i)] = None
self.sm['s' + str(i)] = None
def linear(self, W: np.ndarray, X: np.ndarray, b: np.ndarray) -> np.ndarray:
return np.dot(W.T, X) + b
def relu(self, X: np.ndarray) -> np.ndarray:
return np.maximum(X, np.zeros_like(X))
def softmax(self, X: np.ndarray) -> np.ndarray:
return np.exp(X - np.max(X)) / sum(np.exp(X))
def forward(self, X: np.ndarray) -> np.ndarray:
self.outputs = {}
# For input layer to first hidden layer
# OL - output of linear layer
# OR - output of Relu layer
# OS - output of softmax layer
self.outputs["OL" + str(1)] = self.linear(self.params["W" + str(1)],
X, self.params["b" + str(1)])
self.outputs["OR" + str(1)] = self.relu(self.outputs["OL" + str(1)])
# For all subsequent layers
for i in range(2, self.num_layers + 1):
self.outputs["OL" + str(i)] = self.linear(self.params["W" + str(i)],
self.outputs["OR" + str(i - 1)],
self.params["b" + str(i)])
self.outputs["OR" + str(i)] = self.relu(self.outputs["OL" + str(i)])
self.outputs["OS" + str(self.num_layers + 1)] = self.softmax(
self.outputs["OR" + str(self.num_layers)])
return self.outputs["OS" + str(self.num_layers + 1)]
def softmax_grad(self, s: np.ndarray,
X_in: np.ndarray, y: np.ndarray) -> np.ndarray:
ds = s.copy()
ds[y.astype(np.int32), len(y) - 1] -= 1
# grad = np.dot(X_in, ds.T) / X_in.shape[0]
return ds
def relu_grad(self, grad: np.ndarray, inp: np.ndarray) -> np.ndarray:
return inp.T > 0
def linear_grad(self, h: np.ndarray, grad: np.ndarray,
W: np.ndarray, b: np.ndarray, layer: int, lr: float,
reg: float,
t: int) -> np.ndarray:
local_grad = (np.dot(h, grad)) / grad.shape[1]
upstream_grad = np.dot(W, grad.T) / grad.shape[1]
temp = np.average(grad, axis = 0).reshape((-1, 1))
if self.optim == 'sgd': # sgd implementation
# parameter updates
t1 = lr * local_grad
t2 = self.reg_const * np.sum(self.params["W" + str(layer)])
self.params["W" + str(layer)] -= t1 + t2
self.params["b" + str(layer)] -= lr * temp
else: # adam implementation
# moment initialization if first iter
if self.fm['f' + str(layer)] is None:
self.fm['f' + str(layer)] = np.zeros_like(local_grad)
self.sm['s' + str(layer)] = np.zeros_like(local_grad)
fms = self.fm['f' + str(layer)]
sms = self.sm['s' + str(layer)]
fms = self.beta1 * fms + (1 - self.beta1) * local_grad
sms = self.beta2 * sms + (1 - self.beta2) * local_grad**2
self.fm['f' + str(layer)] = fms
self.sm['s' + str(layer)] = sms
# bias correction
fm_unbias = fms / (1 - self.beta1**(t+1))
sm_unbias = sms / (1 - self.beta2**(t+1))
self.params["W" + str(layer)] -= lr * fm_unbias / (np.sqrt(sm_unbias) + 1e-7)
return upstream_grad
def backward(self, X: np.ndarray, y: np.ndarray,
lr: float, reg: float, t: int) -> float:
self.gradients = {}
loss = 0.0
# one-hot encoding for calculating loss
one_hot = np.zeros((y.size, 10))
rows = np.arange(y.size)
one_hot[rows, y] = 1
y = one_hot
# Loss computation
softmax_layer = self.outputs["OS" + str(self.num_layers + 1)]
p = softmax_layer[y.astype(np.int32), len(y) - 1]
loss = -np.log(p)
loss = np.sum(loss) / len(y) + (2 * self.reg_const * np.sum(self.params["W" + str(self.num_layers)]**2))
self.gradients["G" + str(self.num_layers + 1)] = self.softmax_grad(softmax_layer, self.outputs["OR" + str(self.num_layers)], y)
for i in range(self.num_layers, 1, -1):
self.gradients["G" + str(i)] = self.relu_grad(self.gradients["G" + str(i + 1)],
self.outputs["OL" + str(i)])
self.gradients["G" + str(i)] = self.linear_grad(self.outputs["OR" + str(i - 1)],
self.gradients["G" + str(i)],
self.params["W" + str(i)],
self.params["b" + str(i)],
i, lr, reg, t)
self.gradients["G" + str(1)] = self.relu_grad(self.gradients["G" + str(2)],
self.outputs["OL" + str(1)])
self.gradients["G" + str(1)] = self.linear_grad(X,
self.gradients["G" + str(1)],
self.params["W" + str(1)],
self.params["b" + str(1)],
1, lr, reg, t)
return loss
网络初始化如下：
# Hyperparameters
input_size = 32 * 32 * 3
num_layers = 6
hidden_size = 64
hidden_sizes = [hidden_size] * (num_layers - 1)
num_classes = 10
epochs = 100
batch_size = 128
learning_rate = 1e-4
reg = 1e-6
# Change the optimizer here
optimizer = 'adam' # 'sgd' or 'adam'
# # Initialize a new neural network model
net = NeuralNetwork(input_size, hidden_sizes, num_classes, num_layers, reg, optimizer)
# Variables to store performance for each epoch
train_loss = np.zeros(epochs)
train_accuracy = np.zeros(epochs)
val_accuracy = np.zeros(epochs)
这是培训模块：
# For each epoch...
for epoch in range(epochs):
print('epoch:', epoch)
# Shuffle the dataset
shuffled_indices = [i for i in range(X_train.shape[0])]
shuffle(shuffled_indices)
X_train_new = X_train[shuffled_indices, :] # shuffling samples
y_train_new = y_train[shuffled_indices,] # shuffling labels
# Training
# For each mini-batch...
for batch in range(TRAIN_IMAGES // batch_size):
# Create a mini-batch of training data and labels
X_batch = X_train_new[batch * batch_size : (batch + 1) * batch_size, :]
y_batch = y_train_new[batch * batch_size : (batch + 1) * batch_size,].astype(int)
X_batch = X_batch.astype(float) / 255.
X_batch = X_batch.T
# Run the forward pass of the model to get a prediction and compute the accuracy
res = net.forward(X_batch)
res = np.argmax(res, axis = 0)
acc = 0
for i in range(len(res)):
if res[i] == y_batch[i]:
acc += 1
# Run the backward pass of the model to update the weights and compute the loss
train_loss[epoch] += net.backward(X_batch, y_batch, learning_rate, reg, epoch)
train_accuracy[epoch] += acc
print("Epoch {}: Training accuracy: {:.3f}".format(epoch, train_accuracy[epoch] / (TRAIN_IMAGES // batch_size)))
print("Epoch {}: Training loss: {:.3f}".format(epoch, train_loss[epoch] / (TRAIN_IMAGES // batch_size)))
# Validation
# No need to run the backward pass here, just run the forward pass to compute accuracy
val_accuracy[epoch] += 0
val_acc = 0
for batch in range(VAL_IMAGES // batch_size):
X_batch = X_val[batch * batch_size : (batch + 1) * batch_size, :]
y_batch = y_val[batch * batch_size : (batch + 1) * batch_size,]
X_batch = X_batch.T
res = net.forward(X_batch)
res = np.argmax(res, axis = 0)
for i in range(len(res)):
if res[i] == y_batch[i]:
val_acc += 1
val_accuracy[epoch] += val_acc / (VAL_IMAGES // batch_size)
print("Epoch {} Validation accuracy: {:.3f}".format(epoch, val_accuracy[epoch]))
这是训练输出的样子：
epoch: 0
Epoch 0: Training accuracy: 12.770
Epoch 0: Training loss: 23.072
Epoch 0 Validation accuracy: 14.714
epoch: 1
Epoch 1: Training accuracy: 12.848
Epoch 1: Training loss: 23.040
Epoch 1 Validation accuracy: 14.714
epoch: 2
Epoch 2: Training accuracy: 12.969
Epoch 2: Training loss: 23.035
Epoch 2 Validation accuracy: 14.571
epoch: 3
Epoch 3: Training accuracy: 12.838
Epoch 3: Training loss: 23.033
Epoch 3 Validation accuracy: 14.857
epoch: 4
Epoch 4: Training accuracy: 12.796
Epoch 4: Training loss: 23.031
Epoch 4 Validation accuracy: 14.714
epoch: 5
Epoch 5: Training accuracy: 12.510
Epoch 5: Training loss: 23.030
Epoch 5 Validation accuracy: 15.000
这只是 5 次迭代，但数字变化不大。我尝试更改超参数，但没有太大变化。此外，我尝试增加层数和每层神经元的数量，但仍然没有太大变化。这让我相信我的网络中存在一些错误。
请告诉我我哪里出错了，请解释错误，这将是一个巨大的帮助。谢谢。
这是数据集配置：
TRAIN_IMAGES = 49000
VAL_IMAGES = 1000
TEST_IMAGES = 10000
data = get_CIFAR10_data(TRAIN_IMAGES, VAL_IMAGES, TEST_IMAGES) # you can load this from keras
X_train, y_train = data['X_train'], data['y_train']
X_val, y_val = data['X_val'], data['y_val']
X_test, y_test = data['X_test'], data['y_test']
【问题讨论】：
-
看来问题出在你的反向传播计算上。但是你能用相应的激活函数快速绘制你的网络吗？
标签： python neural-network loss-function