简单的非面向对象的神经网络“跳跃”的代价

Question

我正在 Python 3.4 中使用 numpy 和矩阵构建神经网络的草图，以学习简单的 XOR。 我的符号如下：

a 是一个神经元的活动

z 是一个神经元的输入

W 是一个权重矩阵，大小为 R^{#前一层神经元数}x{#下一层神经元数}

B 是偏差值的向量

在 python 中实现了一个非常简单的网络后，在仅对单个输入向量进行训练时一切正常。然而，在对所有四个异或训练示例进行训练时，误差函数表现出非常奇怪的行为（见图），网络的输出总是大约为 0.5。 更改网络大小、学习率或训练时期似乎没有帮助。

仅对一个训练示例进行训练时的成本 J

使用所有训练示例进行训练时的成本 J

这是网络的代码：

import numpy as np
import time
import matplotlib.pyplot as plt


Js = []
start = time.time()
np.random.seed(2)


#Sigmoid        
def activation(x, derivative = False):
    if(derivative):
        a = activation(x)
        return a * (1 - a)
    else:
        return 1/(1+np.exp(-x))

def cost(output, target):
    return (1/2) * np.sum((target - output)**2)


INPUTS = np.array([
    [0, 1],
    [1, 0],
    [0, 0],
    [1, 1],
])
TARGET = np.array([
    [1],
    [1],
    [0],
    [0],
])

"Hyper-Parameters"
# Layer Structure
LAYER = [2, 3, 1]
LEARNING_RATE = 0.1
ITERATIONS = int(1e3)

# Init Weights
W1 = np.random.rand(LAYER[0], LAYER[1])
W2 = np.random.rand(LAYER[1], LAYER[2])

# Init Biases
B1 = np.random.rand(LAYER[1], 1)
B2 = np.random.rand(LAYER[2], 1)

for i in range(0, ITERATIONS):
    exampleIndex = i % len(INPUTS)
    #exampleIndex = 2
    "Forward Pass"
    # Layer One Activity (Input layer)
    A0 = np.transpose(INPUTS[exampleIndex:exampleIndex+1])

    # Layer Two Activity (Hidden Layer)
    Z1 = np.dot(np.transpose(W1), A0) + B1
    A1 = activation(Z1)

    # Layer Three Activity (Output Layer)
    Z2 = np.dot(np.transpose(W2), A1) + B2
    A2 = activation(Z2)

    # Output
    O = A2

    # Cost J

    # Target Vector T
    T = np.transpose(TARGET[exampleIndex:exampleIndex+1])
    J = cost(O, T)
    Js.append(J)

    print("J = {}".format(J))
    print("I = {}, O = {}".format(A0, O))

    "Backward Pass"

    # Calculate Delta of output layer
    D2 = (O - T) * activation(Z2, True)

    # Calculate Delta of hidden layer
    D1 = np.dot(W2, D2) * activation(Z1, True)

    # Calculate Derivatives w.r.t. W2
    DerW2 = np.dot(A1, np.transpose(D2))
    # Calculate Derivatives w.r.t. W1
    DerW1 = np.dot(A0, np.transpose(D1))

    # Calculate Derivatives w.r.t. B2
    DerB2 = D2
    # Calculate Derivatives w.r.t. B1
    DerB1 = D1

    "Update Weights and Biases"

    W1 -= LEARNING_RATE * DerW1
    B1 -= LEARNING_RATE * DerB1

    W2 -= LEARNING_RATE * DerW2
    B2 -= LEARNING_RATE * DerB2

# Show prediction

print("Time elapsed {}s".format(time.time() - start))    
plt.plot(Js)
plt.ylabel("Cost J")
plt.xlabel("Iterations")
plt.show()

我的实现中出现这种奇怪行为的原因可能是什么？

Answer 1

我认为您的成本函数正在跳跃，因为您在每个样本后执行权重更新。但是，您的网络仍在训练正确的行为：

479997
J = 4.7222501603409765e-05
I = [[1]
 [0]], O = [[ 0.99028172]]
T = [[1]]
479998
J = 7.3205311398742e-05
I = [[0]
 [0]], O = [[ 0.01210003]]
T = [[0]]
479999
J = 4.577485181547362e-05
I = [[1]
 [1]], O = [[ 0.00956816]]
T = [[0]]
480000
J = 4.726257702199439e-05
I = [[0]
 [1]], O = [[ 0.9902776]]
T = [[1]]

成本函数表现出一些有趣的行为：训练过程达到一个点，成本函数的跳跃将变得非常小。 您可以使用下面的代码重现这一点（我只做了轻微的更改；请注意，我训练了更多的时期）：

import numpy as np
import time
import matplotlib.pyplot as plt


Js = []
start = time.time()
np.random.seed(2)


#Sigmoid        
def activation(x, derivative = False):
    if(derivative):
        a = activation(x)
        return a * (1 - a)
    else:
        return 1/(1+np.exp(-x))

def cost(output, target):
    return (1/2) * np.sum((target - output)**2)


INPUTS = np.array([[0, 1],[1, 0],[0, 0],[1, 1]])
TARGET = np.array([[1],[1],[0],[0]])

"Hyper-Parameters"
# Layer Structure
LAYER = [2, 3, 1]
LEARNING_RATE = 0.1
ITERATIONS = int(5e5)

# Init Weights
W1 = np.random.rand(LAYER[0], LAYER[1])
W2 = np.random.rand(LAYER[1], LAYER[2])

# Init Biases
B1 = np.random.rand(LAYER[1], 1)
B2 = np.random.rand(LAYER[2], 1)

for i in range(0, ITERATIONS):
    exampleIndex = i % len(INPUTS)
    # exampleIndex = 2
    "Forward Pass"
    # Layer One Activity (Input layer)
    A0 = np.transpose(INPUTS[exampleIndex:exampleIndex+1])

    # Layer Two Activity (Hidden Layer)
    Z1 = np.dot(np.transpose(W1), A0) + B1
    A1 = activation(Z1)

    # Layer Three Activity (Output Layer)
    Z2 = np.dot(np.transpose(W2), A1) + B2
    A2 = activation(Z2)

    # Output
    O = A2

    # Cost J

    # Target Vector T
    T = np.transpose(TARGET[exampleIndex:exampleIndex+1])
    J = cost(O, T)
    Js.append(J)

    # print("J = {}".format(J))
    # print("I = {}, O = {}".format(A0, O))
    # print("T = {}".format(T))
    if ((i+3) % 20000 == 0):
        print(i)
        print("J = {}".format(J))
        print("I = {}, O = {}".format(A0, O))
        print("T = {}".format(T))
    if ((i+2) % 20000 == 0):
        print(i)
        print("J = {}".format(J))
        print("I = {}, O = {}".format(A0, O))
        print("T = {}".format(T))
    if ((i+1) % 20000 == 0):
        print(i)
        print("J = {}".format(J))
        print("I = {}, O = {}".format(A0, O))
        print("T = {}".format(T))
    if (i % 20000 == 0):
        print(i)
        print("J = {}".format(J))
        print("I = {}, O = {}".format(A0, O))
        print("T = {}".format(T))

    "Backward Pass"

    # Calculate Delta of output layer
    D2 = (O - T) * activation(Z2, True)

    # Calculate Delta of hidden layer
    D1 = np.dot(W2, D2) * activation(Z1, True)

    # Calculate Derivatives w.r.t. W2
    DerW2 = np.dot(A1, np.transpose(D2))
    # Calculate Derivatives w.r.t. W1
    DerW1 = np.dot(A0, np.transpose(D1))

    # Calculate Derivatives w.r.t. B2
    DerB2 = D2
    # Calculate Derivatives w.r.t. B1
    DerB1 = D1

    "Update Weights and Biases"

    W1 -= LEARNING_RATE * DerW1
    B1 -= LEARNING_RATE * DerB1

    W2 -= LEARNING_RATE * DerW2
    B2 -= LEARNING_RATE * DerB2

# Show prediction

print("Time elapsed {}s".format(time.time() - start))    
plt.plot(Js)
plt.ylabel("Cost J")
plt.xlabel("Iterations")
plt.savefig('cost.pdf')
plt.show()

为了减少成本函数的波动，通常在执行更新（一些平均更新）之前使用多个数据样本，但我发现这在仅包含四个不同训练事件的集合中是困难的。 因此，总结这个相当长的答案：您的成本函数会跳跃，因为它是针对每个示例计算的，而不是针对多个示例的平均值。但是，网络输出很好地遵循了 XOR 函数的分布，因此您无需更改它。