使用 fmin_bfgs 在 Logistic 回归中优化 Theta 时出现运行时错误

Question

#Get libraries
import  scipy.optimize as opt
import numpy as np
import pandas
import matplotlib.pyplot as plt


def plotData():
    plt.scatter(X[y==1,0],X[y==1,1], marker='+', c='black',label="Admitted")
    plt.scatter(X[y==0,0],X[y==0,1], marker='.', c='y', label="Not Admitted")
    plt.xlabel("Exam 1 Score")
    plt.ylabel("Exam 2 Score")
    plt.legend(['Admitted', 'Not Admitted'])

def sigmoid(z):
    return 1/(1 + np.exp(-1. * z))

def costFunction(theta, X, y):
    y = np.reshape(y, (len(y), 1))
    cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
    #grad = (1/m)*(X.T@(sigmoid(X@theta) - y))
    return(cost[0])

def costFunctionDer(theta, X, y):
    grad = (1./m)*(X.T@(sigmoid(X@theta) - y))
    return np.ndarray.flatten(grad)

def predict(theta, X):
    return (X@theta >= 0).astype('int')

#Load Data
data = pandas.read_table('ex2data1.txt', sep=',', names=['Exam 1 Score','Exam 2 Score','Admittance'] )
X = data.iloc[:,0:2].to_numpy()
y = data.iloc[:,2].to_numpy()

#Plot Data
plotData()

#Compute Cost and Gradient

#Add intercept
X = np.insert(X, 0, 1, 1)

m, n = X.shape

#Initialise thetas
theta_i = np.zeros((n,1))


cost = costFunction(theta_i, X, y)
grad = costFunctionDer(theta_i, X, y)

#Optimise Algorithm for theta
result = opt.fmin_bfgs(costFunction, theta_i, costFunctionDer,args=(X,y), full_output = True, maxiter=400, retall=True)

theta, cost_min = result[0:2]

#Make a decision Boundary vv smart, take the min and max range of x
# and then calculate the value of score 2 from it as theta[0] + x1theta[1] + x2theta[2] = 0
boundary_xs = np.array([np.min(X[:,1]), np.max(X[:,1])])
boundary_ys = (-1./theta[2])*(theta[0] + theta[1]*boundary_xs)
plt.plot(boundary_xs,boundary_ys,'b-',label='Decision Boundary')
plt.legend()

#Predict
p = predict(theta, X)

print("Train Accuracy {}".format(100*np.sum(p==y)/len(y)))

最近我一直在通过 Andrew Ng 的在线课程学习机器学习。我正在做练习 2，它要求我们实施逻辑回归。虽然该课程是在 Octave/Matlab 中教授的，但我正在尝试使用行业标准的 Python 来学习它。

为了优化 theta 的值，我尝试使用函数 fmin_bfgs，这给了我一个运行时错误。我使用了 min 函数并且代码工作得很好。有人可以帮我找到问题吗？我是 ML 新手，因此对于任何明显的缺陷，我深表歉意。

报错如下

RuntimeWarning: divide by zero encountered in log
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
RuntimeWarning: invalid value encountered in matmul
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
RuntimeWarning: divide by zero encountered in log
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
RuntimeWarning: invalid value encountered in matmul
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
RuntimeWarning: divide by zero encountered in log
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))
RuntimeWarning: invalid value encountered in matmul
  cost = (-1./m) * ( y.T@np.log(sigmoid(X@theta)) + (1 - y).T@np.log(1 - sigmoid(X@theta)))

Answer 1

第一个 RuntimeWarning 给了我需要的线索。对于非常低的 z 值，我的 sigmoid 函数返回 0。为了解决这个问题，我在 sigmoid 函数中手动设置了下限和上限。