逻辑回归的讲解和代码_逻辑回归代码解读

逻辑回归模型是由以下条件概率分布表示的分类模型。

逻辑回归模型源自逻辑分布，其分布函数使S形函数；

逻辑回归：用于分类问题中，预测值为离散值；算法的性质是输出值永远在0和1之间；

逻辑回归的模型假设：，

h(x)的作用：对于给定的输入变量，根据选择的参数计算输出变量=1的可能性，

代价函数：

梯度下降算法：

高级优化算法：共轭梯度法、BFGS变尺度法、L-BFGS限制变尺度法、fminunc无约束最小化函数

正则化：保留所有的特征，减小参数的大小；

其中lamda是正则化参数，lamda越大，参数越小。因为需要最小化代价函数，但是加上了尾部的这一部分，尾部越大，则整个代价函数越大，则theta越小，才能保证最小的代价函数。

代码部分，最重要的是实现代价函数和sigmoid函数。

import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize as opt
from plotData import *
import costFunctionReg as cfr
import plotDecisionBoundary as pdb
import predict as predict
import mapFeature as mf
 
plt.ion()
# Load data
# The first two columns contain the exam scores and the third column contains the label.
data = np.loadtxt('ex2data2.txt', delimiter=',')
X = data[:, 0:2]
y = data[:, 2]
 
plot_data(X, y)
 
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.legend(['y = 1', 'y = 0'])
 
input('Program paused. Press ENTER to continue')
 
# ===================== Part 1: Regularized Logistic Regression =====================
X = mf.map_feature(X[:, 0], X[:, 1])
 
# Initialize fitting parameters
initial_theta = np.zeros(X.shape[1])
 
# Set regularization parameter lambda to 1
lmd = 1
 
# Compute and display initial cost and gradient for regularized logistic regression
cost, grad = cfr.cost_function_reg(initial_theta, X, y, lmd)
 
np.set_printoptions(formatter={'float': '{: 0.4f}\n'.format})
print('Cost at initial theta (zeros): {}'.format(cost))
print('Expected cost (approx): 0.693')
print('Gradient at initial theta (zeros) - first five values only: \n{}'.format(grad[0:5]))
print('Expected gradients (approx) - first five values only: \n 0.0085\n 0.0188\n 0.0001\n 0.0503\n 0.0115')
 
input('Program paused. Press ENTER to continue')
 
# Compute and display cost and gradient with non-zero theta
test_theta = np.ones(X.shape[1])
 
cost, grad = cfr.cost_function_reg(test_theta, X, y, lmd)
 
print('Cost at test theta: {}'.format(cost))
print('Expected cost (approx): 2.13')
print('Gradient at test theta - first five values only: \n{}'.format(grad[0:5]))
print('Expected gradients (approx) - first five values only: \n 0.3460\n 0.0851\n 0.1185\n 0.1506\n 0.0159')
 
input('Program paused. Press ENTER to continue')
 
# ===================== Part 2: Regularization and Accuracies =====================
# Optional Exercise:
# In this part, you will get to try different values of lambda and
# see how regularization affects the decision boundary
#
# Try the following values of lambda (0, 1, 10, 100).
#
# How does the decision boundary change when you vary lambda? How does
# the training set accuracy vary?
#
 
# Initializa fitting parameters
initial_theta = np.zeros(X.shape[1])
 
# Set regularization parameter lambda to 1 (you should vary this)
lmd = 1
 
# Optimize
def cost_func(t):
    return cfr.cost_function_reg(t, X, y, lmd)[0]
 
def grad_func(t):
    return cfr.cost_function_reg(t, X, y, lmd)[1]
 
theta, cost, *unused = opt.fmin_bfgs(f=cost_func, fprime=grad_func, x0=initial_theta, maxiter=400, full_output=True, disp=False) #使用的是优化库函数里面的牛顿法
 
# Plot boundary
print('Plotting decision boundary ...')
pdb.plot_decision_boundary(theta, X, y)
plt.title('lambda = {}'.format(lmd))
 
plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
 
# Compute accuracy on our training set
p = predict.predict(theta, X)
 
print('Train Accuracy: {:0.4f}'.format(np.mean(y == p) * 100))
print('Expected accuracy (with lambda = 1): 83.1 (approx)')
 
input('ex2_reg Finished. Press ENTER to exit')
 
 
 
import numpy as np
from sigmoid import *
 
def cost_function_reg(theta, X, y, lmd):
    m = y.size
 
    hypothesis = sigmoid(np.dot(X, theta))
 
    reg_theta = theta[1:]
 
    cost = np.sum(-y * np.log(hypothesis) - (1 - y) * np.log(1 - hypothesis)) / m \
           + (lmd / (2 * m)) * np.sum(reg_theta * reg_theta)
 
    normal_grad = (np.dot(X.T, hypothesis - y) / m).flatten()
 
    grad[0] = normal_grad[0]
    grad[1:] = normal_grad[1:] + reg_theta * (lmd / m)
 
    # ===========================================================
 
    return cost, grad
 
 
import matplotlib.pyplot as plt
import numpy as np
from plotData import *
from mapFeature import *
 
def plot_decision_boundary(theta, X, y):
    plot_data(X[:, 1:3], y)
 
    if X.shape[1] <= 3:
        # Only need two points to define a line, so choose two endpoints
        plot_x = np.array([np.min(X[:, 1]) - 2, np.max(X[:, 1]) + 2])
 
        # Calculate the decision boundary line
        plot_y = (-1/theta[2]) * (theta[1]*plot_x + theta[0])
 
        plt.plot(plot_x, plot_y)
 
        plt.legend(['Decision Boundary', 'Admitted', 'Not admitted'], loc=1)
        plt.axis([30, 100, 30, 100])
    else:
        # Here is the grid range
        u = np.linspace(-1, 1.5, 50)
        v = np.linspace(-1, 1.5, 50)
 
        z = np.zeros((u.size, v.size))
 
        # Evaluate z = theta*x over the grid
        for i in range(0, u.size):
            for j in range(0, v.size):
                z[i, j] = np.dot(map_feature(u[i], v[j]), theta)
 
        z = z.T
 
        # Plot z = 0
        # Notice you need to specify the range [0, 0]
        cs = plt.contour(u, v, z, levels=[0], colors='r', label='Decision Boundary')
        plt.legend([cs.collections[0]], ['Decision Boundary'])
   
        
import numpy as np
 
def map_feature(x1, x2):
    degree = 6
 
    x1 = x1.reshape((x1.size, 1))
    x2 = x2.reshape((x2.size, 1))
    result = np.ones(x1[:, 0].shape)
 
    for i in range(1, degree + 1):
        for j in range(0, i + 1):
            result = np.c_[result, (x1**(i-j)) * (x2**j)]
 
    return result
 
 
import matplotlib.pyplot as plt
import numpy as np
 
def plot_data(X, y):
    plt.figure()
 
    pos = np.where(y == 1)[0] #输出满足条件的坐标
    neg = np.where(y == 0)[0]
 
    plt.scatter(X[pos, 0], X[pos, 1], marker="+", c='b')
    plt.scatter(X[neg, 0], X[neg, 1], marker="o", c='y')
 
 
import numpy as np
from sigmoid import *
 
def predict(theta, X):
    m = X.shape[0]
    p = np.zeros(m)
    p = sigmoid(np.dot(X, theta))
    pos = np.where(p >= 0.5)
    neg = np.where(p < 0.5)
 
    p[pos] = 1
    p[neg] = 0
 
    # ===========================================================
 
    return p
 
 
import numpy as np
 
def sigmoid(z):
    g = np.zeros(z.size)
    g = 1 / (1 + np.exp(-z))
 
    return g