手推一元线性回归（附代码）

线性回归

用例说明

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression

#体重
weights=[10.05,12.54,14.65,16.64,18.98]
#年龄
ages=[1,2,3,4,5]
df=pd.DataFrame()
df['age']=ages
df['weight']=weights
# 散点图
df.plot(kind='scatter', x='age', y='weight', c=None, s=15) # s：点的大小  c：点的颜色 c =np.squeeze(colors)
plt.title(u'儿童年龄体重对照', fontsize=15,fontdict=dict(family='KaiTi')) #楷体
plt.show()


y = ax + b


f

(

x

)

=

2.25

x

+

7.73

f(x)=2.25x+7.73

f(x)=2.25x+7.73

y

1

^

=

f

(

x

1

)

=

2.25

×

1

+

7.73

=

9.98

hat{y_1}=f(x_1)=2.25times1+7.73=9.98

y1^=f(x1)=2.25×1+7.73=9.98

y

2

^

=

f

(

x

2

)

=

2.25

×

2

+

7.73

=

12.23

hat{y_2}=f(x_2)=2.25times2+7.73=12.23

y2^=f(x2)=2.25×2+7.73=12.23

y

3

^

=

f

(

x

3

)

=

2.25

×

3

+

7.73

=

14.48

hat{y_3}=f(x_3)=2.25times3+7.73=14.48

y3^=f(x3)=2.25×3+7.73=14.48

y

4

^

=

f

(

x

4

)

=

2.25

×

4

+

7.73

=

16.73

hat{y_4}=f(x_4)=2.25times4+7.73=16.73

y4^=f(x4)=2.25×4+7.73=16.73

y

5

^

=

f

(

x

5

)

=

2.25

×

5

+

7.73

=

18.98

hat{y_5}=f(x_5)=2.25times5+7.73=18.98

y5^=f(x5)=2.25×5+7.73=18.98

θ

^

hat{theta}

θ^, 读作theta hat。

##使用程序计算

y_head5_predict=([2.25*x+7.73 for x in df['age']]) #拟合结果


f

(

x

)

=

2.25

x

+

7.73

f(x)=2.25x+7.73

f(x)=2.25x+7.73 的预测结果

[9.98, 12.23, 14.48, 16.73, 18.98]


[ 9.98, 12.23, 14.48, 16.73, 18.98]
[10.05, 12.54, 14.65, 16.64, 18.98]


最小二乘法

损失函数

残差公式

e

=

f

(

x

i

)

y

i

e=f(x_i)-y_i

e=f(xi)yi

y

i

^

y

i

=

ϵ

hat{y_i}-y_i=epsilon

yi^yi=ϵ

损失函数原型

和方差（SSE）

S

S

E

=

i

=

1

m

(

y

i

y

i

^

)

2

SSE=sum_{i=1}^{m}(y_i-hat{y_i})^2

SSE=i=1m(yiyi^)2

均方误差（MSE）

M

S

E

=

S

S

E

N

=

1

N

i

=

1

m

(

y

i

y

i

^

)

2

MSE=frac{SSE}{N}=frac{1}{N}sum_{i=1}^{m}(y_i-hat{y_i})^2

MSE=NSSE=N1i=1m(yiyi^)2

均方根（RMSE）

R

M

S

E

=

M

S

E

=

1

N

i

=

1

m

(

y

i

y

i

^

)

2

Large RMSE=sqrt{MSE}=sqrt{frac{1}{N}sum_{i=1}^{m}(y_i-hat{y_i})^2}

RMSE=MSE

=N1i=1m(yiyi^)2

参数估计——最小二乘法

f

(

x

)

=

a

x

+

b

f(x)=ax+b

f(x)=ax+b

f

(

a

,

b

)

=

i

=

1

m

(

y

i

y

i

^

)

2

=

i

=

1

m

(

y

i

(

a

x

i

+

b

)

)

2

f(a,b)=sum_{i=1}^{m}(y_i-hat{y_i})^2=sum_{i=1}^{m}(y_i-(ax_i+b))^2

f(a,b)=i=1m(yiyi^)2=i=1m(yi(axi+b))2

f

(

a

,

b

)

f(a,b)

f(a,b)分别对a和b求偏导，然后令偏导等于0。

对参数a的求导过程

u

=

y

i

(

a

x

i

+

b

)

u=y_i-(ax_i+b)

u=yi(axi+b)，根据链式法则(注意：x、y和b看成常数)：

(

i

=

1

m

(

y

i

(

a

x

i

+

b

)

)

2

)

big(sum_{i=1}^{m}(y_i-(ax_i+b))^2big)prime

(i=1m(yi(axi+b))2)

=

i

=

1

m

2

(

y

i

(

a

x

i

+

b

)

)

(

y

i

(

a

x

i

+

b

)

)

=sum_{i=1}^{m}2(y_i-(ax_i+b))(y_i-(ax_i+b))prime

=i=1m2(yi(axi+b))(yi(axi+b))

=

i

=

1

m

2

(

y

i

(

a

x

i

+

b

)

)

(

y

i

(

a

x

i

)

b

)

=sum_{i=1}^{m}2(y_i-(ax_i+b))(y_iprime-(ax_i)prime-bprime)

=i=1m2(yi(axi+b))(yi(axi)b)

=

2

i

=

1

m

(

y

i

(

a

x

i

+

b

)

)

(

0

x

i

0

)

=2sum_{i=1}^{m}(y_i-(ax_i+b))(0-x_i-0)

=2i=1m(yi(axi+b))(0xi0)

=

2

i

=

1

m

(

a

x

i

+

b

y

i

)

x

i

=2sum_{i=1}^{m}(ax_i+b-y_i)x_i

=2i=1m(axi+byi)xi

对参数b的求导过程

u

=

y

i

(

a

x

i

+

b

)

u=y_i-(ax_i+b)

u=yi(axi+b)，根据链式法则(注意：x、y和a看成常数)：

(

i

=

1

m

(

y

i

(

a

x

i

+

b

)

)

2

)

big(sum_{i=1}^{m}(y_i-(ax_i+b))^2big)prime

(i=1m(yi(axi+b))2)

=

i

=

1

m

2

(

y

i

(

a

x

i

+

b

)

)

(

y

i

(

a

x

i

+

b

)

)

=sum_{i=1}^{m}2(y_i-(ax_i+b))(y_i-(ax_i+b))prime

=i=1m2(yi(axi+b))(yi(axi+b))

=

i

=

1

m

2

(

y

i

(

a

x

i

+

b

)

)

(

y

i

(

a

x

i

)

b

)

=sum_{i=1}^{m}2(y_i-(ax_i+b))(y_iprime-(ax_i)prime-bprime)

=i=1m2(yi(axi+b))(yi(axi)b)

=

2

i

=

1

m

(

y

i

(

a

x

i

+

b

)

)

(

0

0

1

)

=2sum_{i=1}^{m}(y_i-(ax_i+b))(0-0-1)

=2i=1m(yi(axi+b))(001)

=

2

i

=

1

m

(

a

x

i

+

b

y

i

)

=2sum_{i=1}^{m}(ax_i+b-y_i)

=2i=1m(axi+byi)

估算参数

{

a

ϵ

=

2

i

=

1

m

(

a

x

i

+

b

y

i

)

x

i

=

0

a

ϵ

=

2

i

=

1

m

(

a

x

i

+

b

y

i

)

=

0

begin{cases} frac{partial}{partial{a}}epsilon=2sum_{i=1}^{m}(ax_i+b-y_i)x_i=0\ frac{partial}{partial{a}}epsilon=2sum_{i=1}^{m}(ax_i+b-y_i)=0 end{cases}

{aϵ=2i=1m(axi+byi)xi=0aϵ=2i=1m(axi+byi)=0
(长时间没用过高数，只能用笨办法)

i

=

1

m

(

a

x

i

+

b

y

i

)

x

i

=

0

sum_{i=1}^{m}(ax_i+b-y_i)x_i=0

i=1m(axi+byi)xi=0

i

=

1

m

(

a

x

i

2

+

b

x

i

y

i

x

i

)

=

0

sum_{i=1}^{m}(ax_i^2+bx_i-y_ix_i)=0

i=1m(axi2+bxiyixi)=0

i

=

1

m

a

x

i

2

+

i

=

1

m

b

x

i

i

=

1

m

y

i

x

i

=

0

sum_{i=1}^{m}ax_i^2+sum_{i=1}^{m}bx_i-sum_{i=1}^{m}y_ix_i=0

i=1maxi2+i=1mbxii=1myixi=0

a

i

=

1

m

x

i

2

+

b

i

=

1

m

x

i

i

=

1

m

y

i

x

i

=

0

asum_{i=1}^{m}x_i^2+bsum_{i=1}^{m}x_i-sum_{i=1}^{m}y_ix_i=0

ai=1mxi2+bi=1mxii=1myixi=0

i

=

1

m

(

a

x

i

+

b

y

i

)

=

0

sum_{i=1}^{m}(ax_i+b-y_i)=0

i=1m(axi+byi)=0

i

=

1

m

a

x

i

+

i

=

1

m

b

i

=

1

m

y

i

=

0

sum_{i=1}^{m}ax_i+sum_{i=1}^{m}b-sum_{i=1}^{m}y_i=0

i=1maxi+i=1mbi=1myi=0

a

i

=

1

m

x

i

+

m

b

i

=

1

m

y

i

=

0

asum_{i=1}^{m}x_i+mb-sum_{i=1}^{m}y_i=0

ai=1mxi+mbi=1myi=0

b

=

i

=

1

m

y

i

m

a

i

=

1

m

x

i

m

=

y

ˉ

a

x

ˉ

Large b=frac{sum_{i=1}^{m}y_i}{m}-frac{asum_{i=1}^{m}x_i}{m}=bar{y}-abar{x}

b=mi=1myimai=1mxi=yˉaxˉ

a

i

=

1

m

x

i

2

+

(

y

ˉ

a

x

ˉ

)

i

=

1

m

x

i

i

=

1

m

y

i

x

i

=

0

asum_{i=1}^{m}x_i^2+(bar{y}-abar{x})sum_{i=1}^{m}x_i-sum_{i=1}^{m}y_ix_i=0

ai=1mxi2+(yˉaxˉ)i=1mxii=1myixi=0

a

i

=

1

m

x

i

2

+

y

ˉ

i

=

1

m

x

i

a

x

ˉ

i

=

1

m

x

i

i

=

1

m

y

i

x

i

=

0

asum_{i=1}^{m}x_i^2+bar{y}sum_{i=1}^{m}x_i-abar{x}sum_{i=1}^{m}x_i-sum_{i=1}^{m}y_ix_i=0

ai=1mxi2+yˉi=1mxiaxˉi=1mxii=1myixi=0

a

(

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

)

+

y

ˉ

i

=

1

m

x

i

i

=

1

m

y

i

x

i

=

0

a(sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i)+bar{y}sum_{i=1}^{m}x_i-sum_{i=1}^{m}y_ix_i=0

a(i=1mxi2xˉi=1mxi)+yˉi=1mxii=1myixi=0

a

=

i

=

1

m

y

i

x

i

y

ˉ

i

=

1

m

x

i

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

Large a=frac{sum_{i=1}^{m}y_ix_i-bar{y}sum_{i=1}^{m}x_i}{sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i}

a=i=1mxi2xˉi=1mxii=1myixiyˉi=1mxi

# 最小二乘法估出参数
x_bar = np.mean(ages)
y_bar = np.mean(weights)
a_param = np.dot(ages, weights) - y_bar * np.sum(ages)
a_param = a_param / (np.sum(np.square(ages)) - x_bar * np.sum(ages))
b_param = y_bar - a_param * x_bar


2.1960000000000037


7.9839999999999876

估算过程补充说明
1. 公式

a

=

i

=

1

m

y

i

x

i

y

ˉ

i

=

1

m

x

i

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

Large a=frac{sum_{i=1}^{m}y_ix_i-bar{y}sum_{i=1}^{m}x_i}{sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i}

这个公式若使用手算还可以再转化：

a

=

i

=

1

m

y

i

x

i

1

m

i

=

1

m

y

i

i

=

1

m

x

i

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

Large a=frac{sum_{i=1}^{m}y_ix_i-frac{1}{m}sum_{i=1}^{m}y_isum_{i=1}^{m}x_i}{sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i}

a

=

i

=

1

m

y

i

x

i

x

ˉ

i

=

1

m

y

i

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

Large a=frac{sum_{i=1}^{m}y_ix_i-bar{x}sum_{i=1}^{m}y_i}{sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i}

a

=

i

=

1

m

y

i

(

x

i

x

ˉ

)

i

=

1

m

x

i

2

x

ˉ

i

=

1

m

x

i

Large a=frac{sum_{i=1}^{m}y_i(x_i-bar{x})}{sum_{i=1}^{m}x_i^2-bar{x}sum_{i=1}^{m}x_i}

2. 代码说明
np.dot(ages, weights)


a

=

[

a

1

,

a

2

,

a

3

,

,

a

n

]

a=[a_1,a_2,a_3,ldots,a_n]

a=[a1,a2,a3,,an]

b

=

[

b

1

,

b

2

,

b

3

,

,

b

n

]

b=[b_1,b_2,b_3,ldots,b_n]

b=[b1,b2,b3,,bn]

a

b

=

a

1

b

1

+

a

2

b

2

+

a

3

b

3

+

+

a

n

b

n

Large vec{a}bulletvec{b}=a_1b_1+a_2b_2+a_3b_3+ldots+a_nb_n

a

b

=a1b1+a2b2+a3b3++anbn

i

=

1

m

y

i

x

i

sum_{i=1}^{m}y_ix_i

i=1myixi可以用向量点乘方式来计算。

用程序验证手推结果

# 验证上面手推最小二乘法的结果
# sklearn中，数据都应该是二维矩阵,这里需要转换
x_train = np.array(ages).reshape(-1, 1)
y_train = np.array(weights).reshape(-1, 1)
lr = LinearRegression()
lr.fit(x_train, y_train)
print("斜率:", lr.coef_)
print("截距:", lr.intercept_)


斜率: [[2.196]]



f

(

x

i

)

=

2.196

x

i

+

7.984

Large f(x_i)=2.196x_i+7.984

f(xi)=2.196xi+7.984

多元线性回归

f

(

x

)

=

a

x

+

b

f(x)=ax+b

f(x)=ax+b

f

(

x

i

)

=

ω

1

x

i

1

+

ω

2

x

i

2

+

+

ω

d

x

i

d

+

b

Large f(x_i)=omega_1x_{i1}+omega_2x_{i2}+ldots+omega_dx_{id}+b

f(xi)=ω1xi1+ω2xi2++ωdxid+b

ϵ

epsilon

ϵ，公式不一样但思想相同。

y

=

β

0

+

β

1

x

1

+

β

2

x

2

+

+

β

k

x

k

+

ϵ

Large y=beta_0+beta_1x_1+beta_2x_2+ldots+beta_kx_k+epsilon

y=β0+β1x1+β2x2++βkxk+ϵ

ω

omega

ω和b。而

ω

omega

ω的集合就是机器学习中高大上的参数矩阵

f

(

x

i

)

=

ω

T

x

i

+

b

Large f(x_i)=omega^Tx_i+b

f(xi)=ωTxi+b

ω

omega

ω都变为矩阵。而求解过程也比一元线性回归要复杂得多，这里就不做手工推算了，直接使用sklearn集成好的库，用代码体验一下。

#增加一个身高输入项
ages_highs = [
[1, 2, 3, 4, 5],    #年龄
[76.5,88.5,96.8,104.1,111.3]   #标准身高
]
x_train =np.array(ages_highs).T  #这里需要做矩阵转置
y_train = np.array(weights).reshape(-1, 1)
lr = LinearRegression()
lr.fit(x_train, y_train)
print("斜率:", lr.coef_)
print("截距:", lr.intercept_)
lr.predict([[6,117.7]])


斜率: [[1.67268574 0.06142186]]

array([[20.95730786]])


THE END