# 自适应学习率BP神经网络【代码】

BP_network.py

``````# -*- coding: utf-8 -*

'''
the definition of BP network class
'''

class BP_network:

def __init__(self):

'''
initial variables
'''
# node number each layer
self.i_n = 0
self.h_n = 0
self.o_n = 0

# output value for each layer
self.i_v = []
self.h_v = []
self.o_v = []

# parameters (w, t)
self.ih_w = []  # weight for each link
self.ho_w = []
self.h_t = []  # threshold for each neuron
self.o_t = []

# definition of alternative activation functions and it's derivation
self.fun = {
'Sigmoid': Sigmoid,
'SigmoidDerivate': SigmoidDerivate,
'Tanh': Tanh,
'TanhDerivate': TanhDerivate,

}

# initial the learning rate
self.lr1 = []  # output layer
self.lr2 = []  # hidden layer

def CreateNN(self, ni, nh, no, actfun, learningrate):
'''
建立BP网络并初始化参数
@param ni, nh, no: the neuron number of each layer
@param actfun: string, the name of activation function
@param learningrate: learning rate of gradient algorithm
'''

# dependent packages
import numpy as np

# 初始化每层的结点数
self.i_n = ni  # 输入层
self.h_n = nh  # 隐层
self.o_n = no  # 输出层

# 初始化每层的输出值
self.i_v = np.zeros(self.i_n)
self.h_v = np.zeros(self.h_n)
self.o_v = np.zeros(self.o_n)

# 初始化每层的权重矩阵（随机初始化）
self.ih_w = np.zeros([self.i_n, self.h_n])
self.ho_w = np.zeros([self.h_n, self.o_n])
for i in range(self.i_n):
for h in range(self.h_n):
self.ih_w[i][h] = rand(0, 1)
for h in range(self.h_n):
for j in range(self.o_n):
self.ho_w[h][j] = rand(0, 1)

# 初始化每层的阈值
self.h_t = np.zeros(self.h_n)
self.o_t = np.zeros(self.o_n)
for h in range(self.h_n): self.h_t[h] = rand(0, 1)
for j in range(self.o_n): self.o_t[j] = rand(0, 1)

# 初始化激活函数
self.af = self.fun[actfun]
self.afd = self.fun[actfun + 'Derivate']

# 初始化学习率
self.lr1 = np.ones(self.o_n) * learningrate
self.lr2 = np.ones(self.h_n) * learningrate

def Pred(self, x):
'''
前向传播的过程
predict process through the network
@param x: the input array for input layer
'''

# activate input layer
for i in range(self.i_n):
self.i_v[i] = x[i]

# activate hidden layer
for h in range(self.h_n):
total = 0.0
for i in range(self.i_n):
total += self.i_v[i] * self.ih_w[i][h]
self.h_v[h] = self.af(total - self.h_t[h])

# activate output layer
for j in range(self.o_n):
total = 0.0
for h in range(self.h_n):
total += self.h_v[h] * self.ho_w[h][j]
self.o_v[j] = self.af(total - self.o_t[j])

'''
for fixed learning rate
'''

def BackPropagate(self, x, y):
'''
反向传播
the implementation of BP algorithms on one slide of sample

@param x, y: array, input and output of the data sample
'''

# dependent packages
import numpy as np

# get current network output
self.Pred(x)

# calculate the gradient based on output
o_grid = np.zeros(self.o_n)
for j in range(self.o_n):
o_grid[j] = (y[j] - self.o_v[j]) * self.afd(self.o_v[j])

h_grid = np.zeros(self.h_n)
for h in range(self.h_n):
for j in range(self.o_n):
h_grid[h] += self.ho_w[h][j] * o_grid[j]
h_grid[h] = h_grid[h] * self.afd(self.h_v[h])

# 更新参数
for h in range(self.h_n):
for j in range(self.o_n):
self.ho_w[h][j] += self.lr1[j] * o_grid[j] * self.h_v[h]

for i in range(self.i_n):
for h in range(self.h_n):
self.ih_w[i][h] += self.lr2[h] * h_grid[h] * self.i_v[i]

for j in range(self.o_n):
self.o_t[j] -= self.lr1[j] * o_grid[j]

for h in range(self.h_n):
self.h_t[h] -= self.lr2[h] * h_grid[h]

def TrainStandard(self, data_in, data_out):
'''
standard BP training
@param lr, learning rate, default 0.05
@return: e, accumulated error
@return: e_k, error array of each step
'''
e_k = []
for k in range(len(data_in)):
x = data_in[k]
y = data_out[k]
self.BackPropagate(x, y)

# error in train set for each step
y_delta2 = 0.0
for j in range(self.o_n):
y_delta2 += (self.o_v[j] - y[j]) * (self.o_v[j] - y[j])
e_k.append(y_delta2 / 2)

# total error of training
e = sum(e_k) / len(e_k)

return e, e_k

'''
for dynamic learning rate
'''

def BackPropagate_Dynamic_Lr(self, x, y, d_ho_w_p, d_ih_w_p, d_o_t_p, d_h_t_p, o_grid_p, h_grid_p, alpha):
'''
the implementation of BP algorithms on one slide of sample

@param x, y: array, input and output of the data sample
@param d_ho_w_p, d_ih_w_p, d_o_t_p, d_h_t_p: adjust values (delta) of last step
@param o_grid_p, h_grid_p: gradient of last step
@param alpha: forget factor

@return adjust values (delta) of ho_w, ih_w, o_t, h_t,
and gradient value of o_grid, h_grid for this step
'''

# dependent packages
import numpy as np

# get current network output
self.Pred(x)

# 基于当前输出计算梯度
o_grid = np.zeros(self.o_n)  # 输出层神经元的梯度项
for j in range(self.o_n):
o_grid[j] = (y[j] - self.o_v[j]) * self.afd(self.o_v[j])
# 输出层梯度 = (真实结果 - 输出结果) * y( 1-y )   ( 西瓜书式 5.10 )

h_grid = np.zeros(self.h_n)  # 隐层神经元的梯度项
for h in range(self.h_n):
for j in range(self.o_n):
h_grid[h] += self.ho_w[h][j] * o_grid[j]  # ∑ (输出层权重参数 * 输出层梯度项)
h_grid[h] = h_grid[h] * self.afd(self.h_v[h])
# 隐层梯度 = bh(1-bh)∑ (输出层权重参数 * 输出层梯度项)  ( 西瓜书式 5.15 )
# ( bh为隐层神经元的输出 )

# 输出层更新参数
lamda = np.sign(o_grid * o_grid_p)
# λ = sign(g(t)g(t-1))
# 当x>0，sign(x)=1;  当x=0，sign(x)=0;  当x<0， sign(x)=-1
o_grid_p = o_grid # o_grid是当前梯度  o_grid_p是上一次梯度
for h in range(self.h_n):
for j in range(self.o_n):
# 自适应学习率
o_grid_p[j] = o_grid[j]
lr = self.lr1[j] * (3 ** lamda[j])
# 基本思想：如果连续两次梯度方向一致，λ为正数，学习率会增大，加速收敛
#   如果如果连续两次梯度方向不一致，λ为负数，学习率减小
self.lr1[j] = 0.5 if lr > 0.5 else (0.005 if lr < 0.005 else lr)
# 不让学习率过大或者过小，处于0.005到0.5之间

# 改进后的输出层权重  对应西瓜书式 5.11   alpha是遗忘因子，取0到1，加强当前数据的影响，减少历史数据的影响
d_ho_w_p[h][j] = self.lr1[j] * o_grid[j] * self.h_v[h] + alpha * d_ho_w_p[h][j]
self.ho_w[h][j] += d_ho_w_p[h][j]

# 同上
lamda = np.sign(h_grid * h_grid_p)
h_grid_p = h_grid
for i in range(self.i_n):
for h in range(self.h_n):
lr = self.lr2[h] * (3 ** lamda[h])
self.lr2[j] = 0.5 if lr > 0.5 else (0.005 if lr < 0.005 else lr)

# updating parameter
d_ih_w_p[i][h] = self.lr2[h] * h_grid[h] * self.i_v[i] + alpha * d_ih_w_p[i][h]
self.ih_w[i][h] += d_ih_w_p[i][h]

for j in range(self.o_n):
d_o_t_p[j] = -(self.lr1[j] * o_grid[j] + alpha * d_o_t_p[j])
self.o_t[j] += d_o_t_p[j]

for h in range(self.h_n):
d_h_t_p[h] = -(self.lr2[h] * h_grid[h] + alpha * d_h_t_p[h])
self.h_t[h] += d_h_t_p[h]

return d_ho_w_p, d_ih_w_p, d_o_t_p, d_h_t_p, o_grid_p, h_grid_p

def TrainStandard_Dynamic_Lr(self, data_in, data_out):
'''
standard BP training
@param lr, learning rate, default 0.05
@return: e, accumulated error
@return: e_k, error array of each step
'''
# dependent packages
import numpy as np

d_ih_w_p = np.zeros([self.i_n, self.h_n])  # initial delta values = 0.0
d_ho_w_p = np.zeros([self.h_n, self.o_n])
d_h_t_p = np.zeros(self.h_n)
d_o_t_p = np.zeros(self.o_n)

o_grid_p = np.zeros(self.o_n)  # initial gradient = 0.01
h_grid_p = np.zeros(self.h_n)

e_k = []
for k in range(len(data_in)):
x = data_in[k]
y = data_out[k]
d_ho_w_p, d_ih_w_p, d_o_t_p, d_h_t_p, o_grid_p, h_grid_p
= self.BackPropagate_Dynamic_Lr(x, y, d_ho_w_p, d_ih_w_p, d_o_t_p, d_h_t_p,
o_grid_p, h_grid_p, 0.2)

# error in train set for each step
y_delta2 = 0.0
for j in range(self.o_n):
y_delta2 += (self.o_v[j] - y[j]) * (self.o_v[j] - y[j])
e_k.append(y_delta2 / 2)

# total error of training
e = sum(e_k) / len(e_k)

return e, e_k

def PredLabel(self, X):
'''
predict process through the network

@param X: the input sample set for input layer
@return: y, array, output set (0,1,2... - class) based on [winner-takes-all]
'''
import numpy as np

y = []

for m in range(len(X)):
self.Pred(X[m])
#             if self.o_v[0] > 0.5:  y.append(1)
#             else : y.append(0)
max_y = self.o_v[0]
label = 0
for j in range(1, self.o_n):
if max_y < self.o_v[j]:
label = j
max_y = self.o_v[j]
y.append(label)

return np.array(y)

'''
the definition of activation functions
'''

def Sigmoid(x):
'''
definition of sigmoid function and it's derivation
'''
from math import exp
return 1.0 / (1.0 + exp(-x))

def SigmoidDerivate(y):
return y * (1 - y)

def Tanh(x):
'''
definition of sigmoid function and it's derivation
'''
from math import tanh
return tanh(x)

def TanhDerivate(y):
return 1 - y * y

'''
the definition of random function
'''

def rand(a, b):
'''
random value generation for parameter initialization
@param a,b: the upper and lower limitation of the random value
'''
from random import random
return (b - a) * random() + a

``````

BP_improve.py

``````# -*- coding: utf-8 -*

'''
preparation of data
'''
import pandas as pd
import matplotlib.pyplot as plt

from urllib.request import urlopen

url = "http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
attr = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']

#visualization of data
# import seaborn as sns
# sns.pairplot(dataset, hue='species', vars = ['sepal_length','petal_length'],diag_kind = None)
# plt.show()

# generation of input, output, label

# input variables (assignment directly)
X = dataset.iloc[:, :4].values
#print(X)

# label (generation after transform output to categorical variables)
dataset.iloc[:, -1] = dataset.iloc[:, -1].astype('category')
label = dataset.iloc[:, 4].values.categories

# output 1 (generation after string categorical variables to numerical values)

dataset.iloc[:, 4].cat.categories = [0, 1, 2]  # 三分类

y = dataset.iloc[:, 4].values

# output 2 (generation after one hot encoding)
Y = pd.get_dummies(dataset.iloc[:, 4]).values

'''
split of train set and test set (using sklearn function)
'''
from sklearn.model_selection import  train_test_split

train_X, test_X, train_y, test_y, train_Y, test_Y = train_test_split(X, y, Y, test_size=0.5, random_state=42)

'''
construction of BP network
'''
from BP_network import *

bpn1 = BP_network()  # initial a BP network class
bpn1.CreateNN(4, 5, 3, actfun='Sigmoid', learningrate=0.05)  # build the network

'''
experiment of fixed learning rate
'''

# parameter training with fixed learning rate initial above
e = []
for i in range(1000):
err, err_k = bpn1.TrainStandard(train_X, train_Y)
e.append(err)

# draw the convergence curve of output error by each step of iteration
import matplotlib.pyplot as plt

f1 = plt.figure(1)
plt.xlabel("epochs")
plt.ylabel("error")
plt.ylim(0, 1)
plt.title("training error convergence curve with fixed learning rate")
plt.title("training error convergence curven learning rate = 0.05")
plt.plot(e)
plt.show()

# get the test error in test set
pred = bpn1.PredLabel(test_X)
count = 0
for i in range(len(test_y)):
if pred[i] == test_y[i]: count += 1

test_err = 1 - count / len(test_y)
print("固定学习率的错误率: %.3f" % test_err)

'''
experiment of dynamic learning rate
'''

bpn2 = BP_network()  # initial a BP network class
bpn2.CreateNN(4, 5, 3, actfun='Sigmoid', learningrate=0.05)  # build the network

# parameter training with fixed learning rate initial above
e = []
for i in range(1000):
err, err_k = bpn2.TrainStandard_Dynamic_Lr(train_X, train_Y)
e.append(err)

# draw the convergence curve of output error by each step of iteration
# import matplotlib.pyplot as plt
f2 = plt.figure(2)
plt.xlabel("epochs")
plt.ylabel("error")
plt.ylim(0, 1)
plt.title("training error convergence curve with dynamic learning rate")
plt.plot(e)
plt.show()

# get the test error in test set
pred = bpn2.PredLabel(test_X)
count = 0
for i in range(len(test_y)):
if pred[i] == test_y[i]: count += 1

test_err = 1 - count / len(test_y)
print("动态调整学习率的错误率: %.3f" % test_err)

plt.show()

``````

THE END

)">