构建最简单图神经网络
参考：李金洪老师的《Pytorch深度学习和图神经网络(卷1)基础知识》
使用数据集CORA（典中之典）。主要两个文件，一个Solution,一个Ranger(shiyan3)。Solution里为逐步搭建图神经网络并且逐步解决问题。Ranger以后回来补坑吧，因为暂时没有时间去看所以并不清楚但是它有自带英文注释。
Solution
from scipy.sparse import coo_matrix,csr_matrix,eye,diags
import torch.nn.functional as F
import matplotlib.pyplot as plt
from pathlib import Path
from tqdm import tqdm
from shiyan3 import *
from torch import nn
import pandas as pd
import numpy as np
import torch
#numpy,scipy本质可以通用不必过于区分
#用作归一化的函数
def normalize(matrix):
#虽然不是标准矩阵(csr格式的矩阵)但是一样的用维度
cnt=np.array(np.sum(matrix,axis=1))
jz=(cnt**(-1)).flatten()
jz[np.isinf(jz)]=0
jz=diags(jz)
matrix=np.dot(jz,matrix)
return matrix
#输出运算资源情况。要是没有GPU就用CPU
device=torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
print(device)
#输出路径，检查数据是否存在
path=Path("cora")
print(path)
#获取文章的ID，文章特征向量，文章标签
papers=np.genfromtxt(path/'cora.content',dtype=np.str_)
ids=papers[:,0].astype(np.int64)
features=csr_matrix(papers[:,1:-1],dtype=np.float64)#这里必须为浮点数
labels=papers[:,-1]
#重新处理文章标签
zd={j:i for i,j in enumerate(sorted(np.unique(labels)))}
labels=[zd[_] for _ in labels]
#处理边吧
edges=np.genfromtxt(path/'cora.cites',dtype=np.int64)
ids2={j:i for i,j in enumerate(ids)}
edges=np.array([ids2[_] for _ in edges.flatten()],np.int64).reshape(edges.shape)
#构建邻接矩阵
adj=coo_matrix((np.ones(edges.shape[0]),(edges[:,0],edges[:,1])),shape=(len(labels),len(labels)),dtype=np.float64)#这里必须为浮点数
#生成无向对称矩阵
adj_long=adj.multiply(adj.T<adj)
adj=adj_long+adj_long.T
#处理邻接矩阵以及特征矩阵
features=normalize(features)
adj=normalize(adj+eye(adj.shape[0]))#变成自循环图
#构建张量；划分数据；打乱顺序；分配资源
labels=torch.LongTensor(labels)
features=torch.FloatTensor(features.todense())
adj=torch.FloatTensor(adj.todense())
n_train=200;n_val=300;n_test=len(features)-n_train-n_val
ids3=np.random.permutation(len(features))
ids_train=torch.LongTensor(ids3[:n_train])
ids_val=torch.LongTensor(ids3[n_train:n_train+n_val])
ids_test=torch.LongTensor(ids3[n_train+n_val:])
labels=labels.to(device)
features=features.to(device)
adj=adj.to(device)
ids_train=ids_train.to(device)
ids_val=ids_val.to(device)
ids_test=ids_test.to(device)
#反正就是一个激活函数效果不错
def mish(x):
return x*(torch.tanh(F.softplus(x)))
class GraphConvolution(nn.Module):
def __init__(self,f_in,f_out,use_bias=True,activation=mish):
super().__init__()
self.f_in=f_in
self.f_out=f_out
self.use_bias=use_bias#TF是否使用哑元,默认使用
self.activation=activation#是否采用激活,默认使用
self.w=nn.Parameter(torch.FloatTensor(f_in,f_out))#权重很简单的
self.b=nn.Parameter(torch.FloatTensor(f_out)) if use_bias else None#哑元很简单的
self.initialize_wights()
def initialize_wights(self):
#初始化权重吧，细节理解不了
if self.activation is None:
nn.init.xavier_uniform_(self.w)
else:
nn.init.kaiming_uniform_(self.w,nonlinearity='leaky_relu')
#初始化哑元吧，细节理解不了
if self.use_bias:
nn.init.zeros_(self.b)
def forward(self,input,adj):
#点积结果
output=torch.mm(adj,torch.mm(input,self.w))
#是否加入哑元
if self.use_bias:
output.add_(self.b)
#是否使用激活函数
if self.activation is not None:
output=self.activation(output)
return output
class GCN(nn.Module):
def __init__(self,f_in,n_classes,hidden=[16],dropout=0.5):
super().__init__()
#根据参数构建网络。值得注意默认一层。下面代码并无错误仔细阅读知其精妙
layers=[]
for f_in,f_out in zip([f_in]+hidden[:-1],hidden):
layers+=[GraphConvolution(f_in,f_out)]
self.layers=nn.Sequential(*layers)
#设置参数dropout参数吧，没啥说的
self.dropout=dropout
#构建输出层吧这里就没有激活函数啦
self.out_layer=GraphConvolution(f_out,n_classes,activation=None)
def forward(self,x,adj):
#这里就是不断向前一层一层
for layer in self.layers:
x=layer(x,adj)
F.dropout(x,self.dropout,training=self.training,inplace=True)#这里有个self.training很奇怪，书上这么说的
return self.out_layer(x,adj)
#要开始啦
n_labels=labels.max().item()+1;n_features=features.shape[1]
model=GCN(n_features,n_labels,hidden=[16,32,16]).to(device)
#大概这里output类似onehot编码
def accuracy(output,y):
return (output.argmax(axis=1)==y).type(torch.float64).mean().item()
def step():
model.train() #训练模式
optimizer.zero_grad() #清除梯度
output=model(features,adj) #得到输出
loss=F.cross_entropy(output[ids_train],labels[ids_train])#计算损失(是交叉熵)
acc=accuracy(output[ids_train],labels[ids_train]) #计算准度
loss.backward() #反向传播
optimizer.step() #模型更新
return loss.item(),acc
def evaluate(ids):
model.eval() #评估模式
output=model(features,adj) #得到输出
loss=F.cross_entropy(output[ids],labels[ids])
return loss.item(),accuracy(output[ids],labels[ids]) #返回结果
#这里有个啥优化器，我没有看。MayBe大概可能是优化参数的吧。以后有时间回来填坑吧
optimizer=Ranger(model.parameters()) #parameters()是父类方法
epochs=1000;print_steps=50;train_loss,train_acc=[],[];val_loss,val_acc=[],[]
for i in tqdm(range(epochs)):
tl,ta=step()
train_loss+=[tl];train_acc+=[ta]
if (i+1)%print_steps==0 or i==0:
t1,ta=evaluate(ids_train)
v1,va=evaluate(ids_val)
val_loss+=[v1];val_acc+=[va]
print(f'{i+1:6d}/{epochs}:
train_loss={t1:.4f},train_acc={ta:.4f}'+f',val_loss={v1:.4f},val_acc={va:.4f}')
final_train,final_val,final_test=evaluate(ids_train),evaluate(ids_val),evaluate(ids_test)
print(f'Train :loss={final_train[0]:.4f},accuary={final_train[1]:.4f}')
print(f'Validation:loss= {final_val[0]:.4f},accuary= {final_val[1]:.4f}')
print(f'Test :loss= {final_test[0]:.4f},accuary= {final_test[1]:.4f}')
plt.figure(figsize=(16,9))
plt.subplot(1,2,1)
plt.plot(train_loss[::print_steps]+[train_loss[-1]],label='Train')
plt.plot(val_loss,label='Validation')
plt.legend()
plt.subplot(1,2,2)
plt.plot( train_acc[::print_steps]+ [train_acc[-1]],label='Train')
plt.plot(val_acc,label='Validation')
plt.legend()
plt.show()
output=model(features,adj);samples=10
ids_sample=ids_test[torch.randperm(len(ids_test))[:samples]]
zd={i:j for j,i in zd.items()}
df=pd.DataFrame({'Real':[zd[_] for _ in labels[ids_sample].tolist()],
'Pred':[zd[_] for _ in output[ids_sample].argmax(1).tolist()]})
print(df)
Ranger(shiyan3)
#Ranger deep learning optimizer - RAdam + Lookahead combined.
#https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer
#Ranger has now been used to capture 12 records on the FastAI leaderboard.
#This version = 9.3.19
#Credits:
#RAdam --> https://github.com/LiyuanLucasLiu/RAdam
#Lookahead --> rewritten by lessw2020, but big thanks to Github @LonePatient and @RWightman for ideas from their code.
#Lookahead paper --> MZhang,G Hinton https://arxiv.org/abs/1907.08610
#summary of changes:
#full code integration with all updates at param level instead of group, moves slow weights into state dict (from generic weights),
#supports group learning rates (thanks @SHolderbach), fixes sporadic load from saved model issues.
#changes 8/31/19 - fix references to *self*.N_sma_threshold;
#changed eps to 1e-5 as better default than 1e-8.
import math
import torch
from torch.optim.optimizer import Optimizer, required
import itertools as it
class Ranger(Optimizer):
def __init__(self, params, lr=1e-3, alpha=0.5, k=6, N_sma_threshhold=5, betas=(.95,0.999), eps=1e-5, weight_decay=0):
#parameter checks
if not 0.0 <= alpha <= 1.0:
raise ValueError(f'Invalid slow update rate: {alpha}')
if not 1 <= k:
raise ValueError(f'Invalid lookahead steps: {k}')
if not lr > 0:
raise ValueError(f'Invalid Learning Rate: {lr}')
if not eps > 0:
raise ValueError(f'Invalid eps: {eps}')
#parameter comments:
# beta1 (momentum) of .95 seems to work better than .90...
#N_sma_threshold of 5 seems better in testing than 4.
#In both cases, worth testing on your dataset (.90 vs .95, 4 vs 5) to make sure which works best for you.
#prep defaults and init torch.optim base
defaults = dict(lr=lr, alpha=alpha, k=k, step_counter=0, betas=betas, N_sma_threshhold=N_sma_threshhold, eps=eps, weight_decay=weight_decay)
super().__init__(params,defaults)
#adjustable threshold
self.N_sma_threshhold = N_sma_threshhold
#now we can get to work...
#removed as we now use step from RAdam...no need for duplicate step counting
#for group in self.param_groups:
# group["step_counter"] = 0
#print("group step counter init")
#look ahead params
self.alpha = alpha
self.k = k
#radam buffer for state
self.radam_buffer = [[None,None,None] for ind in range(10)]
#self.first_run_check=0
#lookahead weights
#9/2/19 - lookahead param tensors have been moved to state storage.
#This should resolve issues with load/save where weights were left in GPU memory from first load, slowing down future runs.
#self.slow_weights = [[p.clone().detach() for p in group['params']]
# for group in self.param_groups]
#don't use grad for lookahead weights
#for w in it.chain(*self.slow_weights):
# w.requires_grad = False
def __setstate__(self, state):
print("set state called")
super(Ranger, self).__setstate__(state)
def step(self, closure=None):
loss = None
#note - below is commented out b/c I have other work that passes back the loss as a float, and thus not a callable closure.
#Uncomment if you need to use the actual closure...
#if closure is not None:
#loss = closure()
#Evaluate averages and grad, update param tensors
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('Ranger optimizer does not support sparse gradients')
p_data_fp32 = p.data.float()
state = self.state[p] #get state dict for this param
if len(state) == 0: #if first time to run...init dictionary with our desired entries
#if self.first_run_check==0:
#self.first_run_check=1
#print("Initializing slow buffer...should not see this at load from saved model!")
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
#look ahead weight storage now in state dict
state['slow_buffer'] = torch.empty_like(p.data)
state['slow_buffer'].copy_(p.data)
else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
#begin computations
exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
#compute variance mov avg
exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
#compute mean moving avg
exp_avg.mul_(beta1).add_(1 - beta1, grad)
state['step'] += 1
buffered = self.radam_buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 / (1 - beta2) - 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)
buffered[1] = N_sma
if N_sma > self.N_sma_threshhold:
step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])
else:
step_size = 1.0 / (1 - beta1 ** state['step'])
buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
if N_sma > self.N_sma_threshhold:
denom = exp_avg_sq.sqrt().add_(group['eps'])
p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
else:
p_data_fp32.add_(-step_size * group['lr'], exp_avg)
p.data.copy_(p_data_fp32)
#integrated look ahead...
#we do it at the param level instead of group level
if state['step'] % group['k'] == 0:
slow_p = state['slow_buffer'] #get access to slow param tensor
slow_p.add_(self.alpha, p.data - slow_p) #(fast weights - slow weights) * alpha
p.data.copy_(slow_p) #copy interpolated weights to RAdam param tensor
return loss