kaggle学习笔记-otto-baseline6-使用 RAPIDS TSNE 和项目矩阵分解可视化用户行为
简介
在Kaggle的Otto推荐系统竞赛中，项目ID是匿名的。所以我们不知道每个项目 id 指的是哪个。但是，通过项目矩阵分解，我们可以将匿名项目 ID 转换为有意义的嵌入。那么类似的嵌入将是类似的项目。如果我们将嵌入投影到 2D 平面（使用 TSNE、UMAP、PCA 等）并绘制它们，我们可以看到类似项目的集群。然后一个集群可能是服装，另一个集群可能是电子产品等。
使用项目嵌入的 2D 平面，我们可以绘制点和线，显示每个用户活动的进展。这使我们能够了解用户如何购物。用户是否浏览类似的项目，然后查看不同的项目，然后返回购买原始项目。用户主要探索类似项目吗？还是用户探索各种项目？下面的图将阐明用户行为，帮助我们理解数据并构建更好的功能和模型。
数据处理
import cudf
print('RAPIDS cuDF version',cudf.__version__)
train = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')
test = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')
train_pairs = cudf.concat([train, test])[['session', 'aid']]
del train, test
train_pairs['aid_next'] = train_pairs.groupby('session').aid.shift(-1)
train_pairs = train_pairs[['aid', 'aid_next']].dropna().reset_index(drop=True)
cardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())
print('Cardinality of items is',cardinality_aids)
安装 Merlin 下载器
!pip install merlin-dataloader==0.0.2
from merlin.loader.torch import Loader
train_pairs.to_pandas().to_parquet('train_pairs.parquet') # TRAIN WITH ALL DATA
train_pairs[-10_000_000:].to_pandas().to_parquet('valid_pairs.parquet')
from merlin.loader.torch import Loader
from merlin.io import Dataset
train_ds = Dataset('train_pairs.parquet')
train_dl_merlin = Loader(train_ds, 65536, True)
使用 PyTorch 矩阵分解模型学习项目嵌入
import torch
from torch import nn
class MatrixFactorization(nn.Module):
def __init__(self, n_aids, n_factors):
super().__init__()
self.aid_factors = nn.Embedding(n_aids, n_factors, sparse=True)
def forward(self, aid1, aid2):
aid1 = self.aid_factors(aid1)
aid2 = self.aid_factors(aid2)
return (aid1 * aid2).sum(dim=1)
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def __str__(self):
fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
return fmtstr.format(**self.__dict__)
valid_ds = Dataset('valid_pairs.parquet')
valid_dl_merlin = Loader(valid_ds, 65536, True)
from torch.optim import SparseAdam
num_epochs = 10
lr=0.1
model = MatrixFactorization(cardinality_aids+1, 32)
optimizer = SparseAdam(model.parameters(), lr=lr)
criterion = nn.BCEWithLogitsLoss()
model.to('cuda')
for epoch in range(num_epochs):
for batch, _ in train_dl_merlin:
model.train()
losses = AverageMeter('Loss', ':.4e')
aid1, aid2 = batch['aid'], batch['aid_next']
aid1 = aid1.to('cuda')
aid2 = aid2.to('cuda')
output_pos = model(aid1, aid2)
output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])
output = torch.cat([output_pos, output_neg])
targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])
loss = criterion(output, targets)
losses.update(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
model.eval()
with torch.no_grad():
accuracy = AverageMeter('accuracy')
for batch, _ in valid_dl_merlin:
aid1, aid2 = batch['aid'], batch['aid_next']
output_pos = model(aid1, aid2)
output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])
accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()
accuracy.update(accuracy_batch, aid1.shape[0])
print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f} * Accuracy {accuracy.avg:.3f}')
提取项目嵌入
# EXTRACT EMBEDDINGS FROM MODEL EMBEDDING TABLE
embeddings = model.aid_factors.weight.detach().cpu().numpy()
print('Item Matrix Factorization embeddings have shape',embeddings.shape)
使用 RAPIDS TSNE 可视化用户行为
# IMPORT RAPIDS TSNE
from cuml import UMAP, TSNE, PCA
import matplotlib.pyplot as plt, numpy as np
import matplotlib.patches as mpatches, cuml
print('RAPIDS cuML version',cuml.__version__)
# FIT TRANSFORM TSNE
em_2d = TSNE(n_components=2).fit_transform(embeddings)
print('TSNE embeddings have shape',em_2d.shape)
# LOAD TEST DATA
test = cudf.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')
tmp = test.groupby('session').aid.agg('count').rename('n')
test = test.merge(tmp, left_on='session', right_index=True, how='left')
active_users = test.loc[test.n>20,'session'].unique().to_array()
test = test.sort_values(['session','ts'])
print('Test data shape:', test.shape )
test.head()
使用 TSNE 项目嵌入显示用户活动
对于 50 个随机用户（有 20 个或更多活动），我们将绘制他们的活动随时间的变化。顶部图显示与之交互的项目的类别，底部图显示项目交互的时间。当顶部图中的点彼此靠近时，则项目相似。例如，两件衣服将接近，而一件衣服将与一件电子产品相距甚远。
图中的数字按时间排序。数字 1 是用户与之交互的第一个项目，而数字 10 是第 10 个项目，依此类推。我们只显示一些数字，以便数字可读。橙色点是点击，绿色点是购物车，红点是买入。180万个蓝点都是独一无二的物品。我们观察到代表不同类别物品的蓝点簇。
x-y 平面表示不同的项目类别。如果用户呆在同一区域，那么他们正在购买类似的物品，例如服装部门。当用户的绘图从 x-y 平面的一个区域更改为另一个区域时，用户将更改为不同类别的项目，例如从服装购物转移到电子产品购物。我们观察不同类型的用户。一些用户浏览一个项目类别，而其他用户浏览各种项目类别。
# DISPLAY EDA FOR 50 USERS
for k in range(50):
# SELECT ONE USER WITH 20+ CLICKS
u = np.random.choice(active_users)
dff = test.loc[test.session==u].to_pandas().reset_index(drop=True)
tmp = dff.aid.values
clicks = test.loc[(test.session==u)&(test['type']==0)].to_pandas().aid.values
carts = test.loc[(test.session==u)&(test['type']==1)].to_pandas().aid.values
orders = test.loc[(test.session==u)&(test['type']==2)].to_pandas().aid.values
############
## PLOT HISTORY BY ITEM CATEGORY
############
# PLOT CLICKS, CARTS, ORDERS OVER TSNE ITEM EMBEDDING PLOT
plt.figure(figsize=(15,15))
plt.scatter(em_2d[::25,0],em_2d[::25,1],s=1,label='All 1.8M items!')
plt.plot(em_2d[tmp][:,0],em_2d[tmp][:,1],'-',color='orange')
plt.scatter(em_2d[tmp][:,0],em_2d[tmp][:,1],color='orange',s=25,label='Click')
plt.scatter(em_2d[carts][:,0],em_2d[carts][:,1],color='green',s=100,label='Cart')
plt.scatter(em_2d[orders][:,0],em_2d[orders][:,1],color='red',s=250,label='Order')
# PLOT NUMBERS OF ORDER VISITED
old_xy = []; pos = []
for i,(x,y) in enumerate(zip(em_2d[tmp][:,0],em_2d[tmp][:,1])):
new_location = True
for j in old_xy:
if (np.abs(x-j[0])<5) & (np.abs(y-j[1])<5):
new_location = False
if new_location:
plt.text(x,y,f'{i+1}',size=18)
old_xy.append( (x,y) ); pos.append(i)
# LABEL PLOT
plt.legend()
plt.title(f'Test User {u} - {len(clicks)} clicks, {len(carts)} carts, {len(orders)} orders:',size=18)
#plt.xlabel('Item category',size=16)
plt.ylabel('nnItem category',size=16)
plt.xticks([], [])
plt.yticks([], [])
plt.show()
############
## PLOT HISTORY BY DAY AND HOUR
############
mn = test.ts.min()
dff['day'] = (dff.ts - mn) // (60*60*24)
dff['hour'] = ((dff.ts - mn) % (60*60*24)) // (60*60)
plt.figure(figsize=(15,3))
xx = np.random.uniform(-0.2,0.2,len(dff))
yy = np.random.uniform(-0.5,0.5,len(dff))
plt.scatter(dff.day.values+xx, dff.hour.values+yy, s=25, color='orange')
cidx = dff.loc[dff['type']==1].index.values
oidx = dff.loc[dff['type']==2].index.values
plt.scatter(dff.day.values[cidx]+xx[cidx], dff.hour.values[cidx]+yy[cidx], s=50, color='green')
plt.scatter(dff.day.values[oidx]+xx[oidx], dff.hour.values[oidx]+yy[oidx], s=100, color='red')
old_xy = []
for i in range(len(dff)):
if 1: #i in pos:
x = dff.day.values[i]+xx[i]
y = dff.hour.values[i]+yy[i]
new_location = True
for j in old_xy:
if (np.abs(x-j[0])<0.5) & (np.abs(y-j[1])<4):
new_location = False
if new_location:
plt.text(x, y, f'{i+1}', size=18)
old_xy.append( (x,y) )
plt.ylim((-1,25))
plt.xlim((-1,7))
plt.ylabel('Hour of Day',size=16)
plt.xlabel('Day of Month',size=16)
plt.yticks([0,4,8,12,16,20,24],['12am','4am','8am','noon','4pm','8pm','12am'])
plt.xticks([0,1,2,3,4,5,6],['MonnAug 29th','TuenAug 30rd','WednAug 31st',
'ThrnSep 1st','FrinSep 2nd','SatnSep 3rd','SunnSep 4th'])
c1 = mpatches.Patch(color='orange', label='Click')
c2 = mpatches.Patch(color='green', label='Cart')
c3 = mpatches.Patch(color='red', label='Order')
plt.legend(handles=[c1,c2,c3])
plt.show()
print('nnnnnnn')
这段代码产生的图片较多，这里只放两张作为示意