# kaggle学习笔记-otto-baseline6-使用 RAPIDS TSNE 和项目矩阵分解可视化用户行为

## 数据处理

``````import cudf
print('RAPIDS cuDF version',cudf.__version__)

train_pairs = cudf.concat([train, test])[['session', 'aid']]
del train, test

train_pairs['aid_next'] = train_pairs.groupby('session').aid.shift(-1)
train_pairs = train_pairs[['aid', 'aid_next']].dropna().reset_index(drop=True)

cardinality_aids = max(train_pairs['aid'].max(), train_pairs['aid_next'].max())
print('Cardinality of items is',cardinality_aids)
``````

## 安装 Merlin 下载器

``````!pip install merlin-dataloader==0.0.2
``````
``````from merlin.loader.torch import Loader

train_pairs.to_pandas().to_parquet('train_pairs.parquet') # TRAIN WITH ALL DATA
train_pairs[-10_000_000:].to_pandas().to_parquet('valid_pairs.parquet')

from merlin.io import Dataset

train_ds = Dataset('train_pairs.parquet')
``````

## 使用 PyTorch 矩阵分解模型学习项目嵌入

``````import torch
from torch import nn

class MatrixFactorization(nn.Module):
def __init__(self, n_aids, n_factors):
super().__init__()
self.aid_factors = nn.Embedding(n_aids, n_factors, sparse=True)

def forward(self, aid1, aid2):
aid1 = self.aid_factors(aid1)
aid2 = self.aid_factors(aid2)

return (aid1 * aid2).sum(dim=1)

class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()

def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0

def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count

def __str__(self):
fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
return fmtstr.format(**self.__dict__)

valid_ds = Dataset('valid_pairs.parquet')
``````
``````from torch.optim import SparseAdam

num_epochs = 10
lr=0.1

model = MatrixFactorization(cardinality_aids+1, 32)
criterion = nn.BCEWithLogitsLoss()

model.to('cuda')
for epoch in range(num_epochs):
for batch, _ in train_dl_merlin:
model.train()
losses = AverageMeter('Loss', ':.4e')

aid1, aid2 = batch['aid'], batch['aid_next']
aid1 = aid1.to('cuda')
aid2 = aid2.to('cuda')
output_pos = model(aid1, aid2)
output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])

output = torch.cat([output_pos, output_neg])
targets = torch.cat([torch.ones_like(output_pos), torch.zeros_like(output_pos)])
loss = criterion(output, targets)
losses.update(loss.item())

loss.backward()
optimizer.step()

model.eval()

accuracy = AverageMeter('accuracy')
for batch, _ in valid_dl_merlin:
aid1, aid2 = batch['aid'], batch['aid_next']
output_pos = model(aid1, aid2)
output_neg = model(aid1, aid2[torch.randperm(aid2.shape[0])])
accuracy_batch = torch.cat([output_pos.sigmoid() > 0.5, output_neg.sigmoid() < 0.5]).float().mean()
accuracy.update(accuracy_batch, aid1.shape[0])

print(f'{epoch+1:02d}: * TrainLoss {losses.avg:.3f}  * Accuracy {accuracy.avg:.3f}')
``````

## 提取项目嵌入

``````# EXTRACT EMBEDDINGS FROM MODEL EMBEDDING TABLE
embeddings = model.aid_factors.weight.detach().cpu().numpy()
print('Item Matrix Factorization embeddings have shape',embeddings.shape)
``````

## 使用 RAPIDS TSNE 可视化用户行为

``````# IMPORT RAPIDS TSNE
from cuml import UMAP, TSNE, PCA
import matplotlib.pyplot as plt, numpy as np
import matplotlib.patches as mpatches, cuml
print('RAPIDS cuML version',cuml.__version__)

# FIT TRANSFORM TSNE
em_2d = TSNE(n_components=2).fit_transform(embeddings)
print('TSNE embeddings have shape',em_2d.shape)
``````
``````# LOAD TEST DATA
tmp = test.groupby('session').aid.agg('count').rename('n')
test = test.merge(tmp, left_on='session', right_index=True, how='left')
active_users = test.loc[test.n>20,'session'].unique().to_array()
test = test.sort_values(['session','ts'])
print('Test data shape:', test.shape )
``````

## 使用 TSNE 项目嵌入显示用户活动

x-y 平面表示不同的项目类别。如果用户呆在同一区域，那么他们正在购买类似的物品，例如服装部门。当用户的绘图从 x-y 平面的一个区域更改为另一个区域时，用户将更改为不同类别的项目，例如从服装购物转移到电子产品购物。我们观察不同类型的用户。一些用户浏览一个项目类别，而其他用户浏览各种项目类别。

``````# DISPLAY EDA FOR 50 USERS
for k in range(50):

# SELECT ONE USER WITH 20+ CLICKS
u = np.random.choice(active_users)
dff = test.loc[test.session==u].to_pandas().reset_index(drop=True)
tmp = dff.aid.values
clicks = test.loc[(test.session==u)&(test['type']==0)].to_pandas().aid.values
carts = test.loc[(test.session==u)&(test['type']==1)].to_pandas().aid.values
orders = test.loc[(test.session==u)&(test['type']==2)].to_pandas().aid.values

############
## PLOT HISTORY BY ITEM CATEGORY
############

# PLOT CLICKS, CARTS, ORDERS OVER TSNE ITEM EMBEDDING PLOT
plt.figure(figsize=(15,15))
plt.scatter(em_2d[::25,0],em_2d[::25,1],s=1,label='All 1.8M items!')
plt.plot(em_2d[tmp][:,0],em_2d[tmp][:,1],'-',color='orange')
plt.scatter(em_2d[tmp][:,0],em_2d[tmp][:,1],color='orange',s=25,label='Click')
plt.scatter(em_2d[carts][:,0],em_2d[carts][:,1],color='green',s=100,label='Cart')
plt.scatter(em_2d[orders][:,0],em_2d[orders][:,1],color='red',s=250,label='Order')

# PLOT NUMBERS OF ORDER VISITED
old_xy = []; pos = []
for i,(x,y) in enumerate(zip(em_2d[tmp][:,0],em_2d[tmp][:,1])):
new_location = True
for j in old_xy:
if (np.abs(x-j[0])<5) & (np.abs(y-j[1])<5):
new_location = False
if new_location:
plt.text(x,y,f'{i+1}',size=18)
old_xy.append( (x,y) ); pos.append(i)

# LABEL PLOT
plt.legend()
plt.title(f'Test User {u} - {len(clicks)} clicks, {len(carts)} carts, {len(orders)} orders:',size=18)
#plt.xlabel('Item category',size=16)
plt.ylabel('nnItem category',size=16)
plt.xticks([], [])
plt.yticks([], [])
plt.show()

############
## PLOT HISTORY BY DAY AND HOUR
############

mn = test.ts.min()
dff['day'] = (dff.ts - mn) // (60*60*24)
dff['hour'] = ((dff.ts - mn) % (60*60*24)) // (60*60)

plt.figure(figsize=(15,3))
xx = np.random.uniform(-0.2,0.2,len(dff))
yy = np.random.uniform(-0.5,0.5,len(dff))
plt.scatter(dff.day.values+xx, dff.hour.values+yy, s=25, color='orange')
cidx = dff.loc[dff['type']==1].index.values
oidx = dff.loc[dff['type']==2].index.values
plt.scatter(dff.day.values[cidx]+xx[cidx], dff.hour.values[cidx]+yy[cidx], s=50, color='green')
plt.scatter(dff.day.values[oidx]+xx[oidx], dff.hour.values[oidx]+yy[oidx], s=100, color='red')
old_xy = []
for i in range(len(dff)):
if 1: #i in pos:
x = dff.day.values[i]+xx[i]
y = dff.hour.values[i]+yy[i]
new_location = True
for j in old_xy:
if (np.abs(x-j[0])<0.5) & (np.abs(y-j[1])<4):
new_location = False
if new_location:
plt.text(x, y, f'{i+1}', size=18)
old_xy.append( (x,y) )
plt.ylim((-1,25))
plt.xlim((-1,7))
plt.ylabel('Hour of Day',size=16)
plt.xlabel('Day of Month',size=16)
plt.yticks([0,4,8,12,16,20,24],['12am','4am','8am','noon','4pm','8pm','12am'])
plt.xticks([0,1,2,3,4,5,6],['MonnAug 29th','TuenAug 30rd','WednAug 31st',
'ThrnSep 1st','FrinSep 2nd','SatnSep 3rd','SunnSep 4th'])
c1 = mpatches.Patch(color='orange', label='Click')
c2 = mpatches.Patch(color='green', label='Cart')
c3 = mpatches.Patch(color='red', label='Order')
plt.legend(handles=[c1,c2,c3])
plt.show()

print('nnnnnnn')
``````

THE END