import pandas as pd
import idsw.hdfs as ih
hdfs = ih.client()
reader = hdfs.open('/user/1.csv', 'rb') 
df = pd.read_csv(reader)

with hdfs.open('/user/2.csv', 'wb')  as ff:
    out.to_csv(ff,index=0,header=1)

# do this for train and test dataset
raw_train_df = <uid,label,c1,c2,c3...>
raw_test_df = <uid,c1,c2,c3...>

train_uid=raw_train_df['acc_nbr']
train_y=raw_train_df['label']  # df.iloc[:,1]  df.iloc[:,2:176]  
train_X=raw_train_df.drop(['acc_nbr','label'], axis=1)  # axis=0表示行，axis=1表示列

test_uid=raw_test_df.pop("acc_nbr")
test_X=raw_test_df

from sklearn.model_selection import train_test_split
#from imblearn.datasets import make_imbalance
#X_imb, y_imb = make_imbalance(train_X, train_y,sampling_strategy={1: 50000, 2: 338166, 3: 390000})
#X_train, X_test, y_train, y_test = train_test_split(X_imb, y_imb)
X_train, X_test, y_train, y_test = train_test_split(train_X, train_y)
train_X.shape
train_y=train_y-1

# predict for test dataset
test_fea=torch.from_numpy(test_X.values).float()
outputs = model(test_fea)
_, y_pred = torch.max(outputs.data, dim = 1)
y_pred=y_pred+1
y_pred

submit=pd.merge(test_uid, pd.Series(y_pred, name='label'), left_index=True, right_index=True)

check = pd.merge(submit, Feb_ans, on=['acc_nbr'])
y_predicted = check['label_x']
y_true = check['label_y']

accuracy = np.mean(y_predicted == y_true) * 100
print ("accuracy:",accuracy)
from sklearn.metrics import recall_score
print(recall_score(y_true,y_predicted,average = 'macro'))
from sklearn.metrics import classification_report
print(classification_report(y_true, y_predicted,labels=['1','2','3']))

import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
from torchvision import datasets, transforms

batch_size = 64
train_set = TensorDataset(torch.from_numpy(X_train.values).float(),torch.from_numpy(y_train.values))
test_set = TensorDataset(torch.from_numpy(X_test.values).float(),torch.from_numpy(y_test.values))
train_loader = DataLoader(train_set,batch_size=batch_size,shuffle=True,num_workers=4)
test_loader = DataLoader(test_set,batch_size=batch_size,shuffle=True,num_workers=4)

from torch import nn
class Net(torch.nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.l1 = torch.nn.Linear(174, 256)
        self.l2 = torch.nn.Linear(256, 128)
        self.l3 = torch.nn.Linear(128, 64)
        self.l4 = torch.nn.Linear(64, 8)
        self.l5 = torch.nn.Linear(8, 3)
        
    def forward(self, x):
        x = x.view(-1, 174) # -1自动计算N,将(N,1, 28, 28) 变成 (N, 784)
        x = F.relu(self.l1(x))
        x = F.relu(self.l2(x))
        x = F.relu(self.l3(x))
        x = F.relu(self.l4(x))
        return self.l5(x) # 最后一个不做激活
model = Net()

criterion = torch.nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr = 0.01, momentum = 0.5) # momentum：带冲量的优化算法

#for i_batch, batch_data in enumerate(train_loader):
#   print(i_batch)  # 打印batch编号
#   print(batch_data[0])  # 打印该batch里面src
#   print(batch_data[1])  # 打印该batch里面trg

train_label

def train(epoch):
    running_loss = 0.0
    for batch_idx, data in enumerate(train_loader, 0):
        inputs, target = data  # 多分类中标签用数字的时候标签要从0开始算起
        optimizer.zero_grad() # 优化器清零
        
        outputs = model(inputs) # 获得预测值，forward
        loss = criterion(outputs, target) # 获得损失值
        loss.backward() # backward
        optimizer.step() # update
        
        running_loss += loss.item() # 注意加上item：不构成计算图
        if batch_idx % 300 == 299:
            print('[%d, %5d] loss: %.3f'%(epoch + 1, batch_idx + 1, running_loss / 300))
            running_loss = 0.0
            
def test():
    correct = 0
    total = 0
    with torch.no_grad(): # 不需要计算梯度
        for data in test_loader:
            features, labels = data
            outputs = model(features)
            _, predicted = torch.max(outputs.data, dim = 1) # 每一行的最大值下标
            total += labels.size(0) # 测试了多少个数据
            correct += (predicted == labels).sum().item() # 计算有多少个预测正确
    print('Accuuracy on test set: %d %%' % (100 * correct / total)) # 输出正确率
    
for epoch in range(10):
    train(epoch)
    test()