# -*- encoding: utf-8 -*-

from openerp import models
import base64
from cStringIO import StringIO
from xlwt import Workbook, easyxf
import time

class grp_pep_reporte_necesidades(models.AbstractModel):
    _name = 'grp.pep.reporte.necesidades'

    def generar_excel(self, wizard):
        plan = self.env['grp.pep.anual'].browse(wizard.plan_anual_id.id)
        ids = [x.concepto_id.id for x in plan.lineas_concepto]
        conceptos = self.env['grp.pep.concepto'].browse(ids)
        datos = conceptos.get_producto_necesidad_actualizada()

        lista_productos = []
        for producto_id, necesidad in datos.iteritems():
            producto = self.env['product.product'].browse(producto_id)
            lista_productos.append({'producto': producto.display_name,
                                    'sice': producto.grp_sice_cod,
                                    'unidad': producto.uom_id.name,
                                    'necesidad': necesidad})
        # Creo el libro Excel
        wb = Workbook(encoding='utf-8')
        ws = wb.add_sheet('Sheet 1', cell_overwrite_ok=True)

        ws.protect = False
        ws.password = "/Dd3R$eg(&(=:.$"

        # Creo los 'estilos'
        title_left_bloqueado = easyxf('font: name Calibri, bold True; alignment: horizontal center;')
        header_left_bloqueado = easyxf('font: name Calibri, bold True; alignment: horizontal left;'
                                       ' pattern: pattern solid, fore_colour 0x16;')
        texto_bloqueado = easyxf('font: name Calibri; alignment: horizontal left;')
        texto_editable = easyxf('font: name Calibri; alignment: horizontal left;'
                                ' protection: cell_locked false;')
        integer_editable = easyxf('font: name Calibri; alignment: horizontal right;'
                                  ' protection: cell_locked false;')
        fila = 0

        # Escribo el titulo
        ws.write_merge(fila, 0, fila, 6, u"REPORTE DE NECESIDADES", title_left_bloqueado)
        fila = fila + 1
        ws.write(fila, 5, u"Fecha del reporte", texto_bloqueado)
        ws.write(fila, 6, time.strftime('%d/%m/%Y'), texto_bloqueado)
        fila = fila + 1
        ws.write(fila, 0, u"Nombre", texto_bloqueado)
        ws.write(fila, 1, plan.name, texto_bloqueado)
        fila = fila + 1
        ws.write(fila, 0, u"Año fiscal", texto_bloqueado)
        ws.write(fila, 1, plan.anio_fiscal.name, texto_bloqueado)

        # Cabezal
        fila = fila + 2
        ws.write(fila, 0, u"Producto", header_left_bloqueado)
        ws.write(fila, 1, u"Código SICE", header_left_bloqueado)
        ws.write(fila, 2, u"Unidad de medida", header_left_bloqueado)
        ws.write(fila, 3, u"Cantidad", header_left_bloqueado)
        fila = fila + 1
        #Líneas
        for linea in lista_productos:
            ws.write(fila, 0, linea['producto'], texto_editable)
            ws.write(fila, 1, linea['sice'], integer_editable)
            ws.write(fila, 2, linea['unidad'], texto_editable)
            ws.write(fila, 3, linea['necesidad'], integer_editable)
            fila = fila + 1

        # Salvo hacia un string IO
        fp = StringIO()
        wb.save(fp)

        # Guardo la planilla en un string
        fp.seek(0)
        data = fp.read()

        # Codifico a base 64
        data_to_save = base64.encodestring(data)

        # Creo el Wizard de Descarga con el archivo
        fp.close()
        file_name = 'Export_Necesidades_' + plan.name + '.xls'
        wizard.write({'archivo_nombre': file_name,
                      'archivo_contenido': data_to_save})

grp_pep_reporte_necesidades()


import os
from copy import deepcopy

import torch
import torch.optim as optim
from torch.autograd import Variable
from torchvision import models

import CNN_model
from _utils import prediction, mean_average_precision

def CalcSim(batch_label, train_label):
    S = (batch_label.mm(train_label.t()) > 0).type(torch.FloatTensor)
    return S

def CreateModel(model_name, bit, use_gpu):
    if model_name == 'vgg11':
        vgg11 = models.vgg11(pretrained=True)
        cnn_model = CNN_model.cnn_model(vgg11, model_name, bit)
    if model_name == 'alexnet':
        alexnet = models.alexnet(pretrained=True)
        cnn_model = CNN_model.cnn_model(alexnet, model_name, bit)
    if use_gpu:
        cnn_model = cnn_model.cuda()
    return cnn_model

def AdjustLearningRate(optimizer, epoch, learning_rate):
    lr = learning_rate * (0.1 ** (epoch // 50))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr
    return optimizer

def Logtrick(x, use_gpu):
    if use_gpu:
        lt = torch.log(1 + torch.exp(-torch.abs(x))) + torch.max(x, Variable(torch.FloatTensor([0.]).cuda()))
    else:
        lt = torch.log(1 + torch.exp(-torch.abs(x))) + torch.max(x, Variable(torch.FloatTensor([0.])))
    return lt

def Totloss(U, B, Sim, lamda, num_train):
    theta = U.mm(U.t()) / 2
    t1 = (theta * theta).sum() / (num_train * num_train)
    l1 = (- theta * Sim + Logtrick(Variable(theta), False).data).sum()
    l2 = (U - B).pow(2).sum()
    l = l1 + lamda * l2
    return l, l1, l2, t1

def DPSH_algo(param, train_loader, test_loader, database_loader, logger, gpu_ind=0):
    # parameters setting
    os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_ind)

    learning_rate = param["learning_rate"]
    weight_decay = param["weight_decay"]
    lamda = param['lambda']
    use_gpu = torch.cuda.is_available()

    ### create model
    model = CreateModel(param['model'], param['bit'], use_gpu)
    optimizer = optim.SGD(
        model.parameters(),
        lr=learning_rate,
        weight_decay=weight_decay)

    ### training phase
    # parameters setting
    num_train = len(train_loader.dataset)
    B = torch.zeros(num_train, param['bit'])
    U = torch.zeros(num_train, param['bit'])
    train_labels = train_loader.dataset.get_onehot_targets()

    totloss_record = []
    totl1_record = []
    totl2_record = []
    t1_record = []

    Sim = CalcSim(train_labels, train_labels)

    best_map = 0.0
    best_epoch = 0
    best_checkpoint = None
    count = 0
    for epoch in range(param['epochs']):
        model.train()
        epoch_loss = 0.0
        ## training epoch
        for train_input, train_label, batch_ind in train_loader:
            if use_gpu:
                S = CalcSim(train_label, train_labels)
                train_input = Variable(train_input.cuda())
            else:
                S = CalcSim(train_label, train_labels)
                train_input = Variable(train_input)

            model.zero_grad()
            train_outputs = model(train_input)
            for i, ind in enumerate(batch_ind):
                U[ind, :] = train_outputs.data[i]
                B[ind, :] = torch.sign(train_outputs.data[i])

            Bbatch = torch.sign(train_outputs)
            if use_gpu:
                theta_x = train_outputs.mm(Variable(U.cuda()).t()) / 2
                logloss = (Variable(S.cuda()) * theta_x - Logtrick(theta_x, use_gpu)).sum() / (
                        num_train * len(train_label))
                regterm = (Bbatch - train_outputs).pow(2).sum() / (num_train * len(train_label))
            else:
                theta_x = train_outputs.mm(Variable(U).t()) / 2
                logloss = (Variable(S) * theta_x - Logtrick(theta_x, use_gpu)).sum() / (num_train * len(train_label))
                regterm = (Bbatch - train_outputs).pow(2).sum() / (num_train * len(train_label))

            loss = - logloss + lamda * regterm
            loss.backward()
            optimizer.step()
            epoch_loss += loss.item()

        logger.info('[Train Phase][Epoch: %3d/%3d][Loss: %3.5f]' % (epoch, param['epochs'] - 1, epoch_loss / num_train))
        optimizer = AdjustLearningRate(optimizer, epoch, learning_rate)

        l, l1, l2, t1 = Totloss(U, B, Sim, lamda, num_train)
        totloss_record.append(l)
        totl1_record.append(l1)
        totl2_record.append(l2)
        t1_record.append(t1)

        logger.info('[Total Loss: %10.5f][total L1: %10.5f][total L2: %10.5f][norm theta: %3.5f]' % (l, l1, l2, t1))

        ## testing during epoch
        if (epoch + 1) % 5 == 0 or (epoch + 1) == param['epochs']:
            qB, qL = prediction(model, test_loader)
            rB, rL = prediction(model, database_loader)
            map = mean_average_precision(qB, rB, qL, rL, param['topk'])
            logger.info(
                f"[Evaluation][dataset:{param['dataset']}][bits:{param['bit']}][epoch:{epoch}/{param['epochs'] - 1}][best-mAP@{param['topk']}:{best_map:.7f}][mAP@{param['topk']}:{map:.7f}][count:{0 if map > best_map else (count + 1)}]")

            if map > best_map:
                best_map = map
                best_epoch = epoch
                best_checkpoint = deepcopy(model.state_dict())
                count = 0
            else:
                count += 1
                if count == 10:
                    logger.info(
                        f"without improvement, will save & exit, best mAP: {best_map}, best epoch: {best_epoch}")
                    torch.save(best_checkpoint, f"{param['save_dir']}/e{best_epoch}_{best_map:.3f}.pkl")
                    break
    if count != 10:
        logger.info(f"reach epoch limit, will save & exit, best mAP: {best_map}, best epoch: {best_epoch}")
        torch.save(best_checkpoint, f"{param['save_dir']}/e{best_epoch}_{best_map:.3f}.pkl")

    return best_epoch, best_map


from collections import deque

class TreeNode:
    def __init__(self, val, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

def revLevelOrderTraversal(root):
    # res = []
    res = deque()

    if root == None:
        return res

    # to maintain a list of nodes to be traversed
    # queue for first in and first out
    # queue = []
    queue = deque()
    # starting from root
    queue.append(root)

    while len(queue) != 0:
        # no. of nodes to be traversed in this iteration
        levelSize = len(queue)
        # to store the final total no. of nodes in current level
        currentLevel = []

        for _ in range(levelSize):
            # removing current node from the queue
            # currentNode = queue.pop(0)
            currentNode = queue.popleft()
            # Appending it to the current level
            currentLevel.append(currentNode.val)

            # storing the children of current node in queue for future
            if(currentNode.left != None):
                queue.append(currentNode.left)
            if(currentNode.right != None):
                queue.append(currentNode.right)

        # the current level ends here
        # this is the only difference b/w level order traversal and reverse
        # insert the latest level at the start of res
        # res.insert(0, currentLevel)
        res.appendleft(currentLevel)

    return res

root = TreeNode(1)
root.left = TreeNode(2)
root.right = TreeNode(3)
root.left.left = TreeNode(4)
root.left.right = TreeNode(5)
root.right.left = TreeNode(6)
root.right.right = TreeNode(7)

print(revLevelOrderTraversal(root))

7. Tree Breadth First Search/2_Reverse level order traversal (medium).py


import glfw
import logging
from input import Input
import configuration.programPipeline as programPipeline
from OpenGL.GL import *

logging.basicConfig()

# Makes a window via glfw and renders to it with Opengl
class Window:
    def __init__(self, width, height):
        if width <= 0 or height <= 0:
            raise Exception("Invalid size "+width+","+height)
        self.width = width
        self.height = height
        glfw.set_error_callback(self.error_callback)
        if not glfw.init():
            raise Exception("GLFW couldn't initialize.")
        self.window = self.glfw_create_window(self.width, self.height, "Character Creator")
        if not self.window:
            glfw.terminate()
            raise Exception("GLFW window couldn't initialize.")
        glfw.make_context_current(self.window)
        self.input = Input()
        glfw.set_key_callback(self.window, self.input.key_callback)
        glfw.set_mouse_button_callback(self.window, self.input.mouse_button_callback)
        glfw.set_window_size_callback(self.window, self.window_size_callback)
        glfw.set_cursor_pos_callback(self.window, self.input.cursor_position_callback)
        self.print_opengl_data()
        glViewport(0, 0, self.width, self.height)
        glEnable(GL_DEPTH_TEST)
        self.pipeline = programPipeline.get_cylinder_pipeline(self)

    def window_size_callback(self, window, width, height):
        logging.info("dimensions are now "+str(width)+", "+str(height))
        self.width = width
        self.height = height
        glViewport(0, 0, self.width, self.height)

    def run(self):
        program_data = {'input': self.input}
        while not glfw.window_should_close(self.window):
            glfw.swap_buffers(self.window)
            glfw.poll_events()
            for program in self.pipeline:
                program.run(program_data)
        glfw.terminate()

    @staticmethod
    def glfw_create_window(width, height, window_name):
        glfw.window_hint(glfw.OPENGL_FORWARD_COMPAT, True)
        glfw.window_hint(glfw.OPENGL_PROFILE, glfw.OPENGL_COMPAT_PROFILE)
        glfw.window_hint(glfw.CONTEXT_VERSION_MAJOR, 4)
        glfw.window_hint(glfw.CONTEXT_VERSION_MINOR, 4)
        return glfw.create_window(width, height, window_name, None, None)

    @staticmethod
    def print_opengl_data():
        print("OpenGL version: " + str(glGetString(GL_VERSION)))
        print("GLSL version: " + str(glGetString(GL_SHADING_LANGUAGE_VERSION)))
        print("OpenGL Vendor: " + str(glGetString(GL_VENDOR)))
        print("Renderer: " + str(glGetString(GL_RENDERER)))

    @staticmethod
    def error_callback(code, description):
        raise Exception("GLFW Exception "+code+": "+description)


from functionUtils import *
from sklearn.preprocessing import LabelEncoder
from tqdm import tqdm_notebook, tnrange
from workalendar.america import Brazil
from datetime import date, timedelta
import pandas as pd
import numpy as np
import warnings
import workalendar
import datetime
import gc

DATA_PATH = '../../input/'

print('read data...')
df_train = pd.read_csv(DATA_PATH + 'train.csv')
df_test = pd.read_csv(DATA_PATH + 'test.csv')
df_historical = pd.read_csv(DATA_PATH + 'historical_transactions.csv', dtype={'purchase_date': np.str}, low_memory=True)
df_new_merchant = pd.read_csv(DATA_PATH + 'new_merchant_transactions.csv', dtype={'purchase_date': np.str},
                              low_memory=True)

df_train = reduce_mem_usage(df_train)
df_test = reduce_mem_usage(df_test)
df_historical = reduce_mem_usage(df_historical)
df_new_merchant = reduce_mem_usage(df_new_merchant)

df_train['is_test'] = 0
df_test['is_test'] = 1
# 用户在历史和之后的记录中是否出现过
for df in [df_train, df_test]:
    df['is_in_new_monthlag1'] = df['card_id'].isin(df_new_merchant[df_new_merchant.month_lag == 1].card_id).astype(int)
    df['is_in_new_monthlag2'] = df['card_id'].isin(df_new_merchant[df_new_merchant.month_lag == 2].card_id).astype(int)
df_data = pd.concat([df_train, df_test])
df_data = pd.get_dummies(df_data, columns=['feature_1', 'feature_2', 'feature_3'],
                         prefix=['feature_1', 'feature_2', 'feature_3'])
df_transactions = pd.concat([df_historical, df_new_merchant])

df_transactions['purchase_amount'] = np.round(df_transactions['purchase_amount'] / 0.00150265118 + 497.06, 2)

## id类特征
print('id类特征')
df_temp = df_historical.card_id.value_counts().reset_index()
df_temp.rename(columns={'index': 'card_id', 'card_id': 'hist_card_id_counts'}, inplace=True)
df_temp['hist_card_id_ratio'] = df_temp['hist_card_id_counts'] / df_historical.shape[0]
df_data = df_data.merge(df_temp, on='card_id', how='left')

df_temp = df_new_merchant.card_id.value_counts().reset_index()
df_temp.rename(columns={'index': 'card_id', 'card_id': 'new_card_id_counts'}, inplace=True)
df_temp['new_card_id_ratio'] = df_temp['new_card_id_counts'] / df_new_merchant.shape[0]
df_data = df_data.merge(df_temp, on='card_id', how='left')

df_data['hist/new_card_counts_ratio'] = df_data['hist_card_id_counts'] / df_data['new_card_id_counts']

df_data[['hist_card_id_counts', 'hist_card_id_ratio', 'new_card_id_counts', 'new_card_id_ratio',
         'hist/new_card_counts_ratio']].fillna(0, inplace=True)

del df_temp
gc.collect()

dateHandle = pd.to_datetime(df_data['first_active_month'])
df_data['active_year'] = dateHandle.dt.year
df_data['active_month'] = dateHandle.dt.month
df_data['active_to_base_time'] = (datetime.date(2018, 2, 28) - dateHandle.dt.date).dt.days

print('刷卡时间...')
# hist第一次刷卡时间
df_temp = df_historical.groupby('card_id')['purchase_date'].min().reset_index()
df_temp.rename(columns={'purchase_date': 'hist_first_purchase_date'}, inplace=True)
df_data = df_data.merge(df_temp, on='card_id', how='left')
# 最后一次刷卡时间
df_temp = df_historical.groupby('card_id')['purchase_date'].max().reset_index()
df_temp.rename(columns={'purchase_date': 'hist_last_purchase_date'}, inplace=True)
df_data = df_data.merge(df_temp, on='card_id', how='left')

# new第一次刷卡时间
df_temp = df_new_merchant.groupby('card_id')['purchase_date'].min().reset_index()
df_temp.rename(columns={'purchase_date': 'new_first_purchase_date'}, inplace=True)
df_data = df_data.merge(df_temp, on='card_id', how='left')
# 最后一次刷卡时间
df_temp = df_new_merchant.groupby('card_id')['purchase_date'].max().reset_index()
df_temp.rename(columns={'purchase_date': 'new_last_purchase_date'}, inplace=True)
df_data = df_data.merge(df_temp, on='card_id', how='left')

# hist-new 购买时长
df_data['hist_first_to_base_time'] = (
        pd.to_datetime(df_data['hist_first_purchase_date']).dt.date - datetime.date(2018, 2, 28)).dt.days
df_data['hist_last_to_base_time'] = (
        pd.to_datetime(df_data['hist_last_purchase_date']).dt.date - datetime.date(2018, 2, 28)).dt.days
df_data['hist_last_to_first_time'] = (pd.to_datetime(df_data['hist_last_purchase_date']).dt.date - pd.to_datetime(
    df_data['hist_first_purchase_date']).dt.date).dt.days
df_data['hist_first_active_time'] = (pd.to_datetime(df_data['hist_first_purchase_date']).dt.date - pd.to_datetime(
    df_data['first_active_month']).dt.date).dt.days
df_data['hist_last_active_time'] = (pd.to_datetime(df_data['hist_last_purchase_date']).dt.date - pd.to_datetime(
    df_data['first_active_month']).dt.date).dt.days

df_data['new_first_to_base_time'] = (
        pd.to_datetime(df_data['new_first_purchase_date']).dt.date - datetime.date(2018, 2, 28)).dt.days
df_data['new_last_to_base_time'] = (
        pd.to_datetime(df_data['new_last_purchase_date']).dt.date - datetime.date(2018, 2, 28)).dt.days
df_data['new_first_to_active_time'] = (pd.to_datetime(df_data['new_first_purchase_date']).dt.date - pd.to_datetime(
    df_data['first_active_month']).dt.date).dt.days
df_data['new_last_to_active_time'] = (pd.to_datetime(df_data['new_last_purchase_date']).dt.date - pd.to_datetime(
    df_data['first_active_month']).dt.date).dt.days
df_data['new_last_to_first_time'] = (pd.to_datetime(df_data['new_last_purchase_date']).dt.date - pd.to_datetime(
    df_data['new_first_purchase_date']).dt.date).dt.days

# hist最后一次购买距离new第一次购买的时间差
df_data['new_to_hist_time'] = (pd.to_datetime(df_data['new_first_purchase_date']).dt.date -
                               pd.to_datetime(df_data['hist_last_purchase_date']).dt.date).dt.days
# 单位时间内的刷卡次数
df_data['hist_per_time_purchaseCounts'] = df_data['hist_card_id_counts'] / df_data['hist_last_to_first_time']
df_data['new_per_time_purchaseCounts'] = df_data['new_card_id_counts'] / df_data['new_last_to_first_time']

df_data.drop(columns=['hist_last_purchase_date', 'hist_first_purchase_date', 'new_first_purchase_date',
                      'new_last_purchase_date'], inplace=True)
df_data.replace([np.inf, -np.inf], 0, inplace=True)
df_data.fillna(-1, inplace=True)
del df_train, df_test, df_historical, df_new_merchant, df_temp
gc.collect()

def dateUtils(df=None, timeCol='purchase_date'):
    dateHandle = pd.to_datetime(df[timeCol])
    df['week'] = dateHandle.dt.week
    df['year'] = dateHandle.dt.year
    df['month'] = dateHandle.dt.month
    df['dayofweek'] = dateHandle.dt.dayofweek
    df['weekend'] = (dateHandle.dt.weekday >= 5).astype(int)
    df['hour'] = dateHandle.dt.hour
    df['month_gap'] = (dateHandle.dt.date - datetime.date(2018, 2, 28)).dt.days // 30
    df['day_gap'] = (dateHandle.dt.date - datetime.date(2018, 2, 28)).dt.days
    # cardid用户连续购买之间的时间差
    df['purchase_time'] = dateHandle
    df['purchase_time_diff_days'] = df.groupby('card_id')['purchase_time'].apply(lambda series: series.diff(1)).dt.days
    df['purchase_time_diff_days'].fillna(df.purchase_time_diff_days.mean(), inplace=True)
    df['purchase_time_diff_seconds'] = df.groupby('card_id')['purchase_time'].apply(
        lambda series: series.diff(1)).dt.seconds
    df['purchase_time_diff_seconds'].fillna(df.purchase_time_diff_seconds.mean(), inplace=True)
    df.drop(columns=['purchase_time'], inplace=True)

    def getdate2Holiday(d):
        cal = Brazil()
        holiday = cal.holidays(d.year)
        dis, flag = 0, 0
        year, month, day = d.year, d.month, d.day
        d = datetime.date(year, month, day)
        for i, h in enumerate(holiday):
            if holiday[i][0] > d:
                flag = 1
                dis = (holiday[i][0] - d).days
                return dis
        if flag == 0:
            dis = (holiday[-1][0] - d).days
        return dis

    df['date2Holiday'] = df['purchase_date'].apply(lambda x: getdate2Holiday(pd.to_datetime(x)))

    return df

# 缺失值处理
# C2:区域；C3分期等级；C1为-1城市编码对应Y
df_transactions['category_2'].fillna(6, inplace=True)
df_transactions['category_3'].fillna('B', inplace=True)
df_transactions['merchant_id'].fillna(df_transactions['merchant_id'].value_counts().index[0], inplace=True)
df_transactions['installments'].replace(-1, 1, inplace=True)
df_transactions['installments'].replace(999, 0, inplace=True)

# 交互特征
df_transactions['city_C1'] = ((df_transactions['city_id'] < 0) + 0).astype(np.str) + '_' + df_transactions[
    'category_1'].astype(np.str)
df_transactions['C2_state'] = df_transactions['category_2'].astype(np.str) + '_' + df_transactions['state_id'].astype(
    np.str)
df_transactions['C2_state_subsector'] = df_transactions['category_2'].astype(np.str) + '_' + df_transactions[
    'state_id'].astype(np.str) + '_' + df_transactions['subsector_id'].astype(np.str)
df_transactions['subsector_city'] = df_transactions['subsector_id'].astype(np.str) + '_' + (
        (df_transactions['city_id'] < 0) + 0).astype(np.str)
df_transactions['auth_C3'] = df_transactions['authorized_flag'].astype(np.str) + '_' + df_transactions[
    'category_3'].astype(np.str)

cateCols = ['city_C1', 'C2_state', 'C2_state_subsector', 'subsector_city', 'auth_C3']
print('label_encoding...')
df_transactions = label_encoding(df_transactions, cateCols)

print('category_3...')
for cate in ['A', 'B', 'C']:
    df_transactions['category_3_%s' % cate] = (df_transactions['category_3'] == cate) + 0
print('category_2...')
for cate in [1, 2, 3, 4, 5, 6]:
    df_transactions['category_2_%s' % cate] = (df_transactions['category_2'] == cate) + 0
print('authorized_flag...')
for cate in ['Y', 'N']:
    df_transactions['authorized_flag_%s' % cate] = (df_transactions['authorized_flag'] == cate) + 0

print('map...')
df_transactions['category_1'] = df_transactions['category_1'].map({'Y': 1, 'N': 0})
df_transactions['category_3'] = df_transactions['category_3'].map({'A': 0, 'B': 1, 'C': 2})
df_transactions['authorized_flag'] = df_transactions['authorized_flag'].map({'Y': 1, 'N': 0})

df_transactions.sort_values(by=['card_id', 'purchase_date'], ascending=True, inplace=True)
print('date utils...')
df_transactions = dateUtils(df_transactions, timeCol='purchase_date')
df_transactions = reduce_mem_usage(df_transactions)

df_transactions.sort_values(by=['card_id', 'purchase_date'], ascending=True, inplace=True)
# df_transactions['purchase_diff'] = df_transactions.groupby('card_id')['purchase_amount'].apply(lambda series:series.diff(1)).values
# df_transactions['purchase_diff'].fillna(0,inplace=True)
df_hist_transactions = df_transactions[df_transactions.month_lag <= 0]
df_new_transactions = df_transactions[df_transactions.month_lag > 0]
for df in [df_hist_transactions, df_new_transactions]:
    df['purchase_diff'] = df.groupby('card_id')['purchase_amount'].apply(lambda series: series.diff(1)).values
    df['purchase_diff'].fillna(0, inplace=True)
df_hist_auth_Y_transactions = df_hist_transactions[df_hist_transactions.authorized_flag == 1]
df_hist_auth_N_transactions = df_hist_transactions[df_hist_transactions.authorized_flag == 0]

# 用户月消费记录
def getMonthPurchase(df_data, df_feature, group='month_gap', fea='purchase_amount', name='hist'):
    df_purchase = df_feature.groupby(['card_id', 'month_gap'])['purchase_amount'].sum().reset_index()
    df_purchase.rename(columns={'purchase_amount': 'purchase_amount_sum_month'}, inplace=True)
    df_purchase.sort_values(by=['month_gap'], inplace=True, ascending=True)

    df_temp = df_purchase[['card_id', 'month_gap', 'purchase_amount_sum_month']]
    df_temp.index = df_temp.card_id
    df_temp = df_temp.set_index(['month_gap'], append=True)
    df_temp = pd.Series(df_temp['purchase_amount_sum_month'].values.reshape(len(df_temp['purchase_amount_sum_month'])),
                        index=df_temp.index)
    df_temp = df_temp.unstack()
    df_temp.reset_index(inplace=True)
    cols = ['card_id']
    for index in list(df_purchase['month_gap'].unique()):
        cols.append('%s_month%s_purchase' % (name, index))
    df_temp.columns = cols
    df_temp.fillna(0, inplace=True)
    df_data = df_data.merge(df_temp, on='card_id', how='left')

    del df_purchase, df_temp, cols
    gc.collect()

    return df_data

# 周消费记录
def getWeekPurchase(df_data, df_feature, group='week', fea='purchase_amount', name='hist'):
    df_purchase = df_feature.groupby(['card_id', 'week'])['purchase_amount'].sum().reset_index()
    df_purchase.rename(columns={'purchase_amount': 'purchase_sum_week'}, inplace=True)
    df_purchase.sort_values(by=['week'], inplace=True, ascending=True)
    df_count = df_purchase.groupby(['card_id'])['purchase_sum_week'].count().reset_index()
    df_count.rename(columns={'purchase_sum_week': '%s_purchase_counts' % name}, inplace=True)

    df_temp = df_purchase[['card_id', 'week', 'purchase_sum_week']]
    df_temp.index = df_temp.card_id
    df_temp = df_temp.set_index(['week'], append=True)
    df_temp = pd.Series(df_temp['purchase_sum_week'].values.reshape(len(df_temp['purchase_sum_week'])),
                        index=df_temp.index)
    df_temp = df_temp.unstack()
    df_temp.reset_index(inplace=True)
    cols = ['card_id']
    for index in list(df_purchase['week'].unique()):
        cols.append('%s_week%s_purchase' % (name, index))
    df_temp.columns = cols
    df_temp = df_temp.merge(df_count, on='card_id', how='left')
    df_temp.fillna(0, inplace=True)
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    return df_data

# 用户月分期记录
def getMonthInstallments(df_data, df_feature, group='month_gap', fea='installments', name='hist'):
    df_purchase = df_feature.groupby(['card_id', 'month_gap'])['installments'].sum().reset_index()
    df_purchase.rename(columns={'installments': 'installments_sum_month'}, inplace=True)
    df_purchase.sort_values(by=['month_gap'], inplace=True, ascending=True)

    df_temp = df_purchase[['card_id', 'month_gap', 'installments_sum_month']]
    df_temp.index = df_temp.card_id
    df_temp = df_temp.set_index(['month_gap'], append=True)
    df_temp = pd.Series(df_temp['installments_sum_month'].values.reshape(len(df_temp['installments_sum_month'])),
                        index=df_temp.index)
    df_temp = df_temp.unstack()
    df_temp.reset_index(inplace=True)
    cols = ['card_id']
    for index in list(df_purchase['month_gap'].unique()):
        cols.append('%s_month%s_installments' % (name, index))
    df_temp.columns = cols
    df_temp.fillna(0, inplace=True)
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    del df_purchase, df_temp, cols
    gc.collect()
    return df_data

# 用户周分期记录
def getWeekInstallments(df_data, df_feature, group='week', fea='installments', name='hist'):
    df_purchase = df_feature.groupby(['card_id', 'week'])['installments'].sum().reset_index()
    df_purchase.rename(columns={'installments': 'installments_sum_week'}, inplace=True)
    df_purchase.sort_values(by=['week'], inplace=True, ascending=True)

    df_temp = df_purchase[['card_id', 'week', 'installments_sum_week']]
    df_temp.index = df_temp.card_id
    df_temp = df_temp.set_index(['week'], append=True)
    df_temp = pd.Series(df_temp['installments_sum_week'].values.reshape(len(df_temp['installments_sum_week'])),
                        index=df_temp.index)
    df_temp = df_temp.unstack()
    df_temp.reset_index(inplace=True)
    cols = ['card_id']
    for index in list(df_purchase['week'].unique()):
        cols.append('%s_week%s_installments' % (name, index))
    df_temp.columns = cols
    df_temp.fillna(0, inplace=True)
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    del df_purchase, df_temp, cols
    gc.collect()
    return df_data

df_data = getMonthPurchase(df_data, df_hist_auth_Y_transactions, name='hist_auth_Y')
df_data = getMonthPurchase(df_data, df_hist_auth_N_transactions, name='hist_auth_N')
df_data = getMonthPurchase(df_data, df_new_transactions, name='new')
# df_data = getMonthInstallments(df_data,df_hist_transactions,name='hist')
# df_data = getMonthInstallments(df_data,df_new_transactions,name='new')

# df_data = getWeekPurchase(df_data,df_hist_auth_Y_transactions,name='hist_auth_Y')
# df_data = getWeekPurchase(df_data,df_hist_auth_N_transactions,name='hist_auth_N')
# df_data = getWeekPurchase(df_data,df_new_transactions,name='new')
# df_data = getWeekInstallments(df_data,df_hist_transactions,name='hist')
# df_data = getWeekInstallments(df_data,df_new_transactions,name='new')

df_data.replace([np.inf, -np.inf], np.nan, inplace=True)
df_data.fillna(0, inplace=True)

### tmp01 to_csv
# df_data.to_csv('tmp01_df_data.csv',index=False)
# df_transactions.to_csv('tmp01_df_transactions.csv',index=False)

originalCols = list(df_data.columns)

###持卡人不同授权类型的购买总量
def getAuthorizedPurchase(df_data, df_features, name='hist'):
    df_temp = df_features.groupby(['card_id', 'authorized_flag'])['purchase_amount'].sum().reset_index()
    df_temp = df_temp.pivot(index='card_id', columns='authorized_flag', values='purchase_amount')
    cols = []
    for col in df_temp.columns:
        cols.append(name + df_temp.columns.name + '_' + np.str(col) + '_purchasesum')
    df_temp.columns = cols
    df_temp.columns.name = None
    df_temp.reset_index(inplace=True)
    df_temp.fillna(0, inplace=True)
    if name == 'hist':
        df_temp['%s_Auth_purchase_sum' % name] = df_temp[cols[0]] + df_temp[cols[1]]
        df_temp[cols[0] + '_ratio'] = df_temp[cols[0]] / df_temp['%s_Auth_purchase_sum' % name]
        df_temp[cols[1] + '_ratio'] = df_temp[cols[1]] / df_temp['%s_Auth_purchase_sum' % name]
        df_temp.drop(columns=['%s_Auth_purchase_sum' % name], inplace=True)
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    del df_temp
    gc.collect()
    return df_data

#### 用户对商家的回购率
def getUserRepurchaseRatio(df_data, df_fea, group=['card_id', 'merchant_id'], fea='purchase_amount',
                           name='repurchase_mechant_ratio'):
    df_temp = df_features.groupby(group)[fea].count().reset_index().rename(
        columns={fea: 'purchase_amount_counts'})
    df_temp['repurchase_flag'] = (df_temp['purchase_amount_counts'] > 1) + 0
    df = df_temp.groupby(['card_id'])['purchase_amount_counts'].sum().reset_index().rename(
        columns={'purchase_amount_counts': 'purchase_records'})
    df_temp['repurchase_counts'] = df_temp['repurchase_flag'] * df_temp['purchase_amount_counts']
    df_temp = df_temp.merge(df, on='card_id', how='left')
    df_temp['repurchase_ratio'] = df_temp['repurchase_counts'] / df_temp['purchase_records']
    df_temp = df_temp.groupby('card_id')['repurchase_ratio'].max().reset_index().rename(
        columns={'repurchase_ratio': name})
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    del df, df_temp
    gc.collect()
    return df_data

for flag, df_features in zip(['hist_auth_Y', 'hist_auth_N', 'new'],
                             [df_hist_auth_Y_transactions, df_hist_auth_N_transactions, df_new_transactions]):
    print(".................%s......................" % flag)
    if 'hist' in flag:
        month_lag = [-1, -3, -5, -7, -13]
    else:
        month_lag = [1, 2]
    for index in month_lag:
        print('month_lag:', index)
        df = df_features[df_features.month_lag >= index]
        # 各个阶段的purchase_amount(强特)
        print('purchase_amount...')
        df_data = getMeanStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                         name='%s_purchaseAmountMean_%s' % (flag, index))
        df_data = getMaxStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                        name='%s_purchaseAmountMax_%s' % (flag, index))
        df_data = getMedianStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                           name='%s_purchaseAmountMedian_%s' % (flag, index))
        df_data = getSumStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                        name='%s_purchaseAmountSum_%s' % (flag, index))
        df_data = getStdStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                        name='%s_purcahseAmountStd_%s' % (flag, index))
        df_data = getCountsStaticsFeatures(df_data, df, ['card_id'], 'purchase_amount',
                                           name='%s_purcahseAmountCount_%s' % (flag, index))

        df_data['%s_per_time_purchaseAmountSum_%s' % (flag, index)] = df_data[
                                                                          '%s_purchaseAmountSum_%s' % (flag, index)] / (
                                                                          np.abs(index))
        df_data['%s_per_time_purcahseAmountCount_%s' % (flag, index)] = df_data['%s_purcahseAmountCount_%s' % (
            flag, index)] / (np.abs(index))

        # 分期数
        print('installments...')
        df_data = getMeanStaticsFeatures(df_data, df, ['card_id'], 'installments',
                                         name='%s_installmentsMean_%s' % (flag, index))
        df_data = getMaxStaticsFeatures(df_data, df, ['card_id'], 'installments',
                                        name='%s_installmentsMax_%s' % (flag, index))
        df_data = getSumStaticsFeatures(df_data, df, ['card_id'], 'installments',
                                        name='%s_installmentsSum_%s' % (flag, index))
        df_data = getCountsStaticsFeatures(df_data, df, ['card_id'], 'installments',
                                           name='%s_installmentsCount_%s' % (flag, index))

        df_data['%s_per_time_installmentsSum_%s' % (flag, index)] = df_data['%s_installmentsSum_%s' % (flag, index)] / (
            np.abs(index))
        df_data['%s_per_time_installmentsCount_%s' % (flag, index)] = df_data[
                                                                          '%s_installmentsCount_%s' % (flag, index)] / (
                                                                          np.abs(index))

        # purchase_diff(强特)
        print('purchase_diff...')
        df_data = getMeanStaticsFeatures(df_data, df, ['card_id'], 'purchase_diff',
                                         name='%s_purchaseDiffMean_%s' % (flag, index))
        df_data = getMaxStaticsFeatures(df_data, df, ['card_id'], 'purchase_diff',
                                        name='%s_purchaseDiffMax_%s' % (flag, index))
        df_data = getMedianStaticsFeatures(df_data, df, ['card_id'], 'purchase_diff',
                                           name='%s_purchaseDiffMedian_%s' % (flag, index))
        df_data = getMinStaticsFeatures(df_data, df, ['card_id'], 'purchase_diff',
                                        name='%s_purchaseDiffMin_%s' % (flag, index))
        df_data = getStdStaticsFeatures(df_data, df, ['card_id'], 'purchase_diff',
                                        name='%s_purcahseDiffStd_%s' % (flag, index))

        # day_diff
        #         df_data = getMaxStaticsFeatures(df_data,df_features,['card_id'],'day_diff',name='%s_dayDiffMax_%s'%(flag,index))
        #         df_data = getMinStaticsFeatures(df_data,df_features,['card_id'],'day_diff',name='%s_dayDiffMin_%s'%(flag,index))
        #         df_data = getCategoryFrequenceMax(df_data,df_features,['card_id'],'day_diff',name='%s_dayDiffFrequenceMax_%s'%(flag,index))

        # 信用卡分期类别特征
        print('category_3...')
        for cate in ['category_3_A', 'category_3_B', 'category_3_C']:
            df_data = getSumStaticsFeatures(df_data, df, ['card_id'], cate, name='%s_%s_sum_%s' % (flag, cate, index))

    if 'hist' in flag:
        histflag = 'hist'
    else:
        histflag = 'new'
    # 信用卡授权统计
    print('authorized_flag...')
    for cate in ['Y', 'N']:
        df_data = getSumStaticsFeatures(df_data, df_features, ['card_id'], 'authorized_flag_%s' % cate,
                                        name='%s_authorized_flag_%s_sum_%s' % (flag, cate, index))
        df_data['%s_per_time_authorized_flag_%s_sum_%s' % (flag, cate, index)] = (
                df_data['%s_authorized_flag_%s_sum_%s' % (flag, cate, index)] / df_data[
            '%s_last_to_first_time' % histflag])

    ##date2Holiday
    print('date2Holiday...')
    df_data = getMaxStaticsFeatures(df_data, df_features, ['card_id'], 'date2Holiday', name='%s_date2HolidayMax' % flag)
    df_data = getMinStaticsFeatures(df_data, df_features, ['card_id'], 'date2Holiday', name='%s_date2HolidayMin' % flag)
    df_data = getMeanStaticsFeatures(df_data, df_features, ['card_id'], 'date2Holiday', name='%s_date2Holiday' % flag)

    # AuthorizedPurchase
    print('AuthorizedPurchase...')
    df_data = getAuthorizedPurchase(df_data, df_features, name='%s_' % flag)
    # purchase_is_outlier
    #     df_data = getSumStaticsFeatures(df_data,df_features,['card_id'],'purchase_is_outlier',name='%s_purchaseOutlier_sum'%flag)
    # month
    df_data = getCategoryCounts(df_data, df_features, ['card_id'], 'month', name='%s_activeMonthCount' % flag)
    df_data = getCategoryFrequenceMax(df_data, df_features, ['card_id'], 'month', name='%s_activeMonthMax' % flag)
    df_data = getCategoryFrequenceMaxRatio(df_data, df_features, ['card_id'], 'month',
                                           name='%s_activeMonthMaxRatio' % flag)
    df_data['%s_per_time_activeMonthCount' % flag] = (df_data['%s_activeMonthCount' % flag] /
                                                      df_data['%s_last_to_first_time' % histflag])
    # month_lag
    print('month_lag...')
    df_data = getMinStaticsFeatures(df_data, df_features, ['card_id'], 'month_lag', name='%s_monthLagMin' % flag)
    df_data = getMaxStaticsFeatures(df_data, df_features, ['card_id'], 'month_lag', name='%s_monthLagMax' % flag)
    # month_gap
    print('month_gap...')
    df_data = getCategoryCounts(df_data, df_features, ['card_id'], 'month_gap',
                                name='%s_monthGap_categoryCounts' % flag)
    df_data = getCategoryCountsRatio(df_data, df_features, ['card_id'], 'month_gap',
                                     name='%s_monthGap_categoryCountsRatio' % flag)

    # day_gap
    df_data = getMaxStaticsFeatures(df_data, df_features, ['card_id'], 'day_gap', name='%s_dayGapMax' % flag)
    df_data = getMinStaticsFeatures(df_data, df_features, ['card_id'], 'day_gap', name='%s_dayGapMin' % flag)
    # 回购率
    df_data = getUserRepurchaseRatio(df_data, df_features, name='%s_repurchase_merchant_ratio' % flag)
    df_data = getUserRepurchaseRatio(df_data, df_features, group=['card_id', 'merchant_category_id'],
                                     name='%s_repurchase_merCate_ratio' % flag)
    # purchase shift比例
    print('purchase shift...')
    df_features['%s_purchase_shift' % flag] = df_features.groupby(['card_id'])['purchase_amount'].apply(
        lambda series: series.shift(1)).values
    df_features['%s_purchase_shift' % flag].fillna(0, inplace=True)
    df_features['%s_purchase_shift_ratio' % flag] = df_features['%s_purchase_shift' % flag] / df_features[
        'purchase_amount']
    df_temp = df_features.groupby(['card_id'])['%s_purchase_shift_ratio' % flag].max().reset_index()
    df_data = df_data.merge(df_temp, on='card_id', how='left')
    # hour
    print('hour...')
    df_data = getMeanStaticsFeatures(df_data, df_features, ['card_id'], 'hour', name='%s_HourMean_' % (flag))
    df_data = getSumStaticsFeatures(df_data, df_features, ['card_id'], 'hour', name='%s_HourSum_' % (flag))
    df_data = getCountsStaticsFeatures(df_data, df_features, ['card_id'], 'hour', name='%s_HourCounts_' % (flag))
    # weekend
    df_data = getSumStaticsFeatures(df_data, df_features, ['card_id'], 'weekend', name='%s_weekendSum_' % (flag))
    # dayofweek
    print('dayofweek...')
    df_data = getMeanStaticsFeatures(df_data, df_features, ['card_id'], 'dayofweek', name='%s_dayofweekMean_' % flag)
    df_data = getCategoryCounts(df_data, df_features, ['card_id'], 'dayofweek', name='%s_dayofWeekCounts_' % flag)

    for cate in [1, 2, 3, 4, 5, 6]:
        df_data = getSumStaticsFeatures(df_data, df_features, ['card_id'], 'category_2_%s' % cate,
                                        name='%s_category_2_%s_sum' % (flag, cate))
        df_data['%s_per_time_category_2_%s_sum' % (flag, cate)] = (
                df_data['%s_category_2_%s_sum' % (flag, cate)] / df_data['%s_last_to_first_time' % histflag])

    categoryCols = cateCols + ['authorized_flag', 'merchant_id', 'city_id', 'category_1', 'category_2', 'category_3',
                               'merchant_category_id', 'subsector_id']
    # purchase_time_diff
    print('purchase_time_diff...')
    df_data = getMinStaticsFeatures(df_data, df_features, ['card_id'], 'purchase_time_diff_days',
                                    name='%s_purchase_time_diff_days_min_' % (flag))
    df_data = getMinStaticsFeatures(df_data, df_features, ['card_id'], 'purchase_time_diff_seconds',
                                    name='%s_purchase_time_diff_seconds_min_' % (flag))

    for fea in ['purchase_time_diff_seconds', 'purchase_time_diff_days']:
        print(fea)
        df_data = getCategoryFrequenceMax(df_data, df_features, ['card_id'], fea,
                                          name='%s_%s_frequenceMax' % (flag, fea))
        df_data = getCategoryCounts(df_data, df_features, ['card_id'], fea,
                                    name='%s_%s_categoryCounts' % (flag, fea))
        df_data = getCategoryFrequenceMaxRatio(df_data, df_features, ['card_id'], fea,
                                               name='%s_%s_frequenceMaxRatio' % (flag, fea))
        df_data = getCategoryCountsRatio(df_data, df_features, ['card_id'], fea,
                                         name='%s_%s_categoryCountsRatio' % (flag, fea))
    for fea in categoryCols:
        print('categoryCols:', fea)
        df_data = getCategoryFrequenceMax(df_data, df_features, ['card_id'], fea,
                                          name='%s_%s_frequenceMax' % (flag, fea))
        df_data = getCategoryCounts(df_data, df_features, ['card_id'], fea,
                                    name='%s_%s_categoryCounts' % (flag, fea))
        df_data = getCategoryFrequenceMaxRatio(df_data, df_features, ['card_id'], fea,
                                               name='%s_%s_frequenceMaxRatio' % (flag, fea))
        df_data = getCategoryCountsRatio(df_data, df_features, ['card_id'], fea,
                                         name='%s_%s_categoryCountsRatio' % (flag, fea))

# month_lag=0的消费平均消费水平
print('month_lag=0的消费平均消费水平。。。')
df_temp = df_hist_transactions[df_hist_transactions.month_lag == 0]
df_temp = df_temp.groupby(['card_id'])['purchase_amount'].mean().reset_index().rename(
    columns={'purchase_amount': 'hist_purchaseAmountMean_0'})
df_data = df_data.merge(df_temp, on='card_id', how='left')

del df_temp, df_hist_transactions, df_new_transactions
gc.collect()

df_data['avg_purchase_ratio_-1'] = df_data['hist_auth_Y_purchaseAmountMean_-1'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_-3'] = df_data['hist_auth_Y_purchaseAmountMean_-3'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_-5'] = df_data['hist_auth_Y_purchaseAmountMean_-5'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_-7'] = df_data['hist_auth_Y_purchaseAmountMean_-7'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_-13'] = df_data['hist_auth_Y_purchaseAmountMean_-13'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_1'] = df_data['new_purchaseAmountMean_1'] / df_data['hist_purchaseAmountMean_0']
df_data['avg_purchase_ratio_2'] = df_data['new_purchaseAmountMean_2'] / df_data['hist_purchaseAmountMean_0']

df_data.replace([np.inf, -np.inf], np.nan, inplace=True)
df_data.fillna(0, inplace=True)

# 保存用户统计特征
df_data = reduce_mem_usage(df_data)
df_transactions = reduce_mem_usage(df_transactions)

### tmp02 to_csv
print('save to csv...')
df_data.to_csv('tmp02_df_data.csv', index=False)
df_transactions.to_csv('tmp02_df_transactions.csv', index=False)


import numpy as np
import sys

UNIT_SCALE=1e-03 ##Unit scales = milli
TIME_UNIT=UNIT_SCALE
PARAM_SCALING=1/UNIT_SCALE #Constants for Time, Conductances, Capacitances are up scaled for computation, Potentials remain same

#lists for [a,d] :amplitudes, decay times pairs and, at the last, rho: common rise times
AMPA_d_dlt=np.array([0.3351*1e-03, 1.651*1e-03])                *PARAM_SCALING  #Time terms are upscaled by milli
AMPA_d_rho= 0.3274*1e-03                                        *PARAM_SCALING
AMPA_s_dlt=np.array([0.4*1e-03, 4.899*1e-03, 43.1*1e-03])       *PARAM_SCALING
AMPA_s_rho= 0.5548*1e-03                                        *PARAM_SCALING
NMDA_dlt=np.array([13.52*1e-03, 121.9*1e-03])                   *PARAM_SCALING
NMDA_rho=0.8647*1e-03                                           *PARAM_SCALING
GABA_dlt=np.array([13.52*1e-03, 121.9*1e-03])                   *PARAM_SCALING
GABA_rho=0.8647*1e-03                                           *PARAM_SCALING

AMPA_d_amp=np.array([3.724*1e-09, 0.3033*1e-09])                *PARAM_SCALING*1  #Amp & conductances upscaled by milli
AMPA_s_amp=np.array([0.2487*1e-09,0.2799*1e-09, 0.1268*1e-09])  *PARAM_SCALING*1
NMDA_amp=np.array([17*1e-09, 2.645*1e-09])                      *PARAM_SCALING*1
GABA_amp=np.array([0.1*1e-09, 0.2*1e-09])                       *PARAM_SCALING*1

membrane_capacitance=3.22*1e-12                 *PARAM_SCALING #C_m
leak_conductance=1.06*1e-09                     *PARAM_SCALING #G_m
GABA_conductance=0.438*1e-09                    *PARAM_SCALING #G_GABAR (tonic)

leak_reversal_potential=-79.9*1e-03             #*PARAM_SCALING #E_m
GABA_reversal_potential=-79.1*1e-03             #*PARAM_SCALING #E_GABAR (tonic)
AMPA_reversal_potential=0                       #*PARAM_SCALING #E_AMPAR
NMDA_reversal_potential=0                       #*PARAM_SCALING #E_NMDAR
reset_membrane_potential=-63*1e-03              #*PARAM_SCALING #V_r
activation_threshold=-40*1e-03                  #*PARAM_SCALING #V_t

refractory_interval=2*1e-03                     *PARAM_SCALING #tau

r_AMPA_d=0.1249
r_AMPA_s=0.2792
r_NMDA  =0.0322
r_GABA =0.02

class Spiking_cells:
    def __init__(self,num_dend=1):
        #each refers to Synaptic weghts of each of the components
        self.p_AMPA_d_U=[]
        self.p_AMPA_d_R=[]
        self.p_AMPA_s_U=[]
        self.p_AMPA_s_R=[]
        self.p_NMDA_U  =[]
        self.p_NMDA_R  =[]

        self.p_GABA_U  =[]
        self.p_GABA_R  =[]
        for dend in range(num_dend):
            self.p_AMPA_d_U.append(r_AMPA_d)
            self.p_AMPA_d_R.append(1)
            self.p_AMPA_s_U.append(r_AMPA_s)
            self.p_AMPA_s_R.append(1)
            self.p_NMDA_U  .append(r_NMDA)
            self.p_NMDA_R  .append(1)

            self.p_GABA_U  .append(r_GABA) #GABA added
            self.p_GABA_R  .append(1)

        self.num_dend=num_dend
        self.membrane_potential= reset_membrane_potential
        self.MEM_POT_NOSTP= reset_membrane_potential

    def p_AMPA_d(self, ind_dend): return self.p_AMPA_d_U[ind_dend]*self.p_AMPA_d_R[ind_dend]
    def p_AMPA_s(self, ind_dend): return self.p_AMPA_s_U[ind_dend]*self.p_AMPA_s_R[ind_dend]
    def p_NMDA(self,   ind_dend): return self.p_NMDA_U[ind_dend]*self.p_NMDA_R[ind_dend]
    def p_GABA(self,   ind_dend): return self.p_GABA_U[ind_dend]*self.p_GABA_R[ind_dend] #GABA added

    def PLASTICITY(self, ind_dend, SPIKE):
        p_AMPA_d_U=self.p_AMPA_d_U[ind_dend]
        p_AMPA_s_U=self.p_AMPA_s_U[ind_dend]
        p_NMDA_U  =self.p_NMDA_U  [ind_dend]
        p_AMPA_d_R=self.p_AMPA_d_R[ind_dend]
        p_AMPA_s_R=self.p_AMPA_s_R[ind_dend]
        p_NMDA_R  =self.p_NMDA_R  [ind_dend]

        p_GABA_U  =self.p_GABA_U  [ind_dend] #GABA added
        p_GABA_R  =self.p_GABA_R  [ind_dend]
        if SPIKE:
            self.p_AMPA_d_U[ind_dend]+=r_AMPA_d*(1-p_AMPA_d_U)
            self.p_AMPA_s_U[ind_dend]+=r_AMPA_s*(1-p_AMPA_s_U)
            self.p_NMDA_U  [ind_dend]+=r_NMDA  *(1-p_NMDA_U  )
            self.p_AMPA_d_R[ind_dend]=p_AMPA_d_R*(1-p_AMPA_d_U)
            self.p_AMPA_s_R[ind_dend]=p_AMPA_s_R*(1-p_AMPA_s_U)
            self.p_NMDA_R  [ind_dend]=p_NMDA_R  *(1-p_NMDA_U  )
            self.p_GABA_U  [ind_dend]+=r_GABA   *(1-p_GABA_U  )  #GABA added
            self.p_GABA_R  [ind_dend]= p_GABA_R  *(1-p_GABA_U  )
        else:
            self.p_AMPA_d_U[ind_dend]-=(p_AMPA_d_U-r_AMPA_d)/6.394 #a Potentiation Recovery time constant
            self.p_AMPA_s_U[ind_dend]-=(p_AMPA_s_U-r_AMPA_s)/6.394 # For NMDA, But since no info. for AMPA so.
            self.p_NMDA_U  [ind_dend]-=(p_NMDA_U  -r_NMDA  )/6.394 # All for same
            self.p_AMPA_d_R[ind_dend]-=(p_AMPA_d_R-1)/131
            self.p_AMPA_s_R[ind_dend]-=(p_AMPA_s_R-1)/14.85
            self.p_NMDA_R  [ind_dend]-=(p_NMDA_R  -1)/236.1
            self.p_GABA_U  [ind_dend]-=(p_GABA_U-r_GABA   )/6.394 #GABA added
            self.p_GABA_R  [ind_dend]-=(p_GABA_R -1)/100

    def AMPA_conductance_evaluation(self, t, index_dend):
        d_wave=np.zeros(len(t))

        for ind, _ in enumerate(AMPA_d_amp):
            #d_wave+=AMPA_d_amp[ind]*(np.exp(-t/AMPA_d_rho)-np.exp(-t/AMPA_d_dlt[ind]))
            d_wave+=AMPA_d_amp[ind]*(np.exp(-t/AMPA_d_dlt[ind])-np.exp(-t/AMPA_d_rho))
            d_comp=(AMPA_d_amp[ind]*(np.exp(-t/AMPA_d_dlt[ind])-np.exp(-t/AMPA_d_rho)))
            #plt.plot(t, d_comp, label='d%s'%ind)
        #plt.legend()
        #plt.show()
        s_wave=np.zeros(len(t))
        for ind,_ in enumerate(AMPA_s_amp):
            #s_wave+=AMPA_s_amp[ind]*(np.exp(-t/AMPA_s_rho)-np.exp(-t/AMPA_s_dlt[ind]))
            s_wave+=AMPA_s_amp[ind]*(np.exp(-t/AMPA_s_dlt[ind])-np.exp(-t/AMPA_s_rho))
            #s_comp=(AMPA_s_amp[ind]*(np.exp(-t/AMPA_s_dlt[ind])-np.exp(-t/AMPA_s_rho)))
            #plt.plot(t, s_comp, label='s%s'%ind)
        #plt.legend()
        #plt.show()
        #sys.exit()

        #print('t fo AMPA', t)
        #plt.plot(t, s_wave, label="spillover")
        #plt.plot(t, d_wave, label="direct")
        #plt.title("AMPA wave for a spike")
        #plt.legend()
        #plt.show()
        #sys.exit()
        return self.p_AMPA_d(index_dend)*d_wave + self.p_AMPA_s(index_dend)*s_wave, r_AMPA_d*d_wave + r_AMPA_s*s_wave

    def block(self, STP=True):
        C1=2.07  *1e-03       #scaling term
        C2=0.015 *1e-03
        MG_out=1 *1e-03
        z, T, F_per_R, delta_bind, delta_perm= 2, 308.15, 96485.3329/8.31446261815324, 0.35, 0.53  #scale free
        theta=z*F_per_R/T
        Current_Mem_Pot=self.membrane_potential if STP else self.MEM_POT_NOSTP
        theta_V= theta*Current_Mem_Pot
        block = (
            C1*np.exp(delta_bind*theta_V)+C2*np.exp(-delta_perm*theta_V)
            )\
                /(
                    C1*np.exp(delta_bind*theta_V)+C2*np.exp(-delta_perm*theta_V)\
                    +MG_out*np.exp(-delta_bind*theta_V)
                    )
        import math
        if math.isnan(block) or block>1 or block<0:
            raise Exception('math.isnan(block) or block>1 or block<0\n',
            'block val:', block, 'term: c1', C1*np.exp(delta_bind*theta), 'c2', C2*np.exp(-delta_perm*theta),
            'mg', MG_out*np.exp(-delta_bind*theta), 'theta', theta, 'Mem Voltage:',Current_Mem_Pot)
        return block

    def NMDA_conductance_evaluation(self, t, index_dend):
        NMDA_wave=np.zeros(len(t))
        for ind, _ in enumerate(NMDA_amp):
            #NMDA_wave+=NMDA_amp[ind]*abs(np.exp(-t/NMDA_rho)-np.exp(-t/NMDA_dlt[ind]))
            NMDA_wave+=NMDA_amp[ind]*(np.exp(-t/NMDA_dlt[ind])-np.exp(-t/NMDA_rho))
        return self.p_NMDA(index_dend)*NMDA_wave*self.block(STP=True), r_NMDA*NMDA_wave*self.block(STP=False)

    def GABA_conductance_evaluation(self, t, index_dend):
        GABA_wave=np.zeros(len(t))
        for ind, _ in enumerate(GABA_amp):
            GABA_wave+=GABA_amp[ind]*(np.exp(-t/GABA_dlt[ind])-np.exp(-t/GABA_rho))
        return self.p_GABA(index_dend)*GABA_wave, r_GABA*GABA_wave

    def rate_of_change_membrane_voltage(self, Current_Mem_Pot, current_conductance_AMPA, current_conductance_NMDA,\
                                            current_condunctance_GABA, STP=True):

        dV=-1/membrane_capacitance*(\
                leak_conductance*(Current_Mem_Pot-leak_reversal_potential)\
                #+GABA_conductance*(Current_Mem_Pot-GABA_reversal_potential)\
                +current_condunctance_GABA*(Current_Mem_Pot-GABA_reversal_potential)\
                +current_conductance_AMPA *(Current_Mem_Pot-AMPA_reversal_potential)\
                #block* moved to NMDA evaluation function
                +current_conductance_NMDA*(Current_Mem_Pot-NMDA_reversal_potential)\
                )

        if STP: self.membrane_potential+=dV*TIME_UNIT  # change per milli second
        else: self.MEM_POT_NOSTP+=dV*TIME_UNIT

from RateCodingModel import *
from matplotlib import pyplot as plt
time_step=1000
TIME_RANGE=np.linspace(0,TIME_UNIT*time_step, time_step)
NUM_MFs=4
spiking_rate=80

Spike_trains=[]
Spike_trains+=poisson_spike_generator(80, time_step, 1e-03)  # Firing rate spkies per sec,
Spike_trains+=poisson_spike_generator(80, time_step, 1e-03)
Spike_trains+=poisson_spike_generator(100, time_step, 1e-03)
Spike_trains+=poisson_spike_generator(10, time_step, 1e-03)

spike_pttn_per_bin=np.array(Spike_trains).reshape(NUM_MFs, time_step)

#print(np.shape(spike_pttn_per_bin))
#print(np.shape(spike_pttn_per_bin[0]))
#sys.exit()

'''Input spike plot'''
#raster_plot(spike_pttn_per_bin, [0])
#sys.exit()

SPK_GC=Spiking_cells(num_dend=NUM_MFs)
''' For recording the values of variables as np.zeros'''
AMPA_Conductance=np.zeros((NUM_MFs, time_step))
NMDA_Conductance=np.zeros((NUM_MFs, time_step))
GABA_Conductance=np.zeros((NUM_MFs, time_step))

AMPA_NO_STP=np.zeros((NUM_MFs, time_step))
NMDA_NO_STP=np.zeros((NUM_MFs, time_step))
mem_voltage=np.zeros(time_step)
REFRACTORY_COUNT=0

mem_voltage2=np.zeros(time_step)
REFRACTORY_COUNT2=0

p_ofAMPA_d=np.zeros((NUM_MFs,time_step))
p_ofAMPA_s=np.zeros((NUM_MFs,time_step))
p_ofNMDA  =np.zeros((NUM_MFs,time_step))
p_ofGABA  =np.zeros((NUM_MFs,time_step))
''' Simulation loop start '''
for t, time in enumerate(TIME_RANGE):
    mem_voltage[t]=SPK_GC.membrane_potential

    for dend in range(SPK_GC.num_dend):
        p_ofAMPA_d[dend][t]=SPK_GC.p_AMPA_d(dend)
        p_ofAMPA_s[dend][t]=SPK_GC.p_AMPA_s(dend)
        p_ofNMDA[dend][t]=  SPK_GC.p_NMDA(dend)
        p_ofGABA[dend][t]=  SPK_GC.p_GABA(dend)

    if not REFRACTORY_COUNT==0:  #Refractory Period
        SPK_GC.membrane_potential=reset_membrane_potential
        REFRACTORY_COUNT-=1
    else:
        for dend in range(SPK_GC.num_dend):
            if spike_pttn_per_bin[dend][t]==1: #if there's a spike input for a dendrite
                AMPA_waveform_of_a_spike, _ =SPK_GC.AMPA_conductance_evaluation(np.arange(time_step-t), dend)
                NMDA_waveform_of_a_spike, _ =SPK_GC.NMDA_conductance_evaluation(np.arange(time_step-t), dend)
                GABA_waveform_of_a_spike, _ =SPK_GC.GABA_conductance_evaluation(np.arange(time_step-t), dend)
                AMPA_Conductance[dend][t:]+=AMPA_waveform_of_a_spike
                NMDA_Conductance[dend][t:]+=NMDA_waveform_of_a_spike
                GABA_Conductance[dend][t:]+=GABA_waveform_of_a_spike

            SPK_GC.rate_of_change_membrane_voltage(SPK_GC.membrane_potential,
                                                 AMPA_Conductance[dend][t], NMDA_Conductance[dend][t],\
                                                    GABA_Conductance[dend][t], STP=True)

        if SPK_GC.membrane_potential>=activation_threshold:
            print('SPIKE at time', t)
            REFRACTORY_COUNT=refractory_interval
            SPK_GC.membrane_potential=0          # 0mv Represent spikes (choosed as a arbitrary high value)
        elif SPK_GC.membrane_potential<=leak_reversal_potential:
            print('Limiting minimum potential by leak revers. Pot. at time', t)
            SPK_GC.membrane_potential=leak_reversal_potential

    for dend in range(SPK_GC.num_dend):
        SPK_GC.PLASTICITY(dend, spike_pttn_per_bin[dend][t]) #Short term plasticity

    # No STP simulation for Comparison-------------------------------
    '''
    mem_voltage2[t]=SPK_GC.MEM_POT_NOSTP

    if not REFRACTORY_COUNT2==0:  #Refractory Period
        SPK_GC.MEM_POT_NOSTP=reset_membrane_potential
        REFRACTORY_COUNT2-=1
    else:
        for dend in range(SPK_GC.num_dend):
            if spike_pttn_per_bin[dend][t]==1:
                _, AMPA_WAVE_No_STP=SPK_GC.AMPA_conductance_evaluation(np.arange(time_step-t), dend)
                _, NMDA_WAVE_No_STP=SPK_GC.NMDA_conductance_evaluation(np.arange(time_step-t), dend)
                AMPA_NO_STP[dend][t:]+=AMPA_WAVE_No_STP
                NMDA_NO_STP[dend][t:]+=NMDA_WAVE_No_STP

            SPK_GC.rate_of_change_membrane_voltage(SPK_GC.MEM_POT_NOSTP,
                                                AMPA_NO_STP[dend][t],NMDA_NO_STP[dend][t], STP=False)

        if SPK_GC.MEM_POT_NOSTP>=activation_threshold:
            print('SPIKE at time', t, 'For No STP')
            REFRACTORY_COUNT2=refractory_interval
            SPK_GC.MEM_POT_NOSTP=0   '''

#plt.plot(np.arange(time_step), conductance)
#plt.title('Conductance_ No_NMDA, NO STP')
#plt.show()

#raster_plot(spike_pttn_per_bin, [0])

Plotting_Scale=1 # of conducntance

print('Total spikes arrived:', np.sum(spike_pttn_per_bin), 'shape:', np.shape(spike_pttn_per_bin) )

fig, axs = plt.subplots(2+NUM_MFs, gridspec_kw={'height_ratios': [1]*(NUM_MFs)+[0.1]+[2]})

for ind_mf in range(NUM_MFs):
    axs[ind_mf].plot(np.arange(time_step), AMPA_Conductance[ind_mf]*Plotting_Scale, 'red', label="AMPA")
    axs[ind_mf].plot(np.arange(time_step), NMDA_Conductance[ind_mf]*Plotting_Scale, 'purple', label="NMDA")
    axs[ind_mf].plot(np.arange(time_step), GABA_Conductance[ind_mf]*Plotting_Scale, 'g', label="GABA")
    axs[ind_mf].axhline(y=630*1e-12*PARAM_SCALING, color='c', linestyle=':', label='Peak amplitude') #of both condunctances
    #axs[ind_mf].plot(np.arange(time_step), AMPA_NO_STP[ind_mf]*Plotting_Scale, label="AMPA_NO")
    #axs[ind_mf].plot(np.arange(time_step), NMDA_NO_STP[ind_mf]*Plotting_Scale, label="NMDA_NO")
    if ind_mf==3: axs[ind_mf].legend()

axs[0].set_title('Conductances')

#x,y = np.argwhere(spike_pttn_per_bin == 1).T
#axs[NUM_MFs].scatter(y, x, marker='|', label='SPIKE arrivals')
#axs[NUM_MFs].legend()

axs[NUM_MFs+1].plot(np.arange(time_step), mem_voltage, label='Mem. Pot.')
#axs[NUM_MFs].plot(np.arange(time_step), mem_voltage2, label='NO STP')
axs[NUM_MFs+1].axhline(y=reset_membrane_potential, color='r', linestyle=':', label='Reset_pot')
axs[NUM_MFs+1].axhline(y=activation_threshold, color='b', linestyle=':', label='Thrshld')
axs[NUM_MFs+1].set_ylabel("""Mmbrn Ptnt'l level""")
axs[NUM_MFs+1].legend()

plt.show()

#sys.exit()
# For P development on condunctance terms of STP
fig, axs = plt.subplots(NUM_MFs, gridspec_kw={'height_ratios': [1]*(NUM_MFs)})
for ind_mf in range(NUM_MFs):
    axs[ind_mf].plot(np.arange(time_step), p_ofAMPA_d[ind_mf], color='r', label="AMPA_d")
    axs[ind_mf].plot(np.arange(time_step), p_ofAMPA_s[ind_mf], color='c', label="AMPA_s")
    axs[ind_mf].plot(np.arange(time_step), p_ofNMDA[ind_mf],   color='m', label="NMDA")
    axs[ind_mf].plot(np.arange(time_step), p_ofGABA[ind_mf],   color='g', label="GABA")
    axs[ind_mf].axhline(y=r_AMPA_d, color='r', linestyle=':', label='Init_AMPA_d')
    axs[ind_mf].axhline(y=r_AMPA_s, color='c', linestyle=':', label='Init_AMPA_s')
    axs[ind_mf].axhline(y=r_NMDA,   color='m', linestyle=':', label='Init_NMDA')
    axs[ind_mf].axhline(y=r_GABA,   color='g', linestyle=':', label='Init_GABA')
    if ind_mf==3: axs[ind_mf].legend()

axs[0].set_title('P development')
plt.show()
sys.exit()

Plotting_Scale=1 # of conducntance
plt.plot(np.arange(time_step), AMPA_Conductance*Plotting_Scale, 'red', label="AMPA")
plt.plot(np.arange(time_step), NMDA_Conductance*Plotting_Scale, 'purple', label="NMDA")
#plt.title('Conductance, NO STP, No Mg block')
plt.title('Conductance, NO STP')
#plt.legend(handles=[plt_AMPA,plt_NMDA])
plt.legend()
plt.show()

#plt.plot(np.arange(time_step), mem_voltage/UNIT_SCALE)
plt.plot(np.arange(time_step), mem_voltage)
plt.title('Membrane Voltage')
plt.show()


#!/usr/bin/env python3

import sys

import matplotlib.pyplot as plt
import numpy as np

# Read threshold and output folder from stdin
t = float(sys.argv[1])
output_folder = sys.argv[2]

# Reading histogram file from output folder
bins, norm_hist, accu_hist = np.loadtxt(output_folder + "/histogram.dat",
                                        unpack=True)

# Creating graph for the normalised histogram
plt.figure(dpi=125)
plt.bar(bins, norm_hist, width=1, color='red', linewidth=0)
plt.axvline(x=t, linewidth=1.5, color='black', linestyle='--')
plt.axis([0, 256, 0, norm_hist.max()])
plt.text(t / 2, norm_hist.max() / 2, "pore\nspace", fontsize=16, ha='center')
plt.text(t + (256 - t) / 2, norm_hist.max() / 2, "solid\nspace", fontsize=16,
         ha='center')
plt.title("Normalised histogram", fontsize=18)
plt.xlabel("Greyscale level [0-255]", fontsize=16)
plt.ylabel("Normalised frequency", fontsize=16)
plt.savefig(output_folder + "/norm_hist.png", dpi=125, bbox_inches='tight')

# Creating graph for the accumulated histogram
plt.figure(dpi=125)
plt.bar(bins, accu_hist, width=1, color='red', linewidth=0)
plt.axvline(x=t, linewidth=1.5, color='black', linestyle='--')
plt.axis([0, 256, 0, accu_hist.max()])
plt.text(t/2, accu_hist.max()/2, "pore\nspace", fontsize=16, ha='center')
plt.text(t+(256-t)/2, accu_hist.max()/2, "solid\nspace", fontsize=16,
         ha='center')
plt.title("Accumulated histogram", fontsize=18)
plt.xlabel("Greyscale level [0-255]", fontsize=16)
plt.ylabel("Accumulated frequency", fontsize=16)
plt.savefig(output_folder + "/accu_hist.png", dpi=125, bbox_inches='tight')


import matplotlib.pyplot as plt
import numpy as np
import torch
import json
import os

from argument import parse_args
import torch.optim as optim
import sys

def get_optimizer(model, optimizer, lr,momentum_):
    if optimizer == 'sgd':
        optimizer = optim.SGD(
            filter(lambda p: p.requires_grad, model.parameters()),
            lr,
            momentum = momentum_)
            # weight_decay=1e-4)
            # ,weight_decay=5e-4)
        print("optimizer is SGD")
    elif optimizer == 'adam':
        optimizer = optim.Adam(
            filter(lambda p: p.requires_grad, model.parameters()), lr)
        print("optimizer is Adam")
    elif optimizer == 'adamw':
        optimizer = optim.AdamW(
            filter(lambda p: p.requires_grad, model.parameters()), lr)
        print("optimizer is AdamW")
    elif optimizer == 'rprop':
        optimizer = optim.Rprop(
            filter(lambda p: p.requires_grad, model.parameters()), lr)
        print("optimizer is Rprop")
    elif optimizer == 'rmsprop':
        optimizer = optim.RMSprop(
            filter(lambda p: p.requires_grad, model.parameters()), lr)
        print("optimizer is RMSprop")
    else:
        print("#### unknown optimizer ####")
        sys.exit()
    return optimizer

def save_weight(model,basename):
    args = parse_args()
    # file_name = '%s_%s%s(%s%s)'%(args.model,args.encode,args.weight,args.M,args.K)
    # save_weight_path = 'weights/%s_e%sw%s/(%s%s)'%(args.model,args.encode,args.weightquant,args.M,args.K)
    save_weight_path = 'weights/%s'%(basename)
    os.makedirs(save_weight_path, exist_ok=True)
    save_args_as_json(args, save_weight_path)
    torch.save(model.state_dict(),
                '%s/_best.pth' % (save_weight_path))
    return save_weight_path

def save_args_as_json(args, save_weight_path):
    args_list = [(i, getattr(args, i)) for i in dir(args) if not '_' in i[0]]
    with open('%s/argparse_setting.json' % save_weight_path, 'w') as f:
        for i, j in args_list:
            json.dump({i: j}, f)
    print("#### save argparse as json file ####")

def accuracy(loader, model, device=None):
    total = 0
    correct = 0
    with torch.no_grad():
        for data in loader:
            if device is None:
                images, labels = data
            else:
                images, labels = data[0].to(device), data[1].to(device)
            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    return 100.0 * correct / total

def accuracy_batch(outputs, labels):
    total = 0
    correct = 0
    total = labels.size(0)
    _, predicted = torch.max(outputs.data, 1)
    correct = (predicted == labels).sum().item()
    return 100.0 * correct / total

def accuracy_of_classes(num_classes, loader, model, device=None):
    class_correct = list(0. for i in range(num_classes))
    class_total = list(0. for i in range(num_classes))
    with torch.no_grad():
        for data in loader:
            if device is None:
                images, labels = data
            else:
                images, labels = data[0].to(device), data[1].to(device)
            outputs = model(images)
            _, predicted = torch.max(outputs, 1)
            c = (predicted == labels).squeeze()
            for i in range(len(labels)):
                label = labels[i]
                class_correct[label] += c[i].item()
                class_total[label] += 1
    return [100.0 * correct / total for correct, total in zip(class_correct, class_total)], (100.0 * sum(class_correct) / sum(class_total))

def images_to_probs(net, images, output):
    '''
    Generates predictions and corresponding probabilities from a trained
    network and a list of images
    '''
#     output = net(images.to(device))
    # convert output probabilities to predicted class
    _, preds_tensor = torch.max(output, 1)
    preds = np.squeeze(preds_tensor.cpu().numpy())
    return preds, [F.softmax(el, dim=0)[i].item() for i, el in zip(preds, output)]

def plot_classes_preds(net, images, outputs, labels):
    '''
    Generates matplotlib Figure using a trained network, along with images
    and labels from a batch, that shows the network's top prediction along
    with its probability, alongside the actual label, coloring this
    information based on whether the prediction was correct or not.
    Uses the "images_to_probs" function.
    '''
    preds, probs = images_to_probs(net, images, outputs)
    # plot the images in the batch, along with predicted and true labels
    fig = plt.figure(figsize=(12, 48))
    for idx in np.arange(4):
        ax = fig.add_subplot(1, 4, idx+1, xticks=[], yticks=[])
#         matplotlib_imshow(images[idx], one_channel=True)
        ax.set_title("{0}, {1:.1f}%\n(label: {2})".format(
            classes[preds[idx]],
            probs[idx] * 100.0,
            classes[labels[idx]]),
                    color=("green" if preds[idx]==labels[idx].item() else "red"))
    return fig

def show_table(data):
    # https://stackoverflow.com/questions/35160256/how-do-i-output-lists-as-a-table-in-jupyter-notebook
    display(HTML(tabulate.tabulate(data, tablefmt='html')))

def add_param(writer, net, gstep):

    for name, value in net.state_dict().items():
        # breakpoint()
        writer.add_histogram(name, value, gstep)

class IntermediateOutputWriter(object):
    def __init__(self, writer, net, step):
        super(IntermediateOutputWriter, self).__init__()
        self.writer = writer
        self.net = net
        self.hooks = []
        self.step = step

    def __enter__(self):
        class _f:
            def __init__(self, writer, name, step):
                self.wrote = False
                self.name = name
                self.step = step
                self.writer = writer
            def __call__(self, m, i, o):
                if not self.wrote:
                    self.writer.add_histogram(self.name, o, self.step)
                    self.wrote = True
        for name, module in self.net.named_modules():
#             print("Register: %s" % name)
            self.hooks.append(module.register_forward_hook(
                _f(self.writer, "%s.output" % name, self.step)
            ))
        return self

    def __exit__(self, ex_type, ex_value, trace):
        while len(self.hooks) > 0:
            self.hooks.pop().remove()
        return False


"""
Author:王楠
Date:2022-01-01
Title:特征筛选方法
步骤0 ：读取任务5的数据集，并完成数据划分。
步骤2 ：学习排列重要性，并通过排列重要性计算出最重要的3个特征， 要求你手动实现其过程。
https://scikit-learn.org/stable/modules/permutation_importance.html
https://www.kaggle.com/dansbecker/permutation-importance
"""
import lightgbm as lgb
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
import numpy as np
import random
import lgb_learn.config as config
np.random.seed(2022)

# 步骤0 ：运行以下代码得到训练集和验证集
# 读取数据
# data = pd.read_csv("https://cdn.coggle.club/kaggle-flight-delays/flights_10k.csv.zip")
data = pd.read_csv(r"../data/flight.csv")
# 提取有用的列
data = data[["MONTH","DAY","DAY_OF_WEEK","AIRLINE","FLIGHT_NUMBER","DESTINATION_AIRPORT",
                 "ORIGIN_AIRPORT","AIR_TIME", "DEPARTURE_TIME","DISTANCE","ARRIVAL_DELAY"]]
data.dropna(inplace=True) #去掉空值,默认按行，存在任一空值，即删除改行，axis=1,how='any',thresh=None
# 筛选出部分数据
data["ARRIVAL_DELAY"] = (data["ARRIVAL_DELAY"]>10)*1
# 进行编码
cols = ["AIRLINE","FLIGHT_NUMBER","DESTINATION_AIRPORT","ORIGIN_AIRPORT"]
for item in cols:
    data[item] = data[item].astype("category").cat.codes +1  #类别编码
# 划分训练集和测试集
train, test, y_train, y_test = train_test_split(data.drop(["ARRIVAL_DELAY"], axis=1),
                                                data["ARRIVAL_DELAY"], random_state=10, test_size=0.25)

# 步骤2：提前训练模型
lgb_params = {
        'boosting_type': 'gbdt',
        'objective': 'binary',
        'metric': 'auc',
        'bagging_fraction': 0.85,
        'feature_fraction': 0.76,
        'learning_rate': 0.034,
        'max_depth': 4,
        'num_leaves': 15,
        'verbose': -1,
        'random_state': 2022
    }
# 定义lgb接口的数据集
lgb_train = lgb.Dataset(train, y_train, free_raw_data=False)
lgb_valid = lgb.Dataset(test, y_test, free_raw_data=False)

evals_result = {}
lgb_clf = lgb.train(params=lgb_params,
                    train_set=lgb_train,
                    valid_sets=lgb_valid,
                    num_boost_round=config.num_boost_round,
                    callbacks=[lgb.log_evaluation(10),
                               lgb.early_stopping(config.early_stopping_rounds),
                               lgb.record_evaluation(evals_result),
                               lgb.reset_parameter(learning_rate=lambda iter:0.04*(0.998**iter)) #指数衰减
                               ]
                    )

pred = lgb_clf.predict(test,num_iteration=lgb_clf.best_iteration)
auc_before = roc_auc_score(y_test,pred)
ret = pd.DataFrame(evals_result['valid_0'])

# 步骤3 ：permutation方法计算特征重要性
shuffle_auc_ls = []
for fea in list(train):
    test_copy = test.copy()
    test_copy[fea] = test[fea].sample(frac=1.0).values
    score_after = roc_auc_score(y_test,lgb_clf.predict(test_copy,num_iteration=lgb_clf.best_iteration))
    shuffle_auc_ls.append(score_after)

ret = pd.DataFrame({'feature_name':list(train),
                    'auc_before':[auc_before]*train.shape[1],
                    'auc_after':shuffle_auc_ls
                    })
ret['score_diff'] =  ret['auc_after'] - ret['auc_before']

#筛选shuffle后AUC降低的特征
ret = ret[ret.score_diff<0]

if __name__ == '__main__':
    print(f'{ret}')


import os
import keyboard

recording = True
sd_card_path = 'D://'  # Replace this with the correct path

def write_input_to_file(event):
    global recording

    if event.event_type == keyboard.KEY_DOWN:
        if event.name == 'esc':
            recording = False
        elif event.name == 'enter':
            with open(os.path.join(sd_card_path, "keyboard_input.txt"), 'a') as file:
                file.write('\n')
        else:
            with open(os.path.join(sd_card_path, "keyboard_input.txt"), 'a') as file:
                file.write(event.name)
    elif event.event_type == keyboard.KEY_UP and event.name != 'enter':
        with open(os.path.join(sd_card_path, "keyboard_input.txt"), 'a') as file:
            file.write(' ')

if __name__ == "__main__":
    print("Press ESC + ENTER to stop recording and save.")
    keyboard.on_press(write_input_to_file)

    # Check for both Esc and Enter key presses to stop recording
    while recording:
        if keyboard.is_pressed('esc') and keyboard.is_pressed('enter'):
            recording = False

    keyboard.unhook_all()
    print("Recording stopped. Keyboard input saved to", os.path.join(sd_card_path, "keyboard_input.txt"))

Python Keyboard Reader/Python Keyboard Reader.py


import os, errno
import time
import logging
import torch
import numpy as np

def mkdir(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

def set_log(path):
    if path:
        ts = time.time()
        logging.basicConfig(filename="%s.%s"%(path, ts),
                            format="%(asctime)s %(levelname)s %(module)s: %(message)s",
                            level=logging.INFO)
    else:
        logging.basicConfig(format="%(asctime)s %(levelname)s %(module)s: %(message)s",
                            level=logging.INFO)

def set_device(device, device_id):
    if device == "cuda":
        logging.info("Use CUDA, set gpu device %d as default!"%device_id)
        torch.cuda.set_device(device_id)
    else:
        logging.info("Use CPU!")

def write_to_txt(path, data):
    with open(path, 'w') as f:
      for _list in data:
        for _string in _list:
          f.write(str(_string) + ' ')
        f.write('\n')

def read_from_txt(path):
    imagelist = []
    data = []
    with open(path, 'r') as f:
      for item in f.readlines():
        tmp = item.split()
        imagelist.append(tmp[0])
        data.append(list(map(float, tmp[1:])))
    return imagelist, np.asarray(data)


import numpy as np
import csv
import matplotlib.pyplot as plt
import math
import sys
from mpl_toolkits.mplot3d import Axes3D

with open("linearX.csv") as f1:
    x = np.genfromtxt(f1)
with open("linearY.csv") as f2:
    y = np.genfromtxt(f2)
x=(x-np.mean(x))/np.std(x)

rate=0.001

m=x.size
theta0=0
theta1=0
theta0_list=[]
theta1_list=[]
errors=[]

e=0
k=0
while True:
    eold=e
    deltheta0=np.sum(y-theta0-theta1*x)/m
    deltheta1=np.sum(np.multiply((y-theta0-theta1*x),x))/m

    theta0=theta0+rate*deltheta0
    theta1=theta1+rate*deltheta1
    theta0_list.append(theta0)
    theta1_list.append(theta1)

    e=np.sum((y-theta0-theta1*x)**2)/(2*m)
    errors.append(e)
    k=k+1
    if (abs(e-eold))<0.0000000001:
        break
print('no of iterations:')
print(k)
print('values:')
print(theta0)
print(theta1)
#    (b)
plt.plot(x,y,'o',label='data')
plt.plot(x,theta0+theta1*x,label='Hypothesis Function')
plt.xlabel('x')
plt.ylabel('y')
plt.legend()
#plt.savefig("Q1.png",bbox_inches="tight")
plt.show()

#    (c)

fig = plt.figure()

ax = fig.gca(projection='3d')
ax.set_xlim3d([0.0, 1.0])
ax.set_ylim3d([0.0, 0.002])
ax.set_zlim3d([0.0, 1.0])
ax.set_xlabel('thetha0')
ax.set_ylabel('thetha1')
ax.set_zlabel('errors')
#    ax.set_title('3D Test')

ax.scatter(theta0_list,theta1_list,errors,label='points')
plt.legend()
#plt.savefig("Q1-2.png",bbox_inches="tight")
plt.show()

def fx(rx, ry):
    f=0.0
    for i in range(y.size):
        f=f+(y[i]-ry-x[i]*rx)**2
    return f/200

rx=np.linspace(-3,3,100)
ry=np.linspace(-3,3,100)
rX,rY =np.meshgrid(rx,ry)

rZ=fx(rX,rY)
fig.clf()
fig = plt.figure()
ax = fig.gca(projection='3d')

ax.plot_surface(rX, rY, rZ, linewidth=0,  cmap='Reds_r', alpha=0.2)
#
#for i in range(k):
#    ax.scatter3D(theta0_list[k-i-1], theta1_list[k-i-1],errors[k-i-1] , color='green')
#     plt.pause(0.2)
#plt.show()

#    (d)
fig.clf()
fig = plt.figure()

for i in range(k):
    plt.contour(rX, rY, rZ, [errors[i]])
#    plt.pause(0.1)
plt.show()


#
# CalendarInteraction
#----------------------
# This script is a part of the Master-Schedule program to update the
# master calendar automatically from the docx file to Google Drive
#
# This class is developed in order to interact with the Google Calendar API
# to syncronized the google calendar with the class information
#
# author: Jaime Diez Gonzalez-Pardo (Jaimedgp)
# github: https://github.com/Jaimedgp
################################################################################

from datetime import timedelta

class CalendarInteraction(object):

    """
        Keep Google Calendar up to date with the
        'Master en Ciencia de Datos' calendar uploading
        the class event
    """

    def __init__(self, service):
        """
            Upload the class event to the Google Calendar application
            with the needed information. The program must be able to
            create the new calendar 'Data-Science' if it does not exists
        """

        self._service = service
        self._calendar_id = self.get_id_calendar()

        self.summary, self.description = None, None
        self.dt_start, self.dt_end = None, None

    def get_id_calendar(self):
        """
            Check if 'Data-Science' calendar already exists or create it
            if it is not. Then obtain its id
        """

        calendars_result = self._service.calendarList().list().execute()
        calendars = calendars_result.get('items', [])

        if not calendars:
            return
        for calendar in calendars:
            if calendar['summary'] == 'Data-Science':
                return calendar['id']
        else:
            calendar = {
                    'summary': 'Data-Science',
                    'timeZone': 'Europe/Amsterdam'
            }

            created_calendar = (self._service.calendars().
                                insert(body=calendar).execute())

        return created_calendar['id']

    def set_class_event(self, class_event):
        """
            read and store class event information
        """

        self.summary = class_event.subject
        self.description = "%s by professor %s" %(class_event.topic,
                                                  class_event.teacher)

        self.dt_start = class_event.date.isoformat()
        self.dt_end = (class_event.date
                       + timedelta(hours=class_event.duration)).isoformat()

        self.push_event()

    def event_exists(self):
        """
            Check if there is an event at that time already
        """

        events_result = (self._service.events().list(
                                            calendarId=self._calendar_id,
                                            timeMin=self.dt_start+'Z',
                                            maxResults=2,
                                            singleEvents=True,
                                            orderBy='startTime').execute())

        events = events_result.get('items', [])

        if not events:
            return False
        for event in events:
            start = event['start'].get('dateTime').split("+")[0]
            if start == self.dt_start:
                return event['id']
        else:
            return False

    def push_event(self):
        """
            Push the event to the Google Calendar 'Data-Science'
            If an event at the same time already exists, just upload,
            if not, create it
        """

        is_event = self.event_exists()

        if is_event:
            self._service.events().update(
                                calendarId=self._calendar_id,
                                eventId=is_event,
                                body={
                                    "summary": self.summary,
                                    "description": self.description,
                                    "start": {"dateTime": self.dt_start,
                                            "timeZone": 'Europe/Amsterdam'},
                                    "end": {"dateTime": self.dt_end,
                                            "timeZone": 'Europe/Amsterdam'},
                                     },
                                         ).execute()
        else:
            self._service.events().insert(
                                calendarId=self._calendar_id,
                                body={
                                    "summary": self.summary,
                                    "description": self.description,
                                    "start": {"dateTime": self.dt_start,
                                            "timeZone": 'Europe/Amsterdam'},
                                    "end": {"dateTime": self.dt_end,
                                            "timeZone": 'Europe/Amsterdam'},
                                     },
                                         ).execute()


# -*- coding: utf-8 -*-

from odoo import models, fields, api, _
import logging

_logger = logging.getLogger(__name__)

class PrintNodeJob(models.Model):
    _name = 'print.node.job'
    _description = 'Print Node Job'
    _order = 'id desc'

    name = fields.Char('Job Title')
    job_id = fields.Char('Job ID')
    status = fields.Selection(selection=[
        ('new', 'New'),
        ('sent_to_client', 'Sent'),
        ('queued', 'Queued'),
        ('done', 'Done'),
    ])
    printer_id = fields.Many2one(comodel_name='printing.printer', string="Printer")
    job_response = fields.Text(string="Job Response")
    idempotency_key = fields.Char(string="Idempotency Key", help="Security Key to avoid duplicate printing")