get_ipython().magic('matplotlib inline')

import gym
import itertools
import matplotlib
import numpy as np
import sys
import sklearn.pipeline
import sklearn.preprocessing

if "../" not in sys.path:
  sys.path.append("../")

from lib import plotting
from sklearn.linear_model import SGDRegressor
from sklearn.kernel_approximation import RBFSampler

matplotlib.style.use('ggplot')

env = gym.envs.make("MountainCar-v0")

env.observation_space # 방향이랑 속도? 의 무한대의 집합

env.action_space.n

# Feature Preprocessing: Normalize to zero mean and unit variance
# We use a few samples from the observation space to do this
observation_examples = np.array([env.observation_space.sample() for x in range(10000)])
scaler = sklearn.preprocessing.StandardScaler()
scaler.fit(observation_examples) # (0,sigma^2)로 정규화시키는 전처리..!

# Used to converte a state to a featurizes represenation.
# We use RBF kernels with different variances to cover different parts of the space
featurizer = sklearn.pipeline.FeatureUnion([
        ("rbf1", RBFSampler(gamma=5.0, n_components=100)), # 다른 감마를 사용하여 100차원씩의 특징을 뽑아온다
        ("rbf2", RBFSampler(gamma=2.0, n_components=100)),
        ("rbf3", RBFSampler(gamma=1.0, n_components=100)),
        ("rbf4", RBFSampler(gamma=0.5, n_components=100))
        ])
featurizer.fit(scaler.transform(observation_examples))

observation_examples.shape

featurizer.transform(scaler.transform(observation_examples)).shape

class Estimator():
    """
    Value Function approximator.
    """

    def __init__(self):
        # We create a separate model for each action in the environment's
        # action space. Alternatively we could somehow encode the action
        # into the features, but this way it's easier to code up.
        self.models = []
        for _ in range(env.action_space.n):
            model = SGDRegressor(learning_rate="constant")
            # We need to call partial_fit once to initialize the model
            # or we get a NotFittedError when trying to make a prediction
            # This is quite hacky.

            # stochastic gradient descent를 위한 모델 셋팅
            # action의 수만큼 model 준비? value function을 위해..
            model.partial_fit([self.featurize_state(env.reset())], [0]) # 400차원의 instance를 넣으면 scalar 값 나와 error 계산하도록...
            # X : 400차원의 feature, y : scalar(action-state value) 인 regressor
            self.models.append(model)

    def featurize_state(self, state):
        """
        정규화시킨 후, 2->400차원으로 매핑 커널
        """

        scaled = scaler.transform([state])
        featurized = featurizer.transform(scaled)
        return featurized[0]

    def predict(self, s, a=None):
        """
        Makes value function predictions.

        Args:
            s: state to make a prediction for
            a: (Optional) action to make a prediction for

        Returns
            만약 action a가 주어진다면 하나의 숫자로서의 action-value q(s,a) 를 리턴
            만약 action이 안주어진다면, 모든 action에 대한 v(s) 를 벡터로 리턴

        """
        features = self.featurize_state(s)
        if not a:
            return np.array([m.predict([features])[0] for m in self.models])
        else:
            return self.models[a].predict([features])[0]

    def update(self, s, a, y):
        """
        Updates the estimator parameters for a given state and action towards
        the target y.
        """
        features = self.featurize_state(s)
        self.models[a].partial_fit([features], [y])

def make_epsilon_greedy_policy(estimator, epsilon, nA):
    """
    Creates an epsilon-greedy policy based on a given Q-function approximator and epsilon.

    Args:
        estimator: An estimator that returns q values for a given state
        epsilon: The probability to select a random action . float between 0 and 1.
        nA: Number of actions in the environment.

    Returns:
        A function that takes the observation as an argument and returns
        the probabilities for each action in the form of a numpy array of length nA.

    """
    def policy_fn(observation):
        A = np.ones(nA, dtype=float) * epsilon / nA
        q_values = estimator.predict(observation)
        best_action = np.argmax(q_values)
        A[best_action] += (1.0 - epsilon)
        return A
    return policy_fn

def q_learning(env, estimator, num_episodes, discount_factor=1.0, epsilon=0.1, epsilon_decay=1.0):
    """
    Q-Learning algorithm for fff-policy TD control using Function Approximation.
    Finds the optimal greedy policy while following an epsilon-greedy policy.

    Args:
        env: OpenAI environment.
        estimator: Action-Value function estimator
        num_episodes: Number of episodes to run for.
        discount_factor: Lambda time discount factor.
        epsilon: Chance the sample a random action. Float betwen 0 and 1.
        epsilon_decay: Each episode, epsilon is decayed by this factor

    Returns:
        An EpisodeStats object with two numpy arrays for episode_lengths and episode_rewards.
    """

    # Keeps track of useful statistics
    stats = plotting.EpisodeStats(
        episode_lengths=np.zeros(num_episodes),
        episode_rewards=np.zeros(num_episodes))

    for i_episode in range(num_episodes):

        # behaviour policy (e-greedy) 생성 with epsilon_decay
        policy = make_epsilon_greedy_policy(
            estimator, epsilon * epsilon_decay**i_episode, env.action_space.n)

        # Print out which episode we're on, useful for debugging.
        # Also print reward for last episode
        last_reward = stats.episode_rewards[i_episode - 1]
        sys.stdout.flush()

        # Reset the environment and pick the first action
        state = env.reset()

        # SARSA 일때 사용, not q-learning
        next_action = None # ?

        # One step in the environment
        for t in itertools.count():

            # q=learning일 때,
            if next_action is None:
                action_probs = policy(state) # behaviour policy에서 action_probs을 얻어와서
                action = np.random.choice(np.arange(len(action_probs)), p=action_probs) # e-greedy를 따라 action 취한다.
            else:
                action = next_action

            # 한 스텝 가본다
            next_state, reward, done, _ = env.step(action)

            # Update statistics
            stats.episode_rewards[i_episode] += reward
            stats.episode_lengths[i_episode] = t

            # q-value를 estimator로 approximation 해본다
            q_values_next = estimator.predict(next_state)

            # Use this code for Q-Learning
            # Q-Value TD Target
            td_target = reward + discount_factor * np.max(q_values_next) # TD-target , greedy로 액션 선택

            # Use this code for SARSA TD Target for on policy-training:
            # next_action_probs = policy(next_state)
            # next_action = np.random.choice(np.arange(len(next_action_probs)), p=next_action_probs)
            # td_target = reward + discount_factor * q_values_next[next_action]

            # target을 위한 approximator를 업데이트한다!
            estimator.update(state, action, td_target)

            print("\rStep {} @ Episode {}/{} ({})".format(t, i_episode + 1, num_episodes, last_reward), end="")

            if done:
                break

            state = next_state

    return stats

estimator = Estimator()

# Note: For the Mountain Car we don't actually need an epsilon > 0.0
# because our initial estimate for all states is too "optimistic" which leads
# to the exploration of all states.
stats = q_learning(env, estimator, 100, epsilon=0.0)

plotting.plot_cost_to_go_mountain_car(env, estimator)
plotting.plot_episode_stats(stats, smoothing_window=25)

state = env.reset()
for _ in range(10000):
    env.render()
    q_values_next = estimator.predict(state)
    action = np.argmax(q_values_next)
    next_state, reward, done, _ = env.step(action)

    if done:
        break
    state = next_state

python/Q-Learning with Value Function Approximation Solution.py


import contextlib

from kazoo import exceptions as kazoo_exceptions
from oslo_utils import uuidutils
import testtools
from zake import fake_client

from taskflow import exceptions as exc
from taskflow.persistence import backends
from taskflow.persistence.backends import impl_zookeeper
from taskflow import test
from taskflow.tests.unit.persistence import base
from taskflow.tests import utils as test_utils
from taskflow.utils import kazoo_utils

TEST_PATH_TPL = '/taskflow/persistence-test/%s'
_ZOOKEEPER_AVAILABLE = test_utils.zookeeper_available(
    impl_zookeeper.MIN_ZK_VERSION)

def clean_backend(backend, conf):
    with contextlib.closing(backend.get_connection()) as conn:
        try:
            conn.clear_all()
        except exc.NotFound:
            pass
    client = kazoo_utils.make_client(conf)
    client.start()
    try:
        client.delete(conf['path'], recursive=True)
    except kazoo_exceptions.NoNodeError:
        pass
    finally:
        kazoo_utils.finalize_client(client)

@testtools.skipIf(not _ZOOKEEPER_AVAILABLE, 'zookeeper is not available')
class ZkPersistenceTest(test.TestCase, base.PersistenceTestMixin):
    def _get_connection(self):
        return self.backend.get_connection()

    def setUp(self):
        super(ZkPersistenceTest, self).setUp()
        conf = test_utils.ZK_TEST_CONFIG.copy()
        # Create a unique path just for this test (so that we don't overwrite
        # what other tests are doing).
        conf['path'] = TEST_PATH_TPL % (uuidutils.generate_uuid())
        try:
            self.backend = impl_zookeeper.ZkBackend(conf)
        except Exception as e:
            self.skipTest("Failed creating backend created from configuration"
                          " %s due to %s" % (conf, e))
        else:
            self.addCleanup(self.backend.close)
            self.addCleanup(clean_backend, self.backend, conf)
            with contextlib.closing(self.backend.get_connection()) as conn:
                conn.upgrade()

    def test_zk_persistence_entry_point(self):
        conf = {'connection': 'zookeeper:'}
        with contextlib.closing(backends.fetch(conf)) as be:
            self.assertIsInstance(be, impl_zookeeper.ZkBackend)

@testtools.skipIf(_ZOOKEEPER_AVAILABLE, 'zookeeper is available')
class ZakePersistenceTest(test.TestCase, base.PersistenceTestMixin):
    def _get_connection(self):
        return self._backend.get_connection()

    def setUp(self):
        super(ZakePersistenceTest, self).setUp()
        conf = {
            "path": "/taskflow",
        }
        self.client = fake_client.FakeClient()
        self.client.start()
        self._backend = impl_zookeeper.ZkBackend(conf, client=self.client)
        conn = self._backend.get_connection()
        conn.upgrade()

    def test_zk_persistence_entry_point(self):
        conf = {'connection': 'zookeeper:'}
        with contextlib.closing(backends.fetch(conf)) as be:
            self.assertIsInstance(be, impl_zookeeper.ZkBackend)


import common as cm

def delete_node(head, key):
    if head == None:
        return head
    prev = None
    cur = head
    while cur.next != None:
        if key == cur.val:
            if prev == None:
                cur_next = cur.next
                cur.next = None
                cur = cur_next      # 2.
                head = cur          # 1. purpose
            else:
                prev.next = cur.next
                prev = cur
                curr = cur.next
        else:
            prev = cur
            cur = cur.next

    return head
def main():
    li = [2, 3, 5, 99, 6,8, 9]
    llact = cm.ll_action()
    a = llact.create_list_ll(li)
    llact.show(a)
    print("\n")
    key = 2
    b = delete_node(a,key)
    llact.show(b)

if __name__ == "__main__":
    main()


import numpy as np
import math
from typing import Optional

import oneflow as flow

class __PrinterOptions(object):
    precision: int = 4
    threshold: float = 1000
    edgeitems: int = 3
    linewidth: int = 80
    sci_mode: Optional[bool] = None

PRINT_OPTS = __PrinterOptions()

def _try_convert_to_local_tensor(tensor):
    if tensor.is_consistent:
        tensor = tensor.to_consistent(
            placement=flow.env.all_device_placement(tensor.placement.device_type),
            sbp=flow.sbp.broadcast,
        ).to_local()
    return tensor

class _Formatter(object):
    def __init__(self, tensor):
        self.floating_dtype = tensor.dtype.is_floating_point
        self.int_mode = True
        self.sci_mode = False
        self.max_width = 1
        self.random_sample_num = 50
        tensor = _try_convert_to_local_tensor(tensor)

        with flow.no_grad():
            tensor_view = tensor.reshape(-1)

        if not self.floating_dtype:
            for value in tensor_view:
                value_str = "{}".format(value)
                self.max_width = max(self.max_width, len(value_str))

        else:
            nonzero_finite_vals = flow.masked_select(tensor_view, tensor_view.ne(0))
            if nonzero_finite_vals.numel() == 0:
                # no valid number, do nothing
                return

            nonzero_finite_abs = nonzero_finite_vals.abs()
            nonzero_finite_min = nonzero_finite_abs.min().numpy().astype(np.float64)
            nonzero_finite_max = nonzero_finite_abs.max().numpy().astype(np.float64)

            for value in nonzero_finite_abs.numpy():
                if value != np.ceil(value):
                    self.int_mode = False
                    break

            if self.int_mode:
                # Check if scientific representation should be used.
                if (
                    nonzero_finite_max / nonzero_finite_min > 1000.0
                    or nonzero_finite_max > 1.0e8
                ):
                    self.sci_mode = True
                    for value in nonzero_finite_vals:
                        value_str = (
                            ("{{:.{}e}}").format(PRINT_OPTS.precision).format(value)
                        )
                        self.max_width = max(self.max_width, len(value_str))
                else:
                    for value in nonzero_finite_vals:
                        value_str = ("{:.0f}").format(value)
                        self.max_width = max(self.max_width, len(value_str) + 1)
            else:
                if (
                    nonzero_finite_max / nonzero_finite_min > 1000.0
                    or nonzero_finite_max > 1.0e8
                    or nonzero_finite_min < 1.0e-4
                ):
                    self.sci_mode = True
                    for value in nonzero_finite_vals:
                        value_str = (
                            ("{{:.{}e}}").format(PRINT_OPTS.precision).format(value)
                        )
                        self.max_width = max(self.max_width, len(value_str))
                else:
                    for value in nonzero_finite_vals:
                        value_str = (
                            ("{{:.{}f}}").format(PRINT_OPTS.precision).format(value)
                        )
                        self.max_width = max(self.max_width, len(value_str))

        if PRINT_OPTS.sci_mode is not None:
            self.sci_mode = PRINT_OPTS.sci_mode

    def width(self):
        return self.max_width

    def format(self, value):
        if self.floating_dtype:
            if self.sci_mode:
                ret = (
                    ("{{:{}.{}e}}")
                    .format(self.max_width, PRINT_OPTS.precision)
                    .format(value)
                )
            elif self.int_mode:
                ret = "{:.0f}".format(value)
                if not (math.isinf(value) or math.isnan(value)):
                    ret += "."
            else:
                ret = ("{{:.{}f}}").format(PRINT_OPTS.precision).format(value)
        else:
            ret = "{}".format(value)
        return (self.max_width - len(ret)) * " " + ret

def _scalar_str(self, formatter1):
    return formatter1.format(_try_convert_to_local_tensor(self).tolist())

def _vector_str(self, indent, summarize, formatter1):
    # length includes spaces and comma between elements
    element_length = formatter1.width() + 2
    elements_per_line = max(
        1, int(math.floor((PRINT_OPTS.linewidth - indent) / (element_length)))
    )
    char_per_line = element_length * elements_per_line

    def _val_formatter(val, formatter1=formatter1):
        return formatter1.format(val)

    if summarize and self.size(0) > 2 * PRINT_OPTS.edgeitems:
        left_values = _try_convert_to_local_tensor(
            self[: PRINT_OPTS.edgeitems]
        ).tolist()
        right_values = _try_convert_to_local_tensor(
            self[-PRINT_OPTS.edgeitems :]
        ).tolist()
        data = (
            [_val_formatter(val) for val in left_values]
            + [" ..."]
            + [_val_formatter(val) for val in right_values]
        )
    else:
        values = _try_convert_to_local_tensor(self).tolist()
        data = [_val_formatter(val) for val in values]

    data_lines = [
        data[i : i + elements_per_line] for i in range(0, len(data), elements_per_line)
    ]
    lines = [", ".join(line) for line in data_lines]
    return "[" + ("," + "\n" + " " * (indent + 1)).join(lines) + "]"

def _tensor_str_with_formatter(self, indent, summarize, formatter1):
    dim = self.dim()

    if dim == 0:
        return _scalar_str(self, formatter1)

    if dim == 1:
        return _vector_str(self, indent, summarize, formatter1)

    if summarize and self.size(0) > 2 * PRINT_OPTS.edgeitems:
        slices = (
            [
                _tensor_str_with_formatter(self[i], indent + 1, summarize, formatter1)
                for i in range(0, PRINT_OPTS.edgeitems)
            ]
            + ["..."]
            + [
                _tensor_str_with_formatter(self[i], indent + 1, summarize, formatter1)
                for i in range(self.shape[0] - PRINT_OPTS.edgeitems, self.shape[0])
            ]
        )
    else:
        slices = [
            _tensor_str_with_formatter(self[i], indent + 1, summarize, formatter1)
            for i in range(0, self.size(0))
        ]

    tensor_str = ("," + "\n" * (dim - 1) + " " * (indent + 1)).join(slices)
    return "[" + tensor_str + "]"

def _tensor_str(self, indent):
    summarize = self.numel() > PRINT_OPTS.threshold
    if self.dtype is flow.float16:
        self = self.float()

    # TODO: not support nd sbp tensor for now
    if self.is_consistent and len(self.placement.hierarchy) > 1:
        return "[...]"

    with flow.no_grad():
        formatter = _Formatter(get_summarized_data(self) if summarize else self)
        return _tensor_str_with_formatter(self, indent, summarize, formatter)

def _add_suffixes(tensor_str, suffixes, indent):
    tensor_strs = [tensor_str]
    last_line_len = len(tensor_str) - tensor_str.rfind("\n") + 1
    for suffix in suffixes:
        suffix_len = len(suffix)
        if last_line_len + suffix_len + 2 > PRINT_OPTS.linewidth:
            tensor_strs.append(",\n" + " " * indent + suffix)
            last_line_len = indent + suffix_len
        else:
            tensor_strs.append(", " + suffix)
            last_line_len += suffix_len + 2
    tensor_strs.append(")")
    return "".join(tensor_strs)

def get_summarized_data(self):
    dim = self.dim()
    if dim == 0:
        return self
    if dim == 1:
        if self.size(0) > 2 * PRINT_OPTS.edgeitems:
            return flow.cat(
                (self[: PRINT_OPTS.edgeitems], self[-PRINT_OPTS.edgeitems :])
            )
        else:
            return self
    if self.size(0) > 2 * PRINT_OPTS.edgeitems:
        start = [self[i] for i in range(0, PRINT_OPTS.edgeitems)]
        end = [
            self[i] for i in range(self.shape[0] - PRINT_OPTS.edgeitems, self.shape[0])
        ]
        return flow.stack([get_summarized_data(x) for x in (start + end)])
    else:
        return flow.stack([get_summarized_data(x) for x in self])

def _gen_tensor_str_template(tensor, is_meta):
    is_meta = is_meta or tensor.is_lazy
    prefix = "tensor("
    indent = len(prefix)
    suffixes = []

    # tensor is local or consistent
    if tensor.is_consistent:
        suffixes.append(f"placement={str(tensor.placement)}")
        suffixes.append(f"sbp={str(tensor.sbp)}")
    elif tensor.device.type == "cuda" or tensor.device.type == "gpu":
        suffixes.append("device='" + str(tensor.device) + "'")
    elif tensor.device.type != "cpu":
        raise RunTimeError("unknow device type")
    if tensor.is_lazy:
        suffixes.append("is_lazy='True'")

    # tensor is empty, meta or normal
    if tensor.numel() == 0:
        # Explicitly print the shape if it is not (0,), to match NumPy behavior
        if tensor.dim() != 1:
            suffixes.append("size=" + str(tuple(tensor.shape)))
        tensor_str = "[]"
    elif is_meta:
        tensor_str = "..."
        suffixes.append("size=" + str(tuple(tensor.shape)))
    else:
        tensor_str = _tensor_str(tensor, indent)

    suffixes.append("dtype=" + str(tensor.dtype))
    if tensor.grad_fn is not None:
        name = tensor.grad_fn.name()
        suffixes.append("grad_fn=<{}>".format(name))
    elif tensor.requires_grad:
        suffixes.append("requires_grad=True")

    return _add_suffixes(prefix + tensor_str, suffixes, indent)

def _gen_tensor_str(tensor):
    return _gen_tensor_str_template(tensor, False)

def _gen_tensor_meta_str(tensor):
    # meta
    return _gen_tensor_str_template(tensor, True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 10 10:18:50 2022

@author: mat
"""
from multiprocessing import Process
from multiprocessing import Condition, Lock
from multiprocessing import Value
from multiprocessing import current_process
import time, random

class Table():
    def __init__(self, nphil, manager):
        self.phil = manager.list( [False]*nphil)
        self.eating = Value('i',0)
        self.actual = None

        self.mutex = Lock()
        self.freefork = Condition(self.mutex)

    def set_current_phil(self,i):
        self.actual = i

    def vecinos_libres(self):
        i = self.actual
        return not(self.phil[(i+1)%len(self.phil)]) and not(self.phil[(i-1)%(len(self.phil))])

    def wants_eat(self, i):
        self.mutex.acquire()
        self.freefork.wait_for(self.vecinos_libres)
        #print(self.actual,i)
        self.phil[i]=True
        self.eating.value +=1
        self.mutex.release()

    def wants_think(self,i):
        self.mutex.acquire()
        self.phil[i] = False
        self.eating.value -=1
        self.freefork.notify()
        self.mutex.release()

class CheatMonitor():

    def __init__(self):
        self.eating = Value('i',0)
        self.mutex = Lock()
        self.checkingFriend = Condition(self.mutex)

    def is_eating(self,n):
        self.mutex.acquire()
        self.eating.value += 1
        self.checkingFriend.notify()
        self.mutex.release()

    def readyToThink(self):
        return self.eating.value == 2

    def wants_think(self,n):
        self.mutex.acquire()
        self.checkingFriend.wait_for(self.readyToThink)
        self.eating.value -= 1
        self.mutex.release()


# -*- coding: UTF-8 -*-
import time
from datetime import datetime
from datetime import timedelta
from sklearn.cluster import KMeans
from sklearn.externals import joblib
from sklearn.preprocessing import StandardScaler
import pandas as pd
import pickle
import os
import math
import numpy as np

t_click_file = '../../t_click.csv'
t_loan_sum_file = '../../t_loan_sum.csv'
t_loan_file = '../../t_loan.csv'
t_order_file = '../../t_order.csv'
t_user_file = '../../t_user.csv'

def change_data(x):
    return math.pow(5,x)-1

def conver_time(time):
    return int(time.split('-')[1])

def map_day2period(df,values,action,days):
    def _exch(x):
        if x <= days:
            return '%s_%s' % (action, days)
        else:
            return 'exclude'
    # print(df['days'][:10])
    df['days'] = df['days'].map(lambda x:_exch(x))
    df = df.groupby(['uid','days'],as_index=False).sum()
    df = df.pivot(index='uid', columns='days', values=values).reset_index()
    df = df[['uid','%s_%s' % (action, days)]]
    # del df['uid']
    return df

def map_day2week(df,values,action):
    def _exch(x):
        if x <= 7:
            return '%s_week1' % action
        elif x <= 14:
            return '%s_week2' % action
        elif x <= 21:
            return '%s_week3' % action
        elif x <= 28:
            return '%s_week4' % action
        else:
            return 'other'
    df['weeks'] = df['days'].map(lambda x:_exch(x))
    df = df.groupby(['uid','weeks'],as_index=False).sum()
    df = df.pivot(index='uid', columns='weeks', values=values).reset_index()
    df = df[['uid','%s_week1' % action,'%s_week2' % action,'%s_week3' % action,'%s_week4' % action]]
    df.fillna(0,inplace=True)
    return df

def map_hours2bucket(action,hours):
    if hours>=8 and hours<=11:
        return '%s_hours_01' % action
    if hours >=12 and hours<=15:
        return '%s_hours_02' % action
    if hours >= 16 and hours<=19:
        return '%s_hours_03' % action
    if hours>=20 and hours<=23:
        return '%s_hours_04' % action
    if hours>=4 and hours <= 7:
        return '%s_hours_05' % action
    else:
        return '%s_hours_06' % action

def user_cluster(df,state,model_name):
    """
    使用余弦相似度/kNN计算用户的相似度
    """
    feat_list = list(df.columns)
    feat_list.remove('uid')
    data = df[feat_list].values
    data = StandardScaler().fit_transform(data)

    if state == 'train':
        clf = KMeans(init='k-means++', max_iter=1200,n_clusters=180, n_init=10, n_jobs= -2)
        clf.fit(data)
        joblib.dump(clf , './model/clu_%s.pkl' % model_name)
        cluster_label = clf.labels_ + 1
    else:
        clf = joblib.load('./model/clu_%s.pkl' % model_name)
        cluster_label = clf.predict(data) + 1
    return cluster_label

def gen_basic_user_feat():
    dump_path = './tmp/train_user_feat.pkl'
    if os.path.exists(dump_path):
        df_user = pickle.load(open(dump_path,'rb'))
    else:
        df_user = pd.read_csv(t_user_file,header=0)
        # 训练时的截止日期时11月，以11月初作为计算激活时长终止时间，预测时改为12月
        df_user['a_date'] = df_user['active_date'].map(lambda x: datetime.strptime('2016-11-1','%Y-%m-%d') - datetime.strptime(x, '%Y-%m-%d'))
        df_user['a_date'] = df_user['a_date'].map(lambda x : x.days/7)
        df_user['limit'] = df_user['limit'].map(lambda x: change_data(x))
        del df_user['active_date']
        pickle.dump(df_user, open(dump_path, 'wb'))
    return df_user

def gen_filter_loan_feat():
    dump_path = './tmp/train_filter_loan_feat.pkl'
    if os.path.exists(dump_path):
        df_filter_loan = pickle.load(open(dump_path,'rb'))
    else:
        df_filter_loan = pd.read_csv(t_loan_file,header=0)
        df_filter_loan['month'] = df_filter_loan['loan_time'].map(lambda x: conver_time(x))
        df_filter_loan['loan_amount'] = df_filter_loan['loan_amount'].map(lambda x: round(change_data(x)))
        df_filter_loan = df_filter_loan[df_filter_loan['month'] != 11]
        del df_filter_loan['month']
        # df_filter_loan = df_filter_loan[df_filter_loan['loan_amount'] <= 90000]
        # df_filter_loan = df_filter_loan[df_filter_loan['loan_amount'] > 499]
        # 贷款行为在滑动时间窗口内的贷款总额
        df_filter_loan['days'] = df_filter_loan['loan_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_filter_loan['days'] = df_filter_loan['days'].map(lambda x: int(x.days))

        uid = df_filter_loan['uid'].unique()
        exclu = [1]*len(uid)
        days_df = pd.DataFrame({'uid':uid,'exclu':exclu})
        for day in [1,2,3,5,7,9,15,20,25,30,35,40,45,50,60,70,80,90]:
            df = df_filter_loan[['uid','days','loan_amount']].copy()
            day_df = map_day2period(df,'loan_amount', 'loan', day)
            days_df = pd.merge(days_df,day_df,how='left',on='uid')
        days_df = days_df.fillna(0)
        del days_df['exclu']
        change_list = list(days_df.columns)
        change_list.remove('uid')
        for col in change_list:
            days_df[col] = days_df[col].map(lambda x : math.log(x+1,5))

        weeks_df = map_day2week(df_filter_loan.copy(),'loan_amount','loan_filter')
        weeks_df['filter_loan_min'] = weeks_df[['loan_filter_week1','loan_filter_week2','loan_filter_week3','loan_filter_week4']].apply(lambda x: x.min(),axis=1)
        weeks_df['filter_loan_max'] = weeks_df[['loan_filter_week1','loan_filter_week2','loan_filter_week3','loan_filter_week4']].apply(lambda x: x.max(),axis=1)
        weeks_df['filter_loan_sum'] = weeks_df[['loan_filter_week1','loan_filter_week2','loan_filter_week3','loan_filter_week4']].apply(lambda x: x.sum(),axis=1)
        weeks_df['filter_loan_mean'] = weeks_df[['loan_filter_week1','loan_filter_week2','loan_filter_week3','loan_filter_week4']].apply(lambda x: x.mean(),axis=1)
        weeks_df['filter_loan_std'] = weeks_df[['loan_filter_week1','loan_filter_week2','loan_filter_week3','loan_filter_week4']].apply(lambda x: x.std(),axis=1)
        weeks_df['filter_loan_std'] = weeks_df['filter_loan_std'].map(lambda x : 1/(1+x))

        df_filter_loan = pd.merge(days_df,weeks_df,how='outer',on='uid')
        pickle.dump(df_filter_loan, open(dump_path, 'wb'))
    return df_filter_loan

def gen_basic_loan_feat():
    dump_path = './tmp/train_loan_feat.pkl'
    if os.path.exists(dump_path):
        df_loan = pickle.load(open(dump_path,'rb'))
    else:
        df_loan = pd.read_csv(t_loan_file,header=0)
        df_loan['month'] = df_loan['loan_time'].map(lambda x: conver_time(x))
        df_loan['loan_amount'] = df_loan['loan_amount'].map(lambda x: change_data(x))
        df_loan = df_loan[df_loan['month'] != 11]
        # df_loan = df_loan[df_loan['loan_amount'] <= 90000]
        # df_loan = df_loan[df_loan['loan_amount'] >= 499]

        # 贷款时间分布
        loan_hour_df = df_loan.copy()
        loan_hour_df['loan_time_hours'] = loan_hour_df['loan_time'].map(lambda x: int(x.split(' ')[1].split(':')[0]))
        loan_hour_df['loan_time_hours'] = loan_hour_df['loan_time_hours'].map(lambda x : map_hours2bucket('loan',x))
        loan_hour_df = loan_hour_df.groupby(['uid','loan_time_hours'],as_index=False).count()
        loan_hour_df = loan_hour_df.pivot(index='uid', columns='loan_time_hours', values='loan_amount').reset_index()
        loan_hour_df = loan_hour_df.fillna(0)

        statistic_df = df_loan.copy()
        statistic_df = statistic_df.groupby(['uid','month'],as_index=False).sum()
        statistic_df = statistic_df.pivot(index='uid', columns='month', values='loan_amount').reset_index()
        statistic_df = statistic_df.fillna(0)
        statistic_df['loan_min'] = statistic_df[[8,9,10]].apply(lambda x: x.min(),axis=1)
        statistic_df['loan_max'] = statistic_df[[8,9,10]].apply(lambda x: x.max(),axis=1)
        statistic_df['loan_sum'] = statistic_df[[8,9,10]].apply(lambda x: x.sum(),axis=1)
        statistic_df['loan_mean'] = statistic_df[[8,9,10]].apply(lambda x: x.mean(),axis=1)
        statistic_df['loan_std'] = statistic_df[[8,9,10]].apply(lambda x: x.std(),axis=1)
        statistic_df['loan_std'] = statistic_df['loan_std'].map(lambda x : 1/(1+x))

        # statistic_data = []
        # for uid in set(df_loan['uid']):
        #     _min = df_loan[df_loan['uid'] == uid]['loan_amount'].min()
        #     _max = df_loan[df_loan['uid'] == uid]['loan_amount'].max()
        #     _median = df_loan[df_loan['uid'] == uid]['loan_amount'].median()
        #     _sum = df_loan[df_loan['uid'] == uid]['loan_amount'].sum()
        #     statistic_data.append([uid,_min,_max,_median,_sum])
        # statistic_df = pd.DataFrame(statistic_data,columns=['uid','loan_min','loan_max','loan_median','loan_sum'])

        # 贷款分期特征
        plannum_df = df_loan.copy()
        plannum_df = plannum_df.groupby(['uid','plannum'],as_index=False).count()
        plannum_df = plannum_df.pivot(index='uid',columns='plannum',values='loan_amount').reset_index()
        plannum_df = plannum_df.fillna(0)
        plannum_df.columns = ['uid','plannum_01','plannum_03','plannum_06','plannum_12']
        plannum_df['plannum_min'] = plannum_df[['plannum_01','plannum_03','plannum_06','plannum_12']].apply(lambda x: x.min(),axis=1)
        plannum_df['plannum_max'] = plannum_df[['plannum_01','plannum_03','plannum_06','plannum_12']].apply(lambda x: x.max(),axis=1)
        plannum_df['plannum_sum'] = plannum_df[['plannum_01','plannum_03','plannum_06','plannum_12']].apply(lambda x: x.sum(),axis=1)
        plannum_df['plannum_mean'] = plannum_df[['plannum_01','plannum_03','plannum_06','plannum_12']].apply(lambda x: x.mean(),axis=1)
        plannum_df['plannum_std'] = plannum_df[['plannum_01','plannum_03','plannum_06','plannum_12']].apply(lambda x: x.std(),axis=1)
        plannum_df['plannum_std'] = plannum_df['plannum_std'].map(lambda x : 1/(1+x))
        # 每月贷款次数
        # df_loan['loan_times'] = 1
        # 上次贷款距离现在的时长
        df_loan['loanTime_weights'] = df_loan['loan_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_loan['loanTime_weights'] = df_loan['loanTime_weights'].map(lambda x: 1/(1e-6+x.days/30))
        # 贷款权重
        df_loan['loan_weights'] = df_loan['loan_amount'] * df_loan['loanTime_weights']
        # 11月累计需要还款金额
        month_8 = df_loan[df_loan['month']==8]
        month_8['repay'] = month_8['loan_amount']/month_8['plannum']
        month_8['repay'][month_8['plannum'] < 3] = 0.

        month_9 = df_loan[df_loan['month']==9]
        month_9['repay'] = month_9['loan_amount']/month_9['plannum']
        month_9['repay'][month_9['plannum'] < 2] = 0.

        month_10 = df_loan[df_loan['month']==10]
        month_10['repay'] = month_10['loan_amount']/month_10['plannum']

        df_loan = pd.concat([month_8,month_9,month_10],axis=0,ignore_index=False)
        # 每月贷款间隔特征
        month_df = pd.get_dummies(df_loan['month'], prefix="month")
        df_loan = pd.concat([df_loan,month_df],axis=1)
        df_loan = df_loan.groupby(['uid'],as_index=False).sum()
        df_loan['loan_months'] = df_loan['month_8']+df_loan['month_9']+df_loan['month_10']
        df_loan['loan_12'] = df_loan['month_8']+df_loan['month_9']
        df_loan['loan_12'] = df_loan['loan_12'].map({0:0,1:0,2:1})
        df_loan['loan_13'] = df_loan['month_8']+df_loan['month_10']
        df_loan['loan_13'] = df_loan['loan_13'].map({0:0,1:0,2:1})
        df_loan['loan_23'] = df_loan['month_9']+df_loan['month_10']
        df_loan['loan_23'] = df_loan['loan_23'].map({0:0,1:0,2:1})
        df_loan['loan_123'] = df_loan['month_8']+df_loan['month_9']+df_loan['month_10']
        df_loan['loan_123'] = df_loan['loan_123'].map({0:0,1:0,2:0,3:1})

        del df_loan['month']
        del df_loan['month_8']
        del df_loan['month_9']
        del df_loan['month_10']

        df_loan['per_plannum'] = df_loan['plannum'] / df_loan['loan_months']
        df_loan['per_times_loan'] = df_loan['loan_amount'] /df_loan['loan_months']

        # 每月贷款金额是否超过初始额度
        df_limit = gen_basic_user_feat()[['uid','limit']]
        df_loan = pd.merge(df_loan,df_limit,how='left',on='uid')
        df_loan['exceed_loan'] = df_loan['per_times_loan'] - df_loan['limit']
        def _map_num(x):
            if x >= 0:
                return 1
            else:
                return 0
        df_loan['exceed_loan'] = df_loan['exceed_loan'].map(lambda x : _map_num(x))
        del df_loan['limit']
        # df_loan['per_month_loan'] = df_loan['loan_amount'] /3

        df_loan = pd.merge(df_loan,statistic_df[['uid','loan_min','loan_mean','loan_max','loan_sum','loan_std']], how='outer',on='uid')
        df_loan = pd.merge(df_loan,plannum_df, how='left',on='uid')
        df_loan = pd.merge(df_loan,loan_hour_df, how='left',on='uid')
        df_loan = df_loan.fillna(0)
        loan_cluster_label = user_cluster(df_loan,'train','loan')
        df_loan['loan_cluster_label'] = loan_cluster_label
        pickle.dump(df_loan, open(dump_path, 'wb'))
    return df_loan

def gen_basic_order_feat():
    dump_path = './tmp/train_order_feat.pkl'
    if os.path.exists(dump_path):
        df_order = pickle.load(open(dump_path,'rb'))
    else:
        df_order = pd.read_csv(t_order_file,header=0)
        df_order['month'] = df_order['buy_time'].map(lambda x: conver_time(x))
        df_order = df_order[df_order['month']!=11]
        df_order['price'] = df_order['price'].map(lambda x: change_data(x))
        df_order['discount'] = df_order['discount'].map(lambda x: change_data(x))
        # 购买商品实际支付价格
        df_order['real_price'] = df_order['price']*df_order['qty'] - df_order['discount']
        df_order['real_price'][df_order['real_price']<0] = 0.

        statistic_df = df_order.copy()
        statistic_df = statistic_df.groupby(['uid','month'],as_index=False).sum()
        statistic_df = statistic_df.pivot(index='uid', columns='month', values='real_price').reset_index()
        statistic_df = statistic_df.fillna(0)
        statistic_df['buy_min'] = statistic_df[[8,9,10]].apply(lambda x: x.min(),axis=1)
        statistic_df['buy_max'] = statistic_df[[8,9,10]].apply(lambda x: x.max(),axis=1)
        statistic_df['buy_sum'] = statistic_df[[8,9,10]].apply(lambda x: x.sum(),axis=1)
        statistic_df['buy_mean'] = statistic_df[[8,9,10]].apply(lambda x: x.mean(),axis=1)
        statistic_df['buy_std'] = statistic_df[[8,9,10]].apply(lambda x: x.std(),axis=1)
        statistic_df['buy_std'] = statistic_df['buy_std'].map(lambda x : 1/(1+x))

        df_order['buy_weights'] = df_order['buy_time'].map(lambda x: datetime.strptime('2016-11-1', '%Y-%m-%d') - datetime.strptime(x, '%Y-%m-%d'))
        df_order['buy_weights'] = df_order['buy_weights'].map(lambda x: 1/(1e-6+x.days))
        df_order['cost_weight'] = df_order['real_price'] * df_order['buy_weights']
        df_order = df_order[['uid','buy_weights','cost_weight','real_price','discount']]
        df_order = df_order.groupby(['uid'],as_index=False).sum()
        # 购买商品的折扣率
        df_order['dis_ratio'] = df_order['discount'] / (df_order['discount'] + df_order['real_price'])
        del df_order['discount']

        df_order = pd.merge(df_order,statistic_df[['uid','buy_min','buy_mean','buy_max','buy_sum','buy_std']], how='left',on='uid')
        df_order = df_order.fillna(0)

        order_cluster_label = user_cluster(df_order,'train','order')
        df_order['order_cluster_label'] = order_cluster_label

        pickle.dump(df_order, open(dump_path, 'wb'))
    return df_order

def gen_filter_order_feat():
    dump_path = './tmp/train_filter_order_feat.pkl'
    if os.path.exists(dump_path):
        df_filter_order = pickle.load(open(dump_path,'rb'))
    else:
        df_filter_order = pd.read_csv(t_order_file,header=0)
        df_filter_order['month'] = df_filter_order['buy_time'].map(lambda x: conver_time(x))
        df_filter_order = df_filter_order[df_filter_order['month']!=11]
        df_filter_order['price'] = df_filter_order['price'].map(lambda x: change_data(x))
        df_filter_order['discount'] = df_filter_order['discount'].map(lambda x: change_data(x))
        # 购买商品实际支付价格
        df_filter_order['real_price'] = df_filter_order['price']*df_filter_order['qty'] - df_filter_order['discount']
        df_filter_order['real_price'][df_filter_order['real_price']<0] = 0.

        # 购买行为在滑动时间窗口内的购买总额
        df_filter_order['days'] = df_filter_order['buy_time'].map(lambda x: datetime.strptime('2016-11-1', '%Y-%m-%d') - datetime.strptime(x, '%Y-%m-%d'))
        df_filter_order['days'] = df_filter_order['days'].map(lambda x: int(x.days))
        uid = df_filter_order['uid'].unique()
        exclu = [1]*len(uid)
        days_df = pd.DataFrame({'uid':uid,'exclu':exclu})
        for day in [1,2,3,5,7,9,15,20,25,30,35,40,45,50,60,70,80,90]:
            df = df_filter_order[['uid','days','real_price']].copy()
            day_df = map_day2period(df,'real_price', 'order', day)
            days_df = pd.merge(days_df,day_df,how='left',on='uid')
        days_df = days_df.fillna(0)
        del days_df['exclu']
        change_list = list(days_df.columns)
        change_list.remove('uid')
        for col in change_list:
            days_df[col] = days_df[col].map(lambda x : math.log(x+1,5))
        df_filter_order = days_df
        pickle.dump(df_filter_order, open(dump_path, 'wb'))
    return df_filter_order

def gen_basic_click_feat():
    dump_path = './tmp/train_click_feat.pkl'
    if os.path.exists(dump_path):
        df_click = pickle.load(open(dump_path,'rb'))
    else:
        df_click = pd.read_csv(t_click_file,header=0)
        df_click['month'] = df_click['click_time'].map(lambda x: conver_time(x))
        df_click = df_click[df_click['month'] != 11]

        # 点击时间特征分布
        click_hour_df = df_click.copy()
        click_hour_df['click_time_hours'] = click_hour_df['click_time'].map(lambda x: int(x.split(' ')[1].split(':')[0]))
        click_hour_df['click_time_hours'] = click_hour_df['click_time_hours'].map(lambda x : map_hours2bucket('click',x))
        click_hour_df = click_hour_df.groupby(['uid','click_time_hours'],as_index=False).count()
        click_hour_df = click_hour_df.pivot(index='uid', columns='click_time_hours', values='click_time').reset_index()
        click_hour_df = click_hour_df.fillna(0)
        column_list = list(click_hour_df.columns)
        column_list.remove('uid')
        for fea in column_list:
            click_hour_df[fea] = click_hour_df[fea].map(lambda x:math.log(x+1,5))

        df_click['click_weights'] = df_click['click_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_click['click_weights'] = df_click['click_weights'].map(lambda x: 1/(1e-6+x.days))
        del df_click['click_time']

        pid_df = pd.get_dummies(df_click["pid"], prefix="pid")
        param_df = pd.get_dummies(df_click["param"], prefix="param")
        del df_click['pid']
        del df_click['param']
        df_click = pd.concat([df_click,pid_df,param_df],axis=1)

        column_list = list(df_click.columns)
        column_list.remove('uid')
        column_list.remove('click_weights')
        column_list.remove('month')
        for fea in column_list:
            df_click[fea] = df_click[fea]*df_click['click_weights']

        df_click['click'] = 1
        statistic_df = df_click.groupby(['uid','month'],as_index=False).sum().copy()
        statistic_df = statistic_df.pivot(index='uid', columns='month', values='click').reset_index()
        statistic_df = statistic_df.fillna(0)
        statistic_df['click_min'] = statistic_df[[8,9,10]].apply(lambda x: x.min(),axis=1)
        statistic_df['click_max'] = statistic_df[[8,9,10]].apply(lambda x: x.max(),axis=1)
        statistic_df['click_sum'] = statistic_df[[8,9,10]].apply(lambda x: x.sum(),axis=1)
        statistic_df['click_mean'] = statistic_df[[8,9,10]].apply(lambda x: x.mean(),axis=1)
        statistic_df['click_std'] = statistic_df[[8,9,10]].apply(lambda x: x.std(),axis=1)
        statistic_df['click_std'] = statistic_df['click_std'].map(lambda x : 1/(1+x))

        del df_click['month']
        del df_click['click']
        df_click = df_click.groupby(['uid'],as_index=False).sum()
        df_click = pd.merge(df_click,statistic_df[['uid','click_min','click_max','click_mean','click_sum','click_std']], how='left',on='uid')
        df_click = pd.merge(df_click,click_hour_df,how='left',on='uid')

        click_cluster_label = user_cluster(df_click,'train','click')
        df_click['click_cluster_label'] = click_cluster_label

        pickle.dump(df_click, open(dump_path, 'wb'))
    return df_click

def gen_filter_click_feat():
    dump_path = './tmp/train_filter_click_feat.pkl'
    if os.path.exists(dump_path):
        df_filter_click = pickle.load(open(dump_path,'rb'))
    else:
        df_filter_click = pd.read_csv(t_click_file,header=0)

        df_filter_click['month'] = df_filter_click['click_time'].map(lambda x: conver_time(x))
        df_filter_click = df_filter_click[df_filter_click['month'] != 11]
        # 点击行为在滑动时间窗口内的点击次数
        df_filter_click['days'] = df_filter_click['click_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_filter_click['days'] = df_filter_click['days'].map(lambda x: x.days)
        df_filter_click['click_num'] = 1
        uid = df_filter_click['uid'].unique()
        exclu = [1]*len(uid)
        days_df = pd.DataFrame({'uid':uid,'exclu':exclu})
        for day in [1,2,3,5,7,9,15,20,25,30,35,40,45,50,60,70,80,90]:
            df = df_filter_click[['uid','days','click_num']].copy()
            day_df = map_day2period(df,'click_num', 'click', day)
            days_df = pd.merge(days_df,day_df,how='left',on='uid')
        days_df = days_df.fillna(0)
        del days_df['exclu']

        df_filter_click = days_df

        change_list = list(df_filter_click.columns)
        change_list.remove('uid')
        for col in change_list:
            df_filter_click[col] = df_filter_click[col].map(lambda x : math.log(x+1,5))

        pickle.dump(df_filter_click,open(dump_path,'wb'))
    return df_filter_click

def gen_train_test_user():
    dump_path = './tmp/train_test_user.pkl'
    if os.path.exists(dump_path):
        train_test_user = pickle.load(open(dump_path, 'rb'))
    else:
        import random
        df_loan = pd.read_csv(t_loan_file,header=0)
        df_user = pd.read_csv(t_user_file,header=0)
        df_loan['month'] = df_loan['loan_time'].map(lambda x: conver_time(x))
        # 各月的贷款用户名单
        month_8_uid = set(df_loan[df_loan['month'] == 8]['uid'])
        month_9_uid = set(df_loan[df_loan['month'] == 9]['uid'])
        month_10_uid = set(df_loan[df_loan['month'] == 10]['uid'])
        month_11_uid = set(df_loan[df_loan['month'] == 11]['uid'])
        none_user = set(df_user['uid']) - (month_11_uid | month_10_uid | month_9_uid | month_8_uid)

        # 各月新增用户名单
        new_loan_user_9 = month_9_uid - month_8_uid
        new_loan_user_10 = month_10_uid - (month_9_uid | month_8_uid)
        new_loan_user_11 = month_11_uid - (month_10_uid | month_9_uid | month_8_uid)
        miss_user_11 = (month_10_uid | month_9_uid | month_8_uid) - month_11_uid
        keep_user_11 = month_11_uid - new_loan_user_11

        # xianxia训练集
        model1_user = month_8_uid | month_9_uid | month_10_uid
        model1_test_user1 = random.sample(list(keep_user_11), int(0.2*len(keep_user_11)))
        model1_test_user = random.sample(list(miss_user_11), int(0.2*len(miss_user_11)))
        model1_test_user.extend(model1_test_user1)
        model1_train_user = list(model1_user - set(model1_test_user))

        model2_user = set(df_user['uid']) - model1_user
        model2_test_user1 = random.sample(list(none_user), int(0.2*len(none_user)))
        model2_test_user = random.sample(list(new_loan_user_11), int(0.2*len(new_loan_user_11)))
        model2_test_user.extend(model2_test_user1)
        model2_train_user = list(model2_user - set(model2_test_user))

        # 线上提交测试集
        sub_model1_user = list(month_11_uid | month_9_uid | month_10_uid)
        sub_model2_user = list(set(df_user['uid']) - set(sub_model1_user))

        train_test_user = {'model1_test_user':model1_test_user,'model1_train_user':model1_train_user,\
                            'model2_test_user':model2_test_user,'model2_train_user':model2_train_user,\
                            'sub_model1_user':sub_model1_user,'sub_model2_user':sub_model2_user}

        pickle.dump(train_test_user, open(dump_path, 'wb'))
    return train_test_user

def gen_labels2():
    dump_path = './tmp/train_label_clf.pkl'
    if os.path.exists(dump_path):
        df_label = pickle.load(open(dump_path,'rb'))
    else:
        user_dict = gen_user_dict()
        new_loan_user_11 = list(user_dict['new_loan_user_11'])
        class1 = [1] * len(new_loan_user_11)
        miss_user_11 = list(user_dict['miss_user_11'])
        class2 = [2] * len(miss_user_11)
        keep_user_11 = list(user_dict['keep_user_11'])
        class3 = [3] * len(keep_user_11)

        new_loan_user_11.extend(miss_user_11)
        new_loan_user_11.extend(keep_user_11)

        class1.extend(class2)
        class1.extend(class3)

        data = {'uid':new_loan_user_11, 'label2':class1}
        df_label = pd.DataFrame(data)
        pickle.dump(df_label, open(dump_path, 'wb'))
        print(df_label['label2'].value_counts())
    return  df_label

def gen_labels():
    dump_path = './tmp/train_label_reg.pkl'
    if os.path.exists(dump_path):
        df_label = pickle.load(open(dump_path,'rb'))
    else:
        df_label = pd.read_csv(t_loan_sum_file,header=0)
        df_label['label'] = df_label['loan_sum']
        df_label = df_label[['uid','label']]
        pickle.dump(df_label, open(dump_path, 'wb'))
        print(df_label['label'].describe())
    return  df_label

def cal_percent(list_num):
    list_num = sorted(list_num)
    num_distant = list_num[-1] - list_num[0]
    four_1 = 0.25 * num_distant + list_num[0]
    four_2 = 0.50 * num_distant + list_num[0]
    four_3 = 0.75 * num_distant + list_num[0]
    return four_1,four_2,four_3

def map_fea(four_1,four_2,four_3,feature):
    if feature <= four_1:
        return 1
    elif feature <= four_2:
        return 2
    elif feature <= four_3:
        return 3
    else:
        return 4

def gen_union_feat():
    train_dump_path = './tmp/train_union_feat.pkl'
    test_dump_path = './tmp/test_union_feat.pkl'
    if os.path.exists(train_dump_path):
        train_union_feat = pickle.load(open(train_dump_path,'rb'))
        test_union_feat = pickle.load(open(test_dump_path,'rb'))
    else:
        df_loan = pd.read_csv(t_loan_file,header=0)
        df_loan['month'] = df_loan['loan_time'].map(lambda x: conver_time(x))
        df_loan['loan_amount'] = df_loan['loan_amount'].map(lambda x: round(change_data(x)))
        df_loan = df_loan.groupby(['uid','month'],as_index=False).sum()

        df_order = pd.read_csv(t_order_file,header=0)
        df_order['month'] = df_order['buy_time'].map(lambda x: conver_time(x))
        df_order['price'] = df_order['price'].map(lambda x: change_data(x))
        df_order['discount'] = df_order['discount'].map(lambda x: change_data(x))
        df_order['real_price'] = df_order['price']*df_order['qty'] - df_order['discount']
        df_order['real_price'][df_order['real_price']<0] = 0.
        df_order = df_order.groupby(['uid','month'],as_index=False).sum()

        df_click = pd.read_csv(t_click_file,header=0)
        df_click['month'] = df_click['click_time'].map(lambda x: conver_time(x))
        df_click = df_click.groupby(['uid','month'],as_index=False).count()

        df_loan= df_loan[['uid','month','loan_amount']]
        df_order = df_order[['uid','month','real_price']]
        df_click = df_click[['uid','month','click_time']]
        union_df = pd.merge(df_loan,df_order,how='outer',on=['uid','month'])
        union_df = pd.merge(union_df,df_click,how='outer',on=['uid','month'])
        union_df = union_df.fillna(0)

        change_list = ['loan_amount','real_price','click_time']
        for col in change_list:
            union_df[col] = union_df[col].map(lambda x : math.log(x+1,5))

        four_1,four_2,four_3 = cal_percent(union_df['loan_amount'])
        union_df['loan_amount'] = union_df['loan_amount'].map(lambda x :map_fea(four_1,four_2,four_3,x))

        four_1,four_2,four_3 = cal_percent(union_df['real_price'])
        union_df['real_price'] = union_df['real_price'].map(lambda x :map_fea(four_1,four_2,four_3,x))

        four_1,four_2,four_3 = cal_percent(union_df['click_time'])
        union_df['click_time'] = union_df['click_time'].map(lambda x :map_fea(four_1,four_2,four_3,x))

        union_df = union_df.pivot(index='uid', columns='month').reset_index()
        union_df = union_df.fillna(0)

        train_df = union_df.copy()
        train_df['lp_first'] = train_df['loan_amount'][8]+train_df['real_price'][8]
        train_df['lc_first'] = train_df['loan_amount'][8]+train_df['click_time'][8]
        train_df['pc_first'] = train_df['real_price'][8]+train_df['click_time'][8]
        train_df['lpc_first'] = train_df['loan_amount'][8]+train_df['real_price'][8]+train_df['click_time'][8]

        train_df['lp_two'] = train_df['loan_amount'][9]+train_df['real_price'][9]
        train_df['lc_two'] = train_df['loan_amount'][9]+train_df['click_time'][9]
        train_df['pc_two'] = train_df['real_price'][9]+train_df['click_time'][9]
        train_df['lpc_two'] = train_df['loan_amount'][9]+train_df['real_price'][9]+train_df['click_time'][9]

        train_df['lp_three'] = train_df['loan_amount'][10]+train_df['real_price'][10]
        train_df['lc_three'] = train_df['loan_amount'][10]+train_df['click_time'][10]
        train_df['pc_three'] = train_df['real_price'][10]+train_df['click_time'][10]
        train_df['lpc_three'] = train_df['loan_amount'][10]+train_df['real_price'][10]+train_df['click_time'][10]
        train_df = train_df[['uid','lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']]
        columns = ['uid','lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']
        train_union_feat = pd.DataFrame(train_df.values,columns = columns)
        columns.remove('uid')
        train_union_feat['union_min'] = train_union_feat[columns].apply(lambda x: x.min(),axis=1)
        train_union_feat['union_max'] = train_union_feat[columns].apply(lambda x: x.max(),axis=1)
        train_union_feat['union_sum'] = train_union_feat[columns].apply(lambda x: x.sum(),axis=1)
        train_union_feat['union_mean'] = train_union_feat[columns].apply(lambda x: x.mean(),axis=1)
        train_union_feat['union_std'] = train_union_feat[columns].apply(lambda x: x.std(),axis=1)
        train_union_feat['union_std'] = train_union_feat['union_std'].map(lambda x : 1/(1+x))
        # action_df = pd.get_dummies(train_union_feat[['lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']])
        # train_union_feat = pd.concat([train_union_feat['uid'],action_df],axis=1)

        test_df = union_df.copy()
        test_df['lp_first'] = test_df['loan_amount'][9]+test_df['real_price'][9]
        test_df['lc_first'] = test_df['loan_amount'][9]+test_df['click_time'][9]
        test_df['pc_first'] = test_df['real_price'][9]+test_df['click_time'][9]
        test_df['lpc_first'] = test_df['loan_amount'][9]+test_df['real_price'][9]+test_df['click_time'][9]

        test_df['lp_two'] = test_df['loan_amount'][10]+test_df['real_price'][10]
        test_df['lc_two'] = test_df['loan_amount'][10]+test_df['click_time'][10]
        test_df['pc_two'] = test_df['real_price'][10]+test_df['click_time'][10]
        test_df['lpc_two'] = test_df['loan_amount'][10]+test_df['real_price'][10]+test_df['click_time'][10]

        test_df['lp_three'] = test_df['loan_amount'][11]+test_df['real_price'][11]
        test_df['lc_three'] = test_df['loan_amount'][11]+test_df['click_time'][11]
        test_df['pc_three'] = test_df['real_price'][11]+test_df['click_time'][11]
        test_df['lpc_three'] = test_df['loan_amount'][11]+test_df['real_price'][11]+test_df['click_time'][11]
        test_df = test_df[['uid','lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']]
        columns = ['uid','lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']
        test_union_feat = pd.DataFrame(test_df.values,columns = columns)
        columns.remove('uid')
        test_union_feat['union_min'] = test_union_feat[columns].apply(lambda x: x.min(),axis=1)
        test_union_feat['union_max'] = test_union_feat[columns].apply(lambda x: x.max(),axis=1)
        test_union_feat['union_sum'] = test_union_feat[columns].apply(lambda x: x.sum(),axis=1)
        test_union_feat['union_mean'] = test_union_feat[columns].apply(lambda x: x.mean(),axis=1)
        test_union_feat['union_std'] = test_union_feat[columns].apply(lambda x: x.std(),axis=1)
        test_union_feat['union_std'] = test_union_feat['union_std'].map(lambda x : 1/(1+x))
        # action_df = pd.get_dummies(test_union_feat[['lp_first','lc_first','pc_first','lpc_first','lp_two','lc_two','pc_two','lpc_two','lp_three','lc_three','pc_three','lpc_three']])
        # test_union_feat = pd.concat([test_union_feat['uid'],action_df],axis=1)

        # train_feat = set(train_union_feat.columns)
        # test_feat = set(test_union_feat.columns)
        # comm_feat = train_feat & test_feat
        # comm_feat = sorted(list(comm_feat))

        # # map_comm_feat2num = {feat:i for feat,i in enumerate(comm_feat)}

        # train_union_feat = train_union_feat[comm_feat]
        # test_union_feat = test_union_feat[comm_feat]

        pickle.dump(train_union_feat, open(train_dump_path, 'wb'))
        pickle.dump(test_union_feat, open(test_dump_path, 'wb'))

    return train_union_feat,test_union_feat

def gen_interactive_feat():
    dump_path = './tmp/train_interactive_feat.pkl'
    if os.path.exists(dump_path):
        train_interactive_feat = pickle.load(open(dump_path,'rb'))
    else:
        df_loan_ori = pd.read_csv(t_loan_file,header=0)
        df_loan_ori['month'] = df_loan_ori['loan_time'].map(lambda x: conver_time(x))
        df_loan_ori = df_loan_ori[df_loan_ori['month'] != 11]
        df_loan_ori['days'] = df_loan_ori['loan_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_loan_ori['days'] = df_loan_ori['days'].map(lambda x: int(x.days))
        df_loan_ori['loan_amount'] = df_loan_ori['loan_amount'].map(lambda x: round(change_data(x)))

        df_order_ori = pd.read_csv(t_order_file,header=0)
        df_order_ori['month'] = df_order_ori['buy_time'].map(lambda x: conver_time(x))
        df_order_ori = df_order_ori[df_order_ori['month'] != 11]
        df_order_ori['days'] = df_order_ori['buy_time'].map(lambda x: datetime.strptime('2016-11-1', '%Y-%m-%d') - datetime.strptime(x, '%Y-%m-%d'))
        df_order_ori['days'] = df_order_ori['days'].map(lambda x: int(x.days))
        df_order_ori['price'] = df_order_ori['price'].map(lambda x: change_data(x))
        df_order_ori['discount'] = df_order_ori['discount'].map(lambda x: change_data(x))
        df_order_ori['real_price'] = df_order_ori['price']*df_order_ori['qty'] - df_order_ori['discount']
        df_order_ori['real_price'][df_order_ori['real_price']<0] = 0.

        df_click_ori = pd.read_csv(t_click_file,header=0)
        df_click_ori['month'] = df_click_ori['click_time'].map(lambda x: conver_time(x))
        df_click_ori = df_click_ori[df_click_ori['month'] != 11]
        df_click_ori['days'] = df_click_ori['click_time'].map(lambda x: datetime.strptime('2016-11-1 00:00:00', '%Y-%m-%d %H:%M:%S') - datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))
        df_click_ori['days'] = df_click_ori['days'].map(lambda x: int(x.days))

        df_loan= df_loan_ori[['uid','month','loan_amount']].copy()
        df_loan = df_loan.groupby(['uid','month'],as_index=False).sum()
        df_loan['loan_amount'] = df_loan['loan_amount'].map(lambda x : math.log(x+1,5))
        df_loan = df_loan.pivot(index='uid', columns='month').reset_index()
        df_loan.fillna(0,inplace=True)

        df_order = df_order_ori[['uid','month','real_price']].copy()
        df_order = df_order.groupby(['uid','month'],as_index=False).sum()
        df_order['real_price'] = df_order['real_price'].map(lambda x : math.log(x+1,5))
        df_order = df_order.pivot(index='uid', columns='month').reset_index()
        df_order.fillna(0,inplace=True)

        df_click = df_click_ori[['uid','month','click_time']].copy()
        df_click = df_click.groupby(['uid','month'],as_index=False).count()
        df_click['click_time'] = df_click['click_time'].map(lambda x : math.log(x+1,5))
        df_click = df_click.pivot(index='uid', columns='month').reset_index()
        df_click.fillna(0,inplace=True)

        loan_order_commom_user = sorted(list(set(df_loan['uid']) & set(df_order['uid'])))
        click_order_commom_user = sorted(list(set(df_click['uid']) & set(df_order['uid'])))
        loan_click_commom_user = sorted(list(set(df_loan['uid']) & set(df_click['uid'])))
        loan_click_order_commom_user = sorted(list(set(df_loan['uid']) & set(df_click['uid']) & set(df_order['uid'])))

        loan_ = df_loan[df_loan.uid.isin(loan_order_commom_user)].sort_values(by='uid')
        order_ = df_order[df_order.uid.isin(loan_order_commom_user)].sort_values(by='uid')
        loan_order = loan_.loc[:,[False,True,True,True]].values * order_.loc[:,[False,True,True,True]].values
        loan_order = np.c_[np.array(loan_['uid']),loan_order]
        loan_order = pd.DataFrame(loan_order,columns=['uid','lo_1','lo_2','lo_3'])

        click_ = df_click[df_click.uid.isin(click_order_commom_user)].sort_values(by='uid')
        order_ = df_order[df_order.uid.isin(click_order_commom_user)].sort_values(by='uid')
        clcik_order = click_.loc[:,[False,True,True,True]].values * order_.loc[:,[False,True,True,True]].values
        clcik_order = np.c_[np.array(click_['uid']),clcik_order]
        clcik_order = pd.DataFrame(clcik_order,columns=['uid','co_1','co_2','co_3'])

        loan_ = df_loan[df_loan.uid.isin(loan_click_commom_user)].sort_values(by='uid')
        click_ = df_click[df_click.uid.isin(loan_click_commom_user)].sort_values(by='uid')
        loan_click = loan_.loc[:,[False,True,True,True]].values * click_.loc[:,[False,True,True,True]].values
        loan_click = np.c_[np.array(loan_['uid']),loan_click]
        loan_click = pd.DataFrame(loan_click,columns=['uid','lc_1','lc_2','lc_3'])

        loan_ = df_loan[df_loan.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        click_ = df_click[df_click.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        order_ = df_order[df_order.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        loan_click_order = loan_.loc[:,[False,True,True,True]].values * click_.loc[:,[False,True,True,True]].values * order_.loc[:,[False,True,True,True]].values
        loan_click_order = np.c_[np.array(loan_['uid']),loan_click_order]
        loan_click_order = pd.DataFrame(loan_click_order,columns=['uid','lco_1','lco_2','lco_3'])

        train_interactive_feat = pd.merge(loan_order,clcik_order,how='outer',on='uid')
        train_interactive_feat = pd.merge(train_interactive_feat,loan_click,how='outer',on='uid')
        train_interactive_feat = pd.merge(train_interactive_feat,loan_click_order,how='outer',on='uid')
        train_interactive_feat = train_interactive_feat.fillna(0)

        feature = list(train_interactive_feat.columns)
        feature.remove('uid')
        train_interactive_feat['interactive_min'] = train_interactive_feat[feature].apply(lambda x: x.min(),axis=1)
        train_interactive_feat['interactive_max'] = train_interactive_feat[feature].apply(lambda x: x.max(),axis=1)
        train_interactive_feat['interactive_sum'] = train_interactive_feat[feature].apply(lambda x: x.sum(),axis=1)
        train_interactive_feat['interactive_mean'] = train_interactive_feat[feature].apply(lambda x: x.mean(),axis=1)
        train_interactive_feat['interactive_std'] = train_interactive_feat[feature].apply(lambda x: x.std(),axis=1)
        train_interactive_feat['interactive_std'] = train_interactive_feat['interactive_std'].map(lambda x : 1/(1+x))

        weeks_df = map_day2week(df_loan_ori.copy(),'loan_amount','loan')
        loan_weeks_df = weeks_df.copy()
        loan_weeks_df['loan_weeks_min'] = loan_weeks_df[['loan_week1','loan_week2','loan_week3','loan_week4']].apply(lambda x: x.min(),axis=1)
        loan_weeks_df['loan_weeks_max'] = loan_weeks_df[['loan_week1','loan_week2','loan_week3','loan_week4']].apply(lambda x: x.max(),axis=1)
        loan_weeks_df['loan_weeks_sum'] = loan_weeks_df[['loan_week1','loan_week2','loan_week3','loan_week4']].apply(lambda x: x.sum(),axis=1)
        loan_weeks_df['loan_weeks_mean'] = loan_weeks_df[['loan_week1','loan_week2','loan_week3','loan_week4']].apply(lambda x: x.mean(),axis=1)
        loan_weeks_df['loan_weeks_std'] = loan_weeks_df[['loan_week1','loan_week2','loan_week3','loan_week4']].apply(lambda x: x.std(),axis=1)
        loan_weeks_df['loan_weeks_std'] = loan_weeks_df['loan_weeks_std'].map(lambda x : 1/(1+x))
        loan_weeks_df = loan_weeks_df[['uid','loan_weeks_min','loan_weeks_max','loan_weeks_sum','loan_weeks_mean','loan_weeks_std']]

        weeks_df1 = map_day2week(df_order_ori.copy(),'real_price','order')
        order_weeks_df = weeks_df1.copy()
        order_weeks_df['order_weeks_min'] = order_weeks_df[['order_week1','order_week2','order_week3','order_week4']].apply(lambda x: x.min(),axis=1)
        order_weeks_df['order_weeks_max'] = order_weeks_df[['order_week1','order_week2','order_week3','order_week4']].apply(lambda x: x.max(),axis=1)
        order_weeks_df['order_weeks_sum'] = order_weeks_df[['order_week1','order_week2','order_week3','order_week4']].apply(lambda x: x.sum(),axis=1)
        order_weeks_df['order_weeks_mean'] = order_weeks_df[['order_week1','order_week2','order_week3','order_week4']].apply(lambda x: x.mean(),axis=1)
        order_weeks_df['order_weeks_std'] = order_weeks_df[['order_week1','order_week2','order_week3','order_week4']].apply(lambda x: x.std(),axis=1)
        order_weeks_df['order_weeks_std'] = order_weeks_df['order_weeks_std'].map(lambda x : 1/(1+x))
        order_weeks_df = order_weeks_df[['uid','order_weeks_min','order_weeks_max','order_weeks_sum','order_weeks_mean','order_weeks_std']]

        df_click_ori['ctime'] = 1
        weeks_df2 = map_day2week(df_click_ori.copy(),'ctime','click')
        click_weeks_df = weeks_df2.copy()
        click_weeks_df['click_weeks_min'] = click_weeks_df[['click_week1','click_week2','click_week3','click_week4']].apply(lambda x: x.min(),axis=1)
        click_weeks_df['click_weeks_max'] = click_weeks_df[['click_week1','click_week2','click_week3','click_week4']].apply(lambda x: x.max(),axis=1)
        click_weeks_df['click_weeks_sum'] = click_weeks_df[['click_week1','click_week2','click_week3','click_week4']].apply(lambda x: x.sum(),axis=1)
        click_weeks_df['click_weeks_mean'] = click_weeks_df[['click_week1','click_week2','click_week3','click_week4']].apply(lambda x: x.mean(),axis=1)
        click_weeks_df['click_weeks_std'] = click_weeks_df[['click_week1','click_week2','click_week3','click_week4']].apply(lambda x: x.std(),axis=1)
        click_weeks_df['click_weeks_std'] = click_weeks_df['click_weeks_std'].map(lambda x : 1/(1+x))
        click_weeks_df = click_weeks_df[['uid','click_weeks_min','click_weeks_max','click_weeks_sum','click_weeks_mean','click_weeks_std']]

        action_weeks_df = pd.merge(loan_weeks_df,order_weeks_df,how='outer',on='uid')
        action_weeks_df = pd.merge(action_weeks_df,click_weeks_df,how='outer',on='uid')

        loan_ = weeks_df[weeks_df.uid.isin(loan_order_commom_user)].sort_values(by='uid')
        order_ = weeks_df1[weeks_df1.uid.isin(loan_order_commom_user)].sort_values(by='uid')
        week_loan_order = loan_.loc[:,[False,True,True,True,True]].values * order_.loc[:,[False,True,True,True,True]].values
        week_loan_order = np.c_[np.array(loan_['uid']),week_loan_order]
        week_loan_order = pd.DataFrame(week_loan_order,columns=['uid','week_lo_1','week_lo_2','week_lo_3','week_lo_4'])

        click_ = weeks_df2[weeks_df2.uid.isin(click_order_commom_user)].sort_values(by='uid')
        order_ = weeks_df1[weeks_df1.uid.isin(click_order_commom_user)].sort_values(by='uid')
        week_click_order = click_.loc[:,[False,True,True,True,True]].values * order_.loc[:,[False,True,True,True,True]].values
        week_click_order = np.c_[np.array(click_['uid']),week_click_order]
        week_click_order = pd.DataFrame(week_click_order,columns=['uid','week_co_1','week_co_2','week_co_3','week_co_4'])

        loan_ = weeks_df[weeks_df.uid.isin(loan_click_commom_user)].sort_values(by='uid')
        click_ = weeks_df2[weeks_df2.uid.isin(loan_click_commom_user)].sort_values(by='uid')
        week_loan_click = loan_.loc[:,[False,True,True,True,True]].values * click_.loc[:,[False,True,True,True,True]].values
        week_loan_click = np.c_[np.array(loan_['uid']),week_loan_click]
        week_loan_click = pd.DataFrame(week_loan_click,columns=['uid','week_lc_1','week_lc_2','week_lc_3','week_lc_4'])

        loan_ = weeks_df[weeks_df.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        click_ = weeks_df2[weeks_df2.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        order_ = weeks_df1[weeks_df1.uid.isin(loan_click_order_commom_user)].sort_values(by='uid')
        week_loan_click_order = loan_.loc[:,[False,True,True,True,True]].values * click_.loc[:,[False,True,True,True,True]].values * order_.loc[:,[False,True,True,True,True]].values
        week_loan_click_order = np.c_[np.array(loan_['uid']),week_loan_click_order]
        week_loan_click_order = pd.DataFrame(week_loan_click_order,columns=['uid','week_lco_1','week_lco_2','week_lco_3','week_lco_4'])

        train_interactive_week_feat = pd.merge(week_loan_order,week_click_order,how='outer',on='uid')
        train_interactive_week_feat = pd.merge(train_interactive_week_feat,week_loan_click,how='outer',on='uid')
        train_interactive_week_feat = pd.merge(train_interactive_week_feat,week_loan_click_order,how='outer',on='uid')
        train_interactive_week_feat = train_interactive_week_feat.fillna(0)

        feature = list(train_interactive_week_feat.columns)
        feature.remove('uid')
        train_interactive_week_feat['interactive_min'] = train_interactive_week_feat[feature].apply(lambda x: x.min(),axis=1)
        train_interactive_week_feat['interactive_max'] = train_interactive_week_feat[feature].apply(lambda x: x.max(),axis=1)
        train_interactive_week_feat['interactive_sum'] = train_interactive_week_feat[feature].apply(lambda x: x.sum(),axis=1)
        train_interactive_week_feat['interactive_mean'] = train_interactive_week_feat[feature].apply(lambda x: x.mean(),axis=1)
        train_interactive_week_feat['interactive_std'] = train_interactive_week_feat[feature].apply(lambda x: x.std(),axis=1)
        train_interactive_week_feat['interactive_std'] = train_interactive_week_feat['interactive_std'].map(lambda x : 1/(1+x))

        train_interactive_feat = pd.merge(train_interactive_feat,action_weeks_df,how='outer',on='uid')
        train_interactive_feat = pd.merge(train_interactive_feat,train_interactive_week_feat,how='outer',on='uid')
        train_interactive_feat = train_interactive_feat.fillna(0)
        pickle.dump(train_interactive_feat, open(dump_path, 'wb'))
    return train_interactive_feat

def make_train_set():
    dump_path = './data/training.pkl'
    if os.path.exists(dump_path):
        train_set = pickle.load(open(dump_path,'rb'))
    else:
        df_user = gen_basic_user_feat()
        df_order = gen_basic_order_feat()
        df_filter_order = gen_filter_order_feat()
        df_loan = gen_basic_loan_feat()
        df_filter_loan = gen_filter_loan_feat()
        df_click = gen_basic_click_feat()
        df_filter_click = gen_filter_click_feat()
        train_union_feat,_ = gen_union_feat()
        df_interactive_feat = gen_interactive_feat()
        df_label = gen_labels()
        # df_label2 = gen_labels2()

        train_set = pd.merge(df_user, df_order, how='outer', on='uid')
        train_set = pd.merge(train_set, df_filter_order, how='outer', on='uid')
        train_set = pd.merge(train_set, df_loan, how='outer', on='uid')
        train_set = pd.merge(train_set, df_filter_loan, how='outer', on='uid')
        train_set['repay_cost'] = train_set['repay']+train_set['cost_weight']
        train_set = pd.merge(train_set, df_click, how='outer', on='uid')
        train_set = pd.merge(train_set, df_filter_click, how='outer', on='uid')
        train_set = pd.merge(train_set, train_union_feat, how='outer', on='uid')
        train_set = pd.merge(train_set, df_interactive_feat, how='outer', on='uid')
        train_set = pd.merge(train_set, df_label, how='outer', on='uid')
        # train_set = pd.merge(train_set, df_label2, how='outer', on='uid')
        train_set = train_set.fillna(0)

        change_list = ['cost_weight','real_price','buy_weights','loan_amount','click_weights','click_min','click_max','click_mean','click_sum','click_std',\
                        'per_times_loan','repay','repay_cost','loan_min','loan_max','loan_mean','loan_sum','loan_std','limit',\
                        'buy_min','buy_mean','buy_max','buy_sum','buy_std']
        for col in change_list:
            train_set[col] = train_set[col].map(lambda x : math.log(x+1,5))

        feat = list(train_set.columns)
        feat.remove('label')
        cluster_label = user_cluster(train_set[feat],'train','all')
        train_set['cluster_label'] = cluster_label

        pickle.dump(train_set, open(dump_path, 'wb'))
    feat_id = {i:fea for i ,fea in enumerate(list(train_set.columns))}
    print(feat_id)

    return train_set

def make_training_data():
    dump_path1 = './data/model1_training.pkl'
    dump_path2 = './data/model2_training.pkl'
    if os.path.exists(dump_path1):
        model1_training = pickle.load(open(dump_path1,'rb'))
        model2_training = pickle.load(open(dump_path2,'rb'))
    else:
        df_user = gen_basic_user_feat()
        df_order = gen_basic_order_feat()
        df_filter_order = gen_filter_order_feat()
        df_loan = gen_basic_loan_feat()
        df_filter_loan = gen_filter_loan_feat()
        df_click = gen_basic_click_feat()
        df_filter_click = gen_filter_click_feat()
        train_union_feat,_ = gen_union_feat()
        df_label = gen_labels()

        model1_training = pd.merge(df_user, df_order, how='outer', on='uid')
        model1_training = pd.merge(model1_training, df_filter_order, how='outer', on='uid')
        model1_training = pd.merge(model1_training, df_loan, how='outer', on='uid')
        model1_training = pd.merge(model1_training, df_filter_loan, how='outer', on='uid')
        model1_training['repay_cost'] = model1_training['repay']+model1_training['cost_weight']
        model1_training = pd.merge(model1_training, df_click, how='outer', on='uid')
        model1_training = pd.merge(model1_training, df_filter_click, how='outer', on='uid')
        # model1_training = pd.merge(model1_training, train_union_feat, how='outer', on='uid')
        model1_training = pd.merge(model1_training, df_label, how='outer', on='uid')
        model1_training = model1_training.fillna(0)

        change_list = ['cost_weight','real_price','buy_weights','loan_amount','click_weights','click_min','click_max','click_mean','click_sum','click_std',\
                        'per_times_loan','repay','repay_cost','loan_min','loan_max','loan_mean','loan_sum','loan_std','limit',\
                        'buy_min','buy_mean','buy_max','buy_sum','buy_std']
        for col in change_list:
            model1_training[col] = model1_training[col].map(lambda x : math.log(x+1,5))

        pickle.dump(model1_training, open(dump_path1, 'wb'))
        feat_id = {i:fea for i ,fea in enumerate(list(model1_training.columns))}
        print(feat_id)

        model2_training = pd.merge(df_user, df_order, how='outer', on='uid')
        model2_training = pd.merge(model2_training, df_filter_order, how='outer', on='uid')
        model2_training = pd.merge(model2_training, df_click, how='outer', on='uid')
        model2_training = pd.merge(model2_training, df_filter_click, how='outer', on='uid')
        model2_training = pd.merge(model2_training, train_union_feat, how='outer', on='uid')
        model2_training = pd.merge(model2_training, df_label, how='outer', on='uid')
        model2_training = model2_training.fillna(0)

        change_list = ['cost_weight','real_price','buy_weights','click_weights','click_min','click_max','click_mean','click_sum','click_std',\
                        'buy_min','buy_mean','buy_max','buy_sum','buy_std']
        for col in change_list:
            model2_training[col] = model2_training[col].map(lambda x : math.log(x+1,5))

        pickle.dump(model2_training, open(dump_path2, 'wb'))
        feat_id = {i:fea for i ,fea in enumerate(list(model2_training.columns))}
        print(feat_id)

    return model1_training,model2_training

def make_clf_train_set():
    dump_path = './data/clf_train_set.pkl'
    if os.path.exists(dump_path):
        clf_train_set = pickle.load(open(dump_path,'rb'))
    else:
        user_dict = gen_user_dict()
        df_user = gen_basic_user_feat()
        df_order = gen_basic_order_feat()
        df_click = gen_basic_click_feat()
        df_filter_click = gen_filter_click_feat()
        train_union_feat,_ = gen_union_feat()
        df_label2 = gen_labels2()

        clf_train_set = pd.merge(df_user, df_order, how='outer', on='uid')
        clf_train_set = pd.merge(clf_train_set, df_click, how='outer', on='uid')
        clf_train_set = pd.merge(clf_train_set, df_filter_click, how='outer', on='uid')
        clf_train_set = pd.merge(clf_train_set, train_union_feat, how='left', on='uid')
        clf_train_set = pd.merge(clf_train_set, df_label2, how='left', on='uid')
        clf_train_set = clf_train_set.fillna(0)

        change_list = ['cost_weight','real_price','click_weights',\
                        ]
        for col in change_list:
            clf_train_set[col] = clf_train_set[col].map(lambda x : math.log(x+1,5))

        pickle.dump(clf_train_set, open(dump_path, 'wb'))
    feat_id = {i:fea for i ,fea in enumerate(list(clf_train_set.columns))}
    print(feat_id)

    return clf_train_set

def spilt_train_test():
    X_train_dump_path = './data/X_train.pkl'
    X_test_dump_path = './data/X_test.pkl'
    y_train_dump_path = './data/y_train.pkl'
    y_test_dump_path = './data/y_test.pkl'
    y_train_dump_path2 = './data/y_train2.pkl'
    y_test_dump_path2 = './data/y_test2.pkl'
    if os.path.exists(X_train_dump_path):
        X_train = pickle.load(open(X_train_dump_path,'rb'))
        X_test = pickle.load(open(X_test_dump_path,'rb'))
        y_train = pickle.load(open(y_train_dump_path,'rb'))
        y_test = pickle.load(open(y_test_dump_path,'rb'))
        y_train2 = pickle.load(open(y_train_dump_path2,'rb'))
        y_test2 = pickle.load(open(y_test_dump_path2,'rb'))
    else:
        train_set = make_train_set()

        train_set.sample(frac=1)
        labels = train_set['label'].copy()
        labels = np.array(labels)

        labels2 = train_set['label2'].copy()
        labels2 = np.array(labels2)

        user_uid = train_set[['uid','label']].copy()

        del train_set['label']
        del train_set['label2']
        del train_set['uid']

        feat = sorted(list(train_set.columns))
        train_set = train_set[feat]
        feat_id = {i:fea for i ,fea in enumerate(list(train_set.columns))}
        print(feat_id)

        train_data = np.array(train_set)

        test_ratio = 0.2
        num = int(len(labels)*test_ratio)
        test_uid = user_uid.iloc[:num,]
        test_uid.to_csv('test_uid.csv',index = False)

        X_test = train_data[:num]
        X_train = train_data[num:]

        y_test = labels[:num]
        y_train = labels[num:]

        y_test2 = labels2[:num]
        y_train2 = labels2[num:]

        pickle.dump(X_train, open(X_train_dump_path, 'wb'))
        pickle.dump(X_test, open(X_test_dump_path, 'wb'))
        pickle.dump(y_train, open(y_train_dump_path, 'wb'))
        pickle.dump(y_test, open(y_test_dump_path, 'wb'))

        pickle.dump(y_train2, open(y_train_dump_path2, 'wb'))
        pickle.dump(y_test2, open(y_test_dump_path2, 'wb'))

    return X_train,X_test,y_train,y_test,y_train2,y_test2

def report(label, pred):
    import matplotlib.pyplot as plt
    plt.figure(figsize=(8,6))
    plt.scatter(range(len(pred)), sorted(pred))
    plt.xlabel('uid to reindex', fontsize=12)
    plt.ylabel('predict', fontsize=12)
    plt.show()

    plt.figure(figsize=(8,6))
    plt.scatter(range(len(label)), sorted(label))
    plt.xlabel('uid to reindex', fontsize=12)
    plt.ylabel('label', fontsize=12)
    plt.show()

    label = np.array(label)
    pred = np.array(pred)
    a = pred - label
    b = a * a

    rmse = np.sqrt(np.sum(b)/len(pred))
    print('RMSE:',rmse)

    plt.figure(figsize=(8,6))
    plt.scatter(range(len(a)), sorted(a))
    plt.xlabel('uid to index', fontsize=12)
    plt.ylabel('The difference between pred and label', fontsize=12)
    plt.show()

if __name__ == '__main__':
    make_train_set()
    # make_clf_train_set()
    # make_training_data()
    # gen_train_test_user()


from portrait.transflow.single_account_portrait.trans_flow import transform_class_str
from fileparser.trans_flow.trans_config import MONTH_LIMIT
from logger.logger_util import LoggerUtil
from util.mysql_reader import sql_to_df
import pandas as pd
import datetime

logger = LoggerUtil().logger(__name__)

class TransFlowRawData:
    """
    将流水账户表和流水数据表落库
    author:汪腾飞
    created_time:20200706
    updated_time_v1:20200707新增是否有新增数据字段,若有则有所有后续操作,若无,则无后续操作
    updated_time_v2:20200818添加commit的事务性,若发生错误则全部不提交
    """

    def __init__(self, sql_db, param, title_param, resp, status):
        self.df = None
        self.db = sql_db
        self.param = param
        self.title_param = title_param
        self.raw_list = []
        self.label_list = []
        self.status = status
        self.create_time = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S')
        # 是否有新增数据,若为True,则需要运行save_raw_data,若无则不需要
        self.new_data = True
        self.resp = resp

    def remove_duplicate_data(self, data):
        """
        从数据库里面找到对应银行账号最后一次上传的记录,查找最后一次上传记录与本次记录的时间是否有交集
        如果有则从本次记录中删除上次记录最后时间之前的所有记录,即将本次时间重复的部分删除后再上传
        :return:
        """
        if self.status:
            sql = """select * from trans_flow where account_id in (select id from trans_account where account_name='%s'
            and id_card_no='%s' and account_no='%s' and bank='%s' and create_time > date_sub(now(), interval %d month))
            order by id desc""" % (self.param.get('cusName'), self.param.get('idNo'),
                                   self.param.get('bankAccount'), self.param.get('bankName'), MONTH_LIMIT)
            df = sql_to_df(sql)
            if df.shape[0] == 0:
                return data
            df['trans_time'] = pd.to_datetime(df['trans_time'])
            df['trans_date'] = df['trans_time'].apply(lambda x: x.date())
            data['trans_date'] = data['trans_time'].apply(lambda x: x.date())
            full_date_list = df.groupby('account_id')['trans_date'].agg({'min', 'max'})[['min', 'max']].values.tolist()
            merge_date_list = self.interval_merge(full_date_list)
            not_full_date_list = []
            full_date_string = "data[(data['trans_date'] < pd.to_datetime('%s').date()) | " % \
                               format(merge_date_list[0][0], '%Y-%m-%d')
            for i in range(len(merge_date_list) - 1):
                not_full_date_list.extend(merge_date_list[i])
                temp_str = "((data['trans_date'] > pd.to_datetime('%s').date()) & " \
                           "(data['trans_date'] < pd.to_datetime('%s').date())) | " % \
                           (format(merge_date_list[i][1], '%Y-%m-%d'), format(merge_date_list[i+1][0], '%Y-%m-%d'))
                full_date_string += temp_str
            full_date_string += "(data['trans_date'] > pd.to_datetime('%s').date())]" % \
                                format(merge_date_list[-1][-1], '%Y-%m-%d')
            not_full_date_list.extend(merge_date_list[-1])
            full_date_df = eval(full_date_string)
            not_full_date_df = data[data['trans_date'].isin(not_full_date_list)]
            if not_full_date_df.shape[0] == 0:
                return full_date_df
            not_full_date_df1 = df[df['trans_date'].isin(not_full_date_list)]
            for row in not_full_date_df.itertuples():
                trans_date = getattr(row, 'trans_date')
                trans_amt = getattr(row, 'trans_amt')
                account_balance = getattr(row, 'account_balance')
                opponent_name = getattr(row, 'opponent_name')
                exist_df = not_full_date_df1[(not_full_date_df1['trans_amt'] == trans_amt) &
                                             (not_full_date_df1['trans_date'] == trans_date) &
                                             (not_full_date_df1['account_balance'] == account_balance) &
                                             (not_full_date_df1['opponent_name'] == opponent_name)]
                if exist_df.shape[0] > 0:
                    not_full_date_df.drop(getattr(row, 'Index'), inplace=True)
            full_date_df = pd.concat([full_date_df, not_full_date_df], axis=0, sort=False)
            if full_date_df.shape[0] == 0:
                self.new_data = False
                self.resp['resCode'] = '23'
                self.resp['resMsg'] = '文件重复'
                self.resp['data']['warningMsg'] = ['该流水文件数据已存在于数据库,不再重复录入']
                logger.info('录入数据已存在于数据库,不再重复录入,cus_name: %s,   id_card_no: %s,     time:%s' %
                            (self.param.get('cusName'), self.param.get('idNo'), self.create_time))
            full_date_df.sort_index(inplace=True)
            return full_date_df
        else:
            return data

    @staticmethod
    def interval_merge(intervals):
        if len(intervals) <= 1:
            return intervals
        intervals.sort()
        result = [intervals[0]]
        for x in intervals[1:]:
            if x[0] >= result[-1][-1]:
                result.append(x)
            else:
                result[-1][-1] = max(result[-1][-1], x[-1])
        return result

    def _save_account_data(self):
        """
        将处理过后的流水数据的基本信息存入trans_account表,并将得到的account_id传入trans_flow表
        :return:
        """
        min_trans_time = self.df['trans_time'].min()
        max_trans_time = self.df['trans_time'].max()
        min_trans_time = datetime.datetime.strftime(min_trans_time, '%Y-%m-%d %H:%M:%S')
        max_trans_time = datetime.datetime.strftime(max_trans_time, '%Y-%m-%d %H:%M:%S')
        temp_dict = dict()
        temp_dict['account_name'] = self.param.get('cusName')
        temp_dict['id_card_no'] = self.param.get('idNo')
        temp_dict['id_type'] = self.param.get('idType')
        temp_dict['bank'] = self.title_param.get('bank', self.param.get('bankName'))
        temp_dict['account_no'] = self.param.get('bankAccount')
        temp_dict['start_time'] = self.title_param.get('start_date', min_trans_time)
        temp_dict['end_time'] = self.title_param.get('end_time', max_trans_time)
        temp_dict['trans_flow_type'] = 1 if self.param.get('cusType') == 'PERSONAL' else 2
        temp_dict['update_time'] = self.create_time
        temp_dict['create_time'] = self.create_time
        temp_dict['account_state'] = 1 if self.status else 0
        role = transform_class_str(temp_dict, 'TransAccount')
        self.raw_list.append(role)
        self.db.session.add(role)
        self.db.session.flush()
        return role.id

    def save_raw_data(self):
        account_id = self._save_account_data()
        # 原始数据列名
        col_list = ['trans_time', 'opponent_name', 'trans_amt', 'account_balance', 'currency', 'opponent_account_no',
                    'opponent_account_bank', 'trans_channel', 'trans_type', 'trans_use', 'remark']
        for row in self.df.itertuples():
            temp_dict = dict()
            temp_dict['account_id'] = account_id
            temp_dict['out_req_no'] = self.param.get('outReqNo')
            for col in col_list:
                temp_dict[col] = getattr(row, col)
            temp_dict['create_time'] = self.create_time
            temp_dict['update_time'] = self.create_time
            # 将原始数据落库
            try:
                if self.status:
                    role = transform_class_str(temp_dict, 'TransFlow')
                else:
                    role = transform_class_str(temp_dict, 'TransFlowException')
            except Exception as e:
                self.resp['resCode'] = '1'
                self.resp['resMsg'] = '失败'
                logger.info('导入数据库失败,失败原因:%s' % str(e))
                self.resp['data']['warningMsg'] = ['字符集对应错误']
                return
            self.raw_list.append(role)
        self.db.session.add_all(self.raw_list)
        try:
            self.db.session.commit()
        except Exception as e:
            self.db.session.rollback()
            self.resp['resCode'] = '1'
            self.resp['resMsg'] = '失败'
            logger.info('导入数据库失败,失败原因:%s' % str(e))
            self.resp['data']['warningMsg'] = ['导入数据库失败']


# encoding: utf-8
"""
@author: Dianlei Zhang
@contact: dianlei.zhang@qq.com

@time: 2018/9/21 08:19
@python version:

"""
import json

class Solution:
    def maxArea(self, height):
        """
        :type height: List[int]
        :rtype: int
        """
        height = [1, 8, 6, 2, 5, 4, 8, 3, 7]

        max = 0
        temp_y = 0
        for i in range(len(height)-1):

            if height[i] <= temp_y:
                continue
            else:
                temp_y = height[i]

            for j in range(i+1, len(height)):
                x = j - i

                y = height[i]
                if height[j] < height[i]:
                    y = height[j]
                temp = x * y
                if temp > max:
                    max = temp

        return max

def stringToIntegerList(input):
    return json.loads(input)

def main():
    import sys
    import io
    def readlines():
        for line in io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8'):
            yield line.strip('\n')

    lines = readlines()
    while True:
        try:
            line = next(lines)
            height = stringToIntegerList(line);

            ret = Solution().maxArea(height)

            out = str(ret);
            print(out)
        except StopIteration:
            break

if __name__ == '__main__':
    main()


from common import message as m
def name_to_script(name):
    try:
        import os
        import pandas as pd
        import re
        try:
            a = pd.read_csv('NSE_CM.csv')
            #print("current working directory......",os.getcwd())
           # print(a)
        except:
            os.system('curl -0 https://public.fyers.in/sym_details/NSE_CM.csv')
            #print("exception 1......................")
        finally:
            try:
                a = pd.read_csv('NSE_CM.csv')
                #print("finally1....")
            except:
                os.system('curl -0 https://public.fyers.in/sym_details/NSE_CM.csv')
                a = pd.read_csv('NSE_CM.csv')

        try:
            #print("before dd values......")
            dd=sorted(a[a['NSE:ABAN-EQ'].str.contains(name, flags=re.IGNORECASE)]['NSE:AMARAJABAT-EQ'].to_numpy())[0]

        except:
            try:
                #print("dd2 values..")
                dd=sorted(a[a['ABAN OFFSHORE LTD.'].str.contains(name, flags=re.IGNORECASE)]['NSE:ABAN-EQ'].to_numpy())[0]

            except:
                #print("dd3 values....vv values")
                vv = a[a['INE421A01028'].notna()]
                dd = vv[vv['INE421A01028'].str.contains(name, flags=re.IGNORECASE)]['NSE:ABAN-EQ'].to_numpy()[0]

    except Exception as error :
        m.message(f'Either the stock name is wrong or error while downloading data from fyers website \n{error}')
    try:
        return dd
    except Exception as error:
        m.message(f"can't find the given stock name {error}")
    return False


import os
import sys
import json
import csv
import glob
import pprint
import numpy as np
import random
import argparse
import pandas as pd
from tqdm import tqdm
from .utils import DataProcessor
from .utils import SemEvalSingleSentenceExample
from transformers import (
    AutoTokenizer
)

class SemEvalDataProcessor(DataProcessor):
    """Processor for Sem-Eval 2020 Task 4 Dataset.
    Args:
        data_dir: string. Root directory for the dataset.
        args: argparse class, may be optional.
    """

    def __init__(self, data_dir=None, args=None, **kwargs):
        """Initialization."""
        self.args = args
        self.data_dir = data_dir

    def get_labels(self):
        """See base class."""
        return 2  # Binary.

    def _read_data(self, data_dir=None, split="train"):
        """Reads in data files to create the dataset."""
        if data_dir is None:
            data_dir = self.data_dir
        my_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
        examples = []

        path = my_dir + '/datasets/semeval_2020_task4/' + split + '.csv'

        with open(path, newline = '\n') as csvfile:
            spamreader = csv.DictReader(csvfile)
            guid = 0
            for row in spamreader:
                guid = guid
                text_correct = row['Correct Statement']
                text_incorrect = row['Incorrect Statement']
                right_reason1 = row['Right Reason1']
                right_reason2 = row ['Right Reason2']
                right_reason3 = row ['Right Reason3']
                confusing_reason1 = row['Confusing Reason1']
                confusing_reason2 = row['Confusing Reason2']
                example_1 = SemEvalSingleSentenceExample(
                        guid=guid,
                        text = text_correct,
                        label=1,
                        right_reason1 = right_reason1,
                        right_reason2 = right_reason2,
                        right_reason3 = right_reason3,
                        confusing_reason1 = confusing_reason1,
                        confusing_reason2 = confusing_reason2
                )

                example_2 = SemEvalSingleSentenceExample(
                        guid=guid,
                        text = text_incorrect,
                        label=0,
                        right_reason1 = right_reason1,
                        right_reason2 = right_reason2,
                        right_reason3 = right_reason3,
                        confusing_reason1 = confusing_reason1,
                        confusing_reason2 = confusing_reason2
                )

                examples.append(example_1)
                examples.append(example_2)

        return examples

    def get_train_examples(self, data_dir=None):
        """See base class."""
        return self._read_data(data_dir=data_dir, split="train")

    def get_dev_examples(self, data_dir=None):
        """See base class."""
        return self._read_data(data_dir=data_dir, split="dev")

    def get_test_examples(self, data_dir=None):
        """See base class."""
        return self._read_data(data_dir=data_dir, split="test")

if __name__ == "__main__":

    # Test loading data.
    proc = SemEvalDataProcessor(data_dir="datasets/semeval_2020_task4")
    train_examples = proc.get_train_examples()
    val_examples = proc.get_dev_examples()
    test_examples = proc.get_test_examples()
    print()
    for i in range(3):
        print(test_examples[i])
    print()


import requests
from concurrent.futures import ThreadPoolExecutor

def req():
    print(" req  start")
    r = requests.get("http://localhost:5000/api/lip/?file_path=/root&speechtext=4211")
    print(r.text)

with ThreadPoolExecutor(8) as executor:
        for _ in range(8):
            print(' submit req ')
            executor.submit(req)