import os, time
from tqdm import tqdm

import pandas as pd
import numpy as np
import pickle
from sklearn.decomposition import PCA
from sklearn.metrics import roc_auc_score
from matplotlib import pyplot as plt
from hyperopt import fmin, tpe, hp, space_eval,rand,Trials,partial,STATUS_OK

from pymodule import conf, featuring, rank, eval

def sampling_negtive_samples(user_set, negtive_df, sample_num=10):
    features = pd.DataFrame()
    for user in user_set:
        # TODO 采样比例调优
        # temp = negtive_df[negtive_df['user_id'] == user]  # 第一版错误，使用了user的历时点击item作为负样本
        temp = negtive_df[negtive_df['user_id'] != user]    # 第二版：使用全量点击序中非本user的item作为负样本
        sample_n = temp.shape[0] if temp.shape[0] < sample_num else sample_num
        temp = temp.sample(n=sample_n, replace=False, random_state=1)
        temp.loc[:, 'user_id'] = user
        features = features.append(temp)

    return features

def subsampling_user(sample_df, sample_num):
    # TODO 修改成打散的？
    sub_user_list = sorted(list(set(sample_df['user_id'])))[:sample_num]
    sample_df = sample_df[sample_df['user_id'].isin(sub_user_list)]

    # 用户行为序中点击item少于等于2个的，都删除 由于需要将最后一个item作为标签，所以只剩下一个无法构建item相似度矩阵
    single_item_user_list = list(
        sample_df['user_id'].value_counts()[sample_df['user_id'].value_counts() <= 2].index)
    click_train = sample_df[~sample_df['user_id'].isin(single_item_user_list)]

    return click_train

def save(submit_all, topk=None, file_dir=None):
    import time

    time_str = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

    file_name = 'submit{time_str}.csv'.format(time_str=int(time.time()))
    if file_dir:
        file_path = os.path.join(file_dir, file_name)
    else:
        file_path = os.path.join('./', file_name)

    with open(file_path, 'w') as f:
        for i, row in submit_all.iterrows():
            user_id = str(row['user_id'])

            item_list = str(row['item_id']).split(',')
            if topk:
                item_list = item_list[:topk]

            if topk:
                assert len(set(item_list)) == topk

            line = user_id + ',' + ','.join(item_list) + '\n'
            if topk:
                assert len(line.strip().split(',')) == (topk + 1)

            f.write(line)

    return file_path

def read_train_click(train_path, phase):
    return pd.read_csv(
        train_path + '/underexpose_train_click-{phase}.csv'.format(phase=phase)
        , header=None
        # , nrows=nrows
        , names=['user_id', 'item_id', conf.org_time_name]
        , sep=','
        , dtype={'user_id': np.str, 'item_id': np.str, 'time': np.float}
    )

def read_test_click(test_path, phase):
    return pd.read_csv(
        test_path + '/underexpose_test_click-{phase}/underexpose_test_click-{phase}.csv'.format(phase=phase)
        , header=None
        # , nrows=nrows
        , names=['user_id', 'item_id', conf.org_time_name]
        , sep=','
        , dtype={'user_id': np.str, 'item_id': np.str, 'time': np.float}
    )

def read_qtime(test_path, phase):
    return pd.read_csv(
        test_path + '/underexpose_test_click-{phase}/underexpose_test_qtime-{phase}.csv'.format(phase=phase)
        , header=None
        # , nrows=nrows
        , names=['user_id', conf.org_time_name]
        , sep=','
        , dtype={'user_id': np.str, 'time': np.float}
    )

def two_columns_df2dict(df):
    result_dict = dict(zip(df.iloc[:, 0], df.iloc[:, 1]))
    return result_dict

def del_qtime_future_click(df):
    qtime_user_set = set(df[df['train_or_test'] == 'predict']['user_id'])

    result_df = df[~df['user_id'].isin(qtime_user_set)]
    for qtime_user in tqdm(qtime_user_set):
        tmp = df[df['user_id'] == qtime_user]
        tmp = tmp[tmp['time'] <= tmp[tmp['train_or_test'] == 'predict']['time'].iloc[0]]

        result_df = result_df.append(tmp)

    return result_df

def del_dup(df):
    df = df.drop_duplicates(['user_id', 'item_id', 'time'])
    return df

def process_time(df, time_stamp):
    df[conf.new_time_name] = df[conf.org_time_name] * time_stamp
    return df

def transfer_item_features_df2dict(item_features, dim):
    txt_vec = item_features.apply(lambda x: np.array(list(x.iloc[-dim-dim: -dim])).reshape(-1, ), axis=1)
    img_vec = item_features.apply(lambda x: np.array(list(x.iloc[-dim: ])).reshape(-1, ), axis=1)
    user_key = item_features['item_id']

    assert len(user_key) == len(txt_vec) and len(user_key) == len(img_vec)

    user_features_dict = {}
    user_features_dict['txt_vec'] = dict(zip(user_key, txt_vec))
    user_features_dict['img_vec'] = dict(zip(user_key, img_vec))

    return user_features_dict

def transfer_user_features_df2dict(user_features, dim):
    txt_vec = user_features.apply(lambda x: np.array(list(x.iloc[-dim-dim: -dim])).reshape(-1, ), axis=1)
    img_vec = user_features.apply(lambda x: np.array(list(x.iloc[-dim: ])).reshape(-1, ), axis=1)
    user_key = user_features['user_id']

    assert len(user_key) == len(txt_vec) and len(user_key) == len(img_vec)

    user_features_dict = {}
    user_features_dict['txt_vec'] = dict(zip(user_key, txt_vec))
    user_features_dict['img_vec'] = dict(zip(user_key, img_vec))

    return user_features_dict

def save_pre_as_submit_format_csv(data_df, out_y):
    # 构造submit格式csv
    valid_eval_data = data_df[['user_id', 'item_id']]
    valid_eval_data['pred_prob'] = out_y
    valid_eval_data['rank'] = valid_eval_data.groupby(['user_id'])['pred_prob'].rank(ascending=False, method='first')
    valid_eval_data.sort_values(['rank'], inplace=True)

    valid_submit = valid_eval_data.groupby(['user_id'])['item_id'].agg(lambda x: ','.join(list(x))).reset_index()
    return valid_submit

def get_user2kitem_dict(df, k):
    df = df.sort_values(['user_id', 'time'], ascending=False).reset_index()
    df = df.groupby('user_id').head(k)
    df = df.groupby('user_id').agg({'item_id': lambda x: ','.join(list(x))}).reset_index()
    df.loc[:, 'item_id'] = df.apply(
        lambda x: list(x['item_id'].split(',')),
        axis=1
    )

    result_dict = dict(zip(df['user_id'], df['item_id']))
    return result_dict

def get_user2click_span_dict(df):
    # df = df.sort_values(['user_id', 'time'])
    df = df.groupby('user_id').agg({'time': lambda x: np.max(list(x)) - np.min(list(x))}).reset_index()
    result_dict = dict(zip(df['user_id'], df['time']))

    return result_dict

def get_user2total_deg_dict(df, day):
    tmp = df.groupby('user_id').agg({'{}day_item_deg'.format(day): lambda x: np.sum(list(x))}).reset_index()

    result_dict = dict(zip(tmp['user_id'], tmp['{}day_item_deg'.format(day)]))

    return result_dict

def read_item_user_info():
    ''' 读取item和user属性 '''
    train_underexpose_item_feat_path = os.path.join(conf.train_path, 'underexpose_item_feat.csv')
    train_underexpose_user_feat_path = os.path.join(conf.train_path, 'underexpose_user_feat.csv')

    train_underexpose_item_feat_df_columns = \
        ['item_id'] + \
        ['txt_vec' + str(i) for i in range(conf.org_embedding_dim)] + \
        ['img_vec' + str(i) for i in range(conf.org_embedding_dim)]
    train_underexpose_user_feat_df_columns = \
        ['user_id', 'user_age_level', 'user_gender', 'user_city_level']

    item_info_df = pd.read_csv(
        train_underexpose_item_feat_path,
        names=train_underexpose_item_feat_df_columns,
        dtype={'item_id': np.str}
    )
    user_info_df = pd.read_csv(
        train_underexpose_user_feat_path,
        names=train_underexpose_user_feat_df_columns,
        dtype={'user_id': np.str, 'item_id': np.str, 'time': np.float}
    )

    # 删除[ 和 ]
    item_info_df['txt_vec0'] = \
        item_info_df['txt_vec0'].apply(lambda x: float(str(x)[1:]) if x is not np.nan else x)
    item_info_df['img_vec0'] = \
        item_info_df['img_vec0'].apply(lambda x: float(str(x)[1:]) if x is not np.nan else x)
    item_info_df['txt_vec127'] = \
        item_info_df['txt_vec127'].apply(lambda x: float(str(x)[:-1]) if x is not np.nan else x)
    item_info_df['img_vec127'] = \
        item_info_df['img_vec127'].apply(lambda x: float(str(x)[:-1]) if x is not np.nan else x)

    txt_embedding_df = item_info_df[['txt_vec{}'.format(i) for i in range(conf.org_embedding_dim)]]
    img_embedding_df = item_info_df[['img_vec{}'.format(i) for i in range(conf.org_embedding_dim)]]

    # todo 降维后信息丢失情况
    # item_txt_embedding_dim = item_img_embedding_dim = 32  # 降维后维度
    short_txt_embedding = \
        PCA(n_components=conf.new_embedding_dim).fit_transform(txt_embedding_df.values)
    short_img_embedding = \
        PCA(n_components=conf.new_embedding_dim).fit_transform(img_embedding_df.values)

    item_info_df = item_info_df[['item_id']]
    item_info_df = pd.concat(
        [item_info_df,
         pd.DataFrame(data=short_txt_embedding,
                      columns=['txt_vec{}'.format(i) for i in range(conf.new_embedding_dim)])],
        axis=1
    )
    item_info_df = pd.concat(
        [item_info_df,
         pd.DataFrame(data=short_img_embedding,
                      columns=['img_vec{}'.format(i) for i in range(conf.new_embedding_dim)])],
        axis=1
    )

    return item_info_df

def read_user_info():
    ''' 读取item和user属性 '''
    train_underexpose_user_feat_path = os.path.join(conf.train_path, 'underexpose_user_feat.csv')
    train_underexpose_user_feat_df_columns = \
        ['user_id', 'user_age_level', 'user_gender', 'user_city_level']

    user_info_df = pd.read_csv(
        train_underexpose_user_feat_path,
        names=train_underexpose_user_feat_df_columns,
        dtype={'user_id': np.str, 'item_id': np.str, 'time': np.float}
    )

    return user_info_df

def clean_user_info(df):
    df.loc[:, 'user_gender'] = df['user_gender'].replace({'F': 1, 'M': 2})
    return df

def read_all_phase_click():
    if conf.is_click_cached:
        all_phase_click_666 = pd.read_csv(conf.click_cache_path, dtype={'user_id': np.str, 'item_id': np.str})
        ''' sampling '''
        if conf.subsampling:
            all_phase_click_666 = subsampling_user(all_phase_click_666, conf.subsampling)
        print('load all click, shape:{}'.format(all_phase_click_666.shape))
    else:
        all_phase_click_org = pd.DataFrame()
        for phase in range(0, conf.now_phase + 1):
            one_phase_train_click = read_train_click(conf.train_path, phase)
            one_phase_test_click = read_test_click(conf.test_path, phase)
            one_phase_qtime = read_qtime(conf.test_path, phase)

            one_phase_test_click['phase'] = str(phase)
            one_phase_test_click['train_or_test'] = 'test'
            one_phase_train_click['phase'] = str(phase)
            one_phase_train_click['train_or_test'] = 'train'
            one_phase_qtime['phase'] = str(phase)
            one_phase_qtime['train_or_test'] = 'predict'
            one_phase_qtime['item_id'] = None

            all_phase_click_org = all_phase_click_org.append(one_phase_train_click).reset_index(drop=True)
            all_phase_click_org = all_phase_click_org.append(one_phase_test_click).reset_index(drop=True)
            all_phase_click_org = all_phase_click_org.append(one_phase_qtime).reset_index(drop=True)

        ''' sampling '''
        if conf.subsampling:
            all_phase_click_org = subsampling_user(all_phase_click_org, conf.subsampling)

        # 删除重复点击
        all_phase_click = del_dup(all_phase_click_org)
        # 删除待预测时间点 之后的点击数据 防止数据泄露
        # all_phase_click_666 = utils.del_qtime_future_click(all_phase_click)
        # 时间处理 乘上 1591891140， 否则时间做操作结果太小，防止溢出
        all_phase_click_666 = process_time(all_phase_click, conf.time_puls)

        all_phase_click_666 = all_phase_click_666.sort_values(['user_id', 'time']).reset_index(drop=True)
        all_phase_click_666.to_csv(conf.click_cache_path, index=False)

    return all_phase_click_666

def get_candidate_positive_samples(df):
    tmp = df.groupby('user_id')['item_id'].count().reset_index()
    tmp = tmp[tmp['item_id'] > conf.candidate_positive_num + 2]     # 删掉候选正样本后训练集里面至少有两个点击记录用于训练
    df = df[df['user_id'].isin(tmp['user_id'])].sample(frac=1, random_state=1)
    first_samples = df.groupby('user_id').head(1).reset_index(drop=True)[['user_id', 'item_id', 'time']]
    first_samples['mark'] = 1
    df = df.merge(
        first_samples, on=['user_id', 'item_id', 'time'], how='left'
    ).sort_values(['user_id', 'time'], ascending=True).reset_index(drop=True)

    tmp = df[df.groupby('user_id').apply(
        lambda x: x['time'] >= x[x['mark'] == 1]['time'].iloc[0]
    ).reset_index(drop=True)]

    candidate_positive_sample_df = \
        tmp[tmp.columns.difference(['mark'])].groupby('user_id').head(conf.candidate_positive_num).reset_index(drop=True)
    return candidate_positive_sample_df

def sava_user_features_dict(user_features_dict, save_path):
    result_df = pd.DataFrame(data=user_features_dict).reset_index()
    result_df = result_df.rename(columns={'index': 'user_id'})
    print(result_df)
    result_df.to_csv(save_path, index=False)

def user_features_df2dict(df):
    df = df.set_index('user_id', drop=True)
    df.index.name = ''
    result_dict = df.to_dict()
    return result_dict

def get_features(df, is_label, type):
    ''' 特征列顺序 重新 组织 '''
    pre_columns = ['user_id', 'item_id']
    recall_columns = [conf.ITEM_CF_SCORE]
    # part1_columns = ['click_item_user_sim', 'click_user_item_sim', 'user_click_num',
    #                 'user_click_interval_mean', 'user_click_interval_min',
    #                 'user_click_interval_max', 'item_deg', 'user_item_mean_deg',
    #                 'user_item_min_deg', 'user_item_max_deg',
    #                 '0_item2item_itemcf_score', 'item20_item_itemcf_score',
    #                 '1_item2item_itemcf_score', 'item21_item_itemcf_score',
    #                 '2_item2item_itemcf_score', 'item22_item_itemcf_score',
    #                 '3_item2item_itemcf_score', 'item23_item_itemcf_score',
    #                 '4_item2item_itemcf_score', 'item24_item_itemcf_score',
    #                 'user_avg_click', 'user_span_click', 'user_total_deg', 'user_avg_deg',
    #                 '0_item_deg', '1_item_deg', 'top_1_item_deg', '2_item_deg',
    #                 'top_2_item_deg', '3_item_deg', 'top_3_item_deg', '4_item_deg',
    #                 'top_4_item_deg']
    # part2_columns = ['1_day_user_txt_sim', '1_day_user_img_sim',
    #                 '2_day_user_txt_sim', '2_day_user_img_sim', '3_day_user_txt_sim',
    #                 '3_day_user_img_sim', '7_day_user_txt_sim', '7_day_user_img_sim',
    #                 'all_day_user_txt_sim', 'all_day_user_img_sim']
    part1_columns = ['1day_click_item_user_sim', '1day_click_user_item_sim',
                     '1day_user_click_num', '1day_item_deg', '1day_user_item_mean_deg',
                     '1day_user_item_min_deg', '1day_user_item_max_deg', '1day_user_item_var_deg',
                     '1day_user_item_median_deg', '1day_user_total_deg', '1day_user_avg_deg', '1day_0_item_deg',
                     '1day_1_item_deg', '1day_top_1_item_deg', '1day_2_item_deg', '1day_top_2_item_deg',
                     '1day_3_item_deg', '1day_top_3_item_deg', '1day_4_item_deg', '1day_top_4_item_deg',
                     '1day_user_click_interval_mean', '1day_user_click_interval_min', '1day_user_click_interval_max',
                     '1day_user_click_interval_var', '1day_user_click_interval_median', '1day_user_mean_click',
                     '1day_user_median_click', '1day_user_span_click', '2day_click_item_user_sim',
                     '2day_click_user_item_sim', '2day_user_click_num', '2day_item_deg', '2day_user_item_mean_deg',
                     '2day_user_item_min_deg', '2day_user_item_max_deg', '2day_user_item_var_deg',
                     '2day_user_item_median_deg', '2day_user_total_deg', '2day_user_avg_deg', '2day_0_item_deg',
                     '2day_1_item_deg', '2day_top_1_item_deg', '2day_2_item_deg', '2day_top_2_item_deg',
                     '2day_3_item_deg', '2day_top_3_item_deg', '2day_4_item_deg', '2day_top_4_item_deg',
                     '2day_user_click_interval_mean', '2day_user_click_interval_min', '2day_user_click_interval_max',
                     '2day_user_click_interval_var', '2day_user_click_interval_median', '2day_user_mean_click',
                     '2day_user_median_click', '2day_user_span_click', '3day_click_item_user_sim',
                     '3day_click_user_item_sim', '3day_user_click_num', '3day_item_deg', '3day_user_item_mean_deg',
                     '3day_user_item_min_deg', '3day_user_item_max_deg', '3day_user_item_var_deg',
                     '3day_user_item_median_deg', '3day_user_total_deg', '3day_user_avg_deg', '3day_0_item_deg',
                     '3day_1_item_deg', '3day_top_1_item_deg', '3day_2_item_deg', '3day_top_2_item_deg',
                     '3day_3_item_deg', '3day_top_3_item_deg', '3day_4_item_deg', '3day_top_4_item_deg',
                     '3day_user_click_interval_mean', '3day_user_click_interval_min', '3day_user_click_interval_max',
                     '3day_user_click_interval_var', '3day_user_click_interval_median', '3day_user_mean_click',
                     '3day_user_median_click', '3day_user_span_click', '7day_click_item_user_sim',
                     '7day_click_user_item_sim', '7day_user_click_num', '7day_item_deg', '7day_user_item_mean_deg',
                     '7day_user_item_min_deg', '7day_user_item_max_deg', '7day_user_item_var_deg',
                     '7day_user_item_median_deg', '7day_user_total_deg', '7day_user_avg_deg', '7day_0_item_deg',
                     '7day_1_item_deg', '7day_top_1_item_deg', '7day_2_item_deg', '7day_top_2_item_deg',
                     '7day_3_item_deg', '7day_top_3_item_deg', '7day_4_item_deg', '7day_top_4_item_deg',
                     '7day_user_click_interval_mean', '7day_user_click_interval_min', '7day_user_click_interval_max',
                     '7day_user_click_interval_var', '7day_user_click_interval_median', '7day_user_mean_click',
                     '7day_user_median_click', '7day_user_span_click', 'earlierday_user_click_num',
                     'earlierday_item_deg', 'earlierday_user_item_mean_deg', 'earlierday_user_item_min_deg',
                     'earlierday_user_item_max_deg', 'earlierday_user_item_var_deg', 'earlierday_user_item_median_deg',
                     'earlierday_user_total_deg', 'earlierday_user_avg_deg', 'earlierday_0_item_deg',
                     'earlierday_1_item_deg', 'earlierday_top_1_item_deg', 'earlierday_2_item_deg',
                     'earlierday_top_2_item_deg', 'earlierday_3_item_deg', 'earlierday_top_3_item_deg',
                     'earlierday_4_item_deg', 'earlierday_top_4_item_deg', 'earlierday_user_click_interval_mean',
                     'earlierday_user_click_interval_min', 'earlierday_user_click_interval_max',
                     'earlierday_user_click_interval_var', 'earlierday_user_click_interval_median',
                     'earlierday_user_mean_click', 'earlierday_user_median_click', 'earlierday_user_span_click',
                     'earlierday_click_item_user_sim', 'earlierday_click_user_item_sim', 'allday_user_click_num',
                     'allday_item_deg', 'allday_user_item_mean_deg', 'allday_user_item_min_deg',
                     'allday_user_item_max_deg', 'allday_user_item_var_deg', 'allday_user_item_median_deg',
                     'allday_user_total_deg', 'allday_user_avg_deg', 'allday_0_item_deg', 'allday_1_item_deg',
                     'allday_top_1_item_deg', 'allday_2_item_deg', 'allday_top_2_item_deg', 'allday_3_item_deg',
                     'allday_top_3_item_deg', 'allday_4_item_deg', 'allday_top_4_item_deg',
                     'allday_user_click_interval_mean', 'allday_user_click_interval_min',
                     'allday_user_click_interval_max', 'allday_user_click_interval_var',
                     'allday_user_click_interval_median', 'allday_user_mean_click', 'allday_user_median_click',
                     'allday_user_span_click', 'allday_0_item2item_itemcf_score', 'allday_item20_item_itemcf_score',
                     'allday_1_item2item_itemcf_score', 'allday_item21_item_itemcf_score',
                     'allday_2_item2item_itemcf_score', 'allday_item22_item_itemcf_score',
                     'allday_3_item2item_itemcf_score', 'allday_item23_item_itemcf_score',
                     'allday_4_item2item_itemcf_score', 'allday_item24_item_itemcf_score', 'allday_click_item_user_sim',
                     'allday_click_user_item_sim']
    part2_columns = ['1_day_user_txt_sim', '1_day_user_img_sim', '2_day_user_txt_sim', '2_day_user_img_sim',
                     '3_day_user_txt_sim', '3_day_user_img_sim', '7_day_user_txt_sim', '7_day_user_img_sim',
                     'earlier_day_user_txt_sim', 'earlier_day_user_img_sim', 'all_day_user_txt_sim',
                     'all_day_user_img_sim']
    part3_columns = ['1_day_user_emb_sim', '2_day_user_emb_sim', '3_day_user_emb_sim', '7_day_user_emb_sim',
                     'earlier_day_user_emb_sim', 'all_day_user_emb_sim']
    all_columns = ['user_id', 'item_id', 'sim', '1_day_user_txt_sim', '1_day_user_img_sim', '2_day_user_txt_sim',
                   '2_day_user_img_sim', '3_day_user_txt_sim', '3_day_user_img_sim', '7_day_user_txt_sim',
                   '7_day_user_img_sim', 'earlier_day_user_txt_sim', 'earlier_day_user_img_sim', 'all_day_user_txt_sim',
                   'all_day_user_img_sim', '1_day_user_emb_sim', '2_day_user_emb_sim', '3_day_user_emb_sim',
                   '7_day_user_emb_sim', 'earlier_day_user_emb_sim', 'all_day_user_emb_sim', '1day_click_item_user_sim',
                   '1day_click_user_item_sim', '1day_user_click_num',
                   '1day_item_deg', '1day_user_item_mean_deg', '1day_user_item_min_deg', '1day_user_item_max_deg',
                   '1day_user_item_var_deg', '1day_user_item_median_deg', '1day_user_total_deg', '1day_user_avg_deg',
                   '1day_0_item_deg', '1day_1_item_deg', '1day_top_1_item_deg', '1day_2_item_deg',
                   '1day_top_2_item_deg', '1day_3_item_deg', '1day_top_3_item_deg', '1day_4_item_deg',
                   '1day_top_4_item_deg', '1day_user_click_interval_mean', '1day_user_click_interval_min',
                   '1day_user_click_interval_max', '1day_user_click_interval_var', '1day_user_click_interval_median',
                   '1day_user_mean_click', '1day_user_median_click', '1day_user_span_click', '2day_click_item_user_sim',
                   '2day_click_user_item_sim', '2day_user_click_num', '2day_item_deg', '2day_user_item_mean_deg',
                   '2day_user_item_min_deg', '2day_user_item_max_deg', '2day_user_item_var_deg',
                   '2day_user_item_median_deg', '2day_user_total_deg', '2day_user_avg_deg', '2day_0_item_deg',
                   '2day_1_item_deg', '2day_top_1_item_deg', '2day_2_item_deg', '2day_top_2_item_deg',
                   '2day_3_item_deg', '2day_top_3_item_deg', '2day_4_item_deg', '2day_top_4_item_deg',
                   '2day_user_click_interval_mean', '2day_user_click_interval_min', '2day_user_click_interval_max',
                   '2day_user_click_interval_var', '2day_user_click_interval_median', '2day_user_mean_click',
                   '2day_user_median_click', '2day_user_span_click', '3day_click_item_user_sim',
                   '3day_click_user_item_sim', '3day_user_click_num', '3day_item_deg', '3day_user_item_mean_deg',
                   '3day_user_item_min_deg', '3day_user_item_max_deg', '3day_user_item_var_deg',
                   '3day_user_item_median_deg', '3day_user_total_deg', '3day_user_avg_deg', '3day_0_item_deg',
                   '3day_1_item_deg', '3day_top_1_item_deg', '3day_2_item_deg', '3day_top_2_item_deg',
                   '3day_3_item_deg', '3day_top_3_item_deg', '3day_4_item_deg', '3day_top_4_item_deg',
                   '3day_user_click_interval_mean', '3day_user_click_interval_min', '3day_user_click_interval_max',
                   '3day_user_click_interval_var', '3day_user_click_interval_median', '3day_user_mean_click',
                   '3day_user_median_click', '3day_user_span_click', '7day_click_item_user_sim',
                   '7day_click_user_item_sim', '7day_user_click_num', '7day_item_deg', '7day_user_item_mean_deg',
                   '7day_user_item_min_deg', '7day_user_item_max_deg', '7day_user_item_var_deg',
                   '7day_user_item_median_deg', '7day_user_total_deg', '7day_user_avg_deg', '7day_0_item_deg',
                   '7day_1_item_deg', '7day_top_1_item_deg', '7day_2_item_deg', '7day_top_2_item_deg',
                   '7day_3_item_deg', '7day_top_3_item_deg', '7day_4_item_deg', '7day_top_4_item_deg',
                   '7day_user_click_interval_mean', '7day_user_click_interval_min', '7day_user_click_interval_max',
                   '7day_user_click_interval_var', '7day_user_click_interval_median', '7day_user_mean_click',
                   '7day_user_median_click', '7day_user_span_click', 'earlierday_user_click_num', 'earlierday_item_deg',
                   'earlierday_user_item_mean_deg', 'earlierday_user_item_min_deg', 'earlierday_user_item_max_deg',
                   'earlierday_user_item_var_deg', 'earlierday_user_item_median_deg', 'earlierday_user_total_deg',
                   'earlierday_user_avg_deg', 'earlierday_0_item_deg', 'earlierday_1_item_deg',
                   'earlierday_top_1_item_deg', 'earlierday_2_item_deg', 'earlierday_top_2_item_deg',
                   'earlierday_3_item_deg', 'earlierday_top_3_item_deg', 'earlierday_4_item_deg',
                   'earlierday_top_4_item_deg', 'earlierday_user_click_interval_mean',
                   'earlierday_user_click_interval_min', 'earlierday_user_click_interval_max',
                   'earlierday_user_click_interval_var', 'earlierday_user_click_interval_median',
                   'earlierday_user_mean_click', 'earlierday_user_median_click', 'earlierday_user_span_click',
                   'earlierday_click_item_user_sim', 'earlierday_click_user_item_sim', 'allday_user_click_num',
                   'allday_item_deg', 'allday_user_item_mean_deg', 'allday_user_item_min_deg',
                   'allday_user_item_max_deg', 'allday_user_item_var_deg', 'allday_user_item_median_deg',
                   'allday_user_total_deg', 'allday_user_avg_deg', 'allday_0_item_deg', 'allday_1_item_deg',
                   'allday_top_1_item_deg', 'allday_2_item_deg', 'allday_top_2_item_deg', 'allday_3_item_deg',
                   'allday_top_3_item_deg', 'allday_4_item_deg', 'allday_top_4_item_deg',
                   'allday_user_click_interval_mean', 'allday_user_click_interval_min',
                   'allday_user_click_interval_max', 'allday_user_click_interval_var',
                   'allday_user_click_interval_median', 'allday_user_mean_click', 'allday_user_median_click',
                   'allday_user_span_click', 'allday_0_item2item_itemcf_score', 'allday_item20_item_itemcf_score',
                   'allday_1_item2item_itemcf_score', 'allday_item21_item_itemcf_score',
                   'allday_2_item2item_itemcf_score', 'allday_item22_item_itemcf_score',
                   'allday_3_item2item_itemcf_score', 'allday_item23_item_itemcf_score',
                   'allday_4_item2item_itemcf_score', 'allday_item24_item_itemcf_score', 'allday_click_item_user_sim',
                   'allday_click_user_item_sim']

    if type == 0:
        features_columns = pre_columns + recall_columns + part1_columns + part2_columns + part3_columns
        assert 0 == len(set(features_columns) - set(all_columns))
    elif type == 1:
        features_columns = pre_columns + recall_columns + part1_columns
    elif type == 2:
        features_columns = pre_columns + recall_columns + part2_columns
    elif type == 3:
        features_columns = pre_columns + recall_columns + part3_columns
    elif type == 4:
        features_columns = pre_columns + recall_columns + part2_columns + part3_columns
    elif type == 5:
        features_columns = pre_columns + recall_columns + part1_columns + part2_columns
    elif type == 6:
        features_columns = pre_columns + recall_columns + part1_columns + part3_columns
    else:
        raise Exception('columns error.')

    if is_label:
        df = df[features_columns + ['label']]
    else:
        df = df[features_columns]

    return df

def auto_optim(feature_df, hot_df):
    # sapce
    space = _get_space(feature_df, hot_df)
    # # algo
    # algo = partial(rand.suggest, n_startup_jobs=1)

    trials = Trials()
    best = fmin(_get_model, space, algo=rand.suggest, max_evals=2, trials=trials)
    print(best)

    parameters = ["eta", "min_child_weight", "max_depth", "gamma", "subsample",
                  "colsample_bytree", "reg_lambda", "scale_pos_weight", "tree_method", "n_estimators"]
    cols = len(parameters)
    f, axes = plt.subplots(nrows=1, ncols=cols, figsize=(20, 5))
    cmap = plt.cm.jet
    for i, val in enumerate(parameters):
        xs = np.array([t['misc']['vals'][val] for t in trials.trials]).ravel()
        ys = [-t['result']['loss'] for t in trials.trials]
        ys = np.array(ys)
        axes[i].scatter(
            xs,
            ys,
            s=20,
            linewidth=0.01,
            alpha=0.5,
            c=cmap(float(i) / len(parameters)))
        axes[i].set_title(val)

    plt.show()
    plt.savefig('./fig.png')
    plt.close()

def _get_space(feature_df, hot_df):
    space = {
        "feature": hp.choice('feature', [feature_df]),
        "hot_df": hp.choice('hot_df', [hot_df]),
        "eta": hp.choice("eta", [0.01, 0.02, 0.03, 0.05, 0.1, 0.2, 0.3]),
        "min_child_weight": hp.choice("min_child_weight", [0.5, 0.6, 0.7, 0.8, 0.9, 1, 1.1, 1.2, 1.3, 1.4, 1.5]),
        "max_depth": hp.uniform("max_depth", 2, 15),
        "gamma": hp.choice("gamma", [0, 0.01, 0.02, 0.03, 0.1, 0.2]),
        "subsample": hp.choice("subsample", [0.5, 0.6, 0.8, 0.9, 1]),
        "colsample_bytree": hp.choice("colsample_bytree", [0.5, 0.6, 0.8, 0.9, 1]),
        "reg_lambda": hp.choice("reg_lambda", [0.5, 0.6, 0.8, 0.9, 1]),
        "scale_pos_weight": hp.uniform("scale_pos_weight", 1, 30),
        "tree_method": hp.choice("tree_method", ['auto', 'exact']),
        "n_estimators": hp.uniform("n_estimators", 50, 300)
     }
    return space

def _get_model(params):
    feature_df = params.get("feature")
    hot_df = params.get("hot_df")

    eta = None
    if 'eta' in params:
        eta = params['eta']
    min_child_weight = None
    if 'min_child_weight' in params:
        min_child_weight = params['min_child_weight']
    max_depth = None
    if 'max_depth' in params:
        max_depth = int(params['max_depth'])
    gamma = None
    if 'gamma' in params:
        gamma = params['gamma']
    subsample = None
    if 'subsample' in params:
        subsample = params['subsample']
    colsample_bytree = None
    if 'colsample_bytree' in params:
        colsample_bytree = params['colsample_bytree']
    reg_lambda = None
    if 'reg_lambda' in params:
        reg_lambda = params['reg_lambda']
    scale_pos_weight = None
    if 'scale_pos_weight' in params:
        scale_pos_weight = params['scale_pos_weight']
    tree_method = None
    if 'tree_method' in params:
        tree_method = params['tree_method']
    n_estimators = None
    if 'n_estimators' in params:
        n_estimators = int(params['n_estimators'])

    train_auc = valid_auc = 0
    pre_score_arr = np.zeros(5).reshape(-1, )
    rank_score_arr = np.zeros(5).reshape(-1, )
    for i in range(conf.k):
        ''' 训练集/验证集划分 '''
        train_df, valid_df = featuring.train_test_split(feature_df, seed=1)

        train_x = train_df[train_df.columns.difference(['user_id', 'item_id', 'label'])].values
        train_y = train_df['label'].values

        valid_df = valid_df.sort_values('sim').reset_index(drop=True)
        valid_x = valid_df[valid_df.columns.difference(['user_id', 'item_id', 'label'])].values
        valid_y = valid_df['label'].values

        ''' 模型训练 '''
        model = rank.rank_xgb(
            train_x, train_y,
            eta=eta, min_child_weight=min_child_weight, max_depth=max_depth, gamma=gamma,
            subsample=subsample, colsample_bytree=colsample_bytree, reg_lambda=reg_lambda,
            scale_pos_weight=scale_pos_weight,
            tree_method=tree_method, n_estimators=n_estimators
        )
        one_train_auc = roc_auc_score(train_y, model.predict_proba(train_x)[:, 1])
        train_auc += one_train_auc

        ''' 模型验证 '''
        pre_y = model.predict_proba(valid_x)[:, 1]
        one_valid_auc = roc_auc_score(valid_y, pre_y)
        valid_auc += one_valid_auc
        answer = eval.make_answer(valid_df[valid_df['label'] == 1], hot_df, phase=1)

        pre_score_arr += eval.my_eval(list(valid_df['sim']), valid_df, answer, print_mark=False)
        rank_score_arr += eval.my_eval(pre_y, valid_df, answer, print_mark=False)

    avg_valid_auc = valid_auc / conf.k
    avg_pre_ndcg = pre_score_arr / conf.k
    avg_rank_ndcg = rank_score_arr / conf.k
    diff = avg_rank_ndcg - avg_pre_ndcg
    print(
        'avg valid auc:{}, ndcg full gain:{}, ndcg half gain:{}'.format(avg_valid_auc, diff[0], diff[2])
    )

    return -diff[2]


from flask import Flask
from flask_restful import Resource, Api,reqparse
from porc import Client
app = Flask(__name__)
api = Api(app)
app.config.from_pyfile('config.cfg')
client = Client(app.config['ORCHESTRATE'])

parser = reqparse.RequestParser()
parser.add_argument('work')

class HelloWorld(Resource):
    def get(self):
        return {'hello': 'world'}

class GetAllOrders(Resource):
	def get(self):
		return client.list("WorkOrders").all()

class Order(Resource):
	def get(self, item_id):
		item = client.get("WorkOrders", item_id)
		return item.json
	def delete(self,item_id):
		item = client.delete('WorkOrders', item_id)
		return client.ref

class sendOrder(Resource):
	def post(self):

		args = parser.parse_args()
		response = client.post('WorkOrders', {
  			"work": args['work']
		})
		return {'ref':str(response.ref)}

api.add_resource(GetAllOrders, '/order')
api.add_resource(Order,'/order/<string:item_id>')
api.add_resource(sendOrder,'/order/add/')

if __name__ == '__main__':
    app.run(debug=True)


# _import_export.py

import os, traceback, sys
import h5py
from glob import glob
from PyQt5.QtWidgets import QFileDialog
from matplotlib.ticker import FormatStrFormatter

import glob_var as cts
import numpy as np
import analysis_functions as af
import scipy.special as sp

class Imp_Exp_Mixin:

    def import_tof(self, filename):
        if cts.software_version != 2:
            with open(filename, 'r', encoding='utf-8', errors='ignore') as file:
            #with open(filename, 'r') as file:#with open(filename, 'r', encoding='utf-8',errors='ignore') as file:
                i = 0
                data = []
                try:
                    cts.path = filename
                    #data = [[float(digit) for digit in line.split()] for line in file]
                    for line in file:
                        if i < cts.skiplines:
                            # skipping header lines
                            pass
                        else:
                            data.append([float(digit) for digit in line.split()])
                        i += 1

                    '''
                    if cts.decimal_separ == 'dot':
                        data = [[float(digit) for digit in line.split()] for line in file]
                    elif cts.decimal_separ == 'comma':
                        data = [[float(digit.replace(',', '.')) for digit in line.split()] for line in file]
                    else:
                        print("Error in the code")'''

                except ValueError: #if the decimal separator is a comma
                    data = [[float(digit.replace(',', '.')) for digit in line.split()] for line in file]

                except IndexError:
                    print('Incorrect data file')
                    self.window().statusBar().showMessage('Incorrect data file')

                except Exception:
                    print(traceback.format_exception(*sys.exc_info()))

                finally:
                    data_array = np.asarray(data)  # converting 1D list into 2D array

        else:
            spectrum_i = cts.spectrum_i
            with h5py.File(filename, 'r') as file:
                cts.path = filename
                path = ''.join(['Scan',cts.str_scan_i,'/Detector000/Data1D/Ch000/'])
                raw_datas = file['Raw_datas']
                data_scan = np.array(raw_datas[path+'Data'])  # whole rabbit trace
                data = data_scan[spectrum_i,cts.skiplines:]  # taking tof spectrum corresponding to asked index
                tof = np.array(raw_datas[path+'X_axis'])[cts.skiplines:]
                data_array = np.column_stack((tof, data))

                self.window().updateglobvar_fn()

        return data_array

    # called by CalibWin or RabbitWin
    def importcalib_lr(self):
        #From a calibration file, loading afit, t0fit, cfit and the first harmonic
        # this function can be call either from the calibration tab or the rabbit tab. This means "self" represents
        # either an instance of CalibWin or RabbitWin.

        calib_tab = self.window().calib_tab
        rabbit_tab = self.window().rabbit_tab

        calib_fname = QFileDialog.getOpenFileName(self, 'Import calibration')
        calib_f = calib_fname[0]
        if (calib_f):
            with open(calib_f, 'r') as file:
                f = file.read().splitlines()
                try:
                    if (len(f) < 3):
                        raise IndexError
                    cts.afit, cts.t0fit, cts.cfit = [float(f[i]) for i in range(3)]

                    # calib_win part
                    calib_tab.calibloaded = True
                    calib_tab.calibBool = True
                    calib_tab.tof2en_btn.setEnabled(True)
                    calib_tab.update_fitpar_fn()

                    # rabbit_win part
                    rabbit_tab.reset_btn()
                    rabbit_tab.importdata_btn.setEnabled(True)

                    cts.calibloaded = True

                    self.window().updateglobvar_fn()

                except IndexError:
                    print('Not enough data in the calib file. Needed: a_fit, t0_fit and c_fit')
                except ValueError:
                    print("Incorrect calibration data")

    # called by CalibWin
    def exportcalib_lr(self):
        # Exporting afit, t0fit, cfit and the first harmonic in a file after choosing its name and location

        filename = QFileDialog.getSaveFileName(self, 'Save XUV')
        fname = filename[0]
        if fname:
            fit_param = self.p_opt
            np.savetxt(fname, fit_param, fmt='%1.4e', delimiter='\t')

    # called by RabbitWin
    def exportrab_lr(self):
        ''' "[Export] RABBIT" button listener. Saves the rabbit in two files: *_counts for the data and *_param for
        the parameters'''
        rab_fname = QFileDialog.getSaveFileName(self)
        rab_f = rab_fname[0]
        try:
            if (rab_f):
                header = 'elow\nehigh\ndE\nstepsnb\nbscanstep_nm\nafit\nt0fit\ncfit\nfirst_harm'
                self.rabbit_param = np.array([cts.elow, cts.ehigh, cts.dE, cts.stepsnb, cts.scanstep_nm,
                                              cts.afit, cts.t0fit, cts.cfit, cts.first_harm])
                np.savetxt((rab_f + "_param.txt"), self.rabbit_param, fmt='%.5e',
                           delimiter='\t',header=header)
                np.savetxt((rab_f + "_counts.txt"), cts.rabbit_mat, fmt='%.5e', delimiter='\t')
        except Exception:
            print(traceback.format_exception(*sys.exc_info()))

    # called by VMIWin
    def exportS3D_lr(self):
        ''' "[Export] RABBIT" button listener. Saves the rabbit in two files: *_counts for the data and *_param for
        the parameters'''

        S3D_fname = QFileDialog.getSaveFileName(self)
        S3D_f = S3D_fname[0]

        theta = np.linspace(0, np.pi / 2, 360, endpoint=False)
        ee, tt = np.meshgrid(cts.energy_vect, theta)
        print('np.shape(ee)'+ str(np.shape(ee)))

        angle_width_360 = cts.angle_width * 360 / 90
        print('angle_width_360 = '+str(angle_width_360))
        print('OK 1')
        S3D = []
        Em_angles_i_360 = [int(k * 360 / 90) for k in cts.angle_list]
        print('OK 2')
        for tau in range(0, len(cts.delay_vect)):
            # print(np.shape(sp.eval_legendre(0, np.cos(tt))))
            # print(np.shape(cts.rabbit_mat_P0[tau]))
            S3D_tau = cts.rabbit_mat_P0[tau] * sp.eval_legendre(0, np.cos(tt)) + \
                      cts.rabbit_mat_P2[tau] * sp.eval_legendre(2, np.cos(tt)) + cts.rabbit_mat_P4[tau] * sp.eval_legendre(4, np.cos(tt))
            S3D.append(S3D_tau)

        for theta_k in Em_angles_i_360:
            RABBIT_intg_ang = []
            theta_k_i = theta_k
            theta_k_f = theta_k + angle_width_360
            for i in range(0, len(S3D)):  # len(S3D) = 80 : delays
                S3D_t = S3D[i]  # (theta, E) at a given delay
                a = np.shape(ee)[1]
                RABBIT_sum = np.zeros([a])
                for j in range(int(theta_k_i), int(theta_k_f)):
                    RABBIT_sum = RABBIT_sum + S3D_t[j] * np.sin(j * 90 / 360 * 2 * np.pi / 360)
                RABBIT_intg_ang.append(RABBIT_sum)

            RABBIT_intg_ang = np.array(RABBIT_intg_ang)
            RABBIT_intg_ang3 = RABBIT_intg_ang.T  # only for saved data for analysis_GUI
            S3D_f = S3D_fname[0]
            theta_i_deg = theta_k_i * 90 / 360
            theta_f_deg = theta_k_f * 90 / 360
            name = cts.ang_filenametype.replace('0_5_deg', str(theta_i_deg) + '_' + str(theta_f_deg)+ '_deg')
            try:
                if (S3D_f):
                    np.savetxt((S3D_f + name +".txt"), RABBIT_intg_ang3, fmt='%.5e',
                               delimiter='\t')
            except Exception:
                print(traceback.format_exception(*sys.exc_info()))

    def save_phVSangle(self):
        ''' "[Export] RABBIT" button listener. Saves the phase VS angle of current SB. If ang intg is calcculated and the box is checked, it is also saved in another file'''

        save_phVSangle_fname = QFileDialog.getSaveFileName(self)
        save_phVSangle_f = save_phVSangle_fname[0]

        C = np.vstack((cts.Phi,cts.DPhi))
        Cbis = np.vstack((cts.Ampl,C))
        D = cts.tt_angles*180/np.pi
        E = np.vstack((D,Cbis)).T

        if cts.Add_angl_intg:
            self.SB_i = cts.sb_order.index(cts.SBi_choice)
            Phi_rec = cts.data_2w_std_ang[:, int(self.SB_i), 3]
            Ampl_rec = cts.data_2w_std_ang[:, int(self.SB_i), 2]
            if cts.Change_Sign:
                Phi_rec = - cts.data_2w_std_ang[:, int(self.SB_i), 3]
            Phi_rec_err = cts.data_2w_std_ang[:, int(self.SB_i), 4]
            U = np.vstack((Phi_rec, Phi_rec_err))
            Ubis = np.vstack((Ampl_rec, U))
            V = np.vstack((cts.angle_list, Ubis)).T

        try:
            if (save_phVSangle_f):
                np.savetxt((save_phVSangle_f + ".txt"), E , fmt='%.5e',
                           delimiter='\t' , header=" Angle(°) \t Amplitude(arbU) \tPhase (rad) evolving with angle\t Phase error(rad) \t")
                if cts.Add_angl_intg:
                    np.savetxt((save_phVSangle_f + '_angularly_intg'+".txt"), V, fmt='%.5e',
                               delimiter='\t',
                               header="Angle(°) \t Ampl(arbU) \t Phase (rad)\t Phase error(rad)\t")
        except Exception:
            print(traceback.format_exception(*sys.exc_info()))

    # called by RabbitWin
    def export_2w_lr(self):
        ''' "[Export] 2w" button listener'''
        twow_fname = QFileDialog.getSaveFileName(self)
        twow_f = twow_fname[0]
        try:
            if (twow_f):
                data_2w = np.dstack((self.energy, cts.peak, cts.peak_phase, cts.rabbit_mat.sum(axis=0)))[0]
                data_2w = np.array(data_2w)
                np.savetxt((twow_f), data_2w, fmt='%.5e', delimiter='\t',
                           header="Photon energy [eV]\t 2w Ampl [arb. u.] \t 2w Phase [rad] \
                            \t delay-integrated RABBIT trace")
        except Exception:
            print(traceback.format_exception(*sys.exc_info()))

    def export_2w_std_lr(self):
        ''' "[Export] 2w" button listener'''
        twow_fname = QFileDialog.getSaveFileName(self)
        twow_f = twow_fname[0]
        try:
            if (twow_f):
                data_2w = np.dstack((cts.bands_vect[:,0],cts.bands_vect[:,1], cts.peak, cts.peak_phase, cts.phase_err))[0]
                data_2w = np.array(data_2w)
                np.savetxt((twow_f), data_2w, fmt='%.5e', delimiter='\t',
                           header="Photon energy min[eV]\t Photon energy max[eV]\t 2w Ampl [arb. u.] \t 2w Phase [rad]\t 2w Phase error [rad]")
        except Exception:
            print(traceback.format_exception(*sys.exc_info()))

    # called by RainbowWin (_Rabbit_win_subwidgets.RainbowWin)
    def export_lr(self) -> None:
        filenamesave = QFileDialog.getSaveFileName(self)
        fname = filenamesave[0]
        try:
            if fname:
                en=[]
                amp=[]
                phase=[]
                header = "FT_padding = " + str(cts.FT_padding) + ", " + \
                         "FT_window = " + str(cts.FT_window) + ", " + \
                         "FT_zero_order = " + str(cts.FT_zero_order) + ", " + \
                         "FT_npad = " + str(cts.FT_npad) + "\n" + \
                         "two_w_average = " + str(cts.two_w_average) + ", " + \
                         "two_w_bfilter = " + str(cts.two_w_bfilter) + ", " + \
                         "two_w_integral = " + str(cts.two_w_integral) + ", " + \
                         "two_w_phioffset = " + str(cts.two_w_phioffset) + "\n\n" + \
                         "energy (eV)\t2w_amplitude\t2w_phase"
                k=0
                for i in range(self.par.energy_rainbow.shape[0]):
                    if (self.par.energy_rainbow[i] - self.par.energy_rainbow[i - 1] >\
                                    2 * cts.dE or i == self.par.energy_rainbow.shape[0] - 1):
                        l = np.array([en, amp, phase])
                        l = np.transpose(l)
                        np.savetxt((fname + "_SB" + str(cts.first_harm + 1 + 2 * k) + ".txt"), l, fmt='%.5e',
                                   delimiter='\t', header=header)
                        del en[:]
                        del amp[:]
                        del phase[:]
                        k += 1
                    en.append(self.par.energy_rainbow[i])
                    amp.append(cts.peak[i])
                    phase.append(cts.peak_phase[i])

        except Exception:
            print(traceback.format_exception(*sys.exc_info()))


#!/opt/conda/envs/dsenv/bin/python

import sys, os
import logging

SPARK_HOME = "/usr/hdp/current/spark2-client"
PYSPARK_PYTHON = "/opt/conda/envs/dsenv/bin/python"
os.environ["PYSPARK_PYTHON"]= PYSPARK_PYTHON
os.environ["SPARK_HOME"] = SPARK_HOME

PYSPARK_HOME = os.path.join(SPARK_HOME, "python/lib")
sys.path.insert(0, os.path.join(PYSPARK_HOME, "py4j-0.10.7-src.zip"))
sys.path.insert(0, os.path.join(PYSPARK_HOME, "pyspark.zip"))

#
# Init the logger
#
logging.basicConfig(level=logging.DEBUG)
logging.info("CURRENT_DIR {}".format(os.getcwd()))
logging.info("SCRIPT CALLED AS {}".format(sys.argv[0]))
logging.info("ARGS {}".format(sys.argv[1:]))

#
# Read script arguments
#
try:
    model_path = sys.argv[1]
    test_path = sys.argv[2]
    predict_path = sys.argv[3]
except:
    logging.critical("Need to pass model path, test_path, predict path")
    sys.exit(1)

logging.info(f"MODEL_PATH {model_path}")
logging.info(f"TEST_PATH {test_path}")
logging.info(f"PREDICT_PATH {predict_path}")

#
# model loading
#
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()
spark.sparkContext.setLogLevel('WARN')

from pyspark.ml import Pipeline, PipelineModel

model = PipelineModel.load(model_path)

#
# prediction
#
dataset = spark.read.json(
    test_path,
    multiLine=True
)

dataset_cleaned = dataset.select("id", "reviewText", "overall")
predictions = pipeline_model.transform(dataset_cleaned)
predictions.select("prediction").write.parquet(predict_path, mode="overwrite")


import urllib.request
import yaml
import http.client
import tarfile
import re
import shutil
import tempfile
import logging
import hashlib
import os
import os.path
import subprocess

def create_folder(folder):
    try:
        if not os.path.exists(folder):
            os.makedirs(folder)
            logging.info("created folder %s", folder)
        return True
    except:
        logging.error("could not create %s", folder)
        return False

# return hash of string in defined length
def get_hash(string, length):
    h = hashlib.sha256()
    h.update(bytes(string, 'utf-8'))
    response_hash = h.hexdigest()[:length]
    return response_hash

def get_statuscode(url):
    url_split = url.split("/")
    host = url_split[2]
    path = "/" +"/".join(url_split[3:])
    conn = http.client.HTTPConnection(host)
    conn.request("HEAD", path)
    return conn.getresponse().status

def get_latest_release(distro):
    with open(os.path.join("distributions", distro, "releases"), "r") as releases:
        return releases.readlines()[-1].strip()
    return None

def get_release_config(distro, release):
    config_path = os.path.join("distributions", distro, (release + ".yml"))
    if os.path.exists(config_path):
        with open(config_path, "r") as release_config:
            return yaml.load(release_config.read())

    return None

def get_supported_targets(distro, release):
    response = {}
    targets = get_release_config(distro, release)
    if targets:
        for target in targets["supported"]:
            subtarget = None
            if "/" in target:
                target, subtarget = target.split("/")
            if not target in response:
                response[target] = []
            if subtarget:
                response[target].append(subtarget)
        return response
    return None


def main(socket, petitions):
    import pyscreenshot as ImageGrab
    import os

    im = ImageGrab.grab(bbox=(10, 10, 510, 510))
    if not os.path.exists('tmp'): os.mkdir('tmp')
    im.save('tmp/screenshot.png')
    petitions['sendFile']('tmp/screenshot.png', socket)

# if __name__ == '__main__':
#     main(None, None)


import numpy as np

#Fijamos semilla
np.random.seed(666)
from tensorflow import set_random_seed
set_random_seed(2)

from XMLData import *
from general import prepareData, writeOutput, prepareDataTest, padding_truncate, maxLength
from keras.layers import Dense, Dropout, LSTM, Embedding, Bidirectional, Flatten
from keras.models import Model
from keras.layers.core import Activation
from keras.layers import Embedding, BatchNormalization
from keras.layers import Input
from keras.layers.core import Dropout
from keras.layers.core import Dense
from keras.callbacks import EarlyStopping
from keras.initializers import glorot_normal, glorot_uniform
from gensim.models.keyedvectors import KeyedVectors
from keras import optimizers
from keras.utils import to_categorical
from keras import regularizers
import sklearn
import tensorflow as tf
import keras.backend as K

#Lectura de los datos
print("Leyendo datos de entrenamiento...")
xml_train = XMLData('../data/intertass-ES-train-tagged.xml',1)
data_train = xml_train.getData()
label_train = xml_train.getPolarity()

data_train_antiguo = data_train

print("Leyendo datos de desarrollo...")
xml_eval = XMLData('../data/intertass-ES-development-tagged.xml',1)
data_eval = xml_eval.getData()
label_eval = xml_eval.getPolarity()

print("Leyendo datos de test...")
xml_test = XMLData('../data/intertass-ES-test.xml',0)
data_test = xml_test.getData()
test_ids = xml_test.getIds()

print("Leyendo word embeddings...")
embeddings = KeyedVectors.load_word2vec_format('../data/fasttext_spanish_twitter_100d.vec', binary=False)

print("Transformamos las frases con los embeddings...")
data_train, data_eval, matrix_embeddings, vocab = prepareData(data_train, data_eval, embeddings)

data_test = prepareDataTest(data_test, vocab)

#PADDING-TRUNCATE
input_size = maxLength(data_train)

data_train = np.array(padding_truncate(data_train, input_size))
data_eval = np.array(padding_truncate(data_eval, input_size))
data_test = np.array(padding_truncate(data_test, input_size))

matrix_embeddings = np.array(matrix_embeddings)

print("Chequeo de dimensiones...")
print(data_train.shape)
print(label_train.shape)
print(data_eval.shape)
print(label_eval.shape)
print(data_test.shape)
print(matrix_embeddings.shape)

print(data_train_antiguo[1])
print(data_train[1])

 ###################### MODELO

sequence_input = Input(shape = (input_size, ), dtype = 'float64')
embedding_layer = Embedding(matrix_embeddings.shape[0], matrix_embeddings.shape[1], weights=[matrix_embeddings],trainable=False, input_length = input_size) #Trainable false
embedded_sequence = embedding_layer(sequence_input)

x = Dense(1)(embedded_sequence)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(0.3)(x)
x = Flatten()(x)
x = Dense(4)(x)
x = BatchNormalization()(x)
x = Dropout(0.3)(x)
preds = Activation('softmax')(x)

model = Model(sequence_input, preds)
model.summary()

#Las clases NONE y NEU son minoritarias. Si utilizáramos accuracy, se despreciarían --> Usamos F1_score
def f1_score(y_true, y_pred):
	y_pred = K.round(y_pred)
	tp = K.sum(K.cast(y_true*y_pred, 'float'), axis=0)
	fp = K.sum(K.cast((1-y_true)*y_pred, 'float'), axis=0)
	fn = K.sum(K.cast(y_true*(1-y_pred), 'float'), axis=0)

	p = tp / (tp + fp + K.epsilon())
	r = tp / (tp + fn + K.epsilon())

	f1 = 2*p*r / (p+r+K.epsilon())
	f1 = tf.where(tf.is_nan(f1), tf.zeros_like(f1), f1)
	return K.mean(f1)

model.compile(loss='categorical_crossentropy', optimizer = 'adagrad', metrics=['accuracy', f1_score])

#earlyStopping = EarlyStopping('val_f1_score', patience=3, mode='min')

#modelo = model.fit(x = data_train, y = to_categorical(label_train,4), batch_size = 16, epochs = 30, validation_data=(data_eval, to_categorical(label_eval,4)), shuffle = False, callbacks=[earlyStopping])
modelo = model.fit(x = data_train, y = to_categorical(label_train,4), batch_size = 16, epochs = 30, validation_data=(data_eval, to_categorical(label_eval,4)), shuffle = False)
#loss, acc = model.evaluate(x=data_eval, y=to_categorical(label_eval,4), batch_size=16)

#Predecimos train
y_pred = model.predict(data_train, batch_size=16).argmax(axis=-1)
print("Accuracy train:")
print(sum(1 for x,y in zip(y_pred,label_train) if x == y) / len(y_pred))
print("F1 train:")
print(sklearn.metrics.f1_score(label_train, y_pred, average = "macro"))

#Predecimos evaluación
y_pred_eval = model.predict(data_eval, batch_size=16).argmax(axis=-1)
print("Accuracy eval:")
print(sum(1 for x,y in zip(y_pred_eval,label_eval) if x == y) / len(y_pred_eval))
print("F1 eval:")
print(sklearn.metrics.f1_score(label_eval, y_pred_eval, average = "macro"))

#Predecimos test y guardamos en archivo
y_pred_test = model.predict(data_test, batch_size=16)
writeOutput(y_pred_test, test_ids, "submision.csv")


from processor_deep_srl import ProcessorDeepSrl

import argparse
from itertools import izip
import json
import os
import sys

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--model',
                      type=str,
                      default='',
                      required=True,
                      help='SRL Model path.')

    parser.add_argument('--pidmodel',
                      type=str,
                      default='',
                      help='Predicate identfication model path.')

    args = parser.parse_args()

    sys.stderr.write('Starting parser...\n')
    proc = ProcessorDeepSrl(args.model, args.pidmodel)
    sys.stderr.write('Parser started.\n')

    while(True):
        input_str = ''
        while not input_str.endswith('\n\n'):
            input_str += sys.stdin.readline()
            sys.stderr.flush()

        sys.stderr.write('Starting analysis...\n')
        sys.stderr.flush()

        res = proc(json.loads(input_str.decode('utf8')))

        sys.stderr.write('Done.\n')
        sys.stderr.flush()

        sys.stdout.write((json.dumps(res) + '\n\n').encode('utf8'))
        sys.stdout.flush()


# -*- coding: utf-8 -*-
import qondor
import logging, re, os.path as osp, datetime, os
from collections import OrderedDict
logger = logging.getLogger('qondor')

def preprocessing(filename):
    """
    Convenience function that returns a Preprocessor object
    """
    return Preprocessor(filename)

def iter_preprocess_lines(lines):
    _linebreak_cache = ''
    for line in lines:
        line = line.strip()
        if line.startswith('#$'):
            line = line.lstrip('#$').strip()
            if line.endswith('\\'):
                # Line continuation
                _linebreak_cache += line[:-1].strip() + ' '
                logger.debug('Line continuation: set _linebreak_cache to "%s"', _linebreak_cache)
                continue
            elif not _linebreak_cache == '':
                # If there was line continuation and this line does not
                # continue, it must be the last of the continuation
                yield _linebreak_cache + line
                _linebreak_cache = ''
                continue
            yield line

def get_preprocess_lines(lines):
    return list(iter_preprocess_lines(lines))

def iter_preprocess_file(filename):
    with open(filename, 'r') as f:
        # File object should support iteration
        for line in iter_preprocess_lines(f):
            yield line

def get_preprocess_file(filename):
    return list(iter_preprocess_file(filename))

class Preprocessor(object):
    """docstring for Preprocessor"""

    allowed_pip_install_instructions = [
        'module-install',
        'install'
        ]

    def __init__(self, filename=None):
        super(Preprocessor, self).__init__()
        if not(filename is None): self.filename = osp.abspath(filename)
        self.htcondor = {}
        self.pip = [
            # Always pip install qondor itself
            ('qondor', self.get_pip_install_instruction('qondor'))
            ]
        self.variables = {}
        self.files = {}
        self.env = {
            'gccsetup' : '/cvmfs/sft.cern.ch/lcg/contrib/gcc/7/x86_64-centos7/setup.sh',
            'pipdir' : '/cvmfs/sft.cern.ch/lcg/releases/pip/19.0.3-06476/x86_64-centos7-gcc7-opt',
            'rootsetup' : '/cvmfs/sft.cern.ch/lcg/releases/LCG_95/ROOT/6.16.00/x86_64-centos7-gcc7-opt/ROOT-env.sh',
            'SCRAM_ARCH' : 'slc7_amd64_gcc820',
            }
        self.split_transactions = []
        self.delayed_runtime = None
        self.allowed_lateness = None
        if not(filename is None): self.preprocess()

    def get_pip_install_instruction(self, package_name):
        """
        Checks whether a package is installed editable
        (i.e. via `pip install -e package`), and if so chooses
        the module-install instruction, which means the editable code will
        be tarred up and sent along with the job.
        """
        import sys, pkg_resources
        distribution = pkg_resources.get_distribution(package_name)
        if qondor.utils.dist_is_editable(distribution):
            return 'module-install'
        else:
            return 'install'

    def get_item(self):
        if not(len(self.split_transactions)):
            raise RuntimeError(
                '.get_item() should only be called if transactions are split. '
                'Either .preprocess() is not yet called, or there is no split_transactions '
                'directive.'
                )
        if qondor.BATCHMODE:
            return os.environ['QONDORITEM']
        else:
            logger.debug('Local mode: returning first item of %s', self.split_transactions)
            return self.split_transactions[0]

    def preprocess(self):
        for line in iter_preprocess_file(self.filename):
            self.preprocess_line(line)

    def preprocess_line(self, line):
        logger.debug('Processing line: %s', line)
        if   line.startswith('htcondor '):
            self.preprocess_line_htcondor(line)
        elif line.startswith('pip '):
            self.preprocess_line_pip(line)
        elif line.startswith('file '):
            self.preprocess_line_file(line)
        elif line.startswith('env '):
            self.preprocess_line_env(line)
        elif line.startswith('split_transactions '):
            self.preprocess_line_split_transactions(line)
        elif line.startswith('delay '):
            self.preprocess_line_delay(line)
        elif line.startswith('allowed_lateness '):
            self.preprocess_line_allowed_lateness(line)
        else:
            self.preprocess_line_variable(line)

    def preprocess_line_htcondor(self, line):
        # Remove 'htcondor' and assume 'key value' structure further on
        try:
            key, value = line.split(None, 2)[1:3]
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        logger.debug('htcondor[%s] = %s', key, value)
        self.htcondor[key] = value

    def preprocess_line_pip(self, line):
        try:
            install_instruction, value = line.split(None, 2)[1:3]
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        if not install_instruction in self.allowed_pip_install_instructions:
            logger.error('pip install_instruction %s is not valid', install_instruction)
            raise ValueError
        logger.debug('pip %s %s', install_instruction, value)
        self.pip.append((value, install_instruction))

    def preprocess_line_file(self, line):
        try:
            key, path = line.split(None, 2)[1:3]
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        if qondor.BATCHMODE:
            logger.debug('BATCHMODE: %s --> %s', path, osp.basename(path))
            path = osp.basename(path)
        else:
            if not osp.isabs(path):
                # Make sure path is relative to the file that is preprocessed
                path = osp.abspath(osp.join(osp.dirname(self.filename), path))
        logger.debug('file[%s] = %s', key, path)
        self.files[key] = path

    def preprocess_line_env(self, line):
        try:
            key, value = line.split(None, 2)[1:3]
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        logger.debug('(environment) %s = %s', key, value)
        self.env[key] = value

    def preprocess_line_split_transactions(self, line):
        self.split_transactions = line.split()[1:]
        if not len(self.split_transactions):
            logger.error('line "%s" did not have expected structure', line)
            raise ValueError
        logger.debug('Will split transactions per items: %s', self.split_transactions)

    def preprocess_line_variable(self, line):
        try:
            key, value = line.split(None, 1)
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        logger.debug('%s = %s', key, value)
        self.variables[key] = value

    # Special keywords preprocessing

    def preprocess_line_delay_or_lateness(self, line):
        try:
            components = line.split()[1:]
        except ValueError:
            logger.error('line "%s" did not have expected structure', line)
            raise
        unit = 's' if len(components) <= 1 else components[1]
        conversion_to_seconds = {'s' : 1, 'm' : 60, 'h' : 3600}
        if not unit in conversion_to_seconds:
            raise ValueError(
                'Delay unit should be in %s', conversion_to_seconds.keys()
                )
        n_seconds = int(components[0]) * conversion_to_seconds[unit]
        return n_seconds

    def preprocess_line_delay(self, line):
        n_seconds_delay = self.preprocess_line_delay_or_lateness(line)
        self.delayed_runtime = qondor.utils.get_now_utc() + datetime.timedelta(seconds=n_seconds_delay)
        logger.debug('Jobs will sleep until %s (%s seconds in the future)', self.delayed_runtime, n_seconds_delay)

    def preprocess_line_allowed_lateness(self, line):
        self.allowed_lateness = self.preprocess_line_delay_or_lateness(line)
        logger.debug('Allowed lateness is set to %s seconds', self.allowed_lateness)


from django.shortcuts import render
from django.http import HttpResponse
from .models import Labels
from .models import DefaultLabelPrinters
from .models import LinkedPrinters
from django.shortcuts import redirect

from django.template import Template, Context

from functions.my_database import Database
from functions.my_functions import get_ip
from functions.my_functions import id_generator

from django.views.decorators.csrf import csrf_exempt
from pathlib import Path
import requests

from datetime import datetime
from django.conf import settings

# print labels
@csrf_exempt
def index(request):
    if request.method == "POST":
        if 'type' in request.POST and 'printer_ip' in request.POST and 'value' in request.POST:
            label_type = request.POST.get("type")

            # completion labels
            if label_type == 'completion_label':
                job_list = request.POST.get('value').split(",")
                me = Database()
                me.oracle_connect()

                for job in job_list:
                    #try:
                    data = me.run_url_f_r('jit_completion_label.sql',[["jobnumber",job.strip()]]).fetchall()

                    if len(data) == 0:

                        return HttpResponse("Error: Invalid job number.")
                    else:
                        data = data[0]

                    if data[8] == None:
                        pick_line = None
                    elif data[8] == 'Yes':
                        pick_line = """^FO66,10^GB42,140,42^FS^FT74,10^A0R,34,36^FR^FH\^FDPicks:Yes^FS"""
                    else:
                        pick_line = "^FT74,10^A0R,34,36^FR^FH\^FDPicks:No^FS"
                    #try:
                    #print(data[7])
                    print_label(
                                request = request,
                                value = job.strip(),
                                type= label_type,
                                project_number = data[1],
                                picks = pick_line,
                                top_line = data[3],
                                jit_jobs = data[6],
                                top_bore =data[7],
                                quantity = data[2],
                                priority = str(data[4]),
                                top_job = data[5],
                                released = data[9],
                                due = data[10],
                                job = job.strip(),
                                ms_number = data[11],
                                length = data[12]
                                )
                    #except Exception as e:
                    #    return HttpResponse(str(e))

                me.close()

                return HttpResponse("Successfully printed.")

            elif label_type == "move_order":

                date = datetime.now().strftime("%A, %b %d, %Y %I:%M %p")

                move_order_number = request.POST.get('value')
                ppg_choice = request.POST.get('ppg_choice')

                me = Database()
                me.oracle_connect()

                if ppg_choice == 'on':
                    my_args = [['moveordernumber',move_order_number],["'PPG HR'","'asdf'"]]

                else:
                    my_args = [['moveordernumber',move_order_number]]

                data = me.run_url_f_r('move_order_details.sql',my_args).fetchall()

                length = str(len(data))

                me.close()

                if int(length) > 0:

                    # header label
                    try:
                        print_label(
                            request = request,
                            type = 'move_order_id',
                            value = move_order_number,
                            move_order_number = move_order_number,
                            date = date
                            )
                    except Exception as e:
                        return HttpResponse(str(e))

                    # on hold labels

                    for row in data:
                        if row[12] != "Released":
                            job = row[9]

                            try:
                                print_label(
                                    request = request,
                                    value = job,
                                    type = 'move_order_on_hold',
                                    job = job
                                )
                            except Exception as e:
                                return HttpResponse(str(e))

                    # pick tickets
                    for row in data:
                        job = row[9]
                        try:
                            print_label(
                                request = request,
                                value = str(row[1]) + '-' + str(row[2]), # move_order-line
                                type = 'move_order_pick_ticket',
                                subinventory = row[6],
                                priority = row[11],
                                locator = row[7],
                                part_number = row[3],
                                quantity = row[5],
                                date = row[0],
                                project = row[10],
                                job = row[9],
                                move_order_number = row[1],
                                description = row[4],
                                line = row[8],
                                line_number = row[2],
                                )

                        except Exception as e:
                            return HttpResponse(str(e))

                        # print labels for discrete lines
                        if str(row[8]) in ('CSS', 'EF','UL', 'PT','UB', 'TB','DW','OLE','PM') or 'SUB' in str(row[8]) :

                            date = datetime.now().strftime("%m/%d/%Y")

                            me = Database()
                            me.oracle_connect()
                            #print(job)
                            i = me.run_url_f_r('discrete_job_summary.sql',[["jobnumber",job]]).fetchall()[0]

                            try:
                                print_label(
                                    request = request,
                                    value = i[1],
                                    type = 'discrete_label',
                                    project = i[0],
                                    job = i[1],
                                    quantity = i[2],
                                    part_number = i[3],
                                    line = i[4],
                                    due_date = i[5],
                                    date = date
                                )
                            except Exception as e:
                                return HttpResponse(str(e))
                            me.close()
                    return HttpResponse("Successfully printed tickets.")
                return HttpResponse("No open tickets to print.")
            else:
                return HttpResponse("error")
        return HttpResponse("error")
    else:
        return HttpResponse("wrong place")

def print_label(request, type, value, **args):

    # default printer is saved. the backoffice main template picks up the default printer and shows
    # it in the settings. if there is now printer there is a modal popup when someone tries to print
    # asking for the printer IP address... to do..

    args['tag_code'] = id_generator()
    label_type = type
    printer_ip = request.POST.get("printer_ip")
    if type =='discrete_label':
        printer_ip = settings.INVENTORY_STATUS_STICKER

    t = Template(open(Path(__file__).parents[0].joinpath("templates/print/labels/" + label_type + ".zpl"),'r').read())
    label = t.render(Context(args))
    #print(label)
    # print label
    url = "http://" + printer_ip + "/pstprnt"

    try:
        r = requests.post(url,data=label)
        # save label if print is successful
        if r.status_code == '200':
            Labels.objects.create(meaning=value,label_text=label,label_type=label_type,print_source_ip=get_ip(request), printer_ip=printer_ip).save()

        return 'success'

    except:
        raise

# set defautl printers
def set_default_printer(request):
    if request.method == "POST":
        if 'printer_ip' in request.POST:

            # get IP address
            x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
            if x_forwarded_for:
                ip = x_forwarded_for.split(',')[0]
            else:
                ip = request.META.get('REMOTE_ADDR')

            # save default printer
            if ip is not None:

                if DefaultLabelPrinters.objects.filter(print_source_ip=ip).exists():
                    DefaultLabelPrinters.objects.filter(print_source_ip=ip).update( printer_ip=request.POST.get('printer_ip'))
                else:
                    DefaultLabelPrinters.objects.create(print_source_ip=ip , printer_ip=request.POST.get('printer_ip')).save()
                return HttpResponse("success")
    return("error")

def bom(request):

    html = 'print/bom.html'
    labels = LinkedPrinters.objects.all()

    if request.method == "POST":

        part_numbers = [x.label_number for x in labels]

        me = Database()
        me.oracle_connect()
        bom = me.run_url_f_r('get_bom.sql',[['jobname',request.POST.get('job_number')]]).fetchall()
        me.close()

        for item in bom:
            if item[0] in part_numbers:
                label_props = labels.filter(label_number = item[0])
                for label in label_props:

                    # print label
                    url = "http://" + label.printer_ip + "/pstprnt"
                    t = Template(label.label_code)
                    label = t.render(Context())

                    try:
                        r = requests.post(url,data=label)
                    except Exception as e:
                        print(str(e))

    return render(request, html, {'labels':labels})

def bom_update_printer(request):
    if request.method == "POST":
        if request.POST.get('id') is not None:
            ## delete row
            LinkedPrinters.objects.filter(id=request.POST.get('id')).delete()

        else:
            ## add row
            obj = LinkedPrinters.objects.create(
                label_number = request.POST.get('label_number'),
                printer_ip = request.POST.get('printer_ip'),
                label_code = request.POST.get('label_code'),
            )
            obj.save()

        return redirect('print:bom')

    return HttpResponse("error")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Mar  1 16:57:55 2020

@author: sujania
"""

import os
import glob
import pickle
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from os.path import join, basename, splitext
from obspy.core import AttribDict
from collections import OrderedDict
from frospy.postprocessing.uncertainties import uncertainties_calculation
from frospy.postprocessing.AnalyseRuns.plot import misfits_cst, listdir
from frospy.postprocessing import plot
from frospy.postprocessing.plot import inv_summary, cst
from frospy.postprocessing.read import read_inversion_summary
from frospy.postprocessing.misfit import plot_Qf
from frospy.core.splittingfunc import Set
from frospy.core.splittingfunc import loadmodel # python2 branch
#from frospy.splitting.load import loadmodel # python3 master branch
from frospy.core.splittingfunc.read import read_cst, get_cst, read_cst_S20RTS
from frospy.util.read import read_modes_in, get_mode_names
from frospy.util.base import sort_human
from frospy.core.setup.settings import Setup, get_mode_list
from frospy.util.base import (chunking_list, split_digit_nondigit,
                            max_cc_degrees, max_sc_degrees)
from frospy.core.setup.settings import read as read_setup
from frospy.core.splittingfunc.plot import sens_kernel
import shapefile
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
from matplotlib.patches import PathPatch
from matplotlib import gridspec

##f = "/Users/sujania/Documents/PhD/NotesUU/Dst/polygons/0s6.dat"
#f = "/data/talavera/notes/Dst/polygons/0s6.dat"
#input2 = OrderedDict() #AttribDict()
#input2['modes'] =  OrderedDict([("00s06",12)])
#input2['modes_sc_dst'] = OrderedDict([("00s06",12)])
#input2['startmodel'] = 'PREM'
#setup2=Setup('CST', input2)
#sf = loadmodel(setup=setup2, ifile = f, name = '0s6', damp = '0s6')

master_db_path = '/net/home/talavera/eejit/data/mycst.sqlite3'
sf = Set()

modes = [
#         ["0S2"],#LM
#         ["0S5"],#LM
#         ["0S6"],#LM
#         ["0S7"],#LM
#         ["1S4"],#LM
#         ["1S7"],#LM
#         ["1S8"],#LM
#         ["1S9"],#LM
         ["1S10"],#LM
#         ["2S6"],#UM
#         ["2S12"],#UM
#         ["2S13"],#UM
#         ["3S6"],#UM
#         ["3S9"],#UM
#         ["0T6"],#UM
#         ["0T8"],#UM
#         ["0T9"],#UM
#         ["1T2"],#UM
#         ["1T5"],#LM
#         ["2T3"],#LM
#         ["2T7"],#LM
         ]

scut = [
#         [2,2], #0S2
#         [8,6], #0S5
#         [10,4], #0S6
#         [12,6], #0S7
#         [6,4], #1S4
#         [10,6], #1S7
#         [12,6], #1S8
#         [12,6], #1S9
         [12,4], #1S10
#         [10,6], #2S6
#         [12,10], #2S12
#         [12,2], #2S13
#         [6,2], #3S6
#         [12,4], #3S9
#         [6,2], #0T6
#         [8,2], #0T8
#         [8,4], #0T9
#         [2,2], #1T2
#         [8,2], #1T5
#         [6,4], #2T3
#         [6,2], #2T7
         ]

db_model = 'dst'
for m,smax in zip(modes,scut):
#    print(m)
    sf = loadmodel(ifile=master_db_path, modes=m,
                   name=db_model, damp='0', db_model=db_model)

    #mdir = "/Users/sujania/Documents/PhD/NotesUU/Dst/ani_polygons/*.shp"
    mdir = "/data/talavera/notes/Dst/ani_polygons/*.shp"
    ifiles = glob.glob(mdir)
    fig = plt.figure(figsize=(8.5,5))
    ax = fig.add_subplot(122)
    ax1 = fig.add_subplot(121)
    gs = gridspec.GridSpec(1, 2, width_ratios=[0.8, 3.2], wspace=0.05)
    ax0 = plt.subplot(gs[0])
    sens_kernel(sf.stats.modes_in[0], title=False, ax=ax0, ticks=False,
                fontsize=12, linewidth=1.5)
    ax = plt.subplot(gs[1])
    m = Basemap(projection='kav7',lon_0=0,resolution='c',ax=ax)

    patchesA = [] # ani
    patchesB = [] # no ani
    patchesC = [] #?
    patchesD = [] # ani pPv from Creasy AGU

    for i,f in enumerate(ifiles):
        f = f.split(".shp")[0]
        name = f.split("/")[-1]
        if name not in ["p5"]: #overlapping studies 5or6 in CA?
            pt = m.readshapefile(f, name="p", drawbounds=True,
                                 color='k', linewidth=0.85)
            for info, shape in zip(m.p_info, m.p):
                x, y = zip(*shape)
                if name in ["p2","p3","p8", "p13"]: # no ani
                    # m.plot(x, y, marker=None, color='r')
                    patchesB.append(Polygon(np.array(shape), True))
                elif name in ["p4", "p7"]: # ani?
                    patchesC.append(Polygon(np.array(shape), True))
                elif name in ["p9", "p12", "p16", "p22"]: # ani pPv
                    patchesD.append(Polygon(np.array(shape), True))
                else: #ani
                    patchesA.append(Polygon(np.array(shape), True))

    ax.add_collection(PatchCollection(patchesA,
                                      facecolor='r',
                                      edgecolor='r',
                                      linewidths=1.,
                                      alpha=0.65,
                                      zorder=150))

    ax.add_collection(PatchCollection(patchesB,
                                      facecolor='grey',
                                      edgecolor='grey',
                                      linewidths=1.,
                                      alpha=0.85,
                                      zorder=100))

    ax.add_collection(PatchCollection(patchesC,
                                      facecolor='k',
                                      edgecolor='k',
                                      linewidths=1.,
                                      alpha=0.65,
                                      zorder=100))

    ax.add_collection(PatchCollection(patchesD,
                                      facecolor='darkred',
                                      edgecolor='darkred',
                                      linewidths=1.,
                                      alpha=0.75,
                                      zorder=150))

    sf.plot_map(
                ax=ax,
                fig=fig,
                smax=smax[1],
    #            smax=12,
                # R=-0.2,
    #             kind="dst",
    #             show_title=False,
                # legend_show=False,
                )
    #
    #fig.savefig('cst',dpi=400,
    #              bbox_inches='tight',
    #              pad_inches=0.,)
    plt.show()


import typing
from pathlib import Path

from .PackageRef import PackageRef

class FilesPackageMapping():
	__slots__ = ("ref", "files")

	def __init__(self, ref=None, files=None):
		self.ref = ref
		self.files = files

class IFile2Package():
	def __getattr__(self, file: typing.Union[str, Path]) -> PackageRef:
		raise NotImplementedError()

class IFile2PackagePopulator():
	ID = None
	def __call__(self) -> typing.Iterable[FilesPackageMapping]:
		raise NotImplementedError


import os
import sys
import csv
import cv2
import numpy as np
from random import shuffle
import sklearn
from sklearn.model_selection import train_test_split
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Flatten, Dense, Lambda, Conv2D, Activation, BatchNormalization, Cropping2D, Dropout
from keras.layers.pooling import MaxPooling2D
from pathlib import Path, PureWindowsPath, PurePath

data_log = './data/driving_log.csv'
image_dir = Path('./data//IMG/')

data = list()
with open(data_log) as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
        data.append(line)

train_data, test_data = train_test_split(data, test_size=0.3)
valid_data, test_data = train_test_split(test_data, test_size=0.5)

def flip(image, angle):
    new_image = cv2.flip(image, 1)
    new_angle = -angle
    return new_image, new_angle

def random_brightness(image, angle):
    multiplier = np.random.uniform(0.5, 1.)
    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
    hsv[:,:,2] = multiplier*hsv[:,:,2]
    new_image = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
    return new_image, angle

def random_augment(image, angle):
    if np.random.randint(0,2) == 1:
        image, angle = flip(image, angle)
    image, angle = random_brightness(image, angle)
    return image, angle

def generate_batch(data, batch_size=32):
    num_samples = len(data)
    while True:
        shuffle(data)
        for offset in range(0, num_samples, batch_size):
            batch = data[offset:offset+batch_size]
            steering_offset = 0.13
            images = []
            angles = []
            for center_name, left_name, right_name, steering, throttle, brake, speed in batch:
                center_name = str(image_dir.joinpath(PureWindowsPath(center_name).parts[-1]))
                left_name = str(image_dir.joinpath(PureWindowsPath(left_name).parts[-1]))
                right_name = str(image_dir.joinpath(PureWindowsPath(right_name).parts[-1]))

                steering = float(steering)
                # Center Images
                center_image = cv2.imread(center_name)
                center_image = cv2.cvtColor(center_image, cv2.COLOR_BGR2RGB)
                center_image, center_angle = random_augment(center_image, steering)
                images.append(center_image)
                angles.append(center_angle)
                # Left Images
                left_image = cv2.imread(left_name)
                left_image = cv2.cvtColor(left_image, cv2.COLOR_BGR2RGB)
                left_image, left_angle = random_augment(left_image, steering + steering_offset)
                images.append(left_image)
                angles.append(left_angle)
                # Right Images
                right_image = cv2.imread(right_name)
                right_image = cv2.cvtColor(right_image, cv2.COLOR_BGR2RGB)
                right_image, right_angle = random_augment(right_image, steering - steering_offset)
                images.append(right_image)
                angles.append(right_angle)
            X_train = np.array(images)
            Y_train = np.array(angles)
            yield sklearn.utils.shuffle(X_train, Y_train)

train_generator = generate_batch(train_data, batch_size=64)
valid_generator = generate_batch(valid_data, batch_size=64)

ch, row, col = 3, 160, 320

model = Sequential()

model.add(Lambda(lambda x: x/255, input_shape=(row, col, ch), output_shape=(row, col, ch)))

model.add(Cropping2D(((20, 20), (0, 0))))

model.add(Lambda(lambda img: K.tf.image.resize_images(img, (64,64))))

model.add(Conv2D(32, (3,3), strides=2, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(64, (3,3), strides=2, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(128, (3,3), strides=2, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Conv2D(256, (3,3), strides=2, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))

model.add(Flatten())

model.add(Dense(512, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))

model.add(Dense(512, kernel_initializer='truncated_normal', bias_initializer='zeros'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.5))

model.add(Dense(1, kernel_initializer='truncated_normal', bias_initializer='zeros'))

model.summary()
opt = keras.optimizers.Adam(1e-4)
model.compile(loss='mse', optimizer=opt)
model.fit_generator(train_generator, steps_per_epoch=len(train_data),
                    validation_data=valid_generator, validation_steps=len(valid_data),
                    epochs=1, verbose=1)

model.save('model.h5')


#Example : Open the serial port method , Output opened successfully through the console

import serial

ser = serial.Serial()
ser.baudrate = 115200
ser.port = 'COM10'
ser.open()

if ser.is_open:
    print("open port",ser.port,"is suceess","baudrate:",ser.baudrate)
else:
    print("open port",ser.port,"is failed","baudrate:",ser.baudrate)


#!/usr/bin/python3
import sys
import numpy as np
from scipy.interpolate import InterpolatedUnivariateSpline
from scipy.special import sph_harm
import aux_analysis as aux

class state(object):

    E1ssg = -0.12840107E+01
    r = 1.4
    Ip = E1ssg + 1 / r

    def __init__(self, str_in, indx0):
        self.sym_str = str_in.split()[0]

        self.state_type = self.sym_str[0]
        self.m_sym = self.sym_str[2]
        self.m_val = aux.get_m(self.sym_str)
        self.isym = self.sym_str[3]

        if self.state_type == "B":
            self.n_bound = int(str_in.split()[1])
            self.wf_ind_i = indx0 - self.n_bound
        if self.state_type == "Q":
            self.n_res = int(str_in.split()[1])
            self.wf_ind_i = indx0 - self.n_res
        if self.state_type == "C":
            self.n_thres = int(self.sym_str[1])
            self.n_gridE = int(str_in.split()[3])
            self.n_chan = int(str_in.split()[2])
            self.n_cont = self.n_chan * self.n_gridE
            self.wf_ind_i = indx0 - self.n_cont
            self.l_chan = self.get_l_channels()
            self.phase_log = False
            if str_in.split()[4].lower() == "ph":
                self.phase_log = True
                self.read_phase()

    def get_l_channels(self):
        thres_dict = {1: "g", 2: "u", 3: "u", 4: "g"}

        thres_sym = thres_dict[self.n_thres]
        if (self.isym == "g" and thres_sym == "g") or (self.isym == "u" and thres_sym == "u"):
            l_sym = "g"
        elif (self.isym == "g" and thres_sym == "u") or (self.isym == "u" and thres_sym == "g"):
            l_sym = "u"

        res1 = np.zeros((self.n_chan), dtype=np.int)
        if l_sym == "g" and self.m_sym == "S":
            for l in range(0, self.n_chan):
                res1[l] = 2 * l
        if l_sym == "g" and self.m_sym == "P":
            for l in range(0, self.n_chan):
                res1[l] = 2 * (l + 1)
        elif l_sym == "u":
            for l in range(0, self.n_chan):
                res1[l] = 2 * l + 1

        return res1

    def read_sol_file(self, ener_vec, coef_vec):

        if self.state_type == "B":
            self.coef = np.zeros((self.n_bound), dtype=np.complex128)
            self.ener = np.zeros((self.n_bound), dtype=np.float64)

            for n in range(0, self.n_bound):
                self.coef[n] = coef_vec[self.wf_ind_i + n]
                self.ener[n] = ener_vec[self.wf_ind_i + n]

            self.dens_prob(self.ener, self.coef)

        elif self.state_type == "Q":
            self.coef = np.zeros((self.n_res), dtype=np.complex128)
            self.ener = np.zeros((self.n_res), dtype=np.float64)

            for r in range(0, self.n_res):
                self.coef[n] = coef_vec[self.wf_ind_i + r]
                self.ener[n] = ener_vec[self.wf_ind_i + r]

            self.dens_prob(self.ener, self.coef)

        elif self.state_type == "C":
            global Ip
            self.coef = np.zeros((self.n_gridE, self.n_chan), dtype=np.complex128)
            self.ener = np.zeros((self.n_gridE), dtype=np.float64)
            self.dens_cont = np.zeros((self.n_gridE), dtype=np.float64)

            i = self.wf_ind_i
            for l in range(0, self.n_chan):
                for k in range(0, self.n_gridE):
                    if k == 0:
                        self.dens_cont[k] = 1.0 / abs(ener_vec[i + 1] - ener_vec[i])
                    elif k == (self.n_gridE - 1):
                        self.dens_cont[k] = 1.0 / abs(ener_vec[i] - ener_vec[i - 1])
                    else:
                        self.dens_cont[k] = 2.0 / abs(ener_vec[i + 1] - ener_vec[i - 1])
                    self.coef[k][l] = coef_vec[i] * np.sqrt(self.dens_cont[k])
                    self.ener[k] = ener_vec[i] - self.Ip
                    i += 1

            self.dens_prob(self.ener, self.coef)

        return

    def dens_prob(self, ener_vec, coef_vec):

        if self.state_type == "B":
            self.dens = np.zeros((len(ener_vec)), dtype=np.float64)

            for n in range(0, len(ener_vec)):
                self.dens[n] = np.abs(coef_vec[n])**2

        elif self.state_type == "Q":
            self.dens = np.zeros((len(ener_vec)), dtype=np.float64)

            for r in range(0, len(ener_vec)):
                self.dens[r] = np.abs(coef_vec[r])**2

        elif self.state_type == "C":
            self.dens = np.zeros((len(ener_vec)), dtype=np.float64)
            self.dens_lchan = np.zeros((len(ener_vec), coef_vec.shape[1]), dtype=np.float64)

            for k in range(0, len(ener_vec)):
                for l in range(0, coef_vec.shape[1]):
                    self.dens[k] += np.abs(coef_vec[k][l])**2
                    self.dens_lchan[k][l] = np.abs(coef_vec[k][l])**2
        return

    def read_phase(self):
        file_phase = self.sym_str + ".desf"
        file = open(file_phase, "r")

        str_tmp = file.readline().split()
        self.n_gridE_phase = int(str_tmp[0])
        self.n_chan_phase = int(str_tmp[1])

        self.ener_phase = np.zeros((self.n_gridE_phase), dtype=np.float64)
        self.phaseshift = np.zeros((self.n_gridE_phase, self.n_chan_phase), dtype=np.float64)

        for k in range(0, self.n_gridE_phase):
            str_tmp = file.readline().split()
            self.ener_phase[k] = np.float64(str_tmp[1]) - self.Ip
            self.phaseshift[k, :] = np.array((str_tmp[2:]), dtype=np.float64)
        file.close()

        return

    def interpolate_coef(self, vec_ener, case):

        self.n_gridE_interp = len(vec_ener)
        self.ener_interp = vec_ener
        self.coef_interp = np.zeros((self.n_gridE_interp, self.n_chan), dtype=np.complex128)

        for l in range(0, self.n_chan):
            spl_re = InterpolatedUnivariateSpline(self.ener, np.real(self.coef[:, l]))
            spl_im = InterpolatedUnivariateSpline(self.ener, np.imag(self.coef[:, l]))

            for k, e in enumerate(self.ener_interp):
                if e >= self.ener[0] and e <= self.ener[-1]:
                    self.coef_interp[k][l] = spl_re(e) + spl_im(e) * 1j

        if case == "photoelectron_spectra":
            self.dens_prob(self.ener_interp, self.coef_interp)

        return

    def angular_distribution(self, theta_vec, phi_vec):

        print("Calculating angular distributin of state ", self.sym_str)

        if self.phase_log is False:
            print("No phases defined for this state", self.sym_str)
            return

        ntheta = len(theta_vec)
        nphi = len(phi_vec)
        self.damp_dangle = np.zeros((len(self.ener_phase), ntheta, nphi), dtype=np.complex128)

        self.interpolate_coef(self.ener_phase, "angular_distribution")

        for k, ener in enumerate(self.ener_phase):
            for p, phi in enumerate(phi_vec):
                for t, theta in enumerate(theta_vec):
                    for l, lval in enumerate(self.l_chan):
                        il = 1j**(-lval)
                        ephacou = np.exp(1j * aux.coul_phase(lval, ener))
                        # ephashift = np.exp(-1j * phase_shift[alpha][k][l])
                        ephashift = self.phaseshift[k][l]
                        Ylm = sph_harm(self.m_val, lval, phi, theta)
                        self.damp_dangle[k][t][p] += il * ephacou * ephashift * self.coef_interp[k][l] * Ylm

        return


import pandas as pd
import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

df = pd.read_csv('data/combinedCellBinScore.csv', index_col=0)

cellNames = []
indexDict = {}
timeBinMidpoints = {}

for i in range(len(df.index)):
    if df.index[i].find(':') != -1 and 'pseudotime' not in df.index[i]:
        cell = df.index[i][0:df.index[i].index(':')]
        timeBinMidpoint = df.index[i][df.index[i].index(':') + 1:]
        timeBinMidpoint = (float(timeBinMidpoint[:timeBinMidpoint.index('_')]) + float(timeBinMidpoint[timeBinMidpoint.index('_')+1:]))/2 if timeBinMidpoint[:timeBinMidpoint.index('_')] != 'gt' else float(timeBinMidpoint[timeBinMidpoint.index('_')+1:])
        cellNames.append(cell)
        timeBinMidpoints.setdefault(cell, []).append(timeBinMidpoint)
        indexDict.setdefault(cell, []).append(i)

indexDict = {k: v for k, v in indexDict.items() if len(v) > 1}
timeBinMidpoints = {k: v for k, v in timeBinMidpoints.items() if len(v) > 1}

result_df = pd.DataFrame(index = list(indexDict.keys()), columns = df.columns)
result_df = result_df.astype('object')

for i in range(len(df.columns)):
    for cell, indexList in indexDict.items():
        x = df.iloc[:,i][indexList]
        y = timeBinMidpoints[cell]
        result = list(stats.linregress(x,y))
        result_df[df.columns[i]][cell] = result

result_df.fillna(0, inplace=True)
result_df.to_csv('data/linearRegressionOnTimeBins.csv')