# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import random
import matplotlib.pyplot as plt
import pickle as pickle
import os
from sklearn.model_selection import train_test_split
import time
import argparse
from utils import pred_eval
from base_models import xgt_regressor

# In[ ]:

# read data
# parameters
parser = argparse.ArgumentParser()
parser.add_argument("--gather_point", "-p", help="gather point, in hours", type=int, default = 5)
args = parser.parse_args()

GATHER_POINT = args.gather_point # in hours
INTERVAL = 0.25 # in hours

lr = [0.05]
max_d = [4]
estimators = [10]

name = str(GATHER_POINT)
path_data = "../data/"
path_log  = "../results/xgboost_e/"+name+"_log.txt"
path_result = "../results/xgboost_e/"
#path_models = "../results/xgboost_e/models/"

train_df = pd.read_pickle(path_data + 'train.p')
test_df = pd.read_pickle(path_data + 'test.p')
train_news = pd.read_pickle(path_data+'train_news.p')
test_news = pd.read_pickle(path_data+'test_news.p')

pred_event_test = np.load('../results/neuralpp/'+name+'_output.npy')
pred_event_train = np.load('../results/neuralpp/'+name+'_train_output.npy')
pred_event_test = pred_event_test[:,:2]
pred_event_train = pred_event_train[:,:2]
# In[25]:

# form data
train_df['tweet_amount_5.0h'] = train_df['tw_vector'].map(len)
train_df['delta_t'] = train_df.apply(lambda x: [(x['time_list'][i]-x['time_list'][i-1])/24
                                                for i in range(1,x['tweet_amount_'+str(GATHER_POINT)+'.0h'])],
                                     axis=1)
train_df['len'] = train_df['delta_t'].map(len)
train_df = train_df[train_df['len']>0]
train_df = train_df.reset_index(drop=True)
train_news = train_df[['url','label']].merge(train_news[['url','vector']], on='url', how='left')

X_train = train_df.iloc[:, list(range(1, int(GATHER_POINT / INTERVAL)+2))].values
X_train2 = list(train_news['vector'])
y_train = list(train_df['label'])
dy_train = np.array(y_train).reshape((-1,1))-np.array(pred_event_train)

test_df['tweet_amount_5.0h'] = test_df['tw_vector'].map(len)
test_df['delta_t'] = test_df.apply(lambda x: [(x['time_list'][i]-x['time_list'][i-1])/24
                                                for i in range(1,x['tweet_amount_'+str(GATHER_POINT)+'.0h'])],
                                     axis=1)
test_df['len'] = test_df['delta_t'].map(len)
test_df = test_df[test_df['len']>0]
test_df = test_df.reset_index(drop=True)
test_news = test_df[['url','label']].merge(test_news[['url','vector']], on='url', how='left')

X_test = test_df.iloc[:, list(range(1, int(GATHER_POINT / INTERVAL)+2))].values
X_test2 = list(test_news['vector'])
y_test = list(test_df['label'])
dy_test = np.array(y_test).reshape((-1,1))-np.array(pred_event_test)

# In[ ]:
print(dy_train.shape)

# time boosting
X_tr, X_val, y_tr, y_val = train_test_split(X_train, dy_train, test_size=0.1, random_state=42)

c_l = []
for i in range(dy_train.shape[1]):
    obj = 'reg:linear'
    c  = xgt_regressor(lr, max_d, estimators, X_tr, X_val, y_tr[:,i], y_val[:,i], obj)

    with open(path_log, "a") as text_file:
        text_file.write("\n\n ------ Parameters and Evaluation Results ------- \n")
        text_file.write("time boosting : \n")
        text_file.write("objective type : %s \n"%(obj))
        text_file.write("learning rate : %.4f \n"%(c[1][0]))
        text_file.write("max depth : %.4f \n"%(c[1][1]))
        text_file.write("number of estimators : %.4f \n\n"%(c[1][2]))

    c_l.append(c)

'''pred_time_train = []
for i in range(dy_train.shape[1]):
    pred_time_train.append(c_l[i][0].predict(X_train)+np.array(pred_event_train)[:,i])
pred_time_train = np.array(pred_time_train).transpose()
dy_train2 = np.array(y_train).reshape((-1,1))-np.array(pred_time_train)
'''
pred_time_test = []
for i in range(dy_test.shape[1]):
    pred_time_test.append(c_l[i][0].predict(X_test)+np.array(pred_event_test)[:,i])
pred_time_test = np.array(pred_time_test).transpose()

'''
dy_test2 = np.array(y_test).reshape((-1,1))-np.array(pred_time_test)

# In[56]:
print('time boosting finished!')

# news boosting
X_tr, X_val, y_tr, y_val = train_test_split(X_train2, dy_train2, test_size=0.1, random_state=42)

c_l = []
for i in range(dy_train2.shape[1]):
    obj = 'reg:linear'
    c  = xgt_regressor(lr, max_d, estimators, X_tr, X_val, y_tr[:,i], y_val[:,i], obj)

    with open(path_log, "a") as text_file:
        text_file.write("\n\n ------ Parameters and Evaluation Results ------- \n")
        text_file.write("news boosting : \n")
        text_file.write("objective type : %s \n"%(obj))
        text_file.write("learning rate : %.4f \n"%(c[1][0]))
        text_file.write("max depth : %.4f \n"%(c[1][1]))
        text_file.write("number of estimators : %.4f \n\n"%(c[1][2]))

    c_l.append(c)

pred_news_test = []
for i in range(dy_test2.shape[1]):
    pred_news_test.append(c_l[i][0].predict(X_test2)+np.array(pred_time_test)[:,i])
pred_news_test = np.array(pred_news_test).transpose()

'''
# In[ ]:

# calculate results
rmse, mae, mape = pred_eval(pred_time_test.sum(1)/30, y_test)
# np.save(path_result+name+"_output.npy", pred_news_test)

with open(path_log, "a") as text_file:
    text_file.write("\n\n ------ Evaluation Results ------- \n")

    text_file.write("best rmse : %.4f \n"%(rmse))
    text_file.write("best mae : %.4f \n"%(mae))
    text_file.write("best mape : %.4f \n"%(mape))


import argparse

from . import core

description = """
'--level pixel' means individual pixels for each SAR date are labeled (good for larger scenes).
'--level scene' means whole SAR images are labeled (good for smaller scenes).
For scene level labeling, the variance of each average interferogram is used.
"""

def get_cli_args():
    """ """
    p = argparse.ArgumentParser(description=description)
    p.add_argument(
        "--input-files",
        "-i",
        help=(
            "Provide the list of input interferogram filenames in a text file."
            "Alternate to `--search-path`."
        ),
    )
    p.add_argument(
        "--search-path",
        "-p",
        default=".",
        help="location of igram files. (default=%(default)s)",
    )
    p.add_argument(
        "--ext",
        default=".unw",
        help="filename extension of unwrapped igrams to average (default=%(default)s)",
    )
    p.add_argument(
        "--band",
        type=int,
        default=1,
        help="If using GDAL to load igrams, specify which band represents unwrapped phase (default=%(default)s)",
    )
    p.add_argument(
        "--outfile",
        "-o",
        default="labels.nc",
        help="Location to save final labels (default=%(default)s)",
    )
    p.add_argument(
        "--level",
        default="scene",
        choices=["pixel", "scene"],
        help=("Level at which to label outliers. (default=%(default)s).\n"),
    )
    p.add_argument(
        "--avg-file",
        default="average_ifgs.nc",
        help="Location to save stack of averaged igrams (default=%(default)s)",
    )
    p.add_argument(
        "--deramp-order",
        default=2,
        type=int,
        help="Specify order of surface to remove from phase when averaging. "
        " 1 = linear ramp, 2 = quadratic surface, 0 = no ramp adjustment (default=%(default)s)",
    )
    p.add_argument(
        "--nsigma",
        "-n",
        type=int,
        help=(
            "Number of sigma_mad deviations away from median to label as outlier"
            " (default=%(default)s)"
        ),
        default=5,
    )
    p.add_argument(
        "--rsc-file", help="If using ROI_PAC .rsc files, location of .rsc file"
    )
    p.add_argument(
        "--overwrite",
        action="store_true",
        help="Overwrite existing averaged files (default=%(default)s)",
    )
    p.add_argument(
        "--max-temporal-baseline",
        type=int,
        help=(
            "Maximum temporal baseline to use when averaging interferograms. (default=%(default)s)"
        ),
        default=400,
    )
    p.add_argument(
        "--mask-files",
        nargs="+",
        help="List of binary mask files (e.g. water mask) to apply when creating average ifgs.",
    )
    p.add_argument(
        "--mask-is-zero",
        action="store_true",
        help=(
            "Indicate that the areas to mask are 0 in the files given by `--mask-files`. "
            "The default is the numpy convention where True (1) is masked."
        ),
    )
    p.add_argument(
        "--nodata",
        type=int,
        help="Indicate a nodata value to mask in all interferograms (default=%(default)s)",
    )
    p.add_argument(
        "--no-sign-flip",
        action="store_false",
        dest="do_flip",
        help="Skip the sign-flipping that makes interferogram averages have same direction."
        " Skipping for interferograms will make averages including long term deformation, "
        "but is useful for, e.g., averaging correlation images.",
    )
    return p.parse_args()

def average_and_label():
    """ """
    args = get_cli_args()
    core.create_averages(**vars(args))
    core.label_outliers(
        fname=args.avg_file,
        outfile=args.outfile,
        nsigma=args.nsigma,
        level=args.level,
    )


from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response

from ..models import *
from .serializers import *

@api_view(["GET"])  # only accepts a GET request.
def getRoutes(request):

    routes = [
        "GET api/rooms",
        "GET api/rooms/:name",
    ]

    return Response(routes)

@api_view(["GET"])
def getRooms(request):

    rooms = Room.objects.all()
    serializer = RoomSerializer(rooms, many=True)
    return Response(serializer.data)

@api_view(["GET"])
def getRoom(request, roomname):

    room = Room.objects.get(name=roomname)
    serializer = RoomSerializer(room)
    return Response(serializer.data)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 18-5-15 下午4:29
# @Author  : Luo Yao
# @Site    : http://icode.baidu.com/repos/baidu/personal-code/Luoyao
# @File    : lanenet_cluster.py
# @IDE: PyCharm Community Edition
"""
實現LaneNet中實例分割的聚類部分  lanenet_cluster_沒畫線沒補線
"""
import numpy as np
import glog as log
import matplotlib.pyplot as plt
from sklearn.cluster import MeanShift
from sklearn.cluster import DBSCAN
import time
import warnings
import cv2

from itertools import cycle

try:
    from cv2 import cv2
except ImportError:
    pass

class LaneNetCluster(object):
    """
    實例分割聚類器
    """

    def __init__(self):
        """

        """
        self._color_map = [np.array([0, 0, 255]),   #BGR
                           np.array([0, 255, 0]),
                           np.array([255, 0, 0]),
                           np.array([125, 125, 0]),
                           np.array([0, 125, 125]),
                           np.array([125, 0, 125]),
                           np.array([50, 100, 50]),
                           np.array([100, 50, 100])]
        pass

    @staticmethod
    def _cluster(prediction, bandwidth):
        """
        實現論文SectionⅡ的cluster部分
        :param prediction:
        :param bandwidth:
        :return:
        """
        ms = MeanShift(bandwidth=bandwidth, bin_seeding=False,cluster_all=False)
        # log.info('開始Mean shift聚類 ...')
        tic = time.time()
        try:
            label = ms.fit_predict(prediction)
            #print(label, '!!!!!!!!!!!!!')  #每個點都會回傳自己是哪一類(0到3類這樣)，跟座標點個數一樣
        except ValueError as err:
            log.error(err)
            return 0, [], []
        # log.info('Mean Shift耗時: {:.5f}s'.format(time.time() - tic))
        labels = ms.labels_
        cluster_centers = ms.cluster_centers_

        num_clusters = cluster_centers.shape[0]

        log.info('聚類簇個數為: {:d}'.format(num_clusters))
        ##############可以每張圖都畫一個聚類圖，但只有中間一根??#################
        # plt.figure(1)
        # plt.clf()

        # colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
        # for k, col in zip(range(num_clusters), colors):
        #     my_members = labels == k
        #     cluster_center = cluster_centers[k]
        #     plt.plot(prediction[my_members, 0], prediction[my_members, 1], col + '.')
        #     plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
        #              markeredgecolor='k', markersize=14)
        # plt.title('Estimated number of clusters: %d' % num_clusters)
        # plt.show()
        ######################################################################

        return num_clusters, labels, cluster_centers

    @staticmethod
    def _cluster_v2(prediction):
        """
        dbscan cluster
        :param prediction:
        :return:
        """
        db = DBSCAN(eps=0.7, min_samples=200).fit(prediction)
        db_labels = db.labels_
        unique_labels = np.unique(db_labels)
        unique_labels = [tmp for tmp in unique_labels if tmp != -1]
        log.info('聚類簇個數為: {:d}'.format(len(unique_labels)))

        num_clusters = len(unique_labels)
        cluster_centers = db.components_

        return num_clusters, db_labels, cluster_centers

    @staticmethod
    def _get_lane_area(binary_seg_ret, instance_seg_ret):
        """
        通過二值分割掩碼圖在實例分割圖上獲取所有車道線的特徵向量
        :param binary_seg_ret:
        :param instance_seg_ret:
        :return:
        """
        idx = np.where(binary_seg_ret == 1)

        lane_embedding_feats = []
        lane_coordinate = []
        for i in range(len(idx[0])):
            lane_embedding_feats.append(instance_seg_ret[idx[0][i], idx[1][i]])
            lane_coordinate.append([idx[0][i], idx[1][i]])

        return np.array(lane_embedding_feats, np.float32), np.array(lane_coordinate, np.int64)

    @staticmethod
    def _thresh_coord(coord):  #這裡的coord還是x,y顛倒的狀態
        """
        過濾實例車道線位置坐標點,假設車道線是連續的, 因此車道線點的坐標變換應該是平滑變化的不應該出現跳變
        :param coord: [(x, y)]
        :return:
        """
        pts_x = coord[:, 0]  #其實是對上下做檢查(y軸 1~256)
        pts_y = coord[0, :]
        mean_x = np.mean(pts_x)
        mean_y = np.mean(pts_y)

        idx = np.where(np.abs(pts_x - mean_x) < mean_x/2) or np.where(np.abs(pts_y - mean_y) < mean_y/2)
        # print(coord[idx[0]], 'AAAAAAAAAAAA')

        return coord[idx[0]]

    @staticmethod
    def _lane_fit(lane_pts):
        """
        車道線多項式擬合
        :param lane_pts:
        :return:
        """
        if not isinstance(lane_pts, np.ndarray):
            lane_pts = np.array(lane_pts, np.float32)

        x = lane_pts[:, 0]
        y = lane_pts[:, 1]
        x_fit = []
        y_fit = []
        with warnings.catch_warnings():
            warnings.filterwarnings('error')
            try:
                f1 = np.polyfit(x, y, 3)
                p1 = np.poly1d(f1)
                x_min = int(np.min(x))
                x_max = int(np.max(x))
                x_fit = []
                for i in range(x_min, x_max + 1):
                    x_fit.append(i)
                y_fit = p1(x_fit)
            except Warning as e:
                x_fit = x
                y_fit = y
            finally:
                return zip(x_fit, y_fit)

    def get_lane_mask(self, binary_seg_ret, instance_seg_ret):
        """

        :param binary_seg_ret:
        :param instance_seg_ret:
        :return:
        """
        lane_embedding_feats, lane_coordinate = self._get_lane_area(binary_seg_ret, instance_seg_ret)
        #print(lane_embedding_feats, 'QQQQQQQQQQQQQQQQQQQQQ')
        #print(len(lane_coordinate), 'AAAAAAAAAAAAAAAAAAAAA')   #2值分割裡屬於線的pixel的座標，個數(list長度)跟上面的一樣

        num_clusters, labels, cluster_centers = self._cluster(lane_embedding_feats, bandwidth=0.05)

        # 聚類簇超過八個則選擇其中類內樣本最多的八個聚類簇保留下來
        # if num_clusters > 4:
        #     cluster_sample_nums = []
        #     for i in range(num_clusters):
        #         cluster_sample_nums.append(len(np.where(labels == i)[0]))
        #     sort_idx = np.argsort(-np.array(cluster_sample_nums, np.int64))
        #     cluster_index = np.array(range(num_clusters))[sort_idx[0:4]]
        # else:
        #     cluster_index = range(num_clusters)

        cluster_sample_nums = []
        for i in range(num_clusters):
            cluster_sample_nums.append(len(np.where(labels == i)[0]))     #屬於第i個類的有幾個點
            if cluster_sample_nums[i] < 200: #如果該類的點數太少(誤判為線)，則將誤判的刪掉不塗顏色
                cluster_sample_nums[i] == 0
                continue

            print(cluster_sample_nums, 'WWWWWWWWWWWWWWWWW')# = [1080, 1069, 959, 802, 7]  裡面數字為點數

        sort_idx = np.argsort(-np.array(cluster_sample_nums, np.int64))   #按照大到小排列，會印出該數字的index而不是數字本身
        #print(sort_idx) = [0, 1, 2, 3, 4]
        cluster_index = np.array(range(num_clusters))[sort_idx[0:5]]      #取前4個
        #print(cluster_index, '!!!!!!!!!!!!!!!!!!!') = [0, 1, 2, 3]取前4個

        mask_image = np.zeros(shape=[binary_seg_ret.shape[0], binary_seg_ret.shape[1], 3], dtype=np.uint8)

        for index, i in enumerate(cluster_index):
            idx = np.where(labels == i)
            coord = lane_coordinate[idx]
#            print(coord.shape[0], '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
            coord = self._thresh_coord(coord)
            coord = np.flip(coord, axis=1)
            # coord = (coord[:, 0], coord[:, 1])
            color = (int(self._color_map[index][0]),
                     int(self._color_map[index][1]),
                     int(self._color_map[index][2]))
            coord = np.array([coord])
#            print(coord.shape)

            cv2.polylines(img=mask_image, pts=coord, isClosed=False, color=color, thickness=1)
            # mask_image[coord] = color

        return mask_image

if __name__ == '__main__':
    binary_seg_image = cv2.imread('C:/Users/mediacore/lane_detection/binary_ret.png', cv2.IMREAD_GRAYSCALE)
    binary_seg_image[np.where(binary_seg_image == 255)] = 1
    instance_seg_image = cv2.imread('instance_ret.png', cv2.IMREAD_UNCHANGED)
    ele_mex = np.max(instance_seg_image, axis=(0, 1))
    for i in range(3):
        if ele_mex[i] == 0:
            scale = 1
        else:
            scale = 255 / ele_mex[i]
        instance_seg_image[:, :, i] *= int(scale)
    embedding_image = np.array(instance_seg_image, np.uint8)
    cluster = LaneNetCluster()
    mask_image = cluster.get_lane_mask(instance_seg_ret=instance_seg_image, binary_seg_ret=binary_seg_image)
    plt.figure('embedding')
    plt.imshow(embedding_image[:, :, (2, 1, 0)])
    plt.figure('mask_image')
    plt.imshow(mask_image[:, :, (2, 1, 0)])
    plt.show()


import os
import csv
from collections import defaultdict
import itertools
import ast
import numpy as np
import pandas as pd
import pickle
import sys

def create_final_mov_dict(csv_file, ratings_threshold, usernum_threshold):
    # using usecols in read_csv to exclude the last column of timestamp
    df = pd.read_csv(csv_file, delimiter=',', usecols=[0,1,2])
    df = df[df['rating'] >= ratings_threshold]  # rating thresholding

    movie_user_dict = defaultdict(list)
    for index, row in df.iterrows():
        movId = int(row['movieId'])
        usrId = int(row['userId'])
        # rating = row['rating']
        movie_user_dict[movId].append(usrId)

    # store the final res here, after the usernum thresholding
    final_dict = {}
    # num_users = 0
    for key,val in movie_user_dict.items():
        if len(val) >= usernum_threshold:
            final_dict[key] = val

    return final_dict

def get_total_users(final_dict):
    total_users = set()
    for mov in final_dict.keys():
        row = set(final_dict[mov])
        total_users = total_users.union(row)
    num_users = len(total_users)
    return num_users

def create_count_matrix(final_dict):
    # helper function
    def findnum_common_elements(list1, list2):
        return len(set(list1).intersection(list2))

    count_matrix = {}
    for pair in itertools.combinations(final_dict.keys(), r=2):
        mov1, mov2 = pair
        count_matrix[pair] = findnum_common_elements(final_dict[mov1], final_dict[mov2])

    # adding the marginal counts
    for k in final_dict.keys():
        pair = (k,k)
        count_matrix[pair] = len(final_dict[k])

    return count_matrix

def get_genres(genres_json, id_to_genre):
    '''
    id_to_genre: its like a global dict to store the mappings
                from the genre-id to genre-name
    '''
    if not genres_json:
        return None

    tmp = ast.literal_eval(genres_json)
    ids_list = []
    gen_list = []

    for d in tmp:
        id_num = d['id']
        genre_name = d['name']
        genre_name = genre_name.replace(" ", "-") # change all whitespaces to hyphen
        ids_list.append(id_num)
        gen_list.append(genre_name)

        if id_num not in id_to_genre:
            id_to_genre[id_num] = genre_name

    return ids_list, gen_list, id_to_genre

def create_final_genres_dict(csv_file, movIdsList):
    df_meta = pd.read_csv(csv_file, delimiter=',')
    df_meta = df_meta[['genres', 'id']]

    genid_dict = {}
    genname_dict = {}
    genid_counts = defaultdict(int) # counting the genre-genre pairs
    id_to_genre = {}

    for index, row in df_meta.iterrows():

        try:
            movId = int(row['id'])
            gen_json = row['genres']

            if movId in movIdsList:  # Only consider the thresholded movies
                ids, genres, id_to_genre = get_genres(gen_json, id_to_genre)
                genid_dict[movId] = ids
                genname_dict[movId] = genres

                for i in ids:
                    singleton_pair = (i, i)
                    genid_counts[singleton_pair] += 1

                for pair in itertools.combinations(ids, r=2):
                    id1, id2 = pair
                    genid_counts[pair] += 1

        except Exception as e:
            print(index, row['id'])
            repr(e)

    return genid_dict, genname_dict, genid_counts, id_to_genre

def marginal_prob(obj_id, count_matrix, num_users):
    '''function to get the marginal prob:
        P(movie_id1)
    '''
    margn_count = count_matrix[(obj_id, obj_id)]
    return margn_count * 1.0/num_users

def joint_prob(obj_id1, obj_id2, count_matrix, num_users):
    '''function to get the joint prob:
        P(movie_id1, movie_id2)
    '''
    if (obj_id1, obj_id2) not in count_matrix.keys():
        key = (obj_id2, obj_id1)
    else:
        key = (obj_id1, obj_id2)

    joint_count = count_matrix[key]
    return joint_count * 1.0/num_users

def conditional_prob(obj_id1, obj_id2, count_matrix):
    '''function to get the conditional prob:
        P(movie_id1 | movie_id2)
    '''
    if (obj_id1, obj_id2) not in count_matrix.keys():
        key = (obj_id2, obj_id1)
    else:
        key = (obj_id1, obj_id2)

    joint_count = count_matrix[key]
    margn_count = count_matrix[obj_id2, obj_id2]

    return joint_count * 1.0/margn_count

def create_vocab_marginal_files(movie_count_matrix, movie_num_users, genre_count_matrix, genre_num_users, datadir):

    # Create the marginals file and a separate vocab file (for all the possible movieIds)
    # Consider only the diagonal entries in count_matrix
    # Also, the 2 lists work as follows:
    # marginals[i] == marginal_prob(movieid_vocab[i], count_matrix, num_users)
    # i.e the indices of both lists match on movieid and its marginal probability

    mov_marginals = []
    movieid_vocab = []
    for key_pair in movie_count_matrix.keys():
        k1, k2 = key_pair
        if k1 == k2:
            movieid_vocab.append(k1)
            mov_marginals.append(marginal_prob(k1, movie_count_matrix, movie_num_users))

    # Write out the lists to text files

    fname_mov_marginals = datadir + "movie_marginal_prob.txt"
    with open(fname_mov_marginals, "w") as f:
        for prob in mov_marginals:
            f.write("%s\n" % prob)

    fname_mov_vocab = datadir + "movie_vocabulary.txt"
    with open(fname_mov_vocab, "w") as f:
        for movid in movieid_vocab:
            f.write("%s\n" % movid)

    gen_marginals = []
    gen_vocab = []
    for key_pair in genre_count_matrix.keys():
        k1, k2 = key_pair
        if k1 == k2:
            gen_vocab.append(k1)
            gen_marginals.append(marginal_prob(k1, genre_count_matrix, genre_num_users))

    # Write out the lists to text files
    fname_gen_marginals = datadir + "genre_marginal_prob.txt"
    with open(fname_gen_marginals, "w") as f:
        for prob in gen_marginals:
            f.write("%s\n" % prob)

    fname_gen_vocab = datadir + "genre_vocabulary.txt"
    with open(fname_gen_vocab, "w") as f:
        for gen in gen_vocab:
            f.write("%s\n" % gen)
    return

def create_master_files(movie_count_matrix, genre_count_matrix, genname_dict, datadir):
    REL = "IsA"

    # MOVIES
    print("writing movie master files")
    mov_tup_list = []
    for key_pair in movie_count_matrix.keys():
        k1, k2 = key_pair
        if k1 != k2:
            prob_k2k1 = conditional_prob(k2, k1, movie_count_matrix)
            tmp_tup1 = (REL, k1, k2, prob_k2k1)
            mov_tup_list.append(tmp_tup1)

            prob_k1k2 = conditional_prob(k1, k2, movie_count_matrix)
            tmp_tup2 = (REL, k2, k1, prob_k1k2)
            mov_tup_list.append(tmp_tup2)

    fname_mov_master = datadir + "movie_movie_master.txt"
    with open(fname_mov_master, "w") as f:
        writer = csv.writer(f, delimiter='\t', lineterminator='\n')
        writer.writerows(mov_tup_list)

    # GENRES
    print("writing genre master files")
    gen_tup_list = []
    for key_pair in genre_count_matrix.keys():
        k1, k2 = key_pair
        if k1 != k2:
            prob_k2k1 = conditional_prob(k2, k1, genre_count_matrix)
            tmp_tup1 = (REL, k1, k2, prob_k2k1)
            gen_tup_list.append(tmp_tup1)

            prob_k1k2 = conditional_prob(k1, k2, genre_count_matrix)
            tmp_tup2 = (REL, k2, k1, prob_k1k2)
            gen_tup_list.append(tmp_tup2)

    fname_master = datadir + "genre_genre_master.txt"
    with open(fname_master, "w") as f:
        writer = csv.writer(f, delimiter='\t', lineterminator='\n')
        writer.writerows(gen_tup_list)

    # MOVIE - GENRES
    print("writing movie-genre master files")
    tup_list = []
    for movid in genname_dict:
        genres = genname_dict[movid]
        for g in genres:
            tmp_tup = (REL, movid, g, 1.0)
            tup_list.append(tmp_tup)

    fname_master = datadir + "movie_genre_master.txt"
    with open(fname_master, "w") as f:
        writer = csv.writer(f, delimiter='\t', lineterminator='\n')
        writer.writerows(tup_list)

    return

def main():
    # t_rating = 4
    # t_users = 100
    t_rating = float(sys.argv[1])
    t_users = int(sys.argv[2])

    # rootdir = '/home/nkhargonkar/dsis/'
    rootdir = '/home/ninad/Desktop/Link-to-sem4/dsis/'

    # rawdata_file = rootdir + 'datasets/the-movies-dataset/ratings_small.csv'  # SMALL RATINGS FILE
    rawdata_file = rootdir + 'datasets/the-movies-dataset/ratings.csv'    # FULL RATINGS FILE

    metdata_file = rootdir + 'datasets/the-movies-dataset/movies_metadata.csv'  # for the genres
    datadir = rootdir + 'prob-emb/box-code/data/movie_data/movie_data_' + str(t_rating) + '_' + str(t_users) + '_taxonomy/'

    print(datadir)
    print(t_rating, t_users)

    final_dict = create_final_mov_dict(rawdata_file, t_rating, t_users)
    mov_num_users = get_total_users(final_dict)
    mov_cmatrix = create_count_matrix(final_dict)
    movId_list = list(final_dict.keys())
    print("Done with MOVIE")
    del(final_dict)

    if not os.path.exists(datadir):
        os.makedirs(datadir)

    genid_dict, genname_dict, genid_counts, id_to_genre = create_final_genres_dict(metdata_file, movId_list)
    genname_counts = {}   # the count matrix for genres with their names
    for k,v in genid_counts.items():
        k1, k2 = k
        g1 = id_to_genre[k1]
        g2 = id_to_genre[k2]
        genname_counts[g1, g2] = v
    print("Done with GENRES")

    print("Writing MARG and VOCAB")
    create_vocab_marginal_files(mov_cmatrix, mov_num_users, genname_counts, len(movId_list), datadir)

    print("Writing master files")
    create_master_files(mov_cmatrix, genname_counts, genname_dict, datadir)

    # trn_file = datadir + 'movie_train.txt'
    # trn_eval_file = datadir + 'movie_train_eval.txt'
    # create_trneval_file(trn_file, trn_eval_file, 10)

if __name__ == "__main__":
    main()


import os
import mrcfile
import numpy as np
import pytest
from fibsem_tools.io.core import read_xarray
from fibsem_tools.io.xr import stt_from_array
from fibsem_tools.io.mrc import access, recarray_to_dict, to_dask, to_xarray
from xarray.testing import assert_equal

def test_access_mrc(temp_dir):
    name = "test.mrc"
    mrc_path = os.path.join(temp_dir, name)

    data = np.arange(27, dtype="uint8").reshape((3, 3, 3))
    original = mrcfile.new(mrc_path, data=data, overwrite=True)
    original.flush()
    accessed = access(mrc_path, mode="r")
    assert np.array_equal(accessed.data, original.data)
    assert np.array_equal(to_dask(accessed).compute(), original.data)
    assert np.array_equal(
        to_dask(accessed, chunks=(2, -1, -1)).compute(), original.data
    )

    with pytest.raises(ValueError):
        to_dask(accessed, chunks=(1, 1, 1))

    del original
    del accessed

@pytest.mark.parametrize("attrs", (None, {"foo": 10}))
def test_read_xarray(temp_dir, attrs):
    name = "test.mrc"
    mrc_path = os.path.join(temp_dir, name)
    scales = [1.0, 2.0, 3.0]
    data = np.arange(4 * 5 * 6, dtype="uint8").reshape((4, 5, 6))
    original = mrcfile.new(mrc_path, data=data, overwrite=True)
    original.voxel_size = [x * 10 for x in reversed(scales)]
    original.flush()

    expected = stt_from_array(
        data,
        dims=("z", "y", "x"),
        translates=(0, 0, 0),
        scales=scales,
        units=("nm", "nm", "nm"),
    )

    observed = [to_xarray(original, attrs=attrs), read_xarray(mrc_path, attrs=attrs)]
    for obs in observed:
        assert_equal(obs, expected)
        if attrs is None:
            assert dict(obs.attrs) == recarray_to_dict(original.header)


#!/usr/bin/env python3
import functools
import hashlib
import itertools
import json
import logging
import os
import pathlib
import re
import time
from typing import Optional

import fire
import requests
from sprig import dictutils

logger = logging.getLogger(__name__)

class AnswerNotFoundError(Exception):
    ...

class Session:
    def __init__(
        self,
        session: str,
        cache_location: pathlib.Path = pathlib.Path(__file__).parents[1] / "data",
        user_fingerprint: Optional[str] = None,
    ) -> None:
        self._cookie = session
        self._answers = {}
        self._cache_location = cache_location
        self._answers_location = cache_location / "answers.json"
        self._rate_limited_until = time.monotonic()
        self._user_fingerprint = user_fingerprint

    def __enter__(self):
        with self._answers_location.open() as f:
            self._answers = dictutils.deflate(json.load(f))
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        with self._answers_location.open("w") as f:
            json.dump(dictutils.inflate(self._answers), f, sort_keys=True, indent=2)

    @functools.cache
    def _get_text(self, url: str) -> str:
        delay = max(0.0, self._rate_limited_until - time.monotonic())
        logger.info("Downloading %s after %3.1f second delay", url, delay)
        time.sleep(delay)
        resp = requests.get(
            url,
            cookies={"session": self._cookie},
        )
        # Compromise between me being impatient and not wanting to hammer the server
        self._rate_limited_until = time.monotonic() + 10
        resp.raise_for_status()
        return resp.text

    @functools.cache
    def _input(self, year: int, day: int) -> str:
        return self._get_text(f"https://adventofcode.com/{year}/day/{day}/input")

    @functools.cache
    def _question(self, year: int, day: int) -> str:
        return self._get_text(f"https://adventofcode.com/{year}/day/{day}")

    def user_fingerprint(self) -> str:
        if self._user_fingerprint is None:
            self._user_fingerprint = hashlib.sha1(
                self._input(2020, 2).encode("ascii")
            ).hexdigest()[:10]
        return self._user_fingerprint

    def input(self, year: int, day: int) -> str:
        file_location = (
            self._cache_location
            / "inputs"
            / f"{year:04}"
            / f"{day:02}"
            / f"{self.user_fingerprint()}.txt"
        )
        if not file_location.exists():
            file_location.parent.mkdir(exist_ok=True, parents=True)
            file_location.write_text(self._input(year, day))
        return file_location.read_text()

    def question(self, year: int, day: int) -> str:
        file_location = (
            self._cache_location
            / "questions"
            / f"{year:04}"
            / f"{day:02}"
            / f"{self.user_fingerprint()}.html"
        )
        if not file_location.exists():
            file_location.parent.mkdir(exist_ok=True, parents=True)
            file_location.write_text(self._question(year, day))
        return file_location.read_text()

    def _answer(self, year: int, day: int, part: int) -> str:
        question = self.question(year, day)
        answers = {
            i: m[1]
            for i, m in enumerate(
                re.finditer(r"Your puzzle answer was <code>([^<]+)</code>", question), 1
            )
        }
        try:
            return answers[part]
        except KeyError as e:
            raise AnswerNotFoundError from e

    def answer(self, year: int, day: int, part: int) -> str:
        key = f"{year:04}/{day:02}/{part}/{self.user_fingerprint()}"
        if key not in self._answers:
            self._answers[key] = self._answer(year, day, part)
        return self._answers[key]

def _scrape_answers(session: Session) -> None:
    logger.info(f"Scraping {session.user_fingerprint()} for answers")
    for year in itertools.count(2015):
        for day in range(1, 26):
            for part in [1, 2]:
                try:
                    session.answer(year, day, part)
                except AnswerNotFoundError:
                    logger.debug("No answer for found %s/%s/%s", year, day, part)
                except requests.HTTPError:
                    logger.debug(
                        "Could not retrieve question for %s/%s/%s", year, day, part
                    )
                    return

def _scrape_inputs(session: Session) -> None:
    logger.info(f"Scraping {session.user_fingerprint()} for inputs")
    for year in itertools.count(2015):
        for day in range(1, 26):
            try:
                session.input(year, day)
            except requests.HTTPError:
                logger.info("Could not retrieve input for %s/%s", year, day)
                return

SAVED_COOKIES_PATH = pathlib.Path(__file__).with_suffix(".sessions.json")

def _read_cookies() -> dict[str, str]:
    with SAVED_COOKIES_PATH.open() as f:
        return json.load(f)

def _write_cookies(cookies: dict[str, str]) -> None:
    with SAVED_COOKIES_PATH.open("w") as f:
        json.dump(cookies, f, indent=4, sort_keys=True)

class CLI:
    @staticmethod
    def add_session(session_cookie: str) -> None:
        try:
            cookies = _read_cookies()
        except FileNotFoundError:
            cookies = {}
        with Session(session_cookie) as session:
            user_fingerprint = session.user_fingerprint()
        cookies[user_fingerprint] = session_cookie
        _write_cookies(cookies)

    @staticmethod
    def scrape_answers() -> None:
        for fingerprint, cookie in _read_cookies().items():
            with Session(session=cookie, user_fingerprint=fingerprint) as session:
                _scrape_answers(session)

    @staticmethod
    def scrape_inputs() -> None:
        for fingerprint, cookie in _read_cookies().items():
            with Session(session=cookie, user_fingerprint=fingerprint) as session:
                _scrape_inputs(session)

if __name__ == "__main__":
    logging.basicConfig(level=getattr(logging, os.environ.get("LEVEL", "WARNING")))
    fire.Fire(CLI)


# This script uses a similar approach to the MatLab stereo camera calibrator
# Firstly, the chessboard needs to be detected in each camera.
# Once a chessboard is detected -> pressing the 'b'-button saves the coordinates
# After re-placing the chessboard somewhere in the room: the 'n'-button needs to be pressed.
# Save coordinates using 'b'
# Repeat...

# Lastly: After gathering sufficient 3d-coordinates: press the 't'-button to estimate the affine transformation
# ..using the opencv-function estimateAffine3D

import pyrealsense2 as rs
import numpy as np
import cv2

pTime = 0
cTime = 0
num = 0
posList = []
posList2 = []
# cv2.estimateTranslation3D()
# initialize shared coordinate system lists
coordinates_3d_top_view_lst = []
coordinates_3d_side_view_lst = []

# Configure depth and color streams...
# width, height
width = 1280
height = 720
fps = 30
alpha = 0.5

# misc variables
test_set_ready = True

# Configure depth and color streams...
# ...of Camera 1
pipeline_1 = rs.pipeline()
config_1 = rs.config()
config_1.enable_device("151422254978")
config_1.enable_stream(rs.stream.depth, width, height, rs.format.z16, fps)
config_1.enable_stream(rs.stream.color, width, height, rs.format.bgr8, fps)
# ...of Camera 2
pipeline_2 = rs.pipeline()
config_2 = rs.config()
config_2.enable_device("151422254651")
config_2.enable_stream(rs.stream.depth, width, height, rs.format.z16, fps)
config_2.enable_stream(rs.stream.color, width, height, rs.format.bgr8, fps)

# Start streaming from both cameras
pipeline_profile_1 = pipeline_1.start(config_1)
pipeline_profile_2 = pipeline_2.start(config_2)

# Activate RealSense colorizer
colorizer = rs.colorizer()

# Chessboard specifics
chessboardSize = (9, 7)
frameSize = (width, height)
size_of_chessboard_squares_mm = 25

# Termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((chessboardSize[0] * chessboardSize[1], 3), np.float32)
objp[:, :2] = np.mgrid[0 : chessboardSize[0], 0 : chessboardSize[1]].T.reshape(-1, 2)
objp = objp * size_of_chessboard_squares_mm

# Arrays to store object points and image points from all the images.
objpoints = []  # 3d point in real world space
imgpointsL = []  # 2d points in image plane.
imgpointsR = []  # 2d points in image plane.

# pixelpoints
pixelpoints_top_view_lst = []
pixelpoints_side_view_lst = []
delta_lst = []
shared_3d = []

alpha_0 = 0.5

# stream specific data
depth_scale_1 = pipeline_profile_1.get_device().first_depth_sensor().get_depth_scale()
depth_scale_2 = pipeline_profile_2.get_device().first_depth_sensor().get_depth_scale()

try:
    while True:
        frames_1 = pipeline_1.wait_for_frames()
        depth_frame_1 = frames_1.get_depth_frame()
        color_frame_1 = frames_1.get_color_frame()

        # Intrinsic and extrinsic matrices of camera 1
        color_intrinsic_1 = (
            color_frame_1.profile.as_video_stream_profile().intrinsics
        )  #
        depth_intrinsic_1 = (
            depth_frame_1.profile.as_video_stream_profile().intrinsics
        )  #
        depth_to_color_extrinsic_1 = depth_frame_1.profile.get_extrinsics_to(
            color_frame_1.profile
        )
        color_to_depth_extrinsic_1 = color_frame_1.profile.get_extrinsics_to(
            depth_frame_1.profile
        )

        if not depth_frame_1 or not color_frame_1:
            continue

        # Convert images to numpy arrays
        depth_image_1 = np.asanyarray(depth_frame_1.get_data())
        color_image_1 = np.asanyarray(color_frame_1.get_data())
        color_image_1 = cv2.cvtColor(color_image_1, cv2.COLOR_BGR2RGB)

        # align depth_to_rgb 1
        align_1 = rs.align(rs.stream.color)
        frames_1 = align_1.process(frames_1)
        aligned_depth_frame_1 = frames_1.get_depth_frame()
        colorized_depth_1 = np.asanyarray(
            colorizer.colorize(aligned_depth_frame_1).get_data()
        )

        # Camera 2
        # Wait for a coherent pair of frames: depth and color
        frames_2 = pipeline_2.wait_for_frames()
        depth_frame_2 = frames_2.get_depth_frame()
        color_frame_2 = frames_2.get_color_frame()

        color_intrinsic_2 = (
            color_frame_2.profile.as_video_stream_profile().intrinsics
        )  #
        depth_intrinsic_2 = (
            depth_frame_2.profile.as_video_stream_profile().intrinsics
        )  #
        depth_to_color_extrinsic_2 = depth_frame_2.profile.get_extrinsics_to(
            color_frame_2.profile
        )
        color_to_depth_extrinsic_2 = color_frame_2.profile.get_extrinsics_to(
            depth_frame_2.profile
        )

        if not depth_frame_2 or not color_frame_2:
            continue

        # Convert images to numpy arrays
        depth_image_2 = np.asanyarray(depth_frame_2.get_data())
        color_image_2 = np.asanyarray(color_frame_2.get_data())

        # align depth_to_rgb 2
        align_2 = rs.align(rs.stream.color)
        frames_2 = align_2.process(frames_2)
        aligned_depth_frame_2 = frames_2.get_depth_frame()
        # depth visualization
        # colorized_depth_2 = np.asanyarray(colorizer.colorize(filtered_2).get_data())
        colorized_depth_2 = np.asanyarray(
            colorizer.colorize(aligned_depth_frame_2).get_data()
        )

        # prepare chessboard detection by graying images
        grayL = cv2.cvtColor(color_image_2, cv2.COLOR_BGR2GRAY)
        grayR = cv2.cvtColor(color_image_1, cv2.COLOR_BGR2GRAY)

        # find chessboard corners
        retL, cornersL = cv2.findChessboardCorners(grayL, chessboardSize, None)
        retR, cornersR = cv2.findChessboardCorners(grayR, chessboardSize, None)

        # side view
        if cv2.waitKey(33) == ord("b"):
            if test_set_ready:
                for x in range(len(cornersR)):
                    rs.rs2_project_color_pixel_to_depth_pixel(
                        depth_frame_2.get_data(),
                        depth_scale_2,
                        0.2,
                        1,
                        depth_intrinsic_2,
                        color_intrinsic_2,
                        depth_to_color_extrinsic_2,
                        color_to_depth_extrinsic_2,
                        [cornersR[x][0][0], cornersR[x][0][1]],
                    )
                    coordinates_3d_top_shared = rs.rs2_deproject_pixel_to_point(
                        depth_intrinsic_2,
                        [cornersR[x][0][0], cornersR[x][0][1]],
                        depth_frame_2.get_distance(
                            int(cornersR[x][0][0]), int(cornersR[x][0][1])
                        ),
                    )
                    # coordinates_3d_top_shared = rs.rs2_deproject_pixel_to_point(color_intrinsic_2, [cornersR[x][0][0], cornersR[x][0][1]],
                    #                                                             aligned_depth_frame_2.get_distance(int(cornersR[x][0][0]), int(cornersR[x][0][1])))
                    coordinates_3d_top_view_lst.append(coordinates_3d_top_shared)
                    pixelpoints_top_view_lst.append(
                        [cornersR[x][0][0], cornersR[x][0][1]]
                    )
                # top view
                for y in range(len(cornersL)):
                    rs.rs2_project_color_pixel_to_depth_pixel(
                        depth_frame_1.get_data(),
                        depth_scale_1,
                        0.2,
                        1,
                        depth_intrinsic_1,
                        color_intrinsic_1,
                        depth_to_color_extrinsic_1,
                        color_to_depth_extrinsic_1,
                        [cornersR[y][0][0], cornersR[y][0][1]],
                    )
                    coordinates_3d_side_shared = rs.rs2_deproject_pixel_to_point(
                        depth_intrinsic_1,
                        [cornersR[y][0][0], cornersR[y][0][1]],
                        depth_frame_1.get_distance(
                            int(cornersR[y][0][0]), int(cornersR[y][0][1])
                        ),
                    )

                    coordinates_3d_side_view_lst.append(coordinates_3d_side_shared)
                    pixelpoints_side_view_lst.append(
                        [cornersL[y][0][0], cornersL[y][0][1]]
                    )

                if len(coordinates_3d_top_view_lst) >= 54:

                    test_set_ready = False
                    print("Saved 3d coordinates of corresponding points in each view.")
                    print("Change position of chessboard.")
                    print('Press "n" once ready.')

        # ready up next test_set
        if cv2.waitKey(33) == ord("n"):
            print('Getting new set of coordinates upon "b" press..')
            print("TOP-LIST:", coordinates_3d_top_view_lst)
            print("SIDE-LIST:", coordinates_3d_side_view_lst)
            test_set_ready = True

        # Clear/Empty current calibration check list on 'c' press
        if cv2.waitKey(33) == ord("c"):
            coordinates_3d_top_view_lst.clear()
            coordinates_3d_side_view_lst.clear()
            pixelpoints_side_view_lst.clear()
            pixelpoints_top_view_lst.clear()
            delta_lst.clear()
            print("..cleared lists")

        if cv2.waitKey(33) == ord("t"):
            if coordinates_3d_side_view_lst:
                coordinates_3d_top_np = np.asarray(coordinates_3d_top_view_lst)
                coordinates_3d_side_np = np.asarray(coordinates_3d_side_view_lst)
                print("TOP 3D asarray:", coordinates_3d_top_np)
                print("SIDE 3D asarray:", coordinates_3d_side_np)
                ret_val, transformation_3x4, inliers = cv2.estimateAffine3D(
                    coordinates_3d_side_np, coordinates_3d_top_np
                )
                print("Estimated 3D Affine:")
                print(transformation_3x4)

        if retL and retR:
            cornersL = cv2.cornerSubPix(grayL, cornersL, (11, 11), (-1, -1), criteria)
            imgpointsL.append(cornersL)

            cornersR = cv2.cornerSubPix(grayR, cornersR, (11, 11), (-1, -1), criteria)
            imgpointsR.append(cornersR)

            # Draw and display the corners
            cv2.drawChessboardCorners(color_image_2, chessboardSize, cornersL, retL)
            # cv2.imshow('img top', color_image_2)
            cv2.drawChessboardCorners(color_image_1, chessboardSize, cornersR, retR)
            # cv2.imshow('img side', color_image_1)
            cv2.waitKey(1)

        cv2.imshow("Color Stream Side", color_image_2)
        cv2.imshow("Color Stream Top", color_image_1)

    # cv2.waitKey(1)
    # cv2.destroyAllWindows()

finally:
    pass


import time
import datetime
from haste_storage_client.core import HasteStorageClient, OS_SWIFT_STORAGE, TRASH
from haste_storage_client.core2 import HasteStorageClientMeta
from classes.metadata_class import Ingredient, Author, Project, Metadata, Description

haste_storage_client_config = {
    'haste_metadata_server': {
        # See: https://docs.mongodb.com/manual/reference/connection-string/
        'connection_string': 'mongodb://130.239.81.77:27017'
    },
    'os_swift': {
        # See: https://docs.openstack.org/keystoneauth/latest/
        #   api/keystoneauth1.identity.v3.html#module-keystoneauth1.identity.v3.password
        'username': 's9498',
        'password': '***',
        'project_name': 'SNIC 2018/10-31',
        'user_domain_name': 'snic',
        'auth_url': 'https://hpc2n.cloud.snic.se:5000/v3',
        'project_id': '18b42df05165487984448173891ceeee',
        'project_domain_name':  'snic'
    }
}

# Identifies both the experiment, and the session (ie. unique each time the stream starts),
# for example, this would be a good format - this needs to be generated at the stream edge.
initials = 'Andy_Ishak'
stream_id = datetime.datetime.today().strftime('%Y_%m_%d__%H_%M_%S') + '_exp1_' + initials

print('stream ID is: %s' % stream_id)

# Optionally, specify REST server with interesting model:

# Project ID
#project_id = "1"

#CREATE METADATA
#Initilize 	def __init__(self,name,amount,amountType):
author_list = [["Tony Wang","Uppsala University"],["Andy Ishak", "Uppsala University"]]
ingredient_list = [["magnesium",2,"ml"],["carbon",5,"cl"]]
authors =[]
ingredients = []

for ingredient in ingredient_list:
    ingredient = Ingredient(ingredient[0],ingredient[1],ingredient[2])
    ingredients.append(ingredient)
description = Description("testing out how a cell x reacts to 2ml mercury over y time")

for author in author_list:
	author = Author(author[0],author[1])
	authors.append(author)

metadata = Metadata(ingredients,description,authors)

project = Project("Cell Mercury Testing","testing what mercury in the cell x will do over time of y month", metadata)

client = HasteStorageClientMeta(
                            substream_id=stream_id,
                            config=haste_storage_client_config,
                            project = project,
                            storage_policy=[(0.5, 1.0, OS_SWIFT_STORAGE)],  # map 0.5<=interestingness<=1.0 to OS swift.
                            default_storage=TRASH)  # discard blobs which don't match the policy above.

#blob_bytes = b'this is a binary blob eg. image data.'
timestamp_cloud_edge = time.time()
substream_id = '_B22'  # Group by microscopy well ID.
description = "description test!"
machineNumber = "SQ7333"
institute = "Gothenburg University"
"""
    def save(timestamp_cloud_edge,
             substream_id,
             description,
             machineNumber,
             institute
             ):
             """
client.save(timestamp_cloud_edge,
		substream_id,
        project)

client.close()


from matplotlib import pyplot as plt
from sklearn.datasets import make_classification
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

X, y = make_classification(n_samples=1000, n_features=35, n_redundant=5,
                           n_informative=30, random_state=3)

plt.scatter(X[y==0][:,0], X[y==0][:,1], s=100, edgecolors='k')
plt.scatter(X[y==1][:,0], X[y==1][:,1], s=100, edgecolors='k', marker='^')

plt.show()

model = MLPClassifier(max_iter=100000, hidden_layer_sizes=(200,200),
                      activation='relu')

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
                                                    random_state=33,
                                                    shuffle=True)
model.fit(X_train, y_train)

y_pred_train = model.predict(X_train)
y_pred = model.predict(X_test)

print("Training accuracy:", accuracy_score(y_train, y_train))
print("Testing accuracy:", accuracy_score(y_test, y_pred))


import math

import numpy as np
from pytest import approx, raises
from redax.predicates.dd import BDD

import redax.module as mod
from redax.spaces import DynamicCover, EmbeddedGrid, FixedCover, OutOfDomainError, ContinuousCover

Interface = mod.Interface
CompositeInterface = mod.CompositeInterface

def test_dynamic_module():
    mgr = BDD()

    inputs = {'x': DynamicCover(0, 16),
              'y': DynamicCover(0, 4),
              }
    output = {'z': DynamicCover(0, 4)
             }

    h = Interface(mgr, inputs, output)

    # Rename inputs
    assert set(h.inputs) == {'x','y'}
    assert set(h.outputs) == {'z'}
    g = ('j', 'x') >> h >> ('z', 'r')
    assert set(g.inputs) == {'j','y'}
    assert set(g.outputs) == {'r'}
    assert g == h.renamed(x='j', z='r')

    precision = {'j': 4, 'y': 3, 'r': 3}
    bittotal = sum(precision.values())

    assert g.count_io_space(bittotal) == approx(1024)
    assert g.count_io(bittotal) == approx(0)
    g = g.io_refined( {'j': (2.9,10.1), 'y': (2.4,3.8), 'r': (2.1,3.1)}, nbits = precision)
    assert g.count_io(bittotal) == approx(42) # = 7 * 2 * 3

    # Adding approximately the same transitions twice does nothing due to quantization
    oldpred = g.pred
    g = g.io_refined( {'j': (3.,10.), 'y': (2.5,3.8), 'r': (2.1,3.1)}, nbits = precision)
    assert g.pred == oldpred

    assert (g).pred.support == {'j_0', 'j_1', 'j_2', 'j_3',
                                    'y_0', 'y_1', 'y_2',
                                    'r_0', 'r_1', 'r_2'}
    assert (g  >> ('r', 'z') ).pred.support == {'j_0', 'j_1', 'j_2', 'j_3',
                                                     'y_0', 'y_1', 'y_2',
                                                     'z_0', 'z_1', 'z_2'}

    assert g.nonblock() == g.input_to_abs({'j': (3.,10.), 'y': (2.5,3.8)}, nbits = precision) # No inputs block
    assert g.nonblock() == (g.ohidden(g.outputs).pred)

    # Identity test for input and output renaming
    assert ((g  >> ('r', 'z') ) >> ('z','r')) == g
    assert (('j','w') >> (('w','j') >> g) ) == g

    # Parallel composition
    assert set((g * h).outputs) == {'z','r'}
    assert set((g * h).inputs) == {'x','y','j'}

    # Series composition with disjoint I/O yields parallel composition
    assert (g >> h) == (g * h)
    assert (g >> h) == g.composed_with(h)
    assert (g >> h) == h.composed_with(g)
    assert (g * h) == h.composed_with(g)

    # Out of bounds errors
    with raises(OutOfDomainError):
        g = g.io_refined( {'j': (3.,10.), 'y': (2.5,3.8), 'r': (2.1,4.6)}, silent=False, nbits = precision)

def test_series_comp():

    mgr = BDD()

    inputs = {'x': DynamicCover(0, 4),
              'y': DynamicCover(0, 4),
              }
    output = {'z': DynamicCover(0, 4)
             }

    h = Interface(mgr, inputs, output)
    g = ('j', 'y') >> h >> ('z', 'r')
    precision = {'j': 4, 'x': 3, 'r': 3}
    g = g.io_refined( {'j': (.75,2.5), 'x': (2.5,3.8), 'r': (2.1,3.1)}, nbits = precision)

    h = ('r', 'x') >> h
    h = h.io_refined({'r': (1.3, 3.8), 'y': (1.0, 2.0), 'z': (.9, 3.1)}, nbits = {'r': 3, 'y': 3, 'z': 4})
    assert (g >> h) == g.composed_with(h)
    assert (g >> h) == h.composed_with(g)
    assert (g >> h).nonblock() != mgr.false
    assert (g >> h).count_nb(bits=10) == approx(28)  # 7 * 2 * 2

def test_mixed_module():

    from redax.module import Interface
    from redax.spaces import DynamicCover, FixedCover

    mgr = BDD()
    inputs = {'x': DynamicCover(0, 16),
              'y': FixedCover(-10, 10, 10),
              'theta': DynamicCover(-np.pi, np.pi, periodic=True),
              'v': FixedCover(0, 5, 5),
              'omega': FixedCover(-2, 2, 4)
              }
    outputs = {'xnext': DynamicCover(0, 4),
             'ynext': FixedCover(-10,10,10),
             'thetanext': DynamicCover(-np.pi, np.pi, periodic=True)
             }

    dubins = Interface(mgr, inputs, outputs)

    # Underspecified input-output
    with raises(AssertionError):
        dubins.io_refined( {'v': (3.6,3.7), 'theta': (6,-6), 'y': (2,3), 'ynext': (2.1,3.1)},
                                nbits = {'theta': 3})

    # Test that fixed covers yield correct space cardinality
    assert mgr.count(dubins.inspace(), 4+4+4+3+2) == 16 * 10 * 16 * 5 * 4
    assert mgr.count(dubins.outspace(), 2 + 4 + 4) == 4 * 10 * 16

def test_embeddedgrid_module():

    mgr = BDD()
    inputs = {'x': EmbeddedGrid(4, 0, 3)}
    outputs = {'y': EmbeddedGrid(8, 4, 11)}

    m = Interface(mgr, inputs, outputs)

    mgr.declare("x_0", "x_1", "y_0", "y_1", "y_2")
    x0 = mgr.var("x_0")
    x1 = mgr.var("x_1")
    y0 = mgr.var("y_0")
    y1 = mgr.var("y_1")
    y2 = mgr.var("y_2")
    assert m.io_refined({'x': 2, 'y':4}).pred == (x0 & ~x1) &  (~(x0 & ~x1) | (~y0 & ~y1 & ~y2))

    assert len(mgr.vars) > 0

def test_module_composition():
    mgr = BDD()

    x = DynamicCover(0,10)

    m1 = Interface(mgr, {'a': x}, {'b': x, 'c': x})
    m2 = Interface(mgr, {'i': x, 'j': x}, {'k': x})

    m12 = (m1 >> m2.renamed(i = 'c'))
    assert set(m12.inputs) == set(['a', 'j'])
    assert set(m12.outputs) == set(['c', 'b', 'k'])
    assert m12 == m2.renamed(i = 'c').composed_with(m1)
    assert m12 == (('c', 'i') >> m2 ).composed_with(m1)

    # Renaming is left associative
    assert m12 == m1 >> (('c', 'i') >> m2 )
    assert m12 != (m1 >> ('c', 'i')) >> m2
    assert m12 == ((m1.renamed(c = 'i') >> m2)).renamed(i = 'c')
    assert m12 == ((m1 >> ('c', 'i') >> m2)).renamed(i = 'c')
    assert m12 == m1.renamed(c = 'i').composed_with(m2).renamed(i = 'c')
    assert m12 == m1.renamed(c = 'i').composed_with(m2) >> ('i', 'c')

def test_refinement_and_coarsening():

    mgr = BDD()

    def conc(x):
        return -3*x

    x = DynamicCover(-10, 10)
    y = DynamicCover(20, 20)

    linmod = Interface(mgr, {'x': x}, {'y':y})

    width = 15

    for _ in range(50):
        # Generate random input windows
        f_width = {'x': np.random.rand()*width}
        f_left  = {'x': -10 +  np.random.rand() * (20 - f_width['x'])}
        f_right = {k: f_width[k] + f_left[k] for k in f_width}
        iobox   = {k: (f_left[k], f_right[k]) for k in f_width}

        # Generate output overapproximation
        ur = conc(**f_left)
        ll = conc(**f_right)
        iobox['y'] = (ll, ur)

        # Refine and check abstract relation
        newmod = linmod.io_refined(iobox, nbits = {'x': 8, 'y':8})
        assert linmod <= newmod
        linmod = newmod

        # Check abstract relation relative to coarsened module
        assert linmod.coarsened(x=5,y=5) <= linmod
        assert linmod.coarsened(x=5) <= linmod
        assert linmod.coarsened(y=5) <= linmod
         # Coarsen should do nothing because it keeps many bits
        assert linmod.coarsened({'x':10},y=10) == linmod

def test_sin_sqrt_comp():

    def containszero(left, right):
        """Determine if 0 is contained in a periodic interval [left,right]. Possible that right < left."""
        # Map to interval [-pi,pi]
        left = ((left + np.pi) % (np.pi*2) ) - np.pi
        left = left - 2*np.pi if left > np.pi else left
        right = ((right + np.pi) % (np.pi*2) ) - np.pi
        right = right - 2*np.pi if right > np.pi else right

        if right < left:
            if left <= 0:
                return True
        else:
            if left <= 0 and right >= 0:
                return True

        return False

    def maxmincos(left, right):
        """Compute the maximum and minimum values of cos in an interval."""
        if containszero(left, right) is True:
            maxval = 1
        else:
            maxval = max([np.cos(left), np.cos(right)])

        if containszero(left + np.pi, right + np.pi) is True:
            minval = -1
        else:
            minval = min([np.cos(left), np.cos(right)])

        return (minval, maxval)

    def maxminsin(left, right):
        """Compute the maximum and minimum values of sin in an interval."""
        return maxmincos(left - np.pi/2, right - np.pi/2)

    mgr = BDD()

    sinout = DynamicCover(-1.2,1.2)
    sinin = DynamicCover(-2*np.pi, 2*np.pi, periodic=True)

    # Sin module
    sinmod = Interface(mgr, {'sin': sinin}, {'sout': sinout})

    # Sqrt module
    sqrtout = DynamicCover(0, 1.2)
    sqrtmod = Interface(mgr, {'sout': sinout}, {'sqrt': sqrtout})

    comp = CompositeInterface([sinmod, sqrtmod])

    def random_input_gen(module: Interface, scale: float) -> dict:
        iobox = dict()
        for invar, space in module.inputs.items():
            if isinstance(space, ContinuousCover):
                width = scale * space.width()
                if space.periodic:
                    left = space.lb + np.random.rand() * space.width()
                else:
                    left = space.lb + np.random.rand() * (space.width() - width)
                right = left + width
                iobox.update({invar: (left, right)})
        return iobox

    precision = {'sin': 9, 'sout': 9, 'sqrt': 9}

    # Learn sin module
    from redax.visualizer import scatter2D
    for i in range(200):
        iobox = random_input_gen(sinmod, scale = .01)
        out = maxminsin(iobox['sin'][0], iobox['sin'][1])
        iobox.update({'sout': (out[0], out[1])})

        # No errors should be raised
        sinmod = sinmod.io_refined(iobox, silent=False, nbits=precision)
        comp = comp.io_refined(iobox, nbits=precision)
        assert sinmod == comp.children[0]

    # Learn sqrt module
    for i in range(200):
        iobox = random_input_gen(sqrtmod, scale = .1)
        if iobox['sout'][0] < 0 or iobox['sout'][1] < 0:
            continue
        out = (math.sqrt(iobox['sout'][0]), math.sqrt(iobox['sout'][1]))
        iobox.update({'sqrt': out})
        sqrtmod = sqrtmod.io_refined(iobox, silent=True, nbits=precision)
        comp = comp.io_refined(iobox, nbits=precision)
        assert sqrtmod == comp.children[1]

    # sinroot = (sinmod >> sqrtmod).ohidden(['sout'])
    sinroot = (comp.children[0] >> comp.children[1]).ohidden(['sout'])
    assert set(sinroot.vars) == {'sin', 'sqrt'}
    assert sinroot.pred != mgr.false
    sinroot.check()

    # scatter2D(mgr, ('sin', sinin), ('sout', sqrtout), comp.children[0].coarsened(sout = 6, sin = 6) .pred)
    # scatter2D(mgr, ('sout', sqrtout), ('sqrt', sqrtout), comp.children[1].pred)
    # scatter2D(mgr, ('sin', sinin), ('sqrt', sqrtout), sinroot.pred)

def test_composite_module_topology():

    mgr = BDD()

    x = DynamicCover(0,10)

    m1 = Interface(mgr, {'a': x}, {'b': x, 'i': x})
    m2 = Interface(mgr, {'i': x, 'j': x}, {'k': x})

    m12 = CompositeInterface((m1, m2))
    assert m12.sorted_mods() == ((m1,), (m2,))
    assert set(m12.outputs) == {'k', 'b', 'i'}
    assert set(m12.inputs) == {'a', 'j'}
    assert set(m12.latent) == {'i'}

    m3 = Interface(mgr, {'k': x, 'b': x}, {})

    m123 = CompositeInterface([m1,m2,m3])
    assert m123.sorted_mods() == ((m1,), (m2,), (m3,))
    assert set(m123.outputs) == {'b','i','k'}
    assert set(m123.inputs) == {'a', 'j'}
    assert set(m123.vars) == {'a','j','b','i','k'}

    # Renaming
    m123renamed = m123.renamed(b='r', a='q')
    assert set(m123renamed.outputs) == {'r','i','k'}
    assert set(m123renamed.inputs) == {'q', 'j'}

def test_sinks():

    mgr = BDD()
    x =  DynamicCover(0, 10)

    x1 = x.conc2pred(mgr,'x',[1,4], 5, innerapprox=True)
    x2 = x.conc2pred(mgr,'x',[5,8], 5, innerapprox=True)
    x3 = x.conc2pred(mgr,'x',[0.0001, 5.001], 5, innerapprox=False)
    x4 = x.conc2pred(mgr,'x',[4.9999, 9.999], 5, innerapprox=False)

    m1 = Interface(mgr, {'x': x}, {}, assum=x1)
    m2 = Interface(mgr, {'x': x}, {}, assum=x2)
    m3 = Interface(mgr, {'x': x}, {}, assum=x3)
    m4 = Interface(mgr, {'x': x}, {}, assum=x4)

    assert (m1 * m2).assum == mgr.false
    assert (m1 + m2).assum != mgr.false
    assert (m3 + m4).assum == mgr.true

    assert (m3 + m1) == m3
    assert (m3 * m1)== m1

    assert (m4 * m1) <= m4


import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
from matplotlib import font_manager, rc
from sklearn.preprocessing import OrdinalEncoder
from imblearn.under_sampling import RandomUnderSampler
from sklearn.model_selection import train_test_split

font_path = "C:/Windows/Fonts/batang.ttc"
font = font_manager.FontProperties(fname=font_path).get_name()
rc('font', family=font)

STROKE_MENUS = ["종료", #0
                "데이터구하기",#1
                "한글화",#2
                "타깃변수설정",#3
                "ID변수설정",#4
                "구간확률변수설정",#5
                "범주형변수설정",#6
                "결측값이 50% 초과인 변수 제거",#7
                "이상값 제거",#8
                "csv 저장",#9
                "샘플링"]

stroke_meta = {
    'id':'아이디', 'gender':'성별', 'age':'나이', 'hypertension':'고혈압', 'heart_disease':'심장병',
    'ever_married':'기혼여부', 'work_type':'직종', 'Residence_type':'거주형태',
    'avg_glucose_level':'평균혈당', 'bmi':'체질량지수', 'smoking_status':'흡연상태', 'stroke':'뇌졸중'}
stroke_menu = {
    "1" : lambda t: t.spec(),
    "2" : lambda t: t.rename_meta(),
    "3" : lambda t: t.target(),
    "4" : lambda t: t.id_variable(),
    "5" : lambda t: t.interval_variable(),
    "6" : lambda t: t.categorical_variable(),
    "7" : lambda t: t.remove_null(),
    "8": lambda t: t.remove_outlier(),
    "9": lambda t: t.additional_process(),
    "10": lambda t: t.sampling()
}
'''
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 5110 entries, 0 to 5109
Data columns (total 12 columns):
 #   Column             Non-Null Count  Dtype
---  ------             --------------  -----
 0   id                 5110 non-null   int64
 1   gender             5110 non-null   object
 2   age                5110 non-null   float64
 3   hypertension       5110 non-null   int64
 4   heart_disease      5110 non-null   int64
 5   ever_married       5110 non-null   object
 6   work_type          5110 non-null   object
 7   Residence_type     5110 non-null   object
 8   avg_glucose_level  5110 non-null   float64
 9   bmi                4909 non-null   float64
 10  smoking_status     5110 non-null   object
 11  stroke             5110 non-null   int64
dtypes: float64(3), int64(4), object(5)
memory usage: 479.2+ KB
None
'''

class StrokeService:
    def __init__(self):
        self.stroke = pd.read_csv('../../pythonProject/static/data/dam/ml/healthcare-dataset-stroke-data.csv')
        self.my_stroke = None
        self.df1 = None
        self.cols = None
        self.cols1 = None
        self.df2 = None
    '''
    1.스펙보기
    '''
    def spec(self):
        print(" --- 1.Shape ---")
        print(self.stroke.shape)
        print(" --- 2.Features ---")
        print(self.stroke.columns)
        print(" --- 3.Info ---")
        print(self.stroke.info())
        print(" --- 4.Case Top1 ---")
        print(self.stroke.head(1))
        print(" --- 5.Case Bottom1 ---")
        print(self.stroke.tail(3))
        print(" --- 6.Describe ---")
        print(self.stroke.describe())
        print(" --- 7.Describe All ---")
        print(self.stroke.describe(include='all'))
    '''
    2.한글 메타데이터
    '''
    def rename_meta(self):
        self.my_stroke = self.stroke.rename(columns=stroke_meta)
        print(" --- 2.Features ---")
        print(self.my_stroke.columns)
    '''
    3.타깃변수(=종속변수 dependent, y값) 설정
        입력변수(=설명변수, 확률변수, X값)
        타깃변수명 : stroke(=뇌졸중)
        타깃변수값 : 과거에 한 번이라도 뇌졸중이 발병했으면 1, 아니면 0
    '''
    def target(self):
        self.rename_meta()
        df = self.my_stroke
        print(f'stroke type : {df["뇌졸중"].dtype}')
        print(f'stroke null : {df["뇌졸중"].isnull().sum()}')
        print(f'stroke value counts : \n{df["뇌졸중"].value_counts(dropna=False)}')

    def id_variable(self):
        self.target()
        df = self.my_stroke
        print(df['아이디'].dtypes)
        print(f'id 결측값 : {df["아이디"].isnull().sum()}')
        n = len(pd.unique(df['아이디'])) - len(df['아이디'])
        print(f'중복된 id 개수 : {n}')

    def interval_variable(self):
        self.id_variable()
        df = self.my_stroke
        self.cols = ['나이','평균혈당','체질량지수']
        print(df[self.cols].dtypes)
        pd.options.display.float_format = '{:.2f}'.format
        print(df[self.cols].describe())
        c = df['나이'] > 18
        print(f"나이 > 18 : \n{df[c].head(3)}")
        print(f'df[c]의 자료 개수 : {len(df[c])}')
        print(f'df 대비 df[c]의 자료 개수 비율 : {len(df[c])/len(df)}')
        self.df1 = df[c]
        print(f'df1 shape : {self.df1.shape}')

    def categorical_variable(self):
        self.interval_variable()
        df1 = self.df1
        self.cols1 = ['성별','고혈압','심장병','기혼여부','직종','거주형태','흡연상태']
        print(f'cols1 null 개수 : \n{df1[self.cols1].isnull().sum()}')
        print(f'cols1 type : \n{df1[self.cols1].dtypes}')

    def remove_null(self):
        self.interval_variable()
        df1 = self.df1
        df = self.my_stroke
        cols = self.cols
        print(df1.isna().any()[lambda x:x])
        print(f'rate : {df["체질량지수"].isnull().mean()}')
        print(f"외도 : \n{df1[cols].skew()}")
        print(f"첨도 : \n{df1[cols].kurtosis()}")
        print(f"work type : \n{df1['직종'].value_counts(dropna=False)}")
        print(pd.crosstab(df1['직종'],columns='count'))
        print(pd.crosstab(df1['직종'],columns='ratio',normalize=True))
        print(pd.crosstab(df1['직종'],df1['뇌졸중']))
        print(pd.crosstab(df1['직종'],df1['뇌졸중'], normalize=True))

    def remove_outlier(self):
        self.remove_null()
        df1 = self.df1
        df = self.my_stroke
        fig, axes = plt.subplots(1, 3, figsize=(15, 4))
        sns.histplot(ax=axes[0], data=df, x="나이", kde=True, bins=20)
        sns.histplot(ax=axes[1], data=df, x="평균혈당", kde=True, bins=20)
        sns.histplot(ax=axes[2], data=df, x="체질량지수", kde=True, bins=20)
        plt.show()
        sns.set_style('whitegrid')
        fig, axes = plt.subplots(1, 3, figsize=(15, 4))
        sns.boxplot(ax=axes[0], x='나이', data=df1)
        sns.boxplot(ax=axes[1], x='평균혈당', data=df1)
        sns.boxplot(ax=axes[2], x='체질량지수', data=df1)
        plt.show()
        Q1 = df1[['나이', '평균혈당', '체질량지수']].quantile(0.25)
        Q3 = df1[['나이', '평균혈당', '체질량지수']].quantile(0.75)
        IQR = Q3 - Q1
        print(f'IQR : \n{IQR}')
        print("*" * 50)
        Lower = Q1 - 3.0 * IQR
        Upper = Q3 + 3.0 * IQR
        print(f'Lower : \n{Lower}')
        print(f'Upper : \n{Upper}')
        print("*" * 50)
        c1 = df1['평균혈당'] <= 232.64
        c2 = df1['체질량지수'] <= 60.3
        self.df2 = df1[c1 & c2]
        print(f'shape : {self.df2.shape}')
        print(self.df2.info())

    def additional_process(self):
        df2 = self.df2
        df2['성별'] = OrdinalEncoder().fit_transform(df2['성별'].values.reshape(-1, 1))
        df2['기혼여부'] = OrdinalEncoder().fit_transform(df2['기혼여부'].values.reshape(-1, 1))
        df2['직종'] = OrdinalEncoder().fit_transform(df2['직종'].values.reshape(-1, 1))
        df2['거주형태'] = OrdinalEncoder().fit_transform(df2['거주형태'].values.reshape(-1, 1))
        df2['흡연여부'] = OrdinalEncoder().fit_transform(df2['흡연여부'].values.reshape(-1, 1))
        self.df2 = df2

        print(" ### 프리프로세스 종료 ### ")
        self.stroke.to_csv("../../../static/save/dam/ml/stroke.csvstroke.csv")

    def sampling(self):
        df = pd.read_csv("../../pythonProject/static/save/dam/ml/stroke.csv")
        data = df.drop(['뇌졸중'], axis=1)
        target = df['뇌졸중']
        undersample = RandomUnderSampler(sampling_strategy=0.333, random_state=2)
        data_under, target_under = undersample.fit_resample(data, target)
        print(target_under.value_counts(dropna=True))
        X_train, X_test, y_train, y_test = train_test_split(data_under, target_under,
                                                            test_size=0.5, random_state=42, stratify=target_under)
        print("X_train shape :", X_train.shape)
        print("X_test shape :", X_test.shape)
        print("y_train shape :", y_train.shape)
        print("y_test shape :", y_test.shape)
        print("y_train value ratio :\n", y_train.value_counts(normalize=True))
        print("y_train value count :\n", y_train.value_counts())

if __name__ == '__main__':
    def my_menu(ls):
        for i, j in enumerate(ls):
            print(f"{i}. {j}")
        return input('메뉴선택: ')

    if __name__ == '__main__':
        t = StrokeService()
        while True:
            menu = my_menu(STROKE_MENUS)
            if menu == '0':
                print("종료")
                break
            else:
                stroke_menu[menu](t)
                try:
                    stroke_menu[menu](t)
                except KeyError:
                    print(" ### Error ### ")


import time
import datetime
import traceback

def timeit(method):
    def timed(*args, **kw):

        ts = time.time()
        result = method(*args, **kw)
        te = time.time()

        now = str(datetime.datetime.now())
        msg = ('method : {} (args : {}, {}) {:06.2f} sec').format(method.__name__, args, kw, te-ts)
        f = open('speedlog.txt', 'a')
        f.write('\n'+now+'\n')
        f.write(msg)
        f.close()

        return result
    return timed

def save_err(method):
    def wrapper(*args, **kw):
        try:
            result = method(*args, **kw)
            return result
        except :
            def classification():
                now = str(datetime.datetime.now())
                name = method.__name__
                msg = ('{} method : {} (args : {}, {})').format(now, name, args, kw)
                return msg

            msg = classification()
            trace = str(traceback.format_exc())

            f = open('errorlog.txt', 'a')
            f.write('\n'+msg+'\n')
            f.write(trace)
            f.close()
    return wrapper


from  sklearn import ensemble
from sklearn.preprocessing import LabelEncoder
import pandas as pd

def set_missing(df,estimate_list,miss_col):
    """df要处理的数据帧，estimate_list输入的字段名称,miss_col缺失字段名称;会直接在原来的数据帧上修改"""
    col_list = []
    col_list.extend(estimate_list)
    col_list.append(miss_col)
    process_df = df.loc[:  , col_list]      # 所有列
    # class_le = LabelEncoder()
    # for i in col_list[:-1]:                       # 最后一个以前, 输入字段名称
    #     print('col_list[:-1]',col_list[:-1])
    #     process_df.loc[:,i] = class_le.fit_transform(process_df.loc[:,i].values)
    # 分成已知该特征和未知该特征两部分
    known = process_df[process_df[miss_col].isnull() & process_df[estimate_list[0]].notnull()].values   # 预测并返回集 要预测的列为空，用于预测的列非空  注意  .value
    # known[:, -1]=class_le.fit_transform(known[:, -1])
    unknown = process_df[process_df[miss_col].isnull() & process_df[estimate_list[0]].notnull()]
    all_known = process_df[process_df[miss_col].notnull() & process_df[estimate_list[0]].notnull()].values  # 训练集 要预测的为不为空，用于预测的列也不空

    all_known_x = all_known[:, :-1]    # 训练集x， 用于预测的列
    all_known_y = all_known[:, -1]    # 训练集y， 要预测的列
    # X为特征属性值
    x = known[:, :-1] # 输入字段
    # y为结果标签值
    y = known[:, -1]  # 缺失字段

    # fit到RandomForestRegressor之中
    rfr = ensemble.RandomForestRegressor(random_state=1, n_estimators=1000,max_depth=8,n_jobs=-1)
    rfr.fit(all_known_x,all_known_y)
    score=rfr.score(all_known_x,all_known_y)
    print(score)
    # 用得到的模型进行未知特征值预测
    predicted = rfr.predict(x).round(0).astype(int)
    # predicted = class_le.inverse_transform(predicted)
    # print(predicted)
    # 用得到的预测结果填补原缺失数据
    df.loc[(df[miss_col].isnull() & df[estimate_list[0]].notnull()), miss_col] = predicted   # 缺失值为空，用于预测的值非空
    return df

if __name__=='__main__':
    df = pd.read_csv('C:/Users/41634/Desktop/WMS_1440.csv')
    #df=df.drop(['DATETIME'],axis=1)
    columns=df.columns
    print(columns[1])
    print(columns[2])
    #dff=set_missing(df,columns[:, 1],columns[:, 2])
    #dff=set_missing(df,['F_NH3'],'F_TP')
    dff = set_missing(df, ['F_TN'], 'F_TP')
    dff.to_csv('C:/Users/41634/Desktop/1440.csv')


"""
    4IRC
    Exercice du clone du Jeu de la Vie
    Groupe :
        - Maxime BATTU
        - Eileen BALAGUER
        - Batiste LALOI
"""
import time
import random
import os
import ctypes

import multiprocessing as mp
import numpy as np

TAILLE = 30
NB_PROCESS = 5

ITERATION = 50

def clear(): os.system("clear")
def move_to(lig, col) : print("\033[" + str(lig) + ";" + str(col) + "f",end='')

# Create a shared two dimensionnal array
shared_current_grid = mp.Array(ctypes.c_int, TAILLE * TAILLE)
shared_next_grid = mp.Array(ctypes.c_int, TAILLE * TAILLE)

# create a numpy array from the shared memory block
current_grid = np.frombuffer(shared_current_grid.get_obj(), ctypes.c_int)
next_grid = np.frombuffer(shared_next_grid.get_obj(), ctypes.c_int)

# reshape the array into a two dimensional array
current_grid = current_grid.reshape((TAILLE, TAILLE))
next_grid = next_grid.reshape((TAILLE, TAILLE))

def display_grid(grid):
    print("_" * (2 * TAILLE + 1))
    for line in grid:
        print("|", end="")
        for cell in line:
            if cell == 0:
                print("  ", end="")
            else:
                print("██", end="")
        print("|")
    print("‾" * (2*TAILLE + 1))

def addGlider(x, y, grid):
    glider = np.array([[0,   0, 1],
                       [1,  0, 1],
                       [0,  1, 1]])
    grid[x:x+3, y:y+3] = glider

# Fonction qui renvoie le nombre de voisins vivants d'une cellule donnée
def living_neighbours_count(x, y, grid):
    count = 0
    for i in range(-1, 2):
        for j in range(-1, 2):
            if i == 0 and j == 0:
                continue
            elif x + i < 0 or x + i >= TAILLE or y + j < 0 or y + j >= TAILLE:
                continue
            elif grid[x + i][y + j] == 1:
                count += 1
    return count

# Fonction qui traite une cellule
def render_cell(x, y, current_grid, next_grid):
    living_neighbours = living_neighbours_count(x, y, current_grid)
    if current_grid[x][y] == 1:
        if living_neighbours == 2 or living_neighbours == 3:
            next_grid[x][y] = 1
        else :
            next_grid[x][y] = 0
    else:
        if living_neighbours == 3:
            next_grid[x][y] = 1

# Fonction qui traite une ligne
def render_line(line, current_grid, next_grid):
    for cell in range(TAILLE):
        render_cell(line, cell, current_grid, next_grid)

# Ajout d'une glider dans la grille de départ
# addGlider(1, 1, current_grid)

# On remplis de manière aléatoire la grille, avec 20% de chance d'avoir une cellule vivante
for i in range(TAILLE):
    for j in range(len(current_grid[i])):
        if random.randint(0, 4) == 0:
            current_grid[i][j] = 1

clear()

cpt = 0
while True:

    # Effacer l'écran
    move_to(0, 0)

    # Pack de processus
    for pack in range(TAILLE//NB_PROCESS):
        # print(f"Pack #{pack}")
        for line in range(NB_PROCESS) :
            p = mp.Process(target=render_line, args=(pack*NB_PROCESS + line, current_grid, next_grid))
            p.start()

        for _ in range(NB_PROCESS):
            p.join()

    display_grid(next_grid)

    for i in range(TAILLE):
        for j in range(len(next_grid[i])):
            current_grid[i][j] = next_grid[i][j]

    # time.sleep(0)
    cpt += 1
    print(f"Génération numéro : {cpt}")


import json
from kubernetes import client, config, watch
import logging
import os

DOMAIN = "rules-k8s.bazel.io"

def main():
    config.load_incluster_config()

    # Exposed to us through the downward API.
    namespace = os.environ["MY_NAMESPACE"]

    crds = client.CustomObjectsApi()

    def mark_done(event, obj):
        metadata = obj.get("metadata")
        if not metadata:
            logging.error("No metadata in object, skipping: %s", json.dumps(obj, indent=1))
            return
        name = metadata.get("name")

        obj["spec"]["done"] = True
        obj["spec"]["comment"] = "DEMO "

        logging.error("Updating: %s", name)
        crds.replace_namespaced_custom_object(DOMAIN, "v1", namespace, "todos", name, obj)

    resource_version = ''
    while True:
        stream = watch.Watch().stream(crds.list_namespaced_custom_object,
                                      DOMAIN, "v1", namespace, "todos",
                                      resource_version=resource_version)
        for event in stream:
            obj = event["object"]

            spec = obj.get("spec")
            if not spec:
                logging.error("No 'spec' in object, skipping event: %s", json.dumps(obj, indent=1))
            else:
                if not spec.get("done", True):
                    mark_done(event, obj)

            # Configure where to resume streaming.
            metadata = obj.get("metadata")
            if metadata:
                resource_version = metadata["resourceVersion"]

if __name__ == "__main__":
    main()


import socket
import msgpack
import rospy
import math
import numpy as np

from zzz_cognition_msgs.msg import MapState
from zzz_driver_msgs.utils import get_speed
from carla import Location, Rotation, Transform
from zzz_common.geometry import dense_polyline2d
from zzz_common.kinematics import get_frenet_state

from zzz_planning_msgs.msg import DecisionTrajectory
from zzz_planning_decision_continuous_models.VEG.Werling_planner_RL import Werling
from zzz_planning_decision_continuous_models.common import rviz_display, convert_ndarray_to_pathmsg, convert_path_to_ndarray
from zzz_planning_decision_continuous_models.predict import predict

# PARAMETERS
THRESHOLD = 0.2
OBSTACLES_CONSIDERED = 3
ACTION_SPACE_SYMMERTY = 15/3.6

# Should be the same with those in werling!
KICK_IN_TIME = 2.1
DT = 0.3
KICK_IN_POINT = 7 # 2.1/0.3

class VEG_Planner(object):
    """
    Parameter:
        mode: ZZZ TCP connection mode (client/server)
    """
    def __init__(self, openai_server="127.0.0.1", port=2333, mode="client", recv_buffer=4096):
        self._dynamic_map = None
        self._socket_connected = False
        self._rule_based_trajectory_model_instance = Werling()
        self._buffer_size = recv_buffer
        self._collision_signal = False
        self._collision_times = 0
        self._has_clear_buff = False

        self.reference_path = None
        self.ref_path = None
        self.ref_path_tangets = None

        self.rivz_element = rviz_display()

        if mode == "client":
            rospy.loginfo("Connecting to RL server...")
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect((openai_server, port))
            self._socket_connected = True
            rospy.loginfo("Connected...")
        else:
            # TODO: Implement server mode to make multiple connection to this node.
            #     In this mode, only rule based action is returned to system
            raise NotImplementedError("Server mode is still wating to be implemented.")

    def initialize(self, dynamic_map):
        try:
            if self.reference_path is None: # a fixed reference path
                self.reference_path = dynamic_map.jmap.reference_path.map_lane.central_path_points
                ref_path_ori = convert_path_to_ndarray(self.reference_path)
                self.ref_path = dense_polyline2d(ref_path_ori, 2)
                self.ref_path_tangets = np.zeros(len(self.ref_path))
            return True
        except:
            rospy.logdebug("------> VEG: Initialize fail ")
            return False

    def clear_buff(self, dynamic_map):
        self._rule_based_trajectory_model_instance.clear_buff(dynamic_map)
        self._collision_signal = False
        self.reference_path = None
        self.ref_path = None
        self.ref_path_tangets = None
        self.rivz_element.kick_in_signal = None

        # send done to OPENAI
        if self._has_clear_buff == False:
            ego_x = dynamic_map.ego_state.pose.pose.position.x
            ego_y = dynamic_map.ego_state.pose.pose.position.y
            if math.pow((ego_x+10),2) + math.pow((ego_y-94),2) < 64:  # restart point
                leave_current_mmap = 2
            else:
                leave_current_mmap = 1

            collision = 0
            sent_RL_msg = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] # ego and obs state
            sent_RL_msg.append(collision)
            sent_RL_msg.append(leave_current_mmap)
            sent_RL_msg.append(THRESHOLD) # threshold
            sent_RL_msg.append(0.0) # Rule_based_point.d
            sent_RL_msg.append(0.0 - ACTION_SPACE_SYMMERTY) # Rule_based_point.vs
            print("-----------------------------",sent_RL_msg)

            self.sock.sendall(msgpack.packb(sent_RL_msg))
            try:
                RLS_action = msgpack.unpackb(self.sock.recv(self._buffer_size))
            except:
                pass

            self._has_clear_buff = True
        return None

    def trajectory_update(self, dynamic_map):
        if self.initialize(dynamic_map):
            self._has_clear_buff = False
            self._dynamic_map = dynamic_map

            # wrap states
            sent_RL_msg = self.wrap_state()

            # rule-based planner
            rule_trajectory_msg = self._rule_based_trajectory_model_instance.trajectory_update(dynamic_map)
            RLpoint = self.get_RL_point_from_trajectory(self._rule_based_trajectory_model_instance.last_trajectory_rule)
            sent_RL_msg.append(RLpoint.location.x)
            sent_RL_msg.append(RLpoint.location.y)
            print("-----------------------------",sent_RL_msg)
            self.sock.sendall(msgpack.packb(sent_RL_msg))

            # received RL action and plan a RL trajectory
            try:
                received_msg = msgpack.unpackb(self.sock.recv(self._buffer_size))
                rl_action = [received_msg[0], received_msg[1]]
                rl_q = received_msg[2]
                rule_q = received_msg[3]
                return self.generate_VEG_trajectory(rl_q, rule_q, rl_action, rule_trajectory_msg)

            except:
                rospy.logerr("Continous RLS Model cannot receive an action")
                return rule_trajectory_msg
        else:
            return None

    def wrap_state(self):
        # ego state: ego_x(0), ego_y(1), ego_vx(2), ego_vy(3)
        # obstacle 0 : x0(4), y0(5), vx0(6), vy0(7)
        # obstacle 1 : x0(8), y0(9), vx0(10), vy0(11)
        # obstacle 2 : x0(12), y0(13), vx0(14), vy0(15)
        state = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]

        # ego state
        ego_ffstate = get_frenet_state(self._dynamic_map.ego_state, self.ref_path, self.ref_path_tangets)
        state[0] = ego_ffstate.s
        state[1] = -ego_ffstate.d
        state[2] = ego_ffstate.vs
        state[3] = ego_ffstate.vd

        # obs state
        closest_obs = []
        closest_obs = self.found_closest_obstacles(OBSTACLES_CONSIDERED, self._dynamic_map)
        i = 0
        for obs in closest_obs:
            if i < OBSTACLES_CONSIDERED:
                state[(i+1)*4+0] = obs[5]
                state[(i+1)*4+1] = obs[6]
                state[(i+1)*4+2] = obs[7]
                state[(i+1)*4+3] = obs[8]
                i = i+1
            else:
                break

        # if collision
        collision = int(self._collision_signal)
        self._collision_signal = False
        state.append(collision)

        # if finish
        leave_current_mmap = 0
        state.append(leave_current_mmap)
        state.append(THRESHOLD)

        return state

    def found_closest_obstacles(self, num, dynamic_map):
        closest_obs = []
        obs_tuples = []

        for obs in self._dynamic_map.jmap.obstacles:
            # calculate distance
            p1 = np.array([self._dynamic_map.ego_state.pose.pose.position.x , self._dynamic_map.ego_state.pose.pose.position.y])
            p2 = np.array([obs.state.pose.pose.position.x , obs.state.pose.pose.position.y])
            p3 = p2 - p1
            p4 = math.hypot(p3[0],p3[1])

            # transfer to frenet
            obs_ffstate = get_frenet_state(obs.state, self.ref_path, self.ref_path_tangets)
            one_obs = (obs.state.pose.pose.position.x , obs.state.pose.pose.position.y , obs.state.twist.twist.linear.x ,
                     obs.state.twist.twist.linear.y , p4 , obs_ffstate.s , -obs_ffstate.d , obs_ffstate.vs, obs_ffstate.vd,
                     obs.state.accel.accel.linear.x, obs.state.accel.accel.linear.y)
            obs_tuples.append(one_obs)

        # sort by distance
        sorted_obs = sorted(obs_tuples, key=lambda obs: obs[4])
        i = 0
        for obs in sorted_obs:
            if i < num:
                closest_obs.append(obs)
                i = i + 1
            else:
                break

        return closest_obs

    def get_RL_point_from_trajectory(self, frenet_trajectory_rule):
        RLpoint = Transform()

        if len(frenet_trajectory_rule.t) > KICK_IN_POINT:
            RLpoint.location.x = frenet_trajectory_rule.d[KICK_IN_POINT] #only works when DT param of werling is 0.15
            RLpoint.location.y = frenet_trajectory_rule.s_d[KICK_IN_POINT] - ACTION_SPACE_SYMMERTY
        else:
            RLpoint.location.x = 0
            RLpoint.location.y = 0 - ACTION_SPACE_SYMMERTY

        return RLpoint

    def generate_VEG_trajectory(self, rl_q, rule_q, rl_action, rule_trajectory_msg):

        print("rl_action", rl_action[0], rl_action[1])
        print("rl_q", rl_q)
        print("rule_q", rule_q)

        if rl_q - rule_q > THRESHOLD and rl_action[0] < 2333 and rl_action[1] < 2333:
            rl_action[1] = rl_action[1] + ACTION_SPACE_SYMMERTY
            self.rivz_element.kick_in_signal = self.rivz_element.draw_kick_in_path(self._rule_based_trajectory_model_instance.last_trajectory_array)
            return self._rule_based_trajectory_model_instance.trajectory_update_RL_kick(self._dynamic_map, rl_action)

        else:
            self.rivz_element.kick_in_signal = self.rivz_element.draw_rule_path(self._rule_based_trajectory_model_instance.last_trajectory_array_rule)
            return rule_trajectory_msg


from argparse import ArgumentParser

from . import start_dev, build_app

if __name__ == "__main__":
    parser = ArgumentParser(
        prog="react_cli",
        description="A command-line interface used for developing React-like web applications using Python and PyScript",
    )
    parser.add_argument("command", choices=["dev", "build"])

    args = parser.parse_args()
    if args.command == "dev":
        start_dev()
    elif args.command == "build":
        build_app()


# /usr/bin/python
# -*- coding:utf-8 -*-
import time
from io import BytesIO
import numpy as np
import matplotlib as mpl
from matplotlib.font_manager import *
from matplotlib import pyplot as plt
import logging
myfont = FontProperties(fname='/usr/share/fonts/msyh.ttf')
mpl.rcParams['axes.unicode_minus'] = False

def autolabel(rects, ax, xpos='center'):
    """
    Attach a text label above each bar in *rects*, displaying its height.

    *xpos* indicates which side to place the text w.r.t. the center of
    the bar. It can be one of the following {'center', 'right', 'left'}.
    """

    xpos = xpos.lower()  # normalize the case of the parameter
    ha = {'center': 'center', 'right': 'left', 'left': 'right'}
    offset = {'center': 0.5, 'right': 0.57, 'left': 0.43}  # x_txt = x + w*off

    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width() * offset[xpos], 1.01 * height,
                '{}'.format(height), ha=ha[xpos], va='bottom')

class multiColumn:
    def __init__(self, request):
        self.request = request

    def result(self):
        coordinate = self.request.json.get('coordinate')
        data = self.request.json.get('data')
        logging.error(self.request.json)
        return self.draw(coordinate, data)

    @staticmethod
    def draw(coordinate, data):
        '''
        :param coordinate: {'x':[],'y':'金额'}
        :param data = [
                    {'value':[300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3],'label':'平均值'},
                    {'value':[30.2, 53.3, 75.9, 5862.5, 979.6, 1289.7, 1958.3],'label':'最大值'}
                ]
        :return:文件流
        '''
        try:
            # 柱状图的个数
            n = len(data)
            total_width = 0.8
            ind = np.arange(len(coordinate['x']))
            width = total_width / n
            fig, ax = plt.subplots()
            # y轴
            ax.set_ylabel(coordinate['y'],fontproperties=myfont)
            # x轴
            ax.set_xticks(ind)
            ax.set_xticklabels(coordinate['x'],fontproperties=myfont)

            ind = ind - (total_width - width) / 2
            i = 0
            for x in data:
                temp = ax.bar(ind + i * width, x['value'], width, label=x['label'])
                i += 1
                autolabel(temp, ax)

            ax.legend(prop=myfont)
            # 文件流
            canvas = fig.canvas
            buffer = BytesIO()
            canvas.print_png(buffer)
            stream = buffer.getvalue()
            buffer.close()
            return stream
        except Exception as e:
            return '参数错误，图片生成失败', 400


__author__ = 'rafal'

from client import Client
from wheather import WheatherGenerator
from energy_provider import EnergyProvider
from config import Configuration
from logs import Logger

class Model:

    def __init__(self):
        Configuration.init()
        self.client = Client(Configuration.get_start_time())
        self.state = None

    def tick(self, time):
        self.client.update_time(time)
        params = self.__calculate_parameters()
        self.__log_parameters(params)
        current_time, current_temperature, previous_temperature, water_temperature = params
        self.state = {"T_zm": water_temperature, "T_o": current_temperature}
        return params

    def get_state(self):
        return self.state

    def __calculate_parameters(self):
        """
        Wylicza parametry dostawcy energi.
        """

        current_time = self.client.get_current_time()    # aktualny czas
        previous_time = self.client.get_previous_time()  # czas 8h temu
        current_temperature = WheatherGenerator.get_temperature(current_time)    # temperatura otoczenia
        previous_temperature = WheatherGenerator.get_temperature(previous_time)  # temperatura otoczenia 8h temu
        water_temperature = EnergyProvider.get_water_temperature(previous_temperature)  # temperatura wody

        parameters = (current_time, current_temperature, previous_temperature, water_temperature)
        return parameters

    def __log_parameters(self, params):
        """
        Zapisuje parametry do logów.
        """

        current_time, current_temp, previous_temp, water_temp = params
        current_time = "time="+str(current_time)
        current_temp = "current_temp="+str(current_temp)+" st. C"
        previous_temp = "previous_temp="+str(previous_temp)+" st. C"
        water_temp = "water_temp="+str(water_temp)+" st. C"
        msg = current_time+" "+current_temp+" "+previous_temp+" "+water_temp
        Logger.info(msg)

if __name__ == '__main__':
    import time
    Logger.init()
    Configuration.init()
    provider_model = Model()

    while 1:
        print(provider_model.tick(10))


import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt

def keypoint_match(path_of_image1, path_of_image2):

    img1 = cv.imread(path_of_image1, cv.IMREAD_GRAYSCALE)
    img2 = cv.imread(path_of_image2, cv.IMREAD_GRAYSCALE)

    # Initiate SIFT detector
    sift = cv.xfeatures2d.SIFT_create()

    # find the keypoints and descriptors with SIFT
    kp1, des1 = sift.detectAndCompute(img1, None)
    kp2, des2 = sift.detectAndCompute(img2, None)

    # FLANN parameters
    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
    search_params = dict(checks=50)  # or pass empty dictionary
    flann = cv.FlannBasedMatcher(index_params, search_params)
    matches = flann.knnMatch(des1, des2, k=2)

    # Need to draw only good matches, so create a mask
    matchesMask = [[0, 0] for i in range(len(matches))]
    # ratio test as per Lowe's paper
    c = 0
    for i, (m, n) in enumerate(matches):
        if m.distance < 0.7 * n.distance:
            matchesMask[i] = [1, 0]
            c = c + 1
    # print(matchesMask)
    print(c)
    draw_params = dict(matchColor=(0, 255, 0),
                       singlePointColor=(255, 0, 0),
                       matchesMask=matchesMask,
                       flags=cv.DrawMatchesFlags_DEFAULT)
    img3 = cv.drawMatchesKnn(img1, kp1, img2, kp2, matches, None, **draw_params)
    plt.imshow(img3, ), plt.show()

keypoint_match('D:/MintM/deo_classifier/data/fogg/40001571_2-fogg-fragrance-body-spray-paradise_.jpg',
               'D:/MintM/deo_classifier/data/fogg/40001570_2-fogg-fragrance-body-spray-royal.jpg')


import requests, bs4
from bs4 import BeautifulSoup

def twitter_search(name,pren):
    url = "https://twstalker.com/search/?q={} {}".format(pren,name)
    r = requests.get(url)
    page = r.content
    features = "html.parser"
    soup = BeautifulSoup(page, features)

    usernames = []

    username = soup.find_all('div',{'class':'user-request-dt'})

    for i in username:
        i = str(i).split('@')[1].split('</span')[0].replace('\'','')
        usernames.append('@'+i.strip())

    return usernames