"""Test the helper functions are working correctly."""
from typing import List, Union

import pandas as pd
import pytest

from sqltools import executers, helpers

def test_temp_table() -> None:
    """Test that ``TempTable`` can create a table that can be queried."""
    expected_data = pd.DataFrame({"test": [1]})
    tt = """
    --sql
    SELECT
        1 test INTO ##one;
    """
    one = helpers.TempTable(tt)
    query = """
        --sql
        SELECT
            *
        FROM
            ##one;
    """
    actual_data = executers.run_query(query)
    one.close()
    pd.testing.assert_frame_equal(expected_data, actual_data)

def test_show_temp() -> None:
    """Test that ``show_temp`` is showing all temporary tables."""
    expected_table = "##one"
    tt = """
        --sql
        SELECT
            1 test INTO ##one;
    """
    one = helpers.TempTable(tt)
    table_names = helpers.show_temp()["name"].values
    one.close()
    assert expected_table in table_names

def test_find_cols() -> None:
    """Test that ``find_cols`` runs."""
    expected_columns = ["column_name", "table_name"]
    actual_columns = helpers.find_cols("test").columns.tolist()
    assert expected_columns == actual_columns

def test_find_tables() -> None:
    """Test that ``find_tables`` runs."""
    expected_columns = ["table_name"]
    actual_columns = helpers.find_tables("test").columns.tolist()
    assert expected_columns == actual_columns

@pytest.mark.parametrize(["n", "expected_rows"], [(1, 1), (5, 5)])
def test_head(n: int, expected_rows: int) -> None:
    """Test that ``head`` retruns the top ``n`` rows of a ``sys.tables``."""
    actual_rows = helpers.head("sys.tables", n=n).shape[0]
    assert actual_rows == expected_rows

def test_get_cols() -> None:
    """Test if ``get_cols`` returns the column names of ``sys.tables``."""
    expected_columns = [
        "name",
        "object_id",
        "principal_id",
        "schema_id",
        "parent_object_id",
        "type",
        "type_desc",
        "create_date",
        "modify_date",
        "is_ms_shipped",
        "is_published",
        "is_schema_published",
        "lob_data_space_id",
        "filestream_data_space_id",
        "max_column_id_used",
        "lock_on_bulk_load",
        "uses_ansi_nulls",
        "is_replicated",
        "has_replication_filter",
        "is_merge_published",
        "is_sync_tran_subscribed",
        "has_unchecked_assembly_data",
        "text_in_row_limit",
        "large_value_types_out_of_row",
        "is_tracked_by_cdc",
        "lock_escalation",
        "lock_escalation_desc",
        "is_filetable",
        "is_memory_optimized",
        "durability",
        "durability_desc",
        "temporal_type",
        "temporal_type_desc",
        "history_table_id",
        "is_remote_data_archive_enabled",
        "is_external",
    ]
    actual_columns = helpers.get_cols("sys.tables")
    assert actual_columns == expected_columns

@pytest.mark.parametrize(
    ["python_list", "force_string", "expected_sql_list"],
    [
        ([1, 2, 3, 4, 5], False, "(1, 2, 3, 4, 5)"),
        (["a", "e", "i", "o", "u"], False, "('a', 'e', 'i', 'o', 'u')"),
        ([1, 2, 3, 4, 5], True, "('1', '2', '3', '4', '5')"),
        ("a", False, "('a')"),
    ],
)
def test_to_sql_list(
    python_list: Union[str, int, float, List[Union[str, int, float]]],
    force_string: bool,
    expected_sql_list: str,
) -> None:
    """Test is ``to_sql_list`` propertly converts Python lists to SQL."""
    actual_sql_list = helpers.to_sql_list(python_list, force_string=force_string)
    assert actual_sql_list == expected_sql_list


import numpy as np
import math
from tqdm import tqdm
import pyarrow.parquet as pq
import pandas as pd
import pywt
from scipy import signal, stats
from tsfresh.feature_extraction.feature_calculators import *

def maddest(d, axis=None):
    return np.mean(np.absolute(d - np.mean(d, axis)), axis)

def high_pass_filter(x, low_cutoff=1000, sample_rate=50 * 800000):
    nyquist = 0.5 * sample_rate
    norm_low_cutoff = low_cutoff / nyquist
    sos = signal.butter(10, Wn=[norm_low_cutoff], btype='highpass', output='sos')
    filtered_sig = signal.sosfilt(sos, x)
    return filtered_sig

def wavelet_denoise(x, wavelet='db1', level=1):
    coeff = pywt.wavedec(x, wavelet, mode="per")
    sigma = (1 / 0.6745) * maddest(coeff[-level])
    uthresh = sigma * np.sqrt(2 * np.log(len(x)))
    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])
    return pywt.waverec(coeff, wavelet, mode='per')

def signal_entropy(y):
    for i in range(3):
        max_pos = y.argmax()
        y[max_pos - 1000:max_pos + 1000] = 0.

    return stats.entropy(np.histogram(y, 15)[0])

def detail_coeffs_entropy(x, wavelet='db1'):
    c_a, c_d = pywt.dwt(x, wavelet)

    return stats.entropy(np.histogram(c_d, 15)[0])

def bucketed_entropy(x):
    y = wavelet_denoise(x)

    return np.array([stats.entropy(np.histogram(bucket, 10)[0]) for bucket in np.split(y, 10)])

def peaks(x):
    y = wavelet_denoise(x)
    peaks, properties = signal.find_peaks(y)
    widths = signal.peak_widths(y, peaks)[0]
    prominences = signal.peak_prominences(y, peaks)[0]
    return {
        'count': peaks.size,
        'width_mean': widths.mean() if widths.size else -1.,
        'width_max': widths.max() if widths.size else -1.,
        'width_min': widths.min() if widths.size else -1.,
        'prominence_mean': prominences.mean() if prominences.size else -1.,
        'prominence_max': prominences.max() if prominences.size else -1.,
        'prominence_min': prominences.min() if prominences.size else -1.,
    }

def feature(ts, n_dim=160):
    # https://www.kaggle.com/c/vsb-power-line-fault-detection/discussion/80166
    ts = high_pass_filter(ts, low_cutoff=10000, sample_rate=50 * 800000)
    ts_std = wavelet_denoise(ts)
    bucket_size = int(800000 / n_dim)
    new_ts = []
    for i in range(0, 800000, bucket_size):
        ts_range = ts_std[i:i + bucket_size]

        mean = ts_range.mean()
        std = ts_range.std()
        std_top = mean + std
        std_bot = mean - std
        percentil_calc = np.percentile(ts_range, [0, 1, 25, 30, 50, 60, 75, 99, 100])
        max_range = percentil_calc[-1] - percentil_calc[0]
        kurt = stats.kurtosis(ts_range)
        entropy = signal_entropy(ts_range)
        de_coeffs_entropy = detail_coeffs_entropy(ts_range)
        bucketed_entropy_ = bucketed_entropy(ts_range)
        peak = peaks(ts_range)
        c = peak['count']
        width_mean = peak['width_mean']
        width_max = peak['width_max']
        width_min = peak['width_min']
        prominence_mean = peak['prominence_mean']
        prominence_max = peak['prominence_max']
        prominence_min = peak['prominence_min']
        relative_percentile = percentil_calc - mean
        complexity = cid_ce(ts_range, normalize=True)
        nonlinear = c3(ts_range, lag=50)

        new_ts.append(
            np.concatenate(
                [np.asarray([mean, std, std_top, std_bot, max_range, entropy,
                             kurt, de_coeffs_entropy, c, width_mean, width_max,
                             width_min, prominence_mean, prominence_max, prominence_min,
                             complexity, nonlinear]),
                 percentil_calc,
                 relative_percentile,
                 bucketed_entropy_,
                 ]))
    return np.asarray(new_ts)

def get_score(y_true, y_pred):
    y_true = y_true.reshape(-1)
    y_pred = y_pred.reshape(-1)
    y_pred_pos = np.round(y_pred)
    y_pred_neg = 1 - y_pred_pos
    y_pos = np.round(y_true)
    y_neg = 1 - y_pos

    tp = np.sum(y_pos * y_pred_pos)
    tn = np.sum(y_neg * y_pred_neg)
    fp = np.sum(y_neg * y_pred_pos)
    fn = np.sum(y_pos * y_pred_neg)

    return (tp * tn - fp * fn) / (np.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + 1e-15)

def threshold_search(y_true, y_proba):
    best_threshold = 0
    best_score = 0
    raw_score = get_score(y_true.astype(np.float64), (y_proba > 0.5).astype(np.float64))
    for th in np.linspace(0, 1, 100):
        score = get_score(y_true.astype(np.float64), (y_proba > th).astype(np.float64))
        if score > best_score:
            best_threshold = th
            best_score = score

    return best_threshold, best_score, raw_score

def preprocess(df):
    if 'target' not in df.columns:
        df['target'] = np.zeros(len(df))

    def flatten(data):
        id_measurement = data['id_measurement'].values[0]
        target = f"{data['target'].values[0]}{data['target'].values[1]}{data['target'].values[2]}"
        record = [id_measurement, target] + data['signal_id'].tolist()
        return pd.Series(record, index=['id_measurement', 'target', 'signal_id1', 'signal_id2', 'signal_id3'])

    df = df.sort_values(by=['id_measurement', 'phase'], ascending=True)[['signal_id', 'id_measurement', 'target']]
    df = df.groupby(by=['id_measurement']).apply(flatten)
    return df

def worker(data, dim):
    r = []
    for x in data:
        y = feature(x, n_dim=dim)
        r.append(y)
    return r

def downsample(dim=160):
    import multiprocessing as mp

    def fun(train=True, dim=160):
        f = 'train'
        if not train:
            f = 'test'
        signal_data = pq.read_pandas(f'../data/{f}.parquet').to_pandas().values.T
        pool = mp.Pool()
        num_worker = mp.cpu_count()
        num_samples = len(signal_data)
        avg = 1 + num_samples // num_worker
        res = []
        for i in range(num_worker):
            result = pool.apply_async(worker, args=(signal_data[i * avg:(i + 1) * avg], dim))
            res.append(result)
        pool.close()
        pool.join()

        results = []

        for r in res:
            results += r.get()
        results = np.asanyarray(results)
        np.save(f'../data/downsampling_{dim}_{f}_signal.npy', results)

    fun(train=True, dim=dim)
    fun(train=False, dim=dim)

if __name__ == '__main__':
    # downsample(dim=160)
    t = np.load('../data/downsampling_160_train_signal.npy')
    print(t.shape)
    df = pd.read_csv('../data/metadata_train.csv')
    df = preprocess(df)
    print(df)
    # x = np.random.randn(100)


# 84. Largest Rectangle in Histogram
# Hard
# 13.4K
# 190
# company
# Amazon
# company
# Adobe
# company
# Apple

# Given an array of integers heights representing the histogram's bar height where the width of each bar is 1, return the area of the largest rectangle in the histogram.

# Example 1:

# Input: heights = [2,1,5,6,2,3]
# Output: 10
# Explanation: The above is a histogram where width of each bar is 1.
# The largest rectangle is shown in the red area, which has an area = 10 units.

# Example 2:

# Input: heights = [2,4]
# Output: 4

# Constraints:

#     1 <= heights.length <= 105
#     0 <= heights[i] <= 104

# Accepted
# 635.3K
# Submissions
# 1.5M
# Acceptance Rate
# 42.4%

from typing import List

class Solution:
    def largestRectangleArea(self, heights: List[int]) -> int:

        maxArea = 0
        stack = []

        for i in range(len(heights)):
            start = i
            while stack and stack[-1][1] > heights[i]:
                idx, height = stack.pop()
                maxArea = max(maxArea, height * (i - idx))
                start = idx
            stack.append((start, heights[i]))

        for i, h in stack:
            maxArea = max(maxArea, h * (len(heights) - i))

        return maxArea


from .util.util import *
from telegram.ext import CommandHandler

async def set_names(update, context):
    names = context.args
    context.chat_data.__init__()
    context.chat_data.names = names
    await update.message.reply_text(get_reply_text(context), reply_markup=build_keyboard(names))

group_handler = CommandHandler('group', set_names)


from mycroft import MycroftSkill, intent_file_handler, util as mutil
from pint import UnitRegistry

class UnitConverter(MycroftSkill):
    def __init__(self):
        MycroftSkill.__init__(self)
        self.units = UnitRegistry()
        self.Q_ = self.units.Quantity

    @intent_file_handler('converter.unit.intent')
    def handle_converter_unit(self, message):
        try:
            d = message.data
            (firstu, secondu, seconda) = (d['firstunit'], d['secondunit'], d['secondamount'])
            seconda = '1' if seconda == "a" else seconda
            qfrom = self.Q_(seconda + " * " + secondu);
            qTo = qfrom.to(firstu)
            resDict = {
                'firstAmount': mutil.nice_number(qTo.magnitude),
                'firstUnit': firstu,
                'secondAmount': seconda,
                'secondUnit': secondu
            }
            self.speak_dialog('converter.unit', resDict)
        except:
            self.speak_dialog('converter.unit.failed')

def create_skill():
    return UnitConverter()


import requests
import re

url = 'https://api.bilibili.com/x/v1/dm/list.so?oid=765060141'

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 '
                  'Safari/537.36',
}
response = requests.get(url=url, headers=headers)
# 解决网页数据乱码
response.encoding = 'utf-8'
content_list = re.findall('<d p=".*?">(.*?)</d>', response.text)
print(content_list)
for content in content_list:
    with open('弹幕.txt', mode='a', encoding='utf-8') as f:
        f.write(content)
        # 把数据换行
        f.write('\n')


from django.views import View
from django.shortcuts import redirect, render
# import the csrf_exempt decorator
from django.views.decorators.csrf import csrf_exempt
from django.utils.decorators import method_decorator

decorators = [
    csrf_exempt
]

@method_decorator(decorators, name='dispatch')
class IndexView(View):
    """
    Main page of the website
    """

    template_name = 'login_signup/index.html'

    @staticmethod
    def redirect_to_signup(request):
        """
        Redirect to the signup page dependig on the value of the button.
        The button value is like : `personal&login` or `personal&signup` or `medical&login` or `medical&signup`
        """

        person, direction = request.POST.get('button').split("&")

        if person == 'personal':
            response = redirect('login') if direction == 'login' else redirect('login_signup:signup', 1)
            response.set_cookie('medical', False)
            return response

        if person == 'medical':
            response = redirect('login') if direction == 'login' else redirect('login_signup:signup', 1)
            response.set_cookie('medical', True)
            return response

        return redirect('login_signup:index')

    def get(self, request):

        # Don't render immediately but first set the cookie
        if request.user.is_authenticated:
            return redirect('home:home')
        response = render(request, self.template_name)
        response.set_cookie('medical', False)

        return response

    def post(self, request):
        resonse = self.redirect_to_signup(request)
        return resonse


import numpy as np
from copy import copy
from scipy.spatial import Voronoi
from copy import copy

# drake
from pydrake.all import (FirstOrderTaylorApproximation,
    BasicVector,
    VectorSystem,
    MathematicalProgram,
    SolutionResult)
from pydrake.solvers.gurobi import GurobiSolver

# pympc
from pympc.geometry.polyhedron import Polyhedron
from pympc.dynamics.discrete_time_systems import LinearSystem, AffineSystem, PieceWiseAffineSystem
from pympc.optimization.programs import linear_program

def _voronoi_nd(points):
    """
    Given a list of n-dimensional points, returns the Voronoi partition of the space as a list of Polyhedron (pympc class).
    Uses the scipy wrapper of Voronoi from Qhull.
    Does not work in case of 1-dimensional points.

    Arguments
    ----------
    points : list of numpy.ndarray
        Points for the Voronoi partition.

    Returns
    ----------
    partition : list of Polyhedron
        Halfspace representation of all the cells of the partiotion.
    """

    # loop over points
    partition = []
    for i, point in enumerate(points):

        # h-rep of the cell for the point i
        A = []
        b = []

        # loop over the points that share a facet with point i
        for ridge in Voronoi(points).ridge_points:
            if i in ridge:

                # vector from point i to the neighbor
                bottom = point
                tip = points[ridge[1 - ridge.tolist().index(i)]]

                # hyperplane that separates point i and its neighbor
                Ai = tip - point
                center = bottom + (tip - bottom) / 2.
                bi = Ai.dot(center)
                A.append(Ai)
                b.append(bi)

        # assemble cell i and add to partition
        cell = Polyhedron(np.vstack(A), np.array(b))
        cell.normalize()
        partition.append(cell)

    return partition

def _voronoi_1d(points):
    """
    Given a list of 1-dimensional points, returns the Voronoi partition of the space as a list of Polyhedron (pympc class).

    Arguments
    ----------
    points : list of numpy.ndarray
        Points for the Voronoi partition.

    Returns
    ----------
    partition : list of Polyhedron
        Halfspace representation of all the cells of the partiotion.
    """

    # order point from smallest to biggest
    points = sorted(points)

    # loop from the smaller point
    polyhedra = []
    for i, point in enumerate(points):

        # h-rep of the cell for the point i
        A = []
        b = []

        # get previous and next point (if any)
        tips = []
        if i > 0:
            tips.append(points[i-1])
        if i < len(points)-1:
            tips.append(points[i+1])

        # vector from point i to the next/previous point
        for tip in tips:
            bottom = point
            center = bottom + (tip - bottom) / 2.

            # hyperplane that separates point i and its neighbor
            Ai = tip - point
            bi = Ai.dot(center)
            A.append(Ai)
            b.append(bi)

        # assemble cell i and add to partition
        polyhedron = Polyhedron(np.vstack(A), np.array(b))
        polyhedron.normalize()
        polyhedra.append(polyhedron)

    return polyhedra

def constrained_voronoi(points, X=None):
    """
    Given a list of n-dimensional points, returns the Voronoi partition of the Polyhedron X as a list of Polyhedron.
    If X is None, returns the partition of the whole space.

    Arguments
    ----------
    points : list of numpy.ndarray
        Points for the Voronoi partition.
    X : Polyhedron
        Set we want to partition.

    Returns
    ----------
    partition : list of Polyhedron
        Halfspace representation of all the cells of the partiotion.
    """

    # get indices of non-coincident coordinates
    nx = min(p.size for p in points)
    assert nx == max(p.size for p in points)
    indices = [i for i in range(nx) if not np.isclose(min([p[i] for p in points]), max([p[i] for p in points]))]

    # get voronoi partition without boundaries
    points_lower_dimensional = [p[indices] for p in points]
    if len(indices) == 1:
        vor = _voronoi_1d(points_lower_dimensional)
    else:
        vor = _voronoi_nd(points_lower_dimensional)

    # go back to the higher dimensional space
    partition = [Polyhedron(np.zeros((0,nx)), np.zeros(0)) for i in points]
    for i, cell in enumerate(vor):
        partition[i].add_inequality(cell.A, cell.b, indices=indices)

        # intersect with X is provided
        if X is not None:
            partition[i].add_inequality(X.A, X.b)

    return partition

def pwa_from_RigidBodyPlant(plant, linearization_points, X, U, h, method='zero_order_hold'):
    """

    Arguments
    ----------
    plant : RigidBodyPlant
        RigidBodyPlant of the robot.
    linearization_points : list of numpy.ndarray
        Points in the state space where to linearize the dynamics.
    X : Polyhedron
        Overall bounds of the state space.
    U : Polyhedron
        Overall bounds of the control space.
    h : float
        Sampling time for the time discretization of the PWA system.
    method : string
        Method used to discretize each piece of the PWA dynamics ('explicit_euler' or 'zero_order_hold').

    Returns
    ----------
    PWA : PieceWiseAffineSystem
    """

    # partition of the state space
    X_partition = constrained_voronoi(linearization_points, X)
    domains = [Xi.cartesian_product(U) for Xi in X_partition]

    # create context
    context = plant.CreateDefaultContext()
    state = context.get_mutable_continuous_state_vector()
    input = BasicVector(np.array([0.]))
    context.FixInputPort(0, input)

    # affine systems
    affine_systems = []
    for x in linearization_points:
        state.SetFromVector(x)
        taylor_approx = FirstOrderTaylorApproximation(plant, context)
        affine_system = AffineSystem.from_continuous(
            taylor_approx.A(),
            taylor_approx.B(),
            taylor_approx.f0(),
            h,
            method
            )
        affine_systems.append(affine_system)

    return PieceWiseAffineSystem(affine_systems, domains)

class Controller(VectorSystem):
    """
    Wrapper for the HybridModelPredictiveController class from pympc.
    """

    def __init__(self, S, N, Q, R, P, X_N):
        """
        Arguments
        ----------
        S : PieceWiseAffineSystem
            PWA system to be controlled.
        N : int
            Horizon of the optimal control problem.
        Q : numpy.ndarray
            Quadratic cost for the state.
        R : numpy.ndarray
            Quadratic cost for the input.
        P : numpy.ndarray
            Quadratic cost for the terminal state.
        X_N : Polyhedron
            Terminal set.
        """

        # bouild drake controller
        VectorSystem.__init__(self, S.nx, S.nu)
        self.controller = HybridModelPredictiveController(S, N, Q, R, Q, X_N)

    def _DoCalcVectorOutput(self, context, plant_state, unused, plant_input):
        print('Controller called at time ' + str(context.get_time()) + ' with state ' + str(plant_state) + '          \r'),
        plant_input[:] = self.controller.feedback(plant_state)

class HybridModelPredictiveController(object):

    def __init__(self, S, N, Q, R, P, X_N):

        # store inputs
        self.S = S
        self.N = N
        self.Q = Q
        self.R = R
        self.P = P
        self.X_N = X_N

        # mpMIQP
        self.build_mpmiqp()

    def build_mpmiqp(self):

        # express the constrained dynamics as a list of polytopes in the (x,u,x+)-space
        P = graph_representation(self.S)
        m = big_m(P)

        # initialize program
        self.prog = MathematicalProgram()
        self.x = []
        self.u = []
        self.d = []
        obj = 0.
        self.binaries_lower_bound = []

        # initial conditions (set arbitrarily to zero in the building phase)
        self.x.append(self.prog.NewContinuousVariables(self.S.nx))
        self.initial_condition = []
        for k in range(self.S.nx):
            self.initial_condition.append(self.prog.AddLinearConstraint(self.x[0][k] == 0.).evaluator())

        # loop over time
        for t in range(self.N):

            # create input, mode and next state variables
            self.u.append(self.prog.NewContinuousVariables(self.S.nu))
            self.d.append(self.prog.NewBinaryVariables(self.S.nm))
            self.x.append(self.prog.NewContinuousVariables(self.S.nx))

            # enforce constrained dynamics (big-m methods)
            xux = np.concatenate((self.x[t], self.u[t], self.x[t+1]))
            for i in range(self.S.nm):
                mi_sum = np.sum([m[i][j] * self.d[t][j] for j in range(self.S.nm) if j != i], axis=0)
                for k in range(P[i].A.shape[0]):
                    self.prog.AddLinearConstraint(P[i].A[k].dot(xux) <= P[i].b[k] + mi_sum[k])

            # SOS1 on the binaries
            self.prog.AddLinearConstraint(sum(self.d[t]) == 1.)

            # stage cost to the objective
            obj += .5 * self.u[t].dot(self.R).dot(self.u[t])
            obj += .5 * self.x[t].dot(self.Q).dot(self.x[t])

        # terminal constraint
        for k in range(self.X_N.A.shape[0]):
            self.prog.AddLinearConstraint(self.X_N.A[k].dot(self.x[self.N]) <= self.X_N.b[k])

        # terminal cost
        obj += .5 * self.x[self.N].dot(self.P).dot(self.x[self.N])
        self.objective = self.prog.AddQuadraticCost(obj)

        # set solver
        self.solver = GurobiSolver()
        self.prog.SetSolverOption(self.solver.solver_type(), 'OutputFlag', 1)

    def set_initial_condition(self, x0):
        for k, c in enumerate(self.initial_condition):
            c.UpdateLowerBound(x0[k:k+1])
            c.UpdateUpperBound(x0[k:k+1])

    def feedforward(self, x0):

        # overwrite initial condition
        self.set_initial_condition(x0)

        # solve MIQP
        result = self.solver.Solve(self.prog)

        # check feasibility
        if result != SolutionResult.kSolutionFound:
            return None, None, None, None

        # get cost
        obj = self.prog.EvalBindingAtSolution(self.objective)[0]

        # store argmin in list of vectors
        u = [self.prog.GetSolution(ut) for ut in self.u]
        x = [self.prog.GetSolution(xt) for xt in self.x]
        d = [self.prog.GetSolution(dt) for dt in self.d]

        # retrieve mode sequence and check integer feasibility
        ms = [np.argmax(dt) for dt in d]

        return u, x, ms, obj

    def feedback(self, x0):

        # get feedforward and extract first input
        u_feedforward = self.feedforward(x0)[0]
        if u_feedforward is None:
            return None

        return u_feedforward[0]

def graph_representation(S):
    '''
    For the PWA system S
    x+ = Ai x + Bi u + ci if Fi x + Gi u <= hi,
    returns the graphs of the dynamics (list of Polyhedron)
    [ Fi  Gi  0] [ x]    [ hi]
    [ Ai  Bi -I] [ u] <= [-ci]
    [-Ai -Bi  I] [x+]    [ ci]
    '''
    P = []
    for i in range(S.nm):
        Di = S.domains[i]
        Si = S.affine_systems[i]
        Ai = np.vstack((
            np.hstack((Di.A, np.zeros((Di.A.shape[0], S.nx)))),
            np.hstack((Si.A, Si.B, -np.eye(S.nx))),
            np.hstack((-Si.A, -Si.B, np.eye(S.nx))),
            ))
        bi = np.concatenate((Di.b, -Si.c, Si.c))
        P.append(Polyhedron(Ai, bi))
    return P

def big_m(P_list, tol=1.e-6):
    '''
    For the list of Polyhedron P_list in the from Pi = {x | Ai x <= bi} returns a list of lists of numpy arrays, where m[i][j] := max_{x in Pj} Ai x - bi.

    '''
    m = []
    for i, Pi in enumerate(P_list):
        mi = []
        for j, Pj in enumerate(P_list):
            mij = []
            for k in range(Pi.A.shape[0]):
                sol = linear_program(-Pi.A[k], Pj.A, Pj.b)
                mijk = - sol['min'] - Pi.b[k]
                if np.abs(mijk) < tol:
                    mijk = 0.
                mij.append(mijk)
            mi.append(np.array(mij))
        m.append(mi)
    return m


from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import json
import re

import six
from six.moves import range  # pylint: disable=redefined-builtin

_SNAKE_RE = re.compile(
    '((?<=[a-z0-9])[A-Z]+(?=[A-Z][a-z]|$)|(?!^)[A-Z](?=[a-z]))')

def _GetMetaDict(items, key, value):
  """Gets the dict in items that contains key==value.

  A metadict object is a list of dicts of the form:
    [
      {key: value-1, ...},
      {key: value-2, ...},
      ...
    ]

  Args:
    items: A list of dicts.
    key: The dict key name.
    value: The dict key value.

  Returns:
    The dict in items that contains key==value or None if no match or not a
    metadict.
  """
  try:
    for item in items:
      if item.get(key) == value:
        return item
  except (AttributeError, IndexError, TypeError, ValueError):
    pass
  return None

def _GetMetaDataValue(items, name, deserialize=False):
  """Gets the metadata value for the item in items with key == name.

  A metadata object is a list of dicts of the form:
    [
      {'key': key-name-1, 'value': field-1-value-string},
      {'key': key-name-2, 'value': field-2-value-string},
      ...
    ]

  Examples:
    x.metadata[windows-keys].email
      Deserializes the 'windows-keys' metadata value and gets the email value.
    x.metadata[windows-keys]
      Gets the 'windows-key' metadata string value.
    x.metadata[windows-keys][]
      Gets the deserialized 'windows-key' metadata value.

  Args:
    items: The metadata items list.
    name: The metadata name (which must match one of the 'key' values).
    deserialize: If True then attempt to deserialize a compact JSON string.

  Returns:
    The metadata value for name or None if not found or if items is not a
    metadata dict list.
  """
  item = _GetMetaDict(items, 'key', name)
  if item is None:
    return None
  value = item.get('value', None)
  if deserialize:
    try:
      return json.loads(value)
    except (TypeError, ValueError):
      pass
  return value

def ConvertToCamelCase(name):
  """Converts snake_case name to camelCase."""
  part = name.split('_')
  return part[0] + ''.join(x.title() for x in part[1:])

def ConvertToSnakeCase(name):
  """Converts camelCase name to snake_case."""
  return _SNAKE_RE.sub(r'_\1', name).lower()

def ConvertToAngrySnakeCase(name):
  """Converts camelCase name to ANGRY_SNAKE_CASE."""
  return _SNAKE_RE.sub(r'_\1', name).upper()

def GetMatchingIndex(index, func):
  """Returns index converted to a case that satisfies func."""
  if func(index):
    return index
  if not isinstance(index, six.string_types):
    return None
  for convert in [ConvertToCamelCase, ConvertToSnakeCase]:
    name = convert(index)
    if func(name):
      return name
  return None

def GetMatchingIndexValue(index, func):
  """Returns the first non-None func value for case-converted index."""
  value = func(index)
  if value:
    return value
  if not isinstance(index, six.string_types):
    return None
  for convert in [ConvertToCamelCase, ConvertToSnakeCase]:
    value = func(convert(index))
    if value:
      return value
  return None

def GetMessageFieldType(resource_key, message):
  """Returns the messages module type for key in message and the actual key.

  Handles camelCase/snake_case key name variants for OnePlatform compatibility.
  Indices and slices in resource_key are ignored -- they are not needed for
  repeated field queries.

  Args:
    resource_key: Ordered list of key names/indices, applied left to right. Each
      element in the list may be one of:
        str - A resource property name. This could be a class attribute name or
          a dict index.
        int - A list index. Selects one member is the list. Negative indices
          count from the end of the list, starting with -1 for the last element
          in the list. An out of bounds index is not an error; it produces the
          value None.
        None - A list slice. Selects all members of a list or dict like object.
          A slice of an empty dict or list is an empty dict or list.
    message: The known proto message type if not None.

  Raises:
    KeyError: If key is not in message.

  Returns:
    (type, actual_key), the messages module type for key in message and the
      actual key (names in the proper case, indices omitted).
  """
  actual_key = []
  for name in resource_key:
    if not isinstance(name, six.string_types):
      # Ignore indices and slices.
      continue
    for convert in (lambda x: x, ConvertToCamelCase, ConvertToSnakeCase):
      actual_name = convert(name)
      try:
        message = message.field_by_name(actual_name).type
      except (AttributeError, KeyError):
        pass
      else:
        break
    else:
      raise KeyError('Field {} not in message.'.format(name))
    actual_key.append(actual_name)
  if message == six.integer_types:
    # A distinction we don't need, especially since python 3 only has "int".
    # Also, dealing with a type tuple is a pain for callers.
    message = int
  return message, actual_key

def LookupField(resource_key, fields):
  """Returns the actual_key match of resource_key in fields.

  Handles camelCase/snake_case key name variants for OnePlatform compatibility.
  Indices and slices in resource_key are ignored to normalize the lookup. This
  means that the lookup can determine the existence of an attribute name, but
  not a specific value among all repeated values.

  Args:
    resource_key: Ordered list of key names/indices, applied left to right. Each
      element in the list may be one of:
        str - A resource property name. This could be a class attribute name or
          a dict index.
        int - A list index. Selects one member is the list. Negative indices
          count from the end of the list, starting with -1 for the last element
          in the list. An out of bounds index is not an error; it produces the
          value None.
        None - A list slice. Selects all members of a list or dict like object.
          A slice of an empty dict or list is an empty dict or list.
    fields: The set of dotted field names to match against.

  Returns:
    The actual_key match of resource_key in fields or None if no match.
  """
  for convert in (lambda x: x, ConvertToCamelCase, ConvertToSnakeCase):
    actual_key = [convert(name) if isinstance(name, six.string_types) else name
                  for name in resource_key]
    lookup_key = '.'.join([name for name in actual_key
                           if isinstance(name, six.string_types)])
    if lookup_key in fields:
      return actual_key
  return None

def Get(resource_obj, resource_key, default=None):
  """Gets the value referenced by key in the object resource.

  Since it is common for resource instances to be sparse it is not an error if
  a key is not present in a particular resource instance, or if an index does
  not match the resource type.

  Args:
    resource_obj: The resource object possibly containing a value for key.
    resource_key: Ordered list of key names/indices, applied left to right. Each
      element in the list may be one of:
        str - A resource property name. This could be a class attribute name or
          a dict index.
        int - A list index. Selects one member is the list. Negative indices
          count from the end of the list, starting with -1 for the last element
          in the list. An out of bounds index is not an error; it produces the
          value None.
        None - A list slice. Selects all members of a list or dict like object.
          A slice of an empty dict or list is an empty dict or list.
    default: Get() returns this value if key is not in resource.

  Returns:
    The value, None if any of the given keys are not found. This is
      intentionally not an error. In this context a value can be any data
      object: dict, list, tuple, class, str, int, float, ...
  """
  key = list(resource_key)
  resource = resource_obj
  while key:

    # Get the next key index. Some iterations may access the next key.
    index = key.pop(0)

    # Serialized sets are sorted lists.
    if isinstance(resource, set):
      resource = sorted(resource)

    # This if ordering checks builtin object attributes last. For
    # example, with resource = {'items': ...}, Get() treats 'items' as a dict
    # key rather than the builtin 'items' attribute of resource.

    if resource is None:
      # None is different than an empty dict or list.
      return default

    if hasattr(resource, 'items'):
      # dict-like
      if index is None:
        if key:
          # Inner slice: *.[].*
          return [Get(resource, [k] + key, default) for k in resource]
        # Trailing slice: *.[]
        return resource

      name = GetMatchingIndex(index, lambda x: x in resource)
      if name:
        resource = resource[name]
        continue

      if 'items' in resource:
        # It would be nice if there were a better metadata indicator.
        # _GetMetaDataValue() returns None if resource['items'] isn't really
        # metadata, so there is a bit more verification than just 'items' in
        # resource.

        def _GetValue(index):
          # pylint: disable=cell-var-from-loop
          return _GetMetaDataValue(
              resource['items'], index, deserialize=bool(key))

        resource = GetMatchingIndexValue(index, _GetValue)
        continue

      return default

    if isinstance(index, six.string_types):
      # class-like?
      name = GetMatchingIndex(index, lambda x: hasattr(resource, x))
      if name:
        r = getattr(resource, name, default)
        if not callable(r):
          resource = r
          continue

    if hasattr(resource, '__iter__') or isinstance(resource, six.string_types):
      # list-like
      if index is None:
        if key:
          # explicit inner slice: *.[].*
          return [Get(resource, [k] + key, default)
                  for k in range(len(resource))]
        # explicit trailing slice: *.[]
        return resource

      if not isinstance(index, six.integer_types):
        if isinstance(index, six.string_types) and isinstance(resource, list):
          if len(resource) and isinstance(resource[0], dict):
            if key:
              # See the _GetMetaDict docstring for the proto meta dict layout.
              r = _GetMetaDict(resource, index, key[0])
              if r is not None:
                # meta-dict-like
                resource = r
                index = key.pop(0)
                continue
            else:
              # Handle leaf node [{'key': k, 'value': v}, ...] metadata to
              # support r.k:v filter terms.
              r = _GetMetaDataValue(resource, index)
              if r is not None:
                return r
            if index in resource[0]:
              # implicit inner slice
              # resource is a list. An explicit reference would be
              # "resource[].foo" which would be caught above. Implicit
              # "resource.foo" references are handled here.
              return [Get(resource, [k, index] + key, default)
                      for k in range(len(resource))]

            # This is the last chance for index. If we fell through the index
            # would be ignored and the resource would be returned (incorrect).
            # Instead we return the list of non-None index values from the list
            # of dicts. See
            # resource_property_test.PropertyGetTest.testGetLastDictSlice for
            # an example.
            return ([f for f in [d.get(index) for d in resource] if f]
                    or default)

        # Index mismatch.
        return default

      if index in range(-len(resource), len(resource)):
        resource = resource[index]
        continue

    # Resource or index mismatch.
    return default

  # Sets serialize to sorted lists.
  if isinstance(resource, set):
    resource = sorted(resource)

  return resource

def ResourceContainsKey(resource, key):
  """True if resource contains key, else False."""
  return Get(resource, key, None) is not None

def EvaluateGlobalRestriction(resource, restriction, pattern):
  """Returns True if any attribute value in resource matches the RE pattern.

  This function is called to evaluate a global restriction on a resource. For
  example, --filter="Foo.Bar" results in a call like this on each resource item:

    resource_property.EvaluateGlobalRestriction(
      resource,
      'Foo.Bar',
      re.compile(re.escape('Foo.Bar'), re.IGNORECASE),
    )

  Args:
    resource: The object to check.
    restriction: The global restriction string.
    pattern: The global restriction pattern for matcing resource values.

  Returns:
    True if any attribute value in resource matches the RE pattern.
  """
  if not resource:
    return False
  if isinstance(resource, six.string_types):
    try:
      return bool(pattern.search(resource))
    except TypeError:
      pass
  if isinstance(resource, (float, int)):
    try:
      return bool(pattern.search(str(resource)))
    except TypeError:
      pass
  try:
    for key, value in six.iteritems(resource):
      if not key.startswith('_') and EvaluateGlobalRestriction(
          value, restriction, pattern):
        return True
  except AttributeError:
    try:
      for value in resource:
        if EvaluateGlobalRestriction(value, restriction, pattern):
          return True
      return False
    except TypeError:
      pass
  try:
    for key, value in six.iteritems(resource.__dict__):
      if not key.startswith('_') and EvaluateGlobalRestriction(
          value, restriction, pattern):
        return True
  except AttributeError:
    pass
  return False

def IsListLike(resource):
  """Checks if resource is a list-like iterable object.

  Args:
    resource: The object to check.

  Returns:
    True if resource is a list-like iterable object.
  """
  return (isinstance(resource, list) or
          (hasattr(resource, '__iter__') and
           (hasattr(resource, 'next') or hasattr(resource, '__next__'))))


import sys
import pandas as pd
import argparse
import gzip

parser = argparse.ArgumentParser()
parser.add_argument('--vcf', help='vcf with original statistical phasing', required=True)
parser.add_argument('--rfmix_phased', help='filepath for RFMix re-phased alleles', required=True)
parser.add_argument('--out', help='output filepath for differentially phased alleles', required=True)
args = parser.parse_args()

def check_phasing(row):
    if row.OG_phasing == row.RFMix_phasing:
        return(False)
    return(True)

geno_list = []
with gzip.open(args.vcf, 'rt') as geno_file:
    for line in geno_file:
        if line[0] == "#":
            continue
        else:
            line = line.split()
            pos = int(line[1])
            genotypes = line[9:]
            genotypes = "".join(genotypes).translate({ord('|'): ''})
            geno_list.append([pos, genotypes])
OG_phasing = pd.DataFrame(geno_list, columns=["Position", "OG_phasing"])

rfmix_phasing = pd.read_csv(args.rfmix_phased, names=["RFMix_phasing"])
phasing_df = pd.concat([OG_phasing, rfmix_phasing], axis=1)
phasing_df["Match"] = phasing_df.apply(check_phasing, axis=1)
phasing_df[phasing_df.Match == False].to_csv(args.out, sep='\t', index=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__    = "nagracks"
__date__      = "02-07-2016"
__license__   = "MIT"
__copyright__ = "Copyright © 2016 nagracks"

import os
import re
from argparse import ArgumentParser

class RenameIt(object):

    """Class RenameIt

    Constructor args:
     :filename: Name of file
     :dryrun: Just dry run, not perform any action
     :silent:

    Methods:
     * prefix_it
     * postfix_it
     * lower_it
     * replace_space
     * camel_case
     * rename
    """

    def __init__(self, filename, dryrun, silent):
        self.full_name = filename
        # Get filename and file extension #
        self.fname, self.fext = os.path.splitext(filename)
        # Suppress output?
        self.silent = silent
        # Are we actually doing anything or just preforming a dryrun?
        self.do_dryrun = dryrun

        if self.do_dryrun:
            print("PERFORMING A DRY RUN (NO ACTIONS WILL BE TAKEN)")

    def _print(self, *msg):
        """Print msg if not silent

        :msg: *str, What to print
        :return: None
        """
        if not self.silent:
            print(msg)

    def _rename(self, new_name):
        """Generic rename method with error handling

        :new_name: str, Filename to rename to
        :return: None
        """
        try:
            if not self.do_dryrun:
                os.rename(self.full_name, new_name + self.fext)
            self._print("renaming: {old} --> {new}".format(old=self.full_name,
                                                new=new_name + self.fext))
            # Set after every rename, makes it possible to run multiple
            # arguments
            self.fname = new_name
            self.full_name = self.fname + self.fext
        except OSError as e:
            self._print(
                "Failed to rename {old} --> {new}: {err}".
                format(old=self.full_name, new=new_name, err=e))

    def prefix_it(self, prefix_str):
        """Prefix filename with prefix string

        :prefix_str: str, string to use as prefix in filename
        :returns: None
        """
        old_name = self.fname
        new_name = prefix_str + old_name
        self._rename(new_name)

    def postfix_it(self, postfix_str):
        """Postfix filename with postfix string

        :postfix_str: str, string to use as postfix in filename
        :returns: None
        """
        old_name = self.fname
        new_name = old_name + postfix_str
        self._rename(new_name)

    def lower_it(self):
        """Lowercase the filename
        :returns: None
        """
        old_name = self.fname
        new_name = old_name.lower()
        self._rename(new_name)

    def replace_space(self, fill_char='_'):
        """Replace spaces with fill_char

        :fill_char: default to '_'
        :returns: None
        """
        old_name = self.fname
        new_name = old_name.replace(' ', fill_char)
        self._rename(new_name)

    def camel_case(self):
        """Convert to camel case
        :returns: None
        """
        old_name = self.fname.replace('_', ' ')
        modified_name = re.findall('[\w]+', old_name.lower())
        new_name = ''.join([word.title() for word in modified_name])
        self._rename(new_name)

    def rename(self, rename_string):
        """ renames file to rename_string

        :rename_string: str,  new filename
        :returns: None
        """
        old_name = self.fname
        new_name = rename_string
        self._rename(new_name)

if __name__ == "__main__":
    parser = ArgumentParser(description="Python Rename")
    parser.add_argument(
            '-v',
            '--version',
            action='version',
            version='%(prog)s version 0.1'
            )
    parser.add_argument(
            '-A',
            '--prefix',
            dest='prefix',
            metavar='string',
            action='store',
            help="prefix filename with prefix string"
            )
    parser.add_argument(
            '-B',
            '--postfix',
            dest='postfix',
            metavar='string',
            action='store',
            help="postfix filename with postfix string"
            )
    parser.add_argument(
            '-r',
            '--rename',
            dest='rename',
            metavar='string',
            action='store',
            help="replace filename with string"
            )
    parser.add_argument(
            '-n',
            '--dryrun',
            dest='dryrun',
            action='store_true',
            help="perform a dry run (will not run any actions)"
            )
    parser.add_argument(
            '--lower',
            dest='lower',
            action='store_true',
            help="lowercase the filename"
            )
    parser.add_argument(
            '--remove-space',
            action='store_true',
            help="remove space with underscore"
            )
    parser.add_argument(
            '--camel-case',
            action='store_true',
            help="convert to camel case"
            )
    parser.add_argument(
            '-s',
            '--silent',
            dest='silent',
            action='store_true',
            help="silence output"
            )
    parser.add_argument(
            'filename',
            help="filename"
            )

    args = parser.parse_args()

    rename_it = RenameIt(args.filename, args.dryrun, args.silent)

    if args.rename:
        rename_it.rename(args.rename)
    if args.prefix:
        rename_it.prefix_it(args.prefix)
    if args.postfix:
        rename_it.postfix_it(args.postfix)
    if args.lower:
        rename_it.lower_it()
    if args.remove_space:
        rename_it.replace_space()
    if args.camel_case:
        rename_it.camel_case()


import pickle
from transformers import AutoTokenizer
from tqdm import tqdm
tokenizer = AutoTokenizer.from_pretrained("naver/splade-cocondenser-ensembledistil")
import json

with open("/data/lenam/topics/raw/2022_evaluation_topics_tree_v1.0.json","r") as tc22:
	with open("/data/lenam/topics/raw/2022_evaluation_topics_turn_ids.json","r") as ftids:
		_obj = json.load(tc22)
		obj = dict()
		for _topic in _obj:
			obj[_topic['number']] = dict()
			for _turn in _topic['turn']:
				obj[_topic['number']][_turn['number']] = {k:v for k,v in _turn.items() if k != 'number'}
		tids = json.load(ftids)
		rw = dict()

		for topic in tids:
			rw[int(topic)] = dict()
			for turn in tids[topic]:
				rw[int(topic)][turn] = obj[int(topic)][turn]['manual_rewritten_utterance']

def process(rw):
	tokens = dict()
	for topic in tqdm(rw):
		tokens[topic] = dict()
		for turn in rw[topic]:
			tokens[topic][turn] = tokenizer(rw[topic][turn]).input_ids
	return tokens

with open("/data/lenam/cvst/tc/rw22_tokens.pkl","wb") as f:
	pickle.dump(process(rw),f,protocol=-1)


from socket import socket,AF_INET,SOCK_STREAM
import threading,time
from datetime import datetime
from tkinter import *
import time
# Sunucumuzu açalım :D
#Yaşasın açık kaynaklı kod :))
###############################
#     Mesajlaşma Serveri      #
#      Tt : @canustun__       #
#      İg : @canustun.py      #
#      Tg : @canustunn        #
###############################

host = "localhost"
port = 80
veri = socket(AF_INET,SOCK_STREAM)

veri.bind((host,port))
veri.listen(25)

gonderme_onayi = False

banli_ipler = []

kullanicilar,normal_ipler = [],[]
kullanici_adlari = []
yetkililer = []

"""
def aktif_kullanici_adlar():
    global kullanicilar,kullanici_adlari
    while True:
        adlar = ""
        for i in kullanici_adlari:
            adlar += i+"\n"
        for a in kullanicilar:
            try:
                a.send(bytes(f"Qaktif-kul_ism?{adlar}","utf8"))
            except:pass
        time.sleep(1.7)

def aktif_kullanici():
    global kullanicilar
    while True:
        for i in kullanicilar:
            try:
                i.send(bytes(f"ASAkullanici_adet: {len(kullanicilar)}","utf8"))
            except:
                pass
        time.sleep(1.7)
"""
def veri_al(kullanici_bilgisi,kullanici_ad):
    global kullanicilar,kullanici_adlari#,yetkililer

    while True:
        zaman = datetime.now()
        saat = str(zaman.hour)
        dk = str(zaman.minute)
        saat_dk = saat+":"+dk+" "

        try:
            mesaj = kullanici_bilgisi.recv(5242880)

            try:
                mesaj = mesaj.decode("utf-8")
                if mesaj=="" or mesaj == " " or mesaj == len(mesaj)*" ":
                    continue

            except:
                for i in kullanicilar:
                    if i!=kullanici_bilgisi:
                        i.send(mesaj)
                    else:
                        i.send(bytes("Fotoğraf gönderildi","utf8"))
                continue

            if len(mesaj)>200:
                kullanici_bilgisi.send(bytes("200 Karakter Sınırı Aşıldı !","utf8"))
                continue

            elif mesaj != len(mesaj)*" " and mesaj != "":
                for i in kullanicilar:
                    if i!=kullanici_bilgisi:
                        i.send(bytes(saat_dk+kullanici_ad+" : "+mesaj,"utf8"))
                    else:
                        i.send(bytes(saat_dk+"Siz : "+mesaj,"utf8"))

            elif mesaj == "/yetki_ver" or mesaj == "yetki_ver" or mesaj == "yetki ver":
                for i in kullanicilar:
                    i.send(bytes("I CAN(server botu) : Yetki sistemi bir süreliğine kapatılmıştır.","utf8"))

                """yetki_ver['text'] = f"{kullanici_ad} adlı kullanıcı yetki istiyor."
                threading.Thread(target = yetkilendirme, args=(kullanici_ad,)).start()

            elif len(mesaj) > 4 and mesaj[:4] == "/ban":
                for i in yetkililer:
                    if kullanici_ad == i and not mesaj[5:] in yetkililer:
                        ban_atma(mesaj[5:],kullanici_ad)"""
        except:
            try:
                kullanicilar.remove(kullanici_bilgisi)
                kullanici_adlari.remove(kullanici_ad)
            except:pass

            break

"""def yetkilendirme(kullanici_adi):
    global yetkililer,gonderme_onayi

    if gonderme_onayi:
            yetki = yetki_onayla.get()
            if yetki == "E" or yetki == "e":
                for i in kullanicilar:
                    i.send(bytes(kullanici_adi+" 'I CAN' Tarafından Yetkilendirildi.","utf8"))
                yetkililer.append(kullanici_adi)

            else:
                for i in kullanicilar:
                    i.send(bytes(kullanici_adi+" Kullanıcının Yetki İsteği Reddedildi!","utf8"))

            gonderme_onayi = False
            yetki_ver['text'] = "Şuanlık yetki isteği yok !"
            yetki_onayla.delete(0,'end')"""

def isim_alma(kullanici_bilgisi2,ip):
    global kullanicilar,kullanici_adlari
    try:
        while True:
            ad = kullanici_bilgisi2.recv(1024).decode("utf8")
            if len(ad)>1 and ad != "I C4N" and ad != "I CAN" and ad != "l C4N" and len(ad)<17 and ad!="" and ad!=" " and ad != len(ad)*" " and not ad in kullanici_adlari:
                for i in kullanicilar:
                    i.send(bytes(f"{ad} Adlı Kullanıcı Sohbete Katıldı !","utf8"))

                kullanici_adlari.append(ad)
                kullanicilar.append(kullanici_bilgisi2)
                kullanici_bilgisi2.send(bytes(f"Sohbete Katıldın !","utf8"))

                print(f"{ad} : {ip}  Toplam:{len(kullanicilar)} ")
                threading.Thread(target=veri_al, args=(kullanici_bilgisi2,ad)).start()
                break
            else:
                kullanici_bilgisi2.send(bytes("İsmini tekrar gir !","utf8"))

    except:pass

def ban_atma(banlanacak,banlayan):
    global kullanicilar,kullanici_adlari,banli_ipler,normal_ipler
    try:
        indexi = kullanici_adlari.index(banlanacak)
        kullanici_adlari.remove(banlanacak)
        banli_ipler.append(normal_ipler[indexi])
        normal_ipler.remove(normal_ipler[indexi])
        kullanicilar[indexi].send(bytes(f"{banlayan} Tarafından Banlandın!","utf8"))
        kullanicilar[indexi].close()
        kullanicilar.remove(kullanicilar[indexi])
        for i in kullanicilar:
            i.send(bytes(f"{banlanacak} Adlı Kullanıcı {banlayan} Tarafından Banlandı!","utf8"))
    except:pass

def ban_atma_ceo():
    banlanacak = banla.get()
    banla.delete(0,'end')

    ban_atma(banlanacak,"I CAN")

"""def gonder_cevap():
    global gonderme_onayi
    gonderme_onayi = True"""

def onayla():
    global banli_ipler,normal_ipler

    while True:
        kullanici,adres = veri.accept()
        try:
            ip = kullanici.recv(1024).decode("utf8")
        except:
            continue
        if ip in banli_ipler:
            kullanici.send(bytes("Banlı kullanıcılar giriş yapamaz!","utf8"))
            continue
        normal_ipler.append(ip)

        threading.Thread(target=isim_alma,args=(kullanici,ip)).start()

pencere = Tk()
pencere.geometry("150x150")

banla = Entry()
banla.place(x=10,y=50)

banla_butonu = Button(text="Banla",bg="red")
banla_butonu.config(command = ban_atma_ceo)
banla_butonu.place(x=10,y=70)

"""
yetki_ver = Label(text="Şuanlık yetki isteği yok !")
yetki_ver.place(x=10,y=100)

yetki_onayla = Entry()
yetki_onayla.place(x=10,y=120)

yetki_onayla_butonu = Button(text="Cevabı Gönder",bg="green")
yetki_onayla_butonu.configure(command = gonder_cevap)
yetki_onayla_butonu.place(x=10,y=150)
"""
threading.Thread(target=onayla).start()
pencere.mainloop()
# orda minik bir hata vardı D
# serveri açalım :D


import matplotlib.pyplot as plt
# plt.rcParams.update({'font.size': 8})
from matplotlib.animation import FuncAnimation
import numpy as np
import pandas as pd
import seaborn as sns
import time
import os

reg_est_df = pd.read_csv('~/Dropbox/loki_0.5/analysis/aggregated_data/trial_hddm_estimates.csv')
pc_ls_df = pd.read_csv('~/Dropbox/loki_0.5/analysis/aggregated_data/pc_ls_df.csv')

pc_ls_df_sorted = pc_ls_df.sort_values(by=['subj_id', 'condition'])
reg_est_df_sorted = reg_est_df.sort_values(by=['subj_id', 'condition'])

assert reg_est_df_sorted.shape[0] == pc_ls_df_sorted.shape[0]
assert (reg_est_df_sorted.subj_id == pc_ls_df_sorted.subj_id).sum() == pc_ls_df_sorted.shape[0]
assert (reg_est_df_sorted.condition == pc_ls_df_sorted.condition).sum() == pc_ls_df_sorted.shape[0]

reg_est_df_sorted[['projection_0','projection_1', 'projection_2']] = pc_ls_df_sorted[['projection_0','projection_1', 'projection_2']]

reg_est_df_sorted.to_csv('~/Dropbox/loki_0.5/analysis/aggregated_data/pc_ls_reg_est_df.csv', index=False)

# grp = reg_est_df.groupby(['subj_id', 'epoch_number', 'reward_code'])
#
# grp.epoch_length.unique()

reg_window_df = reg_est_df.loc[(reg_est_df.shifted_epoch_trial <= 8) & (reg_est_df.shifted_epoch_trial >= -1)].reset_index(drop=True).copy()

sample_p = reg_window_df.p_optimal.sample().values[0]
sample_sub = reg_window_df.subj_id.sample().values[0]
sample_lambda_val = reg_window_df.lambda_val.sample().values[0]

# sample data for testing
sample_data = reg_window_df.loc[(reg_window_df.subj_id == sample_sub) &
 (reg_window_df.lambda_val == sample_lambda_val) &
 (reg_window_df.p_optimal == sample_p)].reset_index(drop=True).copy()

# create a new variable to identify the epoch window number.
# this includes the trial previous to the start of the current epoch.
# useful for plotting and analysis.

def create_epoch_window_number(session_data):

    print('processing sub {} session {}'.format(session_data.subj_id.unique(), session_data.condition.unique()))
    session_data['epoch_window_number'] = session_data.epoch_number

    for epoch_number in session_data.epoch_number.unique()[1:]:  # only get the second epoch onward (otherwise no prior trial to include)
        epoch_start_idx = session_data.loc[session_data.epoch_number == epoch_number].index[0]
        epoch_window_start_idx = epoch_start_idx - 1

        print('epoch start idx ', epoch_start_idx)
        print('epoch window start idx ', epoch_window_start_idx)

        session_data.set_value(epoch_window_start_idx, 'epoch_window_number',
        session_data.iloc[epoch_start_idx].epoch_number)

    return session_data

reg_window_dfs = []

# now implement on all data
for subj_id in reg_window_df.subj_id.unique():
    for condition in reg_window_df.condition.unique():
        session_data = reg_window_df.loc[(reg_window_df.subj_id == subj_id) &
        (reg_window_df.condition == condition)].reset_index(drop=True).copy()

        session_data_v2 = create_epoch_window_number(session_data)

        # print(session_data_v2[['epoch_window_number', 'epoch_number', 'shifted_epoch_trial', 'epoch_trial']].head(60))

        reg_window_dfs.append(session_data_v2)

reg_window_df_v2 = pd.concat(reg_window_dfs, axis=0).reset_index(drop=True)
kws = dict(marker='o', linestyle='-', linewidth=2, markersize=5, color='black') #add the line

def plot_av_manifold(x,y, shifted_epoch_trial, **kwargs):

    ax = sns.lineplot(x,y, markers=True, dashes=False, estimator=None)

    x_fmt = [np.round(val,3) for val in x]
    ax.set_xticklabels(x_fmt)
    for i in range(len(x)):
        ax.annotate('t' + str(shifted_epoch_trial.values[i]), xy=(x_fmt[i], y.values[i]),fontsize=8)

# for subj_id in reg_window_df_v2.subj_id.unique():
#     for condition in reg_window_df_v2.condition.unique():
#         session_data = reg_window_df_v2.loc[(reg_window_df_v2.subj_id == subj_id) & (reg_window_df_v2.condition == condition)].copy()
#         g = sns.FacetGrid(col='epoch_window_number', col_wrap=4, data=session_data,
#         sharey=False, sharex=False, hue='shifted_epoch_trial')
#
#         g.map(plot_av_manifold, 'a_est', 'v_est', 'shifted_epoch_trial')
#
#         g.fig.subplots_adjust(top=0.9)
#         g.fig.suptitle('subject {} condition {}'.format(session_data.subj_id.unique()[0], session_data.condition.unique()[0]))

def cart2polar(x,y): # note that this should be implemented for each epoch after nesting within condition and sub

    x_diff, y_diff = x.diff(), y.diff()
    r = np.hypot(x_diff, y_diff)
    theta_radians = np.arctan2(y_diff, x_diff)
    theta_deg = np.rad2deg(theta_radians)
    return r, theta_radians, theta_deg

epoch_datumz = []

for subj_id in reg_window_df_v2.subj_id.unique():
    for condition in reg_window_df_v2.condition.unique():

        session_data = reg_window_df_v2.loc[(reg_window_df_v2.subj_id == subj_id) & (reg_window_df_v2.condition == condition)].copy()
        for epoch_window_number in session_data.epoch_window_number.unique():

            epoch_data = session_data.loc[session_data.epoch_window_number == epoch_window_number].reset_index(drop=True).copy()

            r, theta_radians, theta_deg = cart2polar(epoch_data.a_est, epoch_data.v_est)

            epoch_data['r'] = r
            epoch_data['theta_radians'] = theta_radians
            epoch_data['theta_deg'] = theta_deg

            epoch_datumz.append(epoch_data)

av_manifold_df = pd.concat(epoch_datumz, axis=0)
av_manifold_df.to_csv('~/Dropbox/loki_0.5/analysis/aggregated_data/av_manifold_df.csv', index=False)

def plot_compass(r, theta_radians, epoch_trial_n,  arrowprops=None, r_max=1,
plot_mean_components=False,  n_rows=1, n_cols=8, color='black'): # note that this takes radians but plots in degrees
    fig, ax = plt.subplots(n_rows, n_cols, subplot_kw=dict(polar=True))
    ax_list = ax.flatten()

    kw = dict(arrowstyle="->", color='gray', lw=2)
    mean_kw = dict(arrowstyle="->", color='black', lw=2.5)

    for ax in ax_list:
        [ax.annotate("", xy=(theta, rad), xytext=(0,0), arrowprops=kw) for rad, theta in zip(r, theta_radians)]
        ax.set_ylim(0, r_max)

    mean_rad = np.mean(r)
    mean_theta = np.mean(theta_radians)

    if plot_mean_components:

        rads = np.arange(0, (2*np.pi), 0.01)
        mean_theta_arr = np.ones_like(rads)*mean_theta

        plt.polar(mean_theta, r_max,  '.', color='black', clip_on=False, markersize=20)

        plt.polar(rads, mean_theta_arr, '-', color='black')

    ax.annotate("", xy=(mean_theta, mean_rad), xytext=(0,0), arrowprops=mean_kw)

    plt.title('epoch trial vector '+ str(epoch_trial_n-1) + ':' + str(epoch_trial_n), y=1.08)

    return fig, ax

def evaluate_magnitude_uniformity(trial_data):
    # from astropy.stats import rayleightest ?

    # plot distribution
    plt.figure()
    plt.title('distribution of magnitudes for epoch trial vector {}:{}'.format((trial_data.shifted_epoch_trial.unique()-1)[0],
    trial_data.shifted_epoch_trial.unique()[0]))
    plt.hist(trial_data.r)
    plt.xlabel('magnitude (radius)'); plt.ylabel('frequency')

    # evaluate statistical sig.
    # p_uniform = rayleightest(trial_data.theta_radians) # test expects radians ?

    # return p_uniform
    return None

sub_cond_trial_df = []

for subj_id in av_manifold_df.subj_id.unique():
    for condition in av_manifold_df.condition.unique():

        session_data = av_manifold_df.loc[(av_manifold_df.subj_id == subj_id) & (av_manifold_df.condition == condition)].copy()

        # get all the epoch n trials across epochs and plot those vectors according to r and theta
        # will need to do this via FacetGrid or subplots

        # g=sns.FacetGrid(session_data, col='shifted_epoch_trial')
        # g.map(plot_compass, "r", "theta_radians", "shifted_epoch_trial", r_max=session_data.r.max())

        for shifted_epoch_trial in session_data.shifted_epoch_trial.unique()[1:]: # don't need first one (diff is nan)

            trial_data = session_data.loc[session_data.shifted_epoch_trial == shifted_epoch_trial][['r', 'theta_radians', 'shifted_epoch_trial']].reset_index(drop=True)

            # fig, axes = plt.subplots(2, 2, subplot_kw=dict(polar=True))

            # # should be able to call g.map() using custom plotting fn
            # need to melt to longform first (or refraine from nesting... )
            #
            plot_compass(trial_data.r,trial_data.theta_radians, trial_data.shifted_epoch_trial, r_max=trial_data.r.max())
            #

            # evaluate uniformity of vector direction

            p_uniform, reject_uniform = evaluate_directional_uniformity(trial_data)

            trial_data['p_uniform'] = p_uniform
            trial_data['reject_uniform'] = reject_uniform

            sub_cond_trial_df.append(trial_data)

sub_trial_unif_dfs = []

for subj_id in av_manifold_df.subj_id.unique():

    sub_data = av_manifold_df.loc[(av_manifold_df.subj_id == subj_id)].copy()

    # get all the epoch n trials across epochs and plot those vectors according to r and theta
    # will need to do this via FacetGrid or subplots

    # g=sns.FacetGrid(session_data, col='shifted_epoch_trial')
    # g.map(plot_compass, "r", "theta_radians", "shifted_epoch_trial")

    for shifted_epoch_trial in sub_data.shifted_epoch_trial.unique()[1:]: # don't need first one (diff is nan)

        trial_data = sub_data.loc[sub_data.shifted_epoch_trial == shifted_epoch_trial][['r', 'theta_radians', 'shifted_epoch_trial', 'subj_id']].reset_index(drop=True)

        # fig, axes = plt.subplots(2, 2, subplot_kw=dict(polar=True))

        # # should be able to call g.map() using custom plotting fn
        #
        # plot_compass(trial_data.r,trial_data.theta_radians, r_max=trial_data.r.max())
        #

        # evaluate uniformity of vector direction

        p_uniform, reject_uniform = evaluate_directional_uniformity(trial_data)

        sub_trial_unif_df = pd.DataFrame()

        # add to each plot the value of p

        sub_trial_unif_df['p_uniform'] = p_uniform
        sub_trial_unif_df['reject_uniform'] = reject_uniform
        sub_trial_unif_df['subj_id'] = subj_id
        sub_trial_unif_df['shifted_epoch_trial'] = shifted_epoch_trial

        print(sub_trial_unif_df.head())

        sub_trial_unif_dfs.append(sub_trial_unif_df)

sub_trial_unif_df_all  = pd.concat(sub_trial_unif_dfs, axis=0)

# do this for all data for each subject
# then do this on a condition-wise basis

from matplotlib.animation import FuncAnimation
import numpy as np
import pandas as pd
import seaborn as sns
import time
import os
from matplotlib import rc
import matplotlib.pyplot as plt

rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
rc('text', usetex=True)

av_manifold_df = pd.read_csv('~/Dropbox/loki_0.5/analysis/aggregated_data/av_manifold_df.csv')
av_manifold_df_unit = av_manifold_df.loc[(av_manifold_df.subj_id == 789)].reset_index(drop=True).copy()

n_rows=2; n_cols=5;
n_conditions=9;
plot_mean_components=None
savefig=1
fig_path='/home/krista/Dropbox/loki_0.5/figures/av_manifold_polar_plots'

# from jupyterthemes import jtplot
# jtplot.style(context='talk', fscale=1.1, spines=False, gridlines='--')
from matplotlib.ticker import FormatStrFormatter
import matplotlib.pyplot as plt

def evaluate_directional_uniformity(trial_data, criterion=0.05):
    from astropy.stats import rayleightest

    # plot distribution
    plt.ioff()
    plt.figure()
    plt.title('distribution of angles for epoch trial vector {}:{}'.format((trial_data.shifted_epoch_trial.unique()-1)[0],
     trial_data.shifted_epoch_trial.unique()[0]))
    plt.hist(trial_data.theta_radians)
    plt.xlabel('angle (rad.)'); plt.ylabel('frequency')

    # evaluate statistical sig.
    p_uniform = rayleightest(trial_data.theta_radians) # test expects radians

    reject_uniform = p_uniform <= criterion

    return p_uniform, reject_uniform

# print at max view
# figure out how to check fig size when max.

def plot_compass_v2(av_manifold_df_unit, n_rows=2, ncols=5,
n_conditions=9, plot_mean_components=None, savefig=None, figsize=(10,8), fig_path='/home/krista/Dropbox/loki_0.5/figures/av_manifold_polar_plots'):

    if len(av_manifold_df_unit.condition.unique()) == n_conditions:
        all_conditions = True
    elif len(av_manifold_df_unit.condition.unique()) == 1:
        all_conditions = False
    else:
        raise ValueError('check n_conditions in supplied df')

    fig, main_ax = plt.subplots(n_rows, n_cols, subplot_kw=dict(polar=True), figsize=figsize)
    ax_list = main_ax.flatten()
    kw = dict(arrowstyle="->", color='gray', lw=2)
    mean_kw = dict(arrowstyle="->", color='black', lw=2.5)

    shifted_epoch_trials = np.sort(av_manifold_df_unit.shifted_epoch_trial.unique())[1:]

    for shifted_trial, ax in zip(shifted_epoch_trials, ax_list):
        print(shifted_trial)
        data = av_manifold_df_unit.loc[av_manifold_df_unit.shifted_epoch_trial == shifted_trial].reset_index(drop=True).copy()
        print(data.head())
        r_max = data.r.max()
        print(r_max)
        [ax.annotate("", xy=(theta, rad), xytext=(0,0), arrowprops=kw) for rad, theta in zip(data.r, data.theta_radians)]
        ax.yaxis.set_major_formatter(FormatStrFormatter('%.2f'))
        ax.tick_params(axis='y', which='major', labelsize=8)
        ax.set_ylim(0, r_max)

        mean_rad = np.mean(data.r)
        mean_theta = np.mean(data.theta_radians)

        if plot_mean_components:
            rads = np.arange(0, (2*np.pi), 0.01)
            mean_theta_arr = np.ones_like(rads)*mean_theta
            plt.polar(mean_theta, r_max,  '.', color='black', clip_on=False, markersize=20)
            plt.polar(rads, mean_theta_arr, '-', color='black')
        ax.annotate("", xy=(mean_theta, mean_rad), xytext=(0,0), arrowprops=mean_kw)

        if shifted_trial == 0:
            ax.set_title(r'$\vec{{_{{-\textrm{}:\textrm{}}}}}$'.format(abs(int(shifted_trial-1)), int(shifted_trial)), fontsize=20, y=1.10)
        else:
            ax.set_title(r'$\vec{{_{{\textrm{}:\textrm{}}}}}$'.format(int(shifted_trial)-1, int(shifted_trial)), fontsize=20, y=1.10)

        p_uniform, reject_uniform = evaluate_directional_uniformity(data)

        if reject_uniform:
            ax.annotate('* p = ' + str(np.round(p_uniform, 3)), xy=(0, r_max), xytext=(1.22,r_max+0.01), fontsize=18, color='tomato')

        print('trial {} p_uniform {}'.format(shifted_trial, p_uniform))
        print('reject uniform dist.? {}'.format( reject_uniform))
        # time.sleep(1)

    fig.delaxes(ax_list[-1]) # odd number of plots so delete last subplot
    fig.subplots_adjust(top = 0.90, bottom=0.01, hspace=0.03, wspace=0.2)

    if all_conditions:
        fig_name = (str(int(av_manifold_df_unit.subj_id.unique()[0])) + '_' + 'all_conditions_av_manifold_polar.png')
        fig.suptitle('subject ' + str(int(av_manifold_df_unit.subj_id.unique()[0])) +  ': all data', fontsize=20)
    else:
        fig_name = (str(int(av_manifold_df_unit.subj_id.unique()[0])) + '_' + str(int(av_manifold_df_unit.condition.unique()[0])) +'_av_manifold_polar.png')
        fig.suptitle('subject ' + str(int(av_manifold_df_unit.subj_id.unique()[0])) + ': ' +
        '$\lambda = $' + str(int(av_manifold_df_unit.lambda_val.unique()[0])) + 'p = ' + int(str(av_manifold_df_unit.p_optimal.unique()[0])), fontsize=20)
        fig.show()

    if savefig:
        # fig.tight_layout()
        # figManager = plt.get_current_fig_manager()
        # figManager.window.showMaximized()
        # fig.show()
        fig.savefig(os.path.join(fig_path, fig_name))

    return fig, main_ax, p_uniform_list, reject_uniform_list
#
# for subj_id in av_manifold_df.subj_id.unique():
#     sub_data = av_manifold_df.loc[av_manifold_df.subj_id == subj_id].reset_index(drop=True).copy()

# _____

# epoch_window = 10
#
# assert len(reg_est_df.epoch_trial.unique()) == epoch_window, 'check epoch len'
#
# for subj_id in reg_est_df.subj_id.unique():
#
#     sub_data = reg_est_df.loc[reg_est_df.subj_id == subj_id]
#
#     x = sub_data.a_est.values
#     y = sub_data.v_est.values
#
#     print(x, y)
#
#     fig, ax = plt.subplots()
#     plt.xticks(rotation=45)
#     plt.title('subject ' + str(subj_id))
#     ax.set_ylabel(r'$\hat{v}$')
#     ax.set_xlabel(r'$\hat{a}$')
#     plt.tight_layout()
#     x_init, y_init = [],[]
#     line, = ax.plot(x_init, y_init, '--', markersize=8)
#
    #
    # annotation = ax.annotate(
    #     '', xy=(1,0), xytext=(-1,0),
    #     arrowprops = {'arrowstyle': "->"},
    # )
    #
    # # # create some random data
    # # n_samples = 5000
    # # base_x = np.random.randn(n_samples)
    # # base_y = base_x
    #
    # # y = np.sin(base_x**2 + base_y**2)
    # # x = np.cos(base_x*base_y)
    #
    # # TODO: maybe just use the annotation object instead of line
    #
    # plt.xlim(x.min()-.0001, x.max()+.0001)
    # plt.ylim(y.min()-.012, y.max()+.012)
    #
    # def init():
    #     line.set_data([], [])
    #     return line,
    #
    # def animate(i):
    #     line.set_data(x[:i],y[:i])
    #     annotation.set_position((x[i-1],y[i-1]))
    #     annotation.xy = (x[i-1],y[i-1])
    #     if i > 0:
    #         annotation.set_text('    t' + str(i-2))
    #
    #     return line, annotation
    #
    # anim = FuncAnimation(fig, animate, init_func=init,
    #                                blit=True,  frames=len(x)+1,
    #                                interval=300, repeat=True) # reduce interval to speed up
    # anim.save(str(subj_id) + '_av_trial.mp4', fps=1, dpi=600, bitrate=-1) # bitrate=-1 lets mpl choose


from flask import render_template, request, redirect, Flask
import os
import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import joblib
from sklearn import decomposition
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
import pickle
from sklearn.linear_model import LogisticRegression

##############################################
def Newfeature_groupby(Test_data, op_col2,op, group_col):
    '''
    This function creates a new feature using groupby operation. it groups the test and train data using col1
    and apply the aggregate functions on top of col2
    '''
    for g_col in group_col:
        Test=pd.DataFrame()
        for col in op_col2:
            # new column name for the dataframe
            new_name = 'Per'+''.join(g_col)+'_'+col+'_'+op
            Test[new_name] = Test_data.groupby(g_col)[col].transform(op)

        Test_data=pd.concat([Test,Test_data], axis=1)
    return Test_data

#################################################
def predict_cluster(Test_data):

    X=Test_data.drop(['Provider'],axis=1)
    if 'PotentialFraud' in X.columns:
        X=Test_data.drop(['PotentialFraud', 'Provider'],axis=1)
    with open('std_scaler.pkl', 'rb') as f:
        std_scaler = pickle.load(f)
    std_data=std_scaler.transform(X)
    with open('PCA.pkl', 'rb') as f:
        PCA = pickle.load(f)
    pca_data=PCA.transform(std_data)

    with open('kmeans25.pkl', 'rb') as f:
        Kmeans = pickle.load(f)

    y_pred=Kmeans.predict(pca_data)

    return y_pred

###################################################

def ohc(Final_Test_data):
    print(Final_Test_data)
    print(Final_Test_data.columns)

    return Final_Test_data

######################################################
def preprocess_data(test_provider_data,test_benf_data, test_inpatient, test_outpatient,files):
    test_benf_data[['ChronicCond_Alzheimer', 'ChronicCond_Heartfailure','ChronicCond_KidneyDisease', 'ChronicCond_Cancer',
                   'ChronicCond_ObstrPulmonary', 'ChronicCond_Depression','ChronicCond_Diabetes', 'ChronicCond_IschemicHeart',
                   'ChronicCond_Osteoporasis', 'ChronicCond_rheumatoidarthritis','ChronicCond_stroke']] = test_benf_data[
                   ['ChronicCond_Alzheimer', 'ChronicCond_Heartfailure','ChronicCond_KidneyDisease', 'ChronicCond_Cancer','ChronicCond_ObstrPulmonary',
                   'ChronicCond_Depression','ChronicCond_Diabetes', 'ChronicCond_IschemicHeart','ChronicCond_Osteoporasis', 'ChronicCond_rheumatoidarthritis',
                   'ChronicCond_stroke']].replace(to_replace=2,value=0)

    test_benf_data['risk_score']=test_benf_data['ChronicCond_Alzheimer']+test_benf_data['ChronicCond_Cancer']+test_benf_data['ChronicCond_Depression']\
                              +test_benf_data['ChronicCond_Diabetes']+test_benf_data['ChronicCond_Heartfailure']+test_benf_data['ChronicCond_IschemicHeart']\
                              +test_benf_data['ChronicCond_KidneyDisease']+test_benf_data['ChronicCond_KidneyDisease']+test_benf_data['ChronicCond_Osteoporasis']\
                              +test_benf_data['ChronicCond_Osteoporasis']+test_benf_data['ChronicCond_rheumatoidarthritis']

    test_benf_data['DOB'] = pd.to_datetime(test_benf_data['DOB'] , format = '%Y-%m-%d')
    test_benf_data['DOD'] = pd.to_datetime(test_benf_data['DOD'],format = '%Y-%m-%d',errors='ignore')

    test_benf_data['age']= round(((test_benf_data['DOD'] -test_benf_data['DOB']).dt.days)/365)
    test_benf_data['age']=test_benf_data['age'].fillna(round(((pd.to_datetime('2009-12-01' , format = '%Y-%m-%d') - test_benf_data['DOB']).dt.days)/365))

    test_benf_data['died'] = 0
    test_benf_data.loc[test_benf_data.DOD.notna(), 'died'] = 1
    test_benf_data.loc[test_benf_data.age > 90, 'died'] = 1

    test_inpatient['ClaimStartDt'] = pd.to_datetime(test_inpatient['ClaimStartDt'] , format = '%Y-%m-%d')
    test_inpatient['ClaimEndDt'] = pd.to_datetime(test_inpatient['ClaimEndDt'],format = '%Y-%m-%d')
    test_inpatient['claim_period'] = ((test_inpatient['ClaimEndDt'] - test_inpatient['ClaimStartDt']).dt.days)+1

    test_inpatient['AdmissionDt'] = pd.to_datetime(test_inpatient['AdmissionDt'] , format = '%Y-%m-%d')
    test_inpatient['DischargeDt'] = pd.to_datetime(test_inpatient['DischargeDt'],format = '%Y-%m-%d')
    test_inpatient['Hospitalized_period'] = ((test_inpatient['DischargeDt'] - test_inpatient['AdmissionDt']).dt.days)+1

    test_inpatient['ExtraClaimDays'] = np.where( test_inpatient['claim_period']>test_inpatient['Hospitalized_period'], test_inpatient['claim_period'] - test_inpatient['Hospitalized_period'], 0)
    test_inpatient['same_physician'] = np.where( test_inpatient['AttendingPhysician']==test_inpatient['OperatingPhysician'], 1, 0)

    test_outpatient['ClaimStartDt'] = pd.to_datetime(test_outpatient['ClaimStartDt'] , format = '%Y-%m-%d')
    test_outpatient['ClaimEndDt'] = pd.to_datetime(test_outpatient['ClaimEndDt'],format = '%Y-%m-%d')
    test_outpatient['claim_period'] = ((test_outpatient['ClaimEndDt'] - test_outpatient['ClaimStartDt']).dt.days)+1

    test_outpatient['same_physician'] = np.where( test_outpatient['AttendingPhysician']==test_outpatient['OperatingPhysician'], 1, 0)

    test_inpatient['In_Outpatient'] = 1
    test_outpatient['In_Outpatient'] = 0

    # Merge inpatient and outpatient dataframes based on common columns
    common_columns_test = [ idx for idx in test_outpatient.columns if idx in test_inpatient.columns]
    Inpatient_Outpatient_Merge_Te = pd.merge(test_inpatient, test_outpatient, left_on = common_columns_test, right_on = common_columns_test,how = 'outer')

    # Merge beneficiary details with inpatient and outpatient data
    Inpatient_Outpatient_Beneficiary_Merge_Te = pd.merge(Inpatient_Outpatient_Merge_Te, test_benf_data,left_on='BeneID',right_on='BeneID',how='inner')

    Final_Test_data = pd.merge(Inpatient_Outpatient_Beneficiary_Merge_Te, test_provider_data , how = 'inner', on = 'Provider' )
    Final_Test_data = Final_Test_data.fillna(0)

    columns = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt', 'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'age', 'Hospitalized_period', 'claim_period', 'risk_score']
    new_groupby_columns=['BeneID','AttendingPhysician','OperatingPhysician','OtherPhysician',
                    'ClmAdmitDiagnosisCode','DiagnosisGroupCode',
                    'ClmDiagnosisCode_1', 'ClmDiagnosisCode_2', 'ClmDiagnosisCode_3',
                    'ClmDiagnosisCode_4', 'ClmDiagnosisCode_5', 'ClmDiagnosisCode_6',
                    'ClmDiagnosisCode_7', 'ClmDiagnosisCode_8', 'ClmDiagnosisCode_9',
                    'ClmDiagnosisCode_10', 'ClmProcedureCode_1', 'ClmProcedureCode_2',
                    'ClmProcedureCode_3', 'ClmProcedureCode_4', 'ClmProcedureCode_5',
                    'ClmProcedureCode_6']

    Final_Test_data =  Newfeature_groupby(Final_Test_data, columns, 'median', new_groupby_columns)

    columns = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt', 'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'age', 'Hospitalized_period', 'claim_period', 'risk_score']
    new_groupby_columns=['Provider']
    Final_Test_data =  Newfeature_groupby( Final_Test_data,columns, 'count', new_groupby_columns)

    #https://datagy.io/pandas-get-dummies/
    # Do one hot encoding for gender and Race
    if files==1:
        print("files",files)
        # Convert type of Gender and Race to categorical
        Final_Test_data.Gender=Final_Test_data.Gender.astype('category')
        Final_Test_data.Race=Final_Test_data.Race.astype('category')
        Final_Test_data=pd.get_dummies(Final_Test_data,columns=['Gender','Race'])

    elif files==0:
        Final_Test_data['Race1']=0
        Final_Test_data['Race2']=0
        Final_Test_data['Race3']=0
        Final_Test_data['Race5']=0
        Final_Test_data['Gender1']=0
        Final_Test_data['Gender2']=0
        if 1 in Final_Test_data['Race'].values:
            Final_Test_data['Race1']=1
        elif 2 in Final_Test_data['Race'].values:
            Final_Test_data['Race2']=1
        elif 3 in Final_Test_data['Race'].values:
            Final_Test_data['Race3']=1
        elif 5 in Final_Test_data['Race'].values:
            Final_Test_data['Race5']=1

        if 1 in Final_Test_data['Gender'].values:
            Final_Test_data['Gender1']=1
        elif 2 in Final_Test_data['Gender'].values:
            Final_Test_data['Gender2']=1

        Final_Test_data=Final_Test_data.drop(['Race','Gender'], axis=1)

    Final_Test_data['RenalDiseaseIndicator']=Final_Test_data.RenalDiseaseIndicator.replace(['Y'],1)
    if 'PotentialFraud' in Final_Test_data.columns:
        Final_Test_data['PotentialFraud']=Final_Test_data.PotentialFraud.replace(['Yes','No'],[1,0])

    remove_columns=['BeneID', 'ClaimID', 'ClaimStartDt','ClaimEndDt','AttendingPhysician','OperatingPhysician', 'OtherPhysician',
                'ClmDiagnosisCode_1','ClmDiagnosisCode_2', 'ClmDiagnosisCode_3', 'ClmDiagnosisCode_4','ClmDiagnosisCode_5',
                'ClmDiagnosisCode_6', 'ClmDiagnosisCode_7','ClmDiagnosisCode_8', 'ClmDiagnosisCode_9', 'ClmDiagnosisCode_10',
                'ClmProcedureCode_1', 'ClmProcedureCode_2', 'ClmProcedureCode_3','ClmProcedureCode_4', 'ClmProcedureCode_5',
                'ClmProcedureCode_6','ClmAdmitDiagnosisCode', 'AdmissionDt','DischargeDt', 'DiagnosisGroupCode','DOB', 'DOD','State', 'County']

    Final_Test_data=Final_Test_data.drop(columns=remove_columns, axis=1)

    Final_Test_data['cluster']=predict_cluster(Final_Test_data)
    return Final_Test_data

########################################
def final_fun_1(X, files):

    file=files
    # Load the raw train data
    Test_Provider = X['Test_Provider']
    Test_Beneficiary = X['Test_Beneficiary']
    Test_Inpatient = X['Test_Inpatient']
    Test_Outpatient = X['Test_Outpatient']

    Final_data= preprocess_data(Test_Provider,Test_Beneficiary,Test_Inpatient,Test_Outpatient,file)
    print(Final_data.shape)
    # drop provider column
    test_provider = Final_data[['Provider']]
    test_data = Final_data.drop(axis=1,columns=['Provider'])

    # Standardize the data
    with open('std_scaler1.pkl', 'rb') as f:
        std_scaler = pickle.load(f)
    std_scaler.transform(test_data)

    with open('log_reg.pkl', 'rb') as f:
        best_model = pickle.load(f)

    y_pred=best_model.predict(test_data)
    test_provider['PredictedFraud']=y_pred

    return test_provider

##################################################
import flask
app = Flask(__name__)

@app.route('/')
def index():
    return flask.render_template('page1.html')

@app.route('/forms', methods=["GET", "POST"])
def forms():
    redirect('forms')
    return flask.render_template('forms.html')

@app.route('/forms2', methods=["GET", "POST"])
def forms2():
    redirect('forms2')
    return flask.render_template('forms2.html')

@app.route('/predict', methods=["GET", "POST"])
def predict():

    if request.method == 'POST':
        if request.files:

            uploaded_file = request.files['filename'] # This line uses the same variable and worked fine
            print(uploaded_file.filename)
            filepath = os.path.join('C:/Users/HP/Downloads/Flask', uploaded_file.filename)
            uploaded_file.save(filepath)
            test_provider_data=pd.read_csv(filepath)
            test_wh_frauddata=test_provider_data.drop(['PotentialFraud'], axis=1)

            uploaded_file = request.files['benefilename'] # This line uses the same variable and worked fine
            bene_filepath = os.path.join('C:/Users/HP/Downloads/Flask', uploaded_file.filename)
            uploaded_file.save(bene_filepath)
            test_benf_data= pd.read_csv(bene_filepath)

            uploaded_file = request.files['infilename'] # This line uses the same variable and worked fine
            in_filepath = os.path.join('C:/Users/HP/Downloads/Flask', uploaded_file.filename)
            uploaded_file.save(in_filepath)
            test_inpatient= pd.read_csv(in_filepath)

            uploaded_file = request.files['outfilename'] # This line uses the same variable and worked fine
            out_filepath = os.path.join('C:/Users/HP/Downloads/Flask', uploaded_file.filename)
            uploaded_file.save(out_filepath)
            test_outpatient= pd.read_csv(out_filepath)

            # create a dictionary which will contain all the files
            X = {"Test_Provider":test_wh_frauddata, "Test_Beneficiary":test_benf_data, "Test_Inpatient":test_inpatient, "Test_Outpatient":test_outpatient}

            data= final_fun_1(X, 1)

    return render_template('forms.html',tables=[data.to_html()], titles=[''])

@app.route('/provider', methods=["GET", "POST"])
def provider():
    if request.method == 'POST':
        data=request.form.to_dict()
        test_provider_data= pd.DataFrame([data.values()], columns=data.keys())
        filepath = os.path.join('C:/Users/HP/Downloads/Flask','provider.csv')
        test_provider_data.to_csv(filepath, index=False)
    return render_template('forms2.html',tables1=[test_provider_data.to_html()], titles1=[''])

@app.route('/beneficiary', methods=["GET", "POST"])
def beneficiary():
    if request.method == 'POST':
        data=request.form.to_dict()
        test_benf_data= pd.DataFrame([data.values()], columns=data.keys())
        filepath = os.path.join('C:/Users/HP/Downloads/Flask','beneficiary.csv')
        test_benf_data.to_csv(filepath, index=False)
    return render_template('forms2.html',tables2=[test_benf_data.to_html()], titles2=[''])

@app.route('/inpatient', methods=["GET", "POST"])
def inpatient():
    if request.method == 'POST':
        data=request.form.to_dict()
        test_inpatient= pd.DataFrame([data.values()], columns=data.keys())
        filepath = os.path.join('C:/Users/HP/Downloads/Flask','inpatient.csv')
        test_inpatient.to_csv(filepath, index=False)
    return render_template('forms2.html',tables3=[test_inpatient.to_html()], titles3=[''])

@app.route('/outpatient', methods=["GET", "POST"])
def outpatient():
    if request.method == 'POST':
        data=request.form.to_dict()
        test_outpatient= pd.DataFrame([data.values()], columns=data.keys())
        filepath = os.path.join('C:/Users/HP/Downloads/Flask','outpatient.csv')
        test_outpatient.to_csv(filepath, index=False)
    return render_template('forms2.html',tables4=[test_outpatient.to_html()], titles4=[''])

@app.route('/detect', methods=["GET", "POST"])
def detect():
    test_benf_data=pd.read_csv('beneficiary.csv')
    test_provider_data=pd.read_csv('provider.csv')
    test_inpatient=pd.read_csv('inpatient.csv')
    test_outpatient=pd.read_csv('outpatient.csv')

    X = {"Test_Provider":test_provider_data, "Test_Beneficiary":test_benf_data, "Test_Inpatient":test_inpatient, "Test_Outpatient":test_outpatient}

    data= final_fun_1(X,0)

    return render_template('forms2.html',tables=[data.to_html()], titles=[''])

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

app.config['FILE_UPLOADS'] = "C:\\Users\\HP\\Downloads\\Flask"