import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler, StandardScaler, Imputer

from pandas.tools.plotting import parallel_coordinates

from clumpy import importance
from clumpy.preprocessing import process_data

sns.set_style('white')

CLUSTER_COLOR = sns.color_palette('Reds_d')[5]
MARGINAL_COLOR = sns.color_palette('Blues_d')[5]

def plot_clusters(X, labels=None, centers=None, size=10, aspect=2):
    if labels is None:
        labels = np.zeros(X.shape[0])
    data = pd.DataFrame({'x': X[:, 0], 'y': X[:, 1], 'labels': labels})

    p = sns.lmplot('x', 'y', hue='labels', fit_reg=False, data=data[labels != -1], size=size, aspect=aspect)

    outliers = data[labels == -1]
    plt.scatter(outliers.x.values, outliers.y.values, color='black', alpha=0.25)

    if centers is not None:
        for center in centers:
            plt.plot(center[0], center[1], marker='o', color='black')

    return p

def categorical_plot(y, cluster_labels, data, order='relevance', figsize=(10, 20)):
    cluster_ids = np.unique(cluster_labels)

    relevance = importance.categorical_relevance(y, cluster_labels, data)

    fig, axes = plt.subplots(len(cluster_ids), 1, sharex=True, figsize=figsize)
    for ax, cluster_id in zip(axes, cluster_ids):
        cluster_mask = (cluster_labels == cluster_id)
        sns.countplot(y=y, data=data, color=MARGINAL_COLOR, order=relevance[cluster_id], ax=ax)
        sns.countplot(y=y, data=data[cluster_mask], color=CLUSTER_COLOR, order=relevance[cluster_id], ax=ax)

def cat_plot(y, cluster_labels, cluster_id, data, ax=None):
    if ax is None:
        ax = plt.gca()

    relevance = importance.single_cluster_relevance(y, cluster_labels, cluster_id, data)
    cluster_mask = (cluster_labels == cluster_id)
    sns.countplot(y=y, data=data, color=MARGINAL_COLOR, order=relevance, ax=ax)
    sns.countplot(y=y, data=data[cluster_mask], color=CLUSTER_COLOR, order=relevance, ax=ax)

# this should be a parallel coordinate plot (means of cluster vs. marginal mean)
def quant_plot(y, cluster_labels, cluster_id, data, ax=None, scale=False):
    if ax is None:
        ax = plt.gca()

    width = 1
    cluster_mask = (cluster_labels == cluster_id)

    # for now print statistics
    unscaled_marginal = data[y].mean().to_frame().transpose()
    unscaled_marginal['id'] = 'marginal'
    unscaled_cluster = data[y][cluster_labels == cluster_id].mean().to_frame().transpose()
    unscaled_cluster['id'] = 'cluster'
    print(pd.concat((unscaled_marginal, unscaled_cluster)))

    if scale:
        scaled_data = data.copy()
        #scaled_data[data.columns] = MinMaxScaler().fit_transform(data)
        scaled_data[y], num_cols, cat_cols = process_data(data[y], impute='mean', num_preprocessing='standardize')
    else:
        scaled_data = data

    marginal_means = scaled_data[y].mean().to_frame()
    marginal_means = marginal_means.sort_values(by=0, ascending=False).transpose()
    columns = marginal_means.columns
    marginal_means['id'] = 'marginal'

    cluster_means = scaled_data[y][cluster_mask].mean().to_frame().transpose()
    cluster_means['id'] = 'cluster'.format(cluster_id)

    # sort in descending order of marginal means

    ax = parallel_coordinates(
            pd.concat((marginal_means, cluster_means)), 'id', cols=columns,
            color=[MARGINAL_COLOR, CLUSTER_COLOR],
            ax=ax)
    ax.get_yaxis().set_ticks([])
    plt.setp(ax.get_xticklabels(), rotation=45)
    #marginal_mean = np.mean(data[y])
    #cluster_mean = np.mean(data[y][cluster_mask])

    #if cluster_mean > marginal_mean:
    #    ax.barh(0.5, np.mean(data[y][cluster_mask]), width, color=CLUSTER_COLOR)
    #    ax.barh(0.5, np.mean(data[y]), width, color=MARGINAL_COLOR)
    #else:
    #    ax.barh(0.5, np.mean(data[y]), width, color=MARGINAL_COLOR)
    #    ax.barh(0.5, np.mean(data[y][cluster_mask]), width, color=CLUSTER_COLOR)
    #ax.get_yaxis().set_ticks([])
    #ax.set_xlabel(y)

def plot_cluster_statistics(cluster_labels, cluster_id, data, quant_var=[], qual_var=[], figsize=None, scale=False):
    n_plots = len(qual_var) + 1 if quant_var else len(qual_var)

    fig, axes = plt.subplots(n_plots, 1, sharex=False, figsize=figsize)
    if not isinstance(axes, np.ndarray):
        axes = [axes]

    for ax, var in zip(axes[:-1], qual_var):
        cat_plot(var, cluster_labels, cluster_id, data, ax=ax)

    if quant_var:
        quant_plot(quant_var, cluster_labels, cluster_id, data, scale=scale, ax=axes[-1])


import cv2
from pathlib import Path
import os

def vid_to_frame(cam_id):
    os.makedirs(f'cam_{cam_id}_resized', exist_ok=True)
    cam_id = str(cam_id)
    vid_dir = f'cam_{cam_id}/cam_{cam_id}.mp4'
    vid_dir = str(Path(vid_dir))
    vid = cv2.VideoCapture(vid_dir)
    length = int(vid.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = int(vid.get(cv2.CAP_PROP_FPS))
    frame_num = 0
    while True:
        success, frame = vid.read()
        if success:
            frame = cv2.resize(frame, (448, 448))
            target_dir = f'cam_{cam_id}_resized/{frame_num}.jpg'
            target_dir = str(Path(target_dir))
            cv2.imwrite(target_dir, frame)
            print(f'Wrote {frame_num} to {target_dir}')
        else:
            break
        frame_num += 1
    print(f'Obtained {length} frames with a frame rate of {fps}')


"""
This function is triggered by an Amazon EventBridge rule when a new sns topic is created.
It extracts the identity of the resource creator and tags each newly created topic with this identity
and it's associated tags
"""
import os
import boto3
import json
import logging
from botocore.exceptions import ClientError

log_level = {
                "CRITICAL": logging.CRITICAL,
                "ERROR" : logging.ERROR,
                "WARNING" : logging.WARNING,
                "INFO" : logging.INFO,
                "DEBUG" : logging.DEBUG
            }
log = logging.getLogger(__name__)
log.setLevel(log_level[os.environ.get('LOG_LEVEL',"DEBUG")])

def get_iam_identity_tags(user_name=None, role_name=None):
    """
    Gets and returns a list of tags from an iam user or role
    """
    iam_client = boto3.client('iam')

    if user_name:
        response = iam_client.list_user_tags(UserName=user_name)
    elif role_name:
        response = iam_client.list_role_tags(RoleName=role_name)

    tags = response.get("Tags")
    while response.get("Marker"):
        if user_name:
            response = iam_client.list_user_tags(UserName=user_name, Marker=response.get("Marker"))
        elif role_name:
            response = iam_client.list_role_tags(RoleName=role_name, Marker=response.get("Marker"))
        tags.extend(response.get('Tags'))

    log.debug(f"Tags parsed from IAM identity: {tags}")
    return tags

def get_resource_tag(event):
    """
    Takes in a cloudtrail event, extract IAM identity and returns a list of resource tags
    """
    resource_tags = []
    user_id = event.get("detail").get("userIdentity")

    if user_id.get("type") == "IAMUser" and user_id.get("userName"):
        user_name = user_id.get("userName")
        resource_tags.append( {"Key": "CreatedByUser", "Value": user_name} )
        resource_tags.append( {"Key": "DateCreated", "Value": event.get("detail").get("eventTime")} )
        log.debug(f"IAM user tags parsed from Cloudtrail event: {resource_tags}")
        try:
            tags = get_iam_identity_tags(user_name=user_name)
            resource_tags.extend(tags)
        except ClientError as error:
            log.exception(error)

    elif user_id.get("type") == "AssumedRole" and user_id.get("arn"):
        role_name = user_id.get("arn").split("/")[-2]
        resource_tags.append( {"Key": "CreatedByRole", "Value": role_name} )
        resource_tags.append( {"Key": "DateCreated", "Value": event.get("detail").get("eventTime")} )
        log.debug(f"IAM Role tags parsed from Cloudtrail event: {resource_tags}")
        try:
            tags = get_iam_identity_tags(role_name=role_name)
            resource_tags.extend(tags)
        except ClientError as error:
            log.exception(error)

    return resource_tags

def lambda_handler(event, context):
    """
    If SNS topic is created successfully, get the resource tags
    from the event and append them to the newly created SNS topic
    """
    log.info(json.dumps(event))
    event_name = event.get("detail").get("eventName")

    if "errorCode" not in event.get("detail") and event_name == "CreateTopic":

        resource_tags = get_resource_tag(event)
        if resource_tags:
            log.info(f"Resource tag to be appended\n{resource_tags}")
            topic_arn = event.get("detail").get("responseElements").get("topicArn")
            sns_client = boto3.client('sns')
            try:
                sns_client.tag_resource(ResourceArn=topic_arn,Tags=resource_tags)
                log.info(f'Successfully tagged SNS topic: {topic_arn}')
            except ClientError as error:
                log.exception(error)


from .test_wrapper import *
from keras.datasets import cifar10
from keras.models import load_model
from hlib.frontend import relay_parser
import sys
sys.trackbacklimit = 0

batch = 1
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x = x_test[0:9984, 0:32, 0:32, 0:3] / 255
x_train_keras = np.reshape(x, (-1, batch, 32, 32, 3))
x = np.transpose(x_train_keras, [0, 1, 4, 2, 3])
x = np.reshape(x, (-1, batch, 3, 32, 32))
y = y_test[0:9984].reshape(-1, batch)
test = 1
if(test == 0):
    test_wrapper("cifar10.h5", "relay", "keras", "cifar10",
                 x, y, (batch, 10), {'input_1': (batch, 3, 32, 32)}, batch_size=32)
elif(test == 1):
    keras_model = load_model('cifar10.h5')
    correct = 0
    total = 0
    for i in range(y.shape[0]):
        x_p = keras_model.predict(x_train_keras[i])
        if(np.argmax(x_p, axis=1) == y[i]):
            correct += 1
        total += 1
    print("accuracy:", correct / (total * 1.0))


import numpy as np

from constants import *
from utils import stairs

def a2f_direct(qpoints, smoothing, dtype=np.float16):
    """
    Computes Eliashberg functions a2F directly from lambdas and gammas without any integration.
    """
    freqs, a2f_lambda, a2f_gamma = list(), list(), list()
    for q_point in qpoints:
        dyn_elph = q_point['DynElph']
        lambdas = dyn_elph.lambdas
        gammas = dyn_elph.gammas
        nef = dyn_elph.DOS
        weight = q_point['weight']
        q_freqs = np.sqrt(np.abs(dyn_elph.sqr_freqs)) * np.sign(dyn_elph.sqr_freqs) * k_Ry_THz
        smooth = np.exp(-np.square(smoothing / q_freqs))
        q_a2f_lambda = smooth * lambdas * q_freqs / 2 * weight
        q_a2f_gamma = smooth * gammas / q_freqs / nef / k_Ry_GHz / np.pi / 2 * weight * k_Ry_THz ** 2
        a2f_lambda.append(q_a2f_lambda)
        a2f_gamma.append(q_a2f_gamma)
        freqs.append(q_freqs)
    freqs, a2f_lambda, a2f_gamma = map(lambda x: np.array(x).flatten(), (freqs, a2f_lambda, a2f_gamma))
    idxs = freqs.argsort()
    freqs = freqs[idxs]
    a2f_lambda = a2f_lambda[idxs]
    a2f_gamma = a2f_gamma[idxs]
    a2f_lambda[a2f_lambda <= 0] = 0
    a2f_gamma[a2f_gamma <= 0] = 0
    out_dict = {
        'freqs, THz': freqs,
        'a2f (lambda), THz': a2f_lambda,
        'a2f (gamma), THz': a2f_gamma
    }
    return out_dict

def lambdas_direct(qpoints, smoothing, dtype=np.float16):
    """
    Computes lambdas directly from lambdas and gammas without any integration.
    WARNING: Avoid using dtype=np.float32 or better, sometimes it leads to computational errors.
    """
    freqs_smooth, lambdas_lambda_smooth, lambdas_gamma_smooth = list(), list(), list()
    freqs, lambdas_lambda, lambdas_gamma = list(), list(), list()
    # print(len(qpoints))
    for q_point in qpoints:
        dyn_elph = q_point['DynElph']
        lambdas = dyn_elph.lambdas
        gammas = dyn_elph.gammas
        nef = dyn_elph.DOS
        weight = q_point['weight']
        q_freqs = np.sqrt(np.abs(dyn_elph.sqr_freqs)) * np.sign(dyn_elph.sqr_freqs) * k_Ry_THz
        smooth = np.exp(-np.square(smoothing / q_freqs))
        q_lambdas_lambda_smooth = smooth * lambdas * weight
        q_lambdas_gamma_smooth = smooth * gammas * weight / np.square(q_freqs) / nef * k_Ry_THz / 1000 / np.pi
        q_lambdas_lambda = lambdas * weight
        q_lambdas_gamma = gammas * weight / np.square(q_freqs) / nef * k_Ry_THz / 1000 / np.pi
        lambdas_lambda_smooth.append(q_lambdas_lambda_smooth)
        lambdas_gamma_smooth.append(q_lambdas_gamma_smooth)
        lambdas_lambda.append(q_lambdas_lambda)
        lambdas_gamma.append(q_lambdas_gamma)
        freqs_smooth.append(q_freqs)
        freqs.append(q_freqs)
    freqs_smooth, lambdas_lambda_smooth, lambdas_gamma_smooth = map(lambda x: np.array(x).flatten(), (
        freqs_smooth, lambdas_lambda_smooth, lambdas_gamma_smooth))
    freqs, lambdas_lambda, lambdas_gamma = map(lambda x: np.array(x).flatten(), (freqs, lambdas_lambda, lambdas_gamma))
    idxs = freqs_smooth.argsort()
    lambdas_lambda_smooth = lambdas_lambda_smooth[idxs]
    # print(lambdas_lambda_smooth, np.sum(lambdas_lambda_smooth))
    lambdas_gamma_smooth = lambdas_gamma_smooth[idxs]
    lambda_lambda_stairs_smooth = stairs(lambdas_lambda_smooth)
    lambda_gamma_stairs_smooth = stairs(lambdas_gamma_smooth)
    idxs = freqs.argsort()
    lambdas_lambda = lambdas_lambda[idxs]
    lambdas_gamma = lambdas_gamma[idxs]
    lambda_lambda_stairs = stairs(lambdas_lambda)
    lambda_gamma_stairs = stairs(lambdas_gamma)
    out_dict = {
        'lambdas (lambda)': lambdas_lambda_smooth,
        'lambdas (gamma)': lambdas_gamma_smooth,
        'lambda (lambda)': lambda_lambda_stairs_smooth,
        'lambda (gamma)': lambda_gamma_stairs_smooth,
        'lambda (lambda, not smooth)': lambda_lambda_stairs,
        'lambda (gamma, not smooth)': lambda_gamma_stairs
    }
    return out_dict

def wlogs_direct(direct):
    """
    Computes wlogs directly from lambdas and gammas without any integration.
    First you need to calculate lambdas.
    """
    freqs, lambdas_lambda, lambdas_gamma = direct['freqs, THz'], direct['lambdas (lambda)'], direct['lambdas (gamma)']
    lambda_lambda = np.sum(lambdas_lambda)
    lambda_gamma = np.sum(lambdas_gamma)
    freqs_K = freqs[freqs > 0] / k_Ry_THz * k_Ry_K
    zeros = np.zeros(freqs[freqs <= 0].shape)
    wlogs_lambda = lambdas_lambda[freqs > 0] * np.log(freqs_K) / lambda_lambda
    wlogs_gamma = lambdas_gamma[freqs > 0] * np.log(freqs_K) / lambda_gamma
    wlogs_lambda[wlogs_lambda <= 0] = 0
    wlogs_gamma[wlogs_gamma <= 0] = 0
    wlogs_lambda_stairs = np.concatenate((np.zeros(wlogs_lambda[wlogs_lambda <= 0].shape),
                                          np.exp(stairs(wlogs_lambda[wlogs_lambda > 0]))))
    wlogs_gamma_stairs = np.concatenate((np.zeros(wlogs_gamma[wlogs_gamma <= 0].shape),
                                         np.exp(stairs(wlogs_gamma[wlogs_gamma > 0]))))
    wlogs_lambda, wlogs_gamma, wlogs_lambda_stairs, wlogs_gamma_stairs = \
        map(lambda x: np.concatenate((zeros, x)), (wlogs_lambda, wlogs_gamma, wlogs_lambda_stairs, wlogs_gamma_stairs))
    out_dict = {
        'wlogs (lambda), K': wlogs_lambda,
        'wlogs (gamma), K': wlogs_gamma,
        'wlog (lambda), K': wlogs_lambda_stairs,
        'wlog (gamma), K': wlogs_gamma_stairs,
    }
    return out_dict

def w2s_direct(direct):
    """
    Computes w2s directly from lambdas and gammas without any integration.
    First you need to calculate lambdas.
    """
    freqs, lambdas_lambda, lambdas_gamma = direct['freqs, THz'], direct['lambdas (lambda)'], direct['lambdas (gamma)']
    lambda_lambda = np.sum(lambdas_lambda)
    lambda_gamma = np.sum(lambdas_gamma)
    freqs_K = freqs / k_Ry_THz * k_Ry_K
    w2s_lambda = lambdas_lambda * np.square(freqs_K) / lambda_lambda
    w2s_gamma = lambdas_gamma * np.square(freqs_K) / lambda_gamma
    w2s_lambda_stairs = np.sqrt(stairs(w2s_lambda, dtype=np.longdouble))
    w2s_gamma_stairs = np.sqrt(stairs(w2s_gamma, dtype=np.longdouble))
    out_dict = {
        'w2s (lambda), K': w2s_lambda,
        'w2s (gamma), K': w2s_gamma,
        'w2 (lambda), K': w2s_lambda_stairs,
        'w2 (gamma), K': w2s_gamma_stairs,
    }
    return out_dict


"""
forms.py

Web forms based on Flask-WTForms

See: http://flask.pocoo.org/docs/patterns/wtforms/
     http://wtforms.simplecodes.com/

"""

from flaskext import wtf
from flaskext.wtf import validators
from wtforms.ext.appengine.db import model_form
from models import Post, Category

class ExampleForm(wtf.Form):
    example_name = wtf.TextField('Name', validators=[validators.Required()])
    example_description = wtf.TextAreaField('Description', validators=[validators.Required()])

class TagListField(wtf.Field):
    widget = wtf.TextInput()

    def _value(self):
        if self.data:
            return u', '.join(self.data)
        else:
            return u''

    def process_formdata(self, valuelist):
        if valuelist:
            self.data = [x.strip() for x in valuelist[0].split(',')]
        else:
            self.data = []

class BetterTagListField(TagListField):
    def __init__(self, label='', validators=None, remove_duplicates=True, **kwargs):
        super(BetterTagListField, self).__init__(label, validators, **kwargs)
        self.remove_duplicates = remove_duplicates

    def process_formdata(self, valuelist):
        super(BetterTagListField, self).process_formdata(valuelist)
        if self.remove_duplicates:
            self.data = list(self._remove_duplicates(self.data))

    @classmethod
    def _remove_duplicates(cls, seq):
        """Remove duplicates in a case insensitive, but case preserving manner"""
        d = {}
        for item in seq:
            if item.lower() not in d:
                d[item.lower()] = True
                yield item

BasePostForm = model_form(Post, base_class=wtf.Form)

class PostForm(BasePostForm):
    tags = BetterTagListField(u'Tags')

CategoryForm = model_form(Category, base_class=wtf.Form)


import io

from pytube import YouTube
from pytube.exceptions import VideoUnavailable

async def get_video_meta(video_url):
    video = YouTube(video_url)
    try:
        streams = (
            video.streams.filter(progressive=True, file_extension='mp4')
            .order_by('resolution')
            .desc()
        )
        return {
            'title': video.title,
            'resolutions': [s.resolution for s in streams],
        }
    except VideoUnavailable:
        return {}

async def get_video_url(video_url, resolution):
    video = YouTube(video_url)
    stream = video.streams.filter(res=resolution, progressive=True)[0]
    return {'url': stream.url}

async def get_video(video_url, resolution):
    video = YouTube(video_url)
    stream = video.streams.filter(res=resolution, progressive=True)[0]
    video_buffer = io.BytesIO()
    stream.stream_to_buffer(video_buffer)
    video_buffer.seek(0)

    return video_buffer


from datetime import datetime
import pandas as pd
from fuel_mappings import *

def get_ercot_data():
    all_dfs = []
    for year in range(2019, 2022 + 1):
        for month in list(range(1, 13)):
            newdf = _get_df(year, month)
            all_dfs.append(newdf)
            if year == 2022 and month == 5:
                break
    newdf = pd.concat(all_dfs)
    # Ignore incomplete weeks.
    newdf = newdf.query('Date < "2022-05-29"').copy()

    # Standardize on GWh
    newdf.loc[:, 'Total'] = newdf['Total'].divide(1000).round()
    newdf.loc[:, 'pct'] = newdf['pct'].round(3)
    return newdf

def _get_df(year: int, month: int):
    dateparse = lambda x: datetime.strptime(x, '%m/%d/%Y')
    decimalparse = lambda x: float(x.replace(',',''))

    df = pd.read_table(
        f'ERCOT/IntGenByFuel{year}{month:02d}.tsv',
        usecols=['Date','Fuel','Total', 'Settlement Type'],
        converters={'Date':dateparse,'Total':decimalparse},
    )

        # Find daily total
    alltotal=df.groupby('Date').sum()
    alltotal['Fuel']='ALL SOURCES'
    alltotal['Settlement Type']='Combined'
    alltotal = alltotal.reset_index(drop=False)
    newdf = pd.concat([df, alltotal])

    maxdf = newdf.groupby('Date').max()['Total'].reset_index(drop=False)

    newdf = newdf.merge(maxdf, left_on='Date', right_on='Date', suffixes=['', '_y']).rename({'Total_y': 'DailyMax'}, axis=1)

    newdf = newdf.assign(pct=lambda x: x['Total'] / x['DailyMax'])

    # Generate some type mappings
    newdf['FuelType'] = newdf['Fuel'].apply(get_fuel_type)
    newdf['LoadType'] = newdf['Fuel'].apply(get_load_type)
    newdf['SourceType'] = newdf['LoadType'] + ' ' + newdf['FuelType']
    newdf.loc[newdf['SourceType'] == 'Other Other', 'SourceType'] = 'Other'

    return newdf


'''
Regression ML project on Boston Housing Price

For this project we will investigate the Boston House Price dataset.

 - Problem Definition (Boston house price data).
 - Loading the Dataset.
 - Analyze Data (some skewed distributions and correlated attributes).
 - Evaluate Algorithms (Linear Regression looked good).
 - Evaluate Algorithms with Standardization (KNN looked good).
 - Algorithm Tuning (K=3 for KNN was best).
 - Ensemble Methods (Bagging and Boosting, Gradient Boosting looked good).
 - Tuning Ensemble Methods (getting the most from Gradient Boosting).
 - Finalize Model (use all training data and confirm using validation dataset).

'''

## Load libraries
import numpy
from numpy import arange
from matplotlib import pyplot
from pandas import read_csv
from pandas import set_option
from pandas.tools.plotting import scatter_matrix

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet

from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR

from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import AdaBoostRegressor
from sklearn.metrics import mean_squared_error

## Load dataset
filename = 'housing.csv'
names = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO','B', 'LSTAT', 'MEDV']
## specifying the short names for each attribute to reference them clearly later.
data = read_csv(filename, names = names, delim_whitespace = True)

## Descriptive data
print(data.shape)
#(506, 14)
print(data.dtypes)

print(data.head(20))

set_option('precision', 1)
print(data.describe())
# to summarize the distribution of each attribute.

set_option('precision', 2)
pearson = data.corr(method = 'pearson')
# assume target attr is the last, then remove corr with itself
corr_with_target = pearson.ix[-1][:-1]
# attributes sorted from the most predictive
predictivity = corr_with_target.sort_values(ascending=False)
'''
RM         0.695360
ZN         0.360445
B          0.333461
DIS        0.249929
CHAS       0.175260
AGE       -0.376955
RAD       -0.381626
CRIM      -0.385832
NOX       -0.427321
TAX       -0.468536
INDUS     -0.483725
PTRATIO   -0.507787
LSTAT     -0.737663
Name: MEDV, dtype: float64
'''

# To find the attribute with strongest correlation with output, it would be better to sort the correlations by the absolute value:

'''
>>> corr_with_target[abs(corr_with_target).argsort()[::-1]]
LSTAT     -0.737663
RM         0.695360
PTRATIO   -0.507787
INDUS     -0.483725
TAX       -0.468536
NOX       -0.427321
CRIM      -0.385832
RAD       -0.381626
AGE       -0.376955
ZN         0.360445
B          0.333461
DIS        0.249929
CHAS       0.175260
Name: MEDV, dtype: float64
'''

# It might be interesting to select some strong correlations between attribute pairs.
attrs = pearson.iloc[:-1,:-1] # all except target
# only important correlations and not auto-correlations
threshold = 0.5
# {('LSTAT', 'TAX'): 0.543993, ('INDUS', 'RAD'): 0.595129, ...
important_corrs = (attrs[abs(attrs) > threshold][attrs != 1.0]).unstack().dropna().to_dict()
#     attribute pair  correlation
# 0     (AGE, INDUS)     0.644779
# 1     (INDUS, RAD)     0.595129
# ...

unique_important_corrs = data.DataFrame(
    list(set([(tuple(sorted(key)), important_corrs[key]) \
    for key in important_corrs])), columns=['attribute pair', 'correlation'])

# sorted by absolute value
unique_important_corrs = unique_important_corrs.ix[
    abs(unique_important_corrs['correlation']).argsort()[::-1]]

## Data visualization
#   Unimodal dv
data.hist(sharex = False, sharey = False, xlabelsize = 1, ylabelsize =1)
pyplot.show()

data.plot(kind = 'density', subplots = True, layout = (4,4), sharex = False, legend = False, fontsize = 1)
pyplot.show()
# Add more evidence to our suspicion about possible exponential and bimodal distributions.

data.plot(kind = 'box', subplots = True, layout = (4,4), sharex = False, sharey = False, fontsize = 8)
pyplot.show()
# This helps point out the skew in many distributions so much so that data looks like outliers

#   Multimodal dv [visualizations of the interactions between variables]
scatter_matrix(data)
pyplot.show()

# correlation matrix
fig = pyplot.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(data.corr(), vmin = -1, vmax = 1, interpolation = 'none')
fig.colorbar(cax)
ticks = numpy.arange(0,14,1)
ax.set_xticks(ticks)
ax.set_yticks(ticks)
ax.set_xticklabels(names)
ax.set_yticklabels(names)
pyplot.show()
# We can also see some dark color suggesting candidates removal for improvement of better model accuracy.

# Preparing data
#   - feature selection and removal of correlated attributes
#   - normalization to reduce scaling effect
#   - standardization to reduce distribution effect
#   - bining to improve accuracy for decision tree algorithm

# Split-out validation dataset
array = data.values
X = array[:,0:13]
Y = array[:,13]
validation_ratio = 0.2
seed = 7
X_train, X_validation, Y_train, Y_validation = train_test_split(X, Y, test_size = validation_ratio, random_state = seed)
# To use validation hold-out set as a smoke test to give us confidence on our estimates of accuracy on unseen data.

# Evaluate Algorithm: Baseline

# Use 10-fold cross validation because the dataset is not too small
# Use MSE metric to give idea of how wrong all predictions are
num_folds = 10
#seed = 7
scoring = 'neg_mean_squared_error'

# create a baseline of performance
# spot-check a number of different algorithms:
#   - 3 linear algorithms: LR, LASSO,EN     (with default tuning params)
#   - 3 nonlinear algorithm: CART, SVR, KNN (with default tuning params)
models = []
models.append(('LR', LinearRegression()))
models.append(('LASSO', Lasso()))
models.append(('EN', ElasticNet()))
models.append(('KNN', KNeighborsRegressor()))
models.append(('CART', DecisionTreeRegressor()))
models.append(('SVR', SVR()))

# evaluate each model in turn
results = []
names = []
for name, model in models:
    names.append(name)

    kfold = KFold(n_splits = num_folds, random_state = seed)
    cv_results = cross_val_score(model, X_train, Y_train, cv = kfold, scoring = scoring)
    results.append(cv_results)

    msg = name + ': ' + str(cv_results.mean()) + ' (' + str(cv_results.std()) + ')'
    print(msg)

#   LR: -21.3798557267 (9.41426365698) ************
#   LASSO: -26.4235611084 (11.6511099158)
#   EN: -27.5022593507 (12.3050222641)
#   KNN: -41.8964883902 (13.9016881498)
#   CART: -22.9675067073 (10.9475148015)
#   SVR: -85.5183418393 (31.9947982318)
# Obviously, LR has the lowest MSE, followed closely by CART.

# take a look at the distribution of scores
fig = pyplot.figure()
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
pyplot.boxplot(results)
ax.set_xticklabels(names)
pyplot.show()

# Evaluate Algorithms: Standardization
# suspect that the differing scales of the raw data may be negatively impacting the skill of some of the algorithms.
pipelines = []
pipelines.append(('ScaledLR', Pipeline([('Scaler', StandardScaler()),('LR',LinearRegression())])))
pipelines.append(('ScaledLASSO', Pipeline([('Scaler', StandardScaler()),('LASSO',Lasso())])))
pipelines.append(('ScaledEN', Pipeline([('Scaler', StandardScaler()),('EN',ElasticNet())])))
pipelines.append(('ScaledKNN', Pipeline([('Scaler', StandardScaler()),('KNN',KNeighborsRegressor())])))
pipelines.append(('ScaledCART', Pipeline([('Scaler', StandardScaler()),('CART',DecisionTreeRegressor())])))
pipelines.append(('ScaledSVR', Pipeline([('Scaler', StandardScaler()),('SVR',SVR())])))

results = []
names = []
for name, model in pipelines:
    names.append(name)

    kfold = KFold(n_splits = num_folds, random_state = seed)
    cv_results = cross_val_score(model, X_train, Y_train, cv = kfold, scoring = scoring)
    results.append(cv_results)

    msg = name + ': ' + str(cv_results.mean()) + ' (' + str(cv_results.std()) + ')'
    print(msg)

#   ScaledLR: -21.3798557267 (9.41426365698)
#   ScaledLASSO: -26.6073135577 (8.97876148589)
#   ScaledEN: -27.9323721581 (10.5874904901)
#   ScaledKNN: -20.1076204878 (12.3769491508) ********* effect on knn
#   ScaledCART: -24.4931573171 (10.4662755609)
#   ScaledSVR: -29.6330855003 (17.0091860524)
# We can see that KNN has both a tight distribution of error and has the lowest score.

# take a look at the standardized distribution of scores
fig = pyplot.figure()
fig.suptitle('Scaled Algorithm Comparison')
ax = fig.add_subplot(111)
pyplot.boxplot(results)
ax.set_xticklabels(names)
pyplot.show()

# Improvements: params tuning (e.g. KNN tuning)

# The default value for the number of neighbors in KNN is 7.
# The below tries odd k values from 1 to 21.

scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
k_values = numpy.array([1,3,5,7,9,11,13,15,17,19,21])
param_grid = dict(n_neighbors = k_values)
model = KNeighborsRegressor()
kfold = KFold(n_splits = num_folds, random_state = seed)
grid = GridSearchCV(estimator = model, param_grid = param_grid,scoring = scoring, cv = kfold)
grid_result = grid.fit(rescaledX, Y_train)

print('Best: ' + str(grid_result.best_score_) + ' using ' + str(grid_result.best_params_))

means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']

for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

# Improvements: ensemble methods
#   - 1. Boosting [AdaBoost, Gradient Boosting]
#   - 2. Bagging [Random Forest, Extra Trees]
ensemble = []
ensemble.append(('ScaledAB', Pipeline([('Scaler', StandardScaler()), ('AB', AdaBoostRegressor())])))
ensemble.append(('ScaledGBM', Pipeline([('Scaler', StandardScaler()), ('GBM', GradientBoostingRegressor())])))
ensemble.append(('ScaledRF', Pipeline([('Scaler', StandardScaler()), ('RF', RandomForestRegressor())])))
ensemble.append(('ScaledET', Pipeline([('Scaler', StandardScaler()), ('ET', ExtraTreesRegressor())])))

results = []
names = []
for name, model in ensemble:
  kfold = KFold(n_splits=num_folds, random_state=seed)
  cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
  results.append(cv_results)
  names.append(name)
  msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
  print(msg)

#   ScaledAB: -14.957611 (6.579588)
#   ScaledGBM: -10.054631 (4.522576)
#   ScaledRF: -13.718030 (7.982746)
#   ScaledET: -10.530828 (6.818533)
# Generally we can get better scores than our linear and nonlinear algorithms in previous sections.

# Compare Algorithms
fig = pyplot.figure()
fig.suptitle('Scaled Ensemble Algorithm Comparison')
ax = fig.add_subplot(111)
pyplot.boxplot(results)
ax.set_xticklabels(names)
pyplot.show()

# Tune scaled GBM: to do better with GBM
# The default number of boosting stages to perform is 100.
# Often, the larger the number of boosting stages, the better the performance but the longer the training time
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)

param_grid = dict(n_estimators=numpy.array([50,100,150,200,250,300,350,400]))

model = GradientBoostingRegressor(random_state = seed)
kfold = KFold(n_splits = num_folds, random_state = seed)
grid = GridSearchCV(estimator = model, param_grid = param_grid,scoring = scoring, cv = kfold)

grid_result = grid.fit(rescaledX, Y_train)

print('Best: ' + str(grid_result.best_score_) + ' using ' + str(grid_result.best_params_))

means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']

for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))

# Best: -9.35647101741 using {'n_estimators': 400}

#   -10.812167 (4.724394) with: {'n_estimators': 50}
#   -10.040857 (4.441758) with: {'n_estimators': 100}
#   -9.694096 (4.275762) with: {'n_estimators': 150}
#   -9.539706 (4.270637) with: {'n_estimators': 200}
#   -9.448764 (4.262603) with: {'n_estimators': 250}
#   -9.429946 (4.273791) with: {'n_estimators': 300}
#   -9.369824 (4.254108) with: {'n_estimators': 350}
#   -9.356471 (4.267837) with: {'n_estimators': 400}

# We can see that the best configuration was n estimators=400, better than the untuned method.

# Finalize model
scaler = StandardScaler().fit(X_train)
rescaledX = scaler.transform(X_train)
model = GradientBoostingRegressor(random_state = seed, n_estimators = 400)
model.fit(rescaledX, Y_train)

rescaledValidationX = scaler.transform(X_validation)
predictions = model.predict(rescaledValidationX)
print(mean_squared_error(Y_validation, predictions))
# 11.8752520792


"""
    Main client for hh.ru SDK.
    Provides root interface for working with HeadHunter.
"""
from typing import Callable, Union, Dict, List, Optional

# Components.
from hhru.api import Api
from hhru.auth import Auth
from hhru.consts import VACANCY_SEARCH_MAX_PER_PAGE

class Client:
    """
    ## hh.ru SDK client.
    Main interface for working with HeadHunter.
    Provides auth, api interfaces.

    ### Example use:
    ```python
    import hhru
    client = hhru.Client()
    ```
    """

    # Authentication instance.
    # Used for authentication.
    auth = None

    # API instance.
    # Used for sending API HTTP requests.
    api = None

    def __init__(self):

        # Components.
        self.auth = Auth()
        self.api = Api(auth=self.auth)

    def search_vacancies(self, **kwargs) -> List:
        """Returns list of search results for vacancies"""
        response = self.api.method(name="vacancies", **kwargs)
        return response.items

    def search_vacancies_over_pages(self, *, page_limit: int = 21, **kwargs) -> List:
        """Returns list of search results for vacancies that was collected from all result pages (hh.ru has limit to 20 max pages search depth)."""
        search_current_page = 0
        while True:
            response = self.api.method(
                name="vacancies",
                page=search_current_page,
                per_page=VACANCY_SEARCH_MAX_PER_PAGE,
                **kwargs,
            )
            search_current_page += 1
            if response.raw_response().status_code != 200:
                # Temporary solution.
                print(response.raw_json())
                raise Exception
            for item in response.items:
                yield item
            if (
                search_current_page >= response.pages
                or search_current_page >= page_limit
            ):
                break


from utils import auth_required, admin_required

from flask import request
from flask_restx import Resource, Namespace

from dao.model.user import UserSchema
from implemented import user_service

genre_ns = Namespace('genres')

@genre_ns.route('/')
class UsersView(Resource):
    @auth_required
    def get(self):
        all_users = user_service.get_all()
        res = UserSchema(many=True).dump(all_users)
        return res, 200

    @admin_required
    def post(self):
        req_json = request.json
        user = user_service.create(req_json)
        return "", 201, {"location": f"/users/{user.id}"}

@genre_ns.route('/<int:rid>')
class UserView(Resource):
    @auth_required
    def get(self, rid):
        r = user_service.get_one(rid)
        sm_d = UserSchema().dump(r)
        return sm_d, 200

    @admin_required
    def put(self, rid):
        req_json = request.json
        if "id" not in req_json:
            req_json["id"] = rid

        user_service.update(req_json)
        return "", 204

    @admin_required
    def delete(self, bid):
        user_service.delete(bid)
        return "", 204


# %%

from sklearn.metrics import classification_report
from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import TfidfVectorizer

from keras.preprocessing.text import Tokenizer

from sklearn.model_selection import train_test_split
import re

import pandas as pd

import pickle

# %%
# loading the dataset
initial = pd.read_csv(
    'D:/newCode/Machine Learning/DepressionDetection/tweet_senti.csv', encoding='latin-1')
initial.head()

# %%
# add the only intention and tweet
data = {

    "intention": initial.iloc[:, 0],
    "tweet": initial.iloc[:, 5]

}

df = pd.DataFrame(data)

for i, row in df.iterrows():

    if (df.at[i, 'intention'] != 0):
        df.at[i, 'intention'] = 1

df.head()

# %%
df['intention'].value_counts()

# %%
# Preprocessing function to clean up the dataset

def text_cleaning(text):
    text = re.sub(r'@[A-Za-z0-9]+', '', text)     # removing @mentions
    text = re.sub(r'@[A-Za-zA-Z0-9]+', '', text)  # removing @mentions
    text = re.sub(r'@[A-Za-z]+', '', text)        # removing @mentions
    text = re.sub(r'@[-)]+', '', text)            # removing @mentions
    text = re.sub(r'#', '', text)                # removing '#' sign
    text = re.sub(r'RT[\s]+', '', text)           # removing RT
    text = re.sub(r'https?\/\/\S+', '', text)     # removing the hyper link
    text = re.sub(r'&[a-z;]+', '', text)          # removing '>'

    return text

# %%
# Cleaning the data
df['tweet'] = df['tweet'].apply(text_cleaning)
df.head()

# %%

tfidf = TfidfVectorizer(
    max_features=20000000, ngram_range=(1, 3), analyzer='char')
# %%
X = tfidf.fit_transform(df['tweet'])
Y = df['intention']
# %%
X.shape
# %%
x_train, x_test, y_train, y_test = train_test_split(
    X, Y, test_size=0.2, random_state=0)
# %%
clf = LinearSVC()
clf.fit(x_train, y_train)
# %%
y_pred = clf.predict(x_test)
# %%
print(classification_report(y_test, y_pred))

# %%

pkl_filename = "twitter_NLP_svm.pkl"
with open(pkl_filename, 'wb') as file:
    pickle.dump(clf, file)

# %%
x = 'no one cares about me. i will die alone'
x = text_cleaning(x)
vec = tfidf.transform([x])
clf.predict(vec)
# %%
x = 'today i am so happy. thank a lot'
x = text_cleaning(x)
vec = tfidf.transform([x])
clf.predict(vec)
# %%
x = 'I am so single'
x = text_cleaning(x)
vec = tfidf.transform([x])
clf.predict(vec)
# %%
x = 'I am so HAPPY. i fell so alive'
x = text_cleaning(x)
vec = tfidf.transform([x])
clf.predict(vec)
# %%


from django.conf.urls import url, include
from django.contrib import admin
from loginRegistration import views

urlpatterns = [
    url('admin/', admin.site.urls),

    url(r'^login/$', views.Login, name='Login'),
    url(r'^registration/$', views.Register, name='Register'),
    url(r'^logout/$', views.Logout, name='Logout'),

    url('main/', include("main.urls"), name='main'),
    url('board/', include("board.urls"), name='board'),
]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('testimonial', '0008_auto_20160702_1951'),
    ]

    operations = [
        migrations.CreateModel(
            name='TestimonialVideo',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('video', models.FileField(upload_to=b'testimonial_videos')),
                ('inlist', models.ForeignKey(to='testimonial.TestimonialItem')),
            ],
        ),
    ]


from msrestazure.azure_active_directory import AADTokenCredentials
from azure.mgmt import policyinsights
import sys
import json
import logging
import azure
import requests
import msal
import adal

class MSALClient:
    def __init__(self, client_id, secret):
        self.url = "https://login.microsoftonline.com/"
        self.client_id = client_id
        self.secret = secret

    def authenticate(self, tenant_id, scope):
        app = msal.ConfidentialClientApplication(self.client_id, authority=self.url + tenant_id, client_credential=self.secret)
        token = None
        token = app.acquire_token_silent(scope, account=None)
        if not token:
            logging.info("No suitable token exists in cache. Retriving new one")
            token = app.acquire_token_for_client(scopes=scope)
            print(token)
        return AADTokenCredentials(token, self.client_id)

class ADALClient:
    def __init__(self, client_id, secret):
        self.url = "https://login.microsoftonline.com/"
        self.client_id = client_id
        self.secret = secret

    def authenticate(self, tenant_id, scope):
        app = adal.AuthenticationContext(authority=self.url + tenant_id)
        token = None
        token = app.acquire_token_with_client_credentials(scope, self.client_id, self.secret)
        print(token)
        return AADTokenCredentials(token, self.client_id)

def load_config():
    config = json.load(open("parameters.json"))
    # check for client_id exists ect here
    return config

config = load_config()
client = ADALClient(config["client_id"], config["secret"])

mngmt_creds = client.authenticate(config["tenant_id"], config["management_scope"])

policy_client = policyinsights.PolicyInsightsClient(mngmt_creds)

output = policy_client.policy_states.list_query_results_for_management_group("default", "DevSecOps")

print(output)


import json
import typing
from datetime import datetime

from discord.ext.commands import Greedy
from tortoise.exceptions import OperationalError

from utils.Database import ReactWatch, WatchedEmoji, Guild, BugReportingChannel

import discord
from discord import NotFound, HTTPException, Forbidden, TextChannel
from discord.ext import commands, tasks

from cogs.BaseCog import BaseCog
from utils import Utils, Configuration, Lang, Logging

class ReactMonitor(BaseCog):

    def __init__(self, bot):
        super().__init__(bot)
        self.excluded_channels_key = "react_watch_excluded_channels"
        self.rbu_interrupt_key = "react_watch_interrupt"
        self.rbu_running_key = "react_watch_running"
        self.react_watch_servers = set()
        self.min_react_lifespan = dict()
        self.recent_reactions = dict()
        self.react_removers = dict()
        self.mute_duration = dict()
        self.react_adds = dict()
        self.guilds = dict()
        self.emoji = dict()
        self.mutes = dict()
        self.started = False

    async def on_ready(self):
        for guild in self.bot.guilds:
            await self.init_guild(guild.id)
        if not self.check_reacts.is_running():
            self.check_reacts.start()
        self.started = True

    async def init_guild(self, guild_id):
        watch, created = await ReactWatch.get_or_create(serverid=guild_id)
        self.mutes[guild_id] = Configuration.get_persistent_var(f"react_mutes_{guild_id}", dict())
        self.min_react_lifespan[guild_id] = Configuration.get_persistent_var(f"min_react_lifespan_{guild_id}", 0.5)
        self.mute_duration[guild_id] = watch.muteduration

        # track react add/remove per guild
        self.recent_reactions[guild_id] = dict()
        self.react_removers[guild_id] = dict()
        self.react_adds[guild_id] = dict()

        # list of emoji to watch
        self.emoji[guild_id] = dict()
        for e in await watch.emoji:
            self.emoji[guild_id][e.emoji] = e

        # enable listening if set in db
        if watch.watchremoves:
            await self.activate_react_watch(guild_id)

        self.guilds[guild_id], created = await Guild.get_or_create(serverid=guild_id)

    def cog_unload(self):
        self.check_reacts.cancel()

    @commands.Cog.listener()
    async def on_guild_join(self, guild):
        await self.init_guild(guild.id)

    @commands.Cog.listener()
    async def on_guild_remove(self, guild):
        Configuration.del_persistent_var(f"min_react_lifespan_{guild.id}", True)
        Configuration.del_persistent_var(f"react_mutes_{guild.id}", True)
        del self.mutes[guild.id]
        del self.mute_duration[guild.id]
        del self.min_react_lifespan[guild.id]
        del self.recent_reactions[guild.id]
        del self.react_removers[guild.id]
        del self.react_adds[guild.id]
        del self.emoji[guild.id]
        del self.guilds[guild.id]
        if guild.id in self.react_watch_servers:
            await self.deactivate_react_watch(guild.id)
        watch = await ReactWatch.get(serverid=guild.id)
        for e in await watch.emoji:
            await e.delete()
        await watch.delete()

    @commands.Cog.listener()
    async def on_raw_reaction_add(self, event):
        await self.store_reaction_action(event)

    @commands.Cog.listener()
    async def on_raw_reaction_remove(self, event):
        await self.store_reaction_action(event)

    @commands.Cog.listener()
    async def on_member_join(self, member):
        if member.guild.id not in self.mutes:
            return

        if str(member.id) in self.mutes[member.guild.id]:
            guild = member.guild
            guild_config = await self.bot.get_guild_db_config(guild.id)
            if guild_config and guild_config.mutedrole:
                try:
                    mute_role = guild.get_role(guild_config.mutedrole)
                    await member.add_roles(mute_role)
                    log_msg = f"{Utils.get_member_log_name(member)} joined while still muted for banned reacts\n--- I **muted** them... **again**"
                    await self.bot.guild_log(guild.id, log_msg)
                except Exception as e:
                    await Utils.handle_exception("reactmon failed to mute member", self.bot, e)
            else:
                await self.bot.guild_log(guild.id, "**I can't re-mute for reacts because `!guildconfig` mute role is not set.")

    async def activate_react_watch(self, guild_id):
        # store setting in db, and add to list of listening servers
        watch = await ReactWatch.get(serverid=guild_id)
        watch.watchremoves = True
        await watch.save()
        self.react_watch_servers.add(guild_id)

    async def deactivate_react_watch(self, guild_id):
        # store setting in db, and remove from list of listening servers
        watch = await ReactWatch.get(serverid=guild_id)
        watch.watchremoves = False
        await watch.save()
        self.react_watch_servers.remove(guild_id)

    async def is_user_event_ignored(self, event):
        ignored_channels = [row.channelid for row in await BugReportingChannel.all()]
        is_ignored_channel = event.channel_id in ignored_channels
        guild = self.bot.get_guild(event.guild_id)
        if not guild:
            # Don't listen to DMs
            return True
        is_bot = event.user_id == self.bot.user.id
        member = guild.get_member(event.user_id)

        if member is None:
            return True  # ignore reaction events from departing members

        is_mod = member and member.guild_permissions.ban_members
        is_admin = await self.bot.member_is_admin(event.user_id)
        has_admin = False

        for role in member.roles:
            if role in Configuration.get_var("admin_roles", []):
                has_admin = True

        # ignore bot, ignore mod, ignore admin users and admin roles
        if is_bot or is_mod or is_admin or has_admin or is_ignored_channel:
            return True
        return False

    async def cog_check(self, ctx):
        return ctx.guild and (ctx.author.guild_permissions.ban_members or await self.bot.permission_manage_bot(ctx))

    @tasks.loop(seconds=1.0)
    async def check_reacts(self):
        now = datetime.now().timestamp()
        for guild_id in self.recent_reactions:
            try:
                for user_id, mute_time in dict(self.mutes[guild_id]).items():
                    if float(mute_time) + float(self.mute_duration[guild_id]) < now:
                        try:
                            guild = self.bot.get_guild(guild_id)
                            guild_config = await self.bot.get_guild_db_config(guild_id)
                            if guild_config and guild_config.mutedrole:
                                mute_role = guild.get_role(guild_config.mutedrole)
                                member = guild.get_member(int(user_id))
                                if mute_role in member.roles:
                                    await member.remove_roles(mute_role)
                                del self.mutes[guild_id][user_id]
                        except Exception as e:
                            log_channel = await self.bot.get_guild_log_channel(guild_id)
                            del self.mutes[guild_id][user_id]
                            await log_channel.send(f'Failed to unmute user ({user_id}) <@{user_id}>... did they leave the server?')
                            # await Utils.handle_exception('react watch unmute failure', self.bot, e)
                rr = self.recent_reactions[guild_id]
                adds = {t: e for (t, e) in rr.items() if e.event_type == "REACTION_ADD"}

                # creat list of adds
                for t, e in adds.items():
                    self.react_adds[guild_id][t] = e
                # cull out expired ones
                for t, e in dict(self.react_adds[guild_id]).items():
                    if t + self.min_react_lifespan[guild_id] < now:
                        # add reaction is too far in the past. remove from the list
                        del self.react_adds[guild_id][t]

                # loop over a copy of recent_reactions so we can remove items
                for timestamp, event in dict(self.recent_reactions[guild_id]).items():
                    # remove this one from the list
                    del self.recent_reactions[guild_id][timestamp]

                    p = getattr(self, 'process_'+event.event_type.lower())
                    await p(timestamp, event)

            except Exception as ex:
                await Utils.handle_exception('react watch loop error...', self.bot, ex)

    @commands.group(name="reactmonitor",
                    aliases=['reactmon', 'reactwatch', 'react', 'watcher'],
                    invoke_without_command=True)
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True)
    async def react_monitor(self, ctx: commands.Context):
        """
        List the watched emoji and their settings
        """
        max_fields = 24
        watch = await ReactWatch.get(serverid=ctx.guild.id)
        embed = discord.Embed(
            timestamp=ctx.message.created_at,
            color=Utils.COLOR_LIME,
            title=Lang.get_locale_string("react_monitor/info_title", ctx, server_name=ctx.guild.name))

        embed.add_field(name="Monitor React Removal", value="Yes" if watch.watchremoves else "No")

        if watch.watchremoves:
            embed.add_field(name="Reaction minimum lifespan", value=f"{self.min_react_lifespan[ctx.guild.id]} seconds")

        embed.add_field(name="Mute duration", value=Utils.to_pretty_time(self.mute_duration[ctx.guild.id]))

        embed.add_field(name="__                                             __",
                        value="__                                             __",
                        inline=False)

        if ctx.guild.id in self.emoji:
            for key, emoji in self.emoji[ctx.guild.id].items():
                if len(embed.fields) == max_fields:
                    await ctx.send(embed=embed)
                    embed = discord.Embed(
                        color=Utils.COLOR_LIME,
                        title="...")
                embed.add_field(
                    name=f"{emoji.emoji}",
                    value=self.describe_emoji_watch_settings(emoji),
                    inline=True)

        await ctx.send(embed=embed)

    @staticmethod
    def describe_emoji_watch_settings(emoji):
        flags = [f"__*{name}*__" for name in ['log', 'remove', 'mute'] if getattr(emoji, name)]
        val = ' | '.join(flags) if flags else '__*no action*__'
        return val

    @react_monitor.command(aliases=["new", "edit", "update"])
    @commands.guild_only()
    async def add(self, ctx: commands.Context, emoji, log: bool = True, remove: bool = False, mute: bool = False ):
        """
        Add an emoji to the reaction watchlist

        emoji: The emoji to add
        log: Boolean - Log use of this emoji
        remove: Boolean - Auto-remove when this emoji is used
        mute: Boolean - Auto-mute members who use this emoji
        """
        try:
            watch, created = await ReactWatch.get_or_create(serverid=ctx.guild.id)
            new_emoji, created = await WatchedEmoji.get_or_create(watcher_id=watch.id, emoji=emoji)
            new_emoji.log = log
            new_emoji.remove = remove
            new_emoji.mute = mute
            await new_emoji.save()
            self.emoji[ctx.guild.id][emoji] = new_emoji
        except Exception as e:
            await Utils.handle_exception("failed to add emoji to watch list", self.bot, e)

        await ctx.send(f"`{emoji}` is now on the watch list with settings:\n"
                       f"{self.describe_emoji_watch_settings(self.emoji[ctx.guild.id][emoji])}")

    @react_monitor.group(name="clean", invoke_without_command=True)
    @commands.guild_only()
    async def clean(self, ctx):
        await ctx.send_help(ctx.command)

    @clean.command(aliases=["user", "byuser"])
    @commands.guild_only()
    async def clean_by_user(
            self,
            ctx: commands.Context,
            target: discord.User,
            check_channels: Greedy[discord.TextChannel] = True,
            count: int = 200):
        Configuration.set_persistent_var(self.rbu_interrupt_key, False)
        channels = ctx.guild.channels if check_channels is True else check_channels
        if check_channels is True:
            await ctx.send(f"Looking for reacts on the {count} most recent messages in all available channels")
        else:
            list_of_channels = "\n".join([c.mention for c in channels])
            await ctx.send(f"Looking for reacts on the {count} most recent messages "
                           f"in the following channels:\n{list_of_channels}")
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        for channel in channels:
            if isinstance(channel, TextChannel):
                if channel.id in excluded_channels:
                    await ctx.send(f"<#{channel.id}> skipped")
                    continue
                try:
                    i = 0
                    await ctx.send(f"checking <#{channel.id}>...")
                    async for message in channel.history(limit=count):
                        interrupt = Configuration.get_persistent_var(self.rbu_interrupt_key, False)
                        if interrupt:
                            Configuration.set_persistent_var(self.rbu_interrupt_key, False)
                            await ctx.send(f"__remove react by user__ operation halted.")
                            return
                        i += 1
                        for react in message.reactions:
                            async for user in react.users():
                                if user.id == target.id:
                                    await message.clear_reaction(react.emoji)
                                    await ctx.send(
                                        f"__{i}.__ react {react.emoji} by userid ({user.id}) "
                                        f"removed from message {message.jump_url}")
                                    continue
                except Forbidden:
                    await ctx.send(f"I can't access message history in channel <#{channel.id}>")
        await ctx.send("All done cleaning reacts")

    @clean.command(aliases=["stop", "stopclean", "stop_clean"])
    @commands.guild_only()
    async def stop_clean_by_user(self, ctx):
        Configuration.set_persistent_var(self.rbu_interrupt_key, True)
        await ctx.send(f"attempting to halt __remove react by user__ operation. If this doesn't work, KILL THE BOT!")

    @clean.command(aliases=["excludedchannels", "list_excluded", "listexcluded"])
    @commands.guild_only()
    async def excluded_channels(self, ctx):
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = [self.bot.get_channel(c).mention for c in excluded_channels]
        if not excluded_channels:
            await ctx.send(f"No channels are excluded from react remove_by_user")
        else:
            channel_list = "\n".join(excluded_channels)
            await ctx.send(f"Channels excluded from react remove_by_user:\n{channel_list}")

    @clean.command(aliases=["exclude", "excludechannels", "exclude_channels"])
    @commands.guild_only()
    async def exclude_channel(self, ctx, channels: Greedy[discord.TextChannel]):
        """
        Exclude channel from checks for react spam
        :param ctx:
        :param channels:
        :return:
        """
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = set(excluded_channels)
        added = []
        for channel in channels:
            if channel.id not in excluded_channels:
                excluded_channels.add(channel.id)
                added.append(channel.mention)
        Configuration.set_persistent_var(self.excluded_channels_key, list(excluded_channels))
        if not added:
            await ctx.send(f"No channels added to exclusion list")
        else:
            await ctx.send(f"Added {' '.join(added)} to react spam channel exclusion list")

    @clean.command(aliases=["unexclude", "unexcludechannels", "unexclude_channels"])
    @commands.guild_only()
    async def unexclude_channel(self, ctx, channels: Greedy[discord.TextChannel]):
        """
        Exclude channel from checks for react spam
        :param ctx:
        :param channels:
        :return:
        """
        excluded_channels = Configuration.get_persistent_var(self.excluded_channels_key, [])
        excluded_channels = set(excluded_channels)
        removed = []
        for channel in channels:
            if channel.id in excluded_channels:
                excluded_channels.remove(channel.id)
                removed.append(channel.mention)
        Configuration.set_persistent_var(self.excluded_channels_key, list(excluded_channels))
        if not removed:
            await ctx.send(f"No channels removed from exclusion list")
        else:
            await ctx.send(f"Removed {' '.join(removed)} from react spam channel exclusion list")

    @react_monitor.command(aliases=["rem", "del", "delete"])
    @commands.guild_only()
    async def remove(self, ctx: commands.Context, emoji):
        """
        Remove an emoji from the watch list

        emoji: The emoji to remove
        """
        try:
            watch_row = await WatchedEmoji.get(watcher__serverid=ctx.guild.id, emoji=emoji)
            await watch_row.delete()
            del self.emoji[ctx.guild.id][emoji]
            await ctx.send(f"I removed `{emoji}` from the watch list")
        except OperationalError:
            await ctx.send(f"I couldn't find `{emoji}` on the emoji watch list, so I didn't remove it.")
        except Exception as e:
            await Utils.handle_exception("react remove failed", self.bot, e)

    @react_monitor.command(aliases=['on'])
    @commands.guild_only()
    async def monitor_removal_on(self, ctx: commands.Context):
        """
        Turn ON monitor for spammy fast-removal of reactions
        """
        if ctx.guild.id in self.react_watch_servers:
            await ctx.send("React monitor is already on")
        else:
            await self.activate_react_watch(ctx.guild.id)
            await ctx.send("I'm on the lookout for reaction spam!")

    @react_monitor.command(aliases=['off'])
    @commands.guild_only()
    async def monitor_removal_off(self, ctx: commands.Context):
        """
        Turn OFF monitor for spammy fast-removal of reactions
        """
        if ctx.guild.id in self.react_watch_servers:
            await self.deactivate_react_watch(ctx.guild.id)
            await ctx.send("OK, I'll stop watching for reaction spams")
        else:
            await ctx.send("React monitor is already off")

    @react_monitor.command(aliases=["time", "reacttime"])
    @commands.guild_only()
    async def react_time(self, ctx: commands.Context, react_time: float):
        """
        Reacts removed before this duration will trigger react-watch

        react_time: time in seconds, floating point e.g. 0.25
        """
        self.min_react_lifespan[ctx.guild.id] = react_time
        Configuration.set_persistent_var(f"min_react_lifespan_{ctx.guild.id}", react_time)
        await ctx.send(f"Reactions that are removed before {react_time} seconds have passed will be flagged")

    @react_monitor.command(aliases=["list", "mutes"])
    @commands.guild_only()
    async def list_mutes(self, ctx: commands.Context):
        if self.mutes[ctx.guild.id]:
            react_muted = list()
            guild_config = await self.bot.get_guild_db_config(ctx.guild.id)
            mute_role = ctx.guild.get_role(guild_config.mutedrole)

            for member_id, timestamp in self.mutes[ctx.guild.id].items():
                member = ctx.guild.get_member(int(member_id))
                if member is not None:
                    long_name = Utils.get_member_log_name(member)
                    if mute_role not in member.roles:
                        # panic because role is not present when it should be
                        await ctx.send(f"{long_name} should be muted for banned reacts... but isn't. ***WHY NOT??***")
                    react_muted.append(long_name)

            names = "\n".join(react_muted)
            await ctx.send(f"__Members muted for banned reacts:__\n{names}")
        else:
            await ctx.send(f"Nobody is muted for banned reacts")

    @react_monitor.command(aliases=["purge", "purgemutes"])
    @commands.guild_only()
    async def purge_mutes(self, ctx: commands.Context):
        if self.mutes[ctx.guild.id]:
            react_unmuted = list()
            guild_config = await self.bot.get_guild_db_config(ctx.guild.id)
            mute_role = ctx.guild.get_role(guild_config.mutedrole)

            for member_id, timestamp in dict(self.mutes[ctx.guild.id]).items():
                member = ctx.guild.get_member(int(member_id))
                if member is not None:
                    await member.remove_roles(mute_role)
                    del self.mutes[ctx.guild.id][member_id]
                    long_name = Utils.get_member_log_name(member)
                    react_unmuted.append(long_name)

            names = "\n".join(react_unmuted)
            await ctx.send(f"__React mutes purged:__\n{names}")
        else:
            await ctx.send(f"Nobody is muted for banned reacts. Can't purge.")

    @react_monitor.command(aliases=["mutetime", "mute"])
    @commands.guild_only()
    async def mute_time(self, ctx: commands.Context, mute_time: float):
        """
        Set the duration for mutes given when mute-enabled reacts are used

        mute_time: time in seconds, floating point e.g. 0.25
        """
        self.mute_duration[ctx.guild.id] = mute_time
        watch, created = await ReactWatch.get_or_create(serverid=ctx.guild.id)
        watch.muteduration = mute_time
        await watch.save()
        t = Utils.to_pretty_time(mute_time)
        await ctx.send(f"Members will now be muted for {t} when they use restricted reacts")

    async def store_reaction_action(self, event):
        if not self.started or await self.is_user_event_ignored(event):
            return

        # Add event to dict for tracking fast removal of reactions
        now = datetime.now().timestamp()
        guild = self.bot.get_guild(event.guild_id)
        try:
            self.recent_reactions[guild.id][now] = event
        except KeyError as e:
            Logging.debug(f"React Monitory Key error: {json.dumps(e)}")

    async def process_reaction_add(self, timestamp, event):
        emoji_used = event.emoji
        member = event.member
        guild = self.bot.get_guild(event.guild_id)
        channel = self.bot.get_channel(event.channel_id)
        log_channel = await self.bot.get_guild_log_channel(event.guild_id)

        # Act on log, remove, and mute:
        e_db = None
        if str(emoji_used) in self.emoji[event.guild_id]:
            e_db = self.emoji[event.guild_id][str(emoji_used)]
            if not e_db.log and not e_db.remove and not e_db.mute:
                # No actions to take. Stop processing
                return

        if not e_db:
            return

        # check mute/warn list for reaction_add - log to channel
        # for reaction_add, remove if threshold for quick-remove is passed
        try:
            # message fetch is API call. Only do it if needed
            message = await channel.fetch_message(event.message_id)
        except (NotFound, HTTPException) as e:
            # Can't track reactions on a message I can't find
            # Happens for deleted messages. Safe to ignore.
            # await Utils.handle_exception(f"Failed to get message {channel.id}/{event.message_id}", self, e)
            return

        log_msg = f"{Utils.get_member_log_name(member)} used emoji "\
                  f"[ {emoji_used} ] in #{channel.name}.\n"\
                  f"{message.jump_url}"

        if e_db.remove:
            await message.clear_reaction(emoji_used)
            log_msg = f"{log_msg}\n--- I **removed** the reaction"

        if e_db.mute:
            guild_config = await self.bot.get_guild_db_config(guild.id)
            if guild_config and guild_config.mutedrole:
                try:
                    mute_role = guild.get_role(guild_config.mutedrole)
                    await member.add_roles(mute_role)
                    self.mutes[guild.id][str(member.id)] = timestamp
                    Configuration.set_persistent_var(f"react_mutes_{guild.id}", self.mutes[guild.id])
                    log_msg = f"{log_msg}\n--- I **muted** them"
                except Exception as e:
                    await Utils.handle_exception("reactmon failed to mute member", self.bot, e)
            else:
                await self.bot.guild_log(event.guild_id, "**I can't mute for reacts because `!guildconfig` mute role is not set.")

        if (e_db.log or e_db.remove or e_db.mute) and log_channel:
            await log_channel.send(log_msg)

    async def process_reaction_remove(self, timestamp, event):
        # TODO: Evaluate - count react removal and auto-mute for hitting threshold in given time?
        #  i.e. track react-remove-count per user over time. if count > x: mute/warn

        # Add user_id to dict of recent reaction removers with timestamp
        # now = datetime.now().timestamp()
        # self.react_removers[event.guild_id][event.user_id] = now

        # listening setting only apples to quick-remove
        server_is_listening = event.guild_id in self.react_watch_servers
        if not server_is_listening or await self.is_user_event_ignored(event):
            return

        # check recent reacts to see if they match the remove event
        for t, add_event in self.react_adds[event.guild_id].items():
            # Criteria for skipping an event in the list
            not_message = add_event.message_id != event.message_id
            not_user = add_event.user_id != event.user_id

            age = timestamp - t
            expired = 0 > age > self.min_react_lifespan[event.guild_id]
            if expired or not_message or not_user:
                # message id and user id must match remove event, and must not be expired
                continue

            # This user added a reaction that was removed within the warning time window
            guild = self.bot.get_guild(event.guild_id)
            member = guild.get_member(event.user_id)
            emoji_used = str(event.emoji)
            channel = self.bot.get_channel(event.channel_id)
            log_channel = await self.bot.get_guild_log_channel(guild.id)
            # ping log channel with detail
            if log_channel:
                content = f"{Utils.get_member_log_name(member)} " \
                          f"quick-removed [ {emoji_used} ] react from a message in {channel.mention}"
                try:
                    message = await channel.fetch_message(event.message_id)
                    content = f"{content}\n{message.jump_url}"
                except (NotFound, HTTPException) as e:
                    pass
                await log_channel.send(content)

async def setup(bot):
    await bot.add_cog(ReactMonitor(bot))


import re
import sys
import types
from babel.numbers import parse_decimal, parse_number
from dateutil.parser import parse as parse_datetime
from abc import ABCMeta, abstractmethod
from datetime import date, datetime

# True if we are running on Python 3.
PY3 = sys.version_info[0] == 3

if PY3:
    string_types = str,
    unicode = str
    long = int

else:
    string_types = basestring,
    unicode = unicode
    long = long

remove_space = re.compile(u' ')

class TypeConstructor(object):
    def __init__(self, type, *args, **kwargs):
        self.type = type
        self.args = args
        self.kwargs = kwargs

    def __call__(self, value):
        return self.type(value, *self.args, **self.kwargs)

def new(type, *args, **kwargs):
    return TypeConstructor(type, *args, **kwargs)

class Type(object):

    __metaclass__ = ABCMeta

    def __new__(cls, value, *args, **kwargs):
        if cls.is_empty(value):
            return None

        if cls.is_non_convertable(value):
            return value

        return cls.get_converted_value(value, *args, **kwargs)

    @classmethod
    def is_empty(cls, value):
        return value is None or len(unicode(value).strip()) == 0

    @classmethod
    def is_non_convertable(cls, value):
        return isinstance(value, cls.__base__)

    @classmethod
    def _try(cls, func, args, kwargs = None):
        try:
            return func(*args, **(kwargs or {}))

        except:
            return None

    @classmethod
    def coalesce(cls, tests):
        for test in tests:
            if len(test) == 3:
                func, args, kwargs = test
            else:
                func, args = test
                kwargs = None

            rvalue = cls._try(func, args, kwargs)
            if rvalue is not None:
                return rvalue

        return None

    @classmethod
    @abstractmethod
    def get_converted_value(cls, value):
        pass

class Float(Type, float):
    @classmethod
    def get_converted_value(cls, value, locale = None):
        if locale:
            rvalue = cls._try(
                parse_decimal,
                (remove_space.sub( u'', unicode(value) ), locale,)
            )
            return float(rvalue) if rvalue else None

        return cls.coalesce([
            ( float, (value, ) ),
            ( float, (re.sub(u',', u'.', remove_space.sub(u'', unicode(value))), ) ),
        ])

class Int(Type, long):
    @classmethod
    def get_converted_value(cls, value, locale = None):
        return cls.coalesce([
            ( long, (value,) ),
            ( long, (Float(value),) ),
        ])

class Datetime(Type, datetime):
    @classmethod
    def is_non_convertable(cls, value):
        return value.__class__ is datetime

    @classmethod
    def get_converted_value(cls, value, *args, **kwargs):
        if isinstance(value, (tuple, list)):
            return cls._try( datetime, value )

        if value.__class__ is date:
            return datetime(value.year, value.month, value.day)

        if not isinstance(value, string_types):
            return None

        return cls._try( parse_datetime, (value,), kwargs )

class Date(Type, date):
    @classmethod
    def is_non_convertable(cls, value):
        return value.__class__ is date

    @classmethod
    def get_converted_value(cls, value, *args, **kwargs):
        if isinstance(value, (tuple, list)):
            return cls._try( date, value )

        if value.__class__ is datetime:
            return value.date()

        if not isinstance(value, string_types):
            return None

        rvalue = cls.coalesce([
             ( parse_datetime, (value,), kwargs ),
             ( parse_datetime, (value.split(u' ')[0],), kwargs ),
        ])
        if rvalue:
            return rvalue.date()

class Str(Type, unicode):
    @classmethod
    def get_converted_value(cls, value):
        return unicode(value)

class Bool(Type):
    TRUE_VALUES = ( 't', 'true', 'yes', 'y', )
    FALSE_VALUES = ( 'f', 'false', 'no', 'n', )

    @classmethod
    def is_empty(cls, value):
        return False

    @classmethod
    def get_converted_value(cls, value):
        if value is None:
            return value

        if isinstance(value, string_types) and value.lower().strip() in cls.TRUE_VALUES:
            return True

        elif isinstance(value, string_types) and value.lower().strip() in cls.FALSE_VALUES:
            return False

        elif isinstance(value, string_types) and value.replace(u'.',u'',1).isdigit():
            return bool(Float(value))

        return bool(value)


"""
utils.py
"""
import csv
import sys
import socket
import json
import re
import logging
import hashlib
import requests
import ssdeep
import pefile
import configuration_file as cfg
from werkzeug.utils import secure_filename

def break_csv(lines):
    """
    empty
    """
    for line in lines:
        if line.startswith('#'):
            lines.remove(line)
    reader = csv.reader(lines, quotechar='"',
                        delimiter=',',
                        quoting=csv.QUOTE_ALL,
                        skipinitialspace=True)
    next(reader)
    return reader

def init_connection_urlhaus():
    """
    Initialisation, connection to Url Haus
    Params:
    Returns:
    """
    try:
        logging.info('Fetching latest URLhaus list...')
        request = requests.get(cfg.urlhaus["link"])
        if request.status_code == 200:
            logging.info('Processing URLhaus list...')
            return request
        logging.error('Problem connecting to URLhaus. Status code: %i', request.status_code)

    except requests.exceptions.ConnectionError as error_connection:
        logging.warning('Problem connecting to URLhaus. Error: %i', error_connection)

    except Exception as exception:
        logging.warning('Problem connecting to URLhaus. Aborting task.')
        logging.exception(sys.exc_info())
        logging.exception(type(exception))
        logging.exception(exception.args)
        logging.exception(exception)

    return []

def clean_url(url):
    """
    Remove extraneous characters from URL.
    Params:
    - url: (type: string) URL to clean.
    Returns:
    - url: (type: string) clean URL.
    """
    if url is None:
        return None
    if '??' in url:
        url = url.split('??')[0]
    if url.endswith('?'):
        url = url[:-1]
    if '`' in url:
        url = url.replace('`', '')
    return url

def get_name_file_from_url(url):
    """
    Get name file from url
    Params:
    - url: (type: string).
    Returns:
    - name: (type: string).
    """

    last_slash = url.rfind('/')
    return url[last_slash+1:]

def save_infos_from_csv(item):
    """
    SAVE INFOS FROM CSV
    Params:
    - item: (type: string) URL.
    Returns:
    - id_malware, date_added, url_malware, url_status, threat, tags, urlhaus_link, reporter.
    """
    id_malware = item[0]
    date_added = item[1]
    url_malware = clean_url(item[2])
    if item[3] == "online":
        url_status = True
    else:
        url_status = False
    threat = item[4]
    tags = item[5]
    urlhaus_link = item[6]
    reporter = item[7]
    return id_malware, date_added, url_malware, url_status, threat, tags, urlhaus_link, reporter

def save_vt_report(vt_report, id_malware):
    with open(str(id_malware) + ".vt", "w+") as outfile:
        json.dump(vt_report, outfile)
        outfile.close()

def get_ip_geo_from_url(input):
    """Mofication Parser url et recuperer la partie essentiel pour analyser apres"""
    compile_rule1 = re.compile('^(?:http:\/\/|www\.|https:\/\/)([^\/]+)')
    match1 = re.match(compile_rule1, input)
    """Verifier cela est une addresse IP ou pas"""
    compile_rule = re.compile(r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])')
    match = re.match(compile_rule, match1.group(1))
    """Si cela est une addresse IP"""
    if match:
        ip = match.group()
    else:
        try:
            ip = socket.getaddrinfo(match1.group(1), 'http')[0][4][0]
        except:
            ip = "0.0.0.0"
    """A l aide du site ipstack, lire les données de cette addresse"""
    url = 'http://api.ipstack.com/' + ip + '?access_key=' + cfg.ipstack['apikey']
    try:
        a = requests.get(url)
        data = a.json()
    except:
        return "none", "none", "none", "none", "none", "none", "none", "none", "none", 0.0, 0.0, "none"
    country_name = data.get('country_name')
    ip = data.get('ip')
    continent_code = data.get('continent_code')
    continent_name = data.get('continent_name')
    country_code = data.get('country_code')
    region_code = data.get('region_code')
    region_name = data.get('region_name')
    city = data.get('city')
    zip = data.get('zip')
    latitude = data.get('latitude')
    longitude = data.get('longitude')
    full_report = data
    return ip, country_name, country_code, continent_name, continent_code, region_name, region_code, city, zip, latitude, longitude, full_report

def get_md5(fname):
    """
    Get malwate MD5
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_md5 = hashlib.md5()
    with open(cfg.malware["malware_path"] + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    rslt = hash_md5.hexdigest()
    return rslt

def get_sha256(fname):
    """
    Get malware sha256
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_sha256 = hashlib.sha256()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_sha256.update(chunk)
    rslt = hash_sha256.hexdigest()
    return rslt

def get_sha1(fname):
    """
    Get malware sha1
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_sha1 = hashlib.sha1()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_sha1.update(chunk)
    rslt = hash_sha1.hexdigest()
    return rslt

def get_ssdeep(fname):
    """
    Get malware ssdeep
    Params:
    -fname
    Returns:
    - rslt
    """

    hash_ssdeep = ssdeep.Hash()
    with open(cfg.malware["malware_path"]  + secure_filename(fname), 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_ssdeep.update(chunk)
    rslt = hash_ssdeep.digest()
    return rslt

def get_imphash(fname):
    """
    If the malwre is an exec get the impash
    Params:
    -fname, db
    Returns:
    - imps
    """
    try:
        imps = []
        p = pefile.PE(cfg.malware["malware_path"]  + secure_filename(fname))
        imphash = p.get_imphash()
        imps.append(imphash)
        return str(imps)
    except:
        return None

def is_exe(tags):
    """
    If the malwre is an exec
    Params:
    -tags (source csv urlhaus)
    Returns:
    - boolean
    """
    if "exe" in tags:
        return True
    return False


# include the functions that relate to Hamiltonian's and gates
import numpy as np

def spin_operators(spin):
    op = dict()
    if spin == 'half' or abs(spin - 0.5) < 1e-10:
        op['id'] = np.eye(2)
        op['sx'] = np.zeros((2, 2))
        op['sy'] = np.zeros((2, 2), dtype=np.complex)
        op['sz'] = np.zeros((2, 2))
        op['su'] = np.zeros((2, 2))
        op['sd'] = np.zeros((2, 2))
        op['sx'][0, 1] = 0.5
        op['sx'][1, 0] = 0.5
        op['sy'][0, 1] = 0.5 * 1j
        op['sy'][1, 0] = -0.5 * 1j
        op['sz'][0, 0] = 0.5
        op['sz'][1, 1] = -0.5
        op['su'][0, 1] = 1
        op['sd'][1, 0] = 1
    return op

def hamiltonian_heisenberg(jx, jy, jz, hx, hz):
    op = spin_operators(0.5)
    hamilt = jx*np.kron(op['sx'], op['sx']) + jy*np.kron(op['sy'], op['sy']).real + jz*np.kron(op['sz'], op['sz'])
    hamilt += hx*(np.kron(np.eye(2), op['sx']) + np.kron(op['sx'], np.eye(2)))
    hamilt += hz*(np.kron(np.eye(2), op['sz']) + np.kron(op['sz'], np.eye(2)))
    return hamilt

def interactions_full_connection_two_body(l):
    # return the interactions of the fully connected two-body Hamiltonian
    # interact: [first_site, second_site]
    from scipy.special import comb
    ni = comb(l, 2)
    interact = np.zeros((int(ni), 2))
    n = 0
    for n1 in range(0, l):
        for n2 in range(n1+1, l):
            interact[n, :] = [n1, n2]
            n += 1
    return interact, ni

def positions_nearest_neighbor_1d(l, bound_cond='open'):
    # return the 1D Hamiltonian with nearest-neighbor interactions
    # index: [first_site, second_site]
    nh = l-1
    index = np.zeros((nh + (bound_cond == 'periodic'), 2))
    for n in range(0, nh):
        index[n, 0] = n
        index[n, 1] = n + 1
    if bound_cond == 'periodic':  # default: open boundary condition
        index[nh, 0] = 0
        index[nh, 1] = l - 1
    return index

def positions_nearest_neighbor_square(width, height, bound_cond='open'):
    index = np.zeros((width-1, 2))
    for i in range(0, width-1):  # interactions inside the first row
        index[i, :] = [i, i+1]
    for n in range(1, height):  # interactions inside the n-th row
        tmp = np.zeros((width-1, 2))
        for i in range(0, width-1):
            tmp[i, :] = [n*width + i, n*width + i + 1]
        index = np.vstack((index, tmp))
    for n in range(0, width):
        tmp = np.zeros((height-1, 2))
        for i in range(0, height-1):
            tmp[i, :] = [i*width + n, (i + 1)*width + n]
        index = np.vstack((index, tmp))
    if bound_cond == 'periodic':
        tmp = np.zeros((height, 2))
        for n in range(0, height):
            tmp[n, :] = [n*width, (n + 1)*width - 1]
        index = np.vstack((index, tmp))
        tmp = np.zeros((width, 2))
        for n in range(0, width):
            tmp[n, :] = [n, (height - 1)*width + n]
        index = np.vstack((index, tmp))
        index = index.astype(int)
    return index

def interactions_position2full_index_heisenberg_two_body(index_pos):
    # index: [first_site, second_site, first operator, second operator]
    # the operator is ordered as [id, sx, sy, sz, su, sd]
    nh = index_pos.shape[0]
    index = np.zeros((nh*3, 4))
    for n in range(0, nh):
        index[3 * n, 0] = index_pos[n, 0]
        index[3 * n, 1] = index_pos[n, 1]
        index[3 * n, 2] = 4  # su
        index[3 * n, 3] = 5  # sd
        index[3 * n + 1, 0] = index_pos[n, 0]
        index[3 * n + 1, 1] = index_pos[n, 1]
        index[3 * n + 1, 2] = 5  # sd
        index[3 * n + 1, 3] = 4  # su
        index[3 * n + 2, 0] = index_pos[n, 0]
        index[3 * n + 2, 1] = index_pos[n, 1]
        index[3 * n + 2, 2] = 3  # sz
        index[3 * n + 2, 3] = 3  # sz
        index = index.astype(int)
    return index


import pandas as pd
import os
from datetime import datetime
from pytz import timezone
import json
from application import db
from application.models import Hospital
from sqlalchemy import literal

tz = timezone('US/Eastern')

path = "C:\\Users\\Ron\\Hospifind\\data"
beds = os.path.join(path, "Hospital_BedsHospital1_crosstab.csv")
icus = os.path.join(path, "ICU_BedsHospital1_crosstab.csv")

if os.path.exists(beds) and os.path.exists(icus):
    beds = pd.read_csv(beds, delimiter='\t', encoding='utf-16')
    icus = pd.read_csv(icus, delimiter='\t', encoding='utf-16')

    hospitals = list(beds['ProviderName']); hospitals = [hospital.title() for hospital in hospitals]
    bed_capacity = list(beds['Total Staffed Bed Capacity']); bed_capacity = [int(i.replace(',', '')) if not isinstance(i, int) else i for i in bed_capacity]
    beds_available = list(beds['Available']); beds_available = [int(i.replace(',', '')) if not isinstance(i, int) else i for i in beds_available]
    beds_percent_available = list(beds['Available Capacity']); beds_percent_available = [float(i.replace('%', '')) if isinstance(i, str) else 0 for i in beds_percent_available]

    adult_icu_capacity = list(icus['Adult ICU Census']); adult_icu_capacity = [int(i.replace(',', '')) if not isinstance(i, int) else i for i in adult_icu_capacity]
    adult_icus_available = list(icus['Available Adult ICU']); adult_icus_available = [int(i.replace(',', '')) if not isinstance(i, int) else i for i in adult_icus_available]
    adult_icus_percent_available = list(icus['Available Adult ICU%']); adult_icus_percent_available = [float(i.replace('%', '')) if isinstance(i, str) else 0 for i in adult_icus_percent_available]

    count = 0
    for i, hospital in enumerate(hospitals):
        hospital_name = hospital
        if len(Hospital.query.filter_by(name=hospital).all()) > 0:
            hospital = Hospital.query.filter_by(name=hospital).first()
        elif len(db.session.query(Hospital).filter(literal(hospital).contains(Hospital.name)).all()) == 1:
            hospital = db.session.query(Hospital).filter(literal(hospital).contains(Hospital.name)).first()
        elif len(db.session.query(Hospital).filter(Hospital.name.like("%" + hospital + "%")).all()) == 1:
            hospital = db.session.query(Hospital).filter(Hospital.name.like("%" + hospital + "%")).first()
        else:
            continue
        if hospital.state != "FL":
            continue

        hospital.data = None
        count += 1
        if hospital.data is None:
            data = dict()
        else:
            data = json.loads(hospital.data)
        time = datetime.now(tz)
        time = time.timestamp() * 1000
        print(time)
        data[time] = dict()
        data[time]["Bed Capacity"] = bed_capacity[i]
        data[time]["Beds Available"] = beds_available[i]
        data[time]["Beds Available Percent"] = beds_percent_available[i]
        data[time]["Adult ICU Capacity"] = adult_icus_percent_available[i]
        data[time]["Adult ICUs Available"] = adult_icus_available[i]
        data[time]["Adult ICUs Available Percent"] = adult_icus_percent_available[i]

        data_json = json.dumps(data)
        # print(data_json)
        hospital.data = data_json

    db.session.commit()
    # print(time)
    print(count, len(hospitals))


# Starting with 1 and spiralling anticlockwise in the following way,
# a square spiral with side length 7 is formed.
# 37 36 35 34 33 32 31
# 38 17 16 15 14 13 30
# 39 18  5  4  3 12 29
# 40 19  6  1  2 11 28
# 41 20  7  8  9 10 27
# 42 21 22 23 24 25 26
# 43 44 45 46 47 48 49
# It is interesting to note that the odd squares lie along
# the bottom right diagonal,
# but what is more interesting is that 8 out of the 13 numbers
# lying along both diagonals are prime; that is, a ratio of 8/13 ≈ 62%.

# If one complete new layer is wrapped around the spiral above,
# a square spiral with side length 9 will be formed.
# If this process is continued, what is the side length of the square
# spiral for which the ratio of primes along both diagonals
#     first falls below 10%?

import numpy as np
import utils
sidelengthsmin = 1
sidelengthsmax = 20000
primes = []
maxlast = 1
numdiagprimes = 0
numdiags = 1
sidelength = 2*sidelengthsmax+1
numnums = sum([4*i-4 for i in range(3, sidelength+1, 2)])+1
primes = utils.SieveOfEratosthenes(8000, primes)
for sidelength in range(sidelengthsmin, sidelengthsmax):
    sidelength = 2*sidelength+1
    numnewnums = np.arange(4*sidelength-4)+maxlast+1
    maxlast = numnewnums[-1]
    diags = [numnewnums[sidelength-2], numnewnums[2*sidelength-3],
             numnewnums[3*sidelength-4], numnewnums[-1]]
    numdiags += len(diags)
    if max(diags) > max(primes):
        numdiagprimes += len([d for d in diags if utils.isprime(d)])
    else:
        numdiagprimes += len([d for d in diags if d in primes])
    if (numdiagprimes/numdiags) < 0.1:
        print(3, sidelength)
        break


from typing import Any, Dict, List, Type, TypeVar, Union

import attr

from ..models.subscription_item_status import SubscriptionItemStatus
from ..models.subscription_item_tax_type import SubscriptionItemTaxType
from ..types import UNSET, Unset

T = TypeVar("T", bound="SubscriptionItem")

@attr.s(auto_attribs=True)
class SubscriptionItem:
    """
    Attributes:
        subscription_item_id (int): unique id of the subscription item in the context of a company
        product_id (str): id of the product bought
        title (str): product title presented to the customer
        price (float): price of the subscription
        discount_percentage (int): discount offered to the subscription
        quantity (int): quantity of purchased entities
        plenigo_product_id (Union[Unset, str]): if the product is based on a plenigo offer the plenigo product id is
            provided here - can be identically to the productId
        plenigo_step_id (Union[Unset, str]): if the product is based on a plenigo offer the plenigo step id is provided
            here
        access_right_unique_id (Union[Unset, str]): unique id of the access right this subscription item grants access
            to
        internal_title (Union[Unset, str]): if the product is based on a plenigo offer the product title for internal
            usage is provided here
        tax_type (Union[Unset, SubscriptionItemTaxType]): unique identification of tax type the product represents -
            important for tax handling purposes
        package_title (Union[Unset, str]): if subscription item is correlated to another subscription item in a way that
            both items are presented as one (bundle) this field contains the correlation title
        package_id (Union[Unset, str]): if subscription item is correlated to another subscription item in a way that
            both items are presented as one (bundle) this field contains the correlation id - the id is only unique within a
            subscription
        package_cancellation_locked (Union[Unset, bool]): flag indicating if package elements can only be cancelled
            together
        price_issue_id (Union[Unset, int]): id of the price issue the subscription item's price is based on
        credit_count (Union[Unset, int]): available credit count to use
        credit_wallet_unique_id (Union[Unset, str]): the credit wallet unique id
        status (Union[Unset, SubscriptionItemStatus]): current status of the subscription item
        cost_center (Union[Unset, str]): cost center associated with this subscriptino item
    """

    subscription_item_id: int
    product_id: str
    title: str
    price: float
    discount_percentage: int
    quantity: int
    plenigo_product_id: Union[Unset, str] = UNSET
    plenigo_step_id: Union[Unset, str] = UNSET
    access_right_unique_id: Union[Unset, str] = UNSET
    internal_title: Union[Unset, str] = UNSET
    tax_type: Union[Unset, SubscriptionItemTaxType] = UNSET
    package_title: Union[Unset, str] = UNSET
    package_id: Union[Unset, str] = UNSET
    package_cancellation_locked: Union[Unset, bool] = UNSET
    price_issue_id: Union[Unset, int] = UNSET
    credit_count: Union[Unset, int] = UNSET
    credit_wallet_unique_id: Union[Unset, str] = UNSET
    status: Union[Unset, SubscriptionItemStatus] = UNSET
    cost_center: Union[Unset, str] = UNSET
    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        subscription_item_id = self.subscription_item_id
        product_id = self.product_id
        title = self.title
        price = self.price
        discount_percentage = self.discount_percentage
        quantity = self.quantity
        plenigo_product_id = self.plenigo_product_id
        plenigo_step_id = self.plenigo_step_id
        access_right_unique_id = self.access_right_unique_id
        internal_title = self.internal_title
        tax_type: Union[Unset, str] = UNSET
        if not isinstance(self.tax_type, Unset):
            tax_type = self.tax_type.value

        package_title = self.package_title
        package_id = self.package_id
        package_cancellation_locked = self.package_cancellation_locked
        price_issue_id = self.price_issue_id
        credit_count = self.credit_count
        credit_wallet_unique_id = self.credit_wallet_unique_id
        status: Union[Unset, str] = UNSET
        if not isinstance(self.status, Unset):
            status = self.status.value

        cost_center = self.cost_center

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "subscriptionItemId": subscription_item_id,
                "productId": product_id,
                "title": title,
                "price": price,
                "discountPercentage": discount_percentage,
                "quantity": quantity,
            }
        )
        if plenigo_product_id is not UNSET:
            field_dict["plenigoProductId"] = plenigo_product_id
        if plenigo_step_id is not UNSET:
            field_dict["plenigoStepId"] = plenigo_step_id
        if access_right_unique_id is not UNSET:
            field_dict["accessRightUniqueId"] = access_right_unique_id
        if internal_title is not UNSET:
            field_dict["internalTitle"] = internal_title
        if tax_type is not UNSET:
            field_dict["taxType"] = tax_type
        if package_title is not UNSET:
            field_dict["packageTitle"] = package_title
        if package_id is not UNSET:
            field_dict["packageId"] = package_id
        if package_cancellation_locked is not UNSET:
            field_dict["packageCancellationLocked"] = package_cancellation_locked
        if price_issue_id is not UNSET:
            field_dict["priceIssueId"] = price_issue_id
        if credit_count is not UNSET:
            field_dict["creditCount"] = credit_count
        if credit_wallet_unique_id is not UNSET:
            field_dict["creditWalletUniqueId"] = credit_wallet_unique_id
        if status is not UNSET:
            field_dict["status"] = status
        if cost_center is not UNSET:
            field_dict["costCenter"] = cost_center

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        subscription_item_id = d.pop("subscriptionItemId")

        product_id = d.pop("productId")

        title = d.pop("title")

        price = d.pop("price")

        discount_percentage = d.pop("discountPercentage")

        quantity = d.pop("quantity")

        plenigo_product_id = d.pop("plenigoProductId", UNSET)

        plenigo_step_id = d.pop("plenigoStepId", UNSET)

        access_right_unique_id = d.pop("accessRightUniqueId", UNSET)

        internal_title = d.pop("internalTitle", UNSET)

        _tax_type = d.pop("taxType", UNSET)
        tax_type: Union[Unset, SubscriptionItemTaxType]
        if isinstance(_tax_type, Unset):
            tax_type = UNSET
        else:
            tax_type = SubscriptionItemTaxType(_tax_type)

        package_title = d.pop("packageTitle", UNSET)

        package_id = d.pop("packageId", UNSET)

        package_cancellation_locked = d.pop("packageCancellationLocked", UNSET)

        price_issue_id = d.pop("priceIssueId", UNSET)

        credit_count = d.pop("creditCount", UNSET)

        credit_wallet_unique_id = d.pop("creditWalletUniqueId", UNSET)

        _status = d.pop("status", UNSET)
        status: Union[Unset, SubscriptionItemStatus]
        if isinstance(_status, Unset):
            status = UNSET
        else:
            status = SubscriptionItemStatus(_status)

        cost_center = d.pop("costCenter", UNSET)

        subscription_item = cls(
            subscription_item_id=subscription_item_id,
            product_id=product_id,
            title=title,
            price=price,
            discount_percentage=discount_percentage,
            quantity=quantity,
            plenigo_product_id=plenigo_product_id,
            plenigo_step_id=plenigo_step_id,
            access_right_unique_id=access_right_unique_id,
            internal_title=internal_title,
            tax_type=tax_type,
            package_title=package_title,
            package_id=package_id,
            package_cancellation_locked=package_cancellation_locked,
            price_issue_id=price_issue_id,
            credit_count=credit_count,
            credit_wallet_unique_id=credit_wallet_unique_id,
            status=status,
            cost_center=cost_center,
        )

        subscription_item.additional_properties = d
        return subscription_item

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties