"""
Daemon to collect elasticsearch bulk requests from kafka,
run them against relforge, and send the results back over
kafka.
"""

from argparse import ArgumentParser
from typing import Callable

import mjolnir.kafka.msearch_daemon

def configure(parser: ArgumentParser) -> Callable:
    parser.add_argument(
        '-b', '--brokers', dest='brokers', required=True, type=lambda x: x.split(','),
        help='Kafka brokers to bootstrap from as a comma separated list of <host>:<port>')
    parser.add_argument(
        '-m', '--max-request-size', dest='max_request_size', type=int, default=4*1024*1024*10,
        help='Max size of requsets sent to the kafka broker'
             + 'Defaults to 40MB.')
    parser.add_argument(
        '-w', '--num-workers', dest='n_workers', type=int, default=1,
        help='Number of workers to issue elasticsearch queries in parallel. '
             + 'Defaults to 1.')
    parser.add_argument(
        '--prometheus-port', dest='prometheus_port', default=9171, type=int, required=False,
        help='Port to export prometheus metrics over.')
    parser.add_argument(
        '--input-topic', dest='topic_work', default=mjolnir.kafka.TOPIC_REQUEST,
        help='Topic to read msearch requests from')
    parser.add_argument(
        '--output-topic', dest='topic_result', default=mjolnir.kafka.TOPIC_RESULT,
        help='Topic to write msearch responses to')
    parser.add_argument(
        '--group-id', dest='group_id', default='mjolnir_msearch',
        help='Kafka consumer group to join')
    parser.add_argument(
        '--max-concurrent-searches', dest='max_concurrent_searches', type=int, default=1,
        help='Maximum number of queries in a single msearch request that will run in parallel')
    parser.add_argument(
        '--elastic-endpoint', dest='endpoint', type=str, default='http://localhost:9200',
        help='Elasticsearch endpoint to perform requests against')
    return main

def main(**kwargs):
    mjolnir.kafka.msearch_daemon.Daemon(**kwargs).run()


import csv
import requests
import unicodedata

from argparse import ArgumentParser

from settings import settings

INSERTED = []
FILE = settings['csv']['reader_file']
RESULT = settings['csv']['writer_file']
API_KEY = settings['geocode']['key']
BASE_URL = settings['geocode']['base_url']
ADDRESS = settings['geocode']['address']
LATLNG = settings['geocode']['latlng']
FIELDNAMES = settings['csv']['fieldnames']
COUNTRIES = settings['geocode']['countries']

def slugify(s):
    if s is None:
        return None
    s = '-'.join(s.split())
    replace_acutes_gen = [c for c in unicodedata.normalize('NFD', s)
                          if unicodedata.category(c) != 'Mn']
    return ''.join(replace_acutes_gen).lower()

def get_center(lang, place, country):
    search = '{}-{}'.format(place, country)
    url = '{}?{}'.format(BASE_URL, ADDRESS.format(lang, API_KEY, search))
    response = requests.get(url)
    if response.status_code == 200:
        result = response.json()
        latitude = result['results'][0]['geometry']['location']['lat']
        longitude = result['results'][0]['geometry']['location']['lng']
        return latitude, longitude

    return None, None

def get_location_tree(lang, lat, lng):
    LEVELS = [
        'continent',
        'country',
        'administrative_area_level_1',
        'administrative_area_level_2',
        'locality'
    ]

    location_tree = []

    for level in LEVELS:
        query_url = LATLNG.format(lang, API_KEY, level, lat, lng)
        url = '{}?{}'.format(BASE_URL, query_url)
        response = requests.get(url)
        if response.status_code == 200:
            result = response.json()
            if len(result['results']):
                loc = result['results'][0]
                geometry = loc['geometry']
                name = loc['address_components'][0]
                location_tree.append({
                    'name': name['long_name'],
                    'code': name['short_name'],
                    'slug': slugify(name['long_name']),
                    'type': level,
                    'place_id': loc['place_id'],
                    'latitude': geometry['location']['lat'],
                    'longitude': geometry['location']['lng'],
                    'latitudeSW': geometry['bounds']['southwest']['lat'],
                    'longitudeSW': geometry['bounds']['southwest']['lng'],
                    'latitudeNE': geometry['bounds']['northeast']['lat'],
                    'longitudeNE': geometry['bounds']['northeast']['lng']
                })

    return location_tree

def print_locations(writer, locations):

    def get_level_1(type, name):
        enabled = [
            'administrative_area_level_1',
            'administrative_area_level_2',
            'locality'
        ]
        return name if type in enabled else None

    def get_level_2(type, name):
        enabled = [
            'administrative_area_level_2',
            'locality'
        ]
        return name if type in enabled else None

    def get_locality(type, name):
        return name if type == 'locality' else None

    continent = None
    country = None
    level_1 = None
    level_2 = None
    locality = None

    for location in locations:
        if location['type'] == 'country':
            continent = COUNTRIES[location['code']]
            country = location['name']
        elif location['type'] == 'administrative_area_level_1':
            level_1 = location['name']
        elif location['type'] == 'administrative_area_level_2':
            level_2 = location['name']
        else: locality = location['name']

    for location in locations:
        if location['place_id'] not in INSERTED:
            INSERTED.append(location['place_id'])
            type = location['type']
            row = {
                'Continent': continent,
                'Country': country,
                'Administrative Area Level 1': get_level_1(type, level_1),
                'Administrative Area Level 2': get_level_2(type, level_2),
                'City': get_locality(type, locality),
                'Slug': location['slug'],
                'Place id': location['place_id'],
                'Latitude': location['latitude'],
                'Longitude': location['longitude'],
                'LatitudeSW': location['latitudeSW'],
                'LongitudeSW': location['longitudeSW'],
                'LatitudeNE': location['latitudeNE'],
                'LongitudeNE': location['longitudeNE']
            }

            writer.writerow(row)
            print('Parsed {} : {}'.format(type, location['name']))

def main(lang):
    with open(FILE) as reader_file:
        reader = csv.DictReader(reader_file)
        with open(RESULT, 'w') as writer_file:
            writer = csv.DictWriter(writer_file, fieldnames=FIELDNAMES)
            writer.writeheader()
            for row in reader:
                lat, lng = get_center(lang, row['Place'], row['Country'])
                locations = get_location_tree(lang, lat, lng)
                print_locations(writer, locations)

if __name__ == '__main__':
    parser = ArgumentParser(description='Parse locations')
    parser.add_argument('-l',
                        '--language',
                        help='Language to retrieve location names',
                        required=True,
                        type=str)

    args = parser.parse_args()
    lang = args.language
    main(lang)


from twisted.internet.defer import inlineCallbacks

from conductor_device.conductor_parameter import ConductorParameter

class WmFrequency(ConductorParameter):
    priority = 1
    center_frequency = 368.55468e12
    frequency_span = 200e6

    @inlineCallbacks
    def initialize(self):
        yield self.connect()

    @inlineCallbacks
    def update(self):
        response = yield self.cxn.yesr8_hfwm.get_frequency(1)
        if response is None:
            return

        frequency = float(response['Hz'])
        self.value = frequency

        min_frequency = self.center_frequency - self.frequency_span / 2
        max_frequency = self.center_frequency + self.frequency_span / 2
        if frequency != sorted([min_frequency, frequency, max_frequency])[1]:
            yield self.cxn.beeper.beep('m2 verdi mode hop')


import collections
import random

from .. import base
from .. import preprocessing

Limits = collections.namedtuple('Limits', 'lower upper')

Split = collections.namedtuple('Split', 'feature value')

class Node:

    def __init__(self, depth, size_limit, split=None, left=None, right=None):
        self.split = split
        self.size_limit = size_limit
        self.depth = depth
        self.left = left
        self.right = right
        self.r_mass = 0
        self.l_mass = 0

    @property
    def is_terminal(self):
        return self.left is None

    def next(self, x):
        if x[self.split.feature] < self.split.value:
            return self.left
        return self.right

    def update(self, x):

        # Increment the mass
        self.l_mass += 1

        # Update the next node
        if not self.is_terminal:
            self.next(x).update(x)

    def score(self, x):
        if self.r_mass < self.size_limit or self.is_terminal:
            return self.r_mass * 2 ** self.depth
        return self.next(x).score(x)

    def pivot_masses(self):
        self.r_mass = self.l_mass
        self.l_mass = 0

        if not self.is_terminal:
            self.left.pivot_masses()
            self.right.pivot_masses()

    def __str__(self):
        s = '\t' * self.depth + f'r_mass={self.r_mass}, l_mass={self.l_mass}'

        if self.is_terminal:
            return s

        return f'{s} ({self.split.feature} < {self.split.value:.3f})\n{self.left}\n{self.right}'

def make_limits(rng):
    sq = rng.random()
    return Limits(lower=sq - 2 * max(sq, 1 - sq), upper=sq + 2 * max(sq, 1 - sq))

def make_tree(limits, tree_height, depth, size_limit, rng):
    """Returns a half-space tree with splits occurring between 0 and 1.

    Parameters:
        limits (dict)
        max_depth (int): The desired tree height.
        depth (int)
        size_limit (int)

    """

    if depth == tree_height:
        return Node(depth=depth, size_limit=size_limit)

    # Randomly pick a feature and find the center of it's limits
    feature = rng.choice(list(limits.keys()))
    center = (limits[feature].lower + limits[feature].upper) / 2

    # Build the left node
    tmp = limits[feature]
    limits[feature] = Limits(lower=tmp.lower, upper=center)
    left = make_tree(
        limits=limits,
        tree_height=tree_height,
        depth=depth + 1,
        size_limit=size_limit,
        rng=rng
    )
    limits[feature] = tmp

    # Build the right node
    tmp = limits[feature]
    limits[feature] = Limits(lower=center, upper=tmp.upper)
    right = make_tree(
        limits=limits,
        tree_height=tree_height,
        depth=depth + 1,
        size_limit=size_limit,
        rng=rng
    )
    limits[feature] = tmp

    return Node(
        depth=depth,
        split=Split(feature=feature, value=center),
        left=left,
        right=right,
        size_limit=size_limit
    )

class HalfSpaceTrees(base.OutlierDetector):
    """Half-Space Trees (HST).

    Half-space trees are an online variant of isolation forests. They work well when anomalies are
    spread out in time. They do not work well if anomalies are packed together in windows of time,
    such as in the KDD'99 HTTP and SMTP datasets.

    Parameters:
        n_trees (int): Number of trees to use.
        tree_height (int): Height of each tree.
        window_size (int): Number of observations to use for calculating the mass at each node in
            each tree.
        scale (bool): Whether or not to scale features between 0 and 1. Only set to ``False`` if
            you know that all your features are already contained between 0 and 1.
        seed (int): Random number seed.

    Example:

        ::

            >>> from creme import anomaly

            >>> X = [0.5, 0.45, 0.43, 0.44, 0.445, 0.45, 0.0]
            >>> hst = anomaly.HalfSpaceTrees(
            ...     n_trees=5,
            ...     tree_height=3,
            ...     window_size=3,
            ...     scale=False,
            ...     seed=42
            ... )

            >>> for x in X[:3]:
            ...     hst = hst.fit_one({'x': x})  # Warming up

            >>> for x in X:
            ...     features = {'x': x}
            ...     hst = hst.fit_one(features)
            ...     print(f'Anomaly score for {x:.3f}: {hst.score_one(features)}')
            Anomaly score for 0.500: 120
            Anomaly score for 0.450: 120
            Anomaly score for 0.430: 120
            Anomaly score for 0.440: 120
            Anomaly score for 0.445: 120
            Anomaly score for 0.450: 120
            Anomaly score for 0.000: 0

    References:
        1. `Fast Anomaly Detection for Streaming Data <https://www.ijcai.org/Proceedings/11/Papers/254.pdf>`_

    """

    def __init__(self, n_trees=25, tree_height=15, window_size=250, scale=True, seed=None):
        self.n_trees = n_trees
        self.window_size = window_size
        self.tree_height = tree_height
        self.scale = scale
        self.rng = random.Random(seed)
        self.trees = []
        self.min_max_scaler = preprocessing.MinMaxScaler()
        self.counter = 0

    def fit_one(self, x):

        # Scale the features between 0 and 1
        if self.scale:
            x = self.min_max_scaler.transform_one(x)
            self.min_max_scaler.fit_one(x)

        # The trees are built when the first observation comes in
        if not self.trees:
            self.trees = [
                make_tree(
                    limits={f: make_limits(rng=self.rng) for f in x},
                    tree_height=self.tree_height,
                    depth=0,
                    size_limit=.1 * self.window_size,
                    rng=self.rng
                )
                for _ in range(self.n_trees)
            ]

        # Update each tree
        for tree in self.trees:
            tree.update(x)

        # Pivot the masses if necessary
        self.counter += 1
        if self.counter == self.window_size:
            for tree in self.trees:
                tree.pivot_masses()
            self.counter = 0

        return self

    def score_one(self, x):
        if not self.trees:
            return 0
        return sum(tree.score(x) for tree in self.trees)


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
from typing import Optional

from airflow.kubernetes.volume import Volume
from airflow.kubernetes.volume_mount import VolumeMount
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from airflow.operators.dummy_operator import DummyOperator

from liminal.runners.airflow.config.standalone_variable_backend import get_variable
from liminal.runners.airflow.model import task

_LOG = logging.getLogger(__name__)

class PythonTask(task.Task):
    """
    Python task.
    """

    def __init__(self, dag, liminal_config, pipeline_config, task_config, parent, trigger_rule):
        super().__init__(dag, liminal_config, pipeline_config, task_config, parent, trigger_rule)
        self.task_name = self.task_config['task']
        self.image = self.task_config['image']
        self.volumes = self._volumes()
        self.mounts = self.task_config.get('mounts', [])
        self.resources = self.__kubernetes_resources()
        self.env_vars = self.__env_vars()
        self.kubernetes_kwargs = self.__kubernetes_kwargs()
        self.cmds, self.arguments = self.__kubernetes_cmds_and_arguments()
        self.executors = self.__executors()

    def apply_task_to_dag(self):
        if self.executors == 1:
            return self.__apply_task_to_dag_single_executor()
        else:
            return self.__apply_task_to_dag_multiple_executors()

    def _volumes(self):
        volumes_config = self.liminal_config.get('volumes', [])
        volumes = []
        for volume_config in volumes_config:
            name = volume_config['volume']
            volume = Volume(
                name=name,
                configs={
                    'persistentVolumeClaim': {
                        'claimName': f"{name}-pvc"
                    }
                }
            )
            volumes.append(volume)
        return volumes

    def __apply_task_to_dag_multiple_executors(self):
        start_task = DummyOperator(
            task_id=f'{self.task_name}_parallelize',
            trigger_rule=self.trigger_rule,
            dag=self.dag
        )

        end_task = DummyOperator(
            task_id=self.task_name,
            dag=self.dag
        )

        if self.parent:
            self.parent.set_downstream(start_task)

            for i in range(self.executors):
                split_task = self.__create_pod_operator(
                    image=self.image,
                    task_id=i
                )

                start_task.set_downstream(split_task)

                split_task.set_downstream(end_task)

        return end_task

    def __create_pod_operator(self, image: str, task_id: Optional[int] = None):
        env_vars = self.env_vars

        if task_id is not None:
            env_vars = self.env_vars.copy()
            env_vars['LIMINAL_SPLIT_ID'] = str(task_id)
            env_vars['LIMINAL_NUM_SPLITS'] = str(self.executors)

        return KubernetesPodOperator(
            task_id=f'{self.task_name}_{task_id}' if task_id is not None else self.task_name,
            image=image,
            cmds=self.cmds,
            arguments=self.arguments,
            env_vars=env_vars,
            **self.kubernetes_kwargs
        )

    def __apply_task_to_dag_single_executor(self):
        pod_task = self.__create_pod_operator(image=f'''{self.image}''')

        first_task = pod_task

        if self.parent:
            self.parent.set_downstream(first_task)

        return pod_task

    def __executors(self) -> int:
        executors = 1

        if 'executors' in self.task_config:
            executors = self.task_config['executors']

        return executors

    def __kubernetes_cmds_and_arguments(self):
        cmds = ['/bin/bash', '-c']
        arguments = [self.task_config['cmd']]
        return cmds, arguments

    def __kubernetes_kwargs(self):
        kubernetes_kwargs = {
            'namespace': get_variable('kubernetes_namespace', default_val='default'),
            'name': self.task_name.replace('_', '-'),
            'in_cluster': get_variable('in_kubernetes_cluster', default_val=False),
            'image_pull_policy': get_variable('image_pull_policy', default_val='IfNotPresent'),
            'get_logs': True,
            'is_delete_operator_pod': True,
            'startup_timeout_seconds': 300,
            'image_pull_secrets': 'regcred',
            'resources': self.resources,
            'dag': self.dag,
            'volumes': self.volumes,
            'volume_mounts': [
                VolumeMount(mount['volume'],
                            mount['path'],
                            mount.get('sub_path'),
                            mount.get('read_only', False))
                for mount
                in self.mounts
            ]
        }
        return kubernetes_kwargs

    def __env_vars(self):
        return dict([(k, str(v)) for k, v in self.task_config.get('env_vars', {}).items()])

    def __kubernetes_resources(self):
        resources = {}

        if 'request_cpu' in self.task_config:
            resources['request_cpu'] = self.task_config['request_cpu']
        if 'request_memory' in self.task_config:
            resources['request_memory'] = self.task_config['request_memory']
        if 'limit_cpu' in self.task_config:
            resources['limit_cpu'] = self.task_config['limit_cpu']
        if 'limit_memory' in self.task_config:
            resources['limit_memory'] = self.task_config['limit_memory']

        return resources


from typing import Union, List

import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import lines
from matplotlib.font_manager import FontProperties
from scipy import stats
from seaborn.utils import remove_na

from statannot.StatResult import StatResult
from statannot.comparisons_corrections.ComparisonsCorrection import ComparisonsCorrection
from statannot.comparisons_corrections.utils import assert_valid_correction_name
from statannot.utils import assert_is_in

DEFAULT = object()

def stat_test(
        box_data1,
        box_data2,
        test_name,
        comparisons_correction=None,
        num_comparisons=1,
        verbose=1,
        **stats_params
):
    """Get formatted result of two sample statistical test.

    Arguments
    ---------
    box_data1, box_data2
    test_name: str
        Statistical test to run. Must be one of:
        - `Levene`
        - `Mann-Whitney`
        - `Mann-Whitney-gt`
        - `Mann-Whitney-ls`
        - `t-test_ind`
        - `t-test_welch`
        - `t-test_paired`
        - `Wilcoxon`
        - `Kruskal`
    comparisons_correction: ComparisonCorrection (type 0) or None, default None
        Method to use for multiple comparisons correction.
    num_comparisons: int, default 1
        Number of comparisons to use for multiple comparisons correction.
    verbose: int
        Verbosity parameter, see add_stat_annotation
    stats_params
        Additional keyword arguments to pass to scipy stats functions.

    Returns
    -------
    StatResult object with formatted result of test.

    """
    # Check arguments.
    if comparisons_correction is not None:
        if not isinstance(comparisons_correction, ComparisonsCorrection):
            raise ValueError("comparisons_correction must be a "
                             "ComparisonsCorrection instance")

    # Switch to run scipy.stats hypothesis test.
    if test_name == 'Levene':
        stat, pval = stats.levene(box_data1, box_data2, **stats_params)
        result = StatResult(
            'Levene test of variance', 'levene', 'stat_value', stat, pval
        )
    elif test_name == 'Mann-Whitney':
        u_stat, pval = stats.mannwhitneyu(
            box_data1, box_data2, alternative='two-sided', **stats_params
        )
        result = StatResult(
            'Mann-Whitney-Wilcoxon test two-sided',
            'M.W.W.',
            'U_stat',
            u_stat,
            pval,
        )
    elif test_name == 'Mann-Whitney-gt':
        u_stat, pval = stats.mannwhitneyu(
            box_data1, box_data2, alternative='greater', **stats_params
        )
        result = StatResult(
            'Mann-Whitney-Wilcoxon test greater',
            'M.W.W.',
            'U_stat',
            u_stat,
            pval,
        )
    elif test_name == 'Mann-Whitney-ls':
        u_stat, pval = stats.mannwhitneyu(
            box_data1, box_data2, alternative='less', **stats_params
        )
        result = StatResult(
            'Mann-Whitney-Wilcoxon test smaller',
            'M.W.W.',
            'U_stat',
            u_stat,
            pval,
        )
    elif test_name == 't-test_ind':
        stat, pval = stats.ttest_ind(a=box_data1, b=box_data2, **stats_params)
        result = StatResult(
            't-test independent samples', 't-test_ind', 'stat_value', stat, pval
        )
    elif test_name == 't-test_welch':
        stat, pval = stats.ttest_ind(
            a=box_data1, b=box_data2, equal_var=False, **stats_params
        )
        result = StatResult(
            'Welch\'s t-test independent samples',
            't-test_welch',
            'stat_value',
            stat,
            pval,
        )
    elif test_name == 't-test_paired':
        stat, pval = stats.ttest_rel(a=box_data1, b=box_data2, **stats_params)
        result = StatResult(
            't-test paired samples', 't-test_rel', 'stat_value', stat, pval
        )
    elif test_name == 'Wilcoxon':
        zero_method_default = len(box_data1) <= 20 and "pratt" or "wilcox"
        zero_method = stats_params.get('zero_method', zero_method_default)
        if verbose >= 1:
            print("Using zero_method ", zero_method)
        stat, pval = stats.wilcoxon(
            box_data1, box_data2, zero_method=zero_method, **stats_params
        )
        result = StatResult(
            'Wilcoxon test (paired samples)', 'Wilcoxon', 'stat_value', stat, pval
        )
    elif test_name == 'Kruskal':
        stat, pval = stats.kruskal(box_data1, box_data2, **stats_params)
        result = StatResult(
            'Kruskal-Wallis paired samples', 'Kruskal', 'stat_value', stat, pval
        )
    else:
        result = StatResult(None, '', None, None, np.nan)

    # Optionally, run multiple comparisons correction
    if comparisons_correction is not None and comparisons_correction.type == 0:
        result.pval = comparisons_correction(result.pval, num_comparisons)
        result.correction_method = comparisons_correction.name

    return result

def pval_annotation_text(result: Union[List[StatResult], StatResult], pvalue_thresholds):
    single_value = False

    if isinstance(result, list):
        x1_pval = np.array([res.pval for res in result])
        x1_corr_signif = [res.corrected_significance for res in result]
    else:
        x1_pval = np.array([result.pval])
        x1_corr_signif = [result.corrected_significance]
        single_value = True

    # Sort the threshold array
    pvalue_thresholds = pd.DataFrame(pvalue_thresholds).sort_values(by=0, ascending=False).values
    x_annot = pd.Series(["" for _ in range(len(x1_pval))])

    for i in range(0, len(pvalue_thresholds)):

        if i < len(pvalue_thresholds) - 1:
            condition = (x1_pval <= pvalue_thresholds[i][0]) & (pvalue_thresholds[i + 1][0] < x1_pval)
            x_annot[condition] = pvalue_thresholds[i][1]

        else:
            condition = x1_pval < pvalue_thresholds[i][0]
            x_annot[condition] = pvalue_thresholds[i][1]

    x_annot = pd.Series(["ns" if corr_signif is False else star
                         for star, corr_signif in zip(x_annot, x1_corr_signif)])

    return x_annot if not single_value else x_annot.iloc[0]

def simple_text(result: StatResult, pvalue_format, pvalue_thresholds, test_short_name=None):
    """
    Generates simple text for test name and pvalue
    :param result: StatResult instance
    :param pvalue_format: format string for pvalue
    :param test_short_name: Short name of test to show
    :param pvalue_thresholds: String to display per pvalue range
    :return: simple annotation
    """
    # Sort thresholds
    thresholds = sorted(pvalue_thresholds, key=lambda x: x[0])

    # Test name if passed
    text = test_short_name and test_short_name + " " or ""

    for threshold in thresholds:
        if result.pval < threshold[0]:
            pval_text = "p ≤ {}".format(threshold[1])
            break
    else:
        pval_text = "p = {}".format(pvalue_format).format(result.pval)

    return text + pval_text + result.significance_suffix

def add_stat_annotation(ax, plot='boxplot',
                        data=None, x=None, y=None, hue=None, units=None, order=None,
                        hue_order=None, box_pairs=None, width=0.8,
                        perform_stat_test=True,
                        pvalues=None, test_short_name=None,
                        test=None, text_format='star', pvalue_format_string=DEFAULT,
                        text_annot_custom=None,
                        loc='inside', show_test_name=True,
                        pvalue_thresholds=DEFAULT, stats_params: dict = None,
                        comparisons_correction='bonferroni', num_comparisons='auto',
                        use_fixed_offset=False, line_offset_to_box=None,
                        line_offset=None, line_height=0.02, text_offset=1,
                        color='0.2', linewidth=1.5,
                        fontsize='medium', verbose=1):
    """
    Optionally computes statistical test between pairs of data series, and add statistical annotation on top
    of the boxes/bars. The same exact arguments `data`, `x`, `y`, `hue`, `order`, `width`,
    `hue_order` (and `units`) as in the seaborn boxplot/barplot function must be passed to this function.

    This function works in one of the two following modes:
    a) `perform_stat_test` is True: statistical test as given by argument `test` is performed.
       The `test_short_name` argument can be used to customize what appears before the pvalues
    b) `perform_stat_test` is False: no statistical test is performed, list of custom p-values `pvalues` are
       used for each pair of boxes. The `test_short_name` argument is then used as the name of the
       custom statistical test.

    :param plot: type of the plot, one of 'boxplot' or 'barplot'.
    :param line_height: in axes fraction coordinates
    :param text_offset: in points
    :param box_pairs: can be of either form: For non-grouped boxplot: `[(cat1, cat2), (cat3, cat4)]`. For boxplot grouped by hue: `[((cat1, hue1), (cat2, hue2)), ((cat3, hue3), (cat4, hue4))]`
    :param pvalue_format_string: defaults to `"{.3e}"`
    :param pvalue_thresholds: list of lists, or tuples. Default is: For "star" text_format: `[[1e-4, "****"], [1e-3, "***"], [1e-2, "**"], [0.05, "*"], [1, "ns"]]`. For "simple" text_format : `[[1e-5, "1e-5"], [1e-4, "1e-4"], [1e-3, "0.001"], [1e-2, "0.01"]]`
    :param pvalues: list or array of p-values for each box pair comparison.
    :param stats_params: Parameters for statistical test functions.
    :param comparisons_correction: Method for multiple comparisons correction.
        One of `statsmodel` `multipletests` methods (w/ default FWER), or a ComparisonCorrection instance.
    :param num_comparisons: Override number of comparisons otherwise calculated with number of box_pairs
    """

    def find_x_position_box(b_plotter, boxName):
        """
        boxName can be either a name "cat" or a tuple ("cat", "hue")
        """
        if b_plotter.plot_hues is None:
            cat = boxName
            hue_offset = 0
        else:
            cat = boxName[0]
            hue_level = boxName[1]
            hue_offset = b_plotter.hue_offsets[
                b_plotter.hue_names.index(hue_level)]

        group_pos = b_plotter.group_names.index(cat)
        box_pos = group_pos + hue_offset
        return box_pos

    def get_box_data(b_plotter, boxName):
        """
        boxName can be either a name "cat" or a tuple ("cat", "hue")

        Here we really have to duplicate seaborn code, because there is not
        direct access to the box_data in the BoxPlotter class.
        """
        cat = b_plotter.plot_hues is None and boxName or boxName[0]

        index = b_plotter.group_names.index(cat)
        group_data = b_plotter.plot_data[index]

        if b_plotter.plot_hues is None:
            # Draw a single box or a set of boxes
            # with a single level of grouping
            box_data = remove_na(group_data)
        else:
            hue_level = boxName[1]
            hue_mask = b_plotter.plot_hues[index] == hue_level
            box_data = remove_na(group_data[hue_mask])

        return box_data

    # Set default values if necessary
    if pvalue_format_string is DEFAULT:
        pvalue_format_string = '{:.3e}'
        simple_format_string = '{:.2f}'
    else:
        simple_format_string = pvalue_format_string

    if pvalue_thresholds is DEFAULT:
        if text_format == "star":
            pvalue_thresholds = [[1e-4, "****"], [1e-3, "***"],
                                 [1e-2, "**"], [0.05, "*"], [1, "ns"]]
        else:
            pvalue_thresholds = [[1e-5, "1e-5"], [1e-4, "1e-4"],
                                 [1e-3, "0.001"], [1e-2, "0.01"]]

    if stats_params is None:
        stats_params = dict()

    fig = plt.gcf()

    # Validate arguments
    if perform_stat_test:
        if test is None:
            raise ValueError("If `perform_stat_test` is True, `test` must be specified.")
        if pvalues is not None or test_short_name is not None:
            raise ValueError("If `perform_stat_test` is True, custom `pvalues` "
                             "or `test_short_name` must be `None`.")
        valid_list = ['t-test_ind', 't-test_welch', 't-test_paired',
                      'Mann-Whitney', 'Mann-Whitney-gt', 'Mann-Whitney-ls',
                      'Levene', 'Wilcoxon', 'Kruskal']
        if test not in valid_list:
            raise ValueError("test value should be one of the following: {}."
                             .format(', '.join(valid_list)))
    else:
        if pvalues is None:
            raise ValueError("If `perform_stat_test` is False, custom `pvalues` must be specified.")
        if test is not None:
            raise ValueError("If `perform_stat_test` is False, `test` must be None.")
        if len(pvalues) != len(box_pairs):
            raise ValueError("`pvalues` should be of the same length as `box_pairs`.")

    if text_annot_custom is not None and len(text_annot_custom) != len(box_pairs):
        raise ValueError("`text_annot_custom` should be of same length as `box_pairs`.")

    assert_is_in(
        loc, ['inside', 'outside'], label='argument `loc`'
    )
    assert_is_in(
        text_format,
        ['full', 'simple', 'star'],
        label='argument `text_format`'
    )

    # Comparisons correction
    if comparisons_correction is None:
        pass
    elif isinstance(comparisons_correction, str):
        assert_valid_correction_name(comparisons_correction)
        comparisons_correction = ComparisonsCorrection(comparisons_correction)
    elif not(isinstance(comparisons_correction, ComparisonsCorrection)):
            raise ValueError("comparisons_correction must be a statmodels "
                             "method name or a ComparisonCorrection instance")

    if verbose >= 1 and text_format == 'star':
        print("p-value annotation legend:")
        pvalue_thresholds = pd.DataFrame(pvalue_thresholds).sort_values(by=0, ascending=False).values
        for i in range(0, len(pvalue_thresholds)):
            if i < len(pvalue_thresholds) - 1:
                print('{}: {:.2e} < p <= {:.2e}'.format(pvalue_thresholds[i][1],
                                                        pvalue_thresholds[i + 1][0],
                                                        pvalue_thresholds[i][0]))
            else:
                print('{}: p <= {:.2e}'.format(pvalue_thresholds[i][1], pvalue_thresholds[i][0]))
        print()

    ylim = ax.get_ylim()
    yrange = ylim[1] - ylim[0]

    if line_offset is None:
        if loc == 'inside':
            line_offset = 0.05
            if line_offset_to_box is None:
                line_offset_to_box = 0.06
        # 'outside', see valid_list
        else:
            line_offset = 0.03
            if line_offset_to_box is None:
                line_offset_to_box = line_offset
    else:
        if loc == 'inside':
            if line_offset_to_box is None:
                line_offset_to_box = 0.06
        elif loc == 'outside':
            line_offset_to_box = line_offset
    y_offset = line_offset * yrange
    y_offset_to_box = line_offset_to_box * yrange

    if plot == 'boxplot':
        # Create the same plotter object as seaborn's boxplot
        box_plotter = sns.categorical._BoxPlotter(
            x, y, hue, data, order, hue_order, orient=None, width=width, color=None,
            palette=None, saturation=.75, dodge=True, fliersize=5, linewidth=None)

    elif plot == 'barplot':
        # Create the same plotter object as seaborn's barplot
        box_plotter = sns.categorical._BarPlotter(
            x, y, hue, data, order, hue_order,
            estimator=np.mean, ci=95, n_boot=1000, units=None,
            orient=None, color=None, palette=None, saturation=.75,
            errcolor=".26", errwidth=None, capsize=None, dodge=True)

    # Build the list of box data structures with the x and ymax positions
    group_names = box_plotter.group_names
    hue_names = box_plotter.hue_names
    if box_plotter.plot_hues is None:
        box_names = group_names
        labels = box_names
    else:
        box_names = [(group_name, hue_name) for group_name in group_names for hue_name in hue_names]
        labels = ['{}_{}'.format(group_name, hue_name) for (group_name, hue_name) in box_names]

    box_structs = [
        {
            'box':      box_names[i],
            'label':    labels[i],
            'x':        find_x_position_box(box_plotter, box_names[i]),
            'box_data': get_box_data(box_plotter, box_names[i]),
            'ymax':     (np.amax(get_box_data(box_plotter, box_names[i]))
                         if len(get_box_data(box_plotter, box_names[i])) > 0
                         else np.nan)
        } for i in range(len(box_names))]

    # Sort the box data structures by position along the x axis
    box_structs = sorted(box_structs, key=lambda a: a['x'])
    # Add the index position in the list of boxes along the x axis
    box_structs = [dict(box_struct, xi=i) for i, box_struct in enumerate(box_structs)]
    # Same data structure list with access key by box name
    box_structs_dic = {box_struct['box']: box_struct for box_struct in box_structs}

    # Build the list of box data structure pairs
    box_struct_pairs = []
    for i_box_pair, (box1, box2) in enumerate(box_pairs):
        valid = box1 in box_names and box2 in box_names
        if not valid:
            raise ValueError("box_pairs contains an invalid box pair.")
            pass
        # i_box_pair will keep track of the original order of the box pairs.
        box_struct1 = dict(box_structs_dic[box1], i_box_pair=i_box_pair)
        box_struct2 = dict(box_structs_dic[box2], i_box_pair=i_box_pair)
        if box_struct1['x'] <= box_struct2['x']:
            pair = (box_struct1, box_struct2)
        else:
            pair = (box_struct2, box_struct1)
        box_struct_pairs.append(pair)

    # Draw first the annotations with the shortest between-boxes distance, in order to reduce
    # overlapping between annotations.
    box_struct_pairs = sorted(box_struct_pairs, key=lambda a: abs(a[1]['x'] - a[0]['x']))

    # Build array that contains the x and y_max position of the highest annotation or box data at
    # a given x position, and also keeps track of the number of stacked annotations.
    # This array will be updated when a new annotation is drawn.
    y_stack_arr = np.array([[box_struct['x'] for box_struct in box_structs],
                            [box_struct['ymax'] for box_struct in box_structs],
                            [0 for _ in range(len(box_structs))]])
    if loc == 'outside':
        y_stack_arr[1, :] = ylim[1]
    ann_list = []
    test_result_list = []
    ymaxs = []
    y_stack = []

    # fetch results of all tests
    for box_struct1, box_struct2 in box_struct_pairs:
        box1 = box_struct1['box']
        box2 = box_struct2['box']
        box_data1 = box_struct1['box_data']
        box_data2 = box_struct2['box_data']
        i_box_pair = box_struct1['i_box_pair']

        if perform_stat_test:
            result = stat_test(
                box_data1,
                box_data2,
                test,
                comparisons_correction,
                num_comparisons != "auto" and num_comparisons or len(box_struct_pairs),
                verbose,
                **stats_params
            )
        else:
            test_short_name = test_short_name if test_short_name is not None else ''
            result = StatResult(
                'Custom statistical test',
                test_short_name,
                None,
                None,
                pvalues[i_box_pair]
            )
        result.box1 = box1
        result.box2 = box2

        test_result_list.append(result)

    # Perform other types of correction methods for multiple testing
    if comparisons_correction is not None:
        corr_name = comparisons_correction.name

        # If correction is applied to a set of pvalues
        if comparisons_correction.type == 1:
            original_pvalues = [result.pval for result in test_result_list]

            significant_pvalues = comparisons_correction(original_pvalues)

            for is_significant, result in zip(significant_pvalues, test_result_list):
                result.correction_method = corr_name
                result.corrected_significance = is_significant

        # If correction is applied per pvalue, just compare with alpha
        else:
            alpha = comparisons_correction.alpha
            for result in test_result_list:
                result.correction_method = corr_name
                result.corrected_significance = result.pval < alpha

    # Then annotate
    for box_structs, result in zip(box_struct_pairs, test_result_list):
        x1 = box_structs[0]['x']
        x2 = box_structs[1]['x']
        xi1 = box_structs[0]['xi']
        xi2 = box_structs[1]['xi']
        label1 = box_structs[0]['label']
        label2 = box_structs[1]['label']
        # ymax1 = box_structs[0]['ymax']  # Not used, so do not assign them
        # ymax2 = box_structs[1]['ymax']
        i_box_pair = box_structs[0]['i_box_pair']
        if verbose >= 1:
            print("{} v.s. {}: {}".format(label1, label2, result.formatted_output))

        if text_annot_custom is not None:
            text = text_annot_custom[i_box_pair]
        else:
            if text_format == 'full':
                text = "{} p = {}{}".format('{}', pvalue_format_string, '{}').format(
                    result.test_short_name, result.pval, result.significance_suffix)
            elif text_format is None:
                text = None
            elif text_format is 'star':
                text = pval_annotation_text(result, pvalue_thresholds)
            elif text_format is 'simple':
                test_short_name = result.test_short_name if show_test_name else None
                text = simple_text(result, simple_format_string,
                                   pvalue_thresholds, test_short_name)

        # Find y maximum for all the y_stacks *in between* the box1 and the box2
        i_ymax_in_range_x1_x2 = xi1 + np.nanargmax(
            y_stack_arr[1, np.where((x1 <= y_stack_arr[0, :])
                                    & (y_stack_arr[0, :] <= x2))])
        ymax_in_range_x1_x2 = y_stack_arr[1, i_ymax_in_range_x1_x2]

        yref = ymax_in_range_x1_x2
        yref2 = yref

        # Choose the best offset depending on whether there is an annotation below
        # at the x position in the range [x1, x2] where the stack is the highest
        if y_stack_arr[2, i_ymax_in_range_x1_x2] == 0:
            # there is only a box below
            offset = y_offset_to_box
        else:
            # there is an annotation below
            offset = y_offset
        y = yref2 + offset
        h = line_height * yrange
        line_x, line_y = [x1, x1, x2, x2], [y, y + h, y + h, y]
        if loc == 'inside':
            ax.plot(line_x, line_y, lw=linewidth, c=color)
        elif loc == 'outside':
            line = lines.Line2D(line_x, line_y, lw=linewidth, c=color, transform=ax.transData)
            line.set_clip_on(False)
            ax.add_line(line)

        # why should we change here the ylim if at the very end we set it to the correct range????
        # ax.set_ylim((ylim[0], 1.1*(y + h)))

        if text is not None:
            ann = ax.annotate(
                text, xy=(np.mean([x1, x2]), y + h),
                xytext=(0, text_offset), textcoords='offset points',
                xycoords='data', ha='center', va='bottom',
                fontsize=fontsize, clip_on=False, annotation_clip=False)
            ann_list.append(ann)

            plt.draw()
            y_top_annot = None
            got_mpl_error = False
            if not use_fixed_offset:
                try:
                    bbox = ann.get_window_extent()
                    bbox_data = bbox.transformed(ax.transData.inverted())
                    y_top_annot = bbox_data.ymax
                except RuntimeError:
                    got_mpl_error = True

            if use_fixed_offset or got_mpl_error:
                if verbose >= 1:
                    print("Warning: cannot get the text bounding box. Falling back to a fixed"
                          " y offset. Layout may be not optimal.")
                # We will apply a fixed offset in points,
                # based on the font size of the annotation.
                fontsize_points = FontProperties(size='medium').get_size_in_points()
                offset_trans = mtransforms.offset_copy(
                    ax.transData, fig=fig, x=0,
                    y=1.0 * fontsize_points + text_offset, units='points')
                y_top_display = offset_trans.transform((0, y + h))
                y_top_annot = ax.transData.inverted().transform(y_top_display)[1]
        else:
            y_top_annot = y + h

        y_stack.append(y_top_annot)  # remark: y_stack is not really necessary if we have the stack_array
        ymaxs.append(max(y_stack))
        # Fill the highest y position of the annotation into the y_stack array
        # for all positions in the range x1 to x2
        y_stack_arr[1, (x1 <= y_stack_arr[0, :]) & (y_stack_arr[0, :] <= x2)] = y_top_annot
        # Increment the counter of annotations in the y_stack array
        y_stack_arr[2, xi1:xi2 + 1] = y_stack_arr[2, xi1:xi2 + 1] + 1

    y_stack_max = max(ymaxs)
    if loc == 'inside':
        ax.set_ylim((ylim[0], max(1.03 * y_stack_max, ylim[1])))
    elif loc == 'outside':
        ax.set_ylim((ylim[0], ylim[1]))

    return ax, test_result_list


import collections
import functools
import io
import unicodedata

import discord
from discord.ext import commands
from PIL import Image

from .utils import formats, human_time

class AnyUser(commands.Converter):
    async def convert(self, ctx, arg):
        try:
            return await commands.MemberConverter().convert(ctx, arg)
        except commands.BadArgument:
            pass

        try:
            return await commands.UserConverter().convert(ctx, arg)
        except commands.BadArgument:
            pass

        try:
            return await ctx.bot.fetch_user(arg)
        except:
            raise commands.BadArgument(f"User `{arg}` not found")

class Tools(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.emoji = ":toolbox:"

    @commands.command(name="serverinfo", description="Get info on the server", aliases=["guildinfo"])
    @commands.guild_only()
    async def serverinfo(self, ctx):
        guild = ctx.guild

        async with ctx.typing():
            if not guild.chunked:
                await guild.chunk(cache=True)

            try:
                color = await self.average_image_color(guild.icon_url_as(format="png"))
            except:
                color = discord.Embed.Empty

        em = discord.Embed(color=color)
        em.set_author(name=f"{guild.name} ({guild.id})", icon_url=ctx.guild.icon_url)
        em.set_thumbnail(url=guild.icon_url)

        em.add_field(name=":crown: Owner", value=guild.owner.mention)
        em.add_field(name=":clock1: Created", value=f"{human_time.fulltime(guild.created_at)}")
        em.add_field(name="<:subscriber:808884446739693609> Nitro Boosts", value=guild.premium_tier)
        bots = len([member for member in guild.members if member.bot])
        em.add_field(name=":earth_americas: Region", value=str(guild.region).upper().replace("-", " "))
        em.add_field(name=":family: Members", value=f"{len(guild.members)} ({formats.plural(bots):bot})")
        em.add_field(name=":speech_balloon: Channels", value=f"<:text_channel:822911982319173642> {str(len(guild.text_channels))} \N{BULLET} <:voice_channel:822912006947733504> {str(len(guild.voice_channels))}")

        static_emojis = len([emoji for emoji in guild.emojis if not emoji.animated])
        static_emojis_percentage = int((static_emojis/guild.emoji_limit)*100)
        static_text = f"{static_emojis}/{guild.emoji_limit} ({static_emojis_percentage}%)"
        animated_emojis = len([emoji for emoji in guild.emojis if emoji.animated])
        animated_emojis_percentage = int((animated_emojis/guild.emoji_limit)*100)
        animated_text = f"{animated_emojis}/{guild.emoji_limit} ({animated_emojis_percentage}%)"
        emoji_percentage = int((len(guild.emojis)/(guild.emoji_limit*2))*100)
        em.add_field(name=":pencil: Emojis", value=f"Static: {static_text} \nAnimated: {animated_text} \nTotal: {len(guild.emojis)}/{guild.emoji_limit*2} ({emoji_percentage}%)")
        em.add_field(name=":bookmark: Roles", value=str(len(guild.roles)))

        await ctx.send(embed=em)

    @commands.command(name="userinfo", description="Get info on a user", aliases=["ui", "whois"])
    @commands.guild_only()
    async def userinfo(self, ctx, *, user: AnyUser = None):
        if not user:
            user = ctx.author
        is_member = isinstance(user, discord.Member)

        pubilic_flags_mapping = {
            discord.UserFlags.staff: "<:staff:808882362820984853>",
            discord.UserFlags.partner: "<:partner:808882401085227059>",
            discord.UserFlags.hypesquad: "<:hypersquad:808882441460252702>",
            discord.UserFlags.bug_hunter: "<:bug_hunter:808882493163962398>",
            discord.UserFlags.bug_hunter_level_2: "<:bug_hunter_2:808884426306093076>",
            discord.UserFlags.hypesquad_bravery: "<:hypesquad_bravery:808882576928407593>",
            discord.UserFlags.hypesquad_brilliance: "<:hypesquad_balance:808882618912735243>",
            discord.UserFlags.hypesquad_balance: "<:hypesquad_balance:808882618912735243>",
            discord.UserFlags.early_supporter: "<:early_supporter:808883560752742430>",
            discord.UserFlags.verified_bot_developer: "<:verified_bot_developer:808884369053581323>",
        }

        badges = ""
        for flag in user.public_flags.all():
            emoji = pubilic_flags_mapping.get(flag)
            if emoji:
                badges += emoji

        async with ctx.typing():
            if is_member and not ctx.guild.chunked:
                await ctx.guild.chunk(cache=True)

            try:
                color = await self.average_image_color(user.avatar_url_as(format="png"))
            except:
                color = user.color

        if is_member:
            name = f"{user}{f' ({user.nick})' if user.nick else ''} - {user.id}"
        else:
            name = f"{user} - {user.id}"

        em = discord.Embed(description=badges, color=color)
        em.set_author(name=name, icon_url=user.avatar_url)
        em.set_thumbnail(url=user.avatar_url)

        if user.id == ctx.guild.owner.id:
            em.description += "\n:crown: This user owns the server"
        if user.bot:
            em.description += "\n:robot: This user is a bot"
        if user.id == self.bot.user.id:
            em.description +="\n:wave: This user is me"
        if is_member and user.premium_since:
            em.description += f"\n<:nitro:808884446739693609> This user has been bosting since {human_time.fulltime(user.joined_at)}"

        if user.public_flags.team_user:
            em.description += "\n:family: This user is a team user"
        if user.public_flags.system:
            em.description += "\n:gear: This user is a system user"
        if user.public_flags.verified_bot:
            em.description += "\n:white_check_mark: This user is a verified bot"

        em.add_field(name=":clock3: Created", value=human_time.fulltime(user.created_at))

        if is_member:
            em.add_field(name=":arrow_right: Joined", value=f"{human_time.fulltime(user.joined_at)}")

            sorted_members = sorted(ctx.guild.members, key=lambda x: x.joined_at)
            for position, member in enumerate(sorted_members):
                if member == user:
                    break

            joins = []
            if position > 1:
                joins.append(f"{sorted_members[position-2]}")
            if position > 0:
                joins.append(f"{sorted_members[position-1]}")

            joins.append(f"**{user} (#{position+1})**")

            if position < len(sorted_members) - 1:
                joins.append(f"{sorted_members[position+1]}")
            if position < len(sorted_members) - 2:
                joins.append(f"{sorted_members[position+2]}")

            em.add_field(name=":busts_in_silhouette: Join Order", value=" → ".join(joins), inline=False)

            if len(user.roles) > 1:
                em.add_field(name="Roles", value=" ".join([role.mention for role in reversed(user.roles) if not role.is_default()]))

        shared = [guild for guild in self.bot.guilds if discord.utils.get(guild.members, id=user.id)]
        if shared:
            em.set_footer(text=f"{formats.plural(len(shared)):server} shared")

        await ctx.send(embed=em)

    @commands.command(name="avatar", description="View someone's avatar", usage="[user] [--format FORMAT]")
    async def avatar(self, ctx, *, user = ""):
        if user in ("--format png", "--format jpg", "--format jpeg", "--format webp"):
            if user.endswith(("png", "jpg")):
                view_format = user[-3:]
                user = ""
            elif user.endswith(("jpeg", "webp")):
                view_format = user[-4:]
                user = ""
        elif user.endswith((" --format png", " --format jpg", " --format jpeg", " --format webp")):
            if user.endswith(("png", "jpg")):
                view_format = user[-3:]
                user = user[:len(user)-13]
            elif user.endswith(("jpeg", "webp")):
                view_format = user[-4:]
                user = user[:len(user)-14]

        else:
            view_format = "png"

        if user:
            user = await commands.MemberConverter().convert(ctx, user)
        else:
            user = ctx.author

        async with ctx.typing():
            try:
                color = await self.average_image_color(user.avatar_url_as(format="png"))
            except:
                color = discord.Embed.Empty

        avatar_formats = ["png", "jpg", "webp"]
        if user.is_avatar_animated():
            avatar_formats.append("gif")

        formats_text = [f"[{avatar_format.upper()}]({user.avatar_url_as(format=avatar_format)})" for avatar_format in avatar_formats]

        em = discord.Embed(description=f"View as {formats.join(formats_text, last='or')}", color=color)
        em.set_author(name=user.display_name, icon_url=user.avatar_url)
        em.set_image(url=user.avatar_url_as(static_format=view_format))
        await ctx.send(embed=em)

    @commands.command(name="charinfo", description="Get info on a character")
    async def charinfo(self, ctx, *, text):
        if len(text) > 20:
            return await ctx.send(":x: Your text must be shorter than 20 characters")

        info = []
        for character in text:
            digit = f"{ord(character):x}"
            name = unicodedata.name(character, "Name not found")
            info.append(f"`\\U{digit:>08}`: {name} - {character} \N{EM DASH} <http://www.fileformat.info/info/unicode/char/{digit}>")
        await ctx.send("\n".join(info))

    @commands.command(name="poll", description="Create a poll")
    @commands.guild_only()
    async def poll(self, ctx, title = None, *options):
        possible_reactions = [
            "\N{REGIONAL INDICATOR SYMBOL LETTER A}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER B}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER C}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER D}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER E}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER F}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER G}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER H}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER I}",
            "\N{REGIONAL INDICATOR SYMBOL LETTER J}",
        ]

        Option = collections.namedtuple("Option", ["emoji", "text"])

        if not title:
            options = []

            check = lambda message: message.channel == ctx.author.dm_channel and message.author == ctx.author
            try:
                await ctx.author.send("What is the title of the poll?")
            except discord.Forbidden:
                return await ctx.send(":x: You have DMs disabled")

            message = await self.bot.wait_for("message", check=check)
            title = message.content

            await ctx.author.send("Send me up to 10 poll options. Type `done` to send the poll")
            while len(options) < len(possible_reactions):
                message = await self.bot.wait_for("message", check=check)
                if message.content == "done":
                    if len(options) < 2:
                        return await ctx.author.send(":x: You must have at least 2 options")
                    break
                elif message.content == "abort":
                    return await ctx.author.send("Aborting")

                args = message.content.split(" ")
                if len(args) == 1:
                    options.append(Option(None, message.content))
                    await message.add_reaction("\N{WHITE HEAVY CHECK MARK}")
                else:
                    emoji = args[0]
                    text = " ".join(args[1:])
                    if emoji in [option.emoji for option in options]:
                        await message.add_reaction("\N{CROSS MARK}")
                        await ctx.author.send(":x: You already used that emoji", delete_after=5)
                    elif text in [option.text for option in options]:
                        await message.add_reaction("\N{CROSS MARK}")
                        await ctx.author.send(":x: You already added that option", delete_after=5)
                    else:
                        if emoji in self.bot.default_emojis:
                            options.append(Option(emoji, text))
                        else:
                            options.append(Option(None, message.content))
                        await message.add_reaction("\N{WHITE HEAVY CHECK MARK}")

        else:
            options = [Option(None, text) for text in options]

        if len(options) > len(possible_reactions):
            raise commands.BadArgument(f"You cannot have more then {len(possible_reactions)} options")
        if len(options) < 2:
            return await ctx.send(":x: You must have at least 2 options")

        description = ""
        reactions = []
        for counter, option in enumerate(options):
            if option.emoji:
                emoji = option.emoji
            else:
                emoji = possible_reactions[counter]

            description += f"\n{emoji} {option.text}"
            reactions.append(emoji)

        em = discord.Embed(title=title, description=description)
        em.set_author(name=ctx.author.display_name, icon_url=ctx.author.avatar_url)
        message = await ctx.send(embed=em)

        for reaction in reactions:
            await message.add_reaction(reaction)

    @commands.command(name="snowstamp", description="Get the timestamp from a Discord snowflake", hidden=True)
    async def snowflake(self, ctx, *, snowflake: int):
        time = discord.utils.snowflake_time(snowflake)
        await ctx.send(human_time.format_time(time))

    async def average_image_color(self, icon):
        data = await icon.read()
        image = io.BytesIO(data)

        partial = functools.partial(Image.open, image)
        img = await self.bot.loop.run_in_executor(None, partial)

        partial = functools.partial(img.resize, (1, 1))
        img2 = await self.bot.loop.run_in_executor(None, partial)

        partial = functools.partial(img2.getpixel, (0, 0))
        color = await self.bot.loop.run_in_executor(None, partial)

        return(discord.Color(int("0x{:02x}{:02x}{:02x}".format(*color), 16)))

def setup(bot):
    bot.add_cog(Tools(bot))


# -*- coding: utf-8 -*-
import xlrd

# 읽기 라이브러리를 통해 현재 같은 디렉터리상에 있는 test.xls를 불러와 workbook에 할당합니다.
workbook = xlrd.open_workbook('test.xls')

# 워크북에 할당된 엑셀 데이터의 첫번째시트를 불러옵니다.
worksheet = workbook.sheet_by_index(0)

# nrows에 불러온 첫번째 시트의 행수를 불러옵니다.
nrows = worksheet.nrows

# 불러온 데이터를 저장할 딕셔너리를 선언합니다.
datadict = {}

# 행수만큼의 for loop 를 돌려서 행단위로 데이터를 불러와 datadict에 저장합니다.
for row_num in range(nrows):
    datadict[row_num] = {}
    # field_cnt는 열의 개수입니다. 열갯수는 여러분이 지정하시면 됩니다.
    for col in range(field_cnt):
        # datadict[row_num]에 열숫자별로 셀데이터를 저장합니다.
        datadict[row_num][col] = worksheet.cell_value(row_num, col)

workbookw = xlwt.Workbook(encoding='utf-8')  # utf-8 인코딩 방식의 workbook 생성

workbookw.default_style.font.height = 20 * 11
worksheetw = workbookw.add_sheet('시트 이름')  # 시트 생성

# 2번째 행부터 데이터를 입력합니다.
for row_num in range(nrows):
    for col in range(field_cnt):
        worksheetw.write(row_num, col, datadict[row_num][col])

# 할당된 데이터셋을 'result.xls'로 저장합니다.
workbookw.save("result.xls")


from django.test import TestCase

from contacts.models import Contact

class ContactTests(TestCase):
    """Contact model tests."""

    def test_str(self):

        contact = Contact(first_name='gulsah', last_name='acar')

        self.assertEquals(
            str(contact),
            'gulsahacar',
        )


from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import *  # NOQA
from future.utils import itervalues, iteritems, iterkeys  # NOQA

import pytest
import selfactualize as sa

# test DAG
@pytest.fixture
def dag_1():
    op_1 = sa.wrap_function(lambda x, y=7: x + y, 'x, y? => z', 'op_1')
    op_2 = sa.wrap_function(lambda x: 3 * x, 'x => z', 'op_2')
    dag_1 = (
        sa.DAG(
            name='dag_1', input_names=['x', 'm'], input_required=[True, False],
            output_names=['z'])
        .add_op(op_1)
        .add_op(op_2)
        .add_flow('x', 'op_1.x')
        .add_flow('m', 'op_1.y')
        .add_flow('op_1.z', 'op_2.x')
        .add_flow('op_2.z', 'z')
    )
    return dag_1

@pytest.fixture
def dag_2():
    op_1 = sa.wrap_function(lambda x: (x, x + 1), 'x => y, z', 'op_1')
    op_2 = sa.wrap_function(lambda x: x, 'x => z', 'op_2')
    op_3 = sa.wrap_function(lambda x: x, 'x => z', 'op_3')
    op_4 = sa.wrap_function(lambda a, b: (a, b), 'a, b => x, y', 'op_4')
    op_5 = sa.wrap_function(lambda x: x, 'x => z', 'op_5')
    dag_2 = (
        sa.DAG(
            name='dag_2', input_names=['x'], input_required=[True],
            output_names=['y', 'z'])
        .add_op(op_4)
        .add_op(op_3)
        .add_op(op_2)
        .add_op(op_5)
        .add_op(op_1)
        .add_flow('x', 'op_1.x')
        .add_flow('op_1.y', 'op_2.x')
        .add_flow('op_1.z', 'op_3.x')
        .add_flow('op_2.z', 'op_4.a')
        .add_flow('op_3.z', 'op_4.b')
        .add_flow('op_4.x', 'op_5.x')
        .add_flow('op_4.y', 'y')
        .add_flow('op_5.z', 'z')
    )
    return dag_2

class TestDAG(object):

    def test_execution_order(self, dag_1, dag_2):
        dag_1.run({'x': 10})
        assert dag_1.execution_order == ['op_1', 'op_2']
        dag_2.run({'x': 10})
        assert (
            dag_2.execution_order == ['op_1', 'op_2', 'op_3', 'op_4', 'op_5'] or
            dag_2.execution_order == ['op_1', 'op_3', 'op_2', 'op_4', 'op_5']
        )

    def test_run(self, dag_1, dag_2):
        assert dag_1.run({'x': 3}) == {'z': 30}
        assert dag_1.run({'x': 1, 'm': 1}) == {'z': 6}
        assert dag_2.run({'x': 10}) == {'y': 11, 'z': 10}


import json
import os
import re
from typing import Any, Dict

from . import files as sm_files

def parse_sm_config() -> Dict[str, Any]:
    """Attempt to parse SageMaker configuration.

    Returns:
        A dictionary of SageMaker config keys/values or empty dict if not found.
    """
    conf = {}
    if os.path.exists(sm_files.SM_PARAM_CONFIG) and os.path.exists(
        sm_files.SM_RESOURCE_CONFIG
    ):
        conf["sagemaker_training_job_name"] = os.getenv("TRAINING_JOB_NAME")
        # Hyperparameter searches quote configs...
        for k, v in json.load(open(sm_files.SM_PARAM_CONFIG)).items():
            cast = v.strip('"')
            if re.match(r"^-?[\d]+$", cast):
                cast = int(cast)
            elif re.match(r"^-?[.\d]+$", cast):
                cast = float(cast)
            conf[k] = cast
    return conf


from django.shortcuts import render
from . models import *

def index(request):
	services = Service.objects.all()[:3]
	team_members = TeamMember.objects.all()[:3]

	events = TimelineEvent.objects.all().order_by('date')
	projects = Project.objects.all().order_by('-date')

	pictures = Picture.objects.all().order_by('-date')

	jobs = Job.objects.all()

	return render(request, 'myapp/index.html', {
			'services' : services,
			'team_members' : team_members,
			'events' : events,
			'projects' : projects,
			'pictures' : pictures,
			'jobs' : jobs,
 		})


#!/usr/bin/env python3

# JM: 16 Nov 2018
# generic script to combine multiple netCDF4 files into one average
# do this by opening a grabbed list of files and then averaging it by the length
# of the file list
# only does it per variable, option to output

import numpy as np
import glob, netCDF4, copy, sys

#--------------------------------------------------------
# define the argument parser
import argparse

parser = argparse.ArgumentParser(description =
    """
    Generic script to combine multiple netCDF4 files into one file as an average.
    See flags for options.
    """)

# fixed arguments
parser.add_argument("data_dir", type = str,
                    help = "specify data directory")
parser.add_argument("file",     type = str,
                    help = "specify string in file that matches the data (e.g. \"*MOC_V*\", note the quote marks)")
parser.add_argument("var",      type = str,
                    help = "specify variable name in file (e.g. votemper)")
parser.add_argument("file_out", type = str,
                    help = "specify output name (e.g. kgm_tave.nc)")

# collect arguments
args = parser.parse_args()

#--------------------------------------------------------

# initialisation

# grab file names
filenames = args.data_dir + args.file + ".nc"
file_list = []
for file in glob.glob(filenames):
    file_list.append(file)

if len(file_list) == 0:
  sys.exit("no files grabbed in %s, are you in the right folder?" % filenames)

# sort it according to the timestamps
file_list.sort()

#--------------------------------------------------------
# cycle through the file lists and compile the moc
print("%g files found, cycling through them..." % len(file_list))

for i in range(len(file_list)):
  file = file_list[i].replace(args.data_dir, "") # strip out the data_dir
  print(" ")
  print("working in file = %g / %g" % (i + 1, len(file_list)))
  data = netCDF4.Dataset(file_list[i])
  try:
    var = data.variables[args.var]
  except KeyError:
    print(data)
    data.close()
    sys.exit("key error found, look up the list printed above")

  # work on the assumption that there is a time-dimension
  if i == 0:
    long_name = copy.deepcopy(var.long_name)
    units     = copy.deepcopy(var.units)
    nav_lat   = data.variables["nav_lat"][:, :]
    nav_lon   = data.variables["nav_lon"][:, :]
    t         = data.variables["time_centered"][:]
    for kt in range(len(t)):
      print("processing %s at index %i / %i..."
        % (file_list[i].replace(args.data_dir, ""), kt + 1, len(t))
           )
      if kt == 0:
        array  = var[kt, :, :] / (len(file_list) * len(t))
      else:
        array += var[kt, :, :] / (len(file_list) * len(t))
  else:
    for kt in range(len(t)):
      print("processing %s at index %i / %i..."
        % (file_list[i].replace(args.data_dir, ""), kt + 1, len(t))
           )
      array += var[kt, :, :] / (len(file_list) * len(t))

  data.close()

print(" ")
print("returning final time-averaged field")

#--------------------------------------------------------
# write the file

ncfile = netCDF4.Dataset(args.data_dir + args.file_out, "w", format = "NETCDF4")
ncfile.title = "combined variable %s" % args.var

# create the dimensions.

ncfile.createDimension("x", nav_lat.shape[1])
ncfile.createDimension("y", nav_lat.shape[0])
ncfile.createDimension("time", len(np.asarray([0.0])))

# first argument is name of variable,
# second is datatype,
# third is a tuple with the names of dimensions.

lon_netcdf = ncfile.createVariable("nav_lon", np.dtype("float32").char, ("y", "x"))
lon_netcdf[:] = nav_lon
lon_netcdf.units = "deg"
lon_netcdf.long_name = "x"

lat_netcdf = ncfile.createVariable("nav_lat", np.dtype("float32").char, ("y", "x"))
lat_netcdf[:] = nav_lat
lat_netcdf.units = "deg"
lat_netcdf.long_name = "y"

var_netcdf = ncfile.createVariable(args.var, np.dtype("float32").char, ("time", "y", "x"))
var_netcdf[:] = array
var_netcdf.units = units
var_netcdf.long_name = long_name

print("*** SUCCESS writing to %s! ***" % args.file_out)


import matplotlib.pyplot as plt
import numpy as np
import pickle
import json
from pylab import plot, show, savefig, xlim, figure, ylim, legend, boxplot, setp, axes
from i3Deep import utils
import pandas as pd
import os
import seaborn as sns

def plot_dice_box_plots():
    for i, task in enumerate(tasks):
        fig = plt.figure(constrained_layout=True, figsize=(figsizes[i][0], figsizes[i][1]))
        gs = fig.add_gridspec(4, 4)
        for j, class_name in enumerate(task_classes[i]):
            df_methods = {}
            for k, method in enumerate(methods):
                filename = base_path + task + "/refinement_" + set + "/eval_results/processed/dice/" + method + ".csv"
                if not os.path.isfile(filename):
                    continue
                df = pd.read_csv(filename)
                df = df.iloc[:, j+1].to_numpy()
                df_methods[method_names[k]] = df
            df = pd.DataFrame.from_dict(df_methods, orient='index').T
            sns.set_theme(style="whitegrid")
            ax = fig.add_subplot(gs[gridspec_indices[i][j][0], gridspec_indices[i][j][1]])
            ax = sns.boxplot(data=df, ax=ax)
            ax.tick_params(axis='x', rotation=20)
            ax.set_ylim(0, 1)
            ax.set_title("{}".format(class_name))
        plt.suptitle("{}".format(task_names[i]), fontsize=16)
        # plt.show()
        plt.savefig(base_path + "Evaluation/Results/" + set + "/" + task_names[i] + ".png", bbox_inches='tight')
        plt.clf()

if __name__ == '__main__':
    base_path = "/gris/gris-f/homelv/kgotkows/datasets/nnUnet_datasets/nnUNet_raw_data/nnUNet_raw_data/"
    tasks = ["Task070_guided_all_public_ggo", "Task002_BrainTumour_guided", "Task008_Pancreas_guided"]
    task_names = ["COVID-19", "Brain Tumor", "Pancreas"]
    task_classes = [["GGO"], ["Edema", "Non-enhancing Tumor", "Enhancing Tumor"], ["Pancreas", "Cancer"]]
    methods = ["automatic", "my_method", "P_Net", "watershed", "GraphCut1"]  # ["automatic", "my_method", "P_Net", "watershed", "random_walker", "GraphCut1", "random1", "random2", "random3"]
    method_names = ["Preseg.", "nnU-Net", "P-Net", "Watershed", "Graph-Cut"]  # ["Preseg.", "nnU-Net", "P-Net", "Watershed", "Random Walker", "Graph-Cut", "RS1", "RS2", "RS3"]
    set = "test"
    figsizes = [[8, 5], [29, 10], [12, 5]]
    gridspec_indices = [[[slice(0, 4), slice(0, 4)]],
                        [[slice(0, 2), slice(0, 1)], [slice(0, 2), slice(1, 2)], [slice(0, 2), slice(2, 3)]],
                        [[slice(0, 4), slice(0, 2)], [slice(0, 4), slice(2, 4)]]]

    plot_dice_box_plots()


# -*- coding: utf-8 -*-
import unittest

from pyramid.interfaces import IRendererFactory
from pyramid.renderers import JSON
from pyramid.threadlocal import get_current_registry

from oe_utils.audit import audit
from oe_utils.audit import audit_with_request
from oe_utils.data.data_managers import AuditManager
from oe_utils.data.db_audit_manager import audit_manager
from oe_utils.data.db_audit_manager import includeme
from oe_utils.data.models import Base
from oe_utils.data.models import Wijziging

try:
    from unittest.mock import Mock, MagicMock
except ImportError:
    from mock import Mock, MagicMock  # pragma: no cover
try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import os
from sqlalchemy import engine_from_config
from sqlalchemy.orm import sessionmaker
from zope.sqlalchemy import ZopeTransactionExtension

json_renderer = JSON()

def init_test_db(engine):
    Base.metadata.drop_all(engine)
    Base.metadata.create_all(engine)
    session_maker = sessionmaker(bind=engine)
    session = session_maker()

    session.commit()
    session.close()

class Resource:
    def __init__(self):
        self.id = (None,)
        self.naam = None
        self.voornaam = None

def resource_adapter(obj, request=None):
    return {"id": obj.id, "naam": obj.naam, "voornaam": obj.voornaam}

json_renderer.add_adapter(Resource, resource_adapter)

class DummyParent(object):
    def __init__(self):
        self.request = MagicMock()
        self.request.registry = get_current_registry()
        self.request.registry.settings = {}
        self.request.registry.registerUtility(
            json_renderer, IRendererFactory, name="jsonrenderer"
        )

    @audit()
    def dummy(self):
        resource = Resource()
        resource.id = 123
        resource.naam = "crash test"
        resource.voornaam = "dummy"
        return resource

    @audit()
    def dummy_none(self):
        return "test"

    @audit(result_id_key="naam")
    def dummy_other_id(self):
        resource = Resource()
        resource.id = 123
        resource.naam = "456"
        resource.voornaam = "dummy"
        return resource

    @audit(actie="bijlage bewerken")
    def dummy_bijlage(self):
        return None

@audit_with_request(actie="bijlage blijven bewerken")
def dummy_bijlage_with_request(request):
    return None

class AuditTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        config = configparser.ConfigParser()
        config.read(os.path.join(os.path.dirname(__file__), "test.ini"))
        settings = config.items("app:oe_utils")
        settings = dict((s[0], s[1]) for s in settings)
        cls.engine = engine_from_config(settings, prefix="sqlalchemy.")
        cls.session_maker = sessionmaker(
            bind=cls.engine, extension=ZopeTransactionExtension()
        )

    def setUp(self):
        init_test_db(self.engine)
        self.session = self.session_maker()
        self.dummy_parent = DummyParent()
        self.audit_manager = AuditManager(self.session)
        self.dummy_parent.request.audit_manager = self.audit_manager

    def tearDown(self):
        self.session.close()

    def test_audit_get(self):
        self.dummy_parent.request.user = {
            "actor": {"uri": "uri", "omschrijving": "omschrijving"},
            "attributes": {},
        }
        self.dummy_parent.request.method = "GET"
        self.dummy_parent.request.matchdict = {"id": 123}
        self.dummy_parent.dummy()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("opvragen", wijziging.actie)
        self.assertEqual("omschrijving", wijziging.updated_by_omschrijving)
        self.assertEqual("crash test", wijziging.resource_object_json["naam"])

    def test_audit_put(self):
        self.dummy_parent.request.method = "PUT"
        self.dummy_parent.request.matchdict = {"id": 123}
        self.dummy_parent.dummy()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("bewerken", wijziging.actie)
        self.assertEqual("crash test", wijziging.resource_object_json["naam"])
        self.assertEqual("publiek", wijziging.updated_by_omschrijving)

    def test_audit_post(self):
        self.dummy_parent.request.method = "POST"
        self.dummy_parent.dummy()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("aanmaken", wijziging.actie)
        self.assertEqual("crash test", wijziging.resource_object_json["naam"])
        self.assertEqual("crash test", wijziging.resource_object_json["naam"])

    def test_audit_delete(self):
        self.dummy_parent.request.method = "DELETE"
        self.dummy_parent.request.matchdict = {"id": 123}
        self.dummy_parent.dummy_none()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("verwijderen", wijziging.actie)
        self.assertEqual(123, wijziging.resource_object_id)
        self.assertIsNone(wijziging.resource_object_json)

    def test_audit_other(self):
        self.dummy_parent.request.method = "HEAD"
        self.dummy_parent.request.matchdict = {"id": 123}
        self.dummy_parent.dummy_none()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("onbekend", wijziging.actie)
        self.assertEqual(123, wijziging.resource_object_id)
        self.assertIsNone(wijziging.resource_object_json)

    def test_audit_bijlage(self):
        self.dummy_parent.request.method = "PUT"
        self.dummy_parent.request.matchdict = {"id": 123}
        self.dummy_parent.dummy_bijlage()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("bijlage bewerken", wijziging.actie)
        self.assertEqual(123, wijziging.resource_object_id)
        self.assertIsNone(wijziging.resource_object_json)

    def test_db_audit_manager(self):
        config = Mock()
        includeme(config)
        request = MagicMock()
        request.db = self.session
        manager = audit_manager(request)
        self.assertIsInstance(manager, AuditManager)

    def test_audit_bijlage_with_request(self):
        request = MagicMock()
        request.audit_manager = AuditManager(self.session)
        request.method = "PUT"
        request.matchdict = {"id": 1234}
        dummy_bijlage_with_request(request)
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("bijlage blijven bewerken", wijziging.actie)
        self.assertEqual(1234, wijziging.resource_object_id)
        self.assertIsNone(wijziging.resource_object_json)

    def test_audit_other_id_key(self):
        self.dummy_parent.request.user = {
            "actor": {"uri": "uri", "omschrijving": "omschrijving"},
            "attributes": {},
        }
        self.dummy_parent.request.method = "GET"
        self.dummy_parent.request.matchdict = {"id": None}
        self.dummy_parent.dummy_other_id()
        wijziging = (
            self.session.query(Wijziging).order_by(Wijziging.updated_at.desc()).first()
        )
        self.assertEqual("opvragen", wijziging.actie)
        self.assertEqual("omschrijving", wijziging.updated_by_omschrijving)
        self.assertEqual("456", wijziging.resource_object_json["naam"])
        self.assertEqual(456, wijziging.resource_object_id)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('courseDetails/<int:myid>', views.courseDetails, name='courseDetails'),
    path('enroll/<int:myid>', views.courseEnroll, name='courseEnroll'),
    path('course/', views.course, name='course'),
    path('about/', views.aboutus, name='about'),
    path('contact/', views.contact, name='contact'),
    path('search', views.search, name='search'),
    path('gallery', views.gallery, name='gallery'),
    path('studentReg/<int:myid>', views.studentReg, name='studentReg'),
    path('termsAndConditions', views.terms, name='termsAndConditions'),
    path('premiumcourse', views.premiumreg, name='premiumcourse'),
]


from flask import Flask, jsonify, request

app = Flask(__name__)
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from src.request.ranking_input import RankingInput
from src.response.ranking_response import RankingResponse

def init():
    pass

@app.route('/', methods=['GET', 'POST'])
def hello_world():
    return 'Cleveland-Ranking: Ranking online system for SuperSearch.'

@app.route('/ranking', methods=['POST'])
def ranking():
    # pass-through: return same searchResponse with extra rankScore.
    request_body = request.json
    ranking_input_obj = RankingInput.from_request_json(request_body)
    response_obj = RankingResponse.from_ranking_input(ranking_input_obj)
    # set default rankScore the same as filtrationScore.
    response_obj.set_default_ranking_score()
    return jsonify(response_obj.__dict__)

if __name__ == '__main__':
    init()
    app.run(host='0.0.0.0', port=80)


from django.conf import settings
import requests

API_KEY = settings.DARKSKY_KEY
ENDPOINT = "https://api.darkskyapp.com/v1/forecast/%s/%s,%s"

def forecast(lat, lon):

    url = ENDPOINT % (API_KEY, lat, lon)

    resp = requests.get(url)
    return resp.json


import numpy as np
import math
import scipy.stats as stats
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor

from settree.set_data import apply_operation

DTYPE = np.float32
MAX_INT = np.iinfo(np.int32).max

def complementary(curr_as, record):
    return [i for i in range(len(record)) if i not in curr_as]

def mse(y, weights=None):
    """
    Mean squared error for decision tree (ie., mean) predictions
    """
    return np.average((y - np.average(y, weights=weights)) ** 2, weights=weights)

def entropy(y, weights=None):
    """
    Entropy of a label sequence
    """
    hist = np.bincount(y, weights)
    return stats.entropy(hist)
    # ps = hist / np.sum(hist)
    # return -np.sum([p * np.log2(p) for p in ps if p > 0])

def gini(y, weights=None):
    """
    Gini impurity (local entropy) of a label sequence
    """
    hist = np.bincount(y, weights)
    N = np.sum(hist)
    return 1 - sum([(i / N) ** 2 for i in hist])

class DatasetMemoryQueue():
    """
    FIFO queue for keeping running memory of calculated statistics.
    Is used for avoiding excess computations and increase efficiency.
    """

    def __init__(self, operations, use_attention_set, use_attention_set_comp, attention_set_limit):

        """

        The running memory queue is kept as a list of np.arrays with the structure : [flat_base, flat_as_1, ... flat_as_n]
        Where flat_as_i is an array that contains the aggregated results for the ith stage.

        Parameters
        ----------

        operations : list of numpy functions
            The list of aggregation operators to use when building the tree.

        use_attention_set : bool
            Whether to use the attention-set mechanism.

        use_attention_set_comp : bool
            Whether to use the attention-set compatibles set.

        attention_set_limit : int
            The number of ancestors to scan while scanning split criteria with the attention-sets.
            When attention_set_limit=1, scans only the direct ancestors.
        """

        self.operations = operations
        self.n_operations = len(self.operations)

        self.use_attention_set = use_attention_set
        self.use_attention_set_comp = use_attention_set_comp
        self.attention_set_limit = attention_set_limit
        self.attention_set_states = [False, True] if self.use_attention_set_comp else [False]

        self.X = None

    def _init(self, X_set, mask_inds):
        """
        Calculate the flattened dataset without any attention set
        """
        self.n_features = X_set.shape[1]
        self.base_len = self.n_features * self.n_operations

        self.max_memory = self.n_features * self.n_operations
        if self.use_attention_set:
            if self.use_attention_set_comp:
                self.max_memory = self.max_memory * (1 + np.abs(self.attention_set_limit) * 2)
            else:
                self.max_memory = self.max_memory * (1 + np.abs(self.attention_set_limit))

        X = []
        for i in mask_inds:
            record = X_set.records[i]
            X.append(np.concatenate([op(record) for op in self.operations]))

        self.X = X
        self.records_len = np.full(shape=(len(mask_inds,)), fill_value=self.base_len).astype(int)

    def _get_flat_record(self, record, attention_set):
        record = record[attention_set]
        if len(record):
            flat_record = []
            for op in self.operations:
                flat_record.append(op(record))
            flat_record = np.concatenate(flat_record)
        else:
            flat_record = np.full(shape=(self.n_features * self.n_operations,), fill_value=np.finfo(np.float32).min)
        return flat_record

    def _calc_flatten_dataset_for_current_attention_set(self, X_set, mask_inds):

        flat_record_len = self.base_len * 2 if self.use_attention_set_comp else self.base_len

        for ind in mask_inds:
            as_mem = X_set.attention_set[ind]
            record = X_set.records[ind]

            prev_as = as_mem[-1]
            flat_record = self._get_flat_record(record, prev_as)

            if self.use_attention_set_comp:
                prev_as = complementary(prev_as, record)
                flat_comp_record = self._get_flat_record(record, prev_as)
                flat_record = np.concatenate([flat_record, flat_comp_record])

            # if the memory if full
            # dump the first memory, shift the rest to the right and insert a new memory
            if self.records_len[ind] >= self.max_memory:
                mem_flat_record = self.X[ind]
                mem_base = mem_flat_record[:self.base_len]
                mem_as = mem_flat_record[self.base_len:-flat_record_len]
                self.X[ind] = np.concatenate([mem_base, flat_record, mem_as])
                self.records_len[ind] = self.max_memory

            # if there is already a flatten attention-set in memory
            # shift the current memory to the right and insert a new memory
            elif self.records_len[ind] > self.base_len:
                mem_flat_record = self.X[ind]
                mem_base = mem_flat_record[:self.base_len]
                mem_as = mem_flat_record[self.base_len:]
                self.X[ind] = np.concatenate([mem_base, flat_record, mem_as])
                self.records_len[ind] += flat_record_len

            else:
                mem_base = self.X[ind]
                self.X[ind] = np.concatenate([mem_base, flat_record])
                self.records_len[ind] += flat_record_len

    def _get_feature_number_to_args(self, cur_depth, apply_attention_set=False):
        ind2args = {}
        c = 0
        for op in self.operations:
            for i in range(self.n_features):
                ind2args[c] = {'feature': i,
                               'op': op,
                               'use_attention_set': False,
                               'use_attention_set_comp': False}
                c += 1

        # if the model is using the attention-sets - add it to mapping
        if apply_attention_set:
            for as_level in list(range(-1, max(-cur_depth, -self.attention_set_limit) - 1, -1)):
                for as_comp in self.attention_set_states:
                    for op in self.operations:
                        for i in range(self.n_features):
                            ind2args[c] = {'feature': i,
                                           'op': op,
                                           'use_attention_set': as_level,
                                           'use_attention_set_comp': as_comp}
                            c += 1

        return ind2args

    def reset(self):
        del self.X
        del self.records_len
        self.X = None
        self.records_len = None

    def enqueue_and_get(self, X_set, mask_inds, cur_depth):
        if cur_depth == 0:
            self._init(X_set, mask_inds)
            ind2args = self._get_feature_number_to_args(cur_depth, False)
        else:
            if self.use_attention_set:
                self._calc_flatten_dataset_for_current_attention_set(X_set, mask_inds)
                ind2args = self._get_feature_number_to_args(cur_depth, True)
            else:
                ind2args = self._get_feature_number_to_args(cur_depth, False)

        X = np.stack([self.X[i] for i in mask_inds])
        return X, ind2args

class BaseSplitter():
    """
    Abstract splitter class
    Splitters are called by tree builders to find the best splits.
    """

    def __init__(self, classifier=True, criterion='entropy', max_features=None, min_samples_leaf=1, random_state=None):
        """
        Parameters
        ----------
        classifier : bool
            Whether the tree used for classification or regression

        criterion : {"gini", "entropy", "mse"}, default="entropy"
            The function to measure the quality of a split.

        max_features : int, float, {"auto", "sqrt", "log2"} or None, default="auto"
            The number of features to consider when looking for the best split:
                - If int, then consider `max_features` features at each split.
                - If float, then `max_features` is a fraction and
                  `int(max_features * n_features)` features are considered at each
                  split.
                - If "auto", then `max_features=sqrt(n_features)`.
                - If "sqrt", then `max_features=sqrt(n_features)`.
                - If "log2", then `max_features=log2(n_features)`.
                - If None, then `max_features=n_features`.

        min_samples_leaf : int or float, default=1
            The minimum number of samples required to be at a leaf node.
            A split point at any depth will only be considered if it leaves at
            least ``min_samples_leaf`` training samples in each of the left and
            right branches.  This may have the effect of smoothing the model,
            especially in regression.
            - If int, then consider `min_samples_leaf` as the minimum number.
            - If float, then `min_samples_leaf` is a fraction and
              `ceil(min_samples_leaf * n_samples)` are the minimum
              number of samples for each node.

        min_weight_leaf : double
            The minimal weight each leaf can have, where the weight is the sum
            of the weights of each sample in it.

        random_state : int, RandomState instance or None, default=None
            Used to pick randomly the `max_features` used at each split.
        """

        self.criterion = criterion
        self.classifier = classifier
        self.min_samples_leaf = min_samples_leaf
        self.random_state = random_state
        self.max_features = max_features

    def init(self, n_samples, n_features):

        self.n_samples = n_samples
        self.n_features = n_features

        if isinstance(self.min_samples_leaf, float):
            self.min_samples_leaf = math.ceil(self.min_samples_leaf * self.n_features)

        if self.criterion in CRITERIONS:
            self.criterion = CRITERIONS[self.criterion]
        else:
            raise ValueError('Invalid criterion name {}'.format(self.criterion))

        if isinstance(self.max_features, float):
            self.max_features = math.ceil(self.max_features * n_features)
        elif self.max_features == 'auto' or self.max_features == 'sqrt':
            self.max_features = math.ceil(np.sqrt(self.n_features))
        elif self.max_features == 'log2':
            self.max_features = math.ceil(np.log2(self.n_features))
        elif self.max_features == None:
            self.max_features = self.n_features
        else:
            raise ValueError('Invalid value for max_features, {}'.format(self.max_features))

    def end(self):
        pass

    def node_split(self, X_set, y, mask_inds, cur_depth):
        raise NotImplemented

class ThirdPartySplitter(BaseSplitter):
    """
    Provides common API for external (third party) splitters
    """

    def __init__(self, classifier=True, criterion='entropy', max_features=None, min_samples_leaf=None, random_state=None,
                 operations=None, use_attention_set=True, use_attention_set_comp=True, attention_set_limit=1):

        self.SMALLEST = np.finfo(DTYPE).min
        self.LARGEST = np.finfo(DTYPE).max

        super().__init__(classifier, criterion, max_features, min_samples_leaf, random_state)

        self.operations = operations
        self.use_attention_set = use_attention_set
        self.use_attention_set_comp = use_attention_set_comp
        self.attention_set_limit = attention_set_limit

        self.memory_queue = DatasetMemoryQueue(operations=self.operations,
                                               use_attention_set=self.use_attention_set,
                                               use_attention_set_comp=self.use_attention_set_comp,
                                               attention_set_limit=self.attention_set_limit)

    def init(self, n_samples, n_features):
        self.attention_set_states = [False, True] if self.use_attention_set_comp else [False]
        self.n_samples = n_samples
        self.n_features = n_features * len(self.operations)
        if self.use_attention_set:
            n_attention_set_levels = np.abs(self.use_attention_set)
            if self.use_attention_set_comp:
                self.n_features = self.n_features * (n_attention_set_levels * 2 + 1)
            else:
                self.n_features = self.n_features * (n_attention_set_levels + 1)

        if isinstance(self.min_samples_leaf, float):
            self.min_samples_leaf = math.ceil(self.min_samples_leaf * self.n_features)

        if self.criterion in CRITERIONS:
            self.criterion = CRITERIONS[self.criterion]
        else:
            raise ValueError('Invalid criterion name {}'.format(self.criterion))

    def node_split(self, X_set, y, mask_inds, cur_depth):
        raise NotImplemented

    def end(self):
        self.memory_queue.reset()

class SklearnSetSplitter(ThirdPartySplitter):

    def __init__(self, classifier=True, criterion='entropy', max_features=None, min_samples_leaf=None, random_state=1,
                 operations=None, use_attention_set=False, use_attention_set_comp=True, attention_set_limit=1):

        super().__init__(classifier, criterion, max_features, min_samples_leaf, random_state,
                         operations, use_attention_set, use_attention_set_comp, attention_set_limit)

    def node_split(self, X_set, y, mask_inds, cur_depth):

        y = y.take(mask_inds)
        X, ind2args = self.memory_queue.enqueue_and_get(X_set, mask_inds, cur_depth)
        rand_state = self.random_state.randint(0, MAX_INT)

        if self.classifier:
            clf = DecisionTreeClassifier(criterion='entropy',
                                         splitter='best',
                                         max_depth=1,
                                         min_samples_split=2,
                                         min_samples_leaf=1,
                                         min_weight_fraction_leaf=0.0,
                                         max_features=self.max_features,
                                         random_state=rand_state).fit(X, y)
        else:
            clf = DecisionTreeRegressor(criterion='mse',
                                        splitter='best',
                                        max_depth=1,
                                        min_samples_split=2,
                                        min_samples_leaf=1,
                                        min_weight_fraction_leaf=0.0,
                                        max_features=self.max_features,
                                        random_state=rand_state).fit(X, y)

        threshold = clf.tree_.threshold[0]
        feature_num = clf.tree_.feature[0]
        gain = clf.tree_.impurity[0]

        if feature_num < 0:
            feature_num = 0
            is_leaf = True
        else:
            is_leaf = False

        best_split = ind2args[feature_num]
        best_split['threshold'] = threshold
        best_split['gain'] = gain
        best_split['is_leaf'] = is_leaf
        return best_split

SPLITTERS = {'sklearn': SklearnSetSplitter}
CRITERIONS = {'gini': gini, 'entropy': entropy, 'mse': mse}