import json
import operator
import re

import sqlparse
from datamanage import pizza_settings as settings
from datamanage.pro import exceptions as dm_pro_errors
from datamanage.pro.datamodel.models.model_dict import InnerField, SchedulingEngineType
from datamanage.pro.datamodel.utils import (
    get_sql_fields_alias_mapping,
    get_sql_source_columns_mapping,
)
from datamanage.utils.api import DataflowApi
from django.core.cache import cache

from common.log import logger

from . import time

python_string_type = str

class SQLVerifier(object):

    sql_template = """SELECT {fields} FROM {table_name} {conditions} {appends};"""
    default_map = dict(table_name="table_name", fields="*", conditions="", appends="")
    cast_type_mapping = dict(
        int="int",
        float="float",
        double="double",
        string={
            SchedulingEngineType.FLINK.value: "varchar",
            SchedulingEngineType.SPARK.value: "string",
        },
    )
    requests_clients = {
        SchedulingEngineType.FLINK.value: DataflowApi.flink_sql_check,
        SchedulingEngineType.SPARK.value: DataflowApi.spark_sql_check,
    }

    def __init__(self, bk_biz_id=1, sql=None, sql_parts=None, scopes=None, auto_format=True):
        """
        初始化sql校验模块，传入sql语句或局部sql以及必要参数
        SELECT [fields] FROM [table] WHERE [conditions] [appends]

        :param bk_biz_id: int 业务id
        :param sql: string 完整sql语句 "SELECT * FROM table WHERE 1"
        :param sql_parts: dict 局部sql，传入后需要对sql进行补全(sql参数若不为空，优先于sql_parts)
            {
                "fields": {"field_alias1": "field_name", "field_alias2":"concat(field_name, '_', 'ex')"},
                # fields中，key是输出的字段名称，value是输入表达式(或输入字段名称)
                "conditions": "a >= 1 and b < 2",
                "appends": {"group_by": ["b", "c"]} # OR "GROUP BY b, c"
                "table_name": "591_durant1119"
            }
        :param scopes: dict sql执行可引用的表范围
            {"591_durant1119": [
                    {
                        "field_type": "string",
                        "field_name": "a"
                    }
                ]
            }
        }
        :param auto_format: boolean 是否自动格式化sql内容
        """
        self.bk_biz_id = bk_biz_id
        self.auto_format = auto_format
        self.sql = sql
        self.sql_parts = sql_parts if sql_parts is not None else {}
        self.scopes = dict()
        self.source_table_mapping = dict()
        self._set_scope_table_name(scopes)
        self.sql_alias_exp_mapping = dict()
        self._set_sql_alias_exp_mapping()
        self.logger = logger
        self.err_trace = dict()

    @staticmethod
    def _pre_deal_scope_list(scope_field_list):
        """
        预处理scope字段列表
        1.去除无效字段
        2.添加默认字段

        :param scope_field_list: list
        :return: list
        """
        processed_scope_field_list = list()
        if isinstance(scope_field_list, list):
            processed_scope_field_list = [
                field_item
                for field_item in scope_field_list
                if isinstance(field_item, dict)
                and field_item.get("field_type", "")
                and field_item.get("field_name", "")
            ]
        processed_scope_field_list.append(dict(field_type="string", field_name="verify__bk_data_sql__default"))
        return processed_scope_field_list

    def _set_scope_table_name(self, scopes):
        """
        对scope表的表名称进行预处理

        :param scopes: dict sql执行可引用的表范围
        :return: None
        """

        raw_scopes = scopes if scopes is not None else {}
        # Todo: 因防止缓存竞争冲突，主表名称会增加随机后缀，所以需要对scope中的表名进行替换，当期因需要只替换了主表表名, 完备方案有两套:
        #   1.首选, 推动flink去掉缓存中转的方案，直接支持参数传递schema, 即可取消随机后缀
        #   2.对多输入scope情况中主表外其他所有表进行随机后缀替换，且需要替换sql中对scope表的引用表名, 较复杂，暂未实现
        if self.sql is None and self.sql_parts:
            raw_main_table_name = self.sql_parts.get("table_name", "")
            self.sql_parts["table_name"] = self._gen_table_name(self.bk_biz_id, raw_main_table_name)
            for scope_table_name, scope_table_info in list(raw_scopes.items()):
                tb_name = self.sql_parts["table_name"] if scope_table_name == raw_main_table_name else scope_table_name
                scope_table_info = self._pre_deal_scope_list(scope_table_info)
                self.scopes[tb_name] = scope_table_info
                self.source_table_mapping[tb_name] = scope_table_name
        else:
            self.scopes = raw_scopes

    def _set_sql_alias_exp_mapping(self):
        """
        从输入中获取字段别名和表达式的关系

        :return: None
        """

        fields_part = self.sql_parts.get(str("fields"), {})
        for field_alias, field_exp in list(fields_part.items()):
            if field_alias and field_exp:
                self.sql_alias_exp_mapping[field_alias] = field_exp

    @staticmethod
    def get_func_list(expr, prefix=None):
        """
        从表达式中捕获使用的函数

        :param expr: string 表达式
        :param prefix: string 指定函数名的前缀
        :return: list 捕获函数列表
        """
        func_list = list(set(re.findall(r"([\S]+?)\(", expr)))
        if prefix and isinstance(prefix, str):
            prefix = prefix.lower()
            return list({func_name for func_name in func_list if func_name.lower().startswith(prefix)})
        return func_list

    @classmethod
    def gen_simple_sql(cls, table_name=None, fields=None, conditions=None, appends=None):
        """
        生成一个基础sql

        :param table_name: string 表名
        :param fields: string 字段表达式
        :param conditions: string 条件表达式(包含where)
        :param appends: string 聚合表达式
        :return: string sql表达式
        """
        raw_sql = cls.sql_template.format(
            table_name=table_name if table_name else cls.default_map["table_name"],
            fields=fields if fields else cls.default_map["fields"],
            conditions=conditions if conditions else cls.default_map["conditions"],
            appends=appends if appends else cls.default_map["appends"],
        )
        return sqlparse.format(raw_sql, strip_comments=True, strip_whitespace=True).strip()

    def gen_sql_expression(self, require_schema, engine):
        """
        根据局部sql和参数构建校验用的sql表达式

        :param require_schema: dict 希望sql返回的schema
        :param engine: string 执行sql的引擎
        :return: 返回str的sql语句
        """

        require_schema_fields = list(require_schema.values())[0]
        field_type_mapping = {
            str(field_item["field_name"]): str(field_item["field_type"]) for field_item in require_schema_fields
        }
        raw_sql = self.sql_template.format(
            table_name=self.sql_parts["table_name"],
            fields=self._gen_fields_exp(field_type_mapping, engine),
            conditions=self._gen_conditions_exp(),
            appends=self._gen_appends_exp(),
        )
        self.sql = sqlparse.format(raw_sql, strip_comments=self.auto_format, strip_whitespace=self.auto_format).strip()
        return self.sql

    @staticmethod
    def _gen_table_name(prefix, table_name):
        """
        生成表名

        :param prefix: 业务id前缀(兼容flink接口)
        :param table_name: 表名
        :return: string 字符串表名
        """
        # 默认带上时间戳后缀，避免缓存竞争
        table_name = "{}_{}".format(table_name, int(round(time.time() * 1000)))
        return "{}_{}".format(prefix, table_name) if not table_name.startswith("{}_".format(prefix)) else table_name

    def _gen_fields_exp(self, field_type_mapping, engine):
        """
        SQL片段 fields部分
        {"key1": "field_1", "key2": "sum(a) as field_2"}
        变量解释: field_name: 模型要求的输出字段 field_alias: 从表达式解析出来的别名 field_expr: sql字段表达式

        :param field_type_mapping: dict 字段类型映射字典
        :param engine: string 执行sql的引擎
        :return: string 字段表达式
        """

        field_exp_list = []
        fields_alias_mapping = get_sql_fields_alias_mapping(
            self.sql_alias_exp_mapping,
            strip_comments=self.auto_format,
            strip_whitespace=self.auto_format,
        )
        for field_name, field_alias_item in list(fields_alias_mapping.items()):
            field_alias = field_alias_item["alias"]
            field_expr = field_alias_item["expr"]
            target_field_type = self.get_target_parse_type(field_type_mapping[field_name], engine)
            # 表达式和模型字段名一致(没做处理)
            if field_expr == field_name:
                field_exp_list.append("cast(`{}` as {}) as `{}`".format(field_expr, target_field_type, field_name))
            elif field_alias:
                # 表达式中设置了指定别名
                if field_alias == field_name:
                    field_exp_list.append("cast({} as {}) as `{}`".format(field_expr, target_field_type, field_name))
                # 表达式中没有设置指定别名
                else:
                    raise dm_pro_errors.AliasSetError(message_kv={"alias": field_name, "err_alias": field_alias})
            else:
                # 表达式没有设置别名，自动补充
                field_exp_list.append("cast({} as {}) as `{}`".format(field_expr, target_field_type, field_name))
        return ", ".join(field_exp_list)

    def _gen_conditions_exp(self):
        """
        SQL片段 conditions部分
        "a >= 1 and b < 2" OR dict(暂未实现)

        :return: string 条件表达式
        """

        conditions_part = self.sql_parts.get(str("conditions"), None)
        if conditions_part and isinstance(conditions_part, dict):
            # TODO:暂未实现字典类型传入
            return ""
        conditions_expr = format(conditions_part) if conditions_part else self.default_map["conditions"]
        return (
            "WHERE {}".format(conditions_expr)
            if conditions_expr and not conditions_expr.strip().lower().startswith("where")
            else conditions_expr
        )

    def _gen_appends_exp(self):
        """
        SQL片段 appends部分

        {"group_by": ["b", "c"]} # OR "GROUP BY b, c"
        :return: string 聚合表达式
        """

        appends_part = self.sql_parts.get(str("appends"), None)
        if appends_part and isinstance(appends_part, dict):
            appends_exp_template = "{group_by_exp}"
            group_by_exp_str = ""
            group_by_exp_list = appends_part.get("group_by", [])
            if group_by_exp_list:
                group_by_exp_str = "GROUP BY {}".format(", ".join(group_by_exp_list))
            return appends_exp_template.format(group_by_exp=group_by_exp_str)
        return appends_part if appends_part else self.default_map["appends"]

    @classmethod
    def get_target_parse_type(cls, require_type, engine):
        """
        根据类型和engine获取实际cast的目标类型

        :param require_type: 用户需要的通用字段类型
        :param engine: 执行引擎
        :return: 实际有效的执行字段类型
        """

        require_type = require_type if require_type in cls.cast_type_mapping else "string"
        target_type_item = cls.cast_type_mapping.get(require_type)
        return str(target_type_item[engine]) if isinstance(target_type_item, dict) else str(target_type_item)

    def _get_runner_properties(self, engine, udf_list, ret_table_name_list, agg):
        """
        根据校验引擎构造参数

        :param engine: string 校验引擎
        :param udf_list: list udf列表
        :param ret_table_name_list: list 期望获得的表名列表
        :param agg: boolean 是否需要聚合
        :return: dict 校验参数
        """
        # Todo: 配置化
        param_dict = {
            "flink_sql": {
                "flink.env": "product",
                "flink.bk_biz_id": self.bk_biz_id,
                "flink.source_data": list(self.scopes.keys()),
                "flink.function_name": ", ".join(udf_list),
                "flink.result_table_name": ", ".join(ret_table_name_list),
                "flink.lateness_time": 1,
                "flink.system_fields": [],
                "flink.expired_time": 0,
                "flink.allowed_lateness": False,
                "flink.session_gap": 0,
                "flink.window_length": 0,
                "flink.count_freq": 1,
            },
            "spark_sql": {
                "spark.dataflow_udf_env": "product",
                "spark.bk_biz_id": self.bk_biz_id,
                "spark.input_result_table": [],
                "spark.input_result_table_with_schema": self.scopes,
                "spark.dataflow_udf_function_name": ", ".join(udf_list),
                "spark.result_table_name": ", ".join(ret_table_name_list),
                "spark.self_dependency_config": {},
            },
        }
        properties = param_dict.get(engine, {})
        if agg and engine == SchedulingEngineType.FLINK.value:
            properties["flink.window_type"] = "tumbling"
            properties["flink.window_length"] = 30
        return properties

    @classmethod
    def _pre_deal_validation_message(cls, message):
        """
        对校验api返回的错误信息进行处理后再输出

        :param message: 原始错误信息
        :return: 处理后的错误信息
        """
        match_obj = re.match(
            r'ParseException: Encountered "[^\w]*([\w]+)" at(.+)Was expecting one of',
            message,
            re.S,
        )
        if match_obj:
            message = '"{0}"是一个保留关键字,请在"字段加工逻辑"中用"`{0}`"替换"{0}"来进行转义(仅替换AS之前的部分)'.format(match_obj.group(1))
        return message

    def _check_sql_validation(self, require_schema, engine, agg):
        """
        提取sql中的udf等信息，组装参数，请求sql校验api

        :param require_schema: dict 期望得到的结果schema字典
        :param engine: string sql检查引擎
        :param agg: boolean 是否需要聚合
        :return: tuple (boolean, dict) check后得到的结果和schema
            CASE 成功: (True, {message: succeed, schema: table_name: [
                {field_name: YYY, field_type: int},
                {field_name: XXX, field_type: string}
            ]})
        """

        # if self.sql is None:
        self.gen_sql_expression(require_schema, engine)
        udf_list = self.get_func_list(self.sql, prefix="udf_")
        ret_table_name_list = list(require_schema.keys())
        check_sql = self.sql.encode("utf-8") if isinstance(self.sql, str) else self.sql
        resp = self.requests_clients[engine](
            {
                "sql": check_sql,
                "properties": self._get_runner_properties(engine, udf_list, ret_table_name_list, agg),
            },
            raise_exception=False,
        )
        if not resp.is_success() or not resp.data:
            self.err_trace = dict(
                message=self._pre_deal_validation_message(resp.message),
                code=resp.code,
                sql=check_sql,
            )
            return False, self.err_trace
        # 对返回数据进行格式化清洗
        ret_data = resp.data
        ret_fields_dict = dict()
        for schema_item in ret_data:
            for field_item in schema_item.get("fields", []):
                if schema_item["name"] not in ret_fields_dict:
                    ret_fields_dict[schema_item["name"]] = list()
                if "field_name" not in field_item and "field" in field_item:
                    field_item["field_name"] = field_item["field"]
                if "field_type" not in field_item and "type" in field_item:
                    field_item["field_type"] = field_item["type"]
                if field_item["field_name"] in InnerField.INNER_FIELD_LIST:
                    continue
                ret_fields_dict[schema_item["name"]].append(
                    dict(
                        field_name=field_item["field_name"],
                        field_type=field_item["field_type"],
                    )
                )
        return True, dict(message="check succeed", schema=ret_fields_dict)

    def get_err_trace(self):
        return self.err_trace

    @staticmethod
    def check_schema_consistency(ret_schema, require_schema):
        """
        校验每个返回的表及其字段是否符合预期

        :param ret_schema: 校验返回结果schema
        :param require_schema: 期望得到schema
        :return:  tuple (boolean, None/string)
            CASE 校验成功: (True, None)
            CASE 校验失败: (False, 错误Message)
        """

        matched = False
        match_info = None
        # 校验返回表名列表是否匹配
        if operator.ne(set(require_schema.keys()), set(ret_schema.keys())):
            return False, "require tables: {}, but return tables: {}".format(
                ", ".join(list(require_schema.keys())),
                ", ".join(list(ret_schema.keys())),
            )
        # 校验返回字段信息是否匹配
        for tb_name, fields in list(require_schema.items()):
            require_fields_dict = {str(field["field_name"]): field for field in fields}
            return_fields_dict = {str(ret_field["field_name"]): ret_field for ret_field in ret_schema[tb_name]}
            # 校验字段名列表
            if operator.ne(set(require_fields_dict.keys()), set(return_fields_dict.keys())):
                match_info = "require fields: {}, but return fields: {}".format(
                    ", ".join(list(require_fields_dict.keys())),
                    ", ".join(list(return_fields_dict.keys())),
                )
                break
            # 校验字段类型
            for field in fields:
                require_type = field["field_type"]
                if require_type == "numeric":
                    require_type_list = settings.VALID_NUMERIC_TYPE_LIST
                elif require_type == "float":
                    require_type_list = ["float", "double"]
                else:
                    require_type_list = [require_type]
                ret_field_type = return_fields_dict[field["field_name"]]["field_type"]
                if ret_field_type not in require_type_list:
                    match_info = '{} should be "{}" type, given "{}"'.format(
                        field["field_name"], require_type, ret_field_type
                    )
                    break
            if match_info is not None:
                break
        else:
            matched = True
        return matched, match_info

    def check_res_schema(
        self,
        require_schema,
        engines=SchedulingEngineType.FLINK.value,
        agg=True,
        extract_source_column=False,
    ):
        """
        验证sql是否能得到期望的schema

        :param require_schema: dict 期望得到的结果schema字典
        numeric 指数值类型，int，float，bigint等任一数值类型均可
        {
            "require_result_table_name": [
                {"field_name": "field_name_1", "field_type": "int"},
                {"field_name": "field_name_2", "field_type": "numeric"},
                {"field_name": "field_name_3", "field_type": "text"}
            ]
        }
        :param engines: list/string sql检查引擎
        :param agg: boolean 是否需要聚合
        :param extract_source_column: boolean 是否解析提取source_column
        :return: tuple (boolean, dict/string)
            CASE 校验成功: (True, 校验相关信息)
            CASE 校验失败: (False, 错误Message)
        """
        if not isinstance(engines, list):
            engines = [engines]
        matched = False
        ret_info = dict()
        for engine in engines:
            if engine == SchedulingEngineType.FLINK.value:
                cache_key = None
                try:
                    for scope_table_name, scope_table_info in list(self.scopes.items()):
                        cache_key = settings.SQL_VERIFY_CACHE_KEY.format(table=scope_table_name)
                        cache_info = {
                            "bk_biz_id": self.bk_biz_id,
                            "result_table_name": scope_table_name,
                            "fields": scope_table_info,
                        }
                        cache.set(cache_key, json.dumps(cache_info), 300)
                except Exception as e:
                    raise dm_pro_errors.SetCacheError(message_kv={"cache_key": cache_key, "error": e})
            check_flag, check_info = self._check_sql_validation(require_schema, engine, agg)
            # 校验sql失败
            if not check_flag:
                return False, check_info["message"]
            ret_schema = check_info["schema"]
            matched, match_info = self.check_schema_consistency(ret_schema, require_schema)
            if not matched:
                return False, match_info
        if extract_source_column:
            source_columns_mapping = get_sql_source_columns_mapping(self.sql, formatted=True)
            ret_info["source_columns"] = source_columns_mapping["fields"]
            ret_info["condition_columns"] = source_columns_mapping["conditions"]
        return matched, ret_info

    @property
    def show_sql(self):
        show_sql = self.sql
        for processed_table_name in list(self.scopes.keys()):
            show_sql = show_sql.replace(processed_table_name, self.source_table_mapping[processed_table_name])
        return show_sql

def validate_calculation_content(
    table_name,
    calculation_atom_name,
    field_type,
    calculation_formula,
    schema_list,
    filter_formula=None,
    aggregation_fields=None,
):
    """
    统计口径和指标SQL校验
    :param table_name: {String} 主表名称
    :param calculation_atom_name: {String} 统计口径名称
    :param field_type: {String} 数据类型
    :param calculation_formula: {String} 统计SQL
    :param schema_list: {List} 主表字段列表
    :param filter_formula: {String} 过滤条件
    :param aggregation_fields: {List} 聚合字段列表
    :return:
    """
    schema_dict = {table_name: schema_list}
    calculation_content_dict = {
        "scopes": schema_dict,
        "sql_parts": {
            "fields": {calculation_atom_name: calculation_formula},
            "conditions": filter_formula,
            "appends": {"group_by": aggregation_fields},
            "table_name": table_name,
        },
    }
    require_schema_dict = {table_name: [{"field_type": field_type, "field_name": calculation_atom_name}]}

    verifier = SQLVerifier(**calculation_content_dict)
    matched, match_info = verifier.check_res_schema(
        require_schema_dict,
        [SchedulingEngineType.FLINK.value, SchedulingEngineType.SPARK.value],
        extract_source_column=True,
    )
    if not matched:
        # matched为False时，match_info是错误message
        raise dm_pro_errors.SqlValidateError(
            message_kv={"error_info": "{} [ERROR_SQL: {}]".format(match_info, verifier.show_sql)}
        )
    # matched为True时, match_info是Dict, 例如{'source_columns': {'item_sales_amt': ['channel_id', 'price']}}
    return match_info["source_columns"], match_info["condition_columns"]

def strip_comment(orig_sql):
    """
    删除sql中的注释语句
    :param orig_sql: {String} 原始sql片段
    :return: {String} 删除注释后的sql片段
    """
    return sqlparse.format(orig_sql, strip_comments=True).strip()


from imutils.object_detection import non_max_suppression
import numpy as np
import pytesseract as py
import argparse
import cv2
import re
import os
import matplotlib.pyplot as plt
from PIL import Image, ImageEnhance, ImageFilter

py.pytesseract.tesseract_cmd = r"C:\Program Files (x86)\Tesseract-OCR\tesseract.exe"
IMAGE_PATH = "name_images/"

#psm 13 or psm8 seems best

def clean_string(names):
    cleaned = ''
    names = ''.join(e for e in names if e.isalnum() or e.isspace())
    for i in re.sub(r'\s+', ' ', names).split():
        if len(i) == 1 or i.islower() or i.lower() == 'ff':
            continue
        cleaned = cleaned + i.upper() + " "
    cleaned = cleaned.rstrip()
    numNames = len(cleaned.split())
    if numNames == 6:
        msg = "Success ==> Found " + str(numNames) + " ==> "
    else:
        msg = "Fail ==> Found " + str(numNames) + " ==> "

    print(msg + cleaned)
    return cleaned

def killfeed_name_recognition(img_path):
    name_config = ('-l eng --oem 3 --psm 7')
    left,  right = 950, 1280
    upper, lower = 107, 145
    lower_yellow, upper_yellow  = np.array([20, 100, 100]), np.array([40, 255, 255])
    lower_blue, upper_blue = np.array([70, 100, 100]), np.array([130, 255, 255])

    img = cv2.imread(img_path)
    img = cv2.bilateralFilter(img,9,75,75)
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    mask = cv2.inRange(hsv, lower_yellow, upper_yellow)
    res = cv2.bitwise_and(img, img, mask = mask)

    h, s, v1 = cv2.split(res)

    killfeed_row = v1[upper: lower, 1200: right]
    killfeed_names = py.image_to_string(killfeed_row, config=name_config)
    cv2.imshow('hello', killfeed_row)
    cv2.waitKey(0)

def row_name_recognition(img_path):
    #Variables
    config = ('-l eng --oem 3 --psm 8')
    #name_config = ('-l eng --oem 3 --psm 6')
    left, right = 30, 448
    left1, right1 = 835, 1250
    upper, lower = 75, 90
    left2, right2 = 0, 70

    #Read image and apply bilateral filter
    img = cv2.imread(img_path)
    img = cv2.bilateralFilter(img,9,75,75)
    #Turn image gray, apply Gaussian threshold, Invert Colors
    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 31, 2)
    img = cv2.bitwise_not(img)

    #Crop name section
    left_side = img[upper:lower, left:right]
    right_side = img[upper:lower, left1:right1]

    #Convert to Text
    left_names = py.image_to_string(left_side, config=config)
    left_names = clean_string(left_names)
    right_names = py.image_to_string(right_side, config=config)
    right_names = clean_string(right_names)
    return left_names, right_names

    """
    #Get Individual names
    for i in range(0,12):
        if i == 6:
            left2, right2 = 0, 70
        if i < 6:
            name = left_side[0:15, left2:right2]
        else:
            name = right_side[0:15, left2:right2]
        text = py.image_to_string(name, config=name_config)
        text = ''.join(e for e in text if e.isalnum())
        #print("Name: " + text)
        left2 += 70
        right2 += 70
    """

def make_header(full_path):
    print("On Image: " + full_path)
    print("==================================")

if __name__ == '__main__':

    for img in os.listdir(IMAGE_PATH):
        full_path = IMAGE_PATH + img
        make_header(img)
        #killfeed_name_recognition(full_path)
        row_name_recognition(full_path)
        print("")


from flask import render_template, current_app, request, abort
from flask_login import login_required
from sqlalchemy.orm import subqueryload, contains_eager
from collections import defaultdict

import os
import re

"""
http://docs.sqlalchemy.org/en/latest/orm/loading_relationships.html#what-kind-of-loading-to-use
When using subquery loading, the load of 100 objects will emit two SQL statements.
The second statement will fetch a total number of rows equal to the sum of the size of all collections.
An INNER JOIN is used, and a minimum of parent columns are requested, only the primary keys.
So a subquery load makes sense when the collections are larger.
"""
from app.helpers.history import create_event
from . import reports
from app import db
from app.models import Report
from app.models import Check

@reports.route('/upload', methods=['GET', 'POST'])
@login_required
def upload():
    return render_template('layout/not-ready.html')

@reports.route('/history', methods=['GET', 'POST'])
@login_required
def history():
    return render_template('layout/not-ready.html')

# placeholder route to allow the creation of /reports/ url
@reports.route('/', methods=['GET'])
@login_required
def index():
    return render_template('layout/not-ready.html')

@reports.route('/<report_id>/metrics', methods=['GET', 'POST'])
@login_required
def display_metrics(report_id):
    return render_template('layout/not-ready.html')

@reports.route('/<report_id>/compare', methods=['GET', 'POST'])
@login_required
def compare(report_id):
    return render_template('layout/not-ready.html')

@reports.route('/<report_id>/checks', methods=['GET', 'POST'])
@login_required
def display_checks(report_id, rc=None):
    rc = request.args.get('rc')
    # Getting the checks from the DB
    q = db.session.query(Report).join(Check, Report.checks).options(contains_eager(Report.checks))
    report_name = "show"
    if rc is not None:
        q = q.filter(Check.global_rc == rc)
        report_name = "list"
    q = q.filter(Report.id == report_id).order_by(Check.priority.desc())
    report = q.options(subqueryload('checks', 'datahooks'), subqueryload('checks', 'check_results')).all()
    if report is None or len(report) == 0:
        abort(404)
    # Getting a list of categories
    create_event("checks", "report", [report[0].id])
    categories = defaultdict(int)
    for c in report[0].checks:
        # Building the categories dict
        categories[c.category] += 1
        subcategories = []
        if c.subcategory != "":
            subcategories = c.subcategory.split("/")
        c.subcategory = subcategories
        if c.category not in subcategories:
            subcategories.insert(0, c.category)
        c.all_categories = subcategories
        for s in subcategories:
            categories[s] += 1

        # Building the visual for the plugin state
        c.plugin_html = current_app.config["PLUGIN_STATES"][c.global_rc]
        c.plugin_name = os.path.splitext(os.path.basename(c.plugin_path))[0]

        # Building the visual for the plugin priority
        c.priority_text = "informative"
        c.priority_class = "info"
        if not hasattr(c, 'priority'):
            c.priority = 0
        if c.priority >= 666:
            c.priority_text = "critical"
            c.priority_class = "danger"
        elif c.priority >= 333:
            c.priority_text = "important"
            c.priority_class = "warning"

        # Extracting the bugzilla informations
        if re.match('^\d{7}$', c.plugin_name):
            c.bug_id = c.plugin_name
            c.bugzilla = "https://bugzilla.redhat.com/show_bug.cgi?id=" + str(c.bug_id)
        else:
            try:
                m = re.search('([0-9]+$)', c.bugzilla)
                c.bug_id = m.group(0)
            except:
                pass
    return render_template('reports/' + report_name + '.html', report=report[0], categories=categories)


#! pip install git+https://www.github.com/keras-team/keras-contrib.git

#Import base libraries

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt

import os

import warnings

from tqdm import tqdm_notebook as tqdm

import random

from scipy import interp

import statsmodels.api as sm

warnings.filterwarnings("ignore", category=FutureWarning)

plt.style.use('ggplot')

#Import sklearn libraries

from sklearn.metrics import accuracy_score,roc_auc_score

from sklearn.ensemble import ExtraTreesClassifier

from sklearn.model_selection import KFold,cross_val_predict,cross_validate,cross_val_score,train_test_split

from sklearn.preprocessing import StandardScaler,MinMaxScaler,RobustScaler

#Help functions

def plot_2d_space(X, y, label='Classes'):

    plt.figure(figsize=(8,8))

    markers = ['o', 's']

    for l, m in zip(np.unique(y), markers):

        plt.scatter(

            X[y==l, 0],

            X[y==l, 1]

            , label=l, marker=m

            )

    plt.title(label)

    plt.legend(loc='upper right')

    plt.show()

#ROC/AUC

def plot_roc_curve(fpr, tpr):

    plt.plot(fpr, tpr, color='orange', label='ROC')

    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')

    plt.xlabel('False Positive Rate')

    plt.ylabel('True Positive Rate')

    plt.title('Receiver Operating Characteristic (ROC) Curve')

    plt.legend()

    plt.show()
# GET INDICIES OF REAL TEST DATA FOR FE

#######################

# TAKE FROM YAG320'S KERNEL

# https://www.kaggle.com/yag320/list-of-fake-samples-and-public-private-lb-split

test_path = '../input/test.csv'

train_path = '../input/train.csv'

df_test = pd.read_csv(test_path)

df_test.drop(['ID_code'], axis=1, inplace=True)

df_test = df_test.values

unique_samples = []

unique_count = np.zeros_like(df_test)

for feature in range(df_test.shape[1]):

    _, index_, count_ = np.unique(df_test[:, feature], return_counts=True, return_index=True)

    unique_count[index_[count_ == 1], feature] += 1

# Samples which have unique values are real the others are fake

real_samples_indexes = np.argwhere(np.sum(unique_count, axis=1) > 0)[:, 0]

synthetic_samples_indexes = np.argwhere(np.sum(unique_count, axis=1) == 0)[:, 0]

print('Found',len(real_samples_indexes),'real test')

print('Found',len(synthetic_samples_indexes),'fake test')

###################

d = {}

for i in range(200): d['var_'+str(i)] = 'float32'

d['target'] = 'uint8'

d['ID_code'] = 'object'

train = pd.read_csv('../input/train.csv', dtype=d)

test = pd.read_csv('../input/test.csv', dtype=d)

print('Loaded',len(train),'rows of train')

print('Loaded',len(test),'rows of test')

print('Found',len(real_samples_indexes),'real test')

print('Found',len(synthetic_samples_indexes),'fake test')

###################

d = {}

for i in range(200): d['var_'+str(i)] = 'float32'

d['target'] = 'uint8'

d['ID_code'] = 'object'

train = pd.read_csv(train_path, dtype=d)

test = pd.read_csv(test_path, dtype=d)

print('Loaded',len(train),'rows of train')

print('Loaded',len(test),'rows of test')
# FREQUENCY ENCODE

def encode_FE(df,col,test):

    cv = df[col].value_counts()

    nm = col+'_FE'

    df[nm] = df[col].map(cv)

    test[nm] = test[col].map(cv)

    test[nm].fillna(0,inplace=True)

    if cv.max()<=255:

        df[nm] = df[nm].astype('uint8')

        test[nm] = test[nm].astype('uint8')

    else:

        df[nm] = df[nm].astype('uint16')

        test[nm] = test[nm].astype('uint16')

    return

test['target'] = -1

comb = pd.concat([train,test.loc[real_samples_indexes]],axis=0,sort=True)

for i in range(200):

    encode_FE(comb,'var_'+str(i),test)

train = comb[:len(train)]; del comb

print('Added 200 new magic features!')
del df_test, real_samples_indexes, synthetic_samples_indexes, unique_count, unique_samples, d
#Create DataFrames

# Load data with counts saved in the previous cells

df_train_data = train.drop(columns=['ID_code'])

df_train_X = df_train_data.drop(columns=['target'])

reverse_columns = True

if reverse_columns:

    reverse_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 15, 16, 18, 19, 22, 24, 25, 26,

                    27, 29, 32, 35, 37, 40, 41, 47, 48, 49, 51, 52, 53, 55, 60, 61,

                    62, 65, 66, 67, 69, 70, 71, 74, 78, 79, 82, 84, 89, 90, 91, 94,

                    95, 96, 97, 99, 103, 105, 106, 110, 111, 112, 118, 119, 125, 128,

                    130, 133, 134, 135, 137, 138, 140, 144, 145, 147, 151, 155, 157,

                    159, 161, 162, 163, 164, 167, 168, 170, 171, 173, 175, 176, 179,

                    180, 181, 184, 185, 187, 189, 190, 191, 195, 196, 199,

                    ]

    for j in reverse_list:

        df_train_X[f'var_{j}'] *= -1
#Define X,y

y = df_train_data['target'].values

X = df_train_X.values

print(df_train_X.shape)
#Check the distribution of target classes

df_train_data.groupby(by="target")["target"].count().plot(kind="bar",title="Distribution Of Classes")
# Normalize data

rs = RobustScaler()

X_scaled_rs = rs.fit_transform(X)
#Let`s plot a 2d scatter plot to see the distribution of 2 random dimensions

n = random.randrange(start=0,stop=198)

print(n)

plot_2d_space(X_scaled_rs[:,n:n+2], y[:], 'Imbalanced dataset')
from sklearn.metrics import accuracy_score,classification_report

from sklearn.metrics import confusion_matrix,roc_curve, auc,roc_auc_score

from sklearn.ensemble import ExtraTreesClassifier

from sklearn.model_selection import KFold,cross_validate,cross_val_predict,cross_val_score,StratifiedKFold

import seaborn as sns

ExtraTreeModel = ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',

                                   max_depth=None, max_features='auto', max_leaf_nodes=None,

                                   min_impurity_decrease=0.0, min_impurity_split=None,

                                   min_samples_leaf=1, min_samples_split=2,

                                   min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,

                                   oob_score=False, random_state=None, verbose=0, warm_start=False)
#Train with cross validation

cv = StratifiedKFold(n_splits=5)

tprs = []

aucs = []

mean_fpr = np.linspace(0, 1, 100)

i = 0

for k_train, k_test in tqdm(cv.split(X_scaled_rs, y)):

    probas_ = ExtraTreeModel.fit(X_scaled_rs[k_train], y[k_train]).predict_proba(X_scaled_rs[k_test])

    # Compute ROC curve and area the curve

    fpr, tpr, thresholds = roc_curve(y[k_test], probas_[:, 1])

    tprs.append(interp(mean_fpr, fpr, tpr))

    tprs[-1][0] = 0.0

    roc_auc = auc(fpr, tpr)

    aucs.append(roc_auc)

    plt.plot(fpr, tpr, lw=1, alpha=0.3,

             label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))

    i += 1

plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',

         label='Chance', alpha=.8)

mean_tpr = np.mean(tprs, axis=0)

mean_tpr[-1] = 1.0

mean_auc = auc(mean_fpr, mean_tpr)

std_auc = np.std(aucs)

plt.plot(mean_fpr, mean_tpr, color='b',

         label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),

         lw=2, alpha=.8)

std_tpr = np.std(tprs, axis=0)

tprs_upper = np.minimum(mean_tpr + std_tpr, 1)

tprs_lower = np.maximum(mean_tpr - std_tpr, 0)

plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,

                 label=r'$\pm$ 1 std. dev.')

plt.xlim([-0.05, 1.05])

plt.ylim([-0.05, 1.05])

plt.xlabel('False Positive Rate')

plt.ylabel('True Positive Rate')

plt.title('Receiver operating characteristic')

plt.legend(loc="lower right")

plt.show()
#Import NB model

from sklearn.naive_bayes import GaussianNB

def create_NB_Model():

    model = GaussianNB()

    return model

def fit_NB_Model(Model,X,y,cv_splits=5):

    cv = KFold(n_splits=cv_splits)

    tprs = []

    aucs = []

    mean_fpr = np.linspace(0, 1, 100)

    for train,test in cv.split(X, y):

        probas_ = Model.fit(X[train], y[train]).predict_proba(X[test])

        # Compute ROC curve and area the curve

        fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])

        tprs.append(interp(mean_fpr, fpr, tpr))

        tprs[-1][0] = 0.0

        roc_auc = auc(fpr, tpr)

        aucs.append(roc_auc)

    mean_tpr = np.mean(tprs, axis=0)

    mean_tpr[-1] = 1.0

    mean_auc = auc(mean_fpr, mean_tpr)

    return Model,mean_auc

def get_NB_results(X,y,cv_splits=5):

    model_aucs  = []

    models = {}

    for k in tqdm(range(0,int(X.shape[1]/2))):

        Model = create_NB_Model()

        X_train = X[:,[k,k+200]]#.reshape(-1,1)

        model,auc = fit_NB_Model(Model,X_train,y,cv_splits=cv_splits)

        #Append model

        models["Model_"+str(k)] =model

        model_aucs.append(auc)

        print(str(auc)+str(" AUC Average for Model_")+str(k))

        #Fit each Model

    return models,np.array(model_aucs)

def make_NB_predictions(X_train,y_train,cv,X_test):

    #Create a new array to store results

    train_predictions = np.zeros_like(X_train)

    test_predictions = np.zeros_like(X_test)

    #Fit model

    models,model_aucs = get_NB_results(X_train,y_train,cv_splits=cv)

    #Iterate over the X

    for k in range(0,int(X_train.shape[1]/2)):

        X_train_pred = X_train[:,[k,k+200]]#.reshape(-1,1)

        train_probas_ = models.get("Model_"+str(k)).predict_proba(X_train_pred)[:, 1]

        train_predictions[:, k] = train_probas_

    for j in range(0,int(X_test.shape[1]/2)):

        X_test_pred = X_test[:,[j,j+200]]#.reshape(-1,1)

        test_probas_ = models.get("Model_"+str(j)).predict_proba(X_test_pred)[:, 1]

        test_predictions[:, j] = test_probas_

    return test_predictions,train_predictions

def save_submit_file(predictions, filename, test_filename=test_path, index_column='ID_code', target_column = 'target'):

    df_test_submit = pd.read_csv(test_filename).set_index(index_column)

    df_test_submit[target_column] = predictions

    df_test_submit[[target_column]].to_csv(filename)

    return
_,train_predictions = make_NB_predictions(X_train = X_scaled_rs,y_train = y,cv = 5,X_test=X_scaled_rs)
from sklearn.linear_model import LogisticRegression

ensemble_model = LogisticRegression()

ensemble_model.fit(X = train_predictions, y = y)
fpr, tpr, thresholds = roc_curve(y, ensemble_model.predict_proba(train_predictions)[:, 1])

plot_roc_curve(fpr, tpr)
# Normalize test data

df_test = test.drop(columns=['ID_code', 'target'])

X_test = df_test.values

X_test_scaled_rs = rs.fit_transform(X_test)
test_predictions,train_predictions = make_NB_predictions(X_train = X_scaled_rs,y_train = y,cv = 5,X_test=X_test_scaled_rs)
int(X.shape[1]/2)
ensemble_model.fit(X = train_predictions, y = y)

predictions = ensemble_model.predict(test_predictions)

save_submit_file(predictions, "submission_rev4.csv")
from IPython.display import FileLink

FileLink('submission_rev4.csv')


"""WebSiTi URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
"""

from django.conf.urls import patterns, url, include
from . import views

urlpatterns = patterns('',
        url(r'^$', views.index, name='index'),

        url(r'^blog/', include('blog.urls')),
        url(r'^cms/', include('cms.urls')),

)
from django.conf import settings # New Import
from django.conf.urls.static import static # New Import

if not settings.DEBUG:
        urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

from django.conf import settings

# UNDERNEATH your urlpatterns definition, add the following two lines:
if settings.DEBUG:
    urlpatterns += patterns(
        'django.views.static',
        (r'^media/(?P<path>.*)',
        'serve',
        {'document_root': settings.MEDIA_ROOT}), )


# -*-coding:utf-8-*-

"""
Author: Rage
Date: 19-4-25
Refer: https://mings-blog.readthedocs.io/zh_CN/latest/c02/c02_01.html
https://mings-blog.readthedocs.io/zh_CN/latest/index.html
"""
import requests
import time
from threading import Thread
from next_station.common import get_func_time
from multiprocessing import Process
from next_station.logger import logger

# CPU计算密集型
def count(x=1, y=1):
    # 使程序完成150万计算
    c = 0
    while c < 500000:
        c += 1
        x += x
        y += y

# 磁盘读写IO密集型
def io_disk():
    with open("file.txt", "w") as f:
        for x in range(5000000):
            f.write("python-learning\n")

# 网络IO密集型
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36'}
url = "https://www.baidu.com/"

def io_request():
    try:
        webPage = requests.get(url, headers=header)
        html = webPage.text
        return
    except Exception as e:
        return {"error": e}

# 【模拟】IO密集型
def io_simulation():
    time.sleep(2)

@get_func_time
def multi_thread(func, func_type=""):
    print("type is {0}".format(func_type))
    thread_list = []
    for i in range(10):
        t = Thread(target=func, args=())
        thread_list.append(t)
        t.start()
    e = len(thread_list)

    while True:
        for th in thread_list:
            if not th.is_alive():
                e -= 1
        if e <= 0:
            break

# # 多线程
# multi_thread(count, func_type="CPU计算密集型")
# multi_thread(io_disk, func_type="磁盘IO密集型")
# multi_thread(io_request, func_type="网络IO密集型")
# multi_thread(io_simulation, func_type="模拟IO密集型")

@get_func_time
def multi_process(func, fun_type=""):
    logger.info("multi_process type is {0}".format(fun_type))
    process_list = []
    for x in range(3):
        p = Process(target=func, args=())
        process_list.append(p)
        p.start()
    e = process_list.__len__()

    while True:
        for pr in process_list:
            if not pr.is_alive():
                e -= 1
        if e <= 0:
            break

if __name__ == '__main__':
    fun_list = [count, io_disk, io_request, io_simulation]
    for fun in fun_list:
        # multi_thread(fun)
        multi_process(fun)


#Simulating L&L Course on Theoretical Physics: Classical Mechanics Problem 1.1
import numpy as np
import matplotlib.pyplot as plt

# Var
l1 = 2
l2 = 2
g = 9.8
m1 = 1
m2 = 1

# Function

def dub_pen(theta1_i, theta2_i, ang_vel1_i, ang_vel2_i, tot_time, tau=0.05):
    time_vec = [0]*tot_time
    theta1_vec = [0]*tot_time
    ang_vel1_vec = [0]*tot_time
    theta2_vec = [0]*tot_time
    ang_vel2_vec = [0]*tot_time

    theta1 = theta1_i
    ang_vel1 = ang_vel1_i

    theta2 = theta2_i
    ang_vel2 = ang_vel2_i

    for i in range(0, tot_time):
        ang_vel1_old = ang_vel1
        theta1_old = theta1
        ang_vel2_old = ang_vel2
        theta2_old = theta2

        ang_vel1 = ang_vel1_old+((((m2*g*np.sin(theta2_old)*np.cos(theta1_old-theta2_old))-
                                 (m2*np.sin(theta1_old-theta2_old)*((l1*(ang_vel1_old**2)*np.cos(theta1_old-theta2_old))+l2*(ang_vel2_old**2)))
                                 -((m1+m2)*g*np.sin(theta1_old)))/(l1*(m1+(m2*(np.sin(theta1_old-theta2_old)**2)))))*tau)
        ang_vel2 = ang_vel2_old+(((((m1+m2)*(l1*(ang_vel1_old**2)*np.sin(theta1_old-theta2_old)-
                                           g*np.sin(theta2_old)+g*np.sin(theta1_old)*np.cos(theta1_old-theta2_old)))
                                 +(m2*l2*(ang_vel2_old**2)*np.sin(theta1_old-theta2_old)*np.cos(theta1_old-theta2_old)))/(l2*(m1+(m2*(np.sin(theta1_old-theta2_old)**2)))))*tau)

        theta1 = theta1_old + (ang_vel1_old * tau)
        theta2 = theta2_old + (ang_vel2_old * tau)

        time_vec[i] = i*tau
        theta1_vec[i] = theta1
        theta2_vec[i] = theta2
        ang_vel1_vec[i] = ang_vel1
        ang_vel2_vec[i] = ang_vel2
    return [time_vec, theta1_vec, theta2_vec, ang_vel1_vec, ang_vel2_vec]

pendulum = dub_pen(np.pi/4, np.pi/3, 0, 0, 300)
print(pendulum[1])
plt.plot(pendulum[0],pendulum[1])
plt.plot(pendulum[0],pendulum[2])

# Sim
l = np.arange(0,300,1)

x1 = l1*np.sin(pendulum[1])
y1 = -l1*np.cos(pendulum[1])
x2 = x1 + (l2*np.sin(pendulum[2]))
y2 = y1 - (l2*np.cos(pendulum[2]))
# Division of trail
ns = 20
sec_per_trail = 2
for i in l:
    plt.figure()
    plt.plot(x1[i],y1[i],'bo',markersize=5)
    plt.plot([0,x1[i]],[0,y1[i]])
    plt.plot(x2[i],y2[i],'bo',markersize=5)
    plt.plot([x1[i],x2[i]],[y1[i],y2[i]])
    plt.xlim(-(l1+l2)-0.5,(l1+l2)+0.5)
    plt.ylim(-(l1+l2)-0.5,(l1+l2)+0.5)
    for j in range(ns):
        tau = 0.05
        max_trail = int(sec_per_trail/tau)
        s = max_trail // ns
        imin = i - (ns-j)*s
        if imin < 0:
            continue
        imax = imin + s + 1
        # The fading looks better if we square the fractional length along the
        # trail.
        alpha = (j/ns)**2
        plt.plot(x2[imin:imax], y2[imin:imax], c='r', solid_capstyle='butt',
                lw=2, alpha=alpha)


import sqlite3

connection = sqlite3.connect('movies_genres.db')

cursor = connection.cursor()

# only run this when creating table
sql_command = """CREATE TABLE movies (title VARCHAR, genre VARCHAR, plot VARCHAR, rated VARCHAR, year VARCHAR)"""
cursor.execute(sql_command)

connection.commit()
connection.close()


from flask import request
from flask_socketio import emit,Namespace
from datetime import datetime

online_list = {}

class Chat(Namespace):

    def on_connect(self):
        print("有个小子{}连接上了".format(request.sid))

    def on_disconnect(self):
        # 翻转后删除key
        new_dict = {v: k for k, v in online_list.items()}
        keys = new_dict[request.sid]
        del online_list[keys]
        emit('broad', online_list, namespace='/chat', broadcast=True)
        print("有个小子{}断开了连接".format(request.sid))

    def on_private_message(self,data):
        print('sid is {}'.format(request.sid))
        print("收到有个小子发来的信息{}".format(data))
        data['time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        to_sid = online_list[data['to']]
        emit('private_message', data, room=to_sid)

    def on_bind(self,data):
        print('sid is {}'.format(request.sid))
        uid = data['uid']
        online_list[uid] = request.sid
        emit('broad', online_list , namespace='/chat' , broadcast=True) # 给所有用户
        print("在线用户 {}".format(online_list))


#!/usr/bin/env python

from django.db import models
from django.utils.translation import ugettext_lazy as _

from panda.models.category import Category
from panda.models.dataset import Dataset
from panda.models.user_proxy import UserProxy

class SearchSubscription(models.Model):
    """
    A log of a user search.
    """
    user = models.ForeignKey(UserProxy, related_name='search_subscriptions',
        help_text=_('The user who subscribed to the search.'),
        verbose_name=_('user'))
    dataset = models.ForeignKey(Dataset, related_name='search_subscriptions', null=True, default=None,
        help_text=_('The dataset to be searched or null if all are to be searched.'),
        verbose_name=_('dataset'))
    category = models.ForeignKey(Category, related_name='search_subscriptions', null=True, default=None,
        help_text=_('A category to be searched or null if all are to be searched.'),
        verbose_name=_('category'))
    query = models.CharField(_('query'), max_length=256,
        help_text=_('The search query to executed.'))
    query_url = models.CharField(_('query_url'), max_length=256,
        help_text=_('Query encoded for URL.'))
    query_human = models.TextField(_('query_human'),
        help_text=_('Human-readable description of the query being run.'))
    last_run = models.DateTimeField(_('last_run'), auto_now=True,
        help_text=_('The last time this search this was run.'))

    class Meta:
        app_label = 'panda'
        verbose_name = _('SearchSubscription')
        verbose_name_plural = _('SearchSubscriptions')

    def __unicode__(self):
        if self.dataset:
            return _('%(user)s is searching for %(query)s in %(dataset)s') \
                % {'user': self.user, 'query': self.query, 'dataset': self.dataset}
        else:
            return _('%(user)s is searching for %(query)s in all datasets') \
                % {'user': self.user, 'query': self.query}


#import matplotlib
#matplotlib.use('Agg')
#import matplotlib
import matplotlib.pyplot as plt
import numpy as np

# Data for plotting
t = np.arange(0.0, 2.0, 0.01)
s = 1 + np.sin(2 * np.pi * t)

fig, ax = plt.subplots()
ax.plot(t, s)

ax.set(xlabel='time (s)', ylabel='voltage (mV)',
       title='About as simple as it gets, folks')
ax.grid()

#fig.savefig("test.png")
plt.show()

###Subplot
#fig = plt.figure()
#ax = fig.add_subplot(111)
#ax.plot([1,2,3])
#fig.savefig('test.png')

'''
#####Easiest exmaple#####
# Compute the x and y coordinates for points on a sine curve
x = np.arange(0, 3 * np.pi, 0.1)
y = np.sin(x)
#plt.title("sine wave form")

# Plot the points using matplotlib
plt.plot(x, y)
plt.show()
'''


import pygame
import Animation
from Assets.Attack import Normal

class Player:
    level = 1
    exp = 0
    god = False
    maxHealth = 0
    inventory = list()
    group=pygame.sprite.Group()
    facing = True
    transform = None

    def __init__(self, health, attack, defend):
        self.health = health
        self.maxHealth = health
        self.attack = attack
        self.defend=defend
        self.image = pygame.transform.scale(pygame.image.load('Assets\Player\player.png'), (180, 180))
        self.rect = self.image.get_rect()
        self.rect.midbottom = (100, 600)
        self.vel_y = 0
        self.jumped = False
        self.still = Animation.importFrames("Assets\Player\BlueWizard\\2BlueWizardIdle\\", 180, 180, 19)
        self.last_time=0

    def update(self,screen):
        x_move = 0
        y_move = 0
        key = pygame.key.get_pressed()

        if key[pygame.K_SPACE] and self.jumped is False:
            self.vel_y = -40
            self.jumped = True
        if key[pygame.K_a]:
            x_move -= 8
            if self.facing:
                self.facing = False
        if key[pygame.K_d]:
            x_move += 8
            if not self.facing:
                self.facing = True
        if key[pygame.K_f]:
            self.Attack(screen)

        self.vel_y += 5  # falling speed
        if self.vel_y > 10:  # maximum falling speed
            self.vel_y = 10
        y_move += self.vel_y
        self.rect.x += x_move
        self.rect.y += y_move
        if self.rect.bottom > 600:
            self.jumped = False
            self.rect.bottom = 600
        self.still.update(pygame.time.get_ticks())
        self.image = self.still.image
        self.group.update()
        self.group.draw(screen)
        if not self.facing:
            self.image = pygame.transform.flip(self.image, True, False)

    def Attack(self, screen):
        current_time=pygame.time.get_ticks()
        if current_time < self.last_time + 100:
            return
        self.last_time=current_time
        if(self.facing):
            bullet=Normal(self.attack,1000,'bullet1.png',50,50,20,1,self.rect.center)
        else:
            bullet = Normal(self.attack, 1000, 'bullet1.png', 50, 50, 20,-1, self.rect.center)
        self.group.add(bullet)

    def Defend(self):
        # put animation here

        pass

    def AddInventory(self, equipment):
        self.inventory.append(equipment)

    def Equipt(self, weapon):
        self.attack += weapon.attack
        self.defend += weapon.defend

    def Levelup(self):
        if self.exp == self.level * self.level * 10:
            self.level += 1
            self.maxHealth += 10
            self.health = self.maxHealth

    def GodMode(self):
        self.health = 1000000000
        self.maxHealth = self.health


# This is a sample Duppy server, using custom logic and an in-memory
# database of records.
#
# This is mostly useful for testing Duppy itself, but it nicely
# demonstrates how you would implement support for backends that don't
# fit our simplistic SQL defaults.
#
# See the examples/ folder for more examples.
#
import duppy
import logging

TEST_KEYS = {
    'example.com' : [
        'FM4d4LDAs9jP/N8EkvhhayqtqcO4tUJzvxsPyG20fkCE7g2IizVaTdeAwudLkwvhVECo50te6gJKhoxJkqUMOA==',
        'QlRlQTl4OA46nPX0/QEk65AECEbreeF8K7guyr5bAsk=',
        '+fnQhoAij/FNM0yCANXkKnxZCNIL7XI2yYRJokvTn+U='],
    'example.org' : ['+fnQhoAij/FNM0yCANXkKnxZCNIL7XI2yYRJokvTn+U='],
}

MOCK_ZONE = []

class MockServer(duppy.Server):

    # App settings
    listen_on = '127.0.0.2'

    # No database, we're writing our own Python code!
    sql_db_driver = None

    # We don't support transactions, so we don't implement these
    #   - transaction_start
    #   - transaction_commit
    #   - transaction_rollback

    async def get_keys(self, zone):
        while zone[-1:] == '.':
            zone = zone[:-1]
        if zone in TEST_KEYS:
            return TEST_KEYS[zone]
        else:
            logging.debug('Zone %s unavailable for updates' % zone)
            return []

    async def delete_all_rrsets(self, dbT, zone, dns_name):
        global MOCK_ZONE
        MOCK_ZONE = [rr for rr in MOCK_ZONE if rr[0] == dns_name]
        return True

    async def delete_rrset(self, dbT, zone, dns_name, rtype):
        global MOCK_ZONE
        MOCK_ZONE = [rr for rr in MOCK_ZONE
            if rr[0] == dns_name
            and rr[1] == rtype]
        return True

    async def delete_from_rrset(self, dbT, zone, dns_name, rtype, rdata):
        global MOCK_ZONE
        MOCK_ZONE = [rr for rr in MOCK_ZONE
            if rr[0] == dns_name
            and rr[1] == rtype
            and rr[-1] == rdata]
        return True

    async def add_to_rrset(self, dbT, zone, dns_name, rtype, ttl, i1, i2, i3, rdata):
        global MOCK_ZONE
        rr = [dns_name, rtype, ttl, i1, i2, i3, rdata]
        if rr not in MOCK_ZONE:
            MOCK_ZONE.append(rr)
        return True

    async def notify_changed(self, dbT, zone):
        print('*** Zone changed %s ***' % zone)
        print('%s' % '\n'.join('%s' % rr for rr in MOCK_ZONE))
        print('***')
        return True

try:
    MockServer().run()
finally:
    print('\nMock zone:\n%s' % '\n'.join('%s' % rr for rr in MOCK_ZONE))


from colorama import init
from colorama import Fore, Back, Style
from loggingModule import Logging
import keyboard
import os

init()

class MenuField():
    '''Класс  реализующий поле в меню консоли

    Необходим для закрепления за полем имени и цвета фона поля'''

    def __init__(self, name:str) -> None:
        '''Метод инициализации

        При создании экземпляра класса передается имя экземпляра,
        которое будет выводится на экран при визуализации меню'''

        self.name = name

        self.push_func = None

    def __str__(self) -> str:
        return self.name

    def getName(self) -> str:
        return self.name

    def settingFuncion(self, func) -> None:
        '''Привязка функции к полю меню

        Входящие данные: функция, которая будет исполняться при нажатии на поле меню'''
        self.push_func = func

    def performFunction(self) -> None:
        '''Выполнение присвоенной этой функции

        Входящие данные: None'''
        self.push_func()

class ConsoleMenu():
    '''Класс  реализующий функции управления меню в консоли

    Собирает поля меню (кнопки) из экземпляров класса MenuField'''

    def __init__(self, name="Menu") -> None:
        '''Метод инициализации'''

        self.log = Logging(__file__).getLogger()

        self.menu_name = name
        # Список полей (кнопок)
        self.field_list = []
        # Выделенное поле
        self.target_field = None
        # Функция, срадатывающая при нажатии клавиши Esc
        self.esc_func = None

        # Переменная определяющая непрерывный процесс
        # обработки и изображения полей меню
        self.click_processing = True

        self.log.info(f"Создание эземпляра класса ConsoleMenu")

    def __str__(self) -> str:
        return self.menu_name

    def addField(self, name_field:str) -> MenuField:
        '''Метод создающий новое поле меню

        Входящие данные: имя поля (str)
        Возвращает: экземпляр класса MenuField с полученным именем'''

        new_field = MenuField(name_field)
        if len(self.field_list)==0:
            self.target_field = new_field
        self.field_list.append(new_field)

        self.log.info(f"Создан экземпляр класса MenuField с именем {name_field}")

        return new_field

    def keyProcessing(self):
        '''Обработка нажатия клавиш

        Метод реагирует на нажатие клавишь. При нажатии одной из нужных,
        вызывает функцию перемещения или обработку нажатия клавиши.
        Входящиее данные: None
        Возвращает: None'''

        key_input = keyboard.record("enter")
        if len(key_input)>1:
            last_input = key_input[-2].name
            if last_input=='up':self.moveCursor(mode=-1)
            elif last_input=='down':self.moveCursor()
            elif last_input=='esc':self.moveEsc()
        else:
            self.moveEnter()

    def settingEsc(self, func=None) -> None:
        '''Настройка функции Esc

        Клавише esc присваивается передаваемая функция.
        Если функия не передавалась, то будет использована функция
        по умолчанию - escFuncion().
        Она будет вызываться при нажатии клавиши Esc
        Входящиее данные: Функция (funcion), которая будет вызываться
        при нажатии клавиши Esc
        Возвращает: None'''
        if func==None:
            self.esc_func=self.escFuncion
        else:
            self.esc_func = func
            self.log.debug(f"Клавище Esc присвоена функция {func}")

    def escFuncion(self):
        '''Функция выхода

        Функция выхода из цикла в processStarting.
        При нажатии клавиши esc будет срабатывать эта функция.
        Входящиее данные: None
        Возвращает: None'''
        self.log.debug(f"Вызвана стандартная функция выхода из меню")
        self.click_processing = False

    def moveEsc(self) -> None:
        '''Вызов функции присвоенной клавище Esc

        Входящиее данные: None
        Возвращает: None'''
        self.log.debug(f"Вызвана функция присвоенная клавище Esc")
        self.esc_func()

    def moveEnter(self) -> None:
        '''Выполнение функции присвоенной к полю,
        которое находится в таргете

        Входящиее данные: None
        Возвращает: None'''

        self.log.debug(f"Запущена функция на поле {self.target_field.name}")
        self.target_field.performFunction()

    def moveCursor(self, mode = 1) -> None:
        '''Переместить курсор

        Метод перемещает курсор "вверх" или "вниз", то есть назад или вперед по списку
        в зависимости от знака mode (-1 - спускаемся, 1 - поднимаемся), заменяя значение
        переменной в таргете (MenuField)
        Входящиее данные: None
        Возвращает: None'''
        self.log.debug(f"Запущена функция moveCursorUP")
        number = self.removeCursor()
        if number!=None:
            if number+mode < len(self.field_list)*(mode):
                self.changeCursor(0)
            else:
                self.changeCursor(number+mode)

    def removeCursor(self) -> int:
        '''Получение индекса поля в таргете

        Данный метод вызывается для получения поля, которое находится в таргете.
        Входящиее данные: None
        Возвращает: индекс поля, который сейчас находится в таргете'''
        if self.target_field!=None:
            self.log.debug(f"Запущена функция remobeCursor")
            index = self.field_list.index(self.target_field)
            self.log.debug(f"Функция removeCursor возвращает индекс поля, находящегося в таргете: {index}")
            self.log.debug(f"Сейчас в таргете находится поле {self.target_field.getName()}")
            return index
        return None

    def changeCursor(self, index:int) -> None:
        '''Сменить таргет

        Данный метод меняет значение в переменной target_field текущего экземпляра класса на
        экземпляр класса MenuField из списка field_list с заданным индексом.
        Входящиее данные: индекс нового таргета (int)
        Возвращает: None'''
        self.log.debug(f"Запущена функция changeCursor (смена таргета) c новым индеком {index}")
        self.log.debug(f"Таргет перешел от поля - {self.target_field.getName} на поле  - {self.field_list[index].getName}")
        self.target_field = self.field_list[index]
        self.log.debug(f"В тарегете теперь находится поле: {self.target_field.getName()}")

    def outputOnDisplay(self) -> None:
        '''Вывод полей меню на консоль

        Входящиее данные: None
        Возвращает: None'''

        self.log.debug(f"Запущен вывод меню в консоль")
        print(self)
        for field in self.field_list:
            self.outputFieldOnDisplay(field)

    def outputFieldOnDisplay(self, field:MenuField) -> None:
        '''Вывод поля на консоль

        Входящиее данные: поле (MenuField), которое нужно вывести в консоль
        Возвращает: None'''
        if self.target_field==field:
            print(Back.GREEN + str(field))
        else:
            print(Back.BLACK + str(field))

    def cleaning(self) -> None:
        '''Очистка консоли

        Входящиее данные: None
        Возвращает: None'''
        self.log.debug(f"Запущена очистка консоли")
        os.system(['clear','cls'][os.name == 'nt'])

    def processStarting(self):
        '''Метод запускающий процесс изображения полей и реагирования
        на нажатие клавиш

        Входящиее данные: None
        Возвращает: None'''
        self.log.info(f"Запущена функция processStarting запускающая процесс отображения полей меню и считывания нажатых клавиш")
        self.click_processing = True
        while self.click_processing:
            self.cleaning()
            self.outputOnDisplay()
            self.keyProcessing()
        self.log.info(f"Функция processStarting прекратила работу")


import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from typing import Optional

class FocalLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(FocalLoss, self).__init__()

    def forward(
        self,
        y_pred: torch.Tensor,
        y_true: torch.Tensor,
        alpha: Optional[float] = 0.8,
        gamma: Optional[float] = 2.0,
        smooth: Optional[int] = 1,
    ) -> torch.Tensor:
        y_pred = F.sigmoid(y_pred)
        y_pred = y_pred.view(-1)
        y_true = y_true.view(-1)

        logpt = F.binary_cross_entropy(y_pred, y_true, reduction="mean")
        pt = torch.exp(-logpt)
        focal_term = (1.0 - pt).pow(gamma)
        focal_loss = alpha * focal_term * logpt

        return focal_loss

class DiceLoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(DiceLoss, self).__init__()

    def forward(
        self, y_pred: torch.Tensor, y_true: torch.Tensor, smooth: Optional[int] = 1
    ) -> torch.Tensor:
        y_pred = F.sigmoid(y_pred)

        y_pred = y_pred.view(-1)
        y_true = y_true.view(-1)

        intersection = (y_pred * y_true).sum()
        dice = (2.0 * intersection + smooth) / (y_pred.sum() + y_true.sum() + smooth)
        return 1 - dice

class DiceBCELoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(DiceBCELoss, self).__init__()

    def forward(
        self, y_pred: torch.Tensor, y_true: torch.Tensor, smooth: Optional[int] = 1
    ) -> torch.Tensor:
        y_pred = F.sigmoid(y_pred)

        y_pred = y_pred.view(-1)
        y_true = y_true.view(-1)
        intersection = (y_pred * y_true).sum()
        dice = 1 - (2.0 * intersection + smooth) / (
            y_pred.sum() + y_true.sum() + smooth
        )
        BCE = F.binary_cross_entropy(y_pred, y_true, reduction="mean")
        dice_bce = BCE + dice

        return dice_bce

class IoULoss(nn.Module):
    def __init__(self, weight=None, size_average=True):
        super(IoULoss, self).__init__()

    def forward(
        self, y_pred: torch.Tensor, y_true: torch.Tensor, smooth: Optional[int] = 1
    ) -> torch.Tensor:
        y_pred = F.sigmoid(y_pred)

        y_pred = y_pred.view(-1)
        y_true = y_true.view(-1)

        intersection = (y_pred * y_true).sum()
        union = (y_pred + y_true).sum() - intersection
        IoU = (intersection + smooth) / (union + smooth)
        return 1 - IoU

# Note when alpha and beta = 0.5 this is the same as Dice Loss
class TverskyLoss(nn.Module):
    def __init__(self, weights=None, size_average=True):
        super(TverskyLoss, self).__init__()

    def forward(
        self,
        y_pred: torch.Tensor,
        y_true: torch.Tensor,
        smooth: Optional[int] = 1,
        alpha: Optional[float] = 0.5,
        beta: Optional[float] = 0.5,
    ) -> torch.Tensor:
        y_pred = F.sigmoid(y_pred)

        y_pred = y_pred.view(-1)
        y_true = y_true.view(-1)

        TP = (y_pred * y_true).sum()
        FP = ((1 - y_true) * y_pred).sum()
        FN = (y_true * (1 - y_pred)).sum()

        tversky = (TP + smooth) / (TP + alpha * FP + beta * FN + smooth)

        return 1 - tversky

class FocalTverskyLoss(nn.Module):
    pass

class LovaszHingeLoss(nn.Module):
    pass

class ComboLoss(nn.Module):
    pass


from overrides import overrides
from pandas import DataFrame

from PySide6.QtCore import QAbstractTableModel, Qt, QModelIndex, QPersistentModelIndex, SIGNAL

class PandasTableModel(QAbstractTableModel):
    """
    https://stackoverflow.com/questions/19411101/pyside-qtableview-example
    """
    def __init__(self, parent, df: DataFrame, *args):
        """doc"""
        super().__init__(parent)
        self.df = df # DataFrame& const df; pointer is not allowed to be reassigned
        self.header = self.df.columns

        # Experimental
        self._showindex = True

    # @overrides
    def rowCount(self, parent) -> int:
        """doc"""
        return self.df.shape[0]

    # @overrides
    def columnCount(self, parent) -> int:
        """doc"""
        return self.df.shape[1] if not self._showindex else self.df.shape[1] + 1

    # @overrides
    def data(self, index: QModelIndex, role: Qt.ItemDataRole) -> str | None:
        """doc"""
        if not index.isValid():
            return None
        elif role != Qt.DisplayRole:
            return None
        if not self._showindex:
            return str(self.df.iloc[index.row(),index.column()])
        else:
            return str(self.df.index[index.row()]) if index.column() == 0 else str(self.df.iloc[index.row(),index.column()-1])

    # @overrides
    def headerData(self, col, orientation: Qt.Orientation, role: Qt.ItemDataRole):
        """doc"""
        if orientation == Qt.Horizontal and role == Qt.DisplayRole:
            if not self._showindex:
                return self.header[col]
            else:
                return 'index' if col == 0 else self.header[col-1]
        return None

    # @overrides
    def sort(self, col: int, order: Qt.SortOrder):
        """sort table by given column number col"""
        self.emit(SIGNAL("layoutAboutToBeChanged()"))
        if not self._showindex:
            self.df.sort_values(by=self.df.columns[col], inplace=True, ascending=(order == Qt.AscendingOrder))
        else:
            if col > 0:
                self.df.sort_values(by=self.df.columns[col-1], inplace=True, ascending=(order == Qt.AscendingOrder))
            else:
                self.df.sort_index(inplace=True, ascending=(order == Qt.AscendingOrder))
        self.emit(SIGNAL("layoutChanged()"))

    # @overrides
    def flags(self, index: QModelIndex) -> Qt.ItemFlags:
        """docs"""
        default = Qt.ItemNeverHasChildren | Qt.ItemIsEnabled | Qt.ItemIsSelectable | Qt.ItemIsEditable
        if not self._showindex:
            return default
        else:
            return default if index.column() > 0 else Qt.ItemNeverHasChildren

    def setData(self, index, value, role: Qt.EditRole):
        """docs"""
        index_column = index.column() if not self._showindex else index.column() - 1
        datatype = self.df[self.df.columns[index_column]].dtype
        try:
            self.df.iloc[index.row(), index_column] = datatype.type(value)
            return True
        except ValueError:
            return False


from django.contrib.auth import get_user_model
from django.urls import reverse
from django.test import TestCase

from rest_framework import status
from rest_framework.test import APIClient

from core.models import Project, Organization

from projects.serializers import ProjectSerializer

PROJECT_URL = reverse('projects:project-list')

def detail_url(project_id):
    """Create the detail URL for a project"""
    return reverse('projects:project-detail', args=[project_id])

class PublicProjectApiTest(TestCase):
    """Test the publicly available projects API"""

    def setUp(self):
        self.client = APIClient()

    def test_retrieve_projects_list(self):
        """Test retrieving a list of projects by unauthenticated user"""
        self.user = get_user_model().objects.create_user(
            'test@xemob.com',
            'password123'
        )
        ngo = Organization.objects.create(user=self.user,
                                          name='NGO-1',
                                          country='Spain')
        Project.objects.create(user=self.user,
                               name='Project 1',
                               organization=ngo)
        Project.objects.create(user=self.user,
                               name='Project 2',
                               organization=ngo)

        res = self.client.get(PROJECT_URL)

        projects = Project.objects.all().order_by('-id')
        serializer = ProjectSerializer(projects, many=True)
        self.assertEqual(res.status_code, status.HTTP_200_OK)
        self.assertEqual(len(res.data), 2)
        self.assertEqual(res.data, serializer.data)

class PrivateProjectsApiTests(TestCase):
    """Test the authorized user projects API"""

    def setUp(self):
        self.user = get_user_model().objects.create_user(
            'test@xemob.com',
            'password123'
        )
        self.client = APIClient()
        self.client.force_authenticate(self.user)
        self.ngo = Organization.objects.create(user=self.user,
                                               name='NGO-1',
                                               country='Spain')
        self.user2 = get_user_model().objects.create_user('other@xemob.com',
                                                          'testpass')
        self.ngo2 = Organization.objects.create(user=self.user2,
                                                name='NGO-2',
                                                country='Wonderland')

    def test_create_project_successful(self):
        """Test creating a new project"""
        payload = {'name': 'Test project',
                   'organization': self.ngo.id,
                   'user': self.user.id}
        self.client.post(PROJECT_URL, payload)

        exists = Project.objects.filter(
            user=self.user,
            name=payload['name']
        ).exists()
        self.assertTrue(exists)

    def test_create_project_invalid(self):
        """Create a project with invalid payload"""
        payload = {'name': '', 'organization': self.ngo}
        res = self.client.post(PROJECT_URL, payload)

        self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)

    def test_partial_update_project_successful(self):
        """Test updating a project by owner successful"""
        project = Project.objects.create(user=self.user,
                                         name='Project 1',
                                         organization=self.ngo)
        payload = {'name': 'Updated project'}
        url = detail_url(project.id)
        res = self.client.patch(url, payload)

        project.refresh_from_db()
        self.assertEqual(project.name, payload['name'])
        self.assertEqual(res.status_code, status.HTTP_200_OK)

    def test_partial_update_by_not_owner_invalid(self):
        """Test updating a project by not owner return not authorized"""
        project = Project.objects.create(user=self.user2,
                                         name='Project 1',
                                         organization=self.ngo)
        payload = {'name': 'Updated project'}
        url = detail_url(project.id)
        res = self.client.patch(url, payload)

        project.refresh_from_db()
        self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN)
        self.assertNotEqual(project.name, payload['name'])

    def test_full_update_project_successful(self):
        """Test full updating a project by owner successful"""
        project = Project.objects.create(user=self.user,
                                         name='Project 1',
                                         organization=self.ngo)
        payload = {'name': 'Test project',
                   'organization': self.ngo.id,
                   'user': self.user.id}

        url = detail_url(project.id)
        res = self.client.put(url, payload)

        project.refresh_from_db()
        self.assertEqual(res.status_code, status.HTTP_200_OK)
        self.assertEqual(project.name, payload['name'])

    def test_full_update_by_not_owner_invalid(self):
        """Test updating a project by not owner return not authorized"""

        project = Project.objects.create(user=self.user2,
                                         name='Project 1',
                                         organization=self.ngo)
        payload = {'user': self.user.id,
                   'name': 'Updated project',
                   'organiztion': self.ngo.id}
        url = detail_url(project.id)
        res = self.client.put(url, payload)

        project.refresh_from_db()
        self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN)
        self.assertNotEqual(project.name, payload['name'])

    def test_create_project_with_not_owned_organization_invalid(self):
        """Test that user should be owner of the organization related with
        the project. Otherwise, it raise an error"""
        payload = {'user': self.user.id,
                   'name': 'Rapped Organization',
                   'organization': self.ngo2.id}
        res = self.client.post(PROJECT_URL, payload)

        self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)

    def test_create_project_for_another_user_is_invalid(self):
        """Test that a user can't create a project posting on
        behalf of anther user"""
        payload = {'user': self.user2.id,
                   'name': 'Rapped Organization',
                   'organization': self.ngo2.id}
        res = self.client.post(PROJECT_URL, payload)

        self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST)


import torch

class CNN(torch.nn.Module):

    def __init__(self, channels, conv_kernels, conv_strides, conv_padding, pool_padding, num_classes=15):
        assert len(conv_kernels) == len(channels) == len(conv_strides) == len(conv_padding)
        super(CNN, self).__init__()

        # create conv blocks
        self.conv_blocks = torch.nn.ModuleList()
        prev_channel = 1

        for i in range(len(channels)):
            # add stacked conv layer
            block = []
            for j, conv_channel in enumerate(channels[i]):
                block.append(torch.nn.Conv1d(in_channels=prev_channel, out_channels=conv_channel, kernel_size=conv_kernels[i], stride=conv_strides[i], padding=conv_padding[i]))
                prev_channel = conv_channel
                # add batch norm layer
                block.append(torch.nn.BatchNorm1d(prev_channel))
                # adding ReLU
                block.append(torch.nn.ReLU())
            self.conv_blocks.append(torch.nn.Sequential(*block))

        # create pooling blocks
        self.pool_blocks = torch.nn.ModuleList()
        for i in range(len(pool_padding)):
            # adding Max Pool (drops dims by a factor of 4)
            self.pool_blocks.append(torch.nn.MaxPool1d(kernel_size=4, stride=4, padding=pool_padding[i]))

        # global pooling
        self.global_pool = torch.nn.AdaptiveAvgPool1d(1)
        self.linear = torch.nn.Linear(prev_channel, num_classes)

    def forward(self, inwav):
        for i in range(len(self.conv_blocks)):
            # apply conv layer
            inwav = self.conv_blocks[i](inwav)
            # apply max_pool
            if i < len(self.pool_blocks): inwav = self.pool_blocks[i](inwav)
        # apply global pooling
        out = self.global_pool(inwav).squeeze() # [batch_size, 256, 1] [batch_size, 256]
        out = self.linear(out) #[batch_size, 15]
        return out

class ResBlock(torch.nn.Module):

    def __init__(self, prev_channel, channel, conv_kernel, conv_stride, conv_pad):
        super(ResBlock, self).__init__()
        self.res = torch.nn.Sequential(
            torch.nn.Conv1d(in_channels = prev_channel, out_channels=channel, kernel_size=conv_kernel, stride=conv_stride, padding=conv_pad),
            torch.nn.BatchNorm1d(channel),
            torch.nn.ReLU(),
            torch.nn.Conv1d(in_channels=channel, out_channels=channel, kernel_size=conv_kernel, stride=conv_stride, padding=conv_pad),
            torch.nn.BatchNorm1d(channel),
        )
        self.bn = torch.nn.BatchNorm1d(channel)
        self.relu = torch.nn.ReLU()

    def forward(self, x):
        identity = x
        x = self.res(x)
        if x.shape[1] == identity.shape[1]:
            x += identity
        # repeat the smaller block till it reaches the size of the bigger block
        elif x.shape[1] > identity.shape[1]:
            if x.shape[1] % identity.shape[1] == 0:
                x += identity.repeat(1, x.shape[1]//identity.shape[1], 1)
            else:
                raise RuntimeError("Dims in ResBlock needs to be divisible on the previous dims!!")
        else:
            if identity.shape[1] % x.shape[1] == 0:
                identity += x.repeat(1, identity.shape[1]//x.shape[1], 1)
            else:
                raise RuntimeError("Dims in ResBlock needs to be divisible on the previous dims!!")
            x = identity
        x = self.bn(x)
        x = self.relu(x)
        return x

class CNNRes(torch.nn.Module):

    def __init__(self, channels, conv_kernels, conv_strides, conv_padding, pool_padding, num_classes=15):
        assert len(conv_kernels) == len(channels) == len(conv_strides) == len(conv_padding)
        super(CNNRes, self).__init__()

        # create conv block
        prev_channel = 1
        self.conv_block = torch.nn.Sequential(
            torch.nn.Conv1d(in_channels=prev_channel, out_channels=channels[0][0], kernel_size=conv_kernels[0], stride=conv_strides[0], padding=conv_padding[0]),
            # add batch norm layer
            torch.nn.BatchNorm1d(channels[0][0]),
            # adding ReLU
            torch.nn.ReLU(),
            # adding max pool
            torch.nn.MaxPool1d(kernel_size = 4, stride = 4, padding = pool_padding[0]),
        )

        # create res
        prev_channel = channels[0][0]
        self.res_blocks = torch.nn.ModuleList()
        for i in range(1, len(channels)):
            # add stacked res layer
            block = []
            for j, conv_channel in enumerate(channels[i]):
                block.append(ResBlock(prev_channel, conv_channel, conv_kernels[i], conv_strides[i], conv_padding[i]))
                prev_channel = conv_channel
            self.res_blocks.append(torch.nn.Sequential(*block))

        # create pool blocks
        self.pool_blocks = torch.nn.ModuleList()
        for i in range(1, len(pool_padding)):
            # adding Max Pool (drops dims by a factor of 4)
            self.pool_blocks.append( torch.nn.MaxPool1d(kernel_size = 4, stride = 4, padding = pool_padding[i]) )

        # global pooling
        self.global_pool = torch.nn.AdaptiveAvgPool1d(1)
        self.linear = torch.nn.Linear(prev_channel, num_classes)

    def forward(self, inwav):
        inwav = self.conv_block(inwav)
        for i in range(len(self.res_blocks)):
            # apply conv layer
            inwav = self.res_blocks[i](inwav)
            # apply max_pool
            if i < len(self.pool_blocks): inwav = self.pool_blocks[i](inwav)
        # apply global pooling
        out = self.global_pool(inwav).squeeze()
        out = self.linear(out)
        return out

m6_res = CNNRes(channels = [[128], [128]*2],
         conv_kernels = [80, 3],
         conv_strides = [4, 1],
         conv_padding = [38, 1],
         pool_padding = [0, 0])


import logging
import threading
from collections import deque
from typing import List

import weewx
from user.weatherlink_live.callback import PacketCallback
from user.weatherlink_live.davis_broadcast import WllBroadcastReceiver
from user.weatherlink_live.davis_http import start_broadcast, request_current
from user.weatherlink_live.mappers import AbstractMapping
from user.weatherlink_live.packets import DavisConditionsPacket

log = logging.getLogger(__name__)

class DataHost(object):
    """Base host class for polled as well as broadcasted data"""

    def __init__(self, mappers: List[AbstractMapping], data_event: threading.Event):
        self._mappers = mappers
        self._data_event = data_event

        self.packets = deque()
        self.error = None

    @property
    def has_error(self):
        return self.error is not None

    def raise_error(self):
        if self.has_error:
            raise self.error

    def _create_record(self, packet: DavisConditionsPacket):
        record = dict()

        for mapper in self._mappers:
            mapper.map(packet, record)
        self.packets.append(record)

        record['dateTime'] = packet.timestamp
        record['usUnits'] = weewx.US

        self._data_event.set()

    def notify_error(self, e):
        self.error = e
        self._data_event.set()

class WllPollHost(DataHost):
    """Host object for polling data from WLL"""

    def __init__(self,
                 host: str,
                 mappers: List[AbstractMapping],
                 data_event: threading.Event,
                 http_timeout: float = 20):
        super().__init__(mappers, data_event)
        self.host = host
        self.http_timeout = http_timeout

    def poll(self):
        packet = request_current(self.host, timeout=self.http_timeout)
        log.debug("Polled current conditions")

        self._create_record(packet)

    def close(self):
        pass

class WLLBroadcastHost(DataHost, PacketCallback):
    """Class for triggering UDP broadcasts and receiving them"""

    def __init__(self,
                 host: str,
                 mappers: List[AbstractMapping],
                 data_event: threading.Event,
                 http_timeout: float = 20):
        super().__init__(mappers, data_event)
        self.host = host
        self.http_timeout = http_timeout

        self._receiver = None
        self._port = 22222

    def refresh_broadcast(self, request_duration: float):
        log.debug("Re-requesting UDP broadcast")
        packet = start_broadcast(self.host, request_duration, timeout=self.http_timeout)
        port = packet.broadcast_port

        if self._port != port:
            log.info("Broadcast port changed from %s to %s" % (self._port, port))
            self._port = port

        log.debug("Restarting broadcast reception")
        self._stop_broadcast_reception()
        self._start_broadcast_reception()

    def _start_broadcast_reception(self):
        self._receiver = WllBroadcastReceiver(self.host, self._port, self)

    def _stop_broadcast_reception(self):
        if self._receiver is None:
            return
        self._receiver.close()

    def on_packet_received(self, packet: DavisConditionsPacket):
        log.debug("Received new broadcast packet")
        try:
            self._create_record(packet)
        except Exception as e:
            self.notify_error(e)
            self.close()

    def on_packet_receive_error(self, e: BaseException):
        self.notify_error(e)
        self.close()

    def close(self):
        self._stop_broadcast_reception()


"""
Detect and remove mux scans (for non-Flongles)
"""

from collections import Counter
from pathlib import Path
from typing import List, Tuple
import numpy as np
import pandas as pd
import tqdm
from simreaduntil.seqsum_tools.seqsum_preprocessing import sort_and_clean_seqsum_df

from simreaduntil.shared_utils.utils import is_sorted

def find_mux_scans(start_times, end_times, muxes, max_block_gap=1, max_intv_gap=20, min_mux_frac=0.95) -> List[Tuple[float, float]]:
    """
    Find mux scan windows when the channel is scanning all the wells (muxes) to determine the best well

    **Note**: This does not work for Flongle runs because they only have a single mux.

    The mux scans are characterized by the mux changing from 1, 2, 3, 4 in this order.
    Although they should occur at 90 minutes intervals, this is not always exactly the case.
    We refer to a mux scan as a consecutive set of windows (of reads at most max_block_gap apart), where the dominant
    fraction of muxes (according to min_mux_frac) of window i is i.

    This method works best with many channels because muxes are performed simultaneously
    across all channels and this way, we can detect whether all channels are scanning the same mux at a time.
    Assumes that the sequencing starts at time 0.

    This code was refactored from sim_utils.py in UNCALLED, mostly by documenting the code and renaming variables.
    The code does not seem very robust, but we don't want to modify it too much.

    Args:
        start_times: start times of reads
        end_times: end times of reads
        muxes: muxes, i.e. which pore in the channel is used
        max_block_gap: maximum distance between reads in windows that will be tested for mux scan segment (for a fixed mux 1, 2, 3, 4),
            if two reads are more than max_block_gap apart, they are in different blocks

        max_intv_gap: if two mux scan windows are more than max_intv_gap apart, they are in different mux scan windows
        min_mux_frac: minimum fraction to classify a region as scanning a certain mux (1, 2, 3 or 4)

    Returns:
        2d np.array of shape (m, 2) of (start, end) times of mux scans
"""
    # pd.Series does not work because of indexing (probably)
    assert isinstance(start_times, np.ndarray)
    assert isinstance(end_times, np.ndarray)
    assert isinstance(muxes, np.ndarray)

    order = np.argsort(start_times)
    start_times = start_times[order] # start times of reads
    end_times = end_times[order] # end times of reads
    muxes = muxes[order] # muxes, i.e. which pore in the channel is used

    # first find blocks within which reads are at most max_block_gap apart

    active_blocks = list() # set of (t_start, t_end) describing active periods
    window_start = start_times[0] # start time of current window
    window_end = end_times[0] # end time of current window

    for (rst, ren) in zip(start_times[1:], end_times[1:]):
        if rst - window_end > max_block_gap:
            active_blocks.append( (window_start, window_end) )
            window_start = rst
            window_end = ren
        else:
            window_end = max(ren, window_end)
            #cur_window_end = ren
    active_blocks.append((window_start, window_end))

    # given the blocks, find mux scan windows
    # they are characterized by the mux changing from 1, 2, 3, 4 in this order, so we only record a mux scan window when there are exactly four elements with increasing muxes
    # for the mux=1 window, we merge it with the previous mux=1 window if they are less than max_intv_gap apart; if they are more apart, we discard the previous mux=1 window
    # finally, the mux window is the window preceding the first mux=1 window to the window following the end of the last mux=4 window
    # whenever two windows are more than max_intv_gap apart, they are in different mux scan windows

    mux_scan_windows = list() # list of mux scans, each entry is a scan of four elements
    scan_gaps = list() # list of (float, float) tuples denoting gaps to the previous and next window for the mux=1 window and mux=4 window
    cur_scan_segments = list() # running list of four (start, end) tuples, corresponding to muxes 1,2,3,4 respectively

    prev_window_end = 0
    for (window_start, window_end) in active_blocks:

        if len(cur_scan_segments) > 0 and window_start - cur_scan_segments[-1][1] > max_intv_gap:
            if len(cur_scan_segments) == 4:
                mux_scan_windows.append(cur_scan_segments)
                # todo2: bug? scan_gaps not appended to
                raise Exception("Should not happen")
            cur_scan_segments = list()

        mux_counts = Counter(muxes[(start_times >= window_start) & (start_times < window_end)]) # count muxes in current window across all channels and see if all channels are predominantly using one mux
        mux_counts = [(c, m) for (m, c) in mux_counts.items()]
        top_count, top_mux = max(mux_counts)

        if top_count / sum((c for (c, m) in mux_counts)) >= min_mux_frac:
            # if the window has a dominant mux (>= min_mux_frac = 0.95)

            if top_mux != 4 and len(cur_scan_segments) == 4:
                # out-of-order, but since we went from 1 to 4, we can finish the mux scan window
                mux_scan_windows.append(cur_scan_segments)

                scan_gaps.append((gap_before_scan,
                                  window_start - cur_scan_segments[-1][1]))

                cur_scan_segments = list()

            if len(cur_scan_segments) > 0 and top_mux == len(cur_scan_segments):
                # window has same mux as previous window
                if window_end - cur_scan_segments[-1][1] < max_intv_gap:
                    # merge windows by modifying scan window
                    cur_scan_segments[-1] = (cur_scan_segments[-1][0], window_end)

                elif top_mux == 1:
                    # if it is mux=1 and above max_intv_gap, discard the old window
                    cur_scan_segments[0] = (window_start, window_end)
                    gap_before_scan = window_start - prev_window_end # gap before mux=1 window
                # otherwise: don't take window into account, todo2: should discard window?

            elif top_mux-1 == len(cur_scan_segments):
                # window has mux one higher than previous window
                cur_scan_segments.append( (window_start, window_end) )
                if len(cur_scan_segments) == 1:
                    # mux=1 window in mux scan
                    gap_before_scan = window_start - prev_window_end

            else:
                # in a mux scan, the mux only increases sequentially
                # since this was not the case, discard current scan
                cur_scan_segments = list()
        else:
            if len(cur_scan_segments) == 4:
                mux_scan_windows.append(cur_scan_segments)
                scan_gaps.append((gap_before_scan,
                                  window_start - cur_scan_segments[-1][1]))
            cur_scan_segments = list()

        prev_window_end = window_end

    # enlargen windows to include gaps
    scans = list()
    for (segs, gaps) in zip(mux_scan_windows, scan_gaps):
        scans.append((segs[0][0]-gaps[0], segs[-1][1]+gaps[1]))

    return np.array(scans) if len(scans) > 0 else np.zeros(shape=(0, 2), dtype=np.float64)

def check_mux_scan_windows(seqsum_df, mux_scan_windows):
    """
    Check that each read is either entirely outside or entirely inside a mux scan,
    """
    assert all(np.diff(mux_scan_windows.flatten()) > 0), "windows should have positive length"

    for (start, end) in tqdm.tqdm(mux_scan_windows, "Checking no read partially intersects mux scan window"):
        # check each element either is before mux scan, inside, or after
        assert all(
            ((seqsum_df["start_time"] <= start) & (seqsum_df["end_time"] <= start)) |
            ((seqsum_df["start_time"] >= start) & (seqsum_df["end_time"] <= end)) |
            ((seqsum_df["start_time"] >= end) & (seqsum_df["end_time"] >= end))
        )

def plot_mux_scans(mux_scan_windows, ax):
    """
    Add mux scans as gray regions (axvspan)

    Args:
        mux_scan_windows: list of tuples (start_time, end_time)
        ax: matplotlib axis
    """
    for (start_time, end_time) in mux_scan_windows: # [(10, 20), (25,30)]
        ax.axvspan(start_time, end_time, facecolor='gray', alpha=0.5)
        ax.axvline(start_time, color="black")
        ax.axvline(end_time, color="black")

def _remove_reads_inside_mux_scans(seqsum_df, mux_scan_windows):
    """
    Remove reads that are entirely inside mux scans

    Args:
        seqsum_df: seqsum dataframe
        mux_scan_windows: list of tuples (start_time, end_time)
    """

    # build mask of elements that are not inside mux scans
    removal_mask = np.zeros(len(seqsum_df), dtype=bool)
    for (start, end) in mux_scan_windows:
        removal_mask |= (seqsum_df["start_time"] >= start) & (seqsum_df["end_time"] <= end)
    # remove reads inside a mux scan

    return seqsum_df[~removal_mask]

def remove_mux_scans(seqsum_df, mux_scan_windows):
    """
    Remove the mux scans from the sequencing summary by shifting times

    Removes reads that are entirely inside mux scans
    Adds a column "nb_scans_before", i.e. how many mux scans have been performed before this read

    Args:
        seqsum_df: sequencing summary
        mux_scan_windows: list of (start, end) tuples of mux scan windows

    Returns:
        Tuple of (the modified sequencing summary, mux scan boundaries)
    """

    seqsum_df = seqsum_df.copy()
    mux_scan_windows = mux_scan_windows.copy()

    seqsum_df = _remove_reads_inside_mux_scans(seqsum_df, mux_scan_windows)
    seqsum_df["nb_scans_before"] = 0

    assert is_sorted(mux_scan_windows.flatten()), "mux scan windows should be sorted"

    for i in range(len(mux_scan_windows)):
        start, end = mux_scan_windows[i]
        # remove ith mux scan from times
        window_length = end - start

        mux_scan_windows[mux_scan_windows > start] -= window_length

        seqsum_df.loc[seqsum_df["start_time"] > start, "start_time"] -= window_length
        seqsum_df.loc[seqsum_df["end_time"] > start, "end_time"] -= window_length
        seqsum_df.loc[seqsum_df["template_start"] > start, "template_start"] -= window_length

        seqsum_df.loc[seqsum_df["start_time"] > start, "nb_scans_before"] += 1

    assert np.allclose(mux_scan_windows[:, 0], mux_scan_windows[:, 1]), "mux scan windows should now have length 0"

    return seqsum_df, mux_scan_windows[:, 0]

def get_seqsum_filename_with_removed_mux_scans(seqsum_filename):
    """Filename to remove sequencing summary with removed mux scans, in same directory"""
    seqsum_filename = Path(seqsum_filename)
    return seqsum_filename.parent / ("no_mux_scans_" + seqsum_filename.name)

def remove_mux_scans_from_file(sequencing_summary_file, save_filename=None):
    """
    Remove mux scans from sequencing summary file and write it back to a new file

    Args:
        sequencing_summary_file: path to sequencing summary file
        save_filename: filename to save to; if None, don't save

    Returns:
        Tuple of (original sequencing summary, mux scan boundaries, path to new sequencing summary file)
    """

    df_read = pd.read_csv(sequencing_summary_file, sep="\t")
    seqsum_df = sort_and_clean_seqsum_df(df_read, min_gap_duration=None)#, min_gap_duration=0.05)

    mux_scan_windows = find_mux_scans(seqsum_df["start_time"].values, seqsum_df["end_time"].values, seqsum_df["mux"].values)
    check_mux_scan_windows(seqsum_df, mux_scan_windows)
    print("Removing mux scans")
    seqsum_df_new, mux_scan_boundaries = remove_mux_scans(seqsum_df, mux_scan_windows)

    if save_filename is not None:
        print(f"Saving to '{save_filename}'")
        seqsum_df_new.to_csv(save_filename, sep="\t", index=False)

    return seqsum_df, mux_scan_boundaries