import datetime
import re
import subprocess
from collections import OrderedDict
from enum import Enum

import pyperclip
import requests
from PyQt5 import QtWidgets, QtCore, QtGui

from lib.common import Timer, AbstractView

class ItemAction(Enum):
    EXCLUDE = "exclude"
    COLLAPSE = "collapse"

class Updater(QtCore.QThread):
    ready = QtCore.pyqtSignal(list)

    def __init__(self, base_url, auth):
        QtCore.QThread.__init__(self)

        self.base_url = base_url
        self.auth = auth

    def run(self):
        request = requests.get("{}/folders".format(self.base_url), auth=self.auth)

        folders = {
            -1: "No folder"
        }

        for folder in request.json()["folders"]:
            folders[int(folder["id"])] = folder["name"]

        request = requests.get("{}/feeds".format(self.base_url), auth=self.auth)

        feeds = {}

        for feed in request.json()["feeds"]:
            feeds[int(feed["id"])] = feed

        request = requests.get("{}/items".format(self.base_url), auth=self.auth, params={"type": 3, "getRead": "false", "batchSize": -1})

        items = []

        for item in request.json()["items"]:
            if len(items) >= 1000:
                break

            feed = feeds[int(item["feedId"])]
            folder_id = feed["folderId"]

            if folder_id is None:
                folder_id = -1

            item["folder"] = folders[int(folder_id)]
            item["feed"] = feed["title"]

            items.append(item)

        self.ready.emit(items)

class View(QtWidgets.QTreeWidget, AbstractView):
    def __init__(self, nextcloud_url, username, password, columns=None, item_options=None, update_interval=600):
        super().__init__()

        self.news = {}
        self.base_url = "{}/index.php/apps/news/api/v1-2".format(nextcloud_url)
        self.auth = (username, password)
        self.item_options = item_options

        self.setAlternatingRowColors(True)
        self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
        self.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
        self.setRootIsDecorated(False)
        self.doubleClicked.connect(self.open_selected_items)

        self.setHeaderLabels(["Title", "Date"])

        self.setColumnWidth(1, 250)

        self.header().setStretchLastSection(False)
        self.header().setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch)

        if columns:
            self.setColumnHidden(0, "title" not in columns)
            self.setColumnHidden(1, "date" not in columns)

        open_action = QtWidgets.QAction(QtGui.QIcon.fromTheme("document-open"), "Open", self)
        open_action.setShortcut(QtGui.QKeySequence(QtCore.Qt.Key.Key_Return))
        open_action.setShortcutContext(QtCore.Qt.WidgetShortcut)
        open_action.triggered.connect(self.open_selected_items)
        self.addAction(open_action)

        copy_action = QtWidgets.QAction(QtGui.QIcon.fromTheme("edit-copy"), "Copy URL", self)
        copy_action.setShortcut(QtGui.QKeySequence(QtGui.QKeySequence.StandardKey.Copy))
        copy_action.setShortcutContext(QtCore.Qt.WidgetShortcut)
        copy_action.triggered.connect(self.copy_selected_items)
        self.addAction(copy_action)

        mark_as_read_action = QtWidgets.QAction(QtGui.QIcon.fromTheme("mail-mark-read"), "Mark as read", self)
        mark_as_read_action.setShortcut(QtGui.QKeySequence(QtGui.QKeySequence.StandardKey.Delete))
        mark_as_read_action.setShortcutContext(QtCore.Qt.WidgetShortcut)
        mark_as_read_action.triggered.connect(self.mark_selected_item_as_read)
        self.addAction(mark_as_read_action)

        self.context_menu = QtWidgets.QMenu()
        self.context_menu.addAction(open_action)
        self.context_menu.addAction(copy_action)
        self.context_menu.addSeparator()
        self.context_menu.addAction(mark_as_read_action)

        self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
        self.customContextMenuRequested.connect(self.show_context_menu)

        self.updater_thread = Updater(self.base_url, self.auth)
        self.updater_thread.ready.connect(self.update_data)

        timer = Timer(self, update_interval * 1000, self)
        timer.timeout.connect(self.trigger_update_by_timer)

    def trigger_update_by_timer(self):
        if len(self.get_selected_items()) > 1:
            return

        self.updater_thread.start()

    def show_context_menu(self, position):
        if not self.itemAt(position):
            return

        self.context_menu.exec(self.mapToGlobal(position))

    def check_item_options(self, item_type: str, value: str, action: ItemAction):
        if not self.item_options:
            return False

        for item in self.item_options:
            if item["type"] != item_type:
                continue

            if "value" in item:
                if item["value"] == value and item["action"] == action.value:
                    return True
            elif "regex" in item and item["action"] == action.value:
                if re.match(item["regex"], value):
                    return True

        return False

    def update_data(self, items):
        self.clear()

        self.news = {}

        grouped_items = {}

        for entry in items:
            folder = entry["folder"]
            feed = entry["feed"]

            self.news[entry["id"]] = entry

            if folder not in grouped_items:
                grouped_items[folder] = {}

            if feed not in grouped_items[folder]:
                grouped_items[folder][feed] = []

            grouped_items[folder][feed].append(entry)

        sorted_folders = OrderedDict(sorted(grouped_items.items(), key=lambda item: item[0].lower()))

        for folder_name, feeds in sorted_folders.items():
            if self.check_item_options("folder", folder_name, ItemAction.EXCLUDE):
                continue

            folder_item = QtWidgets.QTreeWidgetItem(self)

            folder_item.setText(0, "{} ({})".format(folder_name, sum([len(entries) for entries in feeds.values()])))
            folder_item.setExpanded(not self.check_item_options("folder", folder_name, ItemAction.COLLAPSE))

            sorted_feeds = OrderedDict(sorted(feeds.items(), key=lambda item: item[0].lower()))

            for feed_name, entries in sorted_feeds.items():
                if self.check_item_options("feed", feed_name, ItemAction.EXCLUDE):
                    continue

                feed_item = QtWidgets.QTreeWidgetItem(folder_item)

                feed_item.setText(0, "{} ({})".format(feed_name, len(entries)))
                feed_item.setExpanded(not self.check_item_options("feed", feed_name, ItemAction.COLLAPSE))

                for entry in entries:
                    if self.check_item_options("entry", entry["title"], ItemAction.EXCLUDE):
                        continue

                    entry_item = QtWidgets.QTreeWidgetItem(feed_item)

                    datetime_string = datetime.datetime.fromtimestamp(entry["pubDate"]).strftime("%c")

                    entry_item.setData(0, QtCore.Qt.UserRole, entry)

                    entry_item.setText(0, entry["title"])
                    entry_item.setData(0, QtCore.Qt.ToolTipRole, entry["title"])
                    entry_item.setText(1, datetime_string)
                    entry_item.setData(1, QtCore.Qt.ToolTipRole, datetime_string)

    def get_selected_items(self):
        selected_model_indexes = self.selectedIndexes()

        item_ids = set()

        for model_index in selected_model_indexes:
            entry = model_index.siblingAtColumn(0).data(QtCore.Qt.UserRole)

            if entry:
                item_ids.add(entry["id"])

        items = []

        for item_id in item_ids:
            item = self.news[item_id]

            items.append(item)

        return items

    def open_selected_items(self):
        items = self.get_selected_items()

        if not items:
            return

        for item in items:
            subprocess.run(["xdg-open", item["url"]])

            self.mark_item_as_read(item)

        self.updater_thread.start()

    def copy_selected_items(self):
        items = self.get_selected_items()

        if not items:
            return

        lines = [item["url"] for item in items]

        pyperclip.copy("\n".join(lines))

    def mark_selected_item_as_read(self):
        items = self.get_selected_items()

        if not items:
            return

        for item in items:
            self.mark_item_as_read(item)

        self.updater_thread.start()

    def mark_item_as_read(self, item):
        requests.put("{}/items/{}/read".format(self.base_url, item["id"]), auth=self.auth)


import json
import re
from copy import deepcopy

import arrow
import six
from django.utils.decorators import available_attrs
from six.moves import filter

from apps.dataflow.models import ResultTableQueryRecord, ResultTableSelectedRecord
from apps.exceptions import DataError

try:
    from functools import wraps
except ImportError:
    from django.utils.functional import wraps

from django.utils.translation import ugettext_lazy as _

from apps import exceptions
from apps.api import (
    AuthApi,
    DatabusApi,
    DataFlowApi,
    DataManageApi,
    DataQueryApi,
    MetaApi,
    StorekitApi,
)
from apps.common.log import logger
from apps.dataflow.datamart.datadict import get_format_tag_list, processing_type_dict
from apps.dataflow.handlers.business import Business
from apps.dataflow.utils import subdigit
from apps.exceptions import ApiResultError
from apps.utils import APIModel
from apps.utils.time_handler import (
    DTEVENTTIMESTAMP_MULTIPLICATOR,
    generate_influxdb_time_range,
    get_active_timezone_offset,
    get_pizza_timestamp,
    list_date_by_a_period_of_time,
    list_year_month_by_a_period_of_time,
    time_format,
    timeformat_to_timestamp,
    timestamp_to_timeformat,
)

class ResultTable(APIModel):
    KEYS = ["result_table_id"]

    # 默认添加的时间字段
    DEFAULT_TIME_FIELDS_DETAILS = [
        {
            "field_name": "dtEventTime",
            "field_alias": _("数据时间，格式：YYYY-mm-dd HH:MM:SS"),
            "field_type": "string",
            "description": _("数据时间，格式：YYYY-mm-dd HH:MM:SS"),
        },
        {
            "field_name": "dtEventTimeStamp",
            "field_alias": _("数据时间戳，毫秒级别"),
            "field_type": "long",
            "description": _("数据时间戳，毫秒级别"),
            "is_dimension": 0,
        },
        {
            "field_name": "localTime",
            "field_alias": _("本地时间，格式：YYYY-mm-dd HH:MM:SS"),
            "field_type": "string",
            "description": _("本地时间，格式：YYYY-mm-dd HH:MM:SS"),
            "is_dimension": 0,
        },
        {
            "field_name": "thedate",
            "field_alias": _("数据时间，格式：YYYYmmdd"),
            "field_type": "int",
            "description": _("数据时间，格式：YYYYmmdd"),
            "is_dimension": 0,
        },
    ]

    # TSDB时间字段
    TSDB_TIME_FIELDS_DETAILS = [
        {
            "field_name": "time",
            "field_alias": _("数据时间戳，毫秒级别"),
            "type": "long",
            "description": _("数据时间戳，毫秒级别"),
            "is_dimension": 0,
        }
    ]

    class Platform(object):
        BKDATA_PLATFORM_NAME = "bkdata"
        TDW_PLATFORM_NAME = "tdw"

    class SourceType(object):
        RT_SOURCE = "rtsource"
        ETL_SOURCE = "etl_source"
        STREAM_SOURCE = "stream_source"
        BATCH_SOURCE = "batch_source"
        KV_SOURCE = "kv_source"
        BATCH_KV_SOURCE = "batch_kv_source"
        TDW_SOURCE = "tdw_source"
        UNIFIED_KV_SOURCE = "unified_kv_source"

    def __init__(self, result_table_id):
        super(ResultTable, self).__init__()
        self.result_table_id = result_table_id
        self._storage_query = []
        self._storage_display = {}

        self._recommend_sql = None
        self._recommend_storage_fields = None
        self._recommend_storage_type = None
        self._storage_type_sql = None

    @classmethod
    def list_as_source_by_project_id(cls, project_id, source_type, bk_biz_id=None):
        """
        根据数据源类型列出项目有权限的结果表列表
        :param project_id: 项目id
        :param source_type: 数据源类型
        :param bk_biz_id: 业务id
        """
        # 获取项目有权限的业务id
        params = {"project_id": project_id}
        if bk_biz_id is not None:
            params["bk_biz_id"] = bk_biz_id
        project_data = AuthApi.projects.data(params)

        biz_ids = list({_data["bk_biz_id"] for _data in project_data})

        auth_biz_ids = [_data["bk_biz_id"] for _data in project_data if not _data.get("result_table_id")]

        auth_result_table_ids = [_data["result_table_id"] for _data in project_data if _data.get("result_table_id")]

        rts = []

        # 先通过业务列表拉取到全部RT列表，本地在进行过滤
        rts_biz = ResultTable.list(bk_biz_id=biz_ids, has_fields=False, source_type=source_type)

        rts_biz = list(
            filter(
                (lambda _rt: _rt["bk_biz_id"] in auth_biz_ids or _rt["result_table_id"] in auth_result_table_ids),
                rts_biz,
            )
        )

        # 目标结果表列表
        rts.extend(rts_biz)

        # 将项目底下存在结果表也加入选择源
        rts_projects = ResultTable.list(project_id, bk_biz_id=bk_biz_id, has_fields=False, source_type=source_type)

        if rts_projects:
            rts.extend(rts_projects)

        cleaned_rts = []
        cleaned_rt_ids = []

        for _rt in rts:
            _o_rt = cls.init_by_data(_rt)
            if _o_rt.result_table_id not in cleaned_rt_ids:
                # 去重
                cleaned_rt_ids.append(_o_rt.result_table_id)
                cleaned_rts.append(_rt)
        return cleaned_rts

    @classmethod
    def list(
        cls, project_id=None, bk_biz_id=None, is_query=False, has_fields=True, source_type="rtsource", action_id=None
    ):
        """
        结果表列表，支持项、业务过滤

        @param {Boolean} is_query 是否仅返回支持查询的结果表列表
        @param {Boolean} has_fields 是否返回字段列表
        @param {Boolean} source_type
        rtsource(结果数据)
        etl_source(清洗)
        stream_source(实时)
        batch_source(离线)
        kv_source(关联)
        """
        # 默认查询包含存储关系和结果表类型的结果表列表
        api_param = {
            "related": ["storages", "result_table_type", "project_name"],
            "need_storage_detail": 0,
        }
        if project_id is not None:
            api_param["project_id"] = project_id
        if bk_biz_id is not None:
            api_param["bk_biz_id"] = bk_biz_id
        if has_fields:
            api_param["related"].append("fields")
        if source_type == "tdw_source":
            api_param["extra"] = True

        if action_id is None:
            rts = MetaApi.result_tables.list(api_param)
        else:
            api_param["action_id"] = action_id
            rts = MetaApi.result_tables.mine(api_param)

        cleaned_rts = []

        # 此处避免 filter_supported_query_storages 重复去调用 get_supported_storage_query
        support_query_storages = cls.get_supported_storage_query()
        if is_query:
            # 有存储的可查询的结果表
            cleaned_rts = []
            for _rt in rts:
                _o_rt = cls.init_by_data(_rt)

                _storages_arr = cls.filter_supported_query_storages(
                    _o_rt.storages_arr, support_storages=support_query_storages
                )

                # _storages_arr = _o_rt.query_storages_arr
                if _storages_arr:
                    _rt["query_storages_arr"] = _storages_arr
                    cleaned_rts.append(_rt)
        else:
            # 根据数据源类别获取rt
            for _rt in rts:
                # TDW表，
                _o_rt = cls.init_by_data(_rt)

                if _o_rt.judge_source_type(source_type):

                    # 作为TDW数据源时，特殊构造中文名
                    if source_type == "tdw_source":
                        _o_rt._data["result_table_name_alias"] = "{}[{}]".format(
                            _o_rt._data["result_table_name_alias"], _o_rt.tdw_table_name
                        )
                    cleaned_rts.append(_rt)

        return cleaned_rts

    @classmethod
    def list_rt_storage_info(cls, l_result_table_ids):
        """
        获取rt对应的存储信息
        :param l_result_table_ids: 结果表id列表
        :return:
        """
        if not l_result_table_ids:
            return []
        api_param = {"result_table_id": l_result_table_ids}
        storage_nodes = DataFlowApi.flow_nodes.storage_nodes(api_param)
        return storage_nodes

    @classmethod
    def list_latest_data(cls, rt_ids):
        """
        查询结果表最新的数据，支持批量
        :param rt_ids:结果表列表
        :return: {
            "rt1": {"time": "2018-01-01 11:11", },
        }
        """
        rt_condition = "logical_tag = '" + "' or logical_tag = '".join(rt_ids) + "'"
        sql = (
            'SELECT *,logical_tag FROM "data_loss_output_total" '
            "where data_inc > 0 and ({}) group by logical_tag, component "
            "order by time desc limit 1".format(rt_condition)
        )
        api_param = {"database": "monitor_data_metrics", "sql": sql}
        try:
            latest_data = DataManageApi.dmonitor_metrics.query(api_param)
        except (exceptions.ApiResultError, exceptions.ApiRequestError):
            latest_data = {"data_format": "error", "series": {"values": [[]]}}

        if latest_data["data_format"] == "series":
            data = [_d.get("values")[0] for _d in latest_data.get("series", [])]
        elif latest_data["data_format"] == "simple" and len(latest_data["series"]) > 0:
            data = [latest_data["series"][0]]
        else:
            data = []

        d_data = {}
        for _d in data:
            _d["time"] = timestamp_to_timeformat(_d.get("time"))
            d_data.update({_d.get("logical_tag"): _d})

        return d_data

    @classmethod
    def dmonitor_metrics(self, sql):
        """查询数据的监控指标"""
        api_param = {
            "database": "monitor_data_metrics",
            "sql": sql,
        }
        try:
            data_counts = DataManageApi.dmonitor_metrics.query(api_param)
        except (exceptions.ApiResultError, exceptions.ApiRequestError) as e:
            data_counts = {"series": []}
            logger.error("dmonitor_metrics api error, error={}".format(e.message))
        return data_counts

    def list_rt_count(self, start_timestamp, end_timestamp, frequency=None):
        """
        查询结果表的数据趋势
        """
        # 生成influxdb需要的时间段
        start_timestamp, end_timestamp = generate_influxdb_time_range(start_timestamp, end_timestamp)

        group_by = "GROUP BY time({})".format(frequency) if frequency else ""

        if self.recommend_storage_type:
            # 查meta的result_tables接口拿到storage_cluster_config_id
            storage_cluster_config_id = (
                self.storages.get(self.recommend_storage_type, "")
                .get("storage_cluster", {})
                .get("storage_cluster_config_id", "")
            )
            rt_condition = "logical_tag = '%s'" % self.result_table_id
            # sql = "select sum(data_inc) as cnt from data_loss_output_total where {} and storage = 'storage_{}' and " \
            #       "module = 'shipper' and time >= {} and  time <= {} {} fill(null)" \
            #     .format(rt_condition, storage_cluster_config_id, start_timestamp, end_timestamp, group_by)
            sql = (
                "select sum(data_inc) as cnt from data_loss_output_total where {} and storage = 'storage_{}' "
                "and time >= {} and  time <= {} {} fill(null)".format(
                    rt_condition, storage_cluster_config_id, start_timestamp, end_timestamp, group_by
                )
            )
            data_counts = self.dmonitor_metrics(sql)
            cnts = [data["cnt"] for data in data_counts.get("series", [])]
            time = [data["time"] for data in data_counts.get("series", [])]

            formated_time = time_format(time)
            return_data = {"cnt": cnts, "time": formated_time, "timezone": get_active_timezone_offset()}
        else:
            return_data = {"cnt": [], "time": []}
        return return_data

    @classmethod
    def query_rt(cls, sql, storage_type=None):
        api_param = {"sql": sql}
        if storage_type:
            api_param["prefer_storage"] = storage_type
        response = DataQueryApi.query(api_param)
        if storage_type == "es":
            data = response["list"]
            data_list = []
            # 正则只保留高亮部分标签，避免第三方数据造成的xss攻击
            regex = r"<(?P<tag>(?!(em|/em))(.*?))>"
            replace_tag = r"<\g<tag>>"
            for _d in data["hits"]["hits"]:
                _source = _d["_source"]
                for key in _d.get("highlight", []):
                    highlight = "".join(_d["highlight"][key])
                    highlight = re.sub(regex, replace_tag, highlight)
                    _source.update({key: highlight})
                data_list.append(_source)
            time_taken = round(float(data["took"]) / float(1000), 3)
        else:
            # 补充字段名称
            data_list = response.get("list", [])
            time_taken = round(response["timetaken"], 3)

        data_field = response.get("select_fields_order", [])
        if len(data_list):
            if len(data_field) == 0 or data_field[0] == "*":
                data_field = list(data_list[0].keys())
        response = dict(
            select_fields_order=data_field, list=data_list, total=response.get("totalRecords", 0), time_taken=time_taken
        )
        return response

    @classmethod
    def list_selected_history(cls, username):
        """
        用户最近选择的结果表
        :param username:
        :return:
        """
        rt_ids = ResultTableSelectedRecord.list_selected_history(username)
        if not rt_ids:
            return []

        history_result_tables = {}

        api_params = {"related": ["storages"], "result_table_ids": rt_ids}
        l_rt_ids = MetaApi.result_tables.list(api_params)
        for _rt_id in l_rt_ids:
            _rt_id["query_storages_arr"] = cls.filter_supported_query_storages(list(_rt_id["storages"].keys()))
            history_result_tables[_rt_id["result_table_id"]] = _rt_id

        rt_info = []

        latest_data = cls.list_latest_data(rt_ids)
        for rt_id in rt_ids:
            # 添加最后一条数据时间
            try:
                data = history_result_tables[rt_id]
            except KeyError:
                # 结果表不存在或被删了，则跳过
                continue

            latest_data_times = []
            for storage_type in data.get("query_storages_arr", []):
                try:
                    data_time = latest_data[rt_id].get("time")
                except KeyError:
                    # 不存在结果表+存储类型对应的数据
                    continue
                else:
                    latest_data_times.append(data_time)

            if latest_data_times:
                latest_data_time = max(latest_data_times)
            else:
                latest_data_time = _("暂无上报数据")

            data.update(latest_data_time=latest_data_time)
            rt_info.append(data)

        return rt_info

    def get_latest_msg(self, source_type):
        """
        根据数据源类型
        :param source_type: 数据源类型
        :return:
        """
        result = _("暂不支持此类数据源的数据查询")
        if source_type == "stream_source":
            result = self.get_kafka_latest_msg()
        if source_type == "batch_source":
            result = self.get_hdfs_latest_msg()
        if source_type == "kv_source":
            result = _("暂不支持关联数据查询")

        return result

    def get_kafka_latest_msg(self):
        """
        从 KAFKA 获取最新一条数据
        """
        if self.has_kafka():
            try:
                latest_msgs = DatabusApi.result_tables.tail({"result_table_id": self.result_table_id})
            except (exceptions.ApiResultError, exceptions.ApiResultError):
                latest_msgs = [{"data": _("获取实时数据失败")}]
        else:
            latest_msgs = []

        return json.dumps(latest_msgs[0]) if latest_msgs else _("暂无数据")

    def get_hdfs_latest_msg(self):
        """
        从 HDFS 中获取最新一条数据
        """
        if self.has_hdfs():
            try:
                _latest_msg = DataFlowApi.batch_hdfs_result_tables.new_line({"result_table_id": self.result_table_id})
            except (exceptions.ApiResultError, exceptions.ApiResultError):
                _latest_msg = _("获取离线数据失败")
            _latest_msg = _latest_msg or _("暂无数据")
        else:
            _latest_msg = _("暂无数据")
        return _latest_msg

    def list_calculate_fields(self):
        """
        返回 RT 的字段列表，计算字段
        """
        fields = self.fields
        fields = self.clear_default_field(fields)
        return fields

    def add_time_field(self, storage_type, fields):
        """
        添加默认时间字段
        :param storage_type:
        :param fields:
        :return:
        """
        if storage_type == "tsdb":
            fields.extend(self.TSDB_TIME_FIELDS_DETAILS)
        else:
            fields.extend(self.DEFAULT_TIME_FIELDS_DETAILS)
        return fields

    def list_storage_fields(self, storage_type="all", has_judge=True):
        """
        返回存储字段列表
        """
        fields = self.fields
        fields = self.clear_default_field(fields)
        fields = self.add_time_field(storage_type, fields)
        if has_judge:
            # 添加图标

            icons = [
                {"judge": "is_dimension", "key": "dimension", "description": _("维度")},
                {"judge": "is_analyzed", "key": "analyzed", "description": _("分词")},
                {"judge": "is_value", "key": "value", "description": _("值")},
                {"judge": "is_time", "key": "time", "description": _("时间")},
                {"judge": "is_index", "key": "index", "description": _("索引")},
                {"judge": "is_main_key", "key": "main_key", "description": _("主键")},
            ]

            for _f in fields:
                _f.update({"icon": []})
                for icon in icons:
                    if _f.get(icon["judge"]):
                        _f["icon"].append(icon)
        return fields

    @property
    def storage_type_sql(self):
        """
        获取rt关联对应物理存储的schema, 默认查询语句, 存储的查询顺序
        :return:
        {
            "storages":[
                "tsdb"
            ],
            "storage":{
                "tsdb":{
                    "fields":[
                        {
                            "physical_field":"ip",
                            "field_type":"string",
                            "field_alias":"未设置",
                            "description":"",
                            "is_dimension":true,
                            "field_index":0,
                            "is_time":false,
                            "field_name":"ip",
                            "physical_field_type":"string"
                        }
                    ],
                    "config":{

                    },
                    "sql":"SELECT ip FROM 0_docker_cpu_summary WHERE time > '1h' ORDER BY time DESC LIMIT 10"
                }
            },
            "order":[
                "tsdb"
            ]
        }
        """
        if not self._storage_type_sql:
            schema_type_sql_dict = StorekitApi.schema_and_sql.list({"result_table_id": self.result_table_id}, raw=True)
            if schema_type_sql_dict.get("result") and isinstance(schema_type_sql_dict.get("data"), dict):
                self._storage_type_sql = schema_type_sql_dict.get("data")

        return self._storage_type_sql

    @property
    def recommend_storage_type(self):
        """
        rt表从存储接口拿到的推荐存储类型
        """
        # 1 查存储
        if not self._recommend_storage_type and self.storage_type_sql:
            storage_type_list = self.storage_type_sql.get("order", [])
            if storage_type_list:
                self._recommend_storage_type = storage_type_list[0]
        return self._recommend_storage_type

    @property
    def recommend_sql(self):
        """
        rt表从存储接口拿到推荐存储对应的sql
        """
        if not self._recommend_sql and self.recommend_storage_type:
            self._recommend_sql = (
                self.storage_type_sql.get("storage", {}).get(self.recommend_storage_type, {}).get("sql", "")
            )
        return self._recommend_sql

    @property
    def recommend_storage_fields(self):
        """
        从存储接口拿到推荐存储rt所有的字段列表，包含dtEventTime字段，如果没有推荐的存储，则拿kafka对应的字段
        :return:[{},{}]
        """
        # 1 查存储,有存储
        # 2 没有存储, 查所有存储对应的信息，然后找到kafka
        if not self._recommend_storage_fields:
            storage_sql_res = {}
            if self.recommend_storage_type and self.storage_type_sql:
                storage_type = self.recommend_storage_type
                storage_sql_res = self.storage_type_sql
            if not self.recommend_storage_type:
                # 如果没有推荐存储类型，去拿kafka对应的schema

                storage_type = "kafka"
                storage_sql_res = self.kafka_schema_sql()

            fields_tmp = storage_sql_res.get("storage", {}).get(storage_type, {}).get("fields", [])
            self._recommend_storage_fields = [
                {
                    "field_name": field["physical_field"],
                    "field_alias": field["field_alias"],
                    "field_type": field["field_type"],
                    "description": field["description"],
                }
                for field in fields_tmp
            ]

        return self._recommend_storage_fields

    def kafka_schema_sql(self):
        """
        获取所有存储包含kafka的schema和默认查询语句
        :return:
        {
            "storages":[],
            "storage":{
                "kafka":{
                    "fields":[],
                    "config":{},
                    "sql":""
                },
                "queue":{},
            },
            "order":[]
        }
        """
        kafka_schema_sql_dict_tmp = StorekitApi.schema_and_sql.list(
            {"result_table_id": self.result_table_id, "flag": "all"}, raw=True
        )
        kafka_schema_sql_dict = kafka_schema_sql_dict_tmp.get("data") if kafka_schema_sql_dict_tmp.get("result") else {}
        return kafka_schema_sql_dict

    def tag_list(self, page=1, page_size=10):
        """
        获取rt包含的标签列表，根据tag类型排序，业务标签 > 来源标签 > 应用标签 > 描述标签 > 管理标签（地域）
        :return:
        [{tag_alias: "中国内地", tag_code: "inland", tag_type: "manage"}]
        """
        target_filter = {"condition": "OR"}
        criterion = [{"k": "result_table_id", "v": self.result_table_id, "func": "eq"}]
        target_filter["criterion"] = criterion
        params = {
            "page": page,
            "page_size": page_size,
            "target_type": "result_table",
            "target_filter": json.dumps([target_filter]),
        }
        tag_search = MetaApi.target_tag_info(params)

        tag_list = []
        if tag_search and tag_search.get("content", []):
            tag_search = tag_search.get("content", [])[0].get("tags", {})
            # 对实体查到的标签进行格式化处理和排序
            tag_list = get_format_tag_list(tag_search)
        return tag_list

    def data_manager(self):
        """
        rt的数据管理员
        :return:
        """
        manager = AuthApi.roles_users.list({"role_id": "result_table.manager", "scope_id": self.result_table_id})
        return manager

    def data_viewer(self):
        """
        rt的数据观察员
        :return:
        """
        viewer = AuthApi.roles_users.list({"role_id": "result_table.viewer", "scope_id": self.result_table_id})
        return viewer

    def data_manager_and_viewer(self):
        """
        rt的数据管理员 & 数据观察员
        :return:
        """
        manager = self.data_manager()
        viewer = self.data_viewer()
        return manager, viewer

    def data_category_alias(self):
        """
        rt对应的数据分类中文名
        :return:
        """
        data_category_alias = None
        if self.data.get("data_category", ""):
            data_category_dict = MetaApi.tag_info({"code": self.data["data_category"]}, raw=True)
            data_category_alias = data_category_dict.get("alias", "") if data_category_dict.get("result", "") else None
        return data_category_alias

    def judge_permission(self, username, action_type="query_data"):
        """查询用户有无结果表的查询权限"""
        object_type = "{}.{}".format("result_table", action_type)
        try:
            permission_dict = AuthApi.check_user_perm(
                {
                    "user_id": username,
                    "action_id": object_type,
                    "object_id": self.result_table_id,
                    "display_detail": True,
                }
            )
            has_permission = permission_dict.get("result", False)
            no_pers_reason = permission_dict.get("message", "")
        except DataError as e:
            logger.warning("rt judge_permission error: [%s]" % e.message)
            has_permission = False
            no_pers_reason = ""
        return has_permission, no_pers_reason

    def expires(self):
        """获取rt表的存储天数"""
        storage_types_dict = {}
        for key, value in six.iteritems(self.data.get("storages", {})):
            storage_types_dict[key] = subdigit(value.get("expires", ""))
        expires = storage_types_dict
        return expires

    def can_search(self):
        """查询rt是否有可以查询的存储"""
        can_search_list = ["tspider", "mysql", "druid", "tdw", "hdfs", "tsdb", "hermes", "es"]
        can_search = False
        for each_storage in self.storages_arr:
            if each_storage in can_search_list:
                can_search = True
                break
        return can_search

    def has_storage(self, storage):
        """
        查看 RT 是否有指定的存储
        """
        _arr = self.storages_arr
        return storage in _arr

    def has_kafka(self):
        """
        是否具有 KAFKA 存储，只要 RT（非离线） 需要入库某一类存储，都认定为具有 KAFKA 存储，因为 RT 要
        入库到存储，必须经过总线分发，而总线的介质层就是 kafka

                               (connector)
        RT ---------> kafka -------------------> Mysql

        """
        return self.has_storage("kafka")

    def has_hdfs(self):
        """
        是否具有 HDFS 存储，存储配置中显性配置以及离线表默认带上 HDFS
        """
        return self.has_storage("hdfs")

    def has_tredis(self):
        """
        是否具有 Tredis 存储
        """
        return self.has_storage("tredis")

    def has_ipredis(self):
        """
        是否具有 IPREDIS 存储
        """
        return self.has_storage("ipredis")

    def has_ignite(self):
        """
        是否具有 Ignite 存储
        """
        return self.has_storage("ignite")

    def has_tdw(self):
        """
        是否具有 tdw 存储
        """
        return self.has_storage("tdw")

    def is_clean(self):
        """
        结果表类型为1，则是clean
        """
        return self.result_table_type == "clean"

    def is_batch(self):
        """
        结果表类型为2，则是batch
        """
        return self.result_table_type == "batch"

    def judge_source_type(self, source_type):
        """
        判断是否为某种数据源类型
        :param source_type: 数据源类型 source_type 见类SourceType
        :return:
        """
        result = False

        if source_type == self.SourceType.RT_SOURCE:
            # 非清洗表就是结果数据
            result = not self.is_clean() and (self.has_kafka() or self.has_hdfs())
        elif source_type == self.SourceType.ETL_SOURCE:
            result = self.is_clean()
        elif source_type == self.SourceType.STREAM_SOURCE:
            result = self.has_kafka()
        elif source_type == self.SourceType.BATCH_SOURCE:
            result = self.has_hdfs()
        elif source_type == self.SourceType.KV_SOURCE:
            result = self.has_tredis() or self.has_ipredis()
        elif source_type == self.SourceType.BATCH_KV_SOURCE:
            result = self.has_ignite()
        elif source_type == self.SourceType.UNIFIED_KV_SOURCE:
            result = self.has_ignite() or self.has_ipredis()
        elif source_type == self.SourceType.TDW_SOURCE:
            result = self.can_be_tdw_source()

        return result

    def can_be_tdw_source(self):
        """
        # 判断能否作为tdw数据源表
        # 可作为TDW数据源的条件：
        # 1. 有TDW存储（storages包含tdw）
        #   1.1 受控表（is_managed=1)
        #   1.2 非受控表（is_managed=0)
        #       1.2.1 目前仅支持周期类型为小时和日的TDW表（self.count_freq_unit in ['H', 'd']）
        """

        def judge_count_freq():
            # @todo 由于产品问题，暂不判断统计频率，若不满足需求，保存节点时flowapi报错即可
            return True
            # return self.count_freq_unit in ['H', 'd']

        return self.has_tdw() and (self.is_managed or (not self.is_managed and judge_count_freq()))

    def is_exist(self):
        try:
            self._data = self._get_data()
            return True
        except ApiResultError:
            return False

    def _get_data(self):
        api_params = {"result_table_id": self.result_table_id, "related": ["storages", "fields", "result_table_type"]}
        return MetaApi.result_tables.retrieve(api_params)

    @classmethod
    def filter_supported_query_storages(cls, storages, support_storages=None):
        """
        过滤出支持在WEB页面查询的存储列表
        """
        # 支持查询的存储
        if support_storages is None:
            support_storages = cls.get_supported_storage_query()

        query_storages = [storage for storage in storages if storage in support_storages]
        return query_storages

    @classmethod
    def get_supported_storage_query(cls):
        """
        支持查询的存储，从StorekitApi中获取
        """
        return StorekitApi.common.list().get("storage_query", [])

    @property
    def storages_arr(self):
        """
        结果表已落的存储列表
        :return:
        """
        return list(self.data["storages"].keys())

    @property
    def storages(self):
        """
        结果表已落的存储信息
        :return:
        """
        return self.data["storages"]

    @property
    def query_storages_arr(self):
        """
        @note： 如果大量RT列表，轮训该属性，需要考虑性能问题
        """
        return self.filter_supported_query_storages(self.storages_arr)

    @property
    def support_storages(self):
        storages = list(self.data["storages"].keys())
        support_storages = [storage for storage in storages if storage in self.get_supported_storage_query()]
        return support_storages

    @property
    def description(self):
        return self.data["description"]

    @property
    def result_table_type(self):
        """
        结果表类型
        id=1: clean
        id=2: batch
        id=3: steam
        """
        return self.data.get("processing_type")

    @property
    def result_table_type_alias(self):
        """
        结果表类型中文名
        """
        # processing_type_dict = get_processing_type_dict()
        processing_type_alias = (
            processing_type_dict.get(self.result_table_type)
            if self.result_table_type and self.result_table_type not in ["storage", "view"]
            else ""
        )
        return processing_type_alias

    @property
    def platform(self):
        """
        结果表所属平台
        bkdata=蓝鲸基础计算平台
        tdw=TDW
        """
        return self.data.get("platform", "bkdata")

    @property
    def fields(self):
        """
        字段列表
        """
        return self.data.get("fields", [])

    @property
    def biz_id(self):
        return int(self.data["bk_biz_id"])

    @property
    def biz_name(self):
        all_biz_dict = Business.get_name_dict()
        if self.data.get("bk_biz_id", ""):
            return all_biz_dict.get(self.biz_id)
        else:
            return ""

    @property
    def project_id(self):
        return int(self.data.get("project_id", 0))

    @property
    def table_name(self):
        return self.data.get("result_table_name", "")

    @property
    def tdw_table_name(self):
        """
        TDW表名
        """
        try:
            return self.data["extra"]["tdw"]["table_name"]
        except KeyError:
            return ""

    @property
    def count_freq_unit(self):
        """
        TDW统计频率单位
        """
        try:
            return self.data["count_freq_unit"]
        except KeyError:
            return ""

    @property
    def count_freq(self):
        """
        TDW统计频率
        """
        try:
            return self.data["count_freq"]
        except KeyError:
            return 0

    @property
    def is_managed(self):
        """
        是否平台受控表
        """
        return self.data.get("is_managed", 1)

    @staticmethod
    def save_query_record(storage_type=None):
        """
        记录结果表查询记录装饰器
        """

        def _wrap(func):
            @wraps(func, assigned=available_attrs(func))
            def _deco(self, request, *args, **kwargs):
                param = request.data
                record = ResultTableQueryRecord()
                record.project_id = kwargs.get("project_id")
                record.result_table_ids = kwargs.get("result_table_id")
                record.operator = request.user.username
                record.storage_type = param.get("storage_type", storage_type)
                try:
                    _f = func(self, request, *args, **kwargs)
                    record.result = True
                    record.err_msg = None
                except exceptions.ApiResultError as e:
                    record.result = False
                    record.err_msg = e.message
                    raise e
                else:
                    record.time_taken = _f.data.get("time_taken")
                    record.total = _f.data.get("total")
                    return _f
                finally:
                    try:
                        defaults = {
                            "time_taken": record.time_taken,
                            "total": record.total,
                            "result": record.result,
                            "err_msg": record.err_msg,
                        }
                        unique = {
                            "project_id": record.project_id,
                            "operator": record.operator,
                            "storage_type": record.storage_type,
                            "result_table_ids": record.result_table_ids,
                        }
                        if record.storage_type == "es":
                            record.keyword = param.get("keyword")
                            # 无关键字时不记录
                            if record.keyword:
                                unique.update(
                                    {
                                        "keyword": record.keyword,
                                        "search_range_start_time": param.get("start_time"),
                                        "search_range_end_time": param.get("end_time"),
                                    }
                                )

                                ResultTableQueryRecord.objects.update_or_create(defaults=defaults, **unique)
                        else:
                            record.sql = param.get("sql")
                            unique.update(
                                {
                                    "sql": record.sql,
                                }
                            )
                            ResultTableQueryRecord.objects.update_or_create(defaults=defaults, **unique)
                    except Exception:
                        pass

            return _deco

        return _wrap

    @staticmethod
    def clear_default_field(fields):
        """
        默认去掉timestamp, offset等内部系统默认字段
        """
        return [_d for _d in fields if _d["field_name"] not in ["timestamp", "offset"]]

    def list_output_count(self, query_params):
        """
        数据输出量
        """
        query_params["data_set_ids"] = self.result_table_id
        try:
            output_data = DataManageApi.dmonitor_metrics.output_count(query_params)
        except (exceptions.ApiResultError, exceptions.ApiRequestError) as e:
            output_data = {"series": []}
            logger.error("dmonitor_metrics_output_data api error, error={}".format(e.message))
        return output_data

    def list_input_count(self, query_params):
        """
        数据输入量
        """
        query_params["data_set_ids"] = self.result_table_id
        try:
            output_data = DataManageApi.dmonitor_metrics.input_count(query_params)
        except (exceptions.ApiResultError, exceptions.ApiRequestError) as e:
            output_data = {"series": []}
            logger.error("dmonitor_metrics_input_data api error, error={}".format(e.message))
        return output_data

class ES(object):
    TIME_MULTIPLE = 1000
    TIME_OFFSET = {
        "7d": 3600 * 24 * 7 * TIME_MULTIPLE,
        "12h": 3600 * 12 * TIME_MULTIPLE,
        "24h": 3600 * 24 * TIME_MULTIPLE,
    }
    BODY_DATA = {
        "highlight": {
            "pre_tags": ["<em>"],
            "post_tags": ["</em>"],
            "fields": {
                "*": {"number_of_fragments": 0},
            },
            "require_field_match": False,
        },
        "query": {
            "bool": {
                "filter": [{"range": {"dtEventTime": {"gt": "", "lte": ""}}}],
                "must": [
                    {
                        "query_string": {
                            "fields": [],
                            "lenient": True,
                            "query": {},
                            "analyze_wildcard": True,
                        }
                    }
                ],
            }
        },
        "sort": [
            {
                "dtEventTimeStamp": {"order": "desc"},
            }
        ],
    }
    SPECIAL_STR_RE = re.compile(r"[\+\-=&|><!(){}\[\]^\"~*?:/]|AND|OR|TO|NOT")

    @classmethod
    def check_special_string(cls, string):
        """
        @summary 是否含有特殊字符
        """
        return re.search(cls.SPECIAL_STR_RE, string)

    @staticmethod
    def create_query_keyword(keyword, fields):
        """
        @summary 组装关键字
        """
        key_list = keyword.split(" ")
        query_list = []

        # 包含特殊字符则不加通配符，否则拿通配符包裹
        for query_key in key_list:
            if not query_key:
                continue
            pattern = "%s" if ES.check_special_string(query_key) else "*%s*"
            query_list.append(pattern % query_key)

        query_str = " ".join(query_list)

        query_dict = {
            "must": [
                {
                    "query_string": {
                        "query": query_str,
                        "fields": fields,
                        "lenient": True,
                        "analyze_wildcard": True,
                    }
                }
            ]
        }
        return dict(query_dict)

    @staticmethod
    def create_query_range(start_time, end_time):
        return {
            "filter": [
                {
                    "range": {
                        "dtEventTimeStamp": {
                            "from": timeformat_to_timestamp(start_time, DTEVENTTIMESTAMP_MULTIPLICATOR),
                            "to": timeformat_to_timestamp(end_time, DTEVENTTIMESTAMP_MULTIPLICATOR),
                        }
                    }
                }
            ]
        }

    @staticmethod
    def create_body_data(size, start, search_key, fields, start_time, end_time):
        """
        日志查询构造请求参数
        """

        body_data = deepcopy(ES.BODY_DATA)
        body_data.update(
            {
                "sort": [{"dtEventTimeStamp": "desc"}],
                "size": size,
                "from": start,
            }
        )
        body_data["query"]["bool"].update(ES.create_query_keyword(search_key, fields))
        body_data["query"]["bool"].update(ES.create_query_range(start_time, end_time))
        if search_key == "":
            body_data["query"]["bool"].pop("must")

        return body_data

    @staticmethod
    def get_index(result_table_id, start_time=None, end_time=None):
        """
        索引由 result_table_id_${日期} 组成，若不限制查询时间，会比较慢，建议加上时间
        :param result_table_id: 结果表
        :param start_time: 开始时间
        :param end_time: 结束时间
        :return: result_table_id_20181201,result_table_id_20190102,result_table_id_20190203
        """
        # 搜索当前结果表的所有es索引

        # 查询时长超过两个月时使用月份精度，小于两个月时使用日期精度
        if (arrow.get(end_time) - arrow.get(start_time)).days < 60:
            date_list = list_date_by_a_period_of_time(start_time, end_time)
            index_list = [
                "{result_table_id}_{date}".format(result_table_id=result_table_id, date=date) for date in date_list
            ]
        else:
            year_mont_list = list_year_month_by_a_period_of_time(start_time, end_time)
            index_list = [
                "{result_table_id}_{year_month}*".format(result_table_id=result_table_id, year_month=year_month)
                for year_month in year_mont_list
            ]
        return ",".join(index_list)

    @staticmethod
    def get_default_dsl(result_table_id):
        index = ES.get_index(result_table_id)
        time_zone = get_active_timezone_offset()
        time_zone = ":".join([time_zone[:3], time_zone[3:]])
        body_data = {
            "aggs": {
                "time_aggs": {
                    "date_histogram": {
                        "field": "dtEventTimeStamp",
                        "interval": "hour",
                        "time_zone": time_zone,
                        # "format": "yyyy-MM-dd"
                    }
                }
            },
            "query": {
                "bool": {
                    "filter": [
                        {
                            "range": {
                                "dtEventTimeStamp": {
                                    "gt": int(get_pizza_timestamp() * ES.TIME_MULTIPLE) - ES.TIME_OFFSET["24h"],
                                    "lte": int(get_pizza_timestamp() * ES.TIME_MULTIPLE),
                                    "format": "epoch_millis",
                                }
                            }
                        }
                    ],
                    "must": [{"query_string": {"query": "*", "analyze_wildcard": True}}],
                }
            },
            "sort": [
                {
                    "dtEventTimeStamp": {"order": "desc"},
                }
            ],
            "from": 0,
            "size": 10,
        }
        sql = json.dumps({"index": index, "body": body_data})
        return sql

    @staticmethod
    def get_search_dsl(size, start, keyword, start_time, end_time, result_table_id):
        """
        @summary:查询日志
        """
        index = ES.get_index(result_table_id, start_time, end_time)
        fields_detail = ResultTable(result_table_id).list_storage_fields()
        l_field_name = [_f["field_name"] for _f in fields_detail]
        body_data = ES.create_body_data(size, start, keyword, l_field_name, start_time, end_time)

        sql = json.dumps({"index": index, "body": body_data})
        return sql

    @staticmethod
    def get_chart_dsl(keyword, interval, start_time, end_time, result_table_id):
        """
        获取以一定时间频率为间隔聚合统计的图表数据
        @param {String} keyword 搜索关键字
        @param {String} date_interval 时间间隔（7d, 12h, 24h）
        """

        index = ES.get_index(result_table_id, start_time, end_time)

        body_dict = {
            "aggs": {
                "time_aggs": {
                    "date_histogram": {
                        "field": "dtEventTimeStamp",
                        "interval": interval,
                        # "time_zone": "+00:00",
                    }
                }
            },
            "query": {
                "bool": {
                    "filter": [
                        {
                            "range": {
                                "dtEventTimeStamp": {
                                    "from": timeformat_to_timestamp(start_time, DTEVENTTIMESTAMP_MULTIPLICATOR),
                                    "to": timeformat_to_timestamp(end_time, DTEVENTTIMESTAMP_MULTIPLICATOR),
                                    # 'format': 'epoch_millis'
                                }
                            }
                        }
                    ],
                    "must": [{"query_string": {"query": "*", "analyze_wildcard": True}}],
                }
            },
        }
        if keyword:
            fields_detail = ResultTable(result_table_id).list_storage_fields()
            l_field_name = [_f["field_name"] for _f in fields_detail]
            query_dict = ES.create_query_keyword(keyword, l_field_name)
            body_dict["query"]["bool"].update(query_dict)
        sql = json.dumps({"index": index, "body": body_dict})
        return sql

    @staticmethod
    def get_es_chart(sql):
        api_param = {"prefer_storage": "es", "sql": sql}
        response = DataQueryApi.query(api_param)
        try:
            buckets = response["list"]["aggregations"]["time_aggs"]["buckets"]
        except KeyError:
            buckets = []
        _time = []
        cnt = []
        for item in buckets:
            _time.append(timestamp_to_timeformat(item["key"], DTEVENTTIMESTAMP_MULTIPLICATOR))
            cnt.append(item["doc_count"])

        data = {"time": _time, "cnt": cnt}
        return data


import numpy
import scipy
import joblib
import sklearn
from sklearn.neighbors import KNeighborsClassifier
from sklearn import tree
from sklearn.linear_model import Perceptron
from sklearn.neighbors import KNeighborsClassifier
import sklearn.metrics

clf_tree = tree.DecisionTreeClassifier()
clf_knn=KNeighborsClassifier()
clf_per=Perceptron()
clf_svc=sklearn.svm.SVC()

# [height, weight, shoe_size]
# https://scikit-learn.org/stable/tutorial/statistical_inference/supervised_learning.html
X = [[181, 80, 44], [177, 70, 43], [160, 60, 38], [154, 54, 37], [166, 65, 40],
     [190, 90, 47], [175, 64, 39],
     [177, 70, 40], [159, 55, 37], [171, 75, 42], [181, 85, 43]]

Y = ['male', 'male', 'female', 'female', 'male', 'male', 'female', 'female',
     'female', 'male', 'male']

data_for_prediction=[[190,70,43]]

# fitting data
clf_tree.fit(X,Y)
clf_knn.fit(X,Y)
clf_per.fit(X,Y)
clf_svc.fit(X,Y)

results_clf_tree = clf_tree.predict(data_for_prediction)
print(results_clf_tree)

tree_res=clf_tree.predict(X)
tree_score=sklearn.metrics.accuracy_score(Y,tree_res)

knn_res=clf_knn.predict(X)
knn_score=sklearn.metrics.accuracy_score(Y,knn_res)

per_res=clf_per.predict(X)
per_score=sklearn.metrics.accuracy_score(Y, per_res)

svc_res=clf_svc.predict(X)
svc_score=sklearn.metrics.accuracy_score(Y, svc_res)

res=[("Tree",tree_res,tree_score),("K Neighbors", knn_res,knn_score),("Perceptron",per_res,per_score),("SVC",svc_res,svc_score)]

print(res)

sorted_res=sorted(res,key=lambda x:x[2])

print("Best score:\n")
best_scr=[i[0] for i in sorted_res]
print(best_scr)


import itertools as it
import torch as th
from ..losses import ptripletN
from .test_common import TestNet
from .pnp import *
from .amd import *
from .est import *
from .ses import *
import pytest

@pytest.mark.skip(reason='this is test helper')
def __test_xxx_training_step(training_step: callable):
    model = TestNet()
    images = th.rand(10, 1, 28, 28)
    labels = th.stack([th.arange(5), th.arange(5)]).T.flatten()
    loss = training_step(model, (images, labels), 0)
    loss.backward()

@pytest.mark.skip(reason='this is test helper')
def __test_hm_training_step(g, r, hm, srch, dsth, ics):
    model = TestNet()
    images = th.rand(10, 1, 28, 28)
    labels = th.stack([th.arange(5), th.arange(5)]).T.flatten()
    loss = hm_training_step(model, (images, labels), 0,
            gradual=g, fix_anchor=r, hm=hm, srch=srch, desth=dsth,
            ics=ics)
    loss.backward()

@pytest.mark.parametrize('ts', [
    pnp_training_step,
    est_training_step,
    rest_training_step,
    ses_training_step,
    ])
def test_xxx_training_step(ts: callable):
    __test_xxx_training_step(ts)

H_MAP = {'r': 'spc2-random',
         'm': 'spc2-semihard',
         's': 'spc2-softhard',
         'd': 'spc2-distance',
         'h': 'spc2-hard'}

@pytest.mark.parametrize('g, r, hm, srch, dsth, ics',
    it.product(
        (False, True),
        (False,),
        ('KL', 'L2', 'ET'),
        'rmsdh',
        'rmsdh',
        (False, True),
        ))
def test_hm_training_step(g, r, hm, srch, dsth, ics):
    __test_hm_training_step(g, r, hm, H_MAP[srch], H_MAP[dsth], ics)


import cv2
import os,sys,time
from IPython import embed
import json
import glob
import labelme2coco
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from skimage.transform import resize, rescale
from tqdm import tqdm
#img_path = "/home/user/storage/kangni/cv_detection/data/camera/image-top/positive/"
#img_path = "/home/user/storage/kangni/cv_detection/data/mate20pro/images/"
#img_path = "."
#files = os.popen("ls %s/*.jpg"%img_path).read().split('\n')
#files.pop()
#
#for file_index,file in enumerate(files[:]):
#    img = cv2.imread("%s"%file)
#    with open(file.replace('.jpg','.json'),'r') as f:
#        shapes = json.load(f)['shapes']
#    for shape_index, shape in enumerate(shapes):
##        embed()
#        x1 = shape['points'][0][0]
#        x2 = shape['points'][1][0]
#        y1 = shape['points'][1][1]
#        y2 = shape['points'][2][1]
#        xmin = min(x1,x2)
#        xmax= max(x1,x2)
#        ymin = min(y1,y2)
#        ymax = max(y1,y2)
#        crop_img = img[ymin:ymax, xmin:xmax]
#        print("On file:(%s of %s, shape %s)"%(file_index,len(files),shape_index), file)
#        cv2.imwrite('%s_%s.jpg'%(file.split('/')[-1].replace('.jpg',''),shape_index), crop_img)
##        cv2.imshow("cropped", crop_img)
#        cv2.waitKey(0)

def on_pick(event):
    artist = event.artist
    xmouse, ymouse = event.mouseevent.xdata, event.mouseevent.ydata
    x, y = artist.get_xdata(), artist.get_ydata()
    ind = event.ind
    print( 'Artist picked:', event.artist)
    print( 'Data point:', x[ind[0]], y[ind[0]], name[i])

known_json_files = 'train/*/*.json'
images_path = './train_images/'

reassign_list = None
reassign_list = {"head":"head", "head1":"head", "head2":"head", "head3":"head", "head4":"head", "head5":"head", "head6":"head", "angle_r":"angle", "angle":"angle", "top_r":"top", "top":"top"}
something_dont_want = None
something_dont_want = ['..', 'bmp']
label_to_id = {"angle":1, "top":2, "head":3}

jsons = glob.glob(known_json_files)
print("Getting data...")
runner = labelme2coco.labelme2coco(jsons, None, reassign_list, something_dont_want, label_to_id)
runner.data_transfer()

#Crop out not labeled data.
print("re-aranging data...")
ids = []
cats = []
bboxs = np.empty(shape=(0,4))
paths = {}
_lefts_ = {}
_rights_ = {}
_ups_ = {}
_downs_ = {}
for i in range(len(runner.images)):
    paths[str(runner.images[i]['id'])] = runner.images[i]['file_name']
    _lefts_[runner.images[i]['file_name']] = []
    _rights_[runner.images[i]['file_name']] = []
    _ups_[runner.images[i]['file_name']] = []
    _downs_[runner.images[i]['file_name']] = []
for i in range(len(runner.annotations)):
    runner.annotations[i]['segmentation'] = np.array(runner.annotations[i]['segmentation'], dtype=np.int).tolist()
    ids.append(paths[str(runner.annotations[i]['image_id'])])
    cats.append(runner.annotations[i]['category_id'])
    bboxs = np.vstack((bboxs, runner.annotations[i]['bbox']))
for idx, name in enumerate(ids):
    #data = plt.imread(images_path+name)
    _lefts_[name].append(bboxs[idx][0])
    _rights_[name].append(bboxs[idx][0]+bboxs[idx][2])
    _ups_[name].append(bboxs[idx][1])
    _downs_[name].append(bboxs[idx][1]+bboxs[idx][3])
all_stuff = {}   #all_stuff is : {"1":[], "2":[], "3":[]}
for i in set(cats):
    all_stuff[str(i)] = []
for index,i in enumerate(cats):
    all_stuff[str(i)].append([ids[index]]+list(np.array(runner.annotations[index]['bbox'], dtype=int)))

objects = np.empty(shape=(50, 0))

#------------Display of Collection of labeled data:-------------
num = int(np.ceil(np.sqrt(len(cats))))
base = np.zeros((num*50, num*50))
i = 0
fig, ax = plt.subplots()
for key in all_stuff.keys():
    for name, a,b,c,d in tqdm(all_stuff[key]):
        data = plt.imread(images_path+name)
        tmp_data = rescale(data[b:b+d, a:a+c], (50/d,50/c))
        base[i//num*50:(i//num+1)*50, i%num*50:(i%num+1)*50] = tmp_data
#        plt.text(i%num*50+25, i//num*50+25, name, size=5, color = 'red')
        i += 1
#----------Display of name:------------
plt.imshow(base)
embed()
fig.canvas.callbacks.connect('pick_event', on_pick)
plt.show()

crop_fields = {}
for i in ids:
    plt.clf()
    _lefts_[i] = np.array(_lefts_[i]).min()
    _rights_[i] = np.array(_rights_[i]).max()
    _ups_[i] = np.array(_ups_[i]).min()
    _downs_[i] = np.array(_downs_[i]).max()
    crop_fields[i] = [int(np.array(_ups_[i]).min()), int(np.array(_downs_[i]).max()), int(np.array(_lefts_[i]).min()), int(np.array(_rights_[i]).max())]
    data = plt.imread(images_path+i)
    base = np.random.random(data.shape)
    base[crop_fields[i][0]:crop_fields[i][1],crop_fields[i][2]:crop_fields[i][3]] = data[crop_fields[i][0]:crop_fields[i][1],crop_fields[i][2]:crop_fields[i][3]]
    plt.imshow(base)
    plt.pause(0.001)

# abort, this is useless


# from bs4 import BeautifulSoup
import csv
import datetime
import json
import lxml
import os
import requests
import time

if not os.path.isdir('data/'):
    os.mkdir('data')

def get_city_links():
    url = 'https://www.puukauppa24.fi/puukaupat/'

    r = requests.get(url=url, headers=headers)
    soup = BeautifulSoup(r.text, 'lxml')

    city_content = soup.select('.kaikki-kaupungit .page_item a')
    city_name = [name.text for name in city_content]
    city_link = [link['href'] for link in city_content]

    with open('data/city_link.csv', 'w') as file:
        writer = csv.writer(file)
        writer.writerow(
                (
                    "City",
                    "link"
                )
            )

    for city, link in zip(city_name, city_link):
        with open('data/city_link.csv', 'a') as file:
            writer = csv.writer(file)
            writer.writerow(
                    (
                        city,
                        link
                    )
                )

def get_city_page():

    if not os.path.exists('data/city_pages'):
        os.mkdir('data/city_pages')

    total = 0
    with open('data/city_link.csv') as file:
        reader = csv.reader(file, delimiter=',')
        next(reader, None)
        for i in reader:
            total += 1

    with open('data/city_link.csv') as file:
        reader = csv.reader(file, delimiter=',')
        next(reader, None)

        count = 0
        data = []
        for city, url  in reader:
            r = requests.get(url=url, headers=headers)
            soup = BeautifulSoup(r.text, 'lxml')
            count += 1

            data.append(
                        {
                            'city_name': city,
                            'html': r.text
                        }
                    )

            with open(f'data/city_pages/{city}.html', 'w') as file:
                file.write(r.text)

            print(f'[INFO] City {count}/{total}')
            time.sleep(0.25)

        with open(f'data/city_pages.json', 'w') as file:
            json.dump(data, file, indent=4, ensure_ascii=False)

def get_shops():

    if not os.path.exists('data/shop_pages'):
        os.mkdir('data/shop_pages')

    # with open(f'data/city_shop_link.csv', 'w') as file:
    #     writer = csv.writer(file)
    #     writer.writerow(
    #             (
    #                 'City',
    #                 'Shop',
    #                 'Link'
    #             )
    #         )

    # with open('data/city_pages.json') as file:
    #     pages = json.load(file)

    # for item in pages:

    #     city_name = item['city_name']
    #     soup = BeautifulSoup(item['html'], 'lxml')

    #     shop_links = soup.select('.kaupunki a')
    #     city_shop_link = []
    #     for shop in shop_links:
    #         city_shop_link.append(
    #                 (
    #                     city_name,
    #                     shop.text,
    #                     'https://www.puukauppa24.fi' + shop['href']
    #                 )
    #             )

    #         with open(f'data/city_shop_link.csv', 'a') as file:
    #             writer = csv.writer(file)
    #             writer.writerow(
    #                     (
    #                         city_name,
    #                         shop.text,
    #                         'https://www.puukauppa24.fi' + shop['href']
    #                     )
    #                 )

    with open('data/city_shop_link.csv') as file:
        reader = csv.reader(file, delimiter=',')
        next(reader, None)
        total = 0
        for i in reader:
            total += 1

    with open('data/city_shop_link.csv') as file:
        reader = csv.reader(file, delimiter=',')
        next(reader, None)

        count = 0
        data = []
        for city, shop, url  in reader:
            r = requests.get(url=url, headers=headers)
            soup = BeautifulSoup(r.text, 'lxml')
            count += 1

            data.append(
                        {
                            'city_name': city,
                            'shop_name': shop,
                            'html': r.text
                        }
                    )

            with open(f'data/shop_pages/{count}.html', 'w') as file:
                file.write(r.text)

            print(f'[INFO] Shop {count}/{total}')
            time.sleep(0.25)
            # break

        with open(f'data/city_shop_pages.json', 'w') as file:
            json.dump(data, file, indent=4, ensure_ascii=False)

def get_shop_info():

    with open('data/final.csv', 'w') as file:
        writer = csv.writer(file)
        writer.writerow(
                (
                    "City",
                    "Shop",
                    "phone",
                    "email"
                )
            )

    files = os.listdir('data/shop_pages')

    data = []
    count = 0
    for file_name in files:
        with open(f'data/shop_pages/{file_name}') as f:
            soup = BeautifulSoup(f.read(), 'lxml')

        content = soup.select('.entry-content b')

        info = dict()
        for name in content:
            info[name.text.strip()[:-1]] = name.next_element.next_element.strip()

        data.append(info)
        count += 1
        if count % 50 == 0:
            print(f'Final step {count} / {len(files)}')

        # break
    with open('data/final.json', 'w') as file:
        json.dump(data, file, indent=4, ensure_ascii=False)

    for item in data:

        if 'Puukaupan nimi' in item:
            shop_name = item['Puukaupan nimi']
        else:
            phone = 'null'

        if 'Sähköpostiosoite' in item:
            email = item['Sähköpostiosoite']
        else:
            email = 'null'

        if 'Puhelinnumero' in item:
            phone = item['Puhelinnumero']

            if phone.startswith('0'):
                phone = '+358' + phone[1:]
        else:
            phone = 'null'

        if 'Osoite' in item:
            city_string = item['Osoite']
            city = city_string.split()[-1]
        else:
            city = 'null'

        with open('data/final.csv', 'a') as file:
            writer = csv.writer(file)
            writer.writerow(
                    (
                        city,
                        shop_name,
                        phone,
                        email
                    )
                )
        # break

if __name__ == '__main__':
    headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"
        }

    # get_city_links()
    # get_city_page()
    # get_shops()
    get_shop_info()


__author__ = "Nuno Lages"
__email__ = "lages@uthscsa.edu"

import cellprofiler.cpmodule as cpm
import cellprofiler.settings as cps
import cellprofiler.cpimage as cpi
import cellprofiler.objects as cpo
import scipy as sp
import scipy.stats as stats
import scipy.io as sio
from time import time

class TruncThresholdObjects(cpm.CPModule):

    variable_revision_number = 1
    module_name = "TruncThresholdObjects"
    category = "Image Processing"

    def create_settings(self):

        self.input_image_name = cps.ImageNameSubscriber(
            # The text to the left of the edit box
            "Input image name:",
            # HTML help that gets displayed when the user presses the
            # help button to the right of the edit box
            doc = """This is the image that the module operates on. You can
            choose any image that is made available by a prior module.

            ImageTemplate will do something to this image.
            """
        )

        self.output_image_name = cps.ImageNameProvider(
            "Output image name:",
            # The second parameter holds a suggested name for the image.
            "OutputImage",
            doc="""This is the image resulting from the operation."""
        )

        self.input_objects_name = cps.ObjectNameSubscriber(
            # The text to the left of the edit box
            "Input objects name:",
            # HTML help that gets displayed when the user presses the
            # help button to the right of the edit box
            doc = """This is the objects that the module operates on. You can
            choose any objects that is made available by a prior module.

            TruncThresholdObjects will do something to this objects.
            """
        )

        self.center = cps.Choice(
            "Center choice:",
            # The choice takes a list of possibilities. The first one
            # is the default - the one the user will typically choose.
            ['median', 'average'],
            doc="""Choose what to use as estimate of the mean of the
                truncated normal distribution."""
        )

        self.scale_r = cps.Float(
            "Truncated normal parameter red channel:",
            # The default value
            0.05,
            doc=""""""
        )

        self.scale_g = cps.Float(
            "Truncated normal parameter green channel:",
            # The default value
            0.05,
            doc=""""""
        )

        self.scale_b = cps.Float(
            "Truncated normal parameter blue channel:",
            # The default value
            0.05,
            doc=""""""
        )

        self.percentile_r = cps.Float(
            "Percentile red channel:",
            # The default value
            0.01,
            doc=""""""
        )

        self.percentile_g = cps.Float(
            "Percentile green channel:",
            # The default value
            0.01,
            doc=""""""
        )

        self.percentile_b = cps.Float(
            "Percentile blue channel:",
            # The default value
            0.0,
            doc=""""""
        )

    def settings(self):
        return [self.input_image_name,
                self.output_image_name,
                self.input_objects_name,
                self.center,
                self.scale_r,
                self.scale_g,
                self.scale_b,
                self.percentile_r,
                self.percentile_g,
                self.percentile_b]

    def run(self, workspace):

        t0 = time()

        diagnostics = dict()

        cent = self.center.get_value()

        input_objects_name = self.input_objects_name.value
        object_set = workspace.object_set
        assert isinstance(object_set, cpo.ObjectSet)

        input_image_name = self.input_image_name.value
        image_set = workspace.image_set
        assert isinstance(image_set, cpi.ImageSet)
        output_image_name = self.output_image_name.value

        input_image = image_set.get_image(input_image_name)# must_be_rgb=True)
        pixels = input_image.pixel_data
        diagnostics['pixels'] = pixels

        input_objects = object_set.get_objects(input_objects_name)

        mask = input_objects.get_segmented()

        new_im = sp.zeros(shape=pixels.shape)

        diagnostics['new_im'] = list()
        diagnostics['nucleus_processed'] = list()
        diagnostics['nucleus_pixels'] = list()
        diagnostics['ci'] = list()
        diagnostics['mu'] = list()
        diagnostics['sigma'] = list()
        diagnostics['sigma2'] = list()
        diagnostics['old_sigma'] = list()
        diagnostics['a'] = list()
        diagnostics['b'] = list()
        diagnostics['x1'] = list()
        diagnostics['x2'] = list()
        diagnostics['cx'] = list()
        diagnostics['yhat'] = list()

        diagnostics['time_first_part'] = time() - t0

        for x in range(1, mask.max()+1):

            t0 = time()

            nucleus_map = mask == x

            nucleus_pixels = \
                sp.multiply(pixels, nucleus_map[:, :, sp.newaxis] > 0)

            diagnostics['times_loop_' + str(x) + '_nditer'] = time() - t0
            t0 = time()

            diagnostics['nucleus_pixels'].append(nucleus_pixels)

            nucleus_pixels_t = sp.transpose(nucleus_pixels)

            # nucleus_ci_r = get_ci(nucleus_pixels_t[0],
            nucleus_ci_r, mu_r, sigma_r, a_r, b_r, old_sigma_r, x1_r, x2_r, \
            cx_r, \
            yhat_r, sigma2_r = get_ci(nucleus_pixels_t[0],
                                  percentile=self.percentile_r.get_value(),
                                  center=cent,
                                  mod=self.scale_r.get_value())

            # nucleus_ci_g = get_ci(nucleus_pixels_t[1],
            nucleus_ci_g, mu_g, sigma_g, a_g, b_g, old_sigma_g, x1_g, x2_g, \
            cx_g, \
            yhat_g, sigma2_g = get_ci(nucleus_pixels_t[1],
                                  percentile=self.percentile_g.get_value(),
                                  center=cent,
                                  mod=self.scale_g.get_value())

            # nucleus_ci_b = get_ci(nucleus_pixels_t[2],
            nucleus_ci_b, mu_b, sigma_b, a_b, b_b, old_sigma_b, x1_b, x2_b, \
            cx_b, \
            yhat_b, sigma2_b = get_ci(nucleus_pixels_t[2],
                                  percentile=self.percentile_b.get_value(),
                                  center=cent,
                                  mod=self.scale_b.get_value())

            diagnostics['times_loop_' + str(x) + '_ci'] = time() - t0
            t0 = time()

            diagnostics['ci'].append((nucleus_ci_r, nucleus_ci_g,
                                      nucleus_ci_b))
            diagnostics['mu'].append((mu_r, mu_g, mu_b))
            diagnostics['sigma'].append((sigma_r, sigma_g, sigma_b))
            diagnostics['sigma2'].append((sigma2_r, sigma2_g, sigma2_b))
            diagnostics['old_sigma'].append(
                (old_sigma_r, old_sigma_g, old_sigma_b))
            diagnostics['a'].append((a_r, a_g, a_b))
            diagnostics['b'].append((b_r, b_g, b_b))
            diagnostics['x1'].append((x1_r, x1_g, x1_b))
            diagnostics['x2'].append((x2_r, x2_g, x2_b))
            diagnostics['cx'].append((cx_r, cx_g, cx_b))
            diagnostics['yhat'].append((yhat_r, yhat_g, yhat_b))

            nucleus_processed = update_image(nucleus_pixels,
                                             nucleus_ci_r,
                                             nucleus_ci_g,
                                             nucleus_ci_b)

            diagnostics['times_loop_' + str(x) + '_update'] = time() - t0

            diagnostics['nucleus_processed'].append(nucleus_processed)

            new_im = new_im + nucleus_processed

            diagnostics['new_im'].append(new_im)

            from os.path import expanduser
            home = expanduser("~")

            sio.savemat(home + '/diagnostics.mat', diagnostics)

        output_image = cpi.Image(new_im, parent_image=input_image)
        image_set.add(output_image_name, output_image)

    def is_interactive(self):
        return False

def var_truncNormal(a, b, mu, sigma, data, mod=3000.0):

    # x1 = (a - mu)/sigma * stats.norm.pdf(a, mu, sigma)
    x1 = (mu - a)/sigma * stats.norm.pdf(a, mu, sigma)
    x2 = (b - mu)/sigma * stats.norm.pdf(b, mu, sigma)

    cx = stats.norm.cdf(b, mu, sigma) - stats.norm.cdf(a, mu, sigma)

    yhat = stats.tvar(data, limits=[mu-mod, mu+mod], inclusive=(False, False))
    sigma2 = yhat/((1+(x1-x2)/cx - ((x1-x2)/cx)**2))
    sigma = sp.sqrt(sigma2)

    return sigma, x1, x2, cx, yhat, sigma2

def update_image(original_im, ci_red, ci_green, ci_blue):

    ci_vec = sp.array((ci_red, ci_green, ci_blue))
    ci_matrix = sp.multiply(sp.ones(original_im.shape), ci_vec)
    new_im = sp.multiply(original_im, original_im > ci_matrix)

    return new_im

def get_ci(im_data, center='median', mod=3000.0, percentile=0.01):

    flattened = sp.concatenate(im_data)
    flattened = flattened[sp.nonzero(flattened)]

    if center == 'median':
        mu = sp.median(flattened)
    elif center == 'mean':
        mu = sp.average(flattened)

    old_sigma = stats.tstd(flattened)

    # mod = sp.float_(mod)
    sigma, x1, x2, cx, yhat, sigma2 = var_truncNormal(mu - mod, mu + mod, mu,
                                              old_sigma, flattened, mod=mod)

    ci = 2 * mu - stats.norm.ppf(percentile, mu, sigma)

    return ci, mu, sigma, mu-mod, mu+mod, old_sigma, x1, x2, cx, yhat, sigma2


import matplotlib.pyplot as plt
from datetime import datetime
import json
import torch
from spellchecker import SpellChecker
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import re
# import string
# from string import digits
import pandas as pd
import seaborn as sns
from collections import Counter

# stop_words = stopwords.words("english")

def new_dir():
    now = datetime.now()
    current_time = now.strftime("%d_%m_%Y_%H_%M_%S")

    nf = current_time + '/'
    return nf

def save_plot(data, name):
    plt.figure()
    plt.plot(data)
    plt.savefig(name)

def dump_hp(nf, hp):
    with open(nf + 'hp.json', 'w') as f:
        json.dump(hp, f)

def save_model(nf, model, model_name):
    torch.save(model.state_dict(), nf + model_name + '.pth')

def correct_spellings(x):
    """correct the missplled words of a given tweet"""
    spell = SpellChecker()
    x = x.split()
    misspelled = spell.unknown(x)
    result = map(lambda word: spell.correction(word) if word in misspelled else word, x)
    return " ".join(result)

def cleaning(x):
    """Apply function to a clean a tweet"""
    x = convert_to_lower(x)

    # remove punctuation
    x = remove_punctuation(x)
    # operator = str.maketrans('', '', string.punctuation)  # ????
    # x = x.translate(operator)
    # x = correct_spellings(x)
    x = word_tokenize(x)
    # x = remove_stopwords(x)
    return x

def convert_to_lower(text):
    # return the reviews after convering then to lowercase
    return text.lower()

def remove_punctuation(text):
    # return the reviews after removing punctuations
    text = re.sub(r"n\'t", " not", text)
    text = re.sub(r"\'re", " are", text)
    text = re.sub(r"\'s", " is", text)
    text = re.sub(r"\'d", " would", text)
    text = re.sub(r"\'ll", " will", text)
    text = re.sub(r"\'ve", " have", text)
    text = re.sub(r"\'m", " am", text)
    text = re.sub(r'[^\w\s]', ' ', text)
    text = re.sub(' +', ' ', text)
    return text

# Not Needed - last 2 lines of remove punctuation does this work
def remove_stopwords(text):
    # return the reviews after removing the stopwords
    exclude = ['$', '&', '+', ':', ';', '=', '@', '|', '<', '>', '^', '*', '%', '-', '#', '\'', '\n']
    exc = set(exclude)
    # remove_digits = str.maketrans('', '', digits)
    # text = text.translate(remove_digits)
    # return ''.join(ch for ch in text if ch not in (exclude))
    filtered_sentence = [w for w in text if w not in exc]
    return ' '.join(filtered_sentence)

def give_len(a):
    return a.size()[0]

def pad_data(data):
    # Find max length of the mini-batch
    # print('=' * 100)
    # print('=' * 100)
    # print(data)
    # print('=' * 100)
    # print(len(data))
    # print('=' * 100)
    if not len(data) <= 2:
        max_len = max(list(map(give_len, (list(zip(*data))[1]))))
        label_list = torch.tensor(list(zip(*data))[0])
        txt_list = list(zip(*data))[1]
        padded_tensors = torch.stack([torch.cat((txt, torch.tensor([0] * (max_len - give_len(txt))).long())) for txt in txt_list])
    else:
        padded_tensors = data[1]
        label_list = data[0]
    return padded_tensors, label_list

def clean_text(df):
    df['reviews'] = df['reviews'].apply(cleaning)
    return df

def get_class_distribution_loaders(obj, device, isSet=False):
    count_dict = Counter()

    if isSet:
        for (y, _) in obj:
            if len(y.shape) >= 1:
                count_dict.update(y.tolist())
            else:
                count_dict.update([y.item()])
        return count_dict

    for (_, y) in obj:
        if len(y.shape) >= 1:
            count_dict.update(y.tolist())
        else:
            count_dict.update([y.item()])
    return count_dict

def plot_data_distr(loaderslist, initdatalist, nf, device):
    train_count = get_class_distribution_loaders(initdatalist[0], device, isSet=True)
    train_count_new = get_class_distribution_loaders(loaderslist[0], device)
    # val_count = get_class_distribution_loaders(initdatalist[1], device, isSet=True)
    # val_count_new = get_class_distribution_loaders(loaderslist[1], device)

    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(18, 7))
    sns.barplot(x=list(train_count.keys()), y=list(train_count.values()), color="salmon", ax=axes[0]).set_title('Train before')
    axes[0].spines['top'].set_visible(False)
    axes[0].spines['right'].set_visible(False)
    axes[0].spines['bottom'].set_visible(False)
    axes[0].spines['left'].set_visible(False)

    sns.barplot(x=list(train_count_new.keys()), y=list(train_count_new.values()), color="salmon", ax=axes[1]).set_title('Train after')
    axes[1].spines['top'].set_visible(False)
    axes[1].spines['right'].set_visible(False)
    axes[1].spines['bottom'].set_visible(False)
    axes[1].spines['left'].set_visible(False)

    plt.savefig(nf + 'train_before_after.png')

    # fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(18, 7))
    # sns.barplot(x=list(val_count.keys()), y=list(val_count.values()), color="salmon", ax=axes[0]).set_title('Val before')
    # axes[0].spines['top'].set_visible(False)
    # axes[0].spines['right'].set_visible(False)
    # axes[0].spines['bottom'].set_visible(False)
    # axes[0].spines['left'].set_visible(False)

    # sns.barplot(x=list(val_count_new.keys()), y=list(val_count_new.values()), color="salmon", ax=axes[1]).set_title('Val after')
    # axes[1].spines['top'].set_visible(False)
    # axes[1].spines['right'].set_visible(False)
    # axes[1].spines['bottom'].set_visible(False)
    # axes[1].spines['left'].set_visible(False)
    # plt.savefig(nf + 'val_before_after.png')


from game_classes import Dragon, Knight, Wizard
import time
import random
import re
import sys
from termcolor import colored
import os

os.system('color')

def main():
    print(colored (f"\nWELCOME TO THE DRAGONSLAYER GAME!\n", 'red'))
    while(True):
        play = input(colored("\nTo start the game type: PLAY otherwise QUIT\n\n", 'green')).title()
        start = st_game(play)
        if start == True:
            break
        else:
            continue
    name = input(colored ("\nFirst of all choose your adventurer name (only letters, max 25 chr and two words)\n\n", 'green'))
    while(True):

        allowed_name = check_name(name)
        if allowed_name == True:
            break
        else:
            print("\nOnly letters, max 25 chr and two words are allowed\n")
            name = input()
            continue
    black_dragon = Dragon()
    print(colored ("\nSecond choose your class: Knight or Wizard?\n", 'green'))
    while(True):
        classe = input().title()
        if classe == "Knight" or classe == "Wizard":
            user = class_choise(black_dragon, name, classe)
            break
        else:
            print(colored("\nWrong selection, choose: Knight or Wizard\n",'green'))

    scene1(black_dragon, user)

# Check if input name respect the restriction about length and type of characters allowed

def st_game(play):
    if play == "Play":
        return True
    elif play == "Quit":
        sys.exit("\n")
    else:
        return False

def check_name(name):
        allowed_name = re.search(r"^[a-zA-Z]+\s?[a-zA-Z]+$", name)
        if allowed_name and len(name) <= 25:
            return True
        else:
            return False

# Creation of the instance containing the selected Class

def class_choise(black_dragon, name, classe):

    while (True):
        if classe == "Knight":
            user = Knight(black_dragon,name)
            return user
        elif classe == "Wizard":
            user = Wizard(black_dragon,name)
            return user
        else:
            return False

# First scene of the game with two possible outcome

def scene1(black_dragon, user):
    print(colored("""

Let's start the adventure!

After a long journey through the Vingaard Mountains and the Guardian Swamp you finally arrive in front of the entrance to the Black Dragon Cave.
The stench of death that comes out of the cave makes you turn up your nose and a deep sense of anguish begins to oppress your chest.

""", 'cyan'))
    print(colored ("Type your choice: Enter or Leave?\n", 'green'))
    c1 = input().lower()
    time.sleep(1)
    ans = 'incorrect'
    while(ans=='incorrect'):
        if(c1 == "enter"):
            print(colored ("\nYou begin to descend into the dark cave, courageously overcoming the fear of the dragon", 'cyan'))
            ans = 'correct'
            scene2(black_dragon, user)
        elif(c1 == "leave"):
            print(colored("\nThe fear of the Dragon invades your heart, you turn around and run away deciding to change jobs and start dedicating yourself to less dangerous adventures (although not easier) such as learning to program!", 'cyan'))
            ans='correct'
        else:
            print(colored ("TYPE THE CORRECT CHOICE! Enter or Leave?"), 'green')
            c1 = input().lower()

# Second scene of the game where it's possible pick up an item usuful for the fight, but depends on which class has been selected

def scene2(black_dragon, user):
    print(colored("""

After hours of wandering in the meanders of the cave, you find a corpse of an adventurer on the ground, stuck in a trapdoor.

Looking carefully you notice three interesting objects on the body, a Spear, a Potion and a Gem. You can choose whether to take one or leave.
""", 'cyan'))
    print(colored ("\nType your choice: Spear,  Potion, Gem or Leave?\n", 'green'))
    loot_item = { "spear" : "Dragonlance", "potion" : "Unstable Potion", "gem": "Shining Gem"}
    c1 = input().lower()
    time.sleep(1)
    ans = 'incorrect'
    while(ans=='incorrect'):
        if(c1 in loot_item):
            user.items.append(loot_item[c1])
            print(colored(f"\nYou carefully pick up the {c1} from the body and step back just in time before the trap door reopened totally swallowing the adventurer with rest of his equipment\n",'cyan'))
            ans = 'correct'
            scene3(black_dragon, user)
        elif(c1 =="leave"):
            print(colored("\nYou decide not to risk your luck with the mortal trap and pass over\n",'cyan'))
            ans='correct'
            scene3(black_dragon, user)
        else:
            print(colored("\nTYPE A VALID CHOICE: Spear,  Potion, Gem or Leave?\n",'green'))
            c1 = input().lower()

# Third and final scene that implement the combat system promting the player for actions and automatically exert the enemy actions

def scene3(black_dragon, user):
    print(colored("""

You finally arrive at end of the undergound labyrinth and a huge cave is opening in front of you. At the center of the cave there is a mountain of shining gold

and all kind of treasure. On top of this there is the biggest Black Dragon you ever seen sleeping.

You start to get close when one of his eye opened, and showing what seems to you a evil reptilian grin he gives you the welcome

Black Dragon: "You little mouse, you made me wait a long time. I have smelled the stench of your kind since before you entered the cave.

But finally my dinner is arrived! "

""", 'cyan'))
    print(colored("FIGHT!\n", 'green'))
    time.sleep(1)
    combat = "started"
    print("\nYou can attack, use an item or check your HP\n")

    #combat full turn with reset of shield function and check of poison status and poison damage
    while (combat =="started"):
        user.shield = False
        if user.turns_poison > 0:
            user.poison_dmg += 5
            user.hp -= 5
            user.turns_poison -= 1
            print("\nYou receive -5 hp damage from poison\n")
        else:
            user.status = "Healthy"
        turn = "ongoing"

        #user turn, where is possible to choose among check healt point or items/attacks list without loosing the turn or select one of the attacks/items available
        while (turn == "ongoing"):
            action = ""
            print(colored("\nTo consult the list of attacks or items available type: ATTACKS LIST or ITEMS LIST, while for use one of them just type its name. To check your current health points type: HP\n",'yellow'))
            action = input().title()
            if action == "Attacks List":
                attacks_list = []
                print("\nAttacks available:\n")
                for key, value in user.attacks.items():
                    attacks_list.append(key)
                print(colored(f"{attacks_list}",'magenta'))

            elif action == "Items List":
                print(colored(f"{user.items}",'magenta'))

            elif action in user.attacks:

                doSomething = getattr(user, user.attacks[action])
                doSomething(black_dragon)
                print(black_dragon.hp)
                turn = "end"

            elif action == "Hp":
                print(colored(f"\nYour HP: {user.hp}\n",'blue'))

            elif action in user.items:
                if action == "Dragonlance" or action == "Shining Gem":
                    user.item_use(action)
                else:
                    user.item_use(action)
                    turn = "end"
            else:
                print(colored("\nWrong command! Try again\n", 'green'))

        if black_dragon.hp <= 0:
            print(colored("\nWith your last blow you kill the Dragon gaining fortune and glory! Well done adventurer!\n",'cyan'))
            # combat = "ended"
            break
        elif black_dragon.hp < 100:
            print(colored("\nThe Black Dragon is gravely wounded\n",'red'))
        elif black_dragon.hp < 200:
            print(colored("\nThe Black Dragon is moderate wounded\n", 'red'))
            time.sleep(1)
        # Check of dragon action if will do a normal attack or prepare for power attack
        dragon_attack = random.randint(1, 100)
        if dragon_attack > 85 and black_dragon.power_attack == False:
            black_dragon.power_attack = True
            print(colored("\nThe Dragon holds his breath and prepare to attack\n", 'green'))
        else:
            if black_dragon.power_attack == True:
                black_dragon.breath(user)
                black_dragon.power_attack = False
            else:
                black_dragon.bite(user)
        if user.hp <= 0:
            print(colored("\nYOU DIED!\n", 'red'))
            combat = "ended"

if __name__ == '__main__':
    main()


from flask import Flask
app = Flask(__name__)
app.config['DEBUG'] = True

import json
from models import Event
from util import utc_to_central, time_to_string

@app.route('/load')
def load():
    with open('cal.json') as f:
        events = json.load(f)
    for data in events:
        if 'RRULE' in data:  # Recurring event
            continue
        start = utc_to_central(data['DTSTART'])
        end = utc_to_central(data['DTEND'])
        date = "%d%02d%02d" % (start.year, start.month, start.day)
        start_time = time_to_string(start)
        end_time = time_to_string(end)

        event = Event(
            summary = data['SUMMARY'].strip(),
            description = data['DESCRIPTION'].strip(),
            date = date,
            start = start_time,
            end = end_time,
            location = data['LOCATION'].strip(),
            uid = data['UID'])
        event.put()
    return 'OK'


"""Unit test for emlib.py

To run:
    PYTHONPATH=../src python test_emlib.py
"""

__author__ = "Mike Pekala"
__copyright__ = "Copyright 2015, JHU/APL"
__license__ = "Apache 2.0"

import unittest
import numpy as np
from sklearn.metrics import precision_recall_fscore_support as smetrics

import emlib

class TestEmlib(unittest.TestCase):
    def test_metrics(self):
        Y = np.random.randint(0,2,size=(2,5,5))
        Yhat = np.random.randint(0,2,size=(2,5,5))

        C,acc,prec,recall,f1 = emlib.metrics(Y, Yhat, display=False)
        prec2, recall2, f12, supp = smetrics(np.reshape(Y, (Y.size,)),
                np.reshape(Yhat, (Yhat.size,)))

        self.assertAlmostEqual(prec, prec2[1])
        self.assertAlmostEqual(recall, recall2[1])
        self.assertAlmostEqual(f1, f12[1])

    def test_mirror_edges(self):
        X = np.random.rand(10,3,3);
        b = 2  # b := border size
        Xm = emlib.mirror_edges(X,b)

        # make sure the result has the proper size
        assert(Xm.shape[0] == X.shape[0]);
        assert(Xm.shape[1] == X.shape[1]+2*b);
        assert(Xm.shape[2] == X.shape[2]+2*b);

        # make sure the data looks reasonable
        self.assertTrue(np.all(Xm[:,:,b-1] == Xm[:,:,b]))
        self.assertTrue(np.all(Xm[:, b:-b, b:-b] == X))

    def test_interior_pixel_generator(self):
        b = 10  # b := border size
        Z = np.zeros((2,100,100), dtype=np.int32)
        for idx, pct  in emlib.interior_pixel_generator(Z,b,30):
            Z[idx[:,0],idx[:,1],idx[:,2]] += 1

        self.assertTrue(np.all(Z[:,b:-b,b:-b]==1))
        Z[:,b:-b,b:-b] = 0
        self.assertTrue(np.all(Z==0))

    def test_stratified_interior_pixel_generator(self):
        b = 10  # b := border size

        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # For a 50/50 split of pixels in the interior, the generator
        # should reproduce the entire interior.
        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        Y = np.zeros((2,100,100))
        Y[:,0:50,:] = 1

        Z = np.zeros(Y.shape)
        for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30):
            Z[idx[:,0],idx[:,1],idx[:,2]] += 1

        self.assertTrue(np.all(Z[:,b:-b,b:-b]==1))
        Z[:,b:-b,b:-b] = 0
        self.assertTrue(np.all(Z==0))

        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # For a random input, should see a 50/50 split of class
        # labels, but not necessarily hit the entire interior.
        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        Y = np.random.rand(2,100,100) > 0.5
        nOne=0; nZero=0;
        for idx,pct in emlib.stratified_interior_pixel_generator(Y,b,30):
            slices = idx[:,0];  rows = idx[:,1];  cols = idx[:,2]
            nOne += np.sum(Y[slices,rows,cols] == 1)
            nZero += np.sum(Y[slices,rows,cols] == 0)
        self.assertTrue(nOne == nZero)

        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        # For an input tensor with "no-ops", the sampler should only
        # return pixels with a positive or negative label.
        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        Y = np.zeros((2,100,100))
        Y[:,0:20,0:20] = 1
        Y[:,50:70,50:70] = -1
        Z = np.zeros(Y.shape)
        nPos=0; nNeg=0; nTotal=0;
        for idx,pct in emlib.stratified_interior_pixel_generator(Y,0,10,omitLabels=[0]):
            slices = idx[:,0];  rows = idx[:,1];  cols = idx[:,2]
            Z[slices,rows,cols] = Y[slices,rows,cols]
            nPos += np.sum(Y[slices,rows,cols] == 1)
            nNeg += np.sum(Y[slices,rows,cols] == -1)
            nTotal += len(slices)

        self.assertTrue(nPos == 20*20*2);
        self.assertTrue(nNeg == 20*20*2);
        self.assertTrue(nTotal == 20*20*2*2);
        self.assertTrue(np.all(Y == Z))

if __name__ == "__main__":
    unittest.main()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4


import os

from setuptools import setup

def read(f):
    return open(os.path.join(os.path.dirname(__file__), f)).read()

def readlines(f):
    return open(os.path.join(os.path.dirname(__file__), f)).readlines()

setup(
    name="python-udp-handler",
    version="0.1",
    include_package_data=True,
    author="Rawand Hawiz",
    author_email="rhawiz@geophy.com",
    description="UDP Handler for logging to logstash.",
    license="BSD",
    keywords="python udp handler logging",
    url="https://github.com/rhawiz/python-udp-handler",
    packages=['logmodule'],
    long_description=read('README.md'),
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Topic :: Utilities",
        "License :: OSI Approved :: BSD License",
    ],
    install_requires=[
    ],
    test_suite='tests'

)


#!/usr/bin/env python3

# Average: O(log(n)) time | O(1) space
# Worst: O(n) time | O(1) space

"""
iterative method:
Same reason as in Sol_1.py. However, this time we don't create stacks for each steps. Therefore no increasing in memory. That means we have O(1) space complexity.
"""

def findClosestValueInBst(tree, target):
    # Write your code here.
    # Tip: Try define another helper function
    #      Try to write it using recursive and iterative method.
    return findClosestValueInBstHelper(tree, target, tree.value)

def findClosestValueInBstHelper(tree, target, closest):
    currentNode = tree
    while currentNode is not None:
        if abs(target - closest) > abs(target - currentNode.value):
            closest = currentNode.value
        if target < currentNode.value:
            currentNode = currentNode.left
        elif target > currentNode.value:
            currentNode = currentNode.right
        else:
            break
    return closest

# Testing
if __name__ == '__main__':
    import json
    import debug_v1
    import tree_class as tcls

    with open('test.json', 'r') as t:
        load_test = json.load(t)
        test_case = load_test['test']
        correct_output = load_test['answer']

    for count, case in enumerate(test_case):
        print(f'Test Case {count+1}:', end=' ')

        for each in case['tree']['nodes']:
            if case['tree']['root'] == each['id']:
                tree  = tcls.treeNode(each['id'], each['value'])
        tree.build_tree(case['tree']['nodes'])

        value = findClosestValueInBst(tree, case['target'])
        debug_v1.debug(case, value, correct_output[count])