import json

from common.local import get_request_username
from common.log import logger
from datahub.common.const import (
    CLUSTER_NAME,
    GRANTED,
    KAFKA,
    PRODUCER,
    REVOKED,
    REVOKING,
)
from datahub.databus.api import StoreKitApi
from datahub.databus.exceptions import (
    AccessResourceNotFound,
    NotFoundError,
    StoreKitApiError,
    TaskInsertDBErr,
)
from datahub.databus.models import (
    AccessRawData,
    AccessResourceInfo,
    DatabusChannel,
    DatabusClean,
    DatabusCluster,
    DataBusClusterRouteConfig,
    DatabusConfig,
    DatabusConnectorTask,
    DatabusOperationLog,
    DatabusShipper,
    DataBusTaskStatus,
    DataStorageConfig,
    QueueChangeEvent,
    QueueClusterConfig,
    QueueConsumerConfig,
    QueueProducerConfig,
)
from datahub.databus.settings import default_time_format, log_item_format
from django.db.models import Q
from django.forms import model_to_dict
from django.utils.translation import ugettext_lazy as _

from . import settings

def get_clean_by_processing_id(processing_id):
    """
    根据processing id获取清洗的配置对象
    :param processing_id: 清洗的processing_id，和rt_id值相同
    :return: 清洗配置对象，或者None值
    """
    try:
        return DatabusClean.objects.get(processing_id=processing_id)
    except DatabusClean.DoesNotExist:
        logger.warning("failed to find databus clean by processing id %s" % processing_id)
        return None

def get_clean_list_by_raw_data_id(raw_data_id):
    """
    根据rawdataid获取对应的清洗任务列表
    :param raw_data_id: rawdataid
    :return: 对应的清洗任务列表
    """
    return DatabusClean.objects.filter(raw_data_id=raw_data_id).values()

def get_raw_data_by_id(raw_data_id, ignore_empty=True):
    """
    根据raw data的id获取rawdata对象
    :param raw_data_id: 数据接入的raw data的id
    :param ignore_empty: 忽略DoesNotExist异常
    :return: rawdata对象，或者None值
    """
    try:
        return AccessRawData.objects.get(id=raw_data_id)
    except AccessRawData.DoesNotExist:
        logger.warning("failed to find raw data by id %s" % raw_data_id)
        if ignore_empty:
            return None
        raise NotFoundError(message_kv={"type": "raw_data_id", "key": raw_data_id})

def get_raw_data_resource_by_id(raw_data_id):
    """
    根据raw data的id获取rawdata对象资源信息
    :param raw_data_id: 数据接入的raw data的id
    :return: rawdata对象资源信息
    """
    res = AccessResourceInfo.objects.filter(raw_data_id=raw_data_id)
    if not res:
        raise AccessResourceNotFound(message_kv={"data_id": raw_data_id})
    return res

def get_channel_by_name(channel_name):
    """
    根据channel名称获取channel的配置信息
    :param channel_name: channel的名称
    :return: channel配置信息，或者None
    """
    try:
        return DatabusChannel.objects.get(cluster_name=channel_name)
    except DatabusChannel.DoesNotExist:
        logger.warning("failed to find databus channel by name %s" % channel_name)
        return None

def get_channel_by_id(channel_id, ignore_empty=True):
    """
    根据channel id获取channel的配置信息
    :param channel_id: channel的id
    :param ignore_empty: 忽略DoesNotExist异常
    :return: channel配置信息，或者None
    """
    try:
        return DatabusChannel.objects.get(id=channel_id)
    except DatabusChannel.DoesNotExist:
        logger.warning("failed to find databus channel by id %s" % channel_id)
        if ignore_empty:
            return None
        raise NotFoundError(message_kv={"type": "channel_id", "key": channel_id})

def get_channel_by_stream_to_id(stream_to_id, ignore_empty=True):
    """
    根据channel id获取channel的配置信息
    :param channel_id: channel的id
    :param ignore_empty: 忽略DoesNotExist异常
    :return: channel配置信息，或者None
    """
    try:
        return DatabusChannel.objects.get(stream_to_id=stream_to_id)
    except DatabusChannel.DoesNotExist:
        logger.warning("failed to find databus channel by stream_to_id %s" % stream_to_id)
        if ignore_empty:
            return None
        raise NotFoundError(message_kv={"type": "stream_to_id", "key": stream_to_id})

def get_cluster_by_name(cluster_name, ignore_empty=True):
    """
    根据cluster名称获取cluster的配置信息
    :param cluster_name: 集群名
    :param ignore_empty: 忽略DoesNotExist异常
    :return: cluster配置信息，或者None
    """
    try:
        return DatabusCluster.objects.get(cluster_name=cluster_name)
    except DatabusCluster.DoesNotExist:
        logger.warning("failed to find databus cluster by name %s" % cluster_name)
        if ignore_empty:
            return None
        raise NotFoundError(message_kv={"type": "cluster", "key": cluster_name})

def get_connector_route(connector_name, ignore_empty=True):
    """
    根据总线任务路由表找总线任务所在集群信息
    :param connector_name: 总线任务名称
    :param ignore_empty: 忽略DoesNotExist异常
    :return: 任务路由信息列表
    """
    try:
        return DatabusConnectorTask.objects.get(connector_task_name=connector_name)
    except DatabusConnectorTask.DoesNotExist:
        logger.warning("failed to find route info for connector %s" % connector_name)
        if ignore_empty:
            return None
        raise NotFoundError(message_kv={"type": "connector_name", "key": connector_name})

def get_connector_route_by_task_type(processing_id, task_type):
    """
    根据总线任务路由表找总线任务所在集群信息
    :param processing_id: 处理节点id（rt）
    :param task_type: （shipper、clean、puller）
    :return: 任务路由信息列表
    """
    return DatabusConnectorTask.objects.filter(processing_id=processing_id, task_type=task_type)

def delete_connector_route(connector_name):
    """
    删除总线任务路由表
    :param connector_name: 总线任务名称
    """
    logger.info("delete %s from databus_connector_task" % connector_name)
    DatabusConnectorTask.objects.filter(connector_task_name=connector_name).delete()

def update_connector_cluster(route_id, dest_cluster):
    """
    更新总线任务所在集群信息
    :param route_id: 总线任务路由表中的id
    :param dest_cluster: 目的地总线集群
    """
    DatabusConnectorTask.objects.filter(id__exact=route_id).update(cluster_name=dest_cluster)

def add_databus_oplog(operation_type, connector, cluster, args, response, created_by=""):
    """
    databus操作日志记录
    :param operation_type: 操作类型
    :param connector: connector名称
    :param cluster: 集群名称
    :param args: 操作参数
    :param created_by: 操作人
    :param response: 操作结果
    """
    DatabusOperationLog.objects.create(
        operation_type=operation_type,
        item=connector,
        target=cluster,
        request=args,
        response=response,
        created_by=created_by,
    )

def update_databus_oplog_by_item(item, status, args):
    """
    databus操作日志记录
    :param item: 名称
    :param status : 状态
    :param args: 操作参数
    """
    DatabusOperationLog.objects.filter(item=item).update(request=status, response=args)

def get_databus_oplog_by_item(item):
    """
    databus操作日志记录
    :param item: 日志项
    :param response: 操作结果
    """
    return DatabusOperationLog.objects.filter(item=item).order_by("-id")

def delete_databus_oplog_by_item(item):
    """
    删除databus操作日志记录
    :param item: 日志项
    """
    DatabusOperationLog.objects.filter(item=item).delete()

OPERATION_TYPE_RAW_DATA = "raw_data"
OPERATION_TYPE_CLEAN = "clean"
OPERATION_TYPE_STORAGE = "storage"

def log_delete_event(operation_type, result_table):
    """
    记录删除操作
    :param operation_type: raw_data/clean/storage
    :param result_table: result_table
    """
    DatabusOperationLog.objects.create(
        operation_type=operation_type,
        item=result_table,
        target="delete",
        request=result_table,
        response="ok",
        created_by=get_request_username(),
    )

def trans_databus_oplog(oplogs, params):
    """
    转换databus操作日志记录
    :param oplogs: 日志记录项
    :param params: 转换参数
    :param response: 操作结果
    """
    if oplogs:
        result = oplogs.values()
        for oplog in result:
            oplog["created_at"] = (
                oplog.get("created_at").strftime(default_time_format) if oplog.get("created_at") else ""
            )
            oplog["updated_at"] = (
                oplog.get("updated_at").strftime(default_time_format) if oplog.get("updated_at") else ""
            )
            oplog["data_type"] = params["data_type"]
            oplog["data_type_alias"] = _(params["data_type"])
            oplog["storage_type"] = params["storage_type"]
            oplog["operation_type_alias"] = _(oplog["operation_type"])
            try:
                if oplog.get("response"):
                    response = json.loads(oplog.get("response"))
                    if response.get("error_msg"):
                        oplog["log"] = (u"{}:{}".format(_(response.get("content")), response.get("error_msg")),)
                    else:
                        oplog["log"] = u"%s" % _(response.get("content"))
                else:
                    oplog["log"] = ""
            except Exception as e:
                logger.warning(u"load response happend exception:%s" % str(e))
                oplog["log"] = oplog.get("response", "")

        return result
    else:
        return None

def get_storage_config_by_data_id(raw_data_id):
    """
    根据原始数据id获取存储配置列表
    :param raw_data_id:原始数据id
    :return: 存储配置列表
    """
    return DataStorageConfig.objects.filter(raw_data_id=raw_data_id)

def get_storage_config_by_data_type(result_table_id):
    """
    根据rt_id获取存储配置列表
    :param result_table_id:rtid
    :return: response:存储配置列表
    """
    return DataStorageConfig.objects.filter(result_table_id=result_table_id, data_type="raw_data")

def get_storage_config_by_data_type_rt(result_table_id, data_type):
    """
    根据rt_id获取存储配置列表
    :param result_table_id:rtid
    :param data_type:(clean/raw_data)
    :return: response:存储配置列表
    """
    return DataStorageConfig.objects.filter(result_table_id=result_table_id, data_type=data_type)

def get_storage_config_by_cond(raw_data_id, result_table_id, storage_type):
    """
    根据唯一条件查询存储配置
    :param raw_data_id: 原始数据id
    :param result_table_id: 结果表id
    :param storage_type: 存储类型
    :return: 存储配置
    """
    try:
        return DataStorageConfig.objects.get(
            raw_data_id=raw_data_id,
            result_table_id=result_table_id,
            cluster_type=storage_type,
        )
    except DataStorageConfig.DoesNotExist:
        logger.warning(
            "failed to find DataStorageConfig info for raw_data_id:%s,result_table_id:%s,cluster_type:%s"
            % (raw_data_id, result_table_id, storage_type)
        )
        return None

def delete_storage_config_by_cond(data_type, storage_type, result_table_id):
    """
    根据唯一条件删除存储配置
    :param data_type: 数据类型
    :param storage_type: 存储类型
    :param result_table_id: 结果表id
    """
    DataStorageConfig.objects.filter(
        data_type=data_type, result_table_id=result_table_id, cluster_type=storage_type
    ).delete()

def delete_storage_config_by_raw_data(data_type, storage_type, raw_data_id):
    """
    根据唯一条件删除存储配置
    :param data_type: 数据类型
    :param storage_type: 存储类型
    :param raw_data_id: 数据源id
    """
    DataStorageConfig.objects.filter(data_type=data_type, raw_data_id=raw_data_id, cluster_type=storage_type).delete()

def create_storage_config(params):
    """
    创建数据入库配置
    :param params: 存储配置参数
    :return: response
    """
    DataStorageConfig.objects.create(
        raw_data_id=params.get("raw_data_id", ""),
        data_type=params["data_type"],
        result_table_id=params.get("result_table_id", ""),
        cluster_type=params["storage_type"],
        cluster_name=params["storage_cluster"],
        expires=params.get("expires", ""),
        created_by=params["bk_username"],
        updated_by=params["bk_username"],
        status=DataBusTaskStatus.STARTED,
    )

def update_storage_config(params):
    """
    更新数据入库配置
    :param params: 存储配置参数
    :return: response
    """
    DataStorageConfig.objects.filter(
        raw_data_id=params["raw_data_id"],
        result_table_id=params["result_table_id"],
        cluster_type=params["cluster_type"],
    ).update(**params)

def get_clean_by_raw_data_id(raw_data_id):
    """
    根据raw_data_id获取清洗的配置对象
    :param raw_data_id: 原始数据id
    :return: 清洗配置对象
    """

    return DatabusClean.objects.filter(raw_data_id=raw_data_id)

def get_shipper_by_name(connector_task_name):
    """
    根据connector_task_name获取分发的配置对象
    :param connector_task_name: 任务名称
    :return: 分发的配置对象
    """
    try:
        return DatabusShipper.objects.get(connector_task_name=connector_task_name)
    except DatabusShipper.DoesNotExist:
        logger.warning("failed to find DatabusShipper by connector_task_name %s" % connector_task_name)
        return None

def delete_shipper_by_name(connector_name):
    """
    删除shipper任务记录
    :param connector_name: connector名称
    """
    logger.info("delete %s from databus_shipper_info" % connector_name)
    DatabusShipper.objects.filter(connector_task_name=connector_name).delete()

def get_cluster_route_config(dimension_name, dimension_type, cluster_type):
    """
    :param dimension_name: 纬度表name标识
    :param dimension_type: 纬度类型
    :param cluster_type: 集群类型
    :return:
    """
    try:
        return DataBusClusterRouteConfig.objects.get(
            dimension_name=dimension_name,
            dimension_type=dimension_type,
            cluster_type=cluster_type,
        )
    except DataBusClusterRouteConfig.DoesNotExist:
        logger.warning(
            "failed to find DataBusClusterRouteConfig by dimension {} {}".format(dimension_type, dimension_name)
        )
        return None

def create_or_update_cluster_route(dimension_name, dimension_type, cluster_name, cluster_type):
    """
    :param dimension_name: 纬度表name标识
    :param dimension_type: 纬度类型
    :param cluster_type: 集群类型
    :param cluster_name: 集群名称
    :return:
    """
    # 查询如果存在，则更新
    route_config = get_cluster_route_config(dimension_name, dimension_type, cluster_type)
    if not route_config:
        DataBusClusterRouteConfig.objects.create(
            dimension_name=dimension_name,
            dimension_type=dimension_type,
            cluster_name=cluster_name,
            cluster_type=cluster_type,
            created_by=get_request_username(),
            updated_by=get_request_username(),
        )
    else:
        update_param = {CLUSTER_NAME: cluster_name}
        DataBusClusterRouteConfig.objects.filter(id=route_config.id).update(**update_param)

def delete_cluster_route_by_condition(dimension_name, dimension_type, cluster_type):
    """
    :param dimension_name: 纬度表name标识
    :param dimension_type: 纬度类型
    :param cluster_type: 集群类型
    """
    route_config = get_cluster_route_config(dimension_name, dimension_type, cluster_type)
    if route_config:
        route_config.delete()
    logger.info(
        "{}: has been delete, dimension_type: {}, cluster_type: {}".format(dimension_name, dimension_type, cluster_type)
    )

def get_inner_channel_by_priority(cluster_type):
    """
    查询inner kafka集群
    :param cluster_type: cluster type. kafka/pulsar
    :return channel info
    """
    return DatabusChannel.objects.filter(cluster_role="inner", cluster_type=cluster_type).order_by("-priority")

def get_inner_channel_by_name(cluster_name):
    """
    cluster_name: 根据名称查询channel
    :return: channel
    """
    try:
        return DatabusChannel.objects.get(cluster_name=cluster_name)
    except DatabusChannel.DoesNotExist:
        logger.warning("failed to find DatabusChannel by cluster_name: %s" % cluster_name)
        return None

def get_databus_storage_oplog_by_item(storage_type, result_table_id):
    """
    databus操作日志记录
    :param item: 日志项
    :param response: 操作结果
    """
    if storage_type == "es":
        item_es = log_item_format % ("es", result_table_id)
        item_eslog = log_item_format % ("eslog", result_table_id)
        return DatabusOperationLog.objects.filter(Q(item=item_es) | Q(item=item_eslog)).order_by("-id")
    else:
        return DatabusOperationLog.objects.filter(item=log_item_format % (storage_type, result_table_id)).order_by(
            "-id"
        )

def delete_storekit_result_table_kafka(result_table):
    """
    删除rt的存储关系, 忽略不存在报错
    :param result_table: result table
    """
    res = StoreKitApi.result_tables.delete({"result_table_id": result_table, "cluster_type": "kafka"})
    if not res.is_success() and res.code != "1500004":  # 1500004:不存在异常
        raise StoreKitApiError(message_kv={"message": "%s" % res.errors()})

def add_databus_task(
    rt_id,
    connector_name,
    cluster_name,
    source,
    source_type,
    dest,
    dest_type,
    task_type=settings.MODULE_SHIPPER,
):
    """
    任务写入db
    :param rt_id: result_table_id
    :param connector_name: connector名称
    :param cluster_name: 运行集群
    :param source: 来源信息
    :param source_type: 来源类型
    :param dest: 目标信息
    :param dest_type: 目标类型
    :param task_type: 集群类型
    """
    try:
        DatabusConnectorTask.objects.create(
            connector_task_name=connector_name,
            task_type=task_type,
            processing_id=rt_id,
            cluster_name=cluster_name,
            status="running",
            data_source=source,
            source_type=source_type,
            data_sink=dest,
            sink_type=dest_type,
            created_by=get_request_username(),
            updated_by=get_request_username(),
        )
    except Exception as e:
        logger.error(
            "failed to add databus task. %s %s %s %s %s %s. Exception: %s"
            % (connector_name, cluster_name, source, source_type, dest, dest_type, e)
        )
        raise TaskInsertDBErr()

def get_databus_task_info(connector_name):
    """
    根据connectors名称获取任务记录
    :param connector_name: connector名称
    :return: {list} db中对应的完整记录列表
    """
    return [
        model_to_dict(row) for row in DatabusConnectorTask.objects.filter(connector_task_name__exact=connector_name)
    ]

def get_queue_consumer_conf_by_cluster(group_id, topic, user, cluster_id):
    """
    根据groupid ,topic, appcode 获取已经授权的记录
    :param group_id: group id
    :param topic: 需要授权的topic
    :param user: 用户名
    :param cluster_id: queue队列集群id
    :return: {list} db中对应的完整记录列表
    """
    try:
        return QueueConsumerConfig.objects.get(
            group_id=group_id,
            topic=topic,
            user_name=user,
            status=GRANTED,
            cluster_id=cluster_id,
        )
    except QueueConsumerConfig.DoesNotExist:
        logger.warning("failed to find queue_consumer_config by cluster %s" % cluster_id)
        return None

def get_queue_consumer_config_by_topic(group_id, topic, user):
    """
    根据groupid ,topic, appcode 获取已经授权的记录
    :param group_id: group id
    :param topic: 需要授权的topic
    :param user: 用户名
    :return: {list} db中对应的完整记录列表
    """

    try:
        return QueueConsumerConfig.objects.get(group_id=group_id, topic=topic, user_name=user, status=GRANTED)
    except QueueConsumerConfig.DoesNotExist:
        logger.warning("failed to find queue_consumer_config by topic %s" % topic)
        return None

def get_queue_consumer_by_cluster_type(group_id, topic, user, cluster_type):
    """
    根据groupid ,topic, appcode 获取已经授权的记录
    :param group_id: group id
    :param topic: 需要授权的topic
    :param user: 用户名
    :return: {list} db中对应的完整记录列表
    """

    try:
        return QueueConsumerConfig.objects.get(
            group_id=group_id,
            topic=topic,
            user_name=user,
            status=GRANTED,
            type=cluster_type,
        )
    except QueueConsumerConfig.DoesNotExist:
        logger.warning("failed to find queue_consumer_config by topic %s" % topic)
        return None

def get_queue_consumer_config_exclude(topic, cluster_type):
    """
    根据groupid ,topic, appcode 获取已经授权的记录
    :param topic: 需要授权的topic
    :param cluster_type: 就请你类型（kafka/pulsar）
    :return: {list} db中对应的完整记录列表
    """
    return QueueConsumerConfig.objects.filter(topic=topic, status=GRANTED).exclude(type=cluster_type)

def get_granted_queue_consumer_config(group_id, topic, version, cluster_type, is_pattern):
    """
    获取已经授权的producer记录
    :param group_id: 需要授权的topic
    :param topic: queue队列 topic
    :param version 版本号
    :param cluster_type 集群类型
    :param is_pattern 是否为正则表示的topic
    :return: {list} db中对应的完整记录列表
    """
    if topic.startswith("^") or is_pattern:
        return QueueConsumerConfig.objects.filter(
            group_id=group_id,
            topic__regex=topic,
            id__gt=version,
            type=cluster_type,
            status__in=[GRANTED, REVOKING, REVOKED],
        ).order_by("-id")
    else:
        return QueueConsumerConfig.objects.filter(
            group_id=group_id,
            topic=topic,
            id__gt=version,
            type=cluster_type,
            status__in=[GRANTED, REVOKING, REVOKED],
        ).order_by("-id")

def get_granted_queue_consumer_by_id(topic, cluster_id):
    """
    获取已经授权的producer记录
    :param topic: queue队列 topic
    :param cluster_id 队列集群id
    :return: {list} db中对应的完整记录列表
    """
    return QueueConsumerConfig.objects.filter(type=KAFKA, status=GRANTED, topic=topic, cluster_id=cluster_id)

def create_queue_consumer_config(
    cluster_type,
    user_name,
    group_id,
    topic,
    cluster_id,
    created_by,
    updated_by,
    status=GRANTED,
):
    """
    创建已经授权的记录
    :param cluster_type: 就请你类型（kafka/pulsar）
    :param user_name: 需要授权的topic
    :param group_id: 用户名
    :param cluster_id: queue队列集群id
    :param topic: queue队列 topic
    :param status 授权状态
    :return: {list} db中对应的完整记录列表
    """
    return QueueConsumerConfig.objects.create(
        type=cluster_type,
        user_name=user_name,
        group_id=group_id,
        status=status,
        topic=topic,
        cluster_id=cluster_id,
        created_by=created_by,
        updated_by=updated_by,
        description="",
    )

def create_queue_producer_config(user_name, topic, cluster_id, created_by, updated_by, status=GRANTED):
    """
    创建已经授权的记录
    :param user_name: 需要授权的topic
    :param cluster_id: queue队列集群id
    :param topic: queue队列 topic
    :param status 授权状态
    :return: {list} db中对应的完整记录列表
    """
    return QueueProducerConfig.objects.create(
        user_name=user_name,
        topic=topic,
        status=status,
        cluster_id=cluster_id,
        created_by=created_by,
        updated_by=updated_by,
        description="",
    )

def create_queue_change_event(topic, src_cluster_id, dst_cluster_id):
    """
    创建队列迁移事件
    :param src_cluster_id: 原始队列集群id
    :param dst_cluster_id: 目标队列集群id
    :param topic: queue队列 topic
    :return: db中对应的完整记录列表
    """
    return QueueChangeEvent.objects.create(
        type=PRODUCER,
        user_name="",
        topic=topic,
        src_cluster_id=src_cluster_id,
        dst_cluster_id=dst_cluster_id,
        ref_id=0,
        end_offsets="",
        description="",
    )

def get_queue_producer_config(user_name, topic, version):
    """
    获取已经授权的producer记录
    :param user_name: 需要授权的topic
    :param topic: queue队列 topic
    :param version 版本号
    :return: {list} db中对应的完整记录列表
    """
    return QueueProducerConfig.objects.filter(
        user_name=user_name,
        topic=topic,
        id__gt=version,
        status__in=[GRANTED, REVOKING, REVOKED],
    ).order_by("-id")

def get_granted_queue_producer_by_id(topic, cluster_id):
    """
    获取已经授权的producer记录
    :param topic: queue队列 topic
    :param cluster_id 队列集群id
    :return: {list} db中对应的完整记录列表
    """
    return QueueProducerConfig.objects.filter(status=GRANTED, topic=topic, cluster_id=cluster_id)

def get_queue_cluster_config(cluster_name, cluster_type):
    """
    获取队列集群信息
    :param cluster_name: 集群名称
    :param cluster_type: 集群类型(kafka/pulsar)
    :return: 队列集群信息
    """
    try:
        return QueueClusterConfig.objects.get(cluster_name=cluster_name, cluster_type=cluster_type)
    except QueueClusterConfig.DoesNotExist:
        logger.warning("failed to find queue_cluster_config by cluster_name %s" % cluster_name)
        return None

def get_queue_cluster_config_by_id(cluster_id, cluster_type):
    """
    获取队列集群信息
    :param cluster_id: 集群名称
    :param cluster_type: 集群类型(kafka/pulsar)
    :return: 队列集群信息
    """
    try:
        return QueueClusterConfig.objects.get(id=cluster_id, cluster_type=cluster_type)
    except QueueClusterConfig.DoesNotExist:
        logger.warning("failed to find queue_cluster_config by id %s" % cluster_id)
        return None

def create_queue_cluster_config(cluster_name, cluster_type, cluster_domain, cluster_port, tenant, namespace):
    """
    获取队列集群信息
    :param cluster_name: 集群名称
    :param cluster_type: 集群类型(kafka/pulsar)
    :param cluster_domain: 集群连接
    :param cluster_port: 集群端口
    :param tenant 租戶
    :param namespace 命名空间
    :return: 队列集群信息
    """
    return QueueClusterConfig.objects.create(
        cluster_name=cluster_name,
        cluster_type=cluster_type,
        cluster_domain=cluster_domain,
        cluster_port=cluster_port,
        tenant=tenant,
        namespace=namespace,
    )

def get_databus_config(conf_key):
    """
    获取databus_config表中conf_key对于的配置项对象
    :param conf_key: 配置的key
    :return: 配置对象
    """
    try:
        return DatabusConfig.objects.get(conf_key=conf_key)
    except DatabusConfig.DoesNotExist:
        logger.warning("failed to find databus_config by conf_key %s" % conf_key)
        return None

def get_databus_config_value(conf_key, default_val):
    """
    获取databus_config表中conf_key对应的conf_value，如果conf_key不存在，则返回默认值
    :param conf_key: 配置的key
    :param default_val: 默认值
    :return: 配置的value
    """
    conf = get_databus_config(conf_key)
    return conf.conf_value if conf else default_val


import requests
from bs4 import BeautifulSoup
import spacy
from spacy import displacy

def text_from_html(data):
    soup = BeautifulSoup(data, 'html.parser')
    paragraphs = soup.find_all('div', {'class': 'ssrcss-7uxr49-RichTextContainer e5tfeyi1'})
    return u" ".join([paragraph.get_text() for paragraph in paragraphs])

def visualize(textdata):
    doc = nlp(textdata)
    displacy.serve(doc, style='ent')

def render(article):
	data = requests.get(article).text
	textdata = text_from_html(data)
	visualize(textdata)

nlp = spacy.load('en_core_web_sm')

url = 'https://www.bbc.com/news/business'
data = requests.get(url).text
soup = BeautifulSoup(data, 'html.parser')
links = soup.find_all('a', href=True)
valid_articles = []
for lnk in links:
    print(lnk['href'])
    if lnk['href'][:15] == '/news/business-':
        valid_articles.append('http://bbc.com' + lnk['href'])
print(valid_articles)

### doesn't work, link is http://bbc.com/news/business-45489065
#render(valid_articles[0])

## this one works, link is http://bbc.com/news/business-61990864
#render(valid_articles[40])


import json

import discord
from discord.ext import commands

from assets import get_color, aiohttp_assets

def get_time_info(user_id: int):
    with open("./storage/time_files/time_tz.json") as timeFile:
        data = json.load(timeFile)
        timezone = data.get(str(user_id))
        return timezone

def get_embed_from_weather_dict(result: dict):
    longitude = result.get("coord").get('lon')
    latitude = result.get("coord").get('lat')

    weather = result.get("weather")[0].get("main")
    icon = result.get("weather")[0].get("icon")
    actual_temp = int(result.get("main").get("temp") - 273.15)
    feels_like = int(result.get("main").get("feels_like") - 273.15)
    temp_min, temp_max = int(result.get("main").get("temp_min") - 273.15), \
                         int(result.get("main").get("temp_max") - 273.15)
    humidity = result.get("main").get("humidity")

    country_code = result.get("sys").get("country")
    name = result.get("name")

    return name, latitude, longitude, icon, weather, actual_temp, feels_like, temp_min, temp_max, humidity, country_code

class Weather(commands.Cog):

    def __init__(self, bot):
        self.bot = bot

        with open("config.json", "r") as configFile:
            data = json.load(configFile)
            self.apikey = data.get("weather_api_key")
            self.weather_url = "https://api.openweathermap.org/data/2.5/weather?q={cityName}&appid={apiKey}"
            self.weather_icon_url = "https://openweathermap.org/img/wn/{code}@2x.png"
        with open("./storage/weather.json", "r") as weatherFile:
            self.weatherData = json.load(weatherFile)

        if self.apikey is None:
            raise ValueError  # stop the cog from loading

    @commands.command(name="weather", aliases=["temperature"],
                      description="Get the weather information for your location or another member")
    async def weather_user(self, ctx, *, user: discord.Member = None):
        gotten_from_tz = False
        if user is None:
            user = ctx.author
        weather_zone = self.weatherData.get(str(user.id))
        if weather_zone is None:
            weather_zone = str(get_time_info(user.id)).split("/")
            if len(weather_zone) == 1:
                return await ctx.send("It seems you have not set your region. "
                                      "You can set your region using the `setweatherlocation` command.")
            weather_zone = weather_zone[-1]
            gotten_from_tz = True
        result = await aiohttp_assets.aiohttp_get(
            url=self.weather_url.format(cityName=weather_zone, apiKey=self.apikey))
        result = json.loads(result)
        if str(result.get("cod")) == "404":
            return await ctx.send("The city was not found. "
                                  "Please update your city using the `setweatherlocation` command")
        name, latitude, longitude, icon, weather, \
            actual_temp, feels_like, temp_min, temp_max, \
            humidity, country_code = get_embed_from_weather_dict(result)

        embed = discord.Embed(title=f"{name}, where {user.display_name} is",
                              description=f"{longitude}_N_, {latitude}_E_",
                              color=get_color.get_color(user))
        embed.add_field(name="Weather", value=weather, inline=True)
        embed.add_field(name="Temperature", value=f"Actual: **{actual_temp}°C**\n"
                                                  f"Feels Like: **{feels_like}°C**", inline=True)
        embed.add_field(name="Min/Max", value=f"Min: **{temp_min}°C**\n"
                                              f"Max: **{temp_max}°C**", inline=True)
        embed.add_field(name="Humidity", value=f"{humidity}%", inline=True)
        embed.add_field(name="Country Code", value=country_code)
        if icon:
            embed.set_thumbnail(url=self.weather_icon_url.format(code=icon))
        if gotten_from_tz:
            embed.set_footer(text="Location fetched from user's timezone info")
        await ctx.send(embed=embed)

    @commands.command(name="setweatherlocation", aliases=["weatherlocation"],
                      description="Sets the location to use for weather info")
    async def set_weather_location(self, ctx, location):
        message = await ctx.send(f"Checking if **{location}** is a valid location...")
        result = await aiohttp_assets.aiohttp_get(self.weather_url.format(cityName=location, apiKey=self.apikey))
        result = (json.loads(result))
        if result.get("message") is not None:
            return await message.edit(content=f"The API I use encounted an error : **{result.get('message')}**")
        self.weatherData[str(ctx.author.id)] = result.get("name")
        with open("./storage/weather.json", "w") as writeFile:
            json.dump(self.weatherData, writeFile)
        await message.edit(content=f"Location set to **{result.get('name')}** successfully!")

    @commands.command(name="locationweather", aliases=["lweather", "weatherl"],
                      description="Gets weather information for a specific city.")
    async def weather_from_city(self, ctx, city):
        message = await ctx.send(f"Gathering weather info for **{city}**...")
        result = await aiohttp_assets.aiohttp_get(url=self.weather_url.format(cityName=city, apiKey=self.apikey))
        result = json.loads(result)
        if result.get("message") is not None:
            return await message.edit(content=f"The API I use encountered an error: **{result.get('message')}**")
        name, latitude, longitude, icon, weather, \
            actual_temp, feels_like, temp_min, temp_max, \
            humidity, country_code = get_embed_from_weather_dict(result)

        embed = discord.Embed(title=f"{name}'s weather info",
                              description=f"{longitude}_N_, {latitude}_E_",
                              color=get_color.get_color(ctx.author))
        embed.add_field(name="Weather", value=weather, inline=True)
        embed.add_field(name="Temperature", value=f"Actual: **{actual_temp}°C**\n"
                                                  f"Feels Like: **{feels_like}°C**", inline=True)
        embed.add_field(name="Min/Max", value=f"Min: **{temp_min}°C**\n"
                                              f"Max: **{temp_max}°C**", inline=True)
        embed.add_field(name="Humidity", value=f"{humidity}%", inline=True)
        embed.add_field(name="Country Code", value=country_code)
        if icon:
            embed.set_thumbnail(url=self.weather_icon_url.format(code=icon))
        await message.edit(embed=embed, content=None)

def setup(bot):
    bot.add_cog(Weather(bot))


#Python program to print topological sorting of a DAG
from collections import defaultdict

#Class to represent a graph
class Graph:
    def __init__(self,vertices):
        self.graph = defaultdict(list) #dictionary containing adjacency List
        self.V = vertices #No. of vertices

    # function to add an edge to graph
    def addEdge(self,u,v):
        self.graph[u].append(v)

    def topologicalSortUtil(self):
        path = []

        while self.zeros:
            pop = self.zeros.pop(0)
            path.append(pop)
            for v in self.graph[pop]:
                self.inDegrees[v] -= 1
                if self.inDegrees[v] == 0:
                    self.zeros.append(v)

        if len(path) == self.V:
            print(1)
        else:
            print(0)
        #print(path)

    def topologicalSort(self):
        # set for topo sort
        self.inDegrees = [0]*self.V
        self.zeros = []

        for u in range(self.V):
            for v in self.graph[u]:
                self.inDegrees[v] += 1

        for i in range(self.V):
            if self.inDegrees[i] == 0:
                self.zeros.append(i)

        print(self.zeros)
        self.topologicalSortUtil()

g= Graph(6)
g.addEdge(5, 2);
g.addEdge(5, 0);
g.addEdge(4, 0);
g.addEdge(4, 1);
g.addEdge(2, 3);
g.addEdge(3, 1);

print("Following is a Topological Sort of the given graph")
g.topologicalSort()
#This code is contributed by Neelam Yadav

'''

def topoSort(n, graph):
    inDegrees = [0]*n
    zeros = []

    for u in range(n):
        for v in graph[u]:
            inDegrees[v] += 1

    for i in range(n):
        if inDegrees[i] == 0:
            zeros.append(i)

    path = []

    while zeros:
        pop = zeros.pop(0)
        path.append(pop)
        for v in graph[pop]:
            inDegrees[v] -= 1
            if inDegrees[v] == 0:
                zeros.append(v)

    if len(path) == N:
        print(1)
    else:
        print(0)

'''


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^comment/list/$',views.ListCommentView.as_view(), name='listComment'),
    url(r'^comment/add/$',views.AddCommentView.as_view(), name='addComment'),
    url(r'^comment/(?P<pk>[0-9]+)/$',views.DetailCommentView.as_view(), name='detailComment'),
    url(r'^comment/(?P<pk>[0-9]+)/update/$',views.UpdateCommentView.as_view(), name='updateComment'),
    url(r'^comment/(?P<pk>[0-9]+)/delete/$',views.DeleteCommentView.as_view(), name='deleteComment'),

    url(r'^reply/add/$',views.AddReplyView.as_view(), name='addReply'),
    url(r'^reply/(?P<pk>[0-9]+)/$',views.DetailReplyView.as_view(), name='detailReply'),
    url(r'^reply/(?P<pk>[0-9]+)/update/$',views.UpdateReplyView.as_view(), name='updateReply'),
    url(r'^reply/(?P<pk>[0-9]+)/delete/$',views.DeleteReplyView.as_view(), name='deleteReply'),

    url(r'^comment/(?P<pk>[0-9]+)/thread/$',views.CommentThreadView.as_view(), name='commentThread'),
]


from flask import Flask
from flask import request
import time
import hashlib
from flask_script import Manager
import xml.etree.ElementTree as ET

import dispacher

app = Flask(__name__)
manager = Manager(app)

@app.route('/')
def valid():
    echostr = request.args.get('echostr')
    signature = request.args.get('signature')
    timestamp = request.args.get('timestamp')
    nonce = request.args.get('nonce')

    api = wechatCallbackApi()
    if(api.check_signature(signature,timestamp,nonce)):
        return echostr
    else:
        return None

@app.route('/',methods=['post'])
def response():
    postStream = request.stream.read()
    api = wechatCallbackApi()
    toUser,fromUser,userContent = api.get_request_text(postStream)
    responseText = dispacher.dispacher(userContent)
    return api.create_response_xml_text(fromUser,toUser,responseText)

class wechatCallbackApi:
    def check_signature(self,signature, timestamp, nonce):
        token = "palexu"
        L = [timestamp, nonce, token]
        L.sort()
        s = L[0] + L[1] + L[2]
        return hashlib.sha1(s.encode('utf-8')).hexdigest() == signature

    def get_request_text(self,PostXML):
        """Get the content of user query"""
        PostXML = ET.fromstring(PostXML)
        ToUser = PostXML.find("ToUserName").text
        FromUser = PostXML.find("FromUserName").text
        UserContent  = PostXML.find("Content").text

        return ToUser,FromUser,UserContent

    def create_response_xml_text(self,toUser,fromUser,RespContent):
        """ format the response text content and response"""
        RespFomat = "<xml><ToUserName><![CDATA[%s]]></ToUserName><FromUserName><![CDATA[%s]]></FromUserName><CreateTime>%s</CreateTime><MsgType><![CDATA[text]]></MsgType><Content><![CDATA[%s]]></Content><FuncFlag>0</FuncFlag></xml>"
        RespXML = RespFomat % (toUser,fromUser,int(time.time()),RespContent)
        return RespXML

if __name__ == '__main__':
    manager.run()


import numpy as np
import cv2
import math
import tensorflow as tf
from tensorflow.keras import Model

from architecture import HSMNIST
from load_data import DataLoader
from train import Trainer

from utils import decode, draw_bbox, xywh2yxyx
from constant import *

class Detector:
    def __init__(self):
        pass

    def postprocess(self, preds):
        '''
        Arguments
            preds: (3, )
            - preds[i] = [N, W, H, 3, 5 + n_class], (cx, cy, w, h, p, class)

        Return
            pred: [M, 6], (x_min, y_min, x_max, y_max, conf, class_id)
                   different grid scales' outputs are resized as the original (e.g., 480x640)
        '''
        valid_scale = [0, np.inf]

        ori_h = HEIGHT
        ori_w = WIDTH

        tmp = []
        # for i in range(1):
        for i in range(ANCHORS_PER_GRID):
            pred = np.array(preds[i])
            tmp.append(pred)

        preds_arr = [np.reshape(x,(-1,np.shape(x)[-1])) for x in tmp]
        preds_arr = np.concatenate(preds_arr, axis=0)

        xywh = preds_arr[:, :4]
        conf = preds_arr[:, 4]
        prob = preds_arr[:, 5:]

        # print("prob:\n",prob)
        # print(np.argmax(prob, axis=-1))
        # print(np.max(prob))

        # (cx, cy, w, h) -> (x_min, y_min, x_max, y_max)
        coor = np.concatenate([xywh[:,:2]-xywh[:,2:]*0.5, xywh[:,:2]+xywh[:,2:]*0.5], axis=-1)

        # check invalid (x_min, y_min, x_max, y_max)
        clip_xy_min = np.maximum(coor[:,:2], [0,0])
        clip_xy_max = np.minimum(coor[:,2:], [ori_w-1, ori_h-1])
        coor = np.concatenate([clip_xy_min, clip_xy_max], axis=-1)
        invalid_mask = np.logical_or((coor[:,0] > coor[:,2]), (coor[:,1] > coor[:,3]))

        # check invalid bboxes
        area = np.sqrt(np.multiply.reduce(coor[:, 2:4] - coor[:, 0:2], axis=-1))
        scale_mask = np.logical_and((valid_scale[0] < area), (area < valid_scale[1]))

        # discard boxes with low scores
        classes = np.argmax(prob, axis=-1)
        #TODO : fix prob values
        # scores = conf * prob[np.arange(len(coor)), classes]
        scores = conf
        score_mask = scores > SCORE_THRES
        mask = np.logical_and(scale_mask, score_mask)

        coor, scores, classes = coor[mask], scores[mask], classes[mask]

        return np.concatenate([coor, scores[:, np.newaxis], classes[:, np.newaxis]], axis=-1)

if __name__ == "__main__":
    model_path = BEST_MODEL
    path = DATA_PATH

    dl = DataLoader(path)
    im, ld, pd, ns = dl.get_dataset()

    batch_size = 1
    nb = math.ceil(ns/batch_size) # number of batch

    im = im.batch(batch_size)
    ld = ld.batch(batch_size)
    pd = pd.batch(batch_size)

    learning_info = {}
    learning_info["steps_per_epoch"] = nb
    learning_info['epochs'] = 1

    model = HSMNIST((WIDTH, HEIGHT, 3), NUM_CLASS).build(training=True)
    # model.trainable = False
    model.load_weights(model_path)

    # print(model.layers[-1].weights)

    detector = Detector()
    for c, l, p in zip(im, ld, pd):
        preds = model(c)
        preds = decode(preds)
        cx = tf.range(0,400,10)
        cy = tf.range(0,400,10)

        im = c[0].numpy()
        # for i in range(ANCHORS_PER_GRID):
        #     x = cx/STRIDES[i]
        #     y = cy/STRIDES[i]

        #     x = np.array(x, dtype=np.int8)
        #     y = np.array(y, dtype=np.int8)

        #     for n in x:

        #         for m in y:
        #             im = c[0].numpy()

        #             xx = tf.reshape(preds[i][0, n, m,:,0], -1)
        #             yy = tf.reshape(preds[i][0, n, m,:,1], -1)
        #             ww = tf.reshape(preds[i][0, n, m,:,2], -1)
        #             hh = tf.reshape(preds[i][0, n, m,:,3], -1)

        #             for j in range(3):
        #                 x_min = int(xx[j] - ww[j]/2)
        #                 y_min = int(yy[j] - hh[j]/2)
        #                 x_max = int(xx[j] + ww[j]/2)
        #                 y_max = int(yy[j] + hh[j]/2)
        #                 if j == 0: color=(0,0,255)
        #                 if j == 1: color=(0,255,0)
        #                 if j == 2: color=(255,0,0)
        #                 cv2.rectangle(im, (x_min,y_min), (x_max,y_max), color, 1)

        # # cv2.imshow("im", image_data[0].numpy())
        #             cv2.imshow("img", im)
        #             cv2.waitKey(100)
        #             cv2.destroyAllWindows()

        result = detector.postprocess(preds)
        # print(result[0][0,0,0,0,:])
        bboxes = result[:,:4]
        scores = result[:,4]

        for i in range(3):
            for j in range(3):
                obj = preds[i][0,:,:,j,4].numpy()
                tf.print("min: ", tf.reduce_min(obj), " max: ", tf.reduce_max(obj))
                obj = cv2.resize(obj,(WIDTH, HEIGHT))

                cv2.imshow("obj", obj)
                cv2.waitKey()
                cv2.destroyAllWindows()

        # prob = preds[0][0,:,:,0,5:].numpy()
        # prob = cv2.resize(prob, (WIDTH, HEIGHT))
        # cv2.imshow("prob", prob)
        # cv2.waitKey()
        # cv2.destroyAllWindows()

        yxyx = xywh2yxyx(bboxes)

        best_bbox_indices = tf.image.non_max_suppression(yxyx, scores,
                                                        max_output_size = 20,
                                                        iou_threshold = 0.85,
                                                        score_threshold = 0.85)
        best_bbox = result[best_bbox_indices,:]
        # print(best_bbox)
        # print(scores)
        img = draw_bbox(c, best_bbox)
        # img = draw_bbox(c, result)
        # img = np.transpose(img, (1,0,2))
        im = cv2.cvtColor(img.astype('float32'), cv2.COLOR_BGR2RGB)
        cv2.imshow("im", img.astype('float32'))
        cv2.waitKey()
        cv2.destroyAllWindows()
        break


from typing import Generator

from pathlib import Path
from typing import Any
from re import Pattern, compile as re_compile, VERBOSE

_LOADER_BEGIN = """
import Error404 from "$lib/components/error.404.svelte"
import Error403 from "$lib/components/error.403.svelte"

/**
 * Dynamically import an existing mdx componet (bundled)
 *
 * @param {String} componentName
 */
export async function fetchComponent (componentName) {
    let EntryComponent = null;
    let error = undefined;
    try {
      switch (componentName) {
        case 'index':
          EntryComponent = (await import('$lib/mawanet/maelstrom_encyclopedia.mdx')).default;
          break;
"""

_LOADER_END = """
        default:
          EntryComponent = Error403;
          break;
      }
    } catch (e) {
      console.error(e);
      EntryComponent = Error404;
      error = e;
    }
    return {
        componentName,
        entryComponent: EntryComponent,
        error,
    }
}
"""

_LOADER_IMPORT = """
        case '{}':
          EntryComponent = (await import('$lib/mawanet/{}.mdx')).default;
          break;\n
"""

class Globber:

    _RE_WIKILINK: Pattern = re_compile(
        r"""\[{2}   # openning doubles
            (?P<filename>[\w\s]+?) # filename
            (?:\#[\w\s\.\d\(\)\[\]]+?)?    # wont capture, we ignore ids
            (?:\|(?P<alias>[\w\s\.\d\(\)\[\]]+?))?    # visible alias
            \]{2}""", VERBOSE,
    )

    _MDX_COMPONENT_LINK_TEMPLATE = '<Link path="/mawanet/{}">{}</Link>'
    _MDX_IMPORTS = '<script>import Link from "$lib/components/link.svelte"</script>\n\n'

    def __init__(self, glob_root: Path, output_dir: Path) -> None:
        """
        Parameters
        ----------
        glob_root : Path
            Path to `.md`s' root directory
        output_dir : Path
            Where files will be moved
        """
        self._glob_root = glob_root
        self._output_dir = output_dir

    def __call__(self) -> Any:
        glob_result: Generator = self._glob_root.glob('**/*.md')
        count = 0
        for count, md in enumerate(glob_result):
            name_normalised: str = md.stem.replace(' ', '_')
            contents = md.read_text()
            with open(
                self._output_dir / (name_normalised + '.mdx'),
                mode='w',
                encoding='utf-8'
            ) as mdx:
                mdx.write(contents)
        print(f'{count} files processed')

    def replace_wikilinks(self):
        """
        Replaces `[[Entry]]`, `[[Entry Whitespaces#id|alias]]` to

            <Link path="/maelstrom/entry">Entry</Link>
        """
        for mdx in self._output_dir.iterdir():
            contents_old = mdx.read_text()
            contents_new = ''
            mappings_cache = []
            for m in self._RE_WIKILINK.finditer(contents_old):
                filename = m.group('filename')
                alias = m.group('alias')
                link = self._MDX_COMPONENT_LINK_TEMPLATE.format(
                    filename.lower().replace(' ', '_'), alias or filename
                )
                mappings_cache.append((m.span(), link))
            contents_new = contents_old
            for mpair in reversed(mappings_cache):
                spans: tuple = mpair[0]
                string = mpair[1]
                contents_new = contents_new[:spans[0]] + string + contents_new[spans[1]:]
            mdx.write_text(self._MDX_IMPORTS + contents_new)

    def generate_js_imports(self) -> None:
        imports = []
        for mdx in self._output_dir.iterdir():
            imports.append(_LOADER_IMPORT.format(mdx.stem, mdx.stem))
        js_loader_contents = _LOADER_BEGIN + ''.join(imports) + _LOADER_END
        (Path().cwd() / 'src/lib/stores/mawanet.loader.js').write_text(js_loader_contents)


import socket
from pyaudio import PyAudio
from time import sleep

server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
host_name = socket.gethostname()
host_ip = socket.gethostbyname(host_name)
#host_ip = "192.168.100.105"
port = 10050

print("host name => ", host_name)
print("host ip => ", host_ip)
print("host port => ", port)

print("socekt created")

server_socket.bind((host_ip, port))
print("socket bind complete")
# server_socket.setblocking(False)

p = PyAudio()
stream = p.open(format=p.get_format_from_width(2),
                    channels=1,
                    rate=44100,
                    input=True)
while 1:
    try:
        packet = server_socket.recvfrom(40960)
    except BlockingIOError as e:
        print(e)
        continue

    #print(type(data)) #<class 'bytes'>
    # data = pickle.loads(data)
    # if not packet:
    #     continue
    #print(type(data)) #<class 'numpy.ndarray'>
    # data = cv2.decode(data, cv2.IMREAD_COLOR)
    print(packet[0])
    stream.write(packet[0])

stream.stop_stream()
stream.close()
p.terminate()


"""empty message

Revision ID: 704f89ce63a2
Revises: 289d7babb5c7
Create Date: 2020-09-14 17:06:42.845315

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '704f89ce63a2'
down_revision = '289d7babb5c7'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('receipt', schema=None) as batch_op:
        batch_op.add_column(sa.Column('tax', sa.Float(asdecimal=True),
                                      nullable=True))

    with op.batch_alter_table('receipt', schema=None) as batch_op:
        batch_op.execute("UPDATE receipt SET tax = 0")

    with op.batch_alter_table('receipt', schema=None) as batch_op:
        batch_op.alter_column('tax', nullable=False)

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('receipt', schema=None) as batch_op:
        batch_op.drop_column('tax')

    # ### end Alembic commands ###


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 13 14:40:54 2019

@author: AZROUMAHLI Chaimae
"""

# Libraries
from numpy import zeros
from keras.models import Sequential
from keras.layers import Embedding
from keras.layers import Flatten, Dense, LSTM, Dropout,GRU,GlobalAveragePooling1D
#import dask.array as da

# Created Methods
#from Reading_dictionnary import get_WE_fasttext
from Reading_dictionnary import get_WE_Glove
from Reading_dictionnary import get_WE_Word2Vec

#the input: The embedding index that contains the model's dictionnary
def get_embedding_matrix(t,vocab_size,dictionnary_file):
    #loading the whole embedding into memory
    #The function here depends on the model (Word2vec or Glove)
    #each model is named using the methods name
    if "Glove" in dictionnary_file:
        embeddings_index=get_WE_Glove(dictionnary_file)
    else:
        embeddings_index=get_WE_Word2Vec(dictionnary_file)
    """else:
        embeddings_index=get_WE_fasttext(dictionnary_file)"""
    vector_dim=len(list(embeddings_index.values())[0])
    # create a weight matrix for words in training docs
    embedding_matrix = zeros((vocab_size, vector_dim))
    #embedding_matrix=da.zeros((vocab_size,vector_dim))
    for word, i in t.word_index.items():
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    return vector_dim,embedding_matrix

def get_model_WE_NER(t,X_train, Y_train, X_test, Y_test, vocab_size, dictionnary_file, max_lenght, Flatten_Layer,GlobalAvergePooling1D_Layer, LSTM_Layer, Dropout_LSTM, GRU_Layer, GRU_dropout, GRU_reccurent_dropout, Dense_activation, compile_optimizer, N_batch_size, N_epochs):
    #number of output (NER classes)
    N_clusters=9
    #to avoid the problem of Memory error, we need to create a separated function that saves the embedding matrix
    vector_dim,embedding_matrix=get_embedding_matrix(t,vocab_size,dictionnary_file)
    #define model
    model = Sequential()
    e = Embedding(vocab_size,vector_dim,weights=[embedding_matrix], input_length=max_lenght,trainable=False)
    model.add(e)
    if Flatten_Layer:
        model.add(Flatten())
    if GlobalAvergePooling1D_Layer:
        model.add(GlobalAveragePooling1D())
    if LSTM_Layer:
        if Dropout_LSTM:
            model.add(Dropout(0.2))
            model.add(LSTM(vector_dim, return_sequences=True,))
            model.add(Dropout(0.2))
        else:
            model.add(LSTM(vector_dim))
    if GRU_Layer:
        model.add(GRU(32,dropout=GRU_dropout,recurrent_dropout=GRU_reccurent_dropout, return_sequences=True))
    #model.add(Flatten())
    try:
        if Flatten_Layer==0 and GlobalAvergePooling1D_Layer==0:
            model.add(Flatten())
    except:
        pass
    model.add(Dense(N_clusters, activation=Dense_activation))
    # compile the model
    model.compile(optimizer=compile_optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
    # summarize the model
    model.fit(X_train, Y_train, batch_size=N_batch_size, epochs=N_epochs, validation_data=(X_test,Y_test), verbose=0)
    return model


#! /usr/bin/python
# -*- coding: utf-8 -*-

"""Split the target-ligand complex in the given PDB file."""

import dataclasses
import json
import logging
import math
import re
from pathlib import Path
from typing import Any, Dict, List, Optional

from Bio.PDB import PDBIO, Select, MMCIFIO
from Bio.PDB.Structure import Structure as PdbStructure
from dataclasses import dataclass, asdict

from . import structure_file, af2_mmcif_parsing
from .common_utils import check_ext, is_empty, norm_cif_empty_value
from .mmcif_utils import get_uniprot_ref
from .pdb_helper_datasets import get_pdb_ccd_info, CcdInfo
from .. import config
from ..exceptions import AF2MmCIFParseError

@dataclass(order=True)
class LigandInfo:
    ligand_id: int
    pdb_id: str
    ccd_id: str
    model_id: int
    chain_id: str
    res_id: int
    insertion_code: str

def _get_ligand_info(pdb_id: str, structure: PdbStructure, all_ccd_info) -> List[LigandInfo]:
    """Get ligand information of this structure."""
    ligand_info = []

    for model in structure:
        model_id = model.id
        for chain in model:
            chain_id = chain.id
            for residue in chain:
                hetero_flag = residue.id[0]
                if hetero_flag.startswith('H_'):
                    ligand_info.append(LigandInfo(
                        ligand_id=len(ligand_info),
                        pdb_id=pdb_id,
                        ccd_id=hetero_flag[2:],
                        model_id=model_id,
                        chain_id=chain_id,
                        res_id=residue.id[1],
                        insertion_code=residue.id[2],
                    ))

    # TODO: Filter some ligands?

    return ligand_info

class MainTargetSelect(Select):
    """Select main target component of the structure (remove water and hetero residues)."""
    def accept_residue(self, residue):
        res = residue.id[0]
        if res == ' ':
            # ' ' means main target residue.
            return 1
        else:
            # 'W' means water molecule.
            # 'H_XXX' means hetero residue.
            return 0

class LigandSelect(Select):
    def __init__(self, model_id, chain_id, seq_id):
        super().__init__()
        self.model_id = model_id
        self.chain_id = chain_id
        self.seq_id = seq_id

    def accept_model(self, model):
        return 1 if model.id == self.model_id else 0

    def accept_chain(self, chain):
        return 1 if chain.id == self.chain_id else 0

    def accept_residue(self, residue):
        hetatm_flag, resseq, icode = residue.get_id()
        if not hetatm_flag.startswith('H_'):
            return 0
        if icode != ' ':
            logging.warning('Icode %s at position %s', icode, resseq)
        if resseq == self.seq_id:
            return 1
        else:
            return 0

def _save_structure(filename, structure, select, ext):
    if ext == '.cif':
        io = MMCIFIO()
    else:
        io = PDBIO()
    io.set_structure(structure)
    try:
        io.save(str(filename), select=select)
    except TypeError as e:
        # Clean bad file.
        if filename.exists():
            filename.unlink()
        if str(e) == '%c requires int or char':
            raise RuntimeError(f'Chain id is not a single char') from e
        else:
            raise

def _dump_split_complex(
        pdb_id: str,
        structure: PdbStructure,
        ligand_info: List[LigandInfo],
        dest_path: Path,
        ext: str,
):

    # Save main target.
    main_select = MainTargetSelect()
    split_dest_filename = dest_path / f'{pdb_id}-no-ligand{ext}'
    _save_structure(split_dest_filename, structure, main_select, ext)

    # Save all ligands.
    for info in ligand_info:
        ligand_filename = dest_path / f'{pdb_id}-ligand{info.ligand_id}{ext}'
        ligand_select = LigandSelect(model_id=info.model_id, chain_id=info.chain_id, seq_id=info.res_id)
        _save_structure(ligand_filename, structure, ligand_select, ext)

@dataclass
class SplitComplexResult:
    """
    target_filename: Path to the target (without ligands and water).
    ligand_info_filename:
    ligand_filenames:
    ligand_info:
    """
    target_filename: Optional[Path]
    ligand_info_filename: Optional[Path]
    ligand_filenames: List[Path]
    ligand_info: List[LigandInfo]

def split_pdb_complex_paths(
        pdb_id: str,
        split_ext: str = '.pdb',
        split_cache_path: Path = None,
        pdb_cache_path: Path = None,
        ccd_cache_path: Path = None,
) -> SplitComplexResult:
    """Remove all heteros from the given PDB file, and return the split target and ligand file paths.

    Args:
        pdb_id:
        split_ext: File extension to store split structure files. Default: .pdb
            (NOTE: smina cannot parse .cif files with single value, so we fallback to .pdb format)
        split_cache_path (Path):
        pdb_cache_path:
        ccd_cache_path:

    Returns:
        SplitComplexResult: Dict of paths and information.

        Keys:
            target_filename (Path): Path to the target (without ligands and water).
            ligand_info_filename (Path):
            ligand_filenames (List[Path]):
            ligand_info (dict):
        All keys will be set to None or empty values if structure file not exists.

    Raises:
        FileNotFoundError is failed to get structure file.

    Notes:
        1. Internally use PdbStructure.
        2. Unlike `get_target_ligand_pairs_from_pdb`, this function will apply *NO* filtration.
        All hetero atoms will be remained (to get more possible docking inputs).
    """
    split_ext = check_ext(split_ext)

    pdb_id = pdb_id.lower()

    empty_result = SplitComplexResult(
        target_filename=None, ligand_info_filename=None,
        ligand_filenames=[], ligand_info=[],
    )

    if split_cache_path is None:
        split_cache_path = config.split_pdb_cache_path()
    if pdb_cache_path is None:
        pdb_cache_path = config.pdb_cache_path()
    if ccd_cache_path is None:
        ccd_cache_path = config.pdb_ccd_path()
    dest_path = split_cache_path / pdb_id[1:3] / pdb_id
    dest_path.mkdir(exist_ok=True, parents=True)

    def _ligand_filenames():
        return [
            dest_path / f'{pdb_id}-ligand{i}{split_ext}'
            for i in sorted(v.ligand_id for v in ligand_info)
        ]

    split_dest_filename = dest_path / f'{pdb_id}-no-ligand{split_ext}'
    ligand_info_filename = dest_path / f'{pdb_id}-ligand-info.json'
    if split_dest_filename.exists():
        assert ligand_info_filename.exists(), 'Ligand information filename not exists'
        with ligand_info_filename.open('r', encoding='utf-8') as f_info:
            ligand_info_dict = json.load(f_info)
            ligand_info = [LigandInfo(**d) for d in ligand_info_dict]
        return SplitComplexResult(
            target_filename=split_dest_filename,
            ligand_info_filename=ligand_info_filename,
            ligand_filenames=_ligand_filenames(),
            ligand_info=ligand_info,
        )

    try:
        structure = structure_file.get_pdb_structure(
            pdb_id, ext='.cif', pdb_cache_path=pdb_cache_path, mmcif_add_dict=True)
    except FileNotFoundError:
        logging.warning(f'Cannot find structure file of {pdb_id}.')
        return empty_result

    all_ccd_info = get_pdb_ccd_info(ccd_cache_path)
    ligand_info = _get_ligand_info(pdb_id, structure, all_ccd_info)
    with ligand_info_filename.open('w', encoding='utf-8') as f_info:
        ligand_info_dict = [asdict(info) for info in ligand_info]
        json.dump(ligand_info_dict, f_info, indent=4)

    _dump_split_complex(pdb_id, structure, ligand_info, dest_path=dest_path, ext=split_ext)

    return SplitComplexResult(
        target_filename=split_dest_filename,
        ligand_info_filename=ligand_info_filename,
        ligand_filenames=_ligand_filenames(),
        ligand_info=ligand_info,
    )

def split_pdb_complex_file(
        pdb_path: Path,
        split_save_path: Path = Path('.'),
        pdb_id: str = 'unknown',
        split_ext: str = '.pdb',
        ccd_cache_path: Path = None,
):
    """Split PDB file into split_save_path."""

    # TODO: Extract core code from `split_pdb_complex_paths` to here.

@dataclass
class SplitComplexPair:
    pdb_id: str
    chain_id: str
    res_id: int
    insertion_code: str
    site_ids: List[str]
    ligand_id: str
    chain_fasta: str
    ligand_inchi: str
    exist_in_uniprot: bool  # This ligand exists in one (or more) referenced UniProt chains.
    uniprot_ids: List[str]

    @classmethod
    def fields(cls):
        return [field.name for field in dataclasses.fields(cls)]

    @staticmethod
    def str2bool(s):
        if s == 'True':
            return True
        return False

    @staticmethod
    def str2list(s):
        if not s:
            return []
        return s.split(';')

    @classmethod
    def from_str_list(cls, str_list) -> 'SplitComplexPair':
        item = cls(*str_list)
        item.res_id = int(item.res_id)
        item.site_ids = cls.str2list(item.site_ids)
        item.exist_in_uniprot = cls.str2bool(item.exist_in_uniprot)
        item.uniprot_ids = cls.str2list(item.uniprot_ids)
        return item

    def to_str_list(self) -> List[str]:
        return [self.pdb_id, self.chain_id, str(self.res_id), self.insertion_code, ';'.join(self.site_ids),
                self.ligand_id, self.chain_fasta, self.ligand_inchi, str(self.exist_in_uniprot),
                ';'.join(self.uniprot_ids)]

_PAT_CARBON = re.compile(r'C |C$|C[0-9]+')

def _has_carbon(formula):
    match = _PAT_CARBON.search(formula)
    return match is not None

def _get_sites(
        mmcif_object: af2_mmcif_parsing.AF2MmcifObject,
        all_ccd_info: Dict[str, CcdInfo],
        query_chain_id: Optional[str],
):
    parsed_info = mmcif_object.raw_string
    pdb_id = mmcif_object.file_id

    # 1. Get all hetero items (some may not be ligands).
    #   [NOTE]: We only take non-polymers into consideration.
    ligands = af2_mmcif_parsing.mmcif_loop_to_dict('_chem_comp.', '_chem_comp.id', parsed_info)

    # 1.1 Filters
    #   1) Only accept non-polymer TODO: This may be relaxed in future (accept some polymers)?
    #   2) Remove some inorganic compounds (e.g. water)
    #   3) Must in CCD IDs
    #   4) Formula weight limited
    #   5) Remove all inorganic compounds (Carbon not in SMILES)
    filtered_ligands = {}
    for ccd_id, v in ligands.items():
        if v['_chem_comp.type'] != 'non-polymer':
            continue
        if ccd_id in config.SKIP_INORGANIC_LIGANDS:
            continue
        ccd_info = all_ccd_info.get(ccd_id, None)
        if ccd_info is None:
            continue
        if ccd_info.formula_weight > config.LIGAND_MAX_FORMULA_WEIGHT or math.isnan(ccd_info.formula_weight):
            continue
        if not _has_carbon(ccd_info.formula):
            continue
        filtered_ligands[ccd_id] = v
    ligands = filtered_ligands

    # 2. Get position of each ligand.
    # Key of `ligand_positions`: (ccd_id, chain_id, res_id, insertion_code)
    # [NOTE]:
    #   We use 'pdb_seq_num' (not 'auth_seq_num'), which is aligned to index mentioned in site details and
    #   other sections.
    # TODO: Need to change '_pdbx_nonpoly_scheme.pdb_seq_num' into '_pdbx_nonpoly_scheme.auth_seq_num'?
    # TODO: In some cases (e.g. 2dhb), pdb_mon_id != auth_mon_id, so we need to use pdb_mon_id (change from auth_mon_id) and rebuild the whole dataset.
    ligand_positions = af2_mmcif_parsing.mmcif_loop_to_dict(
        '_pdbx_nonpoly_scheme',
        ('_pdbx_nonpoly_scheme.auth_mon_id', '_pdbx_nonpoly_scheme.pdb_strand_id', '_pdbx_nonpoly_scheme.pdb_seq_num',
         '_pdbx_nonpoly_scheme.pdb_ins_code'),
        parsed_info)
    new_ligand_positions = {}
    for k, v in ligand_positions.items():
        ccd_id, chain_id, seq_id, insertion_code = k
        if query_chain_id is not None and ccd_id != query_chain_id:
            continue
        if ccd_id not in ligands:
            continue
        try:
            seq_num = int(seq_id)
        except ValueError:
            logging.warning(f'Cannot parse pdb_seq_num {k[2]}')
            continue
        insertion_code = norm_cif_empty_value(insertion_code)
        new_ligand_positions[(ccd_id, chain_id, seq_num, insertion_code)] = v
    ligand_positions = new_ligand_positions

    # 3. Get ligand - binding site relation (L2BS) information.
    ligand2site = {position: set() for position in ligand_positions}

    # 3.1. Get binding sites structures from '_struct_site_gen', and try to get ligand information from them.
    site_structures = af2_mmcif_parsing.mmcif_loop_to_dict('_struct_site_gen.', '_struct_site_gen.id', parsed_info)

    for site_part in site_structures.values():
        # Match each site to previous ligand positions.
        # [NOTE]:
        #   We need to use 'auth_XXX' instead of 'label_XXX' here (to align author IDs of ligands).
        #   In `extract_binding_sites.py`, we need to use 'label_XXX'.
        ccd_id = site_part['_struct_site_gen.auth_comp_id']
        chain_id = site_part['_struct_site_gen.auth_asym_id']
        res_id = int(site_part['_struct_site_gen.auth_seq_id'])
        bind_position = (
            ccd_id, chain_id, res_id,
            norm_cif_empty_value(site_part['_struct_site_gen.pdbx_auth_ins_code']),
        )
        if bind_position in ligand_positions:
            site_id = site_part['_struct_site_gen.site_id']
            ligand2site[bind_position].add(site_id)

    # 3.2. In some mmCIF files, _struct_site will include some L2BS information. E.g., 6uap.cif.
    site_infos = af2_mmcif_parsing.mmcif_loop_to_dict('_struct_site.', '_struct_site.id', parsed_info)
    for site_id, site_info in site_infos.items():
        ligand_ccd_id = site_info['_struct_site.pdbx_auth_comp_id']
        if is_empty(ligand_ccd_id):
            continue
        ligand_chain_id = site_info['_struct_site.pdbx_auth_asym_id']
        if is_empty(ligand_chain_id):
            continue
        ligand_res_id = site_info['_struct_site.pdbx_auth_seq_id']
        if is_empty(ligand_res_id):
            continue
        ligand_res_id = int(ligand_res_id)
        ligand_insertion_code = norm_cif_empty_value(site_info['_struct_site.pdbx_auth_ins_code'])
        bind_position = (ligand_ccd_id, ligand_chain_id, ligand_res_id, ligand_insertion_code)
        if bind_position in ligand_positions:
            ligand2site[bind_position].add(site_id)

    # 3.3. Get L2BS information from details string. E.g., 4fpc.cif.
    for site_id, site_info in site_infos.items():
        details = site_info['_struct_site.details']
        bind_position = None
        if details.startswith('BINDING SITE FOR RESIDUE '):
            if not 31 <= len(details) <= 35:
                continue
            words = details.split()
            if len(words) != 7:
                continue
            ccd_id, chain_id, res_id, insertion_code = words[4], words[5], words[6], '.'
            try:
                res_id = int(res_id.strip())
            except ValueError:
                continue
            bind_position = (ccd_id, chain_id, res_id, insertion_code)
        # TODO: Other patterns.
        if bind_position is not None and bind_position in ligand_positions and not ligand2site[bind_position]:
            # [NOTE]: Evidence from details have low priority. We accept it only when all other methods have failed.
            ligand2site[bind_position].add(site_id)

    # TODO: 3.4. If still cannot get binding sites, try to create it from nearest residues.

    pdb_db_ref = get_uniprot_ref(mmcif_object)

    pairs = []
    for position in ligand_positions:
        ccd_id, chain_id, res_id, insertion_code = position
        if chain_id not in mmcif_object.chain_to_seqres:
            # [NOTE]: Case in 6yws and 4xqn.
            logging.warning(
                f'{pdb_id} chain {chain_id} not found. Maybe this ligand is bounded to a non-protein sequence.')
            continue

        # 4. Get related UniProt IDs.
        uniprot_ids = []
        exist_in_uniprot = False
        for ref in pdb_db_ref.values():
            if ref.chain_id == chain_id and ref.seq_begin <= res_id <= ref.seq_end:
                uniprot_ids.append(ref.uniprot_id)
                exist_in_uniprot = True
        if not uniprot_ids:
            # [NOTE]: We cannot specify the detailed chain position, so we record all mentioned UniProt IDs.
            uniprot_ids = [ref.uniprot_id for ref in pdb_db_ref.values()]

        pairs.append(SplitComplexPair(
            pdb_id=pdb_id,
            chain_id=chain_id,
            res_id=res_id,
            insertion_code=insertion_code,
            site_ids=sorted(ligand2site[position]),
            ligand_id=ccd_id,
            chain_fasta=mmcif_object.chain_to_seqres[chain_id],
            ligand_inchi=all_ccd_info[ccd_id].inchi,
            exist_in_uniprot=exist_in_uniprot,
            uniprot_ids=uniprot_ids,
        ))
    return pairs

def get_target_ligand_pairs_from_pdb(
        pdb_id: str,
        query_chain_id: Optional[str] = None,
        pdb_cache_path: Path = None,
        ccd_cache_path: Path = None,
) -> List[SplitComplexPair]:
    """Get target-ligand pairs from PDB file.

    Args:
        pdb_id:
        query_chain_id: If specified, will only return ligands on this chain.
        pdb_cache_path:
        ccd_cache_path:

    Returns:
        List[SplitComplexPair]: List of pairs in this PDB.

    Notes:
        Internally use AF2MmcifObject.
    """
    pdb_id = pdb_id.lower()

    if pdb_cache_path is None:
        pdb_cache_path = config.pdb_cache_path()
    if ccd_cache_path is None:
        ccd_cache_path = config.pdb_ccd_path()

    try:
        mmcif_object = structure_file.get_af2_mmcif_object(pdb_id, pdb_cache_path)
    except FileNotFoundError as e:
        logging.warning(f'Cannot find structure file of {pdb_id}.')
        return []
    except AF2MmCIFParseError as e:
        logging.warning(f'Fail to parse mmCIF file of {pdb_id}.')
        return []

    if mmcif_object is None:
        # No valid chains.
        logging.warning(f'No protein chains found in mmCIF file {pdb_id}.')
        return []

    all_ccd_info = get_pdb_ccd_info(ccd_cache_path)
    pairs = _get_sites(mmcif_object, all_ccd_info, query_chain_id=query_chain_id)

    return pairs


import bpy
from mathutils import Matrix

from bpy.props import (
    StringProperty,
    BoolProperty,
    FloatProperty,
    EnumProperty,
    CollectionProperty,
)

from bpy_extras.io_utils import (
    ExportHelper,
)

from .. import export

class ExportPRC(bpy.types.Operator, ExportHelper):
    """Export a PearRay file"""
    bl_idname = "export_scene.prc"
    bl_label = "Export PearRay"
    bl_description = "Export to PearRay format"
    bl_options = {'UNDO'}

    filename_ext = ".prc"
    filter_glob: StringProperty(default="*.prc", options={'HIDDEN'})
    use_selection: BoolProperty(
        name="Selection Only",
        description="Export selected objects only",
        default=False)
    external_mesh_files: BoolProperty(
        name="External Mesh Files",
        description="Export meshes into their own prc file in generated/. Recommended",
        default=True)

    check_extension = True

    def draw(self, context):
        pass

    def execute(self, context):
        if not self.filepath:
            raise Exception("filepath not set")

        scene_exporter = export.Exporter(
            self.filepath, context.evaluated_depsgraph_get(),
            enforceTabs=True)
        scene_exporter.EXTERNAL_MESH_FILES = self.external_mesh_files
        scene_exporter.write_scene()

        return {'FINISHED'}

class PR_PT_export_include(bpy.types.Panel):
    bl_space_type = 'FILE_BROWSER'
    bl_region_type = 'TOOL_PROPS'
    bl_label = "Include"
    bl_parent_id = "FILE_PT_operator"

    @classmethod
    def poll(cls, context):
        sfile = context.space_data
        operator = sfile.active_operator

        return operator.bl_idname == "EXPORT_SCENE_OT_prc"

    def draw(self, context):
        layout = self.layout
        layout.use_property_split = True
        layout.use_property_decorate = False  # No animation.

        sfile = context.space_data
        operator = sfile.active_operator

        layout.prop(operator, 'use_selection')

class PR_PT_export_output(bpy.types.Panel):
    bl_space_type = 'FILE_BROWSER'
    bl_region_type = 'TOOL_PROPS'
    bl_label = "Output"
    bl_parent_id = "FILE_PT_operator"

    @classmethod
    def poll(cls, context):
        sfile = context.space_data
        operator = sfile.active_operator

        return operator.bl_idname == "EXPORT_SCENE_OT_prc"

    def draw(self, context):
        layout = self.layout
        layout.use_property_split = True
        layout.use_property_decorate = False  # No animation.

        sfile = context.space_data
        operator = sfile.active_operator

        layout.prop(operator, 'external_mesh_files')

_classes = [ExportPRC,
            PR_PT_export_include, PR_PT_export_output]
_bl_register, _bl_unregister = bpy.utils.register_classes_factory(_classes)

def menu_func_export(self, context):
    self.layout.operator(
        ExportPRC.bl_idname, text="PearRay (.prc)")

def register():
    _bl_register()
    bpy.types.TOPBAR_MT_file_export.append(menu_func_export)

def unregister():
    bpy.types.TOPBAR_MT_file_export.remove(menu_func_export)
    _bl_unregister()


import json
import os

from dotenv import load_dotenv
from utils import json_serializer
load_dotenv()

API_KEY = os.environ.get('API_KEY')
API_KEY_SECRET = os.environ.get('API_KEY_SECRET')
ACCESS_TOKEN = os.environ.get('ACCESS_TOKEN')
ACCESS_TOKEN_SECRET = os.environ.get('ACCESS_TOKEN_SECRET')

BEARER_TOKEN = os.environ.get('BEARER_TOKEN')

import requests
base_url = "https://api.twitter.com/2/"

from producer import get_producer

myproducer = get_producer()

"""
steps to build a stream
- Create a rule
- Add tag to rule
- Add rule to stream
- authenticate your request
- add rule to stream
- Identify and specify which fields you would like to retrieve
- Connect to the stream and review your response

"""

# rule creation
rule_url_suffix = "tweets/search/stream/rules"
stream_url_suffix = "tweets/search/stream"

def add_rules(value=None, tag=None):
    """
    input
        value: str
        tag: str

    output:
        on success: response from twitter api
    """

    sample_rules = [
        {"value": "databases", "tag": "databases pictures"},
        {"value": "plsql has:images", "tag": "plsql pictures"},
    ]

    if value == None:
        payload = {"add": sample_rules}
    else:
        payload = {"add": [{"value": value, "tag": tag if tag != None else ""}]}

    # payload = {"add": sample_rules}
    response = requests.post(
        f"{base_url}{rule_url_suffix}",
        headers= {
            'Content-type': 'application/json',
            'Authorization': f'Bearer {BEARER_TOKEN}'
        },
        data=json.dumps(payload)
    )
    if response.status_code != 201:
        raise Exception(
            f"Cannot add rules (HTTP Response Code - {response.status_code}): {response.text}"
        )
    print(json.dumps(response.json()))

def get_rules():
    response = requests.get(
        f"{base_url}{rule_url_suffix}",
        headers= {
            'Content-type': 'application/json',
            'Authorization': f'Bearer {BEARER_TOKEN}'
        },
    )
    if response.status_code != 200:
        raise Exception(
            f"Cannot get rules (HTTP Response Code - {response.status_code}): {response.text}"
        )
    print(json.dumps(response.json()))
    return response.json()

def delete_rules(rule_ids=None):
    """
    input:
        rule_ids -> list of rule ids
        if rule_ids == None, all rules will be deleted

    """
    try:
        if (type(rule_ids) != list) and (rule_ids != None):
            # print("Provide a varaible of type -> List")
            raise Exception(
                f"Invalid data type {type(rule_ids)} provide.\nProvide a varaible of type -> List"
            )

        if rule_ids == None:
            payload = {
                "delete" : {
                    "ids": [id['id'] for id in get_rules()['data']]
                }
            }
        else:
            payload = {
                "delete": {
                    "ids": rule_ids
                }
            }

        response = requests.post(
            f"{base_url}{rule_url_suffix}",
            headers= {
                'Content-type': 'application/json',
                'Authorization': f'Bearer {BEARER_TOKEN}'
            },
            data=json.dumps(payload)
        )
        if response.status_code != 200:
            raise Exception(
                f"Cannot delete rules (HTTP Response Code - {response.status_code}): {response.text}"
            )
        print(json.dumps(response.json()))
        return response.json()
    except Exception as e:
        print(e)
        return e

def start_stream():
    response = requests.get(
        f"{base_url}{stream_url_suffix}",
        headers= {
            'Content-type': 'application/json',
            'Authorization': f'Bearer {BEARER_TOKEN}'
        },
        stream=True
    )
    if response.status_code != 200:
        raise Exception(
            f"Cannot start stream (HTTP Response Code - {response.status_code}): {response.text}"
        )
    for response_line in response.iter_lines():
        if response_line:
            json_response = json.loads(response_line)
            print(json.dumps(json_response, indent=4, sort_keys=True))
            myproducer.produce(
                'python_topic',
                # value=json.dumps(json_response).encode('utf-8')
                value=json_serializer(json_response)
            )

# start_stream()

# # print("invoking delete rule function \n\n\n")
# delete_rules()
# # print("rules deleted")

# print('\n\n\n')
# # get_rules()
# # print (json.dumps(res))

# # print(type(["1568591011470778370"]))
# # import pprint
# # pprint.pprint(json.dumps(res), indent=4)
# # requests.post()

if __name__ == "__main__":
    # pass
    # delete_rules()
    # add_rules()
    add_rules(value="queen")
    # get_rules()

    start_stream()
    # print(start_stream())

# auth = tweepy.OAuthHandler(API_KEY, API_KEY_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

# api = tweepy.API(auth=auth)

# if api != None:
#     print("Authenticated successfully")

# class MyStreamListener(tweepy.Stream):

#     def on_status(self, status):
#         print(status.text)

    # def on_data(self, data):
    #     """
    #     input:
    #         data: JSON format
    #     """
    #     # send data through socket
    #     try:
    #         #
    #         message = json.loads( data )
    #         print(message)

    #     except Exception as e:
    #         pass
    #         #
    #     return True

    # def on_error(self, status_code):
    #     if status_code == 420:
    #         return False

# class MyStreamClient(tweepy.StreamingClient):
#     def on_data(self, data):
#         print (json.loads( data))

# class MyStreamRule(tweepy.StreamRule):

# if __name__ == "__main__":
    # stream = MyStreamListener(API_KEY, API_KEY_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)

    # stream.filter(track=['databases'])

    # stream = MyStreamClient(bearer_token=BEARER_TOKEN, return_type=dict)
    # stream.add_rules(tweepy.StreamRule('databases'))
    # stream.filter()


import sys
import gc
sys.path.append("..")

import netsquid as ns
import pandas
import pydynaa
import time

from util.RoutingComparison import run_centralized_routing_test, run_QCast_routing_test,run_SLMP_routing_test,run_greedy_routing_test
from matplotlib import pyplot as plt
"""
This program explores routing in equivalent parameter network.

The program take about 2 minutes to complete.
"""
if __name__ == '__main__':

    ################################
    ####  avg_loss_noisy test ###
    ###############################
    fidelity_data = pandas.DataFrame()
    centralized_routing_throughputs = []
    QCast_throughputs = []
    SLMP_throughputs = []
    greedy_throughputs = []
    qchannel_loss_noisy = []
    for i in range(32):
        qchannel_loss_noisy.append(0.001)

    depolar_rates = []
    for i in range(29):
        depolar_rates.append(0.1)

    dephase_rates = []
    for i in range(29):
        dephase_rates.append(0.1)

    qchannel_loss_init = []
    for i in range(32):
        qchannel_loss_init.append(0.01)

    ### run Centralized routing test ###
    fidelity_data_temp,centralized_routing_throughput, centralized_routing_time_cost = run_centralized_routing_test(depolar_rates,dephase_rates,qchannel_loss_init,qchannel_loss_noisy,300)
    fidelity_data = fidelity_data.append(fidelity_data_temp)
    centralized_routing_throughputs.append(centralized_routing_throughput)

    ### run Greedy routing test ###
    fidelity_data_temp,greedy_throughput, greedy_time_cost = run_greedy_routing_test(depolar_rates,dephase_rates,qchannel_loss_init,qchannel_loss_noisy,300)
    fidelity_data = fidelity_data.append(fidelity_data_temp)
    greedy_throughputs.append(greedy_throughput)

    ### run Q-Cast routing test ###
    fidelity_data_temp,QCast_throughput, QCast_time_cost = run_QCast_routing_test(depolar_rates,dephase_rates,qchannel_loss_init,qchannel_loss_noisy,300)
    fidelity_data = fidelity_data.append(fidelity_data_temp)
    QCast_throughputs.append(QCast_throughput)
    print(fidelity_data)

    ### run SLMP routing test ###
    fidelity_data_temp,SLMP_throughput, SLMP_time_cost = run_SLMP_routing_test(depolar_rates,dephase_rates,qchannel_loss_init,qchannel_loss_noisy,300)
    fidelity_data = fidelity_data.append(fidelity_data_temp)
    SLMP_throughputs.append(SLMP_throughput)

    fidelity_data_bar = fidelity_data.groupby("routing_algorithm")['fidelity'].agg(fidelity='mean', sem='sem').reset_index()
    entanglement_pair_consumption_data = fidelity_data.groupby("routing_algorithm")['entanglement_pair_consumption'].agg(entanglement_pair_consumption='mean').reset_index()
    throughput_data_bar = pandas.DataFrame([["Centralized_Routing",centralized_routing_throughputs[0]],
                                            ["Greedy",greedy_throughputs[0]],
                                           ["Q-Cast",QCast_throughputs[0]],
                                           ["SLMP",SLMP_throughputs[0]]],
                                           columns=['routing_algorithm', 'throughput'])

    fig,ax = plt.subplots(1,2,figsize=(25,10))
    plot_style = {'kind': 'bar', 'grid': True,
      'title': "Routing Algorithm Comparison in Equivalent Parameter Network"}
    ax[0].set_ylabel('fidelity')
    ax[1].set_ylabel('throughput(qps)')

    fidelity_data_bar.plot(x='routing_algorithm', y='fidelity',yerr='sem', ax=ax[0], **plot_style)
    ax[0].tick_params(axis='x', labelrotation=0)
    throughput_data_bar.plot(x='routing_algorithm', y='throughput', ax=ax[1], **plot_style)
    ax[1].tick_params(axis='x', labelrotation=0)

    print(fidelity_data_bar)
    print(entanglement_pair_consumption_data)
    print(throughput_data_bar)
    plt.show()


import requests
import torch
import urllib
import clip
from io import BytesIO
import keras_cv

from PIL import Image
from clip.model import build_model
import  tensorflow as tf
import  numpy as np
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
import matplotlib.pyplot as plt
import base64
try:
    from torchvision.transforms import InterpolationMode
    BICUBIC = InterpolationMode.BICUBIC
except ImportError:
    BICUBIC = Image.BICUBIC
def _convert_image_to_rgb(image):
    return image.convert("RGB")
from keras import backend
from keras.engine import base_layer
from keras.engine import base_preprocessing_layer
from keras.layers.preprocessing import preprocessing_utils as utils
from keras.utils import image_utils
from keras.utils import tf_utils
H_AXIS = -3
W_AXIS = -2
def smart_resize(x, size, interpolation='bicubic'):
    """Resize images to a target size without aspect ratio distortion.

    Warning: `tf.keras.preprocessing.image.smart_resize` is not recommended for
    new code. Prefer `tf.keras.layers.Resizing`, which provides the same
    functionality as a preprocessing layer and adds `tf.RaggedTensor` support.
    See the [preprocessing layer guide](
    https://www.tensorflow.org/guide/keras/preprocessing_layers)
    for an overview of preprocessing layers.

    TensorFlow image datasets typically yield images that have each a different
    size. However, these images need to be batched before they can be
    processed by Keras layers. To be batched, images need to share the same
    height and width.

    You could simply do:

    ```python
    size = (200, 200)
    ds = ds.map(lambda img: tf.image.resize(img, size))
    ```

    However, if you do this, you distort the aspect ratio of your images, since
    in general they do not all have the same aspect ratio as `size`. This is
    fine in many cases, but not always (e.g. for GANs this can be a problem).

    Note that passing the argument `preserve_aspect_ratio=True` to `resize`
    will preserve the aspect ratio, but at the cost of no longer respecting the
    provided target size. Because `tf.image.resize` doesn't crop images,
    your output images will still have different sizes.

    This calls for:

    ```python
    size = (200, 200)
    ds = ds.map(lambda img: smart_resize(img, size))
    ```

    Your output images will actually be `(200, 200)`, and will not be distorted.
    Instead, the parts of the image that do not fit within the target size
    get cropped out.

    The resizing process is:

    1. Take the largest centered crop of the image that has the same aspect
    ratio as the target size. For instance, if `size=(200, 200)` and the input
    image has size `(340, 500)`, we take a crop of `(340, 340)` centered along
    the width.
    2. Resize the cropped image to the target size. In the example above,
    we resize the `(340, 340)` crop to `(200, 200)`.

    Args:
      x: Input image or batch of images (as a tensor or NumPy array). Must be in
        format `(height, width, channels)` or `(batch_size, height, width,
        channels)`.
      size: Tuple of `(height, width)` integer. Target size.
      interpolation: String, interpolation to use for resizing. Defaults to
        `'bilinear'`. Supports `bilinear`, `nearest`, `bicubic`, `area`,
        `lanczos3`, `lanczos5`, `gaussian`, `mitchellcubic`.

    Returns:
      Array with shape `(size[0], size[1], channels)`. If the input image was a
      NumPy array, the output is a NumPy array, and if it was a TF tensor,
      the output is a TF tensor.
    """
    if len(size) != 2:
        raise ValueError(
            f"Expected `size` to be a tuple of 2 integers, but got: {size}."
        )
    img = tf.convert_to_tensor(x)
    if img.shape.rank is not None:
        if img.shape.rank < 3 or img.shape.rank > 4:
            raise ValueError(
                "Expected an image array with shape `(height, width, "
                "channels)`, or `(batch_size, height, width, channels)`, but "
                f"got input with incorrect rank, of shape {img.shape}."
            )
    shape = tf.shape(img)
    height, width = shape[-3], shape[-2]
    target_height, target_width = size
    if img.shape.rank is not None:
        static_num_channels = img.shape[-1]
    else:
        static_num_channels = None

    crop_height = tf.cast(
        tf.cast(width * target_height, "float32") / target_width, "int32"
    )
    crop_width = tf.cast(
        tf.cast(height * target_width, "float32") / target_height, "int32"
    )

    # Set back to input height / width if crop_height / crop_width is not
    # smaller.
    crop_height = tf.minimum(height, crop_height)
    crop_width = tf.minimum(width, crop_width)

    crop_box_hstart = tf.cast(
        tf.cast(height - crop_height, "float32") / 2, "int32"
    )
    crop_box_wstart = tf.cast(
        tf.cast(width - crop_width, "float32") / 2, "int32"
    )

    if img.shape.rank == 4:
        crop_box_start = tf.stack([0, crop_box_hstart, crop_box_wstart, 0])
        crop_box_size = tf.stack([-1, crop_height, crop_width, -1])
    else:
        crop_box_start = tf.stack([crop_box_hstart, crop_box_wstart, 0])
        crop_box_size = tf.stack([crop_height, crop_width, -1])

    img = tf.slice(img, crop_box_start, crop_box_size)
    img = tf.image.resize(images=img, size=size, method=interpolation,antialias=True)
    # Apparent bug in resize_images_v2 may cause shape to be lost
    if img.shape.rank is not None:
        if img.shape.rank == 4:
            img.set_shape((None, None, None, static_num_channels))
        if img.shape.rank == 3:
            img.set_shape((None, None, static_num_channels))
    if isinstance(x, np.ndarray):
        return img.numpy()
    return img
def convert_inputs(inputs, dtype=None):
    if isinstance(inputs, dict):
        raise ValueError(
            "This layer can only process a tensor representing an image or "
            f"a batch of images. Received: type(inputs)={type(inputs)}."
            "If you need to pass a dict containing "
            "images, labels, and bounding boxes, you should "
            "instead use the preprocessing and augmentation layers "
            "from `keras_cv.layers`. See docs at "
            "https://keras.io/api/keras_cv/layers/"
        )
    inputs = utils.ensure_tensor(inputs, dtype=dtype)
    return inputs
class Resizing(base_layer.Layer):
    """A preprocessing layer which resizes images.

    This layer resizes an image input to a target height and width. The input
    should be a 4D (batched) or 3D (unbatched) tensor in `"channels_last"`
    format. Input pixel values can be of any range
    (e.g. `[0., 1.)` or `[0, 255]`) and of integer or floating point dtype.
    By default, the layer will output floats.

    This layer can be called on tf.RaggedTensor batches of input images of
    distinct sizes, and will resize the outputs to dense tensors of uniform
    size.

    For an overview and full list of preprocessing layers, see the preprocessing
    [guide](https://www.tensorflow.org/guide/keras/preprocessing_layers).

    Args:
        height: Integer, the height of the output shape.
        width: Integer, the width of the output shape.
        interpolation: String, the interpolation method.
            Defaults to `"bilinear"`.
            Supports `"bilinear"`, `"nearest"`, `"bicubic"`, `"area"`,
            `"lanczos3"`, `"lanczos5"`, `"gaussian"`, `"mitchellcubic"`.
        crop_to_aspect_ratio: If True, resize the images without aspect
            ratio distortion. When the original aspect ratio differs
            from the target aspect ratio, the output image will be
            cropped so as to return the
            largest possible window in the image (of size `(height, width)`)
            that matches the target aspect ratio. By default
            (`crop_to_aspect_ratio=False`), aspect ratio may not be preserved.
    """

    def __init__(
        self,
        height,
        width,
        interpolation="bilinear",
        crop_to_aspect_ratio=False,
        **kwargs,
    ):
        self.height = height
        self.width = width
        self.interpolation = interpolation
        self.crop_to_aspect_ratio = crop_to_aspect_ratio
        self._interpolation_method = image_utils.get_interpolation(
            interpolation
        )
        super().__init__(**kwargs)
        base_preprocessing_layer.keras_kpl_gauge.get_cell("Resizing").set(True)

    def call(self, inputs):
        # tf.image.resize will always output float32
        # and operate more efficiently on float32
        # unless interpolation is nearest, in which case ouput type matches
        # input type.
        if self.interpolation == "nearest":
            input_dtype = self.compute_dtype
        else:
            input_dtype = tf.float32
        inputs = convert_inputs(inputs, dtype=input_dtype)
        size = [self.height, self.width]
        if self.crop_to_aspect_ratio:

            def resize_to_aspect(x):
                if tf_utils.is_ragged(inputs):
                    x = x.to_tensor()
                return smart_resize(
                    x, size=size, interpolation=self._interpolation_method
                )

            if tf_utils.is_ragged(inputs):
                size_as_shape = tf.TensorShape(size)
                shape = size_as_shape + inputs.shape[-1:]
                spec = tf.TensorSpec(shape, input_dtype)
                outputs = tf.map_fn(
                    resize_to_aspect, inputs, fn_output_signature=spec
                )
            else:
                outputs = resize_to_aspect(inputs)
        else:
            outputs = tf.image.resize(
                inputs, size=size, method=self._interpolation_method,antialias=True
            )
        return tf.cast(outputs, self.compute_dtype)

    def compute_output_shape(self, input_shape):
        input_shape = tf.TensorShape(input_shape).as_list()
        input_shape[H_AXIS] = self.height
        input_shape[W_AXIS] = self.width
        return tf.TensorShape(input_shape)

    def get_config(self):
        config = {
            "height": self.height,
            "width": self.width,
            "interpolation": self.interpolation,
            "crop_to_aspect_ratio": self.crop_to_aspect_ratio,
        }
        base_config = super().get_config()
        return dict(list(base_config.items()) + list(config.items()))
class Base64DecoderLayer(tf.keras.layers.Layer):
    """
    Convert a incoming base 64 string into an bitmap with rgb values between 0 and 1
    target_size e.g. [width,height]
    """

    def __init__(self, target_size):
        self.target_size = target_size
        self.mean = tf.constant([0.48145466, 0.4578275, 0.40821073])
        self.std = tf.constant([0.26862954, 0.26130258, 0.27577711])
        self.resize_layer=Resizing(target_size[0],target_size[0],    interpolation='bicubic',
crop_to_aspect_ratio=True)

        super(Base64DecoderLayer, self).__init__()

    def byte_to_img(self, byte_tensor):
        # base64 decoding id done by tensorflow serve, when using b64 json
        byte_tensor = tf.io.decode_base64(byte_tensor)

        imgs_map = tf.io.decode_image(byte_tensor, channels=3)

        imgs_map.set_shape((None, None, 3))
        #imgs_map=tf.image.resize(imgs_map, [224, 224],method='bicubic',antialias=True,preserve_aspect_ratio=True,)
        imgs_map = self.resize_layer(imgs_map)
        #imgs_map =keras_cv.layers.Resizing( 224, 224,interpolation='bicubic',crop_to_aspect_ratio=True)(imgs_map)
        img = tf.cast(imgs_map, dtype=tf.float32) / 255
        img = tf.math.subtract(img, self.mean)
        img = tf.math.divide(img, self.std)
        return img

    def call(self, input, **kwargs):
        with tf.device("/cpu:0"):
            imgs_map = tf.map_fn(self.byte_to_img, input, dtype=tf.float32)
        return imgs_map
def _transform(n_px):
    return Compose([
        Resize(n_px, interpolation=BICUBIC),
        CenterCrop(n_px),
        _convert_image_to_rgb,
        ToTensor(),
        Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
    ])

preprocess= _transform(224)
image = preprocess(
  # Image.open(requests.get(image_url).content)
   Image.open('/Volumes/Data/oysterqaq/Desktop/108557696_p0_master1200.jpg')
).unsqueeze(0)

print(image.shape)
image = image.permute(0, 2, 3, 1).detach().numpy()
plt.imshow(image[0])
plt.show()
print(image)

pic = open("/Volumes/Data/oysterqaq/Desktop/108557696_p0_master1200.jpg", "rb")
pic_base64 = base64.urlsafe_b64encode(pic.read())

img = Base64DecoderLayer([224, 224])(tf.stack([tf.convert_to_tensor(pic_base64)]))

plt.imshow(img[0])
plt.show()
plt.imshow(img[0]-image[0])
plt.show()
print(img.shape)
print((img[0]-image[0])[0])

from PIL import Image
im = Image.open(r"/Volumes/Data/oysterqaq/Desktop/108557696_p0_master1200.jpg")

min_width = 224

# 计算新高度
width, height = im.size
new_height = int(height * (min_width / width))

# 缩小图像并保持原始比例
im = im.resize((min_width, new_height),Image.BICUBIC)
width, height = im.size
left = (width - min_width) // 2
top = (height - min_width) // 2
right = (width + min_width) // 2
bottom = (height + min_width) // 2

# 裁剪图像
im = im.crop((left, top, right, bottom))

im=tf.keras.utils.img_to_array(im)
mean = tf.constant([0.48145466, 0.4578275, 0.40821073])
std = tf.constant([0.26862954, 0.26130258, 0.27577711])
im = tf.cast(im, dtype=tf.float32) / 255
im = tf.math.subtract(im, mean)
im = tf.math.divide(im, std)
plt.imshow(im-image[0])
plt.show()

print(np.isclose(image[0][0],
                    img[0].numpy()[0],
                     atol=1e-5))
import base64