"""
    :module name: retrievers
    :module summary: endpoint classes that deal with data retrieval
    :module author: Nathan Mendoza (nathancm@uci.edu)
"""

import logging
import hashlib
import secrets
import datetime
import json
from typing import Union, List

from sqlalchemy.exc import NoResultFound
from fastapi import status
from treelib import Tree
import jwt

from . import BaseEndpointHandler
from ..config import ServiceConfig
from ..services import RegisteredService
from ..models import (
        CredentialSet,
        AccountRecord,
        AuthorizationToken,
        ProfileView,
        ConceptRequest,
        ConceptSimpleView,
        ConceptFullView,
        ConceptSearchQuery,
        ConceptLineage,
        AccountFollowingRecord,
        ConceptLikingRecord,
        ConceptComment,
        ConceptCommentThreads,
        EndpointErrorMessage,
        EndpointInformationalMessage,
        EndpointResponse,
        )
from ..exceptions import (
        InvalidCredentialsException,
        BaseIdeaBankAPIException,
        RequestedDataNotFound
    )

LOGGER = logging.getLogger(__name__)

class AuthenticationHandler(BaseEndpointHandler):
    """Endpoint handler dealing into account authenticaiton"""

    def _do_data_ops(self, request: CredentialSet) -> str:
        try:
            LOGGER.info(
                    "Looking up account information: %s",
                    request.display_name
                    )
            with self.get_service(RegisteredService.ACCOUNTS_DS) as service:
                service.add_query(service.fetch_authentication_information(
                        display_name=request.display_name,
                    ))
                service.exec_next()
                result = service.results.one()
                self.__verify_credentials(request, result)
                return result.display_name
        except NoResultFound as err:
            LOGGER.error(
                    "No account record found: %s",
                    request.display_name
                    )
            raise InvalidCredentialsException(
                    "Invalid display name or password"
                    ) from err

    def _build_success_response(self, requested_data: str):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=AuthorizationToken(
                    token=self.__generate_token(requested_data),
                    presenter=requested_data
                    )
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):
        if isinstance(exc, InvalidCredentialsException):
            self._result = EndpointResponse(
                    code=status.HTTP_401_UNAUTHORIZED,
                    body=EndpointErrorMessage(
                        err_msg=str(exc)
                        )
                    )
        else:
            super()._build_error_response(exc)

    def __verify_credentials(
            self,
            provided: CredentialSet,
            oracle: AccountRecord
            ):
        LOGGER.info("Comparing provided credentials to stored")
        provided_hash = hashlib.sha256(
                f'{provided.password}{oracle.salt_value}'.encode('utf-8')
                ).hexdigest()
        if secrets.compare_digest(provided_hash, oracle.password_hash):
            LOGGER.debug("Provided credentials match")
            return
        LOGGER.debug("Provided credentials did not match records")
        raise InvalidCredentialsException("Invalid display name or password")

    def __generate_token(self, owner: str) -> str:
        claims = {
                'username': owner,
                'exp': datetime.datetime.utcnow() + datetime.timedelta(days=7),
                'nbf': datetime.datetime.utcnow() + datetime.timedelta(seconds=2)
                }
        return jwt.encode(
                claims,
                ServiceConfig.AuthKey.JWT_SIGNER,
                ServiceConfig.AuthKey.JWT_HASHER
                )

class ProfileRetrievalHandler(BaseEndpointHandler):
    """Endpoint handler dealing with profile retrievals"""

    def _do_data_ops(self, request: str) -> ProfileView:
        try:
            LOGGER.info("Looking up profile information: %s", request)
            with self.get_service(RegisteredService.ACCOUNTS_DS) as service:
                service.add_query(service.fetch_account_profile(
                    display_name=request
                    ))
                service.exec_next()
                result = service.results.one()
                return ProfileView(
                    preferred_name=result.preferred_name,
                    biography=result.biography,
                    avatar_url=service.share_item(f'avatars/{request}')
                        )
        except NoResultFound as err:
            LOGGER.error("No account record found: %s", request)
            raise RequestedDataNotFound(
                    f'Profile for {request} is not available'
                    ) from err

    def _build_success_response(self, requested_data: ProfileView):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):
        if isinstance(exc, RequestedDataNotFound):
            self._result = EndpointResponse(
                    code=status.HTTP_404_NOT_FOUND,
                    body=EndpointErrorMessage(
                        err_msg=str(exc)
                        )
                    )
        else:
            super()._build_error_response(exc)

class SpecificConceptRetrievalHandler(BaseEndpointHandler):
    """Handler for dealing with requests for a particular concept"""

    def _do_data_ops(self, request: ConceptRequest) -> ConceptSimpleView:
        LOGGER.info(
                "Searching for specific concept: %s/%s",
                request.author,
                request.title
                )
        try:
            with self.get_service(RegisteredService.CONCEPTS_DS) as service:
                service.add_query(service.find_exact_concept(
                    title=request.title,
                    author=request.author
                    ))
                service.exec_next()
                result = service.results.one()
                if request.simple:
                    return ConceptSimpleView(
                            identifier=f'{result.author}/{result.title}',
                            thumbnail_url=service.share_item(
                                f'thumbnails/{result.author}/{result.title}'
                                )
                            )
                return ConceptFullView(
                        author=result.author,
                        title=result.title,
                        description=result.description,
                        diagram=json.dumps(result.diagram),
                        thumbnail_url=service.share_item(
                            f'thumbnails/{result.author}/{result.title}'
                            )
                        )
        except NoResultFound as err:
            LOGGER.error(
                    "Did not find a concept matching %s/%s",
                    request.author,
                    request.title
                    )
            raise RequestedDataNotFound(
                    f"No match for `{request.author}/{request.title}`"
                    ) from err

    def _build_success_response(self, requested_data: Union[ConceptFullView, ConceptSimpleView]):
        LOGGER.info("Found a matching concept")
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):
        if isinstance(exc, RequestedDataNotFound):
            LOGGER.error("Did not find a matching concept")
            self._result = EndpointResponse(
                    code=status.HTTP_404_NOT_FOUND,
                    body=EndpointErrorMessage(err_msg=str(exc))
                    )
        else:
            super()._build_error_response(exc)

class ConceptSearchResultHandler(BaseEndpointHandler):
    """Handler for dealing with search queries for relevant queries"""

    def _do_data_ops(self, request: ConceptSearchQuery) -> List[ConceptSimpleView]:
        LOGGER.info(
                "Searching for concepts matching the following criteria:\n%s",
                request.json(indent=4)
                )
        with self.get_service(RegisteredService.CONCEPTS_DS) as service:
            service.add_query(service.query_concepts(
                author=request.author,
                title=request.title,
                not_before=request.not_before,
                not_after=request.not_after,
                fuzzy=request.fuzzy
                ))
            service.exec_next()
            return [
                    ConceptSimpleView(
                        identifier=result.identifier,
                        thumbnail_url=service.share_item(
                            f'thumbnails/{result.identifier}'
                            )

                        )
                    for result in service.results.all()
                    ]

    def _build_success_response(self, requested_data: List[ConceptSimpleView]):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):  # pylint:disable=useless-parent-delegation
        super()._build_error_response(exc)

class ConceptLineageHandler(BaseEndpointHandler):
    """Endpoint handler for dealing with concept lineage retrieval"""

    def _do_data_ops(self, request: ConceptRequest) -> ConceptLineage:
        LOGGER.info(
                "Building lineage for `%s/%s`",
                request.author,
                request.title
                )
        focus = f'{request.author}/{request.title}'
        try:
            with self.get_service(RegisteredService.CONCEPTS_DS) as service:
                service.add_query(service.find_exact_concept(
                    author=request.author,
                    title=request.title
                    ))
                service.exec_next()
                service.results.one()  # Ensure it exists
                lineage = Tree()
                lineage.create_node(
                    tag=focus,
                    identifier=focus,
                    data=ConceptSimpleView(
                        identifier=focus,
                        thumbnail_url=service.share_item(
                            f'thumbnails/{focus}'
                            )
                        )
                    )
                service.add_query(service.find_parent_ideas(
                    identifier=focus,
                    depth=10
                    ))
                service.exec_next()
                for parent in service.results.all():
                    new_lineage = Tree()
                    new_lineage.create_node(
                            tag=parent.ancestor,
                            identifier=parent.ancestor,
                            data=ConceptSimpleView(
                                identifier=parent.ancestor,
                                thumbnail_url=service.share_item(
                                    f'thumbnails/{parent.ancestor}'
                                    )
                                )
                            )
                    new_lineage.paste(parent.ancestor, lineage)
                    lineage = new_lineage

                service.add_query(service.find_child_ideas(
                    identifier=focus,
                    depth=10
                    ))
                service.exec_next()
                for child in service.results.all():
                    lineage.create_node(
                            tag=child.descendant,
                            identifier=child.descendant,
                            parent=child.ancestor,
                            data=ConceptSimpleView(
                                identifier=child.descendant,
                                thumbnail_url=service.share_item(
                                    f'thumbnails/{child.descendant}'
                                    )
                                )
                            )
            return ConceptLineage(
                    nodes=lineage.size(),
                    lineage=lineage.to_dict(with_data=True)
                    )
        except NoResultFound as err:
            LOGGER.error(
                    "No concept record found for `%s/%s`. Unable to build lineage",
                    request.author,
                    request.title
                    )
            raise RequestedDataNotFound(
                    f"Could not build the lineage for {request.author}/{request.title}"
                    ) from err

    def _build_success_response(self, requested_data: ConceptLineage):
        LOGGER.info("Lineage successfully obtained")
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseException):
        if isinstance(exc, RequestedDataNotFound):
            LOGGER.error("No lineage tree resulted from build process")
            self._result = EndpointResponse(
                    code=status.HTTP_404_NOT_FOUND,
                    body=EndpointErrorMessage(
                        err_msg=str(exc)
                        )
                    )
        else:
            super()._build_error_response(exc)

class CheckFollowingStatusHandler(BaseEndpointHandler):
    """Endpoint handler dealing with checking if a user follows another"""

    def _do_data_ops(self, request: AccountFollowingRecord) -> EndpointInformationalMessage:
        LOGGER.info(
                "Checking if %s follows %s",
                request.follower,
                request.followee
                )
        try:
            with self.get_service(RegisteredService.ENGAGE_DS) as service:
                service.add_query(service.check_following(
                    follower=request.follower,
                    followee=request.followee
                    ))
                service.exec_next()
                service.results.one()
                return EndpointInformationalMessage(
                        msg=f"{request.follower} is following {request.followee}"
                        )
        except NoResultFound as err:
            LOGGER.error("Could not find a matching follow record")
            raise RequestedDataNotFound(
                    f"{request.follower} is not following {request.followee}"
                    ) from err

    def _build_success_response(self, requested_data: EndpointInformationalMessage):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):
        if isinstance(exc, RequestedDataNotFound):
            self._result = EndpointResponse(
                    code=status.HTTP_404_NOT_FOUND,
                    body=EndpointErrorMessage(
                        err_msg=str(exc)
                        )
                    )
        else:
            super()._build_error_response(exc)

class CheckLikingStatusHandler(BaseEndpointHandler):
    """Endpoint handler dealing with checking if a user likes a concept"""

    def _do_data_ops(self, request: ConceptLikingRecord) -> EndpointInformationalMessage:
        LOGGER.info(
                "Checking if %s likes %s",
                request.user_liking,
                request.concept_liked
                )
        try:
            with self.get_service(RegisteredService.ENGAGE_DS) as service:
                service.add_query(service.check_liking(
                    account=request.user_liking,
                    concept=request.concept_liked
                    ))
                service.exec_next()
                service.results.one()
                return EndpointInformationalMessage(
                        msg=f"{request.user_liking} does like {request.concept_liked}"
                        )
        except NoResultFound as err:
            LOGGER.error("Could not find a matching liking record")
            raise RequestedDataNotFound(
                    f"{request.user_liking} does not like {request.concept_liked}"
                    ) from err

    def _build_success_response(self, requested_data: EndpointInformationalMessage):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):
        if isinstance(exc, RequestedDataNotFound):
            self._result = EndpointResponse(
                    code=status.HTTP_404_NOT_FOUND,
                    body=EndpointErrorMessage(
                        err_msg=str(exc)
                        )
                    )
        else:
            super()._build_error_response(exc)

class ConceptCommentsSectionHandler(BaseEndpointHandler):
    """Endpoint handler for retrieving the comments section of a concept"""

    def _do_data_ops(self, request: ConceptRequest) -> ConceptCommentThreads:
        comment_tree = ConceptCommentThreads(threads=[])
        comment_tree.threads.extend(self.__thread_starts(
            concept_id=f'{request.author}/{request.title}'
            ))
        for thread in comment_tree.threads:
            self.__gather_responses(
                    concept_id=f'{request.author}/{request.title}',
                    current_thread=thread
                    )
        return comment_tree

    def _build_success_response(self, requested_data: ConceptCommentThreads):
        self._result = EndpointResponse(
                code=status.HTTP_200_OK,
                body=requested_data
                )

    def _build_error_response(self, exc: BaseIdeaBankAPIException):  # pylint:disable=useless-parent-delegation
        super()._build_error_response(exc)

    def __thread_starts(self, concept_id: str):
        with self.get_service(RegisteredService.ENGAGE_DS) as service:
            service.add_query(service.comments_on(concept_id))
            service.exec_next()
            return [
                    ConceptComment(
                        comment_id=c.comment_id,
                        comment_author=c.comment_by,
                        comment_text=c.free_text
                        )
                    for c in service.results.all()
                    ]

    def __gather_responses(self, concept_id: str, current_thread: ConceptComment):
        with self.get_service(RegisteredService.ENGAGE_DS) as service:
            service.add_query(service.comments_on(
                concept_id=concept_id,
                response_to=current_thread.comment_id
                ))
            service.exec_next()
            current_thread.responses.extend(
                    [
                        ConceptComment(
                            comment_id=c.comment_id,
                            comment_author=c.comment_by,
                            comment_text=c.free_text
                            )
                        for c in service.results.all()
                        ]
                    )
        for subthread in current_thread.responses:
            self.__gather_responses(concept_id, subthread)


from datahub.common.const import PARTITION, PENDING
from datahub.storekit.models import (
    StorageClusterConfig,
    StorageCronTask,
    StorageDataLakeLoadTask,
    StorageResultTable,
    StorageScenarioConfig,
)
from datahub.storekit.serializers import (
    ClusterSerializer,
    DataLakeLoadTaskSerializer,
    ResultTableSerializer,
    StorageScenarioConfigSerializer,
)
from datahub.storekit.settings import DJANGO_BATCH_CREATE_OBJS, STORAGE_RT_ACTIVE_TRUE

def get_storage_rts_by_type(cluster_type):
    """
    根据存储类型找到所有的关联此类存储的rt，所有对象serializer后放入列表返回
    :param cluster_type: 存储类型
    :return: 包含此类存储的rt列表
    """
    data = [ResultTableSerializer(i).data for i in get_storage_rt_objs_by_type(cluster_type)]
    return data

def get_all_storage_rt_objs(active=STORAGE_RT_ACTIVE_TRUE):
    """
    获取所有的rt存储配置信息列表
    :param active: 数据的active状态
    :return: 包含存储的rt对象的结果集
    """
    return StorageResultTable.objects.filter(active=active).order_by("-id")

def get_storage_rt_objs_by_rt(result_table_id, active=STORAGE_RT_ACTIVE_TRUE):
    """
    根据rt找到相关的存储配置信息列表
    :param result_table_id: rt名称
    :param active: 数据的active状态
    :return: rt存储配置列表
    """
    return StorageResultTable.objects.filter(result_table_id=result_table_id).filter(active=active).order_by("-id")

def get_storage_rt_objs_by_rt_type(result_table_id, cluster_type, active=STORAGE_RT_ACTIVE_TRUE):
    """
    根据rt找到相关的存储配置信息列表
    :param result_table_id: rt名称
    :param cluster_type: 存储类型
    :param active: 数据的active状态
    :return: rt存储配置
    """
    return (
        StorageResultTable.objects.filter(result_table_id=result_table_id)
        .filter(storage_cluster_config__cluster_type=cluster_type)
        .filter(active=active)
        .order_by("-id")
    )

def get_all_storage_rt_objs_by_rt_type(result_table_id, cluster_type):
    """
    根据rt找到相关的存储配置信息列表
    :param result_table_id: rt名称
    :param cluster_type: 存储类型
    :return: rt存储配置
    """
    return (
        StorageResultTable.objects.filter(result_table_id=result_table_id)
        .filter(storage_cluster_config__cluster_type=cluster_type)
        .order_by("-id")
    )

def get_all_channel_objs_by_rt(result_table_id):
    """
    根据rt找到channel相关的配置信息
    :param result_table_id: rt名称
    :return: rt的channel存储配置
    """
    return (
        StorageResultTable.objects.filter(result_table_id=result_table_id)
        .filter(storage_cluster_config_id=None)
        .order_by("-id")
    )

def get_storage_rt_objs_by_type(cluster_type, active=STORAGE_RT_ACTIVE_TRUE):
    """
    根据存储类型找到所有的关联此类存储的rt对象，可以逐个遍历，避免一次把所有数据加载到内存中。
    :param cluster_type: 存储类型
    :param active: 数据的active状态
    :return: 包含此类存储的rt对象的结果集
    """
    return (
        StorageResultTable.objects.filter(storage_cluster_config__cluster_type=cluster_type)
        .filter(active=active)
        .order_by("-id")
    )

def get_storage_rt_objs_by_name_type(cluster_name, cluster_type, active=STORAGE_RT_ACTIVE_TRUE):
    """
    根据集群名称和存储类型找到所有的关联此类存储的rt对象，可以逐个遍历，避免一次把所有数据加载到内存中。
    :param cluster_name: 存储集群名称
    :param cluster_type: 存储类型
    :param active: 数据的active状态
    :return: 包含此类存储的rt对象的结果集
    """
    return (
        StorageResultTable.objects.filter(
            storage_cluster_config__cluster_name=cluster_name, storage_cluster_config__cluster_type=cluster_type
        )
        .filter(active=active)
        .order_by("-id")
    )

def get_storage_cluster_configs_by_type(cluster_type):
    """
    根据存储集群类型获取所有此类型集群的配置列表
    :param cluster_type: 集群类型
    :return: 指定存储类型的存储集群配置列表
    """
    data = [ClusterSerializer(i).data for i in get_cluster_objs_by_type(cluster_type)]
    return data

def get_storage_cluster_conf_by_group(cluster_type=None, cluster_group=None):
    """
    根据存储集群类型获取所有此类型集群的配置列表
    :param cluster_type: 集群类型
    :param cluster_group: 集群组名称
    :return: 指定存储类型的存储集群配置列表
    """
    if cluster_type and cluster_group:
        objs = StorageClusterConfig.objects.filter(cluster_type=cluster_type).filter(cluster_group=cluster_group)
    elif cluster_type:
        objs = StorageClusterConfig.objects.filter(cluster_type=cluster_type)
    elif cluster_group:
        objs = StorageClusterConfig.objects.filter(cluster_group=cluster_group)
    else:
        objs = StorageClusterConfig.objects.all()
    data = [ClusterSerializer(i).data for i in objs]
    return data

def get_cluster_objs_by_type(cluster_type=None):
    """
    按照存储类型获取此类型的存储集群结果集，当传入参数为None时，获取所有存储类型的集群对象
    :param cluster_type: 集群类型
    :return: 集群对象的结果集
    """
    if cluster_type:
        return StorageClusterConfig.objects.filter(cluster_type=cluster_type)
    else:
        return StorageClusterConfig.objects.all()

def get_cluster_objs_by_id(cluster_id):
    """
    按照存储id获取此类型的存储集群结果集
    :param cluster_id: 集群id
    :return: 集群对象的结果集
    """
    return StorageClusterConfig.objects.get(id=cluster_id)

def get_cluster_obj_by_name_type(cluster_name, cluster_type):
    """
    按照存储集群名称和类型获取指定的集群对象
    :param cluster_name: 集群名称
    :param cluster_type: 集群类型
    :return: 存储集群对象
    """
    objs = get_cluster_objs_by_type(cluster_type).filter(cluster_name=cluster_name)
    return objs[0] if objs else None

def get_storage_cluster_config(cluster_name, cluster_type):
    """
    根据集群名称和类型查找存储集群配置对象
    :param cluster_name: 集群名称
    :param cluster_type: 集群类型
    :return: 存储集群配置对象
    """
    obj = get_cluster_obj_by_name_type(cluster_name, cluster_type)
    return ClusterSerializer(obj).data if obj else None

def update_cron_task(task_id, status, end_time):
    """
    更新定时维护任务状态
    :param task_id: 定时任务的id
    :param status: 状态
    :param end_time: 结束时间
    """
    task = StorageCronTask.objects.get(id=task_id)
    task.status = status if status else task.status
    task.end_time = end_time if end_time else task.end_time
    task.save()

def batch_update_cron_task(tasks, status, end_time):
    """
    更新定时维护任务状态
    :param tasks: 任务列表
    :param status: 状态
    :param end_time: 结束时间
    """
    for task in tasks:
        task.status = status if status else task.status
        task.end_time = end_time if end_time else task.end_time
        task.save()

def get_running_cron_task_by_type(task_type):
    """
    根据定时任务类型获取正在执行的定时任务
    :param task_type: 定时任务类型
    :return: 执行中的一个定时任务
    """
    tasks = StorageCronTask.objects.filter(task_type=task_type).filter(status="running").order_by("-id")
    if tasks:
        return tasks[0]
    else:
        return None

def get_cron_task_by_id(task_id):
    """
    根据任务id获取任务信息
    :param task_id: 任务id
    :return: 任务信息
    """
    tasks = StorageCronTask.objects.filter(id=task_id).order_by("-id")
    return tasks[0] if tasks else None

def get_running_cron_task():
    """
    获取正在执行的定时任务
    :return: 正在执行的定时任务列表
    """
    tasks = StorageCronTask.objects.filter(status="running").order_by("-id")
    return tasks if tasks else None

def get_scenario_configs_by_name_type(name=None, type=None):
    """
    根据名称和类型查找存储场景的配置信息
    :param name: 名称
    :param type: 类型
    :return: 符合条件的配置信息列表
    """
    objs = get_scenario_objs_by_name_type(name, type)
    data = [StorageScenarioConfigSerializer(i).data for i in objs]
    return data

def get_scenario_objs_by_name_type(name=None, type=None):
    """
    根据名称和类型查找存储场景的配置信息，作为对象返回
    :param name: 名称
    :param type: 类型
    :return: 符合条件的对象列表
    """
    if name and type:
        objs = StorageScenarioConfig.objects.filter(storage_scenario_name=name).filter(cluster_type=type)
    elif name:
        objs = StorageScenarioConfig.objects.filter(storage_scenario_name=name)
    elif type:
        objs = StorageScenarioConfig.objects.filter(cluster_type=type)
    else:
        objs = StorageScenarioConfig.objects.all()
    return objs

def bulk_create_load_task(datalake_load_tasks):
    """
    :param datalake_load_tasks: 创建任务任务
    """
    batch_tasks = [
        datalake_load_tasks[i : i + DJANGO_BATCH_CREATE_OBJS]
        for i in range(0, len(datalake_load_tasks), DJANGO_BATCH_CREATE_OBJS)
    ]
    for tasks in batch_tasks:
        StorageDataLakeLoadTask.objects.bulk_create(tasks)

def get_load_task_by_id(task_id):
    """
    根据任务id获取任务信息
    :param task_id: 任务id
    :return: 任务信息
    """
    tasks = StorageDataLakeLoadTask.objects.filter(id=task_id)
    return DataLakeLoadTaskSerializer(tasks[0]).data if tasks else None

def get_load_task_conditions(result_table_id, hdfs_cluster_name, geog_area, limit):
    """
    根据任务id获取任务信息
    :param result_table_id: 结果表
    :param hdfs_cluster_name: hdfs集群
    :param geog_area: 区域
    :param limit: 限制条数
    :return: 任务信息列表
    """
    order = f"{PARTITION}"
    if result_table_id and hdfs_cluster_name:
        objs = StorageDataLakeLoadTask.objects.filter(result_table_id=result_table_id).filter(
            hdfs_cluster_name=hdfs_cluster_name
        )
    elif result_table_id:
        objs = StorageDataLakeLoadTask.objects.filter(result_table_id=result_table_id)
    elif hdfs_cluster_name:
        objs = StorageDataLakeLoadTask.objects.filter(hdfs_cluster_name=hdfs_cluster_name)
    else:
        objs = StorageDataLakeLoadTask.objects

    if geog_area:
        objs = objs.filter(geog_area=geog_area)

    objs = objs.filter(status=PENDING).order_by(order)[0:limit]
    return [DataLakeLoadTaskSerializer(i).data for i in objs]

def get_pending_task_count_by_rt(result_table_id):
    """
    根据pending任务个数
    :param result_table_id: 结果表
    :return: 任务信息列表
    """
    return StorageDataLakeLoadTask.objects.filter(result_table_id=result_table_id).filter(status=PENDING).count()

def get_task_by_rt(result_table_id, order, status=None):
    """
    根据任务id获取任务信息
    :param result_table_id: 结果表
    :param status: 状态
    :param order: 速
    :return: 任务信息列表
    """
    objs = StorageDataLakeLoadTask.objects.filter(result_table_id=result_table_id)
    if status:
        objs = objs.filter(status=status)
    objs = objs.order_by(order)[0:1]

    return DataLakeLoadTaskSerializer(objs[0]).data if objs else None

def update_load_task_status(id, status):
    """
    更新任务状态
    :param id: 任务id
    :param status: 状态
    """
    return StorageDataLakeLoadTask.objects.filter(id=id).update(status=status)


import os
from typing import Dict

import config
from fsm_parse import parse_json, extract_nodes, extract_edges
from verilog_utils.verilog_always import gen_verilog_always
from verilog_utils.verilog_case import gen_verilog_case
from verilog_utils.verilog_if import gen_verilog_if_chain
from verilog_utils.verilog_module_deceleration import gen_verilog_module_deceleration

class VerilogTransformer:

    def __init__(self, in_path, out_path):
        self.outputs_net = []
        self.inputs_net = []
        self.out_path = out_path

        data = parse_json(in_path)
        self.nodes = extract_nodes(data)
        self.edges = extract_edges(data)

        self.resolve_output_nets()
        self.resolve_input_nets()

    def run(self):
        functions = [
            self.gen_module_declaration(),
            self.gen_intermediate_signals(),
            self.gen_positive_clock_action(),
            self.gen_next_state_case(),
            self.gen_output()
        ]
        out = '\n'.join(functions) + "endmodule"

        if self.out_path is None:
            print(out)

        else:
            if os.path.exists(self.out_path):
                with open(self.out_path, 'w') as file:
                    file.write(out)
            else:
                # Check if the directory of the file path exists
                directory = os.path.dirname(self.out_path)
                if not os.path.exists(directory):
                    # Create the directory if it does not exist
                    if directory != '':
                        os.makedirs(directory)

                # Create the file if it does not exist
                if not os.path.exists(self.out_path):
                    with open(self.out_path, 'w') as file:
                        file.write(out)

    def resolve_input_nets(self) -> None:
        for l in self.edges.values():
            for edge in l:
                for key in edge.inputs:
                    if key not in self.inputs_net:
                        self.inputs_net.append(key)

    def resolve_output_nets(self) -> None:
        for node in self.nodes.values():
            for key in node.outputs:
                if key not in self.outputs_net:
                    self.outputs_net.append(key)

    def gen_module_declaration(self):
        nodes_num = len(self.nodes)

        in_bits_num = len(self.inputs_net) - 1
        in_net_size = f"[{in_bits_num}:0]" if in_bits_num > 0 else None

        out_bits_num = len(self.outputs_net) - 1
        out_sig_size = f"[{out_bits_num}:0]" if out_bits_num > 0 else None

        return gen_verilog_module_deceleration(
            module_name=config.MODULE_NAME,
            parameters=[("N_STATE", nodes_num)],
            inputs=[
                (None, config.CLOCK_SIG_NAME),
                (None, config.RESET_SIG_NAME),
                (in_net_size, config.IN_NET_SIG_NAME),
            ],
            outputs=[
                ("reg", out_sig_size, config.OUT_REG_SIG_NAME)
            ]
        )

    def gen_intermediate_signals(self):
        intermediate_signals: str = ""
        # define state and next state
        nodes_bits_num = (len(self.nodes) - 1).bit_length() - 1
        if nodes_bits_num <= 0:
            intermediate_signals += (f"reg {config.INTERMEDIATE_CURRENT_STATE_REG_NAME},"
                                     f" {config.INTERMEDIATE_NEXT_STATE_REG_NAME};\n")
        else:
            intermediate_signals += (f"reg [{nodes_bits_num}:0] {config.INTERMEDIATE_CURRENT_STATE_REG_NAME}, "
                                     f"{config.INTERMEDIATE_NEXT_STATE_REG_NAME};\n")

        return intermediate_signals

    @staticmethod
    def gen_positive_clock_action():
        reset_cond, reset_stm = f"{config.RESET_SIG_NAME}", f"{config.INTERMEDIATE_CURRENT_STATE_REG_NAME} <= 0;"
        empty_cond, else_stm = "", (f"{config.INTERMEDIATE_CURRENT_STATE_REG_NAME} <="
                                    f" {config.INTERMEDIATE_NEXT_STATE_REG_NAME};")
        body = gen_verilog_if_chain(
            "\t",
            (reset_cond, reset_stm),
            (empty_cond, else_stm)
        )

        return gen_verilog_always('', f"posedge {config.CLOCK_SIG_NAME}", body)

    def gen_next_state_case(self):
        cases = []
        for node_id, edges in self.edges.items():
            node = self.nodes[node_id]
            if_chain = []
            for edge in edges:
                in_seq = get_seq(src_obj=self.inputs_net, obj=edge.inputs)
                if_case = (
                    f"{config.IN_NET_SIG_NAME} == {in_seq}",
                    f"{config.INTERMEDIATE_NEXT_STATE_REG_NAME} = {self.nodes[edge.dest_id].verilog_id};"
                )
                if_chain.append(if_case)

            cases.append((node.verilog_id, gen_verilog_if_chain('\t', *if_chain)))

        return gen_verilog_always('', f"{config.IN_NET_SIG_NAME} or {config.INTERMEDIATE_CURRENT_STATE_REG_NAME}",
                                  gen_verilog_case('\t', config.INTERMEDIATE_CURRENT_STATE_REG_NAME, cases))

    def gen_output(self):

        cases = []
        for _, node in self.nodes.items():
            out_seq = get_seq(src_obj=self.outputs_net, obj=node.outputs)
            cases.append(
                (
                    f"{config.INTERMEDIATE_CURRENT_STATE_REG_NAME}=={node.verilog_id}",
                    f"{config.OUT_REG_SIG_NAME} = {out_seq};"
                )
            )

        return gen_verilog_always('', f"{config.INTERMEDIATE_CURRENT_STATE_REG_NAME}",
                                  gen_verilog_if_chain('\t', *cases))

def get_seq(src_obj, obj: Dict[str, str]):
    result_seq = ['0'] * len(src_obj)

    for i in range(len(src_obj)):
        name = src_obj[i]
        if name in obj.keys():
            result_seq[i] = obj[name]

    return f"{len(result_seq)}'b" + "".join([str(elem) for elem in result_seq])


#

# ba_meta require api 7
from __future__ import annotations

import ba
import _ba
from ba._language import Lstr
from ba._stats import Stats as sta, PlayerScoredMessage, PlayerRecord
import settings, coinsystem
from ba._error import print_exception
from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import Any, Sequence

def player_scored(self,
                  player: ba.Player,
                  base_points: int = 1,
                  target: Sequence[float] | None = None,
                  kill: bool = False,
                  victim_player: ba.Player | None = None,
                  scale: float = 1.0,
                  color: Sequence[float] | None = None,
                  title: str | ba.Lstr | None = None,
                  screenmessage: bool = True,
                  display: bool = True,
                  importance: int = 1,
                  showpoints: bool = True,
                  big_message: bool = False) -> int:
    """Register a score for the player.

    Return value is actual score with multipliers and such factored in.
    """
    # FIXME: Tidy this up.
    # pylint: disable=cyclic-import
    # pylint: disable=too-many-branches
    # pylint: disable=too-many-locals
    # pylint: disable=too-many-statements
    from bastd.actor.popuptext import PopupText
    from ba import _math
    from ba._gameactivity import GameActivity
    name = player.getname()
    s_player = self._player_records[name]

    if kill:
        s_player.submit_kill(showpoints=showpoints)

    display_color: Sequence[float] = (1.0, 1.0, 1.0, 1.0)
    if color is not None:
        display_color = color
    elif importance != 1:
        display_color = (1.0, 1.0, 0.4, 1.0)
    points = base_points

    # If they want a big announcement, throw a zoom-text up there.
    if display and big_message:
        try:
            assert self._activity is not None
            activity = self._activity()
            if isinstance(activity, GameActivity):
                name_full = player.getname(full=True, icon=False)
                activity.show_zoom_message(
                    Lstr(resource='nameScoresText',
                         subs=[('${NAME}', name_full)]),
                    color=_math.normalized_color(player.team.color))
        except Exception:
            print_exception('error showing big_message')

        # If we currently have a actor, pop up a score over it.
    if display and showpoints:
        our_pos = player.node.position if player.node else None
        if our_pos is not None:
            if target is None:
                target = our_pos

            # If display-pos is *way* lower than us, raise it up
            # (so we can still see scores from dudes that fell off cliffs).
            display_pos = (target[0], max(target[1], our_pos[1] - 2.0),
                           min(target[2], our_pos[2] + 2.0))
            activity = self.getactivity()
            if activity is not None:
                if title is not None:
                    sval = Lstr(value='+${A} ${B}',
                                subs=[('${A}', str(points)),
                                      ('${B}', title)])
                else:
                    sval = Lstr(value='+${A}',
                                subs=[('${A}', str(points))])
                PopupText(sval,
                          color=display_color,
                          scale=1.2 * scale,
                          position=display_pos).autoretain()
    # Tally kills.
    if kill:
        s_player.accum_kill_count += 1
        s_player.kill_count += 1

    # Report non-kill scorings.
    try:
        if screenmessage and not kill:
            _ba.screenmessage(Lstr(resource='nameScoresText',
                                   subs=[('${NAME}', name)]),
                              top=True,
                              color=player.color,
                              image=player.get_icon())
    except Exception:
        print_exception('error announcing score')

    s_player.score += points
    s_player.accumscore += points

    # Inform a running game of the score.
    if points != 0:
        activity = self._activity() if self._activity is not None else None
        if activity is not None:
            activity.handlemessage(PlayerScoredMessage(score=points))

    id = player._sessionplayer.get_v1_account_id()
    try:
        sett = settings.get_settings_data()
        if sett["currency"]["enabled"]:
            coinsystem.add_coins_by_pbid(id, 2)
    except Exception:
        print_exception("Cant add coins")
        pass

    return points

def enable():
    sta.player_scored = player_scored


# from __future__ import print_function
from Network.disentangle import Disentangle_Network
from Data.read_nyc import load_npy_data
from Data.read_taxibj import load_taxibj
from keras.callbacks import ModelCheckpoint, EarlyStopping
from Network.metrics import metrics
import tensorflow as tf
import numpy as np
import argparse
import random
import os

parser = argparse.ArgumentParser(description='MUSE-Net')
parser.add_argument('--dataset', type=str, default='BikeNYC', help='BikeNYC, TaxiNYC or TaxiBJ')
parser.add_argument('--device', type=str, default='0')
parser.add_argument('--epoch', type=int, default=350)
parser.add_argument('--batch_size', type=int, default=8)
parser.add_argument('--iterate', type=int, default=1)
parser.add_argument('--train_ratio', type=float, default=0.9)
parser.add_argument('--days_test', type=int, default=20)
parser.add_argument('--len_closeness', type=int, default=3)
parser.add_argument('--len_period', type=int, default=4)
parser.add_argument('--len_trend', type=int, default=4)
parser.add_argument('--feat_dim', type=int, default=64)
parser.add_argument('--mu_dim', type=int, default=128)

args = parser.parse_args()
print(args)

# for GPU in Lab
device = args.device
os.environ["CUDA_VISIBLE_DEVICES"] = device
import setproctitle

setproctitle.setproctitle('MUSE-Net Start! @ Jianyang Qin')

NO = 4
# for reproduction
seed = 1
for i in range(NO):
    seed = seed * 10 + 7
seed = seed * 10 + 7

def seed_tensorflow(seed):
    os.environ['PYTHONHASHSEED'] = '0'
    random.seed(seed)
    np.random.seed(seed)
    tf.set_random_seed(seed)

from keras import backend as K

K.set_image_data_format('channels_first')

# hyperparameters
epoch = args.epoch                  # number of epoch at training stage
batch_size = args.batch_size        # batch size
train_ratio = args.train_ratio      # train ratio
lr = 0.0002                         # learning rate

days_test = args.days_test
len_closeness = args.len_closeness  # length of closeness dependent sequence
len_period = args.len_period        # length of peroid dependent sequence
len_trend = args.len_trend          # length of trend dependent sequence

feat_dim = args.feat_dim            # feature dimension
mu_dim = args.mu_dim                # distribution dimension

iterate_num = args.iterate          # the number of repetition and if retrain the model

def seed_tensorflow(seed):
    os.environ['PYTHONHASHSEED'] = '0'
    random.seed(seed)
    np.random.seed(seed)
    tf.set_random_seed(seed)

class CustomStopper(EarlyStopping):
    # add argument for starting epoch
    def __init__(self, monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto', start_epoch=40):
        super().__init__(monitor=monitor, min_delta=min_delta, patience=patience, verbose=verbose, mode=mode)
        self.start_epoch = start_epoch

    def on_epoch_end(self, epoch, logs=None):
        if epoch > self.start_epoch:
            super().on_epoch_end(epoch, logs)

if __name__ == "__main__":
    setproctitle.setproctitle('MUSE-Net @ Jianyang Qin')  # from V1707

    if args.dataset == 'BikeNYC':
        H, W, channel = 10, 20, 2   # grid size
        T = 24 * 2                  # number of time intervals in one day
        len_test = T * days_test
        T_closeness, T_period, T_trend = 1, T, T * 7

        X_train, Y_train, X_test, Y_test, MM, Max, Min = \
            load_npy_data('Data/bike_flow.npy', len_test, len_closeness, len_period, len_trend, T_closeness,
                          T_period, T_trend)
    elif args.dataset == 'TaxiNYC':
        H, W, channel = 10, 20, 2  # grid size
        T = 24 * 2  # number of time intervals in one day
        len_test = T * days_test
        T_closeness, T_period, T_trend = 1, T, T * 7

        X_train, Y_train, X_test, Y_test, MM, Max, Min = \
            load_npy_data('Data/taxi_flow.npy', len_test, len_closeness, len_period, len_trend, T_closeness,
                          T_period, T_trend)
    elif args.dataset == 'TaxiBJ':
        H, W, channel = 32, 32, 2  # grid size
        T = 24 * 2  # number of time intervals in one day
        len_test = T * days_test
        T_closeness, T_period, T_trend = 1, T, T * 7

        X_train, Y_train, X_test, Y_test, MM, Max, Min = \
            load_taxibj('Data/BJ13_M32x32_T30_InOut.h5', len_test, len_closeness, len_period, len_trend, T_closeness,
                          T_period, T_trend, T)
    else:
        raise ValueError('dataset does not exist')

    X_train = np.concatenate((X_train[0], X_train[1], X_train[2]), axis=1)
    X_test = np.concatenate((X_test[0], X_test[1], X_test[2]), axis=1)

    Y_train = np.concatenate([Y_train, X_train], axis=1)
    Y_test = np.concatenate([Y_test, X_test], axis=1)

    import time

    count = 0
    count_sum = iterate_num

    Pickup_RMSE = np.zeros([iterate_num, 1])
    Pickup_MAE = np.zeros([iterate_num, 1])
    Pickup_MAPE = np.zeros([iterate_num, 1])
    Dropoff_RMSE = np.zeros([iterate_num, 1])
    Dropoff_MAE = np.zeros([iterate_num, 1])
    Dropoff_MAPE = np.zeros([iterate_num, 1])

    for iterate_index in range(iterate_num):
        seed_tensorflow(seed)

        count = count + 1

        print("***** conv_model *****")
        model = Disentangle_Network(batch_size=batch_size, H=H, W=W,
                                    channel=channel, c=len_closeness, p=len_period, t=len_trend,
                                    feat_dim=feat_dim, conv=feat_dim, mu_dim=mu_dim,
                                    R_N=2, plus=8, rate=1, drop=0.1, lr=lr)

        file_conv = 'Exps/MUSE-Net_' + args.dataset + '_iter' + str(iterate_index) + '.hdf5'

        # train
        model_checkpoint = ModelCheckpoint(
            filepath=file_conv,
            monitor='val_pickup_rmse',
            verbose=1,
            save_best_only=True,
            save_weights_only=True,
            mode='min',
            period=1
        )

        stop = CustomStopper(monitor='val_pickup_rmse', min_delta=0, patience=30, verbose=0, mode='min', start_epoch=40)

        print('=' * 10)
        print("***** training conv_model *****")
        train_start = time.time()
        history = model.fit(X_train, Y_train,
                            epochs=epoch,
                            batch_size=batch_size,
                            validation_split=(1-train_ratio),
                            callbacks=[model_checkpoint, stop],
                            verbose=0,
                            shuffle=True)
        train_end = time.time()

        # val
        print('=' * 10)
        print('***** evaluate *****')
        val_start = time.time()
        model.load_weights(file_conv)

        train_score = model.evaluate(X_train, Y_train, batch_size=batch_size, verbose=0)
        val_end = time.time()

        print('Outflow:    RMSE    MAE | Inflow:    RMSE    MAE')
        print('Train score:', end=' ')
        np.set_printoptions(precision=6, suppress=True)
        print(np.array(train_score))

        # test
        test_start = time.time()
        test_score = model.evaluate(X_test, Y_test, batch_size=batch_size, verbose=0)
        test_end = time.time()
        print('Test  score:', end=' ')
        np.set_printoptions(precision=6, suppress=True)
        print(np.array(test_score))

        Pickup_RMSE[iterate_index, 0] = test_score[1] * MM / 2
        Pickup_MAE[iterate_index, 0] = test_score[2] * MM / 2
        Dropoff_RMSE[iterate_index, 0] = test_score[4] * MM / 2
        Dropoff_MAE[iterate_index, 0] = test_score[5] * MM / 2

        for_show = np.concatenate([Pickup_RMSE, Pickup_MAE, Dropoff_RMSE, Dropoff_MAE], axis=1)
        np.set_printoptions(precision=4, suppress=True)
        print('Outflow:    RMSE    MAE | Inflow:    RMSE    MAE')
        print(for_show)

        np.save('Exps/MUSE-Net_eval_score_' + args.dataset + '.npy', [Pickup_RMSE, Pickup_MAE, Dropoff_RMSE, Dropoff_MAE])


from typing import TYPE_CHECKING

import pytest
import pytest_asyncio

from simnet.client.components.calculator.genshin import CalculatorClient
from simnet.client.components.chronicle.genshin import GenshinBattleChronicleClient

if TYPE_CHECKING:
    from simnet.client.cookies import Cookies
    from simnet.utils.enum_ import Region

@pytest_asyncio.fixture
async def calculator_client(genshin_player_id: int, account_id: int, region: "Region", cookies: "Cookies"):
    if genshin_player_id is None:
        pytest.skip("Test case test_starrail_battle_chronicle_client skipped: No genshin player id set.")
    async with CalculatorClient(
        player_id=genshin_player_id,
        cookies=cookies,
        account_id=account_id,
        region=region,
    ) as client_instance:
        yield client_instance

@pytest_asyncio.fixture
async def genshin_battle_chronicle_client(
    genshin_player_id: int, account_id: int, region: "Region", cookies: "Cookies"
):
    async with GenshinBattleChronicleClient(
        player_id=genshin_player_id,
        cookies=cookies,
        account_id=account_id,
        region=region,
    ) as client_instance:
        yield client_instance

@pytest.mark.asyncio
class TestCalculatorClient:
    @staticmethod
    async def test_character_details(
        calculator_client: "CalculatorClient", genshin_battle_chronicle_client: "GenshinBattleChronicleClient"
    ):
        characters = await genshin_battle_chronicle_client.get_genshin_characters()
        character_details = await calculator_client.get_character_details(characters[-1].id)
        assert len(character_details.talents) == 6
        for talent in character_details.talents:
            assert talent.level > -1
        for artifact in character_details.artifacts:
            assert artifact.level >= 0


import oneflow as flow
import oneflow.core.serving.saved_model_pb2 as saved_model_pb
import numpy as np
import requests
from PIL import Image
from io import BytesIO
import google.protobuf.text_format as text_format
import os
from imagenet1000_clsidx_to_labels import clsidx_2_labels
from logger import Logger
from service.abstract_inference_service import AbstractInferenceService

log = Logger().logger

class OneFlowInferenceService(AbstractInferenceService):
    """
    oneflow 框架推理service
    """
    def __init__(self, args):
        super().__init__()
        self.args = args
        self.model_name = args.model_name
        self.model_path = args.model_path
        flow.clear_default_session()
        self.infer_session = flow.SimpleSession()
        self.load_model()

    def load_image(self, image_path):
        global im
        if image_path.startswith("http"):
            response = requests.get(image_path)
            response = response.content
            BytesIOObj = BytesIO()
            BytesIOObj.write(response)
            im = Image.open(BytesIOObj)
        else:
            im = Image.open(image_path)

        input_names = self.infer_session.list_inputs()
        batch_size, channel, height, width = self.infer_session.input_info(input_names[0])["shape"]
        im = im.resize((height, width))
        im = im.convert('RGB')  # 有的图像是单通道的，不加转换会报错
        im = np.array(im).astype('float32')
        im = (im - [123.68, 116.779, 103.939]) / [58.393, 57.12, 57.375]
        im = np.transpose(im, (2, 0, 1))
        im = np.expand_dims(im, axis=0)
        log.info("===============> load image success <===============")
        im = np.ascontiguousarray(im, 'float32')
        images = np.repeat(im, batch_size, axis=0).astype(np.float32)
        return images

    def load_model(self):
        log.info("===============> start load oneflow model :" + self.model_path + " <===============")
        model_meta_file_path = os.path.join(
            self.model_path, "saved_model.prototxt"
        )
        # load saved model
        saved_model_proto = saved_model_pb.SavedModel()
        with open(model_meta_file_path, "rb") as f:
            text_format.Merge(f.read(), saved_model_proto)

        checkpoint_path = os.path.join(
            self.model_path, saved_model_proto.checkpoint_dir[0]
        )

        self.infer_session.set_checkpoint_path(checkpoint_path)
        for job_name, signature in saved_model_proto.signatures_v2.items():
            self.infer_session.setup_job_signature(job_name, signature)

        for job_name, net in saved_model_proto.graphs.items():
            with self.infer_session.open(job_name) as session:
                session.compile(net.op)
        self.infer_session.launch()
        log.info("===============> load oneflow model success <===============")
        return saved_model_proto

    def inference(self, image):
        data = {"data_name": image['data_name']}
        log.info("===============> start load " + image['data_name'] + " <===============")
        images = self.load_image(image['data_path'])

        predictions = self.infer_session.run('inference', image=images)

        top_k_idx = predictions[0][0]
        top_5_idx = top_k_idx.argsort()[::-1][0:5]
        data['predictions'] = list()
        for label_id in top_5_idx:
            result = {"label": clsidx_2_labels[label_id], "probability": "{:.3f}".format(top_k_idx[label_id])}
            data['predictions'].append(result)
        return data


from typing import Union, Dict, List
import numpy as np
from qutip import Qobj
from ..systems import QubitSystem, Qubit
from .linalg import get_mat_elem

def to_oper(state_vec: Union[np.ndarray, Qobj]) -> Union[np.ndarray, Qobj]:
    if isinstance(state_vec, Qobj):
        if state_vec.type != "ket":
            raise ValueError(
                "state_vec type expected to be 'ket', "
                "instead got {}".format(state_vec.type)
            )
        return state_vec * state_vec.dag()
    if isinstance(state_vec, np.ndarray):
        oper = np.einsum("i, j-> ij", state_vec, state_vec.conj())
        return oper
    raise ValueError(
        "state_vec expected to be np.ndarray or qutip.Qobj "
        "instance, instead got {}".format(type(state_vec))
    )

def generalized_rabi_frequency(
    levels: List['str'],
    eps: Union[Dict, float],
    drive_frequency: float,
    system: Union[QubitSystem, Qubit]
) -> float:
    if len(levels) != 2:
        raise ValueError('The generalized rabi frequency is defined betwen two levels. '
                        'Please specify the desired two levels.')

    in_energy, in_state = system.state(levels[0])
    out_energy, out_state = system.state(levels[1])

    if isinstance(system, QubitSystem):
        qubit_labels = system.labels
        drive_op = 0

        for label in qubit_labels:
            drive_op += eps[label]*system[label].charge_op()

    elif isinstance(system, Qubit):
        drive_op = eps*system.charge_op()

    energy_transition = np.abs(in_energy - out_energy)
    drive_detuning = energy_transition - drive_frequency

    big_omega_transition = np.abs(get_mat_elem(drive_op, in_state, out_state))

    return np.sqrt(big_omega_transition**2 + drive_detuning**2)


import tweepy
from dateutil import parser
import time
import uuid
import json
import util

# Tweepy class to access Twitter API
class Streamlistener(tweepy.StreamListener):
    # Set dataset
    dataset = []

    def on_connect(self):
        # Get streamming for 60 seconds and exit
        self.start_time = time.time()
        self.limit = 60

        util.logger.info('You are connected to the Twitter API')

    def on_error(self, status):
        if status != 200:
            util.logger.error('error found: {}'.format(status) )
            # Returning false disconnects the stream
            return False

    # Reads in tweet data as Json and extracts the data we want.
    def on_data(self,data):
        if (time.time() - self.start_time) < self.limit:
            try:
                raw_data = json.loads(data)

                if 'text' in raw_data:
                    username = raw_data['user']['screen_name']
                    created_at = parser.parse(raw_data['created_at'])
                    tweet = raw_data['text']
                    retweets = raw_data['retweet_count']
                    location = raw_data['user']['location']

                    if raw_data['place'] is not None:
                        place = raw_data['place']['country']
                        util.logger.info(place)
                    else:
                        place = None

                    util.logger.info('USERNAME: %s \n TWEET: %s \n RETWEET: %d \n LOCATION: %s',username, tweet, retweets, location)
                    util.logger.info(' Tweet colleted at: {} \n'.format(str(created_at)))

                    # appends data in a structured format
                    self.dataset.append({
                        'key':uuid.uuid4(),
                        'username':username,
                        'tweet':tweet,
                        'retweets':retweets,
                        'location':location,
                        'created_at':str(created_at)})

            except BaseException as e:
                util.logger.error(e)
        else:
            return False


#!/usr/bin/env python
"""
elb-dance
======

elb-dance is used to deregister and register instances with ELB.
It can register and block until the instance is back "In Service".

"""

from setuptools import setup, find_packages
from elb_dance.version import VERSION

tests_require = [
    'mock',
    'pep8',
]

install_requires = ['boto>=2.24.0']

setup(
    name='elb-dance',
    version=VERSION,
    author='Ron Cohen',
    author_email='ron@opbeat.com',
    url='http://github.com/opbeat/elb-dance',
    description='elb-dance is used to deregister and register instances with ELB',
    long_description=__doc__,
    packages=find_packages(exclude=("tests",)),
    zip_safe=False,
    install_requires=install_requires,
    tests_require=tests_require,
    include_package_data=True,
    entry_points={
        'console_scripts': [
            'elb-dance = elb_dance.cli:cli_entry',
        ],
    },
    classifiers=[
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Operating System :: OS Independent',
        'Topic :: Software Development'
    ],
)


#! /usr/bin/env python

import pymongo

# 取点赞最多的前10条短评
with pymongo.MongoClient(host = "192.168.0.105") as client:
    comments = client.douban.movie_26752088_comments

    for doc in comments.find().sort([("vote", -1)]).limit(10):
        print("author = {}, date = {}, vote = {}, comment = {}".format(
            doc.get("author"),
            doc.get("date"),
            doc.ger("vote"),
            doc.get("comment")
        ))


import os

from troposphere import Template, Ref, Parameter
from troposphere.ec2 import SecurityGroup, Instance

t = Template("Create EC2")
t.set_version()

bucket_name = t.add_parameter(Parameter(
    'BucketName',
    Type='String'
))

security_group_ingress = []

cidrs = ["10.0.1.0/24",
         "10.0.2.0/24",
         "10.0.3.0/24",
         "10.0.4.0/24",
         "10.0.5.0/24",
         "10.0.10.0/24"]
for cidr_ip in cidrs:
    security_group_ingress.append({'FromPort': 22,
                                   'ToPort': 22,
                                   'IpProtocol': 'tcp',
                                   'CidrIp': cidr_ip})

security_group = t.add_resource(SecurityGroup(
    'XKESecurityGroup',
    SecurityGroupIngress=security_group_ingress,
    VpcId='vpc-1234',
    GroupDescription='Enable SSH access to the EC2 machine'
))

environment = os.getenv('ENVIRONMENT', 'dev')

def get_instance_type():
    if environment == 'dev':
        return 't2.micro'
    elif environment == 'prod':
        return 'm5.large'
    raise ValueError("Environment {} unknown".format(environment))

t.add_resource(Instance(
    'XKEInstance',
    ImageId='ami-951945d0',
    DisableApiTermination=True,
    InstanceType=get_instance_type(),
    KeyName='xke-key',
    SecurityGroupIds=[Ref(security_group)],
    SubnetId='subnet-1234',
))

print(t.to_yaml())


'''font.py: Class to manage individual fonts.'''
import os
import codecs
from typing import Dict, Any, Optional

from fontTools.ttLib import TTFont
from fonty.lib.variants import FontAttribute
from fonty.lib.font_name_ids import FONT_NAMEID_FAMILY, FONT_NAMEID_FAMILY_PREFFERED, \
                                    FONT_NAMEID_VARIANT, FONT_NAMEID_VARIANT_PREFFERED
from .font_format import FontFormat

class Font(object):
    '''Class to manage individual fonts.'''

    # Class Properties ------------------------------------------------------- #
    path_to_font: str
    family: str
    variant: FontAttribute
    name_table: Optional[Dict[Any, Any]] = None

    # Constructor ------------------------------------------------------------ #
    def __init__(
            self,
            path_to_font: str,
            family: str = None,
            variant: FontAttribute = None
        ) -> None:
        self.path_to_font = path_to_font

        # Get family name
        self.family = family if family else self.get_family_name()

        # Get variant
        self.variant = variant if variant else self.get_variant()

    # Class Methods ----------------------------------------------------------- #
    def install(self):
        '''Installs this font to the system.'''
        from fonty.lib.install import install_fonts

        # Install the font on to the system
        installed_font = install_fonts(self)

        return installed_font

    def generate_filename(self, ext: str = None) -> str:
        '''Generate a suitable filename from this font's name tables.'''
        family_name = self.get_family_name()
        variant = self.get_variant()

        if ext is None:
            _, ext = os.path.splitext(self.path_to_font)
            ext = ext if ext is not '' else '.otf' # Fallback to .otf

        return '{family}-{variant}{ext}'.format(
            family=family_name,
            variant=variant.print(long=True),
            ext=ext
        )

    def parse(self) -> 'Font':
        '''Parse the font's metadata from the font's name table.'''
        if not self.path_to_font or not os.path.isfile(self.path_to_font):
            raise Exception

        font = TTFont(file=self.path_to_font)
        if self.name_table is None:
            self.name_table = {}

        # Parse font file and retrieve family name and variant
        for record in font['name'].names:
            # Decode bytes
            if b'\x00' in record.string:
                data = record.string.decode('utf-16-be')
            elif b'\xa9' in record.string:
                data = codecs.decode(record.string, errors='ignore')
            else:
                data = codecs.decode(record.string, errors='ignore')

            self.name_table[str(record.nameID)] = data

        return self

    def get_name_data_from_id(self, name_id: str) -> str:
        '''Gets data from the font's name table via the name id.'''
        if self.name_table is None:
            self.parse()
        return self.name_table.get(name_id, None)

    def get_family_name(self) -> str:
        '''Get family name from the font's name tables.'''
        if self.name_table is None:
            self.parse()

        family_name = self.get_name_data_from_id(FONT_NAMEID_FAMILY)
        family_name_preferred = self.get_name_data_from_id(FONT_NAMEID_FAMILY_PREFFERED)

        return family_name_preferred if family_name_preferred else family_name

    def get_variant(self) -> FontAttribute:
        '''Get the font attributes from the font's name tables.'''
        if self.name_table is None:
            self.parse()

        variant = self.get_name_data_from_id(FONT_NAMEID_VARIANT)
        variant_preferred = self.get_name_data_from_id(FONT_NAMEID_VARIANT_PREFFERED)
        variant = variant_preferred if variant_preferred else variant

        return FontAttribute.parse(variant)

    def convert(self, path: str, font_format: 'FontFormat' = None) -> str:
        '''Converts this font to either woff or woff2 formats.'''
        _, ext = os.path.splitext(os.path.basename(self.path_to_font))
        font = TTFont(file=self.path_to_font)

        # Get font flavor
        if font_format:
            if font_format == FontFormat.WOFF:
                font.flavor = 'woff'
                ext = '.woff'
            elif font_format == FontFormat.WOFF2:
                font.flavor = 'woff2'
                ext = '.woff2'
            else:
                raise Exception # Only woff and woff2 supported for now

        # Create output directory if it doesn't exist
        path = os.path.abspath(path)
        if not os.path.exists(path):
            os.makedirs(path, exist_ok=True)
        if os.path.isdir(path):
            path = os.path.join(path, '') # Append trailing slash

        # Generate output paths
        output_path = os.path.join(os.path.dirname(path), self.generate_filename(ext))

        # Convert and save
        font.save(file=output_path)

        return output_path


from django import forms
import datetime
from .models import Akun

class AkunForm(forms.ModelForm):

    nama = forms.CharField()
    foto_src = forms.CharField(widget=forms.HiddenInput, required=False)
    kota_domisili = forms.CharField()
    alamat = forms.CharField(widget=forms.Textarea, required=False)
    jenis_kelamin = forms.ChoiceField(widget=forms.RadioSelect)
    aktif = forms.ChoiceField(widget=forms.RadioSelect)
    tanggal_lahir = forms.DateField(initial=datetime.date.today)
    email = forms.EmailField()
    tempat_lahir = forms.CharField()
    no_telepon = forms.IntegerField(min_value=10)
    status = forms.ChoiceField(widget=forms.Select)
    verification_code = forms.IntegerField(disabled=True)
    verification = forms.ChoiceField(widget=forms.Select)

    nama.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Nama lengkap.'
    })
    kota_domisili.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Kota yang ditempati sekarang.'
    })
    alamat.widget.attrs.update({
        'class': 'form-control',
        'placeholder': 'Alamat tempat tinggal sekarang.',
        'rows': 4
    })
    email.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Alamat Email.'
    })
    tempat_lahir.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Tempat Lahir.'
    })
    no_telepon.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Nomor Telepon/Handphone.'
    })
    status.widget.attrs.update({
        'class': 'form-control', 'placeholder': 'Status Akun'
    })
    status.choices = [
        ('admin', 'Admin'),
        ('staf', 'Staf'),
        ('member', 'Member'),
    ]

    jenis_kelamin.widget.attrs.update({
        'class': 'form-check-input',
    })
    jenis_kelamin.widget.choices = [
        (False, 'Perempuan'),
        (True, 'Laki-laki')
    ]
    aktif.widget.attrs.update({
        'class': 'form-check-input',
    })
    aktif.widget.choices = [
        (True, 'Ya'),
        (False, 'Tidak')
    ]
    verification.choices = [
        (True, 'Ya'),
        (False, 'Tidak')
    ]

    tanggal_lahir.widget.attrs.update({
        'class': 'form-control',
        'placeholder': 'Tanggal Lahir.',
        'id': 'id_tanggal_lahir',
    })

    class Meta:
        model = Akun
        fields = ['nama', 'foto_src', 'kota_domisili', 'alamat', 'tanggal_lahir',
                  'jenis_kelamin', 'aktif', 'email', 'tempat_lahir', 'no_telepon', 'status', 'verification_code']


import scrapy

class HouseSpider(scrapy.Spider):
    name = "HouseSpider"

    def start_requests(self):
        urls = [
            'https://www.rent.com/texas/dallas-apartments/the-wilson-reviews-5-4929771',
            # 'https://www.rent.com/texas/dallas-apartments/the-wilson-4-4929771',
            # 'https://www.rent.com/alabama/alabaster-apartments/the-view-apartment-homes-4-100017784',
            # 'https://www.rent.com/texas/texas-city-apartments',
            # 'https://www.rent.com/texas/texas-city-houses',
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        # print('response', response.css('._348mM._3UPGk').get())

        # for item in response.css('._348mM._3UPGk'):

        #     PRICE_SELECTOR_CSS = 'div._3csYd.Kv8CE._1_8Hm'
        #     PRICE_SELECTOR_XPATH = './/span/text()'
        #     item_price = item.css(PRICE_SELECTOR_CSS).xpath(PRICE_SELECTOR_XPATH).extract_first()

        #     yield {
        #         'price': item_price,
        #     }

        # NEXT_PAGE_SELECTOR = '._qkIB a::attr(href)'
        # next_page = response.css(NEXT_PAGE_SELECTOR).extract_first()
        # print('next page', next_page)
        # if next_page:
        #     yield scrapy.Request(
        #         response.urljoin(next_page),
        #         callback = self.parse
        #     )
        filename = 'quotes-reviews-house-item.html'
        with open(filename, 'wb') as f:
            f.write(response.body)
        self.log('Saved file %s' % filename)


"""Python implementation of the TextRank algorithm.
From this paper:
    https://web.eecs.umich.edu/~mihalcea/papers/mihalcea.emnlp04.pdf
Based on:
    https://gist.github.com/voidfiles/1646117
    https://github.com/davidadamojr/TextRank
"""
import editdistance
import itertools
import networkx as nx

def filter_for_tags(tagged, tags=None):
    """Apply syntactic filters based on POS tags."""
    if tags is None:
        tags = ['NOUN', 'ADJ', 'PROPN']

    return [item for item in tagged if item[1] in tags]

def normalize(tagged):
    """Return a list of tuples with the first item's periods removed."""
    return [(item[0].replace('.', ''), item[1]) for item in tagged]

def unique_everseen(iterable, key=None):
    """List unique elements in order of appearance.
    Examples:
        unique_everseen('AAAABBBCCDAABBB') --> A B C D
        unique_everseen('ABBCcAD', str.lower) --> A B C D
    """
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in [x for x in iterable if x not in seen]:
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

def build_graph(nodes):
    """Return a networkx graph instance.
    :param nodes: List of hashables that represent the nodes of a graph.
    """
    gr = nx.Graph()  # initialize an undirected graph
    gr.add_nodes_from(nodes)
    node_pairs = list(itertools.combinations(nodes, 2))

    # add edges to the graph (weighted by Levenshtein distance)
    for pair in node_pairs:
        first_string = pair[0]
        second_string = pair[1]
        lev_distance = editdistance.eval(first_string, second_string)
        gr.add_edge(first_string, second_string, weight=lev_distance)

    return gr

def extract_key_phrases(pos_tagged, keywords_length=10, keywords_length_pct=0.1):
    """Return a set of key phrases.
    """
    text_list = [(x[0], x[2]) for x in pos_tagged]

    tagged = filter_for_tags(pos_tagged)
    tagged = normalize(tagged)

    unique_word_set = unique_everseen([x[0] for x in tagged])
    word_set_list = list(unique_word_set)

    # this will be used to determine adjacent words in order to construct
    # key phrases with two words

    graph = build_graph(word_set_list)

    # pageRank - initial value of 1.0, error tolerance of 0,0001,
    calculated_page_rank = nx.pagerank(graph, weight='weight')

    # most important words in ascending order of importance
    key_phrases = sorted(calculated_page_rank, key=calculated_page_rank.get, reverse=True)

    # the number of key phrases returned will be relative to the size of the
    # text (based on percentage)
    index = min(keywords_length, int(len(key_phrases) * keywords_length_pct))
    key_phrases = key_phrases[0:index + 1]

    # take key phrases with multiple words into consideration as done in the
    # paper - if two words are adjacent in the text and are selected as
    # keywords, join them together
    modified_key_phrases = set([])
    # keeps track of individual keywords that have been joined to form a
    # key phrase
    dealt_with = set([])
    i = 0
    j = 1
    while j < len(text_list):
        first_lemma = text_list[i][0]
        first_word = text_list[i][1]
        second_lemma = text_list[j][0]
        second_word = text_list[j][1]
        if first_lemma in key_phrases and second_lemma in key_phrases:
            key_phrase = first_word + ' ' + second_word
            modified_key_phrases.add(key_phrase)
            dealt_with.add(first_lemma)
            dealt_with.add(second_lemma)
        else:
            if first_lemma in key_phrases and first_lemma not in dealt_with:
                modified_key_phrases.add(first_word)

            # if this is the last word in the text, and it is a keyword, it
            # definitely has no chance of being a key phrase at this point
            if j == len(text_list) - 1 and second_lemma in key_phrases and second_lemma not in dealt_with:
                modified_key_phrases.add(second_word)

        i = i + 1
        j = j + 1

    return modified_key_phrases

def extract_sentences(sentence_tokens, summary_length=100, summary_length_pct=0.1, clean_sentences=True):
    """Return a paragraph formatted summary of the source text.
    :param summary_length_pct: Desired summary length as percentage from the total
    :param sentence_tokens: List of sentences
    :param clean_sentences: When True will only return complete sentences
    :param summary_length: Desired summary length as absolute value (if this is smaller than percentage it will win)
    """
    graph = build_graph(sentence_tokens)

    calculated_page_rank = nx.pagerank(graph, weight='weight')

    # most important sentences in ascending order of importance
    sentences = sorted(calculated_page_rank, key=calculated_page_rank.get, reverse=True)

    # return a summary_length word summary
    summary = ' '.join(sentences)
    summary_words = summary.split()

    # Pick the minimum from summary_length and the summary_length_pct but it must have at least one complete sentence
    # if possible
    summary_length_pct = int(len(summary_words) * summary_length_pct)
    summary_words_pct = summary_words[0:summary_length_pct]
    dot_indices_pct = [idx for idx, word in enumerate(summary_words_pct) if word.find('.') != -1]

    summary_words_abs = summary_words[0:summary_length]
    dot_indices_abs = [idx for idx, word in enumerate(summary_words_abs) if word.find('.') != -1]

    there_are_no_dots = len(dot_indices_pct) == 0 and len(dot_indices_abs) == 0
    there_are_dots_in_both = len(dot_indices_abs) > 0 and len(dot_indices_pct) > 0

    if there_are_no_dots or there_are_dots_in_both:
        # Doesn't matter, pick the smallest one
        summary_length = min(summary_length_pct, summary_length)

    if summary_length_pct > summary_length and (not dot_indices_abs):
        summary_length = summary_length_pct

    # Find closest dot position
    if clean_sentences:
        before = summary_words[0:summary_length]
        before_dot_indices = [idx for idx, word in enumerate(before) if word.find('.') != -1]

        after = summary_words[summary_length:]
        after_dot_indices = [idx for idx, word in enumerate(after) if word.find('.') != -1]

        diff_before = 0
        if before_dot_indices:
            dot_before = max(before_dot_indices)
            diff_before = summary_length - dot_before

        diff_after = summary_length
        if after_dot_indices:
            diff_after = min(after_dot_indices)

        if diff_after <= diff_before:
            summary_length = summary_length + diff_after
        else:
            summary_length = dot_before

    summary_words = summary_words[0:summary_length + 1]
    summary = ' '.join(summary_words)

    return summary


import pathlib
import parse

def parse_input(puzzle_input):
    '''take the puzzle input data and convert it into usable data'''
    fish_dictionary = {}
    for x in range(-1, 9):
        fish_dictionary[x] = 0
    fish_list = list(map(int, puzzle_input.split(',')))

    for fish in fish_list:
        if fish in fish_dictionary:
            fish_dictionary[fish] += 1

    return fish_dictionary

def part1(puzzle_data):
    '''solve part 1 and return answer'''
    duration = 80
    for _ in range(duration):
        for x in range(0, 9):
            puzzle_data[x - 1] = puzzle_data[x]
        puzzle_data[8] = puzzle_data[-1]
        puzzle_data[6] += puzzle_data[-1]
        puzzle_data[-1] = 0
    return sum(puzzle_data.values())

def part2(puzzle_data):
    '''solve part 2 and return answer'''
    duration = 256
    for _ in range(duration):
        for x in range(0, 9):
            puzzle_data[x - 1] = puzzle_data[x]
        puzzle_data[8] = puzzle_data[-1]
        puzzle_data[6] += puzzle_data[-1]
        puzzle_data[-1] = 0
    return sum(puzzle_data.values())

def solve(puzzle_input):
    """Solve the puzzle for the given input"""
    data = parse_input(puzzle_input)
    solution1 = part1(parse_input(puzzle_input))
    solution2 = part2(parse_input(puzzle_input))
    return data, solution1, solution2

if __name__ == '__main__':
    PUZZLE_DIR = pathlib.Path(__file__).parent
    puzzle_input = (PUZZLE_DIR / "input.txt").read_text().strip()
    data, solution1, solution2 = solve(puzzle_input)
    print(solution1)
    print(solution2)


import os
from typing import Sequence

import cv2
import torch
import numpy as np
import cv2 as cv
import time
from torch.backends import cudnn
from typing import List
import torch.nn as nn

from sort import Sort
from utils import attempt_load, non_max_suppression, scale_coords, letterbox, plotBbox

cudnn.benchmark = True

def time_synchronized():
    # pytorch-accurate time
    if torch.cuda.is_available():
        torch.cuda.synchronize()
    return time.time()

def extract_image_patch(image, bbox, patch_shape):
    """Extract image patch from bounding box.

    Parameters
    ----------
    image : ndarray
        The full image.
    bbox : array_like
        The bounding box in format (x, y, width, height).
    patch_shape : Optional[array_like]
        This parameter can be used to enforce a desired patch shape
        (height, width). First, the `bbox` is adapted to the aspect ratio
        of the patch shape, then it is clipped at the image boundaries.
        If None, the shape is computed from :arg:`bbox`.

    Returns
    -------
    ndarray | NoneType
        An image patch showing the :arg:`bbox`, optionally reshaped to
        :arg:`patch_shape`.
        Returns None if the bounding box is empty or fully outside of the image
        boundaries.

    """
    bbox = np.array(bbox)
    if patch_shape is not None:
        # correct aspect ratio to patch shape
        target_aspect = float(patch_shape[1]) / patch_shape[0]
        new_width = target_aspect * bbox[3]
        bbox[0] -= (new_width - bbox[2]) / 2
        bbox[2] = new_width

    # convert to top left, bottom right
    bbox[2:] += bbox[:2]
    bbox = bbox.astype(np.int)

    # clip at image boundaries
    bbox[:2] = np.maximum(0, bbox[:2])
    bbox[2:] = np.minimum(np.asarray(image.shape[:2][::-1]) - 1, bbox[2:])
    if bbox[0] > bbox[2] or bbox[1] > bbox[3]:
        print("None")
        return None
    sx, sy, ex, ey = bbox
    image = image[sy:ey, sx:ex]
    image = cv.resize(image, tuple(patch_shape[::-1]))
    return image

class YOLOv7(nn.Module):

    #def __init__(self, model_path: os.PathLike, cpu: bool = False):
    def __init__(self, model_path: os.PathLike):
        super().__init__()
        self.model_path = model_path
        self.device = torch.device('cuda:0')
       # self.device = torch.device('cuda:0' if not cpu else 'cpu')
        #self.half = not cpu

    def start(self, img_size: int = 640, stride: int = 32):
        self.img_size = img_size
        self.stride = stride
        # print('1')
        self.model = attempt_load(self.model_path, map_location=self.device)
        # print('2')
        self.model.eval()
        # print('3')
        if self.half:
            self.model.half()

    def forward(self,
        img: np.ndarray,
        classes: Sequence[int] ,
        conf_thres: float = .5,
        iou_thres: float = .45,

        agnostic_nms: bool = False
        ):
        # Preprocess
        shape = img.shape
        img = letterbox(img, self.img_size, stride=self.stride)[0]
        img = img[:, :, ::-1].transpose(2, 0, 1)  # BGR to RGB, to 3x416x416\
        img = np.ascontiguousarray(img)
        img = torch.from_numpy(img).to(self.device)
        img = img.half() if self.half else img.float()
        img /= 255.0
        # ret = [torch.empty(0)] * len(img)
        if img.ndimension() == 3:
            img = img.unsqueeze(0)
        with torch.no_grad():
            pred = self.model(img, augment=False)[0]
            pred = non_max_suppression(pred, conf_thres, iou_thres, agnostic=agnostic_nms, classes=classes)[0]
            pred[:, :4] = scale_coords(img.shape[2:], pred[:, :4], shape).round()
        return pred

    def forward_batches(self,
        imgs: List[np.ndarray],
        conf_thres: float = .25,
        iou_thres: float = .45,
        classes: Sequence[int] = [0],
        agnostic_nms: bool = False
        ):
        shape = imgs[0].shape
        imgs = [letterbox(img, self.img_size, stride=self.stride)[0] for img in imgs]
        imgs = [img[:, :, ::-1].transpose(2, 0, 1) for img in imgs]  # BGR to RGB, to 3x416x416\
        imgs = np.array(imgs)
        imgs = torch.from_numpy(imgs).to(self.device)
        imgs = imgs.half() if self.half else imgs.float()
        imgs /= 255.0

        with torch.no_grad():
            preds = self.model(imgs, augment=False)[0]
            # nms
            preds = non_max_suppression(preds, conf_thres, iou_thres, classes=classes, agnostic=agnostic_nms)
            for i in range(len(preds)):
                preds[i][:, :4] = scale_coords(imgs.shape[2:], preds[i][:, :4], shape).round()
        return preds

class MOT(object):

    def __init__(self, yolo_model: os.PathLike, cpu: bool = False):
        self.yolo_model = YOLOv7(yolo_model, cpu)
        self.device = torch.device('cuda:0' if not cpu else 'cpu')
        MOT.GLOBAL_MOT = self

    def start(self, img_size: int = 640):
        self.yolo_model.start(img_size)

    def run_with_sort(self, video: cv.VideoCapture, max_age: int = 20, output: os.PathLike = None):
        """
        Running YOLOv7 and SORT
        """
        tracking_results = {}
        self.tracker = Sort(max_age)
        if output is not None:
            fourcc = cv.VideoWriter_fourcc(*'XVID')
            fps = int(video.get(cv.CAP_PROP_FPS))
            shape = video.get(cv.CAP_PROP_FRAME_WIDTH), video.get(cv.CAP_PROP_FRAME_HEIGHT)
            shape = tuple(map(int, shape))

            video_writer = cv.VideoWriter(output, fourcc, fps, shape)
        i_frame = 0
        while True:
            ret, frame = video.read()
            if not ret:
                break
            det = self.yolo_model.forward(frame)
            # Update tracker.
            if det.shape[0] > 0:
                bboxes = self.tracker.update(det.cpu())
                for box in bboxes:
                    person_dict = tracking_results.setdefault(int(box[-1]),
                        {'bbox': [], 'frames': []})
                    person_dict['bbox'].append(box)
                    person_dict['frames'].append(i_frame)
            if output is not None:
                plotBbox(bboxes, frame)
                video_writer.write(frame)
            i_frame += 1
        for pid in tracking_results.keys():
          tracking_results[pid]['bbox'] = np.asarray(tracking_results[pid]['bbox'])
          tracking_results[pid]['frames'] = np.asarray(tracking_results[pid]['frames'])
        if output is not None:
            video_writer.release()
        return tracking_results

    def run_with_sort_batches(self, video: cv.VideoCapture, max_age: int = 20, batches: int = 16, output: os.PathLike = None):
        """
        Running YOLOv7 and SORT
        """
        if output is not None:
            fourcc = cv.VideoWriter_fourcc(*'XVID')
            fps = int(video.get(cv.CAP_PROP_FPS))
            shape = video.get(cv.CAP_PROP_FRAME_WIDTH), video.get(cv.CAP_PROP_FRAME_HEIGHT)
            shape = tuple(map(int, shape))

            video_writer = cv.VideoWriter(output, fourcc, fps, shape)
        tracking_results = {}
        self.tracker = Sort(max_age)
        i_frame = 0
        one_batch = []
        valid = True
        while valid:
            valid, frame = video.read()
            if valid:
                one_batch.append(frame)
            # empty batch
            if len(one_batch) == 0:
                break
            if len(one_batch) == batches or (not valid and len(one_batch) > 0):
                bat_num = len(one_batch)
                dets = self.yolo_model.forward_batches(one_batch)
                for i in range(bat_num):
                    det = dets[i]
                    # Update tracker.
                    if det.shape[0] > 0:
                        bboxes = self.tracker.update(det.cpu())
                        for box in bboxes:
                            person_dict = tracking_results.setdefault(int(box[-1]),
                                {'bbox': [], 'frames': []})
                            person_dict['bbox'].append(box[:4])
                            person_dict['frames'].append(i_frame - bat_num + i + 1)
                        if output is not None:
                            cf = one_batch[i]
                            plotBbox(bboxes, cf)
                            video_writer.write(cf)
                one_batch.clear()
            i_frame += 1
        if output is not None:
            video_writer.release()
        return tracking_results

if __name__ == '__main__':
    det = YOLOv7('./yolov7.pt', True)
    video = cv2.VideoCapture(0)
    img = cv.imread('./test_img.jpg')
    det.start()
    res = det(img, classes=[0, 2, 3, 7])
    plotBbox(res, img)
    cv.imshow("p", img)
    cv.waitKey(0)
    cv.destroyAllWindows()


import logging
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
import math
import pandas as pd
import numpy as np
import argparse
import json
import gc
import re
import copy
import random
import datetime
nowdate = datetime.datetime.now()
from tqdm import tqdm
from IPython import get_ipython

from dataclasses import dataclass, field
from datasets import load_dataset
from datasets import load_metric
from typing import Dict, List, Optional, Tuple

from Library.bart_to_long import *
from Library.data_preprocess import *

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from transformers import PreTrainedTokenizerFast, LEDForConditionalGeneration, AutoModel
from transformers import BartForConditionalGeneration, BartConfig
from transformers.models.bart.modeling_bart import BartLearnedPositionalEmbedding
from transformers.models.longformer.modeling_longformer import LongformerSelfAttention
from transformers import get_linear_schedule_with_warmup, AdamW, TrainingArguments

from torchinfo import summary
import torch
import torch.nn as nn
import torch.optim as optim
from torch.nn import CrossEntropyLoss, MSELoss
from torch.nn import functional as F
from torch.utils.data import DataLoader, Dataset

import pytorch_lightning as pl
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor
from pytorch_lightning import *
from pytorch_lightning.core.lightning import LightningModule
from torch.utils.data import random_split
from pytorch_lightning import loggers as pl_loggers
from transformers.optimization import AdamW, get_cosine_schedule_with_warmup
import pytorch_lightning.metrics.functional as FM
from pytorch_lightning.loggers import WandbLogger, TensorBoardLogger, MLFlowLogger

import mlflow
from pyngrok import ngrok
from mlflow.tracking import MlflowClient

device = torch.device('gpu : 0' if torch.cuda.is_available() else 'cpu')

batch_size = 5
epochs = 2
attention_window = 512
max_pos = 4104
max_seq_len = 4096

class LongformerKobart(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.save_path = '../LED_KoBART/model'
        self.model = LongformerBartForConditionalGeneration.from_pretrained(self.save_path)
        self.tokenizer = PreTrainedTokenizerFast.from_pretrained(self.save_path)
        self.bos_token = '<s>'
        self.eos_token = '</s>'
        self.pad_token_id = 3
        self.ignore_token_id= -100
        self.lr = 1e4

    def forward(self, inputs) :
        input_ids = inputs['input_ids'].to( device)
        attention_mask = inputs['attention_mask'].to( device)
        decoder_input_ids = inputs['decoder_input_ids'].to( device)
        decoder_attention_mask = inputs['decoder_attention_mask'].to( device)
        labels = inputs['labels'].to( device)

        with torch.cuda.amp.autocast():
            output = self.model(input_ids=input_ids,
                        attention_mask=attention_mask,
                        decoder_input_ids=decoder_input_ids,
                        decoder_attention_mask=decoder_attention_mask,
                        labels=labels, use_cache=False, return_dict=True)

        return output

    def training_step(self, batch, batch_idx) :
        outs = self(batch)
        loss = outs.loss
        self.log('train_loss', loss, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx) :
        outs = self(batch)
        loss = outs.loss
        self.log("val_loss",loss, prog_bar=True)
        return loss

    def validation_epoch_end(self, outputs):
        losses = []
        for loss in outputs:
            losses.append(loss)
        return self.log('val_loss', torch.stack(losses).mean(), prog_bar=True)

    def test_step(self, batch, batch_idx) :
        _, _, _, label_str = batch
        outs = self(batch)
        loss = outs.loss
        self.log("test_loss", loss, prog_bar=True)
        return loss

    def test_epoch_end(self, outputs):
        losses = []
        for loss in outputs:
            losses.append(loss)
        return self.log('test_loss', torch.stack(losses).mean(), prog_bar=True)

    def configure_optimizers(self):
        # Prepare optimizer
        optimizer = AdamW(self.model.parameters(), lr=1e-4, correct_bias=True)
        lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95)

        lr_scheduler = {"scheduler" : lr_scheduler,
        "monitor" : "loss",
        "interval" : "step",
        "frequency" : 1}

        return [optimizer], [lr_scheduler]

def load_data():
    tar_train = pd.read_csv('../LED_KoBART/data/tar_train.csv', encoding='utf-8', index_col = False)
    train_data, val_data = train_test_split(tar_train, test_size=0.3, shuffle=True, stratify=tar_train['Subject'], random_state=42)
    return train_data, val_data

if __name__ == "__main__":
    filename = 'KoBART_LED' + '_' + nowdate.strftime('%Y-%m-%d')
    wandb_logger = WandbLogger(project = filename,name=f'Longformer')
    tb_logger = pl_loggers.TensorBoardLogger(os.path.join('../LED_KoBART/Logger', 'tb_logs'))
    client = MlflowClient()
    exp_id = client.create_experiment(filename)
    client.set_experiment_tag(exp_id, "nlp flow", "Longformer KoBART Model")
    exp_name = client.get_experiment(exp_id)
    print("Name: {}".format(exp_name.name))
    print("Experiment_id: {}".format(exp_name.experiment_id))
    print("Artifact Location: {}".format(exp_name.artifact_location))
    print("Tags: {}".format(exp_name.tags))
    print("Lifecycle_stage: {}".format(exp_name.lifecycle_stage))
    mlf_logger = MLFlowLogger(experiment_name=exp_name, tracking_uri="file:./mlruns")

    train_data, val_data = load_data()
    model = LongformerKobart()

    gc.collect()
    torch.cuda.empty_cache()
    dm = KobartLedDataModule(batch_size = 1,
                            num_workers = 10,
                            tokenizer = tokenizer,
                            train_data = train_data,
                            val_data = val_data)
    checkpoint_callback = pl.callbacks.ModelCheckpoint(monitor = 'train_loss', dirpath = '../LED_KoBART/Logger/checkpoints',
                                                        filename='LEDKoBART/{epoch:02d}-{val_accuracy:.3f}',
                                                        verbose = True, save_last = True,
                                                        mode = 'min', save_top_k = -1)
    torch.cuda.empty_cache()
    lr_logger = pl.callbacks.LearningRateMonitor()
    trainer = pl.Trainer(
        default_root_dir='../LED_KoBART/Logger/checkpoints',
        logger = [wandb_logger, tb_logger, mlf_logger],
        #gpus = -1,
        accumulate_grad_batches = 16,
        gradient_clip_val=0.0,
        log_every_n_steps=20, # logging frequency in training step
        val_check_interval=0.1, # validation 몇번마다 돌릴것인지? > 0.1 epoch에 1번)
        callbacks = checkpoint_callback,
        max_epochs=2,
    )
    # 학습 진행
    with mlflow.start_run() as run:

        # Terminate open tunnels if exis
        trainer.fit(model, dm)
        mlflow.pytorch.log_model(model,"models")
        print("Logged data and model in run {}".format(run.info.run_id))

mlflow.end_run()


from django.db import models
import core.personality as p

# Create your models here.
class Note(models.Model):
    body = models.TextField()
    created_at = models.DateTimeField(auto_now=True)
    personality = models.TextField()

#   A string representation of just the body of the Note object
    def __str__(self):
        return self.body

#   My official representation of the object with body and personality
    def __repr__(self):
        return " Text: {}, HELLO: {}".format(self.body, self.personality)

#   Gets the description of the personality type of the Note object
    def get_description(self):
        mb = self.personality
        mbti = {'INTJ': ('Architect: Imaginative and strategic thinkers, ' +
                         'with a plan for everything'),
                'INTP': ('Logician: Innovative inventors with an ' +
                         'unquenchable thirst for knowledge'),
                'ENTJ': ('Commander: Bold, imaginative and strong-willed ' +
                         'leaders, always finding a way – or making one'),
                'ENTP': ('Debater: Smart and curious thinkers who cannot ' +
                         'resist an intellectual challenge'),
                'INFJ': ('Advocate: Quiet and Mystical, yet very inspiring ' +
                         'and tireless idealists'),
                'INFP': ('Mediator: Poetic, kind, altruistic. Always eager' +
                         'to help a good cause'),
                'ENFJ': ('Protagonist: Charismatic and inspiring leaders, ' +
                         'able to mesmerize their listeners'),
                'ENFP': ('Campaigner: Enthusiastic, creative, and sociable' +
                         ' free spirits; can always find a reason to smile'),
                'ISTJ': ('Logistician: Practical and fact-minded individuals' +
                         ' whose reliability cannot be doubted'),
                'ISFJ': ('Defender: Very dedicated and warm protectors, ' +
                         'always ready to defend their loved ones'),
                'ESTJ': ('Executive: Excellent administrators, unsurpassed ' +
                         'at managing things or people'),
                'ESFJ': ('Consul: Extraordinarily caring, social and popular' +
                         ' people, always eager to help'),
                'ISTP': ('Bold and practical experimenters, masters of all ' +
                         'kinds of tools'),
                'ISFP': ('Adventurer: Flexible and charming artists, always ' +
                         ' ready to explore and experience something new'),
                'ESTP': ('Entreprenuer: Smart, energetic and very perceptive' +
                         ' people, who truly enjoy living on the edge'),
                'ESFP': ('Entertainer: Spontaneous, energetic and enthusiastic'
                         + ' people – life is never boring around them.')}
        return mbti.get(mb)


import os
import json

def load_data_set_info_from_json(data_set_id):
    data_set_path = os.environ['DCNN_DATASETS_PATH']
    data_set_dict = {}
    for data_set in os.listdir(data_set_path):
        if not os.path.isdir(os.path.join(data_set_path,data_set)):
            continue
        data_set_desc = '{}/{}/annotations/data_set_description.json'.format(data_set_path, data_set)
        with open(data_set_desc, 'r') as f:
            data_set_dict[data_set] = json.load(f)
    return data_set_dict[data_set_id]

def load_data_set_info_from_dir(top_dir):
    def _get_num_images(top_dir):
        num_images = 0
        for root, directories, filenames in os.walk(top_dir):

            for directory in directories:
                dir = os.path.join(root, directory)
                if os.path.isdir(dir):
                    print(root, directory)
                    num_images += len(os.listdir(os.path.join(root, directory)))

        return num_images

    def _get_label_map(labels):
        map = {}
        for i, label in enumerate(labels):
            map[label] = i

        return map

    train_dir = os.path.join(top_dir, 'train')
    val_dir = os.path.join(top_dir, 'validation')
    train_labels = [x for x in os.listdir(train_dir) if os.path.isdir(os.path.join(train_dir, x))]
    val_labels = [x for x in os.listdir(val_dir) if os.path.isdir(os.path.join(val_dir, x))]
    assert(train_labels == val_labels)
    labels = train_labels

    dataset_size = {'train': _get_num_images(train_dir),
                    'validation': _get_num_images(val_dir)}

    #print(dataset_size)
    #print(labels)

    # phases = os.listdir(top_dir)
    # labels = []
    # data_set_size = {}
    # for phase  in phases:
    #     path = os.path.join(top_dir, phase)
    #     labels = os.listdir(path)
    #
    #     data_set_size = os.path.listdir(os.path.join(path, ))

    data_set_dict = {
        'data_set_name': top_dir,
        'data_set_dir': top_dir,
        'train_dir': train_dir,
        'validation_dir': val_dir,
        'label_map': _get_label_map(labels),
        'classes': list(_get_label_map(labels).values()),
        'data_set_type': 'classification',
        'data_set_size': dataset_size,
        'data_set_description': "Test"
    }

    return data_set_dict