import logging
import math
import sys
import time
from collections import OrderedDict, deque
from datetime import datetime
from typing import Dict, List, Optional

import botocore

from samcli.commands._utils.table_print import MIN_OFFSET, newline_per_item, pprint_column_names, pprint_columns
from samcli.commands.deploy import exceptions as deploy_exceptions
from samcli.commands.deploy.exceptions import (
    ChangeSetError,
    DeployBucketInDifferentRegionError,
    DeployFailedError,
    DeployStackOutPutFailedError,
    DeployStackStatusMissingError,
)
from samcli.lib.deploy.utils import DeployColor, FailureMode
from samcli.lib.package.local_files_utils import get_uploaded_s3_object_name, mktempfile
from samcli.lib.package.s3_uploader import S3Uploader
from samcli.lib.utils.colors import Colored, Colors
from samcli.lib.utils.s3 import parse_s3_url
from samcli.lib.utils.time import to_datetime, utc_to_timestamp

LOG = logging.getLogger(__name__)

DESCRIBE_STACK_EVENTS_FORMAT_STRING = (
    "{ResourceStatus:<{0}} {ResourceType:<{1}} {LogicalResourceId:<{2}} {ResourceStatusReason:<{3}}"
)
DESCRIBE_STACK_EVENTS_DEFAULT_ARGS = OrderedDict(
    {
        "ResourceStatus": "ResourceStatus",
        "ResourceType": "ResourceType",
        "LogicalResourceId": "LogicalResourceId",
        "ResourceStatusReason": "ResourceStatusReason",
    }
)

DESCRIBE_STACK_EVENTS_TABLE_HEADER_NAME = "CloudFormation events from stack operations (refresh every {} seconds)"

DESCRIBE_CHANGESET_FORMAT_STRING = "{Operation:<{0}} {LogicalResourceId:<{1}} {ResourceType:<{2}} {Replacement:<{3}}"
DESCRIBE_CHANGESET_DEFAULT_ARGS = OrderedDict(
    {
        "Operation": "Operation",
        "LogicalResourceId": "LogicalResourceId",
        "ResourceType": "ResourceType",
        "Replacement": "Replacement",
    }
)

DESCRIBE_CHANGESET_TABLE_HEADER_NAME = "CloudFormation stack changeset"

OUTPUTS_FORMAT_STRING = "{Outputs:<{0}}"
OUTPUTS_DEFAULTS_ARGS = OrderedDict({"Outputs": "Outputs"})

OUTPUTS_TABLE_HEADER_NAME = "CloudFormation outputs from deployed stack"

# 500ms of sleep time between stack checks and describe stack events.
DEFAULT_CLIENT_SLEEP = 0.5

class Deployer:
    def __init__(self, cloudformation_client, changeset_prefix="samcli-deploy", client_sleep=DEFAULT_CLIENT_SLEEP):
        self._client = cloudformation_client
        self.changeset_prefix = changeset_prefix
        try:
            self.client_sleep = float(client_sleep)
        except ValueError:
            self.client_sleep = DEFAULT_CLIENT_SLEEP
        if self.client_sleep <= 0:
            self.client_sleep = DEFAULT_CLIENT_SLEEP
        # 2000ms of backoff time which is exponentially used, when there are exceptions during describe stack events
        self.backoff = 2
        # Maximum number of attempts before raising exception back up the chain.
        self.max_attempts = 3
        self.deploy_color = DeployColor()
        self._colored = Colored()

    # pylint: disable=inconsistent-return-statements
    def has_stack(self, stack_name):
        """
        Checks if a CloudFormation stack with given name exists

        :param stack_name: Name or ID of the stack
        :return: True if stack exists. False otherwise
        """
        try:
            resp = self._client.describe_stacks(StackName=stack_name)
            if not resp["Stacks"]:
                return False

            # When you run CreateChangeSet on a a stack that does not exist,
            # CloudFormation will create a stack and set it's status
            # REVIEW_IN_PROGRESS. However this stack is cannot be manipulated
            # by "update" commands. Under this circumstances, we treat like
            # this stack does not exist and call CreateChangeSet will
            # ChangeSetType set to CREATE and not UPDATE.
            stack = resp["Stacks"][0]
            return stack["StackStatus"] != "REVIEW_IN_PROGRESS"

        except botocore.exceptions.ClientError as e:
            # If a stack does not exist, describe_stacks will throw an
            # exception. Unfortunately we don't have a better way than parsing
            # the exception msg to understand the nature of this exception.

            if "Stack with id {0} does not exist".format(stack_name) in str(e):
                LOG.debug("Stack with id %s does not exist", stack_name)
                return False

            LOG.debug("Unknown ClientError recieved: %s. Cannot determine if stack exists.", str(e))
            raise DeployFailedError(stack_name=stack_name, msg=str(e)) from e
        except botocore.exceptions.BotoCoreError as e:
            # If there are credentials, environment errors,
            # catch that and throw a deploy failed error.

            LOG.debug("Botocore Exception : %s", str(e))
            raise DeployFailedError(stack_name=stack_name, msg=str(e)) from e

        except Exception as e:
            # We don't know anything about this exception. Don't handle
            LOG.debug("Unable to get stack details.", exc_info=e)
            raise e

    def create_changeset(
        self, stack_name, cfn_template, parameter_values, capabilities, role_arn, notification_arns, s3_uploader, tags
    ):
        """
        Call Cloudformation to create a changeset and wait for it to complete

        :param stack_name: Name or ID of stack
        :param cfn_template: CloudFormation template string
        :param parameter_values: Template parameters object
        :param capabilities: Array of capabilities passed to CloudFormation
        :param role_arn: the Arn of the role to create changeset
        :param notification_arns: Arns for sending notifications
        :param s3_uploader: S3Uploader object to upload files to S3 buckets
        :param tags: Array of tags passed to CloudFormation
        :return:
        """
        if not self.has_stack(stack_name):
            changeset_type = "CREATE"
            # When creating a new stack, UsePreviousValue=True is invalid.
            # For such parameters, users should either override with new value,
            # or set a Default value in template to successfully create a stack.
            parameter_values = [x for x in parameter_values if not x.get("UsePreviousValue", False)]
        else:
            changeset_type = "UPDATE"
            # UsePreviousValue not valid if parameter is new
            summary = self._client.get_template_summary(StackName=stack_name)
            existing_parameters = [parameter["ParameterKey"] for parameter in summary["Parameters"]]
            parameter_values = [
                x
                for x in parameter_values
                if not (x.get("UsePreviousValue", False) and x["ParameterKey"] not in existing_parameters)
            ]

        # Each changeset will get a unique name based on time.
        # Description is also setup based on current date and that SAM CLI is used.
        kwargs = {
            "ChangeSetName": self.changeset_prefix + str(int(time.time())),
            "StackName": stack_name,
            "TemplateBody": cfn_template,
            "ChangeSetType": changeset_type,
            "Parameters": parameter_values,
            "Description": "Created by SAM CLI at {0} UTC".format(datetime.utcnow().isoformat()),
            "Tags": tags,
        }

        kwargs = self._process_kwargs(kwargs, s3_uploader, capabilities, role_arn, notification_arns)
        return self._create_change_set(stack_name=stack_name, changeset_type=changeset_type, **kwargs)

    @staticmethod
    def _process_kwargs(
        kwargs: dict,
        s3_uploader: Optional[S3Uploader],
        capabilities: Optional[List[str]],
        role_arn: Optional[str],
        notification_arns: Optional[List[str]],
    ) -> dict:
        # If an S3 uploader is available, use TemplateURL to deploy rather than
        # TemplateBody. This is required for large templates.
        if s3_uploader:
            with mktempfile() as temporary_file:
                temporary_file.write(kwargs.pop("TemplateBody"))
                temporary_file.flush()
                remote_path = get_uploaded_s3_object_name(file_path=temporary_file.name, extension="template")
                # TemplateUrl property requires S3 URL to be in path-style format
                parts = parse_s3_url(s3_uploader.upload(temporary_file.name, remote_path), version_property="Version")
                kwargs["TemplateURL"] = s3_uploader.to_path_style_s3_url(parts["Key"], parts.get("Version", None))

        # don't set these arguments if not specified to use existing values
        if capabilities is not None:
            kwargs["Capabilities"] = capabilities
        if role_arn is not None:
            kwargs["RoleARN"] = role_arn
        if notification_arns is not None:
            kwargs["NotificationARNs"] = notification_arns
        return kwargs

    def _create_change_set(self, stack_name, changeset_type, **kwargs):
        try:
            resp = self._client.create_change_set(**kwargs)
            return resp, changeset_type
        except botocore.exceptions.ClientError as ex:
            if "The bucket you are attempting to access must be addressed using the specified endpoint" in str(ex):
                raise DeployBucketInDifferentRegionError(f"Failed to create/update stack {stack_name}") from ex
            raise ChangeSetError(stack_name=stack_name, msg=str(ex)) from ex

        except Exception as ex:
            LOG.debug("Unable to create changeset", exc_info=ex)
            raise ChangeSetError(stack_name=stack_name, msg=str(ex)) from ex

    @pprint_column_names(
        format_string=DESCRIBE_CHANGESET_FORMAT_STRING,
        format_kwargs=DESCRIBE_CHANGESET_DEFAULT_ARGS,
        table_header=DESCRIBE_CHANGESET_TABLE_HEADER_NAME,
    )
    def describe_changeset(self, change_set_id, stack_name, **kwargs):
        """
        Call Cloudformation to describe a changeset

        :param change_set_id: ID of the changeset
        :param stack_name: Name of the CloudFormation stack
        :param kwargs: Other arguments to pass to pprint_columns()
        :return: dictionary of changes described in the changeset.
        """
        paginator = self._client.get_paginator("describe_change_set")
        response_iterator = paginator.paginate(ChangeSetName=change_set_id, StackName=stack_name)
        changes = {"Add": [], "Modify": [], "Remove": []}
        changes_showcase = {"Add": "+ Add", "Modify": "* Modify", "Remove": "- Delete"}
        changeset = False
        for item in response_iterator:
            cf_changes = item.get("Changes")
            for change in cf_changes:
                changeset = True
                resource_props = change.get("ResourceChange")
                action = resource_props.get("Action")
                changes[action].append(
                    {
                        "LogicalResourceId": resource_props.get("LogicalResourceId"),
                        "ResourceType": resource_props.get("ResourceType"),
                        "Replacement": "N/A"
                        if resource_props.get("Replacement") is None
                        else resource_props.get("Replacement"),
                    }
                )

        for k, v in changes.items():
            for value in v:
                row_color = self.deploy_color.get_changeset_action_color(action=k)
                pprint_columns(
                    columns=[
                        changes_showcase.get(k, k),
                        value["LogicalResourceId"],
                        value["ResourceType"],
                        value["Replacement"],
                    ],
                    width=kwargs["width"],
                    margin=kwargs["margin"],
                    format_string=DESCRIBE_CHANGESET_FORMAT_STRING,
                    format_args=kwargs["format_args"],
                    columns_dict=DESCRIBE_CHANGESET_DEFAULT_ARGS.copy(),
                    color=row_color,
                )

        if not changeset:
            # There can be cases where there are no changes,
            # but could be an an addition of a SNS notification topic.
            pprint_columns(
                columns=["-", "-", "-", "-"],
                width=kwargs["width"],
                margin=kwargs["margin"],
                format_string=DESCRIBE_CHANGESET_FORMAT_STRING,
                format_args=kwargs["format_args"],
                columns_dict=DESCRIBE_CHANGESET_DEFAULT_ARGS.copy(),
            )

        return changes

    def wait_for_changeset(self, changeset_id, stack_name):
        """
        Waits until the changeset creation completes

        :param changeset_id: ID or name of the changeset
        :param stack_name:   Stack name
        """
        sys.stdout.write("\n\nWaiting for changeset to be created..\n\n")
        sys.stdout.flush()

        # Wait for changeset to be created
        waiter = self._client.get_waiter("change_set_create_complete")
        # Use default client_sleep to set the delay between polling
        # To override use SAM_CLI_POLL_DELAY environment variable
        waiter_config = {"Delay": self.client_sleep}
        try:
            waiter.wait(ChangeSetName=changeset_id, StackName=stack_name, WaiterConfig=waiter_config)
        except botocore.exceptions.WaiterError as ex:
            resp = ex.last_response
            status = resp.get("Status")
            reason = resp.get("StatusReason")

            if not status or not reason:
                # not a CFN DescribeChangeSet response, re-raising
                LOG.debug("Failed while waiting for changeset: %s", ex)
                raise ex

            if (
                status == "FAILED"
                and "The submitted information didn't contain changes." in reason
                or "No updates are to be performed" in reason
            ):
                raise deploy_exceptions.ChangeEmptyError(stack_name=stack_name)

            raise ChangeSetError(stack_name=stack_name, msg=f"ex: {ex} Status: {status}. Reason: {reason}") from ex

    def execute_changeset(self, changeset_id, stack_name, disable_rollback):
        """
        Calls CloudFormation to execute changeset

        :param changeset_id: ID of the changeset
        :param stack_name: Name or ID of the stack
        :param disable_rollback: Preserve the state of previously provisioned resources when an operation fails.
        """
        try:
            self._client.execute_change_set(
                ChangeSetName=changeset_id, StackName=stack_name, DisableRollback=disable_rollback
            )
        except botocore.exceptions.ClientError as ex:
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

    def get_last_event_time(self, stack_name, default_time=time.time()):
        """
        Finds the last event time stamp that presents for the stack, if not return the default_time
        :param stack_name: Name or ID of the stack
        :param default_time: the default unix epoch time to be returned in case if the stack provided does not exist
        :return: unix epoch
        """
        try:
            return utc_to_timestamp(
                self._client.describe_stack_events(StackName=stack_name)["StackEvents"][0]["Timestamp"]
            )
        except KeyError:
            return default_time

    @pprint_column_names(
        format_string=DESCRIBE_STACK_EVENTS_FORMAT_STRING,
        format_kwargs=DESCRIBE_STACK_EVENTS_DEFAULT_ARGS,
        table_header=DESCRIBE_STACK_EVENTS_TABLE_HEADER_NAME,
        display_sleep=True,
    )
    def describe_stack_events(
        self, stack_name: str, time_stamp_marker: float, on_failure: FailureMode = FailureMode.ROLLBACK, **kwargs
    ):
        """
        Calls CloudFormation to get current stack events
        :param stack_name: Name or ID of the stack
        :param time_stamp_marker: last event time on the stack to start streaming events from.
        :param on_failure: The action to take if the stack fails to deploy
        :param kwargs: Other arguments to pass to pprint_columns()
        """

        stack_change_in_progress = True
        events = set()
        retry_attempts = 0

        while stack_change_in_progress and retry_attempts <= self.max_attempts:
            try:
                # Only sleep if there have been no retry_attempts
                LOG.debug("Trial # %d to get the stack %s create events", retry_attempts, stack_name)
                time.sleep(0 if retry_attempts else self.client_sleep)
                paginator = self._client.get_paginator("describe_stack_events")
                response_iterator = paginator.paginate(StackName=stack_name)

                # Event buffer
                new_events = deque()  # type: deque

                for event_items in response_iterator:
                    for event in event_items["StackEvents"]:
                        LOG.debug("Stack Event: %s", event)
                        # Skip already shown old event entries or former deployments
                        if utc_to_timestamp(event["Timestamp"]) <= time_stamp_marker:
                            LOG.debug(
                                "Skip previous event as time_stamp_marker: %s is after the event time stamp: %s",
                                to_datetime(time_stamp_marker),
                                event["Timestamp"],
                            )
                            break
                        if event["EventId"] not in events:
                            events.add(event["EventId"])
                            # Events are in reverse chronological order
                            # Pushing in front reverse the order to display older events first
                            new_events.appendleft(event)
                    else:  # go to next loop (page of events) if not break from inside loop
                        LOG.debug("Still in describe_stack_events loop, got to next page")
                        continue
                    break  # reached here only if break from inner loop!
                LOG.debug("Exit from the describe event loop")
                # Override timestamp marker with latest event (last in deque)
                if len(new_events) > 0:
                    time_stamp_marker = utc_to_timestamp(new_events[-1]["Timestamp"])

                for new_event in new_events:
                    row_color = self.deploy_color.get_stack_events_status_color(status=new_event["ResourceStatus"])
                    pprint_columns(
                        columns=[
                            new_event["ResourceStatus"],
                            new_event["ResourceType"],
                            new_event["LogicalResourceId"],
                            new_event.get("ResourceStatusReason", "-"),
                        ],
                        width=kwargs["width"],
                        margin=kwargs["margin"],
                        format_string=DESCRIBE_STACK_EVENTS_FORMAT_STRING,
                        format_args=kwargs["format_args"],
                        columns_dict=DESCRIBE_STACK_EVENTS_DEFAULT_ARGS.copy(),
                        color=row_color,
                    )
                    # Skip events from another consecutive deployment triggered during sleep by another process
                    if self._is_root_stack_event(new_event) and self._check_stack_not_in_progress(
                        new_event["ResourceStatus"]
                    ):
                        LOG.debug(
                            "Stack %s is not in progress. Its status is %s, and event is %s",
                            stack_name,
                            new_event["ResourceStatus"],
                            new_event,
                        )
                        stack_change_in_progress = False
                        break

                # Reset retry attempts if iteration is a success to use client_sleep again
                retry_attempts = 0
            except botocore.exceptions.ClientError as ex:
                if (
                    "Stack with id {0} does not exist".format(stack_name) in str(ex)
                    and on_failure == FailureMode.DELETE
                ):
                    LOG.debug("Stack %s does not exist", stack_name)
                    return

                LOG.debug("Trial # %d failed due to exception %s", retry_attempts, str(ex))

                retry_attempts = retry_attempts + 1
                if retry_attempts > self.max_attempts:
                    LOG.error("Describing stack events for %s failed: %s", stack_name, str(ex))
                    return
                # Sleep in exponential backoff mode
                time.sleep(math.pow(self.backoff, retry_attempts))

    @staticmethod
    def _is_root_stack_event(event: Dict) -> bool:
        return bool(
            event["ResourceType"] == "AWS::CloudFormation::Stack"
            and event["StackName"] == event["LogicalResourceId"]
            and event["PhysicalResourceId"] == event["StackId"]
        )

    @staticmethod
    def _check_stack_not_in_progress(status: str) -> bool:
        return "IN_PROGRESS" not in status

    def wait_for_execute(
        self,
        stack_name: str,
        stack_operation: str,
        disable_rollback: bool,
        on_failure: FailureMode = FailureMode.ROLLBACK,
        time_stamp_marker: float = 0,
    ) -> None:
        """
        Wait for stack operation to execute and return when execution completes.
        If the stack has "Outputs," they will be printed.

        Parameters
        ----------
        stack_name : str
            The name of the stack
        stack_operation : str
            The type of the stack operation, 'CREATE' or 'UPDATE'
        disable_rollback : bool
            Preserves the state of previously provisioned resources when an operation fails
        on_failure : FailureMode
            The action to take when the operation fails
        time_stamp_marker:
            last event time on the stack to start streaming events from.
        """
        sys.stdout.write(
            "\n{} - Waiting for stack create/update "
            "to complete\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        )
        sys.stdout.flush()

        self.describe_stack_events(stack_name, time_stamp_marker, on_failure)

        # Pick the right waiter
        if stack_operation == "CREATE":
            waiter = self._client.get_waiter("stack_create_complete")
        elif stack_operation == "UPDATE":
            waiter = self._client.get_waiter("stack_update_complete")
        else:
            raise RuntimeError("Invalid stack operation type {0}".format(stack_operation))

        # Poll every 30 seconds. Polling too frequently risks hitting rate limits
        # on CloudFormation's DescribeStacks API
        waiter_config = {"Delay": 30, "MaxAttempts": 120}

        try:
            waiter.wait(StackName=stack_name, WaiterConfig=waiter_config)
        except botocore.exceptions.WaiterError as ex:
            LOG.debug("Execute stack waiter exception", exc_info=ex)
            if disable_rollback and on_failure is not FailureMode.DELETE:
                # This will only display the message if disable rollback is set or if DO_NOTHING is specified
                msg = self._gen_deploy_failed_with_rollback_disabled_msg(stack_name)
                LOG.info(self._colored.color_log(msg=msg, color=Colors.FAILURE), extra=dict(markup=True))

            raise deploy_exceptions.DeployFailedError(stack_name=stack_name, msg=str(ex))

        try:
            outputs = self.get_stack_outputs(stack_name=stack_name, echo=False)
            if outputs:
                self._display_stack_outputs(outputs)
        except DeployStackOutPutFailedError as ex:
            # Show exception if we aren't deleting stacks
            if on_failure != FailureMode.DELETE:
                raise ex

    def create_and_wait_for_changeset(
        self, stack_name, cfn_template, parameter_values, capabilities, role_arn, notification_arns, s3_uploader, tags
    ):
        try:
            result, changeset_type = self.create_changeset(
                stack_name, cfn_template, parameter_values, capabilities, role_arn, notification_arns, s3_uploader, tags
            )
            self.wait_for_changeset(result["Id"], stack_name)
            self.describe_changeset(result["Id"], stack_name)
            return result, changeset_type
        except botocore.exceptions.ClientError as ex:
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

    def create_stack(self, **kwargs):
        stack_name = kwargs.get("StackName")
        try:
            resp = self._client.create_stack(**kwargs)
            return resp
        except botocore.exceptions.ClientError as ex:
            if "The bucket you are attempting to access must be addressed using the specified endpoint" in str(ex):
                raise DeployBucketInDifferentRegionError(f"Failed to create/update stack {stack_name}") from ex
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

        except Exception as ex:
            LOG.debug("Unable to create stack", exc_info=ex)
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

    def update_stack(self, **kwargs):
        stack_name = kwargs.get("StackName")
        try:
            resp = self._client.update_stack(**kwargs)
            return resp
        except botocore.exceptions.ClientError as ex:
            if "The bucket you are attempting to access must be addressed using the specified endpoint" in str(ex):
                raise DeployBucketInDifferentRegionError(f"Failed to create/update stack {stack_name}") from ex
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

        except Exception as ex:
            LOG.debug("Unable to update stack", exc_info=ex)
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

    def sync(
        self,
        stack_name: str,
        cfn_template: str,
        parameter_values: List[Dict],
        capabilities: Optional[List[str]],
        role_arn: Optional[str],
        notification_arns: Optional[List[str]],
        s3_uploader: Optional[S3Uploader],
        tags: Optional[Dict],
        on_failure: FailureMode,
    ):
        """
        Call the sync command to directly update stack or create stack

        Parameters
        ----------
        :param stack_name: The name of the stack
        :param cfn_template: CloudFormation template string
        :param parameter_values: Template parameters object
        :param capabilities: Array of capabilities passed to CloudFormation
        :param role_arn: the Arn of the role to create changeset
        :param notification_arns: Arns for sending notifications
        :param s3_uploader: S3Uploader object to upload files to S3 buckets
        :param tags: Array of tags passed to CloudFormation
        :param on_failure: FailureMode enum indicating the action to take on stack creation failure
        :return:
        """
        exists = self.has_stack(stack_name)

        if not exists:
            # When creating a new stack, UsePreviousValue=True is invalid.
            # For such parameters, users should either override with new value,
            # or set a Default value in template to successfully create a stack.
            parameter_values = [x for x in parameter_values if not x.get("UsePreviousValue", False)]
        else:
            summary = self._client.get_template_summary(StackName=stack_name)
            existing_parameters = [parameter["ParameterKey"] for parameter in summary["Parameters"]]
            parameter_values = [
                x
                for x in parameter_values
                if not (x.get("UsePreviousValue", False) and x["ParameterKey"] not in existing_parameters)
            ]

        kwargs = {
            "StackName": stack_name,
            "TemplateBody": cfn_template,
            "Parameters": parameter_values,
            "Tags": tags,
        }

        kwargs = self._process_kwargs(kwargs, s3_uploader, capabilities, role_arn, notification_arns)

        try:
            disable_rollback = False
            if on_failure == FailureMode.DO_NOTHING:
                disable_rollback = True
            msg = ""

            if exists:
                kwargs["DisableRollback"] = disable_rollback  # type: ignore
                # get the latest stack event, and use 0 in case if the stack does not exist
                marker_time = self.get_last_event_time(stack_name, 0)
                result = self.update_stack(**kwargs)
                self.wait_for_execute(
                    stack_name, "UPDATE", disable_rollback, on_failure=on_failure, time_stamp_marker=marker_time
                )
                msg = "\nStack update succeeded. Sync infra completed.\n"
            else:
                # Pass string representation of enum
                kwargs["OnFailure"] = str(on_failure)

                result = self.create_stack(**kwargs)
                self.wait_for_execute(stack_name, "CREATE", disable_rollback, on_failure=on_failure)
                msg = "\nStack creation succeeded. Sync infra completed.\n"

            LOG.info(self._colored.color_log(msg=msg, color=Colors.SUCCESS), extra=dict(markup=True))

            return result
        except botocore.exceptions.ClientError as ex:
            raise DeployFailedError(stack_name=stack_name, msg=str(ex)) from ex

    @staticmethod
    @pprint_column_names(
        format_string=OUTPUTS_FORMAT_STRING, format_kwargs=OUTPUTS_DEFAULTS_ARGS, table_header=OUTPUTS_TABLE_HEADER_NAME
    )
    def _display_stack_outputs(stack_outputs: List[Dict], **kwargs) -> None:
        for counter, output in enumerate(stack_outputs):
            for k, v in [
                ("Key", output.get("OutputKey")),
                ("Description", output.get("Description", "-")),
                ("Value", output.get("OutputValue")),
            ]:
                pprint_columns(
                    columns=["{k:<{0}}{v:<{0}}".format(MIN_OFFSET, k=k, v=v)],
                    width=kwargs["width"],
                    margin=kwargs["margin"],
                    format_string=OUTPUTS_FORMAT_STRING,
                    format_args=kwargs["format_args"],
                    columns_dict=OUTPUTS_DEFAULTS_ARGS.copy(),
                    color=Colors.SUCCESS,
                    replace_whitespace=False,
                    break_long_words=False,
                    drop_whitespace=False,
                )
            newline_per_item(stack_outputs, counter)

    def get_stack_outputs(self, stack_name, echo=True):
        try:
            stacks_description = self._client.describe_stacks(StackName=stack_name)
            try:
                outputs = stacks_description["Stacks"][0]["Outputs"]
                if echo:
                    sys.stdout.write("\nStack {stack_name} outputs:\n".format(stack_name=stack_name))
                    sys.stdout.flush()
                    self._display_stack_outputs(stack_outputs=outputs)
                return outputs
            except KeyError:
                return None

        except botocore.exceptions.ClientError as ex:
            raise DeployStackOutPutFailedError(stack_name=stack_name, msg=str(ex)) from ex

    def rollback_delete_stack(self, stack_name: str):
        """
        Try to rollback the stack to a sucessful state, if there is no good state then delete the stack

        Parameters
        ----------
        :param stack_name: str
            The name of the stack
        """
        kwargs = {
            "StackName": stack_name,
        }
        current_state = self._get_stack_status(stack_name)

        try:
            if current_state == "UPDATE_FAILED":
                LOG.info("Stack %s failed to update, rolling back stack to previous state...", stack_name)

                # get the latest stack event
                marker_time = self.get_last_event_time(stack_name, 0)
                self._client.rollback_stack(**kwargs)
                self.describe_stack_events(stack_name, marker_time, FailureMode.DELETE)
                self._rollback_wait(stack_name)

                current_state = self._get_stack_status(stack_name)

            failed_states = ["CREATE_FAILED", "ROLLBACK_COMPLETE", "ROLLBACK_FAILED"]

            if current_state in failed_states:
                LOG.info("Stack %s failed to create/update correctly, deleting stack", stack_name)

                # get the latest stack event
                marker_time = self.get_last_event_time(stack_name, 0)
                self._client.delete_stack(**kwargs)

                # only a stack that failed to create will have stack events, deleting
                # from a ROLLBACK_COMPLETE state will not return anything
                # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation.html#CloudFormation.Client.delete_stack
                if current_state == "CREATE_FAILED":
                    self.describe_stack_events(stack_name, marker_time, FailureMode.DELETE)

                waiter = self._client.get_waiter("stack_delete_complete")
                waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 30, "MaxAttempts": 120})

                LOG.info("\nStack %s has been deleted", stack_name)
            else:
                LOG.info("Stack %s has rolled back successfully", stack_name)
        except botocore.exceptions.ClientError as ex:
            raise DeployStackStatusMissingError(stack_name) from ex
        except botocore.exceptions.WaiterError:
            LOG.error(
                "\nStack %s failed to delete properly! Please manually clean up any persistent resources.",
                stack_name,
            )
        except KeyError:
            LOG.info("Stack %s is not found, skipping", stack_name)

    def _get_stack_status(self, stack_name: str) -> str:
        """
        Returns the status of the stack

        Parameters
        ----------
        :param stack_name: str
            The name of the stack

        Parameters
        ----------
        :return: str
            A string representing the status of the stack
        """
        stack = self._client.describe_stacks(StackName=stack_name)
        stack_status = str(stack["Stacks"][0]["StackStatus"])

        return stack_status

    def _rollback_wait(self, stack_name: str, wait_time: int = 30, max_retries: int = 120):
        """
        Manual waiter for rollback status, waits until we get *_ROLLBACK_COMPLETE or ROLLBACK_FAILED

        Parameters
        ----------
        :param stack_name: str
            The name of the stack
        :param wait_time: int
            The time to wait between polls, default 30 seconds
        :param max_retries: int
            The number of polls before timing out
        """
        status = ""
        retries = 0

        while retries < max_retries:
            status = self._get_stack_status(stack_name)

            if "ROLLBACK_COMPLETE" in status or status == "ROLLBACK_FAILED":
                return

            retries = retries + 1
            time.sleep(wait_time)

        LOG.error(
            "Stack %s never reached a *_ROLLBACK_COMPLETE or ROLLBACK_FAILED state, we got %s instead.",
            stack_name,
            status,
        )

    @staticmethod
    def _gen_deploy_failed_with_rollback_disabled_msg(stack_name):
        return """\nFailed to deploy. Automatic rollback disabled for this deployment.\n
Actions you can take next
=========================
[*] Fix issues and try deploying again
[*] Roll back stack to the last known stable state: aws cloudformation rollback-stack --stack-name {stack_name}
""".format(
            stack_name=stack_name
        )


# INPUT.TXT содержит две строки. В первой строке задано время отправления, а во второй строке – время в пути.
# Время отправления задается в формате «HH:MM», где HH время в часах, которое принимает значение от 00 до 23,
# ММ – время в минутах, которое принимает значение от 00 до 59.
# Время в пути задается двумя неотрицательными целыми числами – количество часов и количество минут.
# Числа разделяются одним пробелом. Количество часов не превышает 120, минут – 59.
# OUTPUT.TXT должен содержать одну строку – время прибытия поезда на конечную станцию.
# Формат вывода этого времени совпадает с форматом ввода времени отправления.

def If(arr):
    if len(arr[0]) < 2:
        res = '0' + arr[0] + ':' + arr[1]
    else: res = arr[0] + ':' + arr[1]
    return res

def Calc(arr):
    t_otp = arr.splitlines()[0].split(':')
    t_put = arr.splitlines()[1].split()
    from datetime import timedelta
    tm3 = timedelta(hours = int(t_otp[0]), minutes = int(t_otp[1]))
    tm4 = timedelta(hours = int(t_put[0]), minutes = int(t_put[1]))
    dif = str(tm3 + tm4)
    if len(dif) > 8:
        temp = dif.split()[2].split(':')
        res = If(temp)
    elif len(dif) <= 8:
        temp = dif.split(':')
        res = If(temp)
    return res

with open('D:\Works\IT\Python_Start\Tasks\Difficulty_11_20\Task_331\input.txt') as input_data:
    arr = input_data.read()
with open('D:\Works\IT\Python_Start\Tasks\Difficulty_11_20\Task_331\output.txt', 'w', encoding= 'utf-8') as output_data:
    output_data.write('Результат: ' + Calc(arr))

print(Calc(arr))


import pathlib

from bang.UnpackParser import UnpackParser, check_condition
from bang.UnpackParserException import UnpackParserException
from kaitaistruct import ValidationFailedError
from . import android_fbpk

# v1 test file: redfin-rd1a.200810.020-factory-c3ea1715.zip
# v2 test file: raven-sd1a.210817.015.a4-factory-bd6cb030.zip
class AndroidFbpkUnpackParser(UnpackParser):
    extensions = []
    signatures = [
        (0, b'FBPK')
    ]
    pretty_name = 'android_fbpk'

    def parse(self):
        try:
            self.data = android_fbpk.AndroidFbpk.from_io(self.infile)
            # version 2 needs some extra sanity checks
            # read data to trigger evaluation
            if self.data.header.version == 2:
                for entry in self.data.body.entries:
                    # read the parsed partition to trigger
                    # parsing for FBPT entries
                    data = entry.partition_parsed
        except (Exception, ValidationFailedError) as e:
            raise UnpackParserException(e.args)
        self.unpacked_size = self.data.body.total_file_size

    # make sure that self.unpacked_size is not overwritten
    def calculate_unpacked_size(self):
        pass

    def unpack(self, meta_directory):
        seen_partitions = set()
        for entry in self.data.body.entries:
            out_labels = []
            # only consider "real" partitions
            if entry.type == 0:
                continue

            is_renamed = False
            orig_name = ''

            partition_name = entry.partition_name
            # there can be duplicates, so rename
            if partition_name in seen_partitions:
                counter = 1
                while True:
                    new_partition_name = "%s-renamed-%d" % (entry.partition_name, counter)
                    if new_partition_name not in seen_partitions:
                        partition_name = new_partition_name
                        is_renamed = True
                        orig_name = entry.partition_name
                        break
                    counter += 1

            file_path = pathlib.Path(partition_name)
            with meta_directory.unpack_regular_file(file_path) as (unpacked_md, outfile):
                outfile.write(entry.partition)

                if is_renamed:
                    with unpacked_md.open(open_file=False):
                        out_labels.append('renamed')
                        unpacked_md.info['labels'] = out_labels
                        unpacked_md.info['metadata'] = {'name': orig_name}
                yield unpacked_md
            seen_partitions.add(partition_name)

    labels = ['android', 'fbpk']
    metadata = {}


import collections
import datetime

class UfoInfoManager:
    PATH = 'nlo.csv'
    DELIMITER = ','

    def __init__(self, path: str = None):
        self.path = path or self.__class__.PATH
        self.data = collections.defaultdict(list)

    def read_data(self) -> None:
        with open(self.path, 'r') as ufo_data:
            keys = ufo_data.readline().strip().split(self.__class__.DELIMITER)
            for data_line in ufo_data.readlines():
                values = data_line.strip().split(self.__class__.DELIMITER)
                for key, value in zip(keys, values):
                    self.data[key].append(value)

    def get_country_with_most_observations(self) -> str:
        return collections.Counter(self.data['country']).most_common(1)[0][0]

    def get_most_often_ufo_month(self) -> str:
        months = list()
        for date_string in self.data['datetime']:
            date_string = date_string.split()[0]
            date_obj = datetime.datetime.strptime(date_string, '%m/%d/%Y')
            months.append(date_obj.strftime("%B"))
        return collections.Counter(months).most_common(1)[0][0]

ufo = UfoInfoManager()
ufo.read_data()
most_ufo_country = ufo.get_country_with_most_observations()
most_month = ufo.get_most_often_ufo_month()
print(most_ufo_country)
print(most_month)


import numpy as np
# for plotting graphs
import matplotlib.pyplot as plt
#for importing data sets and managing them
import pandas as pd

#we have to set the working directory#

#importing dataset#

dataset=pd.read_csv('Data.csv')
#creating an independent variable matrix
X=dataset.iloc[:,:-1].values
#creating a dependent variable vector
y=dataset.iloc[:,3].values

#taking care of missing data#

from sklearn.preprocessing import Imputer
#press ctrl+I for user guide and cleaer understanding
imputer= Imputer(missing_values='NaN', strategy='mean' , axis=0)
#[:,1:3] upper bound is not included so columns actually included=1,2
imputer=imputer.fit(X[:, 1:3])
X[:,1:3]=imputer.transform(X[:,1:3])

#encoding categorical data#

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X=LabelEncoder()
X[:,0]=labelencoder_X.fit_transform(X[:,0])
onehotencoder_X=OneHotEncoder(categorical_features=[0])
X=onehotencoder_X.fit_transform(X).toarray()
labelencoder_y=LabelEncoder()
y=labelencoder_y.fit_transform(y)


import os
import sys
import subprocess
import importlib.util as il
spec = il.spec_from_file_location("config", snakemake.params.config)
config = il.module_from_spec(spec)
sys.modules[spec.name] = config
spec.loader.exec_module(config)
sys.path.append(snakemake.config['args']['mcc_path'])
import scripts.mccutils as mccutils
import scripts.output as output

def main():
    mccutils.log("temp2","running TEMP2 post processing")
    insert_bed = snakemake.input.insert_bed
    absence_summary = snakemake.input.absence_summary
    te_gff = snakemake.input.te_gff
    reference_fasta = snakemake.input.reference_fasta
    log = snakemake.params.log
    sample_name = snakemake.params.sample_name
    chromosomes = snakemake.params.chromosomes.split(",")
    out_dir = snakemake.params.out_dir
    status_log = snakemake.params.status_log
    vcf_options = snakemake.params.vcf.split(",")

    prev_steps_succeeded = mccutils.check_status_file(status_log)

    if prev_steps_succeeded:
        insertions = read_insertions(insert_bed, sample_name, chromosomes, config)
        absence_bed = make_absence_bed(absence_summary, sample_name, out_dir)
        non_absent_ref_insertions = get_non_absent_ref_tes(te_gff, absence_bed, sample_name, chromosomes, out_dir, log)
        insertions += non_absent_ref_insertions

        if len(insertions) > 0:
            insertions = output.make_redundant_bed(insertions, sample_name, out_dir, method="temp2")
            insertions = output.make_nonredundant_bed(insertions, sample_name, out_dir, method="temp2")
            output.write_vcf(insertions, reference_fasta, sample_name, "temp2", out_dir, vcf_options)
        else:
            mccutils.run_command(["touch", out_dir+"/"+sample_name+"_temp2_redundant.bed"])
            mccutils.run_command(["touch", out_dir+"/"+sample_name+"_temp2_nonredundant.bed"])
    else:
        mccutils.run_command(["touch", out_dir+"/"+sample_name+"_temp2_redundant.bed"])
        mccutils.run_command(["touch", out_dir+"/"+sample_name+"_temp2_nonredundant.bed"])

    mccutils.log("temp2","TEMP2 postprocessing complete")

def read_insertions(insert_bed, sample_name, chromosomes, config):
    insertions = []
    with open(insert_bed, "r") as inf:
        for x,line in enumerate(inf):
            if x > 0:
                insert = output.Insertion(output.Temp2())
                split_line = line.split("\t")
                if len(split_line) == 15:
                    insert.chromosome = split_line[0]
                    insert.start = int(split_line[1])+1
                    insert.end = int(split_line[2])
                    insert.family = split_line[3].split(":")[0]
                    insert.type = "non-reference"
                    insert.support_info.support["frequency"].value = float(split_line[4])
                    insert.strand = split_line[5]
                    insert.support_info.support["class"].value = split_line[6]
                    insert.support_info.support["supportreads"].value = float(split_line[7])
                    insert.support_info.support["referencereads"].value = float(split_line[8])
                    insert.support_info.support["fiveprimesupport"].value = float(split_line[9])
                    insert.support_info.support["threeprimesupport"].value = float(split_line[10])
                    insert.support_info.support["reliability"].value = float(split_line[12].replace("%","")) # rare enties have a % sign for some reason
                    insert.support_info.support["fiveprimejunctionsupport"].value = float(split_line[13])
                    insert.support_info.support["threeprimejunctionsupport"].value = float(split_line[14])

                    insert.name = insert.family+"|non-reference|"+str(insert.support_info.support['frequency'].value)+"|"+sample_name+"|temp2|"

                    if insert.support_info.support["fiveprimejunctionsupport"].value > 0 and insert.support_info.support["threeprimejunctionsupport"].value > 0:
                        insert.name += "sr|"
                    else:
                        insert.name += "rp|"

                    if (
                        insert.chromosome in chromosomes and
                        insert.support_info.support["frequency"].value >= config.PARAMS["frequency_threshold"] and
                        insert.support_info.support["class"].value in config.PARAMS["acceptable_insertion_support_classes"]
                    ):
                        insertions.append(insert)

    return insertions

def make_absence_bed(summary_file, sample, out):
    out_bed = out+"/"+sample+".absent.bed"
    lines = []
    with open(summary_file, "r") as inf:
        for x,line in enumerate(inf):
            if x > 0:
                split_line = line.split("\t")
                new_line = "\t".join([split_line[0], split_line[1], split_line[2]])
                new_line += "\n"
                lines.append(new_line)

    if len(lines) < 1:
        lines.append("empty\t0\t1\n")

    with open(out_bed,"w") as bed:
        for line in lines:
            bed.write(line)

    return out_bed

def get_non_absent_ref_tes(te_gff, absence_bed, sample, chromosomes, out, log):
    insertions = []
    tmp_gff = out+"/tmp.ref_nonabs.gff"
    command = ["bedtools", "subtract", "-A", "-a", te_gff, "-b", absence_bed]
    mccutils.run_command_stdout(command, tmp_gff, log=log)

    with open(tmp_gff,"r") as gff:
        for line in gff:
            if "#" not in line:
                line = line.replace(";","\t")
                split_line = line.split("\t")
                insert = output.Insertion(output.Temp2())
                insert.chromosome = split_line[0]
                insert.start = int(split_line[3])
                insert.end = int(split_line[4])
                insert.name = split_line[9].split("=")[1]+"|reference|NA|"+sample+"|temp2|nonab|"
                insert.strand = split_line[6]
                insert.type = "reference"

                if insert.chromosome in chromosomes:
                    insertions.append(insert)

    mccutils.remove(tmp_gff)

    return insertions

if __name__ == "__main__":
    main()


# import the necessary packages
from __future__ import print_function
from utils import remove_overlap
import matplotlib.pyplot as plt
import numpy as np
import argparse
import imutils
import glob
import cv2
import copy

'''template matching using tradiontinal template matching algorithm'''
def template_matcher(template, wave, method = cv2.TM_CCOEFF_NORMED, threshold = 0.8):
    template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
    wave = cv2.cvtColor(wave, cv2.COLOR_BGR2GRAY)
    # loop over the images to find the template in
    matches = cv2.matchTemplate(wave, template, method)

    # Store the coordinates of matched area in a numpy array
    w, h = template.shape[::-1]
    # If the method is TM_SQDIFF or TM_SQDIFF_NORMED, take minimum
    if method == cv2.TM_SQDIFF or method == cv2.TM_SQDIFF_NORMED:
        locs = np.where(matches <= threshold)
    else:
        locs = np.where(matches >= threshold)
    results = []
    for loc in zip(*locs[::-1]):
        top_left = loc
        bottom_right = (top_left[0] + w, top_left[1] + h)
        results.append([top_left, bottom_right])
    cmp = lambda result: -matches[result[0][1], result[0][0]]
    results.sort(key = cmp)
    return results

'''template matching using template matching algorithm
combine with image pyramid for multiscale matching'''
def template_matcher_multiscale(template, wave, threshold = 0.8, scales = [0.5, 0.3]):
    # Convert to grayscale
    # image =  copy.deepcopy(wave)
    wave = cv2.cvtColor(wave, cv2.COLOR_BGR2GRAY)
    template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)

    # Store width and height of template in w and h
    w, h = template.shape[::-1]
    found = None
    results = []
    temps = []
    for scale in scales:
        # resize the image according to the scale, and keep track
        # of the ratio of the resizing
        resized_template = imutils.resize(template, width = int(template.shape[1] * scale))

        matches = cv2.matchTemplate(wave, resized_template, cv2.TM_CCOEFF_NORMED)
        # get location of all matches
        locs = np.where(matches >= threshold)

        for loc in zip(*locs[::-1]):
            temps.append([matches[loc[1], loc[0]], loc, scale])

    cmp = lambda item: -item[0]
    temps.sort(key = cmp)
    results = []
    for _, top_left, r in temps:
        bottom_right = (top_left[0] + w * r, top_left[1] + h * r)
        results.append([top_left, bottom_right])

    return results

'''Feature matching using ORB detector and Brute Force matching'''
def ORB_detector_BF_matching(template, wave):
    # ORB Detector
    orb = cv2.ORB_create()
    #orb = cv2.SIFT.create()
    kp1, des1 = orb.detectAndCompute(template, None)
    kp2, des2 = orb.detectAndCompute(wave, None)
    # Brute Force Matching
    bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
    matches = bf.match(des1, des2)
    matches = sorted(matches, key = lambda x:x.distance)
    #drawing the matches
    matching_result = cv2.drawMatches(template, kp1, wave, kp2, matches[:50], None, flags=2)
    cv2.imshow('Good Matches', matching_result)
    cv2.waitKey()

'''Template matching using SIFT detector and FLANN matching'''
def SIFT_detector_FLANN_matching(template, wave, threshold = 0.75, visualize = False):
    # Initiate SIFT detector
    sift = cv2.SIFT_create()

    # find the keypoints and descriptors with SIFT
    kp1, des1 = sift.detectAndCompute(template,None)
    kp2, des2 = sift.detectAndCompute(wave,None)

    # FLANN parameters
    FLANN_INDEX_KDTREE = 1
    index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
    search_params = dict(checks = 50)   # or pass empty dictionary
    flann = cv2.FlannBasedMatcher(index_params, search_params)
    matches = flann.knnMatch(des1, des2, k = 2)

    # Need to draw only good matches, so create a mask
    matchesMask = [[0,0] for i in range(len(matches))]

    # ratio test as per Lowe's paper
    results = []
    for i, (m, n) in enumerate(matches):
        if m.distance < threshold * n.distance:
            matchesMask[i] = [1,0]
            results.append(kp2[m.trainIdx].pt)

    if visualize:
        draw_params = dict(matchColor = (0, 255, 0),
                        singlePointColor = (255, 0, 0),
                        matchesMask = matchesMask,
                        flags = cv2.DrawMatchesFlags_DEFAULT)
        img3 = cv2.drawMatchesKnn(template, kp1, wave, kp2, matches, None, **draw_params)
        cv2.imshow('Good Matches', img3)
        cv2.waitKey()

    return results


import wx
import os
import diffKeywords
import createSelectFilePanel
import wx.lib.scrolledpanel as scrolled

class diffKeywordsPanel(scrolled.ScrolledPanel):
  def __init__(self, parent):
    super(diffKeywordsPanel, self).__init__(parent)
    self.SetupScrolling()

    self.selectKeywordsFileNew = createSelectFilePanel.createSelectFilePanel(self,'Select New Keywords Files',0,0)
    self.selectKeywordsFileOld = createSelectFilePanel.createSelectFilePanel(self,'Select Old Keywords Files',0,100)

    self.btnRun = wx.Button(self, label='Run', pos=(10, 200))
    self.btnRun.Bind(wx.EVT_BUTTON, self.on_run)

    self.btnStart = wx.Button(self, label='Start/Next', pos=(100, 200))
    self.btnStart.Bind(wx.EVT_BUTTON, self.on_start)

    wx.StaticText(self,label="Old Keyword",pos=(10, 265))
    wx.StaticText(self,label="New Keyword",pos=(10, 515))

    self.textOldKeyword = wx.TextCtrl(self, pos=(150, 250),size=(800,200), style=wx.TE_MULTILINE)
    self.textNewKeyword = wx.TextCtrl(self, pos=(150, 500),size=(800,45))

    self.btnUpdate = wx.Button(self, label='Update', pos=(10, 550))
    self.btnUpdate.Bind(wx.EVT_BUTTON, self.on_update)

    self.btnSave = wx.Button(self, label='Save', pos=(100, 550))
    self.btnSave.Bind(wx.EVT_BUTTON, self.on_save)

    self.keywordsObj = diffKeywords.diffKeywords()
    self.wordIndex = 0

    self.disableUpdate()

  def disableUpdate(self):
    self.btnUpdate.Disable()
    self.btnStart.Disable()

  def enableUpdate(self):
    self.btnUpdate.Enable()
    self.btnStart.Enable()

  def disableSelect(self):
    self.btnRun.Disable()
    self.selectKeywordsFileOld.disable()
    self.selectKeywordsFileNew.disable()

  def enableSelect(self):
    self.btnRun.Enable()
    self.selectKeywordsFileOld.enable()
    self.selectKeywordsFileNew.enable()

  def on_run(self, event):
    self.keywordsObj.getKeywords(self.selectKeywordsFileNew.getFileName(),self.selectKeywordsFileOld.getFileName())
    self.enableUpdate()

  def on_start(self, event):
    data,index = self.keywordsObj.getNewWords()
    self.disableSelect()

    if data:
      self.wordIndex = index
      self.textOldKeyword.SetValue(data)
      self.textNewKeyword.SetValue("")
      self.btnUpdate.Enable()

    else:
      self.textNewKeyword.SetValue("Done! No New Words")
      self.btnUpdate.Disable()

  def on_update(self,event):
    self.keywordsObj.updateKeyword(self.wordIndex, self.textNewKeyword.GetValue())
    self.btnUpdate.Disable()

  def on_save(self,event):
    self.keywordsObj.writeKeywords(self.selectKeywordsFileNew.getFileName().replace(".xlsx","New.xlsx"))
    self.enableSelect()


import sqlite3
import os, sys, time, datetime, random, string
import urllib.request, urllib.error
import configparser
from flask import Flask, request, session, redirect
from flask import render_template, g, flash, url_for
from contextlib import closing
from pxepagi import Pagi

pxeat_config = configparser.RawConfigParser()

try:
    pxeat_config.read('./pxeat.cfg')
except configparser.Error:
    print("Get configuration failed!")

general_opts = {}
pxefile_opts = {}
for option in pxeat_config.options('general'):
    try:
        general_opts[option] = pxeat_config.get('general', option)
        if general_opts[option] == -1:
            DebugPrint("skip: %s" % option)
    except:
        print("exception on %s!" % option)
        general_opts[option] = None

for option in pxeat_config.options('pxe_file'):
    try:
        pxefile_opts[option] = pxeat_config.get('pxe_file', option).split('\\n')
        if pxefile_opts[option] == -1:
            DebugPrint("skip: %s" % option)
    except:
        print("exception on %s!" % option)
        pxefile_opts[option] = None

SECRET_KEY = general_opts['secret_key']

postfix_kernelfn = '-0'
postfix_initrdfn = '-1'

# Defaults
items = {}

app = Flask(__name__)
app.config.from_object(__name__)

def chk_input(chk_string, chk_type):

    if chk_type == 'pxe_title':
        if chk_string == '':
            raise ValueError("The title can not be empty!")
        return
    elif chk_type == 'file_path':
        if chk_string[0] != '/' or chk_string[-1] == '/':
            raise ValueError("Path format is invalid!")
        return
    elif chk_type == 'repo_url':
        chk_elements = chk_string.split('//')
        if chk_elements[1] == '':
            raise ValueError("The repository can not be empty!")
            return
        elif chk_elements[0] not in ['http:','https:']:
            raise ValueError("Invalid format!"+\
                             " (Only support http:// or https://)")
            return
    else:
        sys.exit("chk_type error!")

def prt_help():
    print("This is help info.:")

def grab_file(base_url, file_path, saved_file):

    file_url = base_url + file_path

    try:
        f = urllib.request.urlopen(file_url)
        local_file = open(saved_file, "wb")
        local_file.write(f.read())
        local_file.close()
    except urllib.error.HTTPError as e:
        return str(e.code) + " " + str(e.reason)
    except urllib.error.URLError as e:
        return str(e.reason)
    except urllib.error:
        return

def boot_opts_gen(opt_flag):
    if opt_flag == "vnc":
        return(general_opts['default_boot_opts'] + \
                         " vnc=1 vncpassword=" + \
                         general_opts['vnc_passwd'])
    elif opt_flag == "ssh":
        return(general_opts['default_boot_opts'] + \
                         " usessh=1 sshpassword=" + \
                         general_opts['ssh_passwd'])
    else:
        return(general_opts['default_boot_opts'])

def connect_db():
    return sqlite3.connect(general_opts['database'])

def init_db():
    with closing(connect_db()) as db:
        with app.open_resource('schema.sql', mode='r') as f:
            db.cursor().executescript(f.read())
        db.commit()

@app.before_request
def before_request():
    g.db = connect_db()

@app.teardown_request
def teardown_request(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        db.close()

@app.route('/')
def form():
    return render_template('form.html')

app.jinja_env.globals['repo_kernel_default'] = general_opts['repo_kernel_default']
app.jinja_env.globals['repo_initrd_default'] = general_opts['repo_initrd_default']

@app.route('/history/', defaults={'page': 1})
@app.route('/history/page/<int:page>')
def history(page):
    count=50
    per_page=10
    pagination = Pagi(page, per_page, count)

    try:
        cur = g.db.execute('select pxe_title,\
                                repo_url,\
                                repo_kernel,\
                                repo_initrd,\
                                pxe_comment,\
                                unix_time,\
                                inst_flag from pxeitems order by id desc')
    except sqlite3.Error as e:
        return render_template('failed.html', failed_msg = "Database error: "+str(e))

    history_entries = [ dict(pxe_title=row[0], \
                    repo_url=row[1], \
                    repo_kernel=row[2], \
                    repo_initrd=row[3], \
                    pxe_comment=row[4], \
                    unix_time=datetime.datetime.fromtimestamp(int(row[5])), \
                    inst_flag=row[6]) \
            for row in cur.fetchall()[(page-1)*per_page:page*per_page]\
            ]

    if not history_entries and page != 1:
        #Shoud do something here other than pass or abort(404)
        pass

    return render_template('history.html',\
                            pagination=pagination,\
                            history_entries=history_entries)

@app.route('/about')
@app.route('/about/')
def about():
    return render_template('about.html')

# For the pagination
def url_for_other_page(page):
    args = request.view_args.copy()
    args['page'] = page
    return url_for(request.endpoint, **args)
app.jinja_env.globals['url_for_other_page'] = url_for_other_page

@app.route('/confirm', methods=['POST'])
def confirm_entry():

    #Input checking
    try:
        for x,y in [[request.form['pxe_title'],'pxe_title'], \
                  [request.form['repo_url'], 'repo_url'], \
                  [request.form['repo_kernel'], 'file_path'], \
                  [request.form['repo_initrd'], 'file_path']]:
            chk_input(x,y)
    except ValueError as e:
        flash(e.args[0],'error')
        return redirect(url_for('form'))
        #return render_template('form.html',ss="red")

    # Assign to the dictionary
    items['repo_kernel'] = request.form['repo_kernel']
    items['repo_url'] = request.form['repo_url']
    items['repo_initrd'] = request.form['repo_initrd']
    items['pxe_title'] = request.form['pxe_title']
    items['pxe_comment'] = request.form['pxe_comment']
    items['inst_flag'] = request.form['inst_method']

    # Generate a random string
    items['random_str'] = ''.join(random.choice(string.ascii_lowercase) for _ in range(4))

    items['unix_time'] = ''

    # Show the entry which will be generated on the confirm page
    gen_format = ["menu label ^a - " + items['pxe_title'], \
              "kernel " + general_opts['loader_path'] + "[random]" + postfix_kernelfn, \
              "append initrd=" + general_opts['loader_path'] + "[random]" + postfix_initrdfn + " " + \
              boot_opts_gen(items['inst_flag']) + " " + \
              "install=" + items['repo_url']]

    return render_template('confirm.html', cfm_entries=items, cfm_fmt=gen_format)

@app.route('/add', methods=['POST'])
def add_entry():
    items['unix_time'] = str(int(time.time()))
    id_random = items['unix_time'] + items['random_str']

    for f_name,i in [[items['repo_kernel'], postfix_kernelfn],\
                     [items['repo_initrd'], postfix_initrdfn]]:
        ret = grab_file(items['repo_url'],\
                        f_name,\
                        general_opts['loader_path'] + id_random + i)
        if ret:
            return render_template('failed.html',\
                                    failed_msg = f_name + ": " + str(ret))
        else:
            pass

    # Save new entry to database

    try:
        g.db.execute('INSERT INTO pxeitems (\
                   pxe_title, \
                   repo_url, \
                   repo_kernel, \
                   repo_initrd, \
                   pxe_comment, \
                   unix_time, \
                   random_str, \
                   inst_flag) values (?, ?, ?, ?, ?, ?, ?, ?)', \
                  [items['pxe_title'], \
                   items['repo_url'], \
                   items['repo_kernel'], \
                   items['repo_initrd'], \
                   items['pxe_comment'], \
                   items['unix_time'], \
                   items['random_str'], \
                   items['inst_flag']\
                   ])
    except sqlite3.Error as e:
        #Remove downloaded files here
        for i in (postfix_kernelfn, postfix_initrdfn):
            os.remove(general_opts['loader_path'] + id_random + i)
        return render_template('failed.html', failed_msg = "Database error: " + str(e))
    g.db.commit()

    # Fetch first 10 entires from the database

    cur = g.db.execute('SELECT pxe_title,\
                            repo_url,\
                            repo_kernel,\
                            repo_initrd,\
                            inst_flag FROM pxeitems order by id desc')
    pxe_entries = [ dict(pxe_title=row[0], \
                    repo_url=row[1], \
                    repo_kernel=row[2], \
                    repo_initrd=row[3], \
                    inst_flag=row[4]) for row in cur.fetchall()[:10]\
                   ]

    # Write the entries to PXE configure file

    try:
        fpxe = open(general_opts['pxe_file'],'w')
    except IOError as e:
        for i in ("0","1"):
            os.remove(general_opts['loader_path'] + id_random + "-" + i)
        g.db.execute('DELETE FROM pxeitems WHERE id = (SELECT max(id) FROM pxeitems)')
        return render_template('failed.html', failed_msg = e)

    for i in pxefile_opts['pxe_header']:
        fpxe.write(i + '\n')

    pxe_index = 'a'
    for pxe_entry in pxe_entries:
        fpxe.write('label {0}\n  menu label ^{0} - {1}\n  menu indent 2\n  kernel {2}\n  append initrd=loader{3} {4} install={5}\n\n'.format(\
                pxe_index,\
                pxe_entry['pxe_title'],\
                general_opts['loader_path'] + items['unix_time'] + items['random_str'] + postfix_kernelfn, \
                general_opts['loader_path'] + items['unix_time'] + items['random_str'] + postfix_initrdfn, \
                boot_opts_gen(pxe_entry['inst_flag']),items['repo_url']))
        pxe_index = chr(ord(pxe_index)+1)

    for i in pxefile_opts['pxe_footer']:
        fpxe.write(i + '\n')
    fpxe.close

    # Remove the out of service kernel files
    # Fetch the 11th entries, get the file name
    cur = g.db.execute('SELECT unix_time, random_str\
                            FROM pxeitems order by id desc LIMIT 1 OFFSET 10')

    try:
        id_random_toberm=''.join(cur.fetchone())
    except:
        #If can not fetch (only less then 10 items), then pass
        pass

    for i in [postfix_kernelfn, postfix_initrdfn]:
        try:
            os.remove(general_opts['loader_path'] + id_random_toberm + i)
        except:
            #If the file not exist, then pass
            pass

    flash(u'New entry was successfully posted','green')
    return redirect(url_for('form'))

if __name__ == '__main__':
    if len(sys.argv) == 2:
        if sys.argv[1] == '--initdb':
            init_db()
        elif sys.argv[1] == 'server':

            if not os.path.isfile(general_opts['database']):
                print("Database is not available!\nCreate with --initdb")
                sys.exit()
            if not os.path.isfile(general_opts['pxe_file']):
                print("PXE file is not available!\nPlease check the configuration")
                sys.exit()

#            app.debug = True
            app.run(host='0.0.0.0')
        else:
            prt_help()
    else:
        prt_help()


from django.urls import path
from django.conf.urls import url
from . import views

app_name = 'printer'

urlpatterns = [
    path('', views.login_view),
    path('home/', views.home_view),
    path('history/', views.history_view),
    path('vehicles/', views.vehicle_view),
    path('logout/', views.logout_view),
    ]


from flask import make_response, request
from config.flask_config import app
from service import qc_service
from util.fetch import fetch_json

URL_BASE = "https://caliber2-mock.revaturelabs.com:443/mock/"

@app.route("/associate/<associate_email>")
def associate_listing(associate_email):
    # localhost:5000/associate/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com
    url = f"{URL_BASE}training/associate/{associate_email}"
    batches = fetch_json(url)
    return {**batches}, 200

@app.route("/associate/<associate_email>/batch")
def associate_batch_listing(associate_email):
    # localhost:5000/associate/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com/batch
    url = f"{URL_BASE}training/associate/{associate_email}/batch"
    batches = fetch_json(url)
    return {**batches}, 200

@app.route("/grades/reports/individual/<associate_email>", methods=["GET"])
def get_trainee_grades_compared_to_batch(associate_email):
    # localhost:5000/grades/reports/individual/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com
    url = f"{URL_BASE}evaluation/grades/reports/individual/{associate_email}"
    return {**fetch_json(url)}

@app.route("/grades/reports/individual/<associate_email>/<int:week>", methods=["GET"])
def get_trainee_grades_compared_to_batch_on_week(associate_email, week):
    # localhost:5000/grades/reports/individual/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com/2
    url = f"{URL_BASE}evaluation/grades/reports/individual/{associate_email}/{week}"
    return {**fetch_json(url)}

@app.route("/grades/assessment/<int:assessment_id>", methods=["GET"])
def findGradesByAssessment(assessment_id):
    # localhost:5000/grades/assessment/2169
    url = "%s%s%d" % (URL_BASE, "evaluation/grades/assessment/", assessment_id)
    return {"assessments": fetch_json(url)}, 200

@app.route("/grades/average", methods=["GET"])
def findAverageGradeByWeek():
    # http://127.0.0.1:5000/grades/average?batch=TR-1190&week=1
    # http://127.0.0.1:5000/grades/average?assessment=2169
    assessment = request.args.get("assessment", None)
    batch = request.args.get("batch", None)
    week = request.args.get("week", None)
    url_response = None

    if assessment:
        url_response = f"?assessment={assessment}"
    elif batch and week:
        url_response = f"?batch={batch}&week={week}"
    else:
        return make_response("Malformed inputs: You must provide assessment or (batch and week)", 400)

    url = f"{URL_BASE}evaluation/grades/average{url_response}"
    return {"average": fetch_json(url)}, 200

@app.route("/grades/batch/<batch_id>", methods=["GET"])
def findGradesByBatch(batch_id):
    # localhost:5000/grades/batch/TR-1190
    url = f"{URL_BASE}evaluation/grades/batch/{batch_id}"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/batch/<batch_id>/<int:week>", methods=["GET"])
def getGradeComparisonForBatchOnWeek(batch_id, week):
    # localhost:5000/grades/reports/batch/TR-1190/1
    url = f"{URL_BASE}evaluation/grades/reports/batch/{batch_id}/{week}"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/individual/<associate_email>", methods=["GET"])
def getTraineeGradesComparedToBatch(associate_email):
    # localhost:5000/grades/reports/individual/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com
    url = f"{URL_BASE}evaluation/grades/reports/individual/{associate_email}"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/individual/<associate_email>/<int:week>", methods=["GET"])
def getTraineeGradesComparedToBatchOnWeek(associate_email, week):
    # localhost:5000/grades/reports/individual/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com/2
    url = f"{URL_BASE}evaluation/grades/reports/individual/{associate_email}/{week}"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/<batch_id>/overall", methods=["GET"])
def getGradeReportForBatch(batch_id):
    # localhost:5000/grades/reports/TR-1190/overall
    url = f"{URL_BASE}evaluation/grades/reports/{batch_id}/overall"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/<batch_id>/overall/<int:week>", methods=["GET"])
def getWeeklyGradeReportForBatch(batch_id, week):
    # localhost:5000/grades/reports/TR-1190/overall/1
    url = f"{URL_BASE}evaluation/grades/reports/{batch_id}/overall/{week}"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/<batch_id>/spider", methods=["GET"])
def getSpiderGraphDataForBatch(batch_id):
    # localhost:5000/grades/reports/TR-1190/spider
    url = f"{URL_BASE}evaluation/grades/reports/{batch_id}/spider"
    return {"batch_grades": fetch_json(url)}

@app.route("/grades/reports/<batch_id>/spider/<associate_email>", methods=["GET"])
def getSpiderGraphDataForTraineeAndBatch(batch_id, associate_email):
    # localhost:5000/grades/reports/TR-1190/spider/mock8.associatef4c8d0c5-ecaf-4127-a459-7bf3617118a6@mock.com
    url = f"{URL_BASE}evaluation/grades/reports/{batch_id}/spider/{associate_email}"
    return {"batch_grades": fetch_json(url)}

# @app.route("/qa/notes/trainee/<associate_id>", methods=["GET"])
# def getQANotesForTrainee(associate_id):
#     # localhost:5000/qa/notes/trainee/SF-2681
#     url_ind = f"{URL_BASE}qa/notes/trainee/{associate_id}"
#     qa_json_ind = fetch_json(url_ind)
#     batch_id = qa_service.getbatchID(qa_json_ind)
#     url_batch = f"{URL_BASE}qa/notes/batch/{batch_id}"
#     qa_json_batch = fetch_json(url_batch)
#
#     return qa_service.parseTraineeJson(qa_json_ind, qa_json_batch)


import logging
import vobject

from pylons import request, response, session, tmpl_context as c
from zkpylons.lib.helpers import redirect_to
from pylons.decorators import validate
from pylons.decorators.rest import dispatch_on

from formencode import validators, htmlfill, ForEach, Invalid
from formencode.variabledecode import NestedVariables

from zkpylons.lib.base import BaseController, render
from zkpylons.lib.ssl_requirement import enforce_ssl
from zkpylons.lib.validators import BaseSchema
import zkpylons.lib.helpers as h

from authkit.authorize.pylons_adaptors import authorize
from authkit.permissions import ValidAuthKitUser

from zkpylons.lib.mail import email

from zkpylons.model import meta
from zkpylons.model.location import Location
from zkpylons.model.config import Config

log = logging.getLogger(__name__)

class LocationSchema(BaseSchema):
    display_name = validators.String(not_empty=True)
    display_order = validators.Int()
    capacity = validators.Int()

class NewLocationSchema(BaseSchema):
    location = LocationSchema()
    pre_validators = [NestedVariables]

class EditLocationSchema(BaseSchema):
    location = LocationSchema()
    pre_validators = [NestedVariables]

class LocationController(BaseController):

    @enforce_ssl(required_all=True)
    def __before__(self, **kwargs):
        c.can_edit = True

    @authorize(h.auth.has_organiser_role)
    @dispatch_on(POST="_new")
    def new(self):
        return render('/location/new.mako')

    @authorize(h.auth.has_organiser_role)
    @validate(schema=NewLocationSchema(), form='new', post_only=True, on_get=True, variable_decode=True)
    def _new(self):
        results = self.form_result['location']

        c.location = Location(**results)
        meta.Session.add(c.location)
        meta.Session.commit()

        h.flash("Location created")
        redirect_to(action='index', id=None)

    @authorize(h.auth.has_organiser_role)
    def view(self, id):
        c.location = Location.find_by_id(id)
        return render('/location/view.mako')

    @authorize(h.auth.has_organiser_role)
    def index(self):
        c.location_collection = Location.find_all()
        return render('/location/list.mako')

    @authorize(h.auth.has_organiser_role)
    @dispatch_on(POST="_edit")
    def edit(self, id):
        c.location = Location.find_by_id(id)

        defaults = h.object_to_defaults(c.location, 'location')

        form = render('/location/edit.mako')
        return htmlfill.render(form, defaults)

    @authorize(h.auth.has_organiser_role)
    @validate(schema=EditLocationSchema(), form='edit', post_only=True, on_get=True, variable_decode=True)
    def _edit(self, id):
        location = Location.find_by_id(id)

        for key in self.form_result['location']:
            setattr(location, key, self.form_result['location'][key])

        # update the objects with the validated form data
        meta.Session.commit()
        h.flash("The Location has been updated successfully.")
        redirect_to(action='index', id=None)

    @authorize(h.auth.has_organiser_role)
    @dispatch_on(POST="_delete")
    def delete(self, id):
        """Delete the location

        GET will return a form asking for approval.

        POST requests will delete the item.
        """
        c.location = Location.find_by_id(id)
        return render('/location/confirm_delete.mako')

    @authorize(h.auth.has_organiser_role)
    @validate(schema=None, form='delete', post_only=True, on_get=True, variable_decode=True)
    def _delete(self, id):
        c.location = Location.find_by_id(id)
        meta.Session.delete(c.location)
        meta.Session.commit()

        h.flash("Location has been deleted.")
        redirect_to('index')

    def ical(self, id):
        c.schedule_collection = Location.find_by_id(id).schedule

        ical = vobject.iCalendar()
        for schedule in c.schedule_collection:
            if not schedule.time_slot.heading:
                event = ical.add('vevent')
                event.add('uid').value = str(schedule.id) + '@' + Config.get('event_host')
                # Created
                tz = timezone(Config.get('time_zone'))
                event.add('created').value = schedule.creation_timestamp.replace(tzinfo=tz)
                # Last Modified
                event.add('dtstamp').value = schedule.last_modification_timestamp.replace(tzinfo=tz)
                event.add('last-modified').value = schedule.last_modification_timestamp.replace(tzinfo=tz)
                # Start and End Time
                event.add('dtstart').value = schedule.time_slot.start_time.replace(tzinfo=tz)
                event.add('dtend').value = schedule.time_slot.end_time.replace(tzinfo=tz)
                # Title and Author (need to add Author here)
                event.add('summary').value = schedule.event.computed_title() + '. ' + h.list_to_string(schedule.event.computed_speakers())
                # Abstract, if we have one
                event.add('description').value = schedule.event.computed_abstract()
                # Add a URL
                if schedule.event.proposal:
                    event.add('url').value = h.url_for(qualified=True, controller='schedule', action='view_talk', id=schedule.event.proposal.id)
                elif not (schedule.event.url is None or schedule.event.url == ''):
                    if schedule.event.url.startswith('https://') or schedule.event.url.startswith('http://'):
                        event.add('url').value = h.url_for(str(schedule.event.url))
                    else:
                        event.add('url').value = h.url_for(str(schedule.event.url), qualified=True)

                concurrent_schedules = schedule.event.schedule_by_time_slot(schedule.time_slot)
                for concurrent_schedule in concurrent_schedules:
                    if concurrent_schedule != schedule:
                        if concurrent_schedule in c.schedule_collection:
                            c.schedule_collection.remove(concurrent_schedule)

                locations = [concurrent_schedule.location.display_name for concurrent_schedule in concurrent_schedules]
                event.add('location').value = h.list_to_string(locations)

        response.charset = 'utf8'
        response.headers['content-type'] = 'text/calendar; charset=utf8'
        response.headers.add('content-transfer-encoding', 'binary')
        response.headers.add('Pragma', 'cache')
        response.headers.add('Cache-Control', 'max-age=3600,public')
        return ical.serialize()


import math
import torch

from torch import nn, Tensor
import torch.nn.functional as F
from signjoey.helpers import freeze_params
from signjoey.layers import  DenseBayesian,LWTA

####################################################

from torch.hub import load_state_dict_from_url

model_urls = {
    'alexnet': 'https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth',
}

class AlexNet(nn.Module):

    def __init__(self, num_classes: int = 1000) -> None:
        super(AlexNet, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(64, 192, kernel_size=5, padding=2),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, num_classes),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = self.features(x)
        x = self.avgpool(x)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

def alexnet(pretrained: bool = True, progress: bool = True, **kwargs) -> AlexNet:
    r"""AlexNet model architecture from the
    `"One weird trick..." <https://arxiv.org/abs/1404.5997>`_ paper.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
        progress (bool): If True, displays a progress bar of the download to stderr
    """
    model = AlexNet(**kwargs)
    if pretrained:
        state_dict = load_state_dict_from_url(model_urls['alexnet'],
                                              progress=progress)
        model.load_state_dict(state_dict)

    return model

###############################################################################################3

def get_activation(activation_type):
    if activation_type == "relu":
        return nn.ReLU()
    elif activation_type == "relu6":
        return nn.ReLU6()
    elif activation_type == "prelu":
        return nn.PReLU()
    elif activation_type == "selu":
        return nn.SELU()
    elif activation_type == "celu":
        return nn.CELU()
    elif activation_type == "gelu":
        return nn.GELU()
    elif activation_type == "sigmoid":
        return nn.Sigmoid()
    elif activation_type == "softplus":
        return nn.Softplus()
    elif activation_type == "softshrink":
        return nn.Softshrink()
    elif activation_type == "softsign":
        return nn.Softsign()
    elif activation_type == "tanh":
        return nn.Tanh()
    elif activation_type == "tanhshrink":
        return nn.Tanhshrink()
    else:
        raise ValueError("Unknown activation type {}".format(activation_type))

class MaskedNorm(nn.Module):
    """
        Original Code from:
        https://discuss.pytorch.org/t/batchnorm-for-different-sized-samples-in-batch/44251/8
    """

    def __init__(self, norm_type, num_groups, num_features):
        super().__init__()
        self.norm_type = norm_type
        if self.norm_type == "batch":
            self.norm = nn.BatchNorm1d(num_features=num_features)
        elif self.norm_type == "group":
            self.norm = nn.GroupNorm(num_groups=num_groups, num_channels=num_features)
        elif self.norm_type == "layer":
            self.norm = nn.LayerNorm(normalized_shape=num_features)
        else:
            raise ValueError("Unsupported Normalization Layer")

        self.num_features = num_features

    def forward(self, x: Tensor, mask: Tensor):
        if self.training:
            reshaped = x.reshape([-1, self.num_features])
            reshaped_mask = mask.reshape([-1, 1]) > 0
            selected = torch.masked_select(reshaped, reshaped_mask).reshape(
                [-1, self.num_features]
            )
            batch_normed = self.norm(selected)
            scattered = reshaped.masked_scatter(reshaped_mask, batch_normed)
            return scattered.reshape([x.shape[0], -1, self.num_features])
        else:
            reshaped = x.reshape([-1, self.num_features])
            batched_normed = self.norm(reshaped)
            return batched_normed.reshape([x.shape[0], -1, self.num_features])

class EmbeddingsEnsemble(nn.Module):
    def __init__(self,*args, **kwargs):
        super().__init__()
        N=10
        self.Decoders=[Embeddings_(*args, **kwargs) for i in range(N) ]
        self.Decoders=nn.ModuleList(self.Decoders)
        self.embedding_dim=self.Decoders[0].embedding_dim
        self.lut=self.Decoders[0].lut
        for param in self.Decoders[0].parameters():
            param.requires_grad = False
    def      forward(self,*args, **kwargs):
        return   [self.Decoders[i](*args, **kwargs) for i in range(10)]
# TODO (Cihan): Spatial and Word Embeddings are pretty much the same
#       We might as well convert them into a single module class.
#       Only difference is the lut vs linear layers.
class Embeddings(nn.Module):

    """
    Simple embeddings class
    """

    # pylint: disable=unused-argument
    def __init__(
        self,
        embedding_dim: int = 64,
        num_heads: int = 8,
        scale: bool = False,
        scale_factor: float = None,
        norm_type: str = None,
        activation_type: str = None,
        vocab_size: int = 0,
        padding_idx: int = 1,
        freeze: bool = False,
        bayesian : bool = False,
        **kwargs
    ):
        """
        Create new embeddings for the vocabulary.
        Use scaling for the Transformer.

        :param embedding_dim:
        :param scale:
        :param vocab_size:
        :param padding_idx:
        :param freeze: freeze the embeddings during training
        """
        super().__init__()

        self.bayesian=bayesian
        self.embedding_dim = embedding_dim
        self.vocab_size = vocab_size
        if bayesian:
            self.ln = DenseBayesian( self.embedding_dim, self.embedding_dim, competitors = 2,
                                activation = activation_type, prior_mean=0, prior_scale=1. )
        self.lut = nn.Embedding(vocab_size, self.embedding_dim, padding_idx=padding_idx)

        self.norm_type = norm_type
        if self.norm_type:
            self.norm = MaskedNorm(
                norm_type=norm_type, num_groups=num_heads, num_features=embedding_dim
            )

        self.activation_type = activation_type
        if self.activation_type and not self.bayesian :
            self.activation = get_activation(activation_type)

        self.scale = scale
        if self.scale:
            if scale_factor:
                self.scale_factor = scale_factor
            else:
                self.scale_factor = math.sqrt(self.embedding_dim)

        if freeze:
            freeze_params(self)

    # pylint: disable=arguments-differ
    def forward(self, x: Tensor, mask: Tensor = None) -> Tensor:
        """
        Perform lookup for input `x` in the embedding table.

        :param mask: token masks
        :param x: index in the vocabulary
        :return: embedded representation for `x`
        """

        x = self.lut(x)
        if self.bayesian:
            x=self.ln(x)
        if self.norm_type:
            x = self.norm(x, mask)

        if self.activation_type and not self.bayesian:
            x = self.activation(x)

        if self.scale:
            return x * self.scale_factor
        else:
            return x

    def __repr__(self):
        return "%s(embedding_dim=%d, vocab_size=%d)" % (
            self.__class__.__name__,
            self.embedding_dim,
            self.vocab_size,
        )

class SpatialEmbeddingsEnsemble(nn.Module):
    def __init__(self,*args, **kwargs):
        super().__init__()
        N=10

        self.Decoders=[SpatialEmbeddings_(*args, **kwargs) for i in range(N) ]
        self.Decoders=nn.ModuleList(self.Decoders)
        self.embedding_dim =self.Decoders[0].embedding_dim
        for param in self.Decoders[0].parameters():
            param.requires_grad = False
    def      forward(self,*args, **kwargs):
        return [self.Decoders[i](*args, **kwargs) for i in range(10)]

class SpatialEmbeddings(nn.Module):

    """
    Simple Linear Projection Layer
    (For encoder outputs to predict glosses)
    """

    # pylint: disable=unused-argument
    def __init__(
        self,
        embedding_dim: int,
        input_size: int,
        num_heads: int,
        freeze: bool = False,
        norm_type: str = None,
        activation_type: str = None,
        scale: bool = False,
        scale_factor: float = None,
        bayesian : bool = False,
        ibp : bool = False,
        **kwargs
    ):
        """
        Create new embeddings for the vocabulary.
        Use scaling for the Transformer.

        :param embedding_dim:
        :param input_size:
        :param freeze: freeze the embeddings during training
        """
        super().__init__()

        self.embedding_dim = embedding_dim
        self.input_size = input_size
        self.bayesian = bayesian

        ###################################

        # Intialize Alexnet
        self.alexnet = alexnet()
        self.alexnet.classifier[6] = nn.Linear(4096, embedding_dim)

        #####################################

        if bayesian:
            self.ln = DenseBayesian(self.input_size, self.embedding_dim, competitors =2 ,
                                activation = activation_type, prior_mean=0, prior_scale=1. , kl_w=0.1, ibp = ibp)

        else:
            self.ln = nn.Linear(self.input_size, self.embedding_dim)
            #self.ln = nn.Conv1d(self.input_size,self.embedding_dim,5,padding=2)
          #  print('\n\n\n\n\n\n CONV  \n\n\n\n\n')
        self.norm_type = norm_type
        if self.norm_type:
            self.norm = MaskedNorm(
                norm_type=norm_type, num_groups=num_heads, num_features=embedding_dim
            )

        self.activation_type = activation_type
        if bayesian:
            self.activation_type = False
        else:
            self.activation_type = activation_type
        if self.activation_type:
            self.activation = get_activation(activation_type)

        self.scale = scale
        if self.scale:
            if scale_factor:
                self.scale_factor = scale_factor
            else:
                self.scale_factor = math.sqrt(self.embedding_dim)

        if freeze:
            freeze_params(self)

    # pylint: disable=arguments-differ
    def forward(self, x: Tensor, mask: Tensor) -> Tensor:
        """
        :param mask: frame masks
        :param x: input frame features
        :return: embedded representation for `x`
        """

        frames_input = False

        if frames_input:
            ###################################

            temp_batch = x.shape[0]
            temp_frames = x.shape[1]
            # x = torch.ones([181, 3, 227, 227], device='cuda:0')
            x = x.reshape((temp_batch * temp_frames, 3, 227, 227))

            # Intialize Alexnet
            x = self.alexnet(x)
            x = x.reshape((temp_batch, temp_frames, self.embedding_dim))
            # self.ln = nn.Linear(self.input_size, self.embedding_dim)

            #####################################

        else:
            # x = self.ln(x.transpose(-1,-2)).transpose(-1,-2)
            # nn.Linear(self.input_size, self.embedding_dim)
            x = self.ln(x)

        if self.norm_type:
            x = self.norm(x, mask)

        if  self.activation_type and (not self.bayesian):
            x = self.activation(x)

        if self.scale:
            return x * self.scale_factor
        else:
            return x

    def __repr__(self):
        return "%s(embedding_dim=%d, input_size=%d)" % (
            self.__class__.__name__,
            self.embedding_dim,
            self.input_size,
        )


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sys
import xgboost
import pandas as pd
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.metrics import log_loss
import subprocess

sys.path.append('xgboost/wrapper/')

train = pd.read_pickle('../data/train.pkl')
test = pd.read_pickle('../data/test.pkl')

featureset1 = ['len_q1c', 'len_q2c', 'words_q1c', 'words_q2c', 'chars_q1c', 'chars_q2c', 'wordshare']
featureset2 = ['qratio', 'wratio', 'partial_ratio', 'partial_tokenset', 'tokenset', 'partial_tokensort']

features = featureset1 + featureset2
print (features)

X_train, X_test, y_train, y_test = train_test_split(train[features], train['is_duplicate'], test_size=0.2)

def gridsearch():
    clf = xgboost.XGBRegressor(
        objective='binary:logistic',
        nthread=4,
        silent=False,
    )
    parameters = {
        'n_estimators': [50, 100, 250, 500],
        'learning_rate': [0.05, 0.1, 0.3],
        'max_depth': [4, 6, 9, 12, 24],
        'subsample': [0.9, 1.0],
        'colsample_bylevel': [1.0]
    }

    clf = GridSearchCV(clf, parameters, n_jobs=1, cv=4)

    clf.fit(X_train, y_train)

    print ("Best parameters", clf.best_params_)

    # print "CV results"
    # print clf.cv_results_

    print('predicted:', clf.predict([[1, 1]]))

    subprocess.call(['speech-dispatcher'])  # start speech dispatcher
    subprocess.call(['spd-say', '"your process has finished"'])

if __name__ == '__main__':
    gridsearch()


from django.shortcuts import get_object_or_404
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from .models import Article, Review
from .serializers import *
from drf_yasg.utils import swagger_auto_schema
from movies.models import Movie
from .schemas import *

@swagger_auto_schema(
    method="get",
    operation_description="커뮤니티의 모든 게시물 정보를 반환합니다.",
    manual_parameters=[token_param],
    responses=article_list_schema,
)
@swagger_auto_schema(
    method="post",
    operation_description="새로운 게시물을 생성합니다.",
    manual_parameters=[token_param],
    request_body=ArticleCreateSerializer,
    responses=article_res_schema(201),
)
@api_view(["GET", "POST"])
def article_list_create(request):
    def article_list():
        articles = Article.objects.all()
        serializer = ArticleSerializer(articles, many=True)
        data = {
            "count": len(serializer.data),
            "articles": serializer.data,
        }
        return Response(data, status.HTTP_200_OK)

    def article_create(request):
        movie = get_object_or_404(Movie, pk=request.data["movie_pk"])
        latitude = request.data["latitude"]
        longitude = request.data["longitude"]
        article = Article.objects.create(author=request.user, movie=movie, latitude=latitude, longitude=longitude)
        return Response(ArticleSerializer(article).data, status=status.HTTP_201_CREATED)

    if request.user.is_authenticated:
        if request.method == "GET":
            return article_list()
        else:
            return article_create(request)
    return Response(status=status.HTTP_401_UNAUTHORIZED)

@swagger_auto_schema(
    method="put",
    operation_description="해당 게시물의 마감 여부를 변경합니다.",
    manual_parameters=[token_param],
    # request_body=ArticleUpdateSerializer,
    responses=article_res_schema(200),
)
@swagger_auto_schema(
    method="delete",
    operation_description="해당 게시물을 삭제합니다.",
    manual_parameters=[token_param],
    responses=delete_schema,
)
@api_view(["PUT", "DELETE"])
def article_update_delete(request, article_pk):
    article = get_object_or_404(Article, pk=article_pk)
    if request.user != article.author:
        return Response(status=status.HTTP_401_UNAUTHORIZED)

    def article_update(request):
        article.is_finished = not article.is_finished
        article.save()
        serializer = ArticleSerializer(article)
        return Response(serializer.data)

    def article_delete():
        article.delete()
        data = {
            "message": f"{article_pk}번 게시물이 삭제되었습니다.",
        }
        return Response(data, status=status.HTTP_204_NO_CONTENT)

    if request.method == "PUT":
        return article_update(request)
    else:
        return article_delete()

@swagger_auto_schema(
    method="get",
    operation_description="특정 게시물의 모든 리뷰를 반환합니다.",
    manual_parameters=[token_param],
    responses=review_list_schema,
)
@swagger_auto_schema(
    method="post",
    operation_description="특정 게시물의 새로운 리뷰를 생성합니다.",
    manual_parameters=[token_param],
    request_body=ReviewBodySerializer,
    # responses=review_res_schema(201),
)
@api_view(["GET", "POST"])
def review_list_create(request, article_pk):
    article = get_object_or_404(Article, pk=article_pk)

    def review_list():
        reviews = article.reviews.all()
        serializer = ReviewSerializer(reviews, many=True)
        data = {
            "article_id": article_pk,
            "reviews": serializer.data,
        }
        return Response(data)

    def review_create(request):
        content = request.data["content"]
        new_review = Review.objects.create(author=request.user, article=article, content=content)
        serializer = ReviewSerializer(new_review)
        data = {
            "article_id": article_pk,
            "review": serializer.data,
        }
        return Response(data, status=status.HTTP_201_CREATED)

    if request.user.is_authenticated:
        if request.method == "GET":
            return review_list()
        else:
            return review_create(request)
    return Response(status=status.HTTP_401_UNAUTHORIZED)

@swagger_auto_schema(
    method="delete",
    operation_description="해당 리뷰를 삭제합니다.",
    manual_parameters=[token_param],
    responses=delete_schema,
)
@swagger_auto_schema(
    method="put",
    operation_description="리뷰의 내용을 변경합니다.",
    manual_parameters=[token_param],
    request_body=ReviewBodySerializer,
    responses=review_res_schema(200),
)
@api_view(["PUT", "DELETE"])
def review_update_delete(request, review_pk):
    if not request.user.is_authenticated:
        return Response(status=status.HTTP_401_UNAUTHORIZED)

    review = get_object_or_404(Review, pk=review_pk)
    if review.author != request.user:
        return Response(status=status.HTTP_400_BAD_REQUEST)

    def review_update():
        review.content = request.data["content"]
        review.save()
        serializer = ReviewSerializer(review)
        data = {
            "article_id": review.article.pk,
            "review": serializer.data,
        }
        return Response(data)

    def review_delete():
        review.delete()
        data = {
            "message": f"{review_pk}번 리뷰가 삭제되었습니다.",
        }
        return Response(data, status=status.HTTP_204_NO_CONTENT)

    if request.method == "PUT":
        return review_update()
    else:
        return review_delete()