import logging
from typing import Optional, List, Dict, Any, Tuple, Union
from collections import defaultdict

from pydcop.algorithms import ComputationDef, AlgoParameterDef
from pydcop.computations_graph.factor_graph import (
    FactorComputationNode,
    VariableComputationNode,
)
from pydcop.dcop.objects import Variable, VariableNoisyCostFunc
from pydcop.dcop.relations import Constraint, generate_assignment_as_dict
from pydcop.infrastructure.computations import (
    DcopComputation,
    SynchronousComputationMixin,
    VariableComputation,
    register,
    Message,
)

GRAPH_TYPE = "factor_graph"
logger = logging.getLogger("pydcop.maxsum")

SAME_COUNT = 4

STABILITY_COEFF = 0.1

HEADER_SIZE = 0
UNIT_SIZE = 1

# constants for memory costs and capacity
FACTOR_UNIT_SIZE = 1
VARIABLE_UNIT_SIZE = 1

def build_computation(comp_def: ComputationDef):
    if comp_def.node.type == "VariableComputation":
        logger.debug(f"Building variable computation {comp_def}")
        return MaxSumVariableComputation(comp_def=comp_def)
    if comp_def.node.type == "FactorComputation":
        logger.debug(f"Building factor computation {comp_def}")
        return MaxSumFactorComputation(comp_def=comp_def)

def computation_memory(
    computation: Union[FactorComputationNode, VariableComputationNode]
) -> float:
    """Memory footprint associated with the maxsum computation node.

    Notes
    -----
    Two formulations of the memory footprint are possible for factors :
    * If the constraint is given by a function (intentional), the factor
      only needs to keep the costs sent by each variable and the footprint
      is the total size of these cost vectors.
    * If the constraints is given extensively the size of the hypercube of
      costs must also be accounted for.

    Parameters
    ----------
    computation: FactorComputationNode or VariableComputationNode
        A computation node for a factor or a variable in the factor-graph.

    Returns
    -------
    float:
        the memory footprint of the computation.
    """
    if isinstance(computation, FactorComputationNode):
        # Memory footprint associated with the factor computation f.
        # For Maxsum, it depends on the size of the domain of the neighbor
        # variables.
        m = 0
        for v in computation.variables:
            domain_size = len(v.domain)
            m += domain_size * FACTOR_UNIT_SIZE
        return m

    elif isinstance(computation, VariableComputationNode):
        # For Maxsum, the memory footprint a variable computations depends
        #  on the number of  neighbors in the factor graph.
        domain_size = len(computation.variable.domain)
        num_neighbors = len(list(computation.links))
        return num_neighbors * domain_size * VARIABLE_UNIT_SIZE

    raise ValueError(
        "Invalid computation node type {}, maxsum only defines "
        "VariableComputationNodeand FactorComputationNode".format(computation)
    )

def communication_load(
    src: Union[FactorComputationNode, VariableComputationNode], target: str
) -> float:
    """The communication cost of an edge between a variable and a factor.

    Parameters
    ----------
    src: VariableComputationNode
        The ComputationNode for the source variable.
    target: str
        the name of the other variable `src` is sending messages to

    Return
    ------
    float:
        the size of messages between computation and target.
    """
    if isinstance(src, VariableComputationNode):
        d_size = len(src.variable.domain)
        return UNIT_SIZE * d_size + HEADER_SIZE

    elif isinstance(src, FactorComputationNode):
        for v in src.variables:
            if v.name == target:
                d_size = len(v.domain)
                return UNIT_SIZE * d_size + HEADER_SIZE
        raise ValueError(
            "Could not find variable {} in constraint of factor "
            "{}".format(target, src)
        )

    raise ValueError(
        "maxsum communication_load only supports "
        "VariableComputationNode and FactorComputationNode, "
        "invalid computation: " + str(src)
    )

algo_params = [
    AlgoParameterDef("damping", "float", None, 0.5),
    AlgoParameterDef(
        "damping_nodes", "str", ["vars", "factors", "both", "none"], "both"
    ),
    AlgoParameterDef("stability", "float", None, STABILITY_COEFF),
    AlgoParameterDef("noise", "float", None, 0.01),
    AlgoParameterDef("start_messages", "str", ["leafs", "leafs_vars", "all"], "leafs"),
]

class MaxSumMessage(Message):
    def __init__(self, costs: Dict):
        super().__init__("max_sum", None)
        self._costs = costs

    @property
    def costs(self):
        return self._costs

    @property
    def size(self):
        # Max sum messages are dictionaries from values to costs:
        return len(self._costs) * 2

    def __str__(self):
        return "MaxSumMessage({})".format(self._costs)

    def __repr__(self):
        return "MaxSumMessage({})".format(self._costs)

    def __eq__(self, other):
        if type(other) != MaxSumMessage:
            return False
        if self.costs == other.costs:
            return True
        return False

    def _simple_repr(self):
        r = {"__module__": self.__module__, "__qualname__": self.__class__.__qualname__}

        # When building the simple repr when transform the dict into a pair
        # of list to avoid problem when serializing / de-serializing the repr.
        # The costs dic often contains int as key, when converting to an from
        # json (which only support string for keys in dict), we would
        # otherwise loose the type information and restore the dict with str
        # keys.
        vals, costs = zip(*self._costs.items())
        r["vals"] = vals
        r["costs"] = costs
        return r

    @classmethod
    def _from_repr(cls, r):
        vals = r["vals"]
        costs = r["costs"]

        return MaxSumMessage(dict(zip(vals, costs)))

# Some semantic type definition, to make things easier to read and check:
VarName = str
FactorName = str
VarVal = Any
Cost = float

class MaxSumFactorComputation(SynchronousComputationMixin, DcopComputation):
    def __init__(self, comp_def: ComputationDef):
        assert comp_def.algo.algo == "maxsum"
        super().__init__(comp_def.node.factor.name, comp_def)
        self.logger.warning(f"Neiborghs {self.neighbors}")

        self.mode = comp_def.algo.mode
        self.factor = comp_def.node.factor
        self.variables = self.factor.dimensions

        # costs : messages for our variables, used to store the content of the
        # messages received from our variables.
        # {v -> {d -> costs} }
        # For each variable, we keep a dict mapping the values for this
        # variable to an associated cost.
        self._costs: Dict[VarName, Dict[VarVal:Cost]] = {}

        self.damping = comp_def.algo.params["damping"]
        self.damping_nodes = comp_def.algo.params["damping_nodes"]
        self.stability_coef = comp_def.algo.params["stability"]
        self.start_messages = comp_def.algo.params["start_messages"]
        self.logger.info(f"Running maxsum with params: {comp_def.algo.params}")

        # A dict var_name -> (message, count)
        self._prev_messages = defaultdict(lambda: (None, 0))

    def on_start(self):

        # Only unary factors (leaf in the graph) needs to send their costs at
        # init.Each leaf factor sends his costs to its only variable.
        # When possible it is better to use a variable with integrated costs
        # instead of a variable with an unary relation representing costs.
        if len(self.variables) == 1 and self.start_messages in ["leafs", "leafs_vars"]:
            for v in self.variables:
                costs_v = factor_costs_for_var(
                    self.factor, v, self._costs, self.mode
                )
                self.post_msg(v.name, MaxSumMessage(costs_v))
                self.logger.info(
                    f"Sending init messages from factor {self.name} -> {v.name} : {costs_v}"
                )
        elif self.start_messages == "all":
            for v in self.variables:
                costs_v = factor_costs_for_var(
                    self.factor, v, self._costs, self.mode
                )
                self.post_msg(v.name, MaxSumMessage(costs_v))
                self.logger.info(
                    f"Sending init messages from factor {self.name} -> {v.name} : {costs_v}"
                )

    @register("max_sum")
    def on_msg(self, variable_name, recv_msg, t):
        # No implementation here, simply used to declare the kind of message supported
        # by this computation
        pass

    def footprint(self) -> float:
        return computation_memory(self.computation_def.node)

    def on_new_cycle(self, messages, cycle_id) -> Optional[List]:

        # Collect costs messages from neighbor variables for this cycle (aka iteration)
        for sender, (message, t) in messages.items():
            self._costs[sender] = message.costs

        for v in self.variables:
            costs_v = factor_costs_for_var(self.factor, v, self._costs, self.mode)
            prev_costs, count = self._prev_messages[v.name]

            # Apply damping to computed costs:
            if self.damping_nodes in ["factors", "both"]:
                costs_v = apply_damping(
                    costs_v, prev_costs, self.damping
                )

            # Check if there was enough change to send the message
            if not approx_match(
                    costs_v, prev_costs, self.stability_coef
            ):
                # Not same as previous : send
                self.logger.debug(
                    f"Sending first time from factor {self.name} -> {v.name} : {costs_v}"
                )
                self.post_msg(v.name, MaxSumMessage(costs_v))
                self._prev_messages[v.name] = costs_v, 1

            elif count < SAME_COUNT:
                # Same as previous, but not yet sent SAME_COUNT times: send
                self.logger.debug(
                    f"Sending {count} time from variable {self.name} -> {v.name} : {costs_v}"
                )
                self.post_msg(v.name, MaxSumMessage(costs_v))
                self._prev_messages[v.name] = costs_v, count + 1
            else:
                # Same and already sent SAME_COUNT times: no-send
                self.logger.debug(
                    f"Not sending (similar) from {self.name} -> {v.name} : {costs_v}"
                )

        return None

def factor_costs_for_var(factor: Constraint, variable: Variable, recv_costs, mode: str):
    """
    Computes the marginals to be send by a factor to a variable

    The content of this message is a table d -> mincost where
    * d is a value of the domain of the variable v
    * mincost is the minimum value of f when the variable v take the
      value d

    Parameters
    ----------
    factor: Constraint
        the factor that will send these cost to `variable`
    variable: Variable

    recv_costs: Dict
        a dict containing the costs received from other variables
    mode: str
        "min" or "max"

    Returns
    -------
    Dict:
        a dict that associates a cost to each value in the domain of `variable`

    """
    # TODO: support passing list of valid assignment as param
    costs = {}
    other_vars = factor.dimensions[:]
    other_vars.remove(variable)
    for d in variable.domain:
        # for each value d in the domain of v, calculate min cost (a)
        # where a is any assignment where v = d
        # cost (a) = f(a) + sum( costvar())
        # where costvar is the cost received from our other variables

        optimal_value = float("inf") if mode == "min" else -float("inf")

        for assignment in generate_assignment_as_dict(other_vars):
            assignment[variable.name] = d
            f_val = factor(**assignment)

            sum_cost = 0
            # sum of the costs from all other variables
            for another_var, var_value in assignment.items():
                if another_var == variable.name:
                    continue
                if another_var in recv_costs:
                    if var_value not in recv_costs[another_var]:
                        continue
                    sum_cost += recv_costs[another_var][var_value]
                else:
                    # we have not received yet costs from variable v
                    pass

            current_val = f_val + sum_cost
            if (optimal_value > current_val and mode == "min") or (
                optimal_value < current_val and mode == "max"
            ):

                optimal_value = current_val

        costs[d] = optimal_value

    return costs

class MaxSumVariableComputation(SynchronousComputationMixin, VariableComputation):
    """

    """

    def __init__(self, comp_def: ComputationDef):
        super().__init__(comp_def.node.variable, comp_def)
        assert comp_def.algo.algo == "maxsum"
        self.mode = comp_def.algo.mode
        self.damping = comp_def.algo.params["damping"]
        self.damping_nodes = comp_def.algo.params["damping_nodes"]
        self.stability_coef = comp_def.algo.params["stability"]
        self.start_messages = comp_def.algo.params["start_messages"]
        self.logger.info(f"Running maxsum with params: {comp_def.algo.params}")

        # The list of factors (names) this variables is linked with
        self.factors = [link.factor_node for link in comp_def.node.links]

        # costs : this dict is used to store, for each value of the domain,
        # the associated cost sent by each factor this variable is involved
        # with. { factor : {domain value : cost }}
        self.costs = {}

        # to store previous messages, necessary to detect convergence
        self._prev_messages = defaultdict(lambda: (None, 0))

        # Add noise to the variable, on top of cost if needed
        if comp_def.algo.params["noise"] != 0:
            self.logger.info(
                f"Adding noise on variable {comp_def.algo.params['noise']}"
            )
            self._variable = VariableNoisyCostFunc(
                self.variable.name,
                self.variable.domain,
                cost_func=lambda x: self.variable.cost_for_val(x),
                initial_value=self.variable.initial_value,
                noise_level=comp_def.algo.params["noise"],
            )

    @register("max_sum")
    def on_msg(self, variable_name, recv_msg, t):
        # No implementation here, simply used to declare the kind of message supported
        # by this computation
        pass

    def on_start(self) -> None:
        # Select our initial value
        if self.variable.initial_value is not None:
            self.value_selection(self.variable.initial_value)
        else:
            self.value_selection(*select_value(self.variable, self.costs, self.mode))
        self.logger.info(f"Initial value selected {self.current_value}")

        if len(self.factors) == 1 and self.start_messages == "leafs":
            # Only send costs if we are a leaf:
            single_factor = self.factors[0]
            costs_f = costs_for_factor(
                self.variable, single_factor, self.factors, self.costs
            )
            self.logger.info(
                f"Sending init msg from leaf variable {self.name} to single factor {single_factor} : {costs_f}"
            )
            self.post_msg(single_factor, MaxSumMessage(costs_f))

        elif self.start_messages in ["leafs_vars", "all"]:
            # in "leafs_vars" mode, send our costs to all the factors we depends on.
            for f in self.factors:
                costs_f = costs_for_factor(
                    self.variable, f, self.factors, self.costs
                )
                self.logger.info(
                    f"Sending init msg from variable {self.name} to factor {f} : {costs_f}"
                )
                self.post_msg(f, MaxSumMessage(costs_f))

    def on_new_cycle(self, messages, cycle_id) -> Optional[List]:

        # Collect costs messages from neighbor factors for this cycle (aka iteration)
        for sender, (message, t) in messages.items():
            self.costs[sender] = message.costs

        # select our value, based on new costs
        self.value_selection(*select_value(self.variable, self.costs, self.mode))

        # Compute and send our own costs to  factors.

        for f_name in self.factors:
            costs_f = costs_for_factor(self.variable, f_name, self.factors, self.costs)
            prev_costs, count = self._prev_messages[f_name]

            # Apply damping to computed costs:
            if self.damping_nodes in ["vars", "both"]:
                costs_f = apply_damping(costs_f, prev_costs, self.damping)

            # Check if there was enough change to send the message
            if not approx_match(costs_f, prev_costs, self.stability_coef):
                # Not same as previous : send
                self.logger.debug(
                    f"Sending first time from variable {self.name} -> {f_name} : {costs_f}"
                )
                self.post_msg(f_name, MaxSumMessage(costs_f))
                self._prev_messages[f_name] = costs_f, 1

            elif count < SAME_COUNT:
                # Same as previous, but not yet sent SAME_COUNT times: send
                self.logger.debug(
                    f"Sending {count} time from variable {self.name} -> {f_name} : {costs_f}"
                )
                self.post_msg(f_name, MaxSumMessage(costs_f))
                self._prev_messages[f_name] = costs_f, count + 1
            else:
                # Same and already sent SAME_COUNT times: no-send
                self.logger.debug(
                    f"Not sending (similar) from {self.name} -> {f_name} : {costs_f}"
                )
        return None

    def _match_previous(self, f_name, costs):
        """
        Check if a cost message for a factor f_name match the previous message
        sent to that factor.

        :param f_name: factor name
        :param costs: costs sent to this factor
        :return:
        """
        prev_costs, count = self._prev_messages[f_name]
        if prev_costs is not None:
            same = approx_match(costs, prev_costs, self.stability_coef)
            return same, count
        else:
            return False, 0

def select_value(
    variable: Variable, costs: Dict[str, Dict], mode: str
) -> Tuple[Any, float]:
    """
    Select the value for `variable` with the best cost / reward (depending on `mode`)

    Parameters
    ----------
    variable: Variable
        the variable for which we need to select a value
    costs: Dict
        a dict { factorname : { value : costs}} representing the cost messages received from factors
    mode: str
        min or max
    Returns
    -------
    Tuple:
        a Tuple containing the selected value and the corresponding cost for
        this computation.
    """

    # Select a value from the domain, based on the variable cost and
    # the costs received from neighbor factors
    d_costs = {d: variable.cost_for_val(d) for d in variable.domain}
    for d in variable.domain:
        for f_costs in costs.values():
            d_costs[d] += f_costs[d]

    from operator import itemgetter

    # print(f" ### On selecting value for {variable.name} : {d_costs}")
    if mode == "min":
        optimal_d = min(d_costs.items(), key=itemgetter(1))
    else:
        optimal_d = max(d_costs.items(), key=itemgetter(1))

    return optimal_d[0], optimal_d[1]

def costs_for_factor(
    variable: Variable, factor: FactorName, factors: List[Constraint], costs: Dict
) -> Dict[VarVal, Cost]:
    """
    Produce the message that must be sent to factor f.

    The content if this message is a d -> cost table, where
    * d is a value from the domain
    * cost is the sum of the costs received from all other factors except f
    for this value d for the domain.

    Parameters
    ----------
    variable: Variable
        the variable sending the message
    factor: str
        the name of the factor the message will be sent to
    factors: list of Constraints
        the constraints this variables depends on
    costs: dict
        the accumulated costs received by the variable from all factors

    Returns
    -------
    Dict:
        a dict containing a cost for each value in the domain of the variable
    """
    # If our variable has integrated costs, add them
    msg_costs = {d: variable.cost_for_val(d) for d in variable.domain}

    sum_cost = 0
    for d in variable.domain:
        for f in factors:
            if f == factor or f not in costs:
                continue
            # [f for f in factors if f != factor and f in costs]:
            f_costs = costs[f]
            if d not in f_costs:
                continue
            c = f_costs[d]
            sum_cost += c
            msg_costs[d] += c

    # Experimentally, when we do not normalize costs the algorithm takes
    # more cycles to stabilize
    # return {d: c for d, c in msg_costs.items() }

    # Normalize costs with the average cost, to avoid exploding costs
    avg_cost = sum_cost / len(msg_costs)
    normalized_msg_costs = {
        d: c - avg_cost for d, c in msg_costs.items()
    }

    return normalized_msg_costs

def apply_damping(costs_f, prev_costs, damping):
    damped_costs = {}
    if prev_costs is not None:
        for d, c in costs_f.items():
            damped_costs[d] = damping * prev_costs[d] + (1 - damping) * c
        return damped_costs
    return costs_f

def approx_match(costs, prev_costs, stability_coef):
    """
    Check if a cost message match the previous message.

    Costs are considered to match if the variation is bellow STABILITY_COEFF.

    :param costs: costs as a dict val -> cost
    :param prev_costs: previous costs as a dict val -> cost
    :return: True if the cost match
    """
    if prev_costs is None:
        return False

    for d, c in costs.items():
        prev_c = prev_costs[d]
        if prev_c != c:
            delta = abs(prev_c - c)
            if prev_c + c != 0:
                if not ((2 * delta / abs(prev_c + c)) < stability_coef):
                    return False
            else:
                return False
    return True

def _valid_assignments(constraint: Constraint, infinity_value):
    """
    Return a list of all valid assignments for the Constraint
    """
    valid_assignments = []
    for assignment in generate_assignment_as_dict(constraint.dimensions[:]):
        if abs(constraint(**assignment)) != infinity_value:
            valid_assignments.append(assignment)
    return valid_assignments


import logging
import paho.mqtt.client as paho
import paho.mqtt.publish as pahopub
import os

logger = logging.getLogger()

class Mqtt():

    def __init__(self, smarthome, host, port):
        self._sh = smarthome
        self.broker_ip = host
        self.broker_port = int(port)
        self.clients = []
        self.items = {}
        self.logics = {}
        self.publisher = self.create_client('main')

    def run(self):
        self.alive = True

    def stop(self):
        self.alive = False
        for client in self.clients:
            logger.debug('Stopping mqtt client {0}'.format(client._client_id))
            client.loop_stop()

    def parse_item(self, item):
        if 'mqtt_topic' in item.conf:
            item.conf['mqtt_topic_in'] = item.conf['mqtt_topic']
            item.conf['mqtt_topic_out'] = item.conf['mqtt_topic']
            logger.debug("parse item: {0}".format(item))

        if 'mqtt_topic_in' in item.conf:
            client = self.create_client(item.id())
            client.on_message = lambda client, obj, msg: self.items[msg.topic](msg.payload, 'MQTT')
            client.on_connect = lambda client, obj, rc: client.subscribe(item.conf['mqtt_topic_in'], 2)
            client.loop_start()
            self.items[item.conf['mqtt_topic_in']] = item
            logger.debug('Item [{0}] is listening for messages on topic [{1}]'.format(item, item.conf['mqtt_topic_in']))

        if 'mqtt_topic_out' in item.conf:
            return self.update_item

    def parse_logic(self, logic):
        if 'mqtt_topic' in logic.conf:
            client = self.create_client(logic.name)
            client.on_message = lambda client, obj, msg: self.logics[msg.topic].trigger('MQTT', msg.topic, msg.payload)
            client.subscribe(logic.conf['mqtt_topic'], 2)
            client.loop_start()
            self.logics[logic.conf['mqtt_topic']] = logic
            logger.debug('Logic [{0}] is listening for messages on topic [{1}]'.format(logic.name, logic.conf['mqtt_topic']))

    def update_item(self, item, caller=None, source=None, dest=None):
        pahopub.single(topic=item.conf['mqtt_topic_out'], payload=str(item()), qos=2, hostname=self.broker_ip)
        logger.info("update item: {0}".format(item.id()))
        logger.debug("Mqtt caller item: {0} \t Source: {1} \t Destination:{2}".format(caller, source, dest))
        logger.info("update topic: {0}\t{1}".format(item.conf['mqtt_topic_out'], str(item())))

    def create_client(self, name):
        client = paho.Client('{0}.{1}'.format(os.uname()[1], name))
        client.connect(self.broker_ip, self.broker_port, 60)
        self.clients.append(client)
        return client


import logging
import os
import time
from PyQt5 import QtWidgets, QtCore

import matplotlib
import six
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT

import options_figure
import options_utils
from gui_utils import get_icon

# Log Dialog Window ##########################################################

LOG_FORMAT = "%(asctime)s %(levelname)s %(message)s"
DATE_FORMAT = '%d/%m/%Y %H:%M:%S'

class LogDialog(QtWidgets.QDialog, logging.StreamHandler):
    """ Logging dialog window, created by Jaime """

    _update_signal = QtCore.pyqtSignal(str)

    def __init__(self, fmt=LOG_FORMAT, datefmt=DATE_FORMAT, parent=None):
        QtWidgets.QDialog.__init__(self, parent=parent)
        logging.Handler.__init__(self)

        self.resize(855, 655)
        layout = QtWidgets.QVBoxLayout()
        self._log_text = QtWidgets.QPlainTextEdit(parent)
        self._log_text.setReadOnly(True)
        layout.addWidget(self._log_text)
        self.setLayout(layout)

        self.setFormatter(logging.Formatter(fmt, datefmt))
        self.setLevel(logging.DEBUG)

        self._update_signal.connect(self._update_log, QtCore.Qt.QueuedConnection)

    def _update_log(self, msg):
        self._log_text.appendPlainText(msg)

    def emit(self, record):
        msg = self.format(record)
        self._update_signal.emit(msg)

class LogStatusBar(QtWidgets.QStatusBar, logging.StreamHandler):
    """ Logging dialog window, created by Jaime """

    _update_signal = QtCore.pyqtSignal(str)

    def __init__(self, fmt=LOG_FORMAT, datefmt=DATE_FORMAT, parent=None):
        QtWidgets.QStatusBar.__init__(self, parent=parent)
        logging.Handler.__init__(self)

        self._status_text = QtWidgets.QLabel()
        self.addWidget(self._status_text, 1)

        self.setFormatter(logging.Formatter(fmt, datefmt))
        self.setLevel(logging.INFO)

        self._update_signal.connect(self.showMessage, QtCore.Qt.QueuedConnection)

    def emit(self, record):
        msg = self.format(record)
        self._update_signal.emit(msg)

# Navigation Toolbar #########################################################

class NavigationToolbar(NavigationToolbar2QT):
    """ Customized Navigation Toolbar """

    ICON_SIZE = 24

    def __init__(self, canvas,
                 save_fun=None, load_fun=None, export_fun=None, import_fun=None, parent=None):
        self.load_data = load_fun
        self.export_plot = export_fun
        self.import_plot = import_fun

        self.toolitems = list(self.toolitems)

        if hasattr(canvas, "move_legend") and hasattr(canvas, "update_legend"):
            self.update_legend = canvas.update_legend
            self.move_legend = canvas.move_legend

            self.toolitems.insert(7, (None, None, None, None))

            self.toolitems.insert(8, (
                "Move Legend", "Move the legend location to predefined settings.",
                "arrows", "move_legend"
            ))

            self.toolitems.insert(9, (
                "Update Legend", "Update the legend.",
                "refresh", "update_legend"
            ))

        if save_fun is not None:
            self.save_figure = save_fun  # otherwise defined by super-class

        if self.load_data is not None:
            self.toolitems.append((
                    "Load Data", "Plot data from tfs files.",
                    "folder_open", "load_data"
                ))

        if self.export_plot is not None:
            self.toolitems.append((
                    "Export Plot", "Export plot to .dly file format.",
                    "export", "export_plot"
                ))

        if self.import_plot is not None:
            self.toolitems.append((
                    "Import Plot", "Import plot from .dly file format.",
                    "import", "import_plot"
                ))

        super(NavigationToolbar, self).__init__(canvas, parent)

    def _init_toolbar(self):
        """ Called from the super function """
        self.basedir = os.path.join(matplotlib.rcParams['datapath'], 'images')

        for text, tooltip_text, image_file, callback in self.toolitems:
            if text is None:
                self.addSeparator()
            else:
                try:
                    icon = get_icon(image_file)
                except IOError:
                    icon = self._icon(image_file + '.png')

                a = self.addAction(icon,
                                   text, getattr(self, callback))
                self._actions[callback] = a
                if callback in ['zoom', 'pan']:
                    a.setCheckable(True)
                if tooltip_text is not None:
                    a.setToolTip(tooltip_text)
                if text == 'Subplots':
                    a = self.addAction(self._icon("qt4_editor_options.png"),
                                       'Customize', self.edit_parameters)
                    a.setToolTip('Edit axis, curve and image parameters')

        self.buttons = {}

        # Add the x,y location widget at the right side of the toolbar
        # The stretch factor is 1 which means any resizing of the toolbar
        # will resize this label instead of the buttons.
        if self.coordinates:
            self.locLabel = QtWidgets.QLabel("", self)
            self.locLabel.setAlignment(
                QtCore.Qt.AlignRight | QtCore.Qt.AlignTop)
            self.locLabel.setSizePolicy(
                QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding,
                                      QtWidgets.QSizePolicy.Ignored))
            labelAction = self.addWidget(self.locLabel)
            labelAction.setVisible(True)

        # reference holder for subplots_adjust window
        self.adj_window = None

        # Aesthetic adjustments - we need to set these explicitly in PyQt5
        # otherwise the layout looks different - but we don't want to set it if
        # not using HiDPI icons otherwise they look worse than before.

        self.setIconSize(QtCore.QSize(NavigationToolbar.ICON_SIZE, NavigationToolbar.ICON_SIZE))
        self.layout().setSpacing(12)

    def edit_parameters(self):
        allaxes = self.canvas.figure.get_axes()
        if not allaxes:
            QtWidgets.QMessageBox.warning(
                self.parent, "Error", "There are no axes to edit.")
            return
        elif len(allaxes) == 1:
            axes, = allaxes
        else:
            titles = []
            for axes in allaxes:
                name = (axes.get_title() or
                        " - ".join(filter(None, [axes.get_xlabel(),
                                                 axes.get_ylabel()])) or
                        "<anonymous {} (id: {:#x})>".format(
                            type(axes).__name__, id(axes)))
                titles.append(name)
            item, ok = QtWidgets.QInputDialog.getItem(
                self.parent, 'Customize', 'Select axes:', titles, 0, False)
            if ok:
                axes = allaxes[titles.index(six.text_type(item))]
            else:
                return

        options_figure.figure_edit(axes, self)

# Figure Canvas ################################################################

class FigureCanvasExt(FigureCanvas):
    """ Extended FigureCanvas.

    TODO: Multiaxes
    - Legend positions can be easily saved and restored

    """

    LEGEND_LOCATIONS = [0, 1, 2, 3, 4, None]

    def __init__(self, figure):
        super(FigureCanvasExt, self).__init__(figure)
        self._legend_location_index = 0
        self._legend_locations = list(FigureCanvasExt.LEGEND_LOCATIONS)

    def move_legend(self):
        self._legend_location_index = (
                                              self._legend_location_index + 1
                                      ) % len(self._legend_locations)
        self._set_legend_loc()

    def update_legend(self):
        axes = self.figure.gca()
        options_utils.regenerate_legend(axes, force_new=True)
        self._save_legend_loc()
        self.draw()

    def _set_legend_loc(self):
        legend = self.figure.gca().get_legend()
        if legend is not None:
            loc = self._legend_locations[
                self._legend_location_index
            ]
            if loc is None:
                legend.set_visible(False)
            else:
                legend.set_visible(True)
                legend._set_loc(loc)

            self.draw()

    def _save_legend_loc(self):
        legend = self.figure.gca().get_legend()
        if legend:
            self._legend_location_index = 0
            self._legend_locations = ([legend._get_loc()] +
                                      list(FigureCanvasExt.LEGEND_LOCATIONS))

    def update_figure(self, figure):
        """ Change Figure for this canvas """
        figure.canvas = self
        self.figure = figure
        self._save_legend_loc()
        self._set_legend_loc()
        self.draw()

    def set_pickers(self, tol=0.5):
        """ Set all artists to send pick events when they are clicked """
        axes = self.figure.get_axes()
        for ax in axes:
            for l in ax.lines:
                width = max(1.1*l.get_markersize(), 1.5*l.get_linewidth())
                l.set_picker(width)

            for a in [ax.xaxis, ax.yaxis]:
                a.set_picker(tol)
                for c in a.get_children():
                    c.set_picker(True)

            for t in ax.texts:
                t.set_picker(True)

            legend = ax.get_legend()
            if legend is not None:
                legend.set_picker(True)

            ax.title.set_picker(True)
            # ax.set_picker(True)

    def unset_pickers(self):
        for c in self.figure.get_children():
            try:
                c.set_picker(False)
                c.set_draggable(False)
            except AttributeError:
                pass

# Figure Canvas ################################################################

class DragHandler(object):
    """ Handles the dragging of objects around the screen """
    time_tol = 0.1  # time between drag'n'drop to be registered (in sec)

    def __init__(self, artist, mouseevent):
        self.artist = artist
        self.time = time.time()
        self.pick_pos_data = [mouseevent.xdata, mouseevent.ydata]
        self.pick_pos_disp = [mouseevent.x, mouseevent.y]
        self.artist_pos_data = artist.get_position()
        self.artist_pos_disp = artist._get_xy_display()

    def move(self, mouseevent):
        # if (time.time() - self.time) < self.time_tol:
        #     return

        mouse_pos_disp = [mouseevent.x, mouseevent.y]
        mouse_pos_data = [mouseevent.xdata, mouseevent.ydata]

        if any(pos is None for pos in self.pick_pos_data + mouse_pos_data):
            # something was outside of the plotting area -> use figure coodinates
            return  # TODO: DOES NOT WORK AS EXPECTED
            to_pos = self._get_new_position(self.artist_pos_disp,
                                            mouse_pos_disp,
                                            self.pick_pos_disp)
            self.artist.transform = self.artist.figure.transFigure
            to_pos = self.artist.transform.inverted().transform(to_pos)
        else:
            # keep everything inside the plotting area
            to_pos = self._get_new_position(self.artist_pos_data,
                                            mouse_pos_data,
                                            self.pick_pos_data)

        self.artist.set_position(to_pos)

    def _get_new_position(self, from_pos, mouse_pos, pick_pos):
        return [fp + mp - pp for fp, mp, pp in zip(from_pos, mouse_pos, pick_pos)]


from flask import Flask, jsonify
from flask import request
import librosa
from pydub import AudioSegment
import ffmpeg
app = Flask(__name__)

@app.route('/',methods = ['GET','POST'])
def calcluateBPM():
	if request.method == 'POST':
		src = request.files['src']
		return 'file uploaded '

	if request.method == 'GET':
		d = {}
		src
		try:
			y, sr = librosa.load(src)
		except:
			print("Unable to find file")
		original_tempo, beat_frames = librosa.beat.beat_track(y=y, sr=sr)
		d['bpm']= '{:2f}'.format(original_tempo)
		return d

if __name__ == "__main__":
    app.run(debug=True)


"""
Print bigrams where first word is adjective and second word is noun

typical usage:

% python3 bigrams_pos.py ../../data/brown_txt/*.txt > bigrams_pos_JN.tsv

each output line corresponds to a bigram:

word_A word_B frequency_of_bigram
"""

import sys
import re
from collections import defaultdict

def tokenize(line, l_n):
    '''Return a list of tokens and POS values extracted from line'''
    tokens = []
    poses = []
    for j, word in enumerate(line.rstrip().split()):
        fields = word.split('/')
        if len(fields) != 2:
            print(
                f'Bad input: word "{word}", which is the {j + 1}-th word in line {l_n + 1}')
            sys.exit(1)
        token, pos = word.split('/')
        tokens.append(token)
        poses.append(pos)
    return tokens, poses

def dofile(fname, bigramfreq, regex1, regex2):
    '''Read a corpus and update bigram frequencies if the tokens that comprise the
    bigram have the appropriate POS tags.'''
    for i, line in enumerate(open(fname, encoding='utf-8', errors='surrogateescape')):
        sentence, pos = tokenize(line, i)
        for j in range(len(sentence) - 1):
            if re.match(regex1, pos[j]) and re.match(regex2, pos[j+1]):
                bigram = f"{sentence[j]}\t{sentence[j+1]}"
                bigramfreq[bigram] += 1

def bigrams_pos():
    """Main function that prints bigrams where first word is adjective and second word is noun"""
    # first word is adjective
    regex_pos1 = "JJ.*"
    # second word is noun
    regex_pos2 = "N.*"

    # bigram frequencies { bigram_form : freq }
    bigramfreq = defaultdict(int)
    for fname in sys.argv[1:]:
        print(fname, file=sys.stderr)
        # update bigram freqs and word freqs
        dofile(fname, bigramfreq, regex_pos1, regex_pos2)

        sorted_bigramfreq = dict(sorted(bigramfreq.items(), key=lambda item: item[1], reverse=True))

    for bigram, fbigram in sorted_bigramfreq.items():
        print(f"{bigram}\t{fbigram}")

if __name__ == "__main__":
    bigrams_pos()


###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Plugin for uploading pdf files of a certain markup, provided by BZK
"""

from datetime import date
import re

from amcat.scripts.article_upload.upload import UploadScript
from amcat.models.article import Article
from .pdf import PDFParser
from .bzk_aliases import BZK_ALIASES as MEDIUM_ALIASES

class BZKPDFScraper(UploadScript):
    def _scrape_unit(self, unit):
        parser = PDFParser()
        self.index = []
        article_lines = []
        headline = ""
        doc = parser.load_document(self.options['file'])
        for i, p in enumerate(parser.process_document(doc)):
            #is this page an index page?
            index_pattern = re.compile("^[^\(]+\([^\)]+\)..+[0-9]+$")
            if any([index_pattern.match(line.get_text()) for line in parser.get_textlines(p)]):
                for line in parser.get_textlines(p):
                    pattern = re.compile("([^\(]+)(\([0-9]+\))? \(([^\)]+)\).+")
                    text = line.get_text()
                    result = pattern.search(text)
                    if result:
                        h = result.group(1); m = result.group(3)
                        self.index.append((h, m))
                continue

            #if not, scrape lines on page for current article
            for line in parser.get_textlines(p):
                text = line.get_text()
                if text.lower().strip() in [i[0].lower().strip() for i in self.index]:

                    # title is recognized. yield old article, start new
                    if len(headline) > 0:
                        article =  self.getarticle(headline, article_lines)
                        yield article

                    headline = text
                    article_lines = []

                article_lines.append(text)

            #last article
            yield self.getarticle(headline, article_lines)

    def getarticle(self, headline, lines):
        article = Article(headline = headline)
        text = ""
        for line in lines[2:]:
            if len(line) > 2:
                text += "\n" + line

        text = text.replace("-\n","")
        text = text.replace("  "," ")
        text = text.replace("\n"," ")

        article.text = text
        date_pattern = re.compile("([0-9]{2,2})\-([0-9]{2,2})\-([0-9]{4,4})")
        result = date_pattern.search(lines[1])
        article.date = date(
            int(result.group(3)),
            int(result.group(2)),
            int(result.group(1)))
        pagenum_pattern = re.compile("\(p.([0-9]+)([0-9\-]+)?\)")
        result = pagenum_pattern.search(lines[1])
        if result:

            article.pagenr = int(result.group(1))

        for h, medium in self.index:
            if article.headline.lower().strip() in h.lower().strip():
                article.set_property("medium", self.get_medium(medium))

        return article

    def get_medium(self, medium):
        if not medium or len(medium) < 1:
            medium = "unknown"
        return MEDIUM_ALIASES.get(medium, medium)

if __name__ == "__main__":
    from amcat.scripts.tools import cli
    cli.run_cli(BZKPDFScraper)


import math as m
import matplotlib.pyplot as plt

#zadanie 1
#funckja 8

Tc=4
fs = 8000
f = 16
fi = 20

N=Tc*fs

x1=[]
y1=[]

def x(t):
    wynik = (1-t)*m.sin(2*m.pi*f*t+fi)*m.cos(4*m.pi*t)
    return wynik

for i in range(fs):
    x1.append(i/fs)
    y1.append(x(i/fs))

plt.plot(x1,y1)
plt.title("Zadanie 1")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

#zadanie 2
#funkcja 5

def y(t):
    wynik = (2*t*m.sin(0.5*t*m.pi)+1.5)*m.cos(9*m.pi*t+m.pi*t)
    return wynik

x2=[]
y2=[]
for i in range(fs):
    x2.append(i/fs)
    y2.append(y(i/fs))

plt.plot(x2,y2)
plt.title("Zadanie 2")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

def z(t):
    wynik = y(t)*x(t)+m.fabs(x(t)+2)*(m.pow(y(t), 2)+ 0.32)
    return wynik

x3=[]
y3=[]
for i in range(fs):
    x3.append(i/fs)
    y3.append(z(i/fs))

plt.plot(x3,y3)
plt.title("Zadanie 3")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

def v(t):
    wynik = m.sqrt(m.fabs(x(t)*z(t)+10))*(m.fabs(y(t))+1.2)*m.sin(2*m.pi*t)
    return wynik

x4=[]
y4=[]
for i in range(fs):
    x4.append(i/fs)
    y4.append(v(i/fs))

plt.plot(x4,y4)
plt.title("Zadanie 4")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

#zadanie 3
#funckja 4

def u(t):
    if t < 0.5 and t >=0:
        wynik = 0.9*m.sin(2*m.pi*t*8-m.pi/3)+m.log2(m.fabs(m.cos(7*m.pow(t, 2))+2.2))
    elif t < 1.9 and t >= 0.5:
        wynik = (m.sin(2*m.cos(4*m.pi*t)*m.pi*t))/(2*m.pow(t, 2)+1)
    elif t < 3.7 and t >= 1.9:
        wynik = m.pow((t-1.9), 2)-m.cos(13*t)
    elif t < 4.9 and t >= 3.7:
        wynik = 0.5*m.pow(t, 0.7)*m.sin(8*t)
    elif t < 6.4 and t >= 4.9:
        wynik = (2+m.sin(18*t))/(3+m.cos(28*t))
    return wynik

x5=[]
y5=[]
for i in range(fs):
    x5.append(i/fs)
    y5.append(u(i/fs))

plt.plot(x5,y5)
plt.title("Zadanie 5")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

#zadanie 4
#funckja 7

fs = 22050
Tc = 1

def b1(t):
    for h in range(1,2):
        wynik = (m.sin(m.pi*t*(2*m.pow(h, 2)+h)))/(2*m.pow(h, 2)*(2.5+m.cos(h*m.pi)-1))
    return wynik

x6=[]
y6=[]
for i in range(fs):
    x6.append(i/fs)
    y6.append(b1(i/fs))

plt.plot(x6,y6)
plt.title("Zadanie 6")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

def b2(t):
    for h in range(1,4):
        wynik = (m.sin(m.pi*t*(2*m.pow(h, 2)+h)))/(2*m.pow(h, 2)*(2.5+m.cos(h*m.pi)-1))
    return wynik

x7=[]
y7=[]
for i in range(fs):
    x7.append(i/fs)
    y7.append(b2(i/fs))

plt.plot(x7,y7)
plt.title("Zadanie 7")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

def b3(t):
    for h in range(1,20):
        wynik = (m.sin(m.pi*t*(2*m.pow(h, 2)+h)))/(2*m.pow(h, 2)*(2.5+m.cos(h*m.pi)-1))
    return wynik

x8=[]
y8=[]
for i in range(fs):
    x8.append(i/fs)
    y8.append(b3(i/fs))

plt.plot(x8,y8)
plt.title("Zadanie 8")
plt.xlabel("t(s)")
plt.ylabel("x(t)")
plt.show()

#wykresy są 1 sekundowe sprawdzić to i uzupełnić pospisy osi i wykresów


"""Factor Of 3 Codevita 9 Solution
Problem Description
Given an array arr, of size N, find whether it is possible to rearrange the elements of array such that sum of no two adjacent elements is divisible by 3.

Constraints
1 <= T <= 10

2 <= N <= 10^5

1 <= arr[i] <= 10^5

Input
First line contains integer T denoting the number of testcases.

Each test cases consists of 2 lines as follows-

First line contains integer N denoting the size of the array.

Second line contains N space separated integers.

Output
For each test case print either ""Yes"" or ""No"" (without quotes) on new line.

Time Limit
1

Examples
Example 1

Input

1

4

1 2 3 3

Output

Yes

Explanation

Some of the rearrangements can be {2,1,3,3}, {3,3,1,2}, {2,3,3,1}, {1,3,3,2},...

We can see that there exist at least 1 combination {3,2,3,1} where sum of 2 adjacent number is not divisible by 3. Other combinations can be {1,3,2,3}, {2,3,1,3}.

Hence the output is Yes.

Example 2

Input

1

4

3 6 1 9

Output

No

Explanation

All possible combination of {3,6,1,9} are

{1,3,6,9}, {1,3,9,6}, {1,6,9,3}, {1,6,3,9}, {1,9,3,6}, {1,9,6,3},

{6,1,3,9}, {6,1, 9,3}, {6,3,1,9}, {6,3,9,1}, {6,9,1,3}, {6,9,3,1},

{3,1,6,9}, {3,1,9,6}, {3,9,1,6}, {3,9,6,1}, {3,6,1,9}, {3,6,9,1},

{9,1,3,6}, {9,1,6,3}, {9,3,1,6}, {9,3,6,1}, {9,6,1,3}, {9,6,3,1}.

Since none of these combinations satisfy the condition, the output is No
# A Python program to print all
# permutations using library function"""
from itertools import permutations
t=int(input())
n=int(input())
IntList=[]
inputs = list(map(str, input().split()))
for i in range(n):
    y=int(inputs[i])
    IntList.append(y)

perm = permutations(IntList)
count=0
mega_list=[]
# Print the obtained permutations
for i in list(perm):
    #print(i)
    listi=list(i)
    #print(listi)
    mega_list.append(listi)
count=0
for listi in mega_list:
    for i in range(len(listi)):
        if i!=(len(listi)-1):
            if (listi[i] + listi[i + 1]) % 3 != 0:
                count=count+1
    if count==3:
        break
    else:
        count=0
if count:
    print("yes")
else:
    print("no")


from rest_framework import status
from rest_framework.reverse import reverse
from rest_framework.test import APITestCase

from restapi.models.web_radio import WebRadio

class TestGetDetails(APITestCase):

    def setUp(self):
        super(TestGetDetails, self).setUp()
        self.webradio = WebRadio.objects.create(name="test", url="http://test.com")
        self.url = reverse('api:webradios:retrieve_update_destroy', args=[self.webradio.id])

    def test_get_details(self):
        response = self.client.get(self.url, format='json')
        expected_response = {'id': self.webradio.id, 'is_default': False, 'name': 'test', 'url': 'http://test.com'}
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(expected_response, response.json())


import unittest
import os
import cv2
import numpy as np

from style_transfer_train import TrainStyleTransfer

class TestStyleTransfer(unittest.TestCase):

    def test_transfer_style(self):
        # Initialize the style transferer
        transferer = TrainStyleTransfer()

        # Load example content and style images
        content_path = os.path.join(os.getcwd(), 'example_images', 'content.jpg')
        content = cv2.imread(content_path)
        style_path = os.path.join(os.getcwd(), 'example_images', 'style.jpg')
        style = cv2.imread(style_path)

        # Transfer the style
        stylized_image = transferer.transfer_style(content, style)

        # Make sure the output has the expected shape
        self.assertIsInstance(stylized_image, np.ndarray)
        self.assertEqual(stylized_image.dtype, np.uint8)
        self.assertEqual(stylized_image.shape, content.shape)

        # Test with a different style image
        style_path = os.path.join(os.getcwd(), 'example_images', 'style2.jpg')
        style = cv2.imread(style_path)
        stylized_image = transferer.transfer_style(content, style)
        self.assertIsInstance(stylized_image, np.ndarray)
        self.assertEqual(stylized_image.dtype, np.uint8)
        self.assertEqual(stylized_image.shape, content.shape)

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/python
import datetime
import re
import urllib2

def parse_item(html):
    tds = re.findall('(.*?)	', html)
    match = re.match('.*', tds[2])
    id = match.group(1)
    return datetime.datetime.strptime(tds[1], "%Y-%m-%d"), id

def fetch_webpage(url):
    proxy_handler = urllib2.ProxyHandler({})
    opener = urllib2.build_opener(proxy_handler)
    result = opener.open(url)
    return result.read()

def get_trs_from_whole_webpage(html):
    trs = re.findall('
.*?
', html)
    return trs

def get_all_commits(html):
    trs = get_trs_from_whole_webpage(html)
    commits = {}
    for tr in trs:
        d, idx = parse_item(tr)
        commits[idx] = d
    return commits

def load_html_file(fname):
    f = open(fname)
    html = f.read().replace('\n', '')
    return html

if __name__ == "__main__":
    pass


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 15 09:29:00 2018

10.4 日志
启用 logging 模块
logging.basicConfig(level=logging.DEBUG, format = '%(asctime)s - %(levelname)s - %(message)s')

不要使用print调试，否则一个一个删很麻烦

日志的好处在于，可以随心所欲在程序中添加。。。

只要加入 logging.disable(logging.CRITICAL)调用，就可以禁日志

日志的级别有 DEBUG\INFO\WARING\ERROR\CRITICAL

将日志记录到文件
import logging
logging.basicConfig(filename='myProgramlog.txt',level=logging.DEBUG, format=
'%(asctime)s -%(levelname)s -%(message)s')

@author: NEVERGUVEIP
"""

import logging

logging.basicConfig(level=logging.DEBUG,
                    format = '%(asctime)s - %(levelname)s - %(message)s')
logging.debug('start of program')

def factorial(n):
    logging.debug('start of factorial(%s%%)' %(n))
    total = 1
    for i in range(n+1):
        total*=1
        logging.debug('i is '+str(i)+ ', total is '+str(total))
    logging.debug('End of factorial(%s%%)'%(n))
    return total

print(factorial(5))
logging.debug('end of program')


import json
import click

from click_option_group import optgroup

from corgie.log import logger as corgie_logger
from corgie.layers import get_layer_types, str_to_layer_type, \
        DEFAULT_LAYER_TYPE
from corgie.data_backends import get_data_backends, str_to_backend, \
        DEFAULT_DATA_BACKEND
from corgie import exceptions

from corgie.stack import Stack

corgie_optgroup = optgroup.group

def corgie_option(*args, **kwargs):
    return optgroup.option(*args, show_default=True, **kwargs)

LAYER_HELP_STR = """
   [required]\n
   Format: JSON *string*. \n
   Optional keys: \n
   \t "type": str from {} ; \n
   \t "name": str, DEFAULT -- same as layer type. """\
   """**Required** if given more than 1 layer of the same type ; \n
   \t "data_backend": str from {} ; \n
   \t "args": a dictionary with additional layer parameters,
   eg binarization scheme for masks data_mip_ranges, etc; \n
   Required keys: "path". \n
   """.format(get_layer_types(), get_data_backends())

   #\t "data_mip_ranges": array of [int, int] tupples specifying MIP ranges where the layer has data. DEFAULT -- all MIPs have data.\n

def create_layer_from_spec(spec_str, **kwargs):
    return create_layer_from_dict(json.loads(spec_str), **kwargs)

def create_layer_from_dict(param_dict, reference=None, caller_name=None,
        allowed_types=None, default_type=None, **kwargs):
    if default_type is None:
        default_type = DEFAULT_LAYER_TYPE
    default_param_dict = {
            "path": None,
            "name": None,
            "type": default_type,
            "data_backend": DEFAULT_DATA_BACKEND,
            "args": {},
            "readonly": False
            }
    for k in param_dict.keys():
        if k not in default_param_dict:
            raise exceptions.CorgieException(f"Unkown layer parameter '{k}'")
    params = {**default_param_dict, **param_dict}

    if params["path"] is None:
        arg_spec = '"path" key in layer specification'
        if caller_name is not None:
            arg_spec += ' of {}'.format(caller_name)
        raise exceptions.ArgumentError(arg_spec, 'not given')

    layer_path = params["path"]
    layer_type = params["type"]
    layer_args = params["args"]
    data_backend = params["data_backend"]
    corgie_logger.debug("Parsing layer path: {}".format(
        layer_path
        ))

    if allowed_types is not None and layer_type not in allowed_types:
        raise exceptions.ArgumentError("layer_type",
                f'must be of type in {allowed_types}')

    backend = str_to_backend(data_backend)
    layer = backend.create_layer(path=layer_path, layer_type=layer_type,
            reference=reference, layer_args=layer_args,
            **kwargs)

    name = params["name"]
    if name is None:
        name = layer.get_layer_type()
    layer.name = name

    return layer

def create_stack_from_spec(
    spec_str_list, name,
    reference=None, readonly=False,
    **kwargs
):
    stack = Stack(name=name)
    if len(spec_str_list) == 0:
        if reference is None:
            return None
        for l in reference.get_layers():
            l.readonly = readonly
            stack.add_layer(l)
    else:
        for spec in spec_str_list:
            layer = create_layer_from_spec(spec, readonly=readonly, **kwargs)
            stack.add_layer(layer)

    return stack


#!/usr/bin/env python3

import sys

# https://goshippo.com/blog/measure-real-size-any-python-object/
# https://gist.github.com/bosswissam/a369b7a31d9dcab46b4a034be7d263b2#file-pysize-py
def getsize(obj, seen=None):
    """Recursively finds size of objects"""
    size = sys.getsizeof(obj)
    if seen is None:
        seen = set()
    obj_id = id(obj)
    if obj_id in seen:
        return 0
    # Important mark as seen *before* entering recursion to gracefully handle
    # self-referential objects
    seen.add(obj_id)
    if isinstance(obj, dict):
        size += sum([get_size(v, seen) for v in obj.values()])
        size += sum([get_size(k, seen) for k in obj.keys()])
    elif hasattr(obj, '__dict__'):
        size += get_size(obj.__dict__, seen)
    elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):
        size += sum([getsize(i, seen) for i in obj])
    return size
'''
aa = [[],['a'],['a','b'],[1], [123], [123,456]]
for a in aa:
    print("%s size=%d" %(a, getsize(a)))

#output
[] size=64
['a'] size=122
['a', 'b'] size=180
[1] size=100
[123] size=100
[123, 456] size=136
'''

# https://code.activestate.com/recipes/577504/
#from __future__ import print_function
from sys import getsizeof, stderr
from itertools import chain
from collections import deque
try:
    from reprlib import repr
except ImportError:
    pass

def total_size(o, handlers={}, verbose=False):
    """ Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,
                    set: iter,
                    frozenset: iter,
                   }
    all_handlers.update(handlers)     # user handlers take precedence
    seen = set()                      # track which object id's have already been seen
    default_size = getsizeof(0)       # estimate sizeof object without __sizeof__

    def sizeof(o):
        if id(o) in seen:       # do not double count the same object
            return 0
        seen.add(id(o))
        s = getsizeof(o, default_size)

        if verbose:
            print(s, type(o), repr(o), file=stderr)

        for typ, handler in all_handlers.items():
            if isinstance(o, typ):
                s += sum(map(sizeof, handler(o)))
                break
        return s

    return sizeof(o)
'''
##### Example call #####

if __name__ == '__main__':
    d = dict(a=1, b=2, c=3, d=[4,5,6,7], e='a string of chars')
    print(total_size(d, verbose=True))
'''

if __name__ == '__main__':
    print('Use `getsize` or `asizeof` from "pympler".\n')
    aa = [[],['a'],['a','b'],[1], [123], [123,456]]
    for a in aa:
        print("%s size=%d" %(a, getsize(a)))
    print('------\nAnother example for old version of asizeof:\n')
    d = dict(a=1, b=2, c=3, d=[4,5,6,7], e='a string of chars')
    print(total_size(d, verbose=True))


import json
import network
from machine import unique_id
from binascii import hexlify
import time
import sys

try:
    from hostname import NAME
except Exception:
    NAME = f"{sys.platform}-{hexlify(unique_id())}"

def setup_network(timeout=10, hostname=NAME, notify=True):
    n = 0
    wlan = network.WLAN(network.STA_IF)
    wlan.active(True)
    scan = network.WLAN(network.STA_IF).scan()
    scan_tuples = [(x[0].decode(), x[3]) for x in scan]
    # Sort by dBm
    scan_tuples.sort(key=lambda x: x[1], reverse=True)
    # Set device hostname
    wlan.config(dhcp_hostname=hostname)
    try:
        with open("wpa_supplicant.config", "r") as wpa_conf:
            wifi_config = json.load(wpa_conf)
    except Exception as e:
        sys.print_exception(e, sys.stdout)
        return
    _APs_in_range = [x[0] for x in scan_tuples if x[0] in wifi_config.keys()]
    if _APs_in_range:
        _ssid = _APs_in_range[0]
        if not wlan.isconnected():
            if notify:
                print("Connecting to network...")
            wlan.connect(_ssid, wifi_config[_ssid])
            while not wlan.isconnected():
                n += 1
                time.sleep(1)
                if n > timeout:
                    return False
        if notify:
            print(f"Connected to {_ssid}")
            print("Network Config:", wlan.ifconfig())

        return _ssid


import scipy as sc
import scipy.optimize as scopt
import scipy.stats as scs
import numpy as np

def cst_fit(x, y):
    af = sc.polyfit(x, y, 0)
    y_fit = np.ones(len(x))*af
    x_fit = x
    return af, x_fit, y_fit

def linear_fit(x, y):
    [af, bf] = sc.polyfit(x, y, 1)
    y_fit = sc.polyval([af, bf], x)
    x_fit = x
    return af, bf, x_fit, y_fit

def exp_fit(x, y, cst=False):
    if cst is False or cst is None:

        fct = lambda t, a, b: b*np.exp(a*t)
        res = scopt.curve_fit(fct,  x,  y, p0=(-1, 0))
        af, bf = res[0]
        x_fit = x
        y_fit = fct(x_fit, af, bf)
        return af, bf, x_fit, y_fit
    else:
        fct = lambda t, a, b, c: b*np.exp(a*t)+c
        res = scopt.curve_fit(fct,  x,  y, p0=cst)
        af, bf, cf = res[0]
        x_fit = x
        y_fit = fct(x_fit, af, bf, cf)
        return af, bf, cf, x_fit, y_fit

def exp_fct(x, a, b, c=0):
    return b*np.exp(a*x)+c

def power_fit(x, y):
    log_x = np.log10(x)
    log_y = np.log10(y)
    [af, bf] = sc.polyfit(log_x, log_y, 1)
    log_y_fit = sc.polyval([af, bf], log_x)
    x_fit = x
    y_fit = 10 ** log_y_fit
    return af, bf, x_fit, y_fit

def log_fit(x, y):
    log_x = np.log(x)
    [af, bf] = sc.polyfit(log_x, y, 1)
    y_fit = sc.polyval([af, bf], log_x)
    x_fit = x
    return af, bf, x_fit, y_fit

def inverse_fit(x, y):
    inverse_y = 1/y
    [af, bf] = sc.polyfit(x, inverse_y, 1)
    inverse_y_fit = sc.polyval([af, bf], x)
    x_fit = x
    y_fit = 1/inverse_y_fit
    return af, bf, x_fit, y_fit

def fct_inverse_cst(x, a, b):
    return a/x+b

def inverse_cst_fit(x, y, p0):
    popt, _ = scopt.curve_fit(fct_inverse_cst, x, y, p0=p0)
    a, b = popt

    x_fit = x
    y_fit = fct_inverse_cst(x, a, b)
    return a, b, x_fit, y_fit

def fct_gauss(x, c, x0, s):
    return c*np.exp(-(x-x0)**2/(2*s**2))

def fct_gauss_cst(x, c, x0, s, d):
    return c*np.exp(-(x-x0)**2/(2*s**2))+d

def centered_fct_gauss(x, c, s):
    return c*np.exp(-x**2/(2*s**2))

def centered_fct_gauss_cst(x, c, s, d):
    return c*np.exp(-x**2/(2*s**2))+d

def fct_von_mises(x, kappa):
    return scs.vonmises.pdf(x, kappa=kappa)

def fct_von_mises_cst(x, kappa, b, k):
    return b*np.exp(kappa*np.cos(x))+k

def gauss_fit(x, y):
    popt, _ = scopt.curve_fit(fct_gauss, x, y)
    c, x0, s = popt

    x_fit = x
    y_fit = fct_gauss(x, c, x0, s)
    return c, x0, s, x_fit, y_fit

def gauss_cst_fit(x, y):
    popt, _ = scopt.curve_fit(fct_gauss_cst, x, y)
    c, x0, s, d = popt

    x_fit = x
    y_fit = fct_gauss_cst(x, c, x0, s, d)
    return c, x0, s, d, x_fit, y_fit

def centered_gauss_fit(x, y, p0=None):
    popt, _ = scopt.curve_fit(centered_fct_gauss, x, y, p0=p0)
    c, s = popt

    x_fit = x
    y_fit = centered_fct_gauss(x, c, s)
    return c, s, x_fit, y_fit

def vonmises_fit(x, y, p0=None):
    popt, _ = scopt.curve_fit(fct_von_mises, x, y, p0=p0)
    kappa = popt

    x_fit = x
    y_fit = fct_von_mises(x, kappa)
    return kappa, x_fit, y_fit

def vonmises_cst_fit(x, y, p0=None):
    popt, _ = scopt.curve_fit(fct_von_mises_cst, x, y, p0=p0)
    kappa, b, k = popt

    x_fit = x
    y_fit = fct_von_mises_cst(x, kappa, b, k)
    return kappa, b, k, x_fit, y_fit

def centered_gauss_cst_fit(x, y, p0=None):
    popt, _ = scopt.curve_fit(centered_fct_gauss_cst, x, y, p0=p0)
    c, s, d = popt

    x_fit = x
    y_fit = centered_fct_gauss_cst(x, c, s, d)
    return c, s, d, x_fit, y_fit

def laplace_fct(x, a, b):
    return b*np.exp(-a*np.abs(x))

def laplace_fit(x, y, p0=None):
    popt, _ = scopt.curve_fit(laplace_fct, x, y, p0=p0)
    a, b = popt

    x_fit = x
    y_fit = laplace_fct(x, a, b)
    return a, b, x_fit, y_fit

def uniform_vonmises_fct(x, q, kappa):
    return q*scs.vonmises.pdf(x, kappa)+(1-q)/2/np.pi

def uniform_vonmises_fit(x, y, p0=None, get_err=False):
    popt, cov = scopt.curve_fit(uniform_vonmises_fct, x, y, p0=p0, bounds=((0, 0), (1, np.inf)))
    q, kappa = popt

    x_fit = x
    y_fit = uniform_vonmises_fct(x, q, kappa)

    if get_err is True:
        return q, kappa, np.sqrt(np.diagonal(cov)), x_fit, y_fit
    else:
        return q, kappa, x_fit, y_fit

def uniform_vonmises_q_fit(x, y, q, p0=None):

    def uniform_vonmises_q_fct(x0, kappa0):
        return q*scs.vonmises.pdf(x0, kappa0)+(1-q)/2/np.pi

    popt, _ = scopt.curve_fit(uniform_vonmises_q_fct, x, y, p0=p0, bounds=(0, 5))
    kappa = popt[0]
    x_fit = x
    y_fit = uniform_vonmises_fct(x, q, kappa)
    return kappa, x_fit, y_fit


import csv
import multiprocessing
import os
import pickle
import signal
import subprocess
from syslog import syslog
from time import sleep

class Runner:
    # Subclasses are expected to override their perf events and filters. A pair
    # (event, filter) is expected. If filter is None, no filter is added.
    # The following alias are understood to ease filter creation:
    #   $payload_len (current payload len; 18 is added to make up for headers)
    #   $phy_name (name of the physical interface)
    #   $vlan_tci (vlan tag control information, tailored for socket priority)
    _perf_events = []

    def __init__(self, cmd_socket, results_dir, iface_name, dest_addr,
                 run_stress, isol_core, intermediate_latency, keep_perf_data,
                 talker_ip):
        self.command = []
        self.cmd_socket = cmd_socket
        self.results_dir = results_dir
        self.iface_name = iface_name
        self.dest_addr = dest_addr
        self.run_stress = run_stress
        self.isol_core = isol_core
        self.intermediate_latency = intermediate_latency
        self.keep_perf_data = keep_perf_data
        self.stress_cmd = [
            'stress-ng',
            '--cpu', str(multiprocessing.cpu_count()),
            '--cpu-method', 'loop'
        ]
        self.talker_ip = talker_ip
        self.interference_process = None

    def run(self):
        raise NotImplementedError('Must implement run()')

    def _insert_cmd(self, cmd):
        cmd.extend(self.command)
        self.command = cmd

    def _get_tai_offset(self):
        timer_list = open('/proc/timer_list', 'r')
        timer_list_str = timer_list.read()

        start, part, end = timer_list_str.partition('ktime_get_clocktai\n')
        clock_tai_str = end.splitlines()[0]

        return int(clock_tai_str.split()[1])

    def _generate_perf_cmd(self, perf_output_name, payload_size, socket_prio):
        # Here we use --mmap-pages argument to ensure that no events are being
        # dropped by `perf record`.
        perf_cmd = [
            'perf', 'record', '-a',
            '--mmap-pages', '128M',
            '-k', 'CLOCK_MONOTONIC',
            '-o', perf_output_name
        ]

        if len(self._perf_events) == 0:
            raise NotImplementedError('Must override _perf_events')

        for (event, filter_) in self._perf_events:
            perf_cmd.extend(['-e', event])
            if filter_ is not None:
                payload_size = int(payload_size) + 18  # To account for headers
                iface_name = self._get_phy_iface_name(self.iface_name)
                # The 5 below is vlan id - currently hardcoded, if it stops to
                # be so, this needs to be fixed
                vlan_tci = ((int(socket_prio) & 0x3) << 13) | 5

                filter_ = (filter_.replace('$payload_len', str(payload_size))
                                  .replace('$phy_name', iface_name)
                                  .replace('$vlan_tci', str(vlan_tci)))
                perf_cmd.extend(['--filter', filter_])

        return perf_cmd

    def _process_intermediate_tstamps(self, iface_name, tai_mono_offset,
                                      perf_output_name, perf_script_name):
        trace_file_name = '/tmp/trace_out.txt'

        # Collect Intermediate Timestamps
        parse_cmd = [
            'perf', 'script',
            '-i', perf_output_name,
            '-s', perf_script_name,
            trace_file_name,
            self._get_phy_iface_name(iface_name),
            str(tai_mono_offset)
        ]

        parse_process = subprocess.Popen(parse_cmd)
        parse_process.wait()

        dataset = self._read_csv(trace_file_name)

        os.remove(trace_file_name)

        return dataset

    def _get_phy_iface_name(self, vlan_iface_name):
        cmd = ['ip', 'link', 'show', vlan_iface_name]
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True)
        out, _ = process.communicate()
        return out.split('@')[1].split(':')[0]

    def _read_csv(self, f):
        close_on_exit = False
        if isinstance(f, str):
            f = open(f, 'r', newline='')
            close_on_exit = True

        data = []
        reader = csv.reader(f)
        for row in reader:
            data.append(row)

        if close_on_exit:
            f.close()

        return data

    def _start_stress(self):
        if self.run_stress:
            self.stress_process = subprocess.Popen(self.stress_cmd)

    def _stop_stress(self):
        if self.run_stress:
            self.stress_process.send_signal(signal.SIGTERM)
            self.stress_process.wait()
            self.stress_process = None

    def _insert_isol_core(self):
        if self.isol_core is not None:
            syslog(f'Isolating listener on core {self.isol_core}')
            self._insert_cmd(['taskset', '-c', str(self.isol_core)])

    def _insert_intermediate_latency(self, factors):
        perf_output_name = None
        if self.intermediate_latency:
            perf_output_name = (f'{self.results_dir}/perf-'
                                f'{factors["PayloadSize"]}-'
                                f'{factors["TransmissionInterval"]}.data')
            self._insert_cmd(self._generate_perf_cmd(perf_output_name,
                                                     factors['PayloadSize'],
                                                     factors['SO_PRIORITY']))

        return perf_output_name

    def _start_network_interference(self):
        raise NotImplementedError('Must implement '
                                  '_start_network_interference()')

    def _stop_interference(self):
        if self.interference_process is not None:
            self.interference_process.send_signal(signal.SIGTERM)
            self.interference_process.wait()
            self.interference_process = None

class TalkerRunner(Runner):
    _cmd_name = 'tsn-talker'

    def __init__(self, *args, iterations, network_interference):
        super(TalkerRunner, self).__init__(*args)
        self.interference_cmd = ['iperf3', '-s']
        self.iterations = iterations
        self.network_interference = network_interference

    def _transfer_intermediate_tstamps(self, dataset):
        # Transfer intermediate timestamp data to the Listener
        self.cmd_socket.send(b'INTERMEDIATE_TSTAMPS_INCOMING')
        self.cmd_socket.send(pickle.dumps(dataset))

    def _calculate_iterations(self, factors):
        iterations = self.iterations
        if isinstance(self.iterations, str):
            if self.iterations[-1:] == 's':
                iterations = int(int(self.iterations[:-1]) * 1000000000 /
                                 int(factors['TransmissionInterval']))
            else:
                iterations = self.iterations

        return iterations

    def _insert_run_command(self, factors, iterations):
        cmd = [
            'chrt', '--fifo', '98',
            self._cmd_name,
            '-i', self.iface_name,
            '-d', self.dest_addr,
            '-n', str(iterations),
            '-p', factors['SO_PRIORITY'],
            '-s', factors['PayloadSize'],
            '-D', factors['TransmissionInterval']]
        self._insert_cmd(cmd)

    def run(self, factors):
        out_file = open(f'{self.results_dir}/output_file.txt', 'a')
        err_file = open(f'{self.results_dir}/errors_file.txt', 'a')

        data = self.cmd_socket.getmsg()
        while data != b'START_TALKER':
            data = self.cmd_socket.getmsg()

        tai_mono_offset = self._get_tai_offset()

        self._start_stress()
        self._start_network_interference()
        iterations = self._calculate_iterations(factors)
        self._insert_run_command(factors, iterations)
        self._insert_isol_core()
        perf_output_name = self._insert_intermediate_latency(factors)

        syslog(
            f'Commencing talker experiment. {iterations} iterations, '
            f'{factors["PayloadSize"]} payload size and '
            f'{factors["TransmissionInterval"]} transmission interval\n'
            f'TAI-monotonic offset {tai_mono_offset}')
        process = subprocess.Popen(self.command, stdout=out_file,
                                   stderr=err_file)
        process.wait()
        syslog('Completed talker experiment')

        self._stop_interference()
        self._stop_stress()

        if tai_mono_offset != self._get_tai_offset():
            print("WARNING: The offset between CLOCK_TAI and CLOCK_MONOTONIC "
                  "has changed. Data might be invalid")

        self.cmd_socket.send(b'STOP_LISTENER')

        if self.intermediate_latency:
            dataset = self._process_intermediate_tstamps(self.iface_name,
                                                         tai_mono_offset,
                                                         perf_output_name)
            self._transfer_intermediate_tstamps(dataset)
            if not self.keep_perf_data:
                os.remove(perf_output_name)
        else:
            self.cmd_socket.send(b'NO_INTERMEDIATE_TSTAMPS')

        self.command = []

        err_file.close()
        out_file.close()

    def _start_network_interference(self):
        if self.network_interference:
            self.interference_process = subprocess.Popen(self.interference_cmd)
            self.cmd_socket.send(b'START_NETWORK_INTERFERENCE')
        else:
            self.cmd_socket.send(b'NO_NETWORK_INTERFERENCE')

class AFPacketTalkerRunner(TalkerRunner):
    _perf_events = [
        ('syscalls:sys_enter_sendto', f"comm == '{TalkerRunner._cmd_name}'"),
        ('net:net_dev_queue', 'len <= $payload_len'),
        ('net:net_dev_start_xmit', 'protocol == 0x22f0 || '
                                   'vlan_tci == $vlan_tci'),
        ('net:net_dev_xmit', 'len <= $payload_len')
    ]

    def _process_intermediate_tstamps(self, *args):
        perf_script_name = 'tx-intermediate-perf-script.py'

        return (super(AFPacketTalkerRunner, self).
                _process_intermediate_tstamps(*args, perf_script_name))

class AFXDPTalkerRunner(TalkerRunner):
    def __init__(self, *args, **kwargs):
        self.xdp_hw_queue = kwargs.pop('xdp_hw_queue')
        self.vlan_priority = kwargs.pop('vlan_priority')
        self.needs_wakeup = kwargs.pop('needs_wakeup')
        self.xdp_mode = kwargs.pop('mode')
        self.xdp_copy_mode = kwargs.pop('copy_mode')
        super(AFXDPTalkerRunner, self).__init__(*args, **kwargs)

        if self.intermediate_latency is True:
            NotImplementedError('Intermediate latency metrics not supported on'
                                'AF_XDP Socket.')

    def _insert_run_command(self, *args):
        super(AFXDPTalkerRunner, self)._insert_run_command(*args)

        syslog(f'Running XDP socket on queue {self.xdp_hw_queue}'
               f'with VLAN Priority {self.vlan_priority}, '
               f'mode: {self.xdp_mode} and copy mode: {self.xdp_copy_mode}')
        self.command.extend(['-X', str(self.xdp_hw_queue),
                             '-V', str(self.vlan_priority)])

        if self.needs_wakeup:
            self.command.append('-w')

        if self.xdp_mode == 'Native':
            self.command.append('-N')
        else:
            self.command.append('-S')

        if self.xdp_copy_mode == 'Copy':
            self.command.append('-C')
        else:
            self.command.append('-Z')

class ListenerRunner(Runner):
    _cmd_name = 'tsn-listener'

    def __init__(self, *args):
        super(ListenerRunner, self).__init__(*args)
        self.interference_cmd = ['iperf3', '-c', self.talker_ip, '-t', '0',
                                 '-R']

    def _receive_intermediate_tstamps(self):
        dumpfile = self.cmd_socket.getmsg()
        intr_data = pickle.loads(dumpfile)
        return intr_data

    def _write_dataset(self, dataset, intr_data_talker, intr_data_listener,
                       csv_file):
        csv_writer = csv.writer(csv_file)

        # On the very first write, we add the headers
        if csv_file.tell() == 0:
            header = []
            header.append(dataset[0][0])  # sw tx ts
            if intr_data_talker is not None:
                header.extend(intr_data_talker[0])
            header.append(dataset[0][1])  # hw rx ts
            if intr_data_listener is not None:
                header.extend(intr_data_listener[0])
            header.append(dataset[0][2])  # sw rx ts
            csv_writer.writerow(header)

        for i in range(1, len(dataset)):
            row = []
            row.append(dataset[i][0])  # sw tx ts
            if intr_data_talker is not None:
                row.extend(intr_data_talker[i])
            row.append(dataset[i][1])  # hw rx ts
            if intr_data_listener is not None:
                row.extend(intr_data_listener[i])
            row.append(dataset[i][2])  # sw rx ts
            csv_writer.writerow(row)

    def _insert_run_command(self, factors):
        cmd = [
            'chrt', '--fifo', '98',
            self._cmd_name,
            '-i', self.iface_name,
            '-s', factors['PayloadSize']]
        self._insert_cmd(cmd)

    def run(self, factors):
        intr_data_talker = None
        intr_data_listener = None
        out_file = open(f'{self.results_dir}/.out_file', 'w+')
        err_file = open(f'{self.results_dir}/errors_file.txt', 'a')

        self._start_stress()
        self._insert_run_command(factors)
        self._insert_isol_core()
        perf_output_name = self._insert_intermediate_latency(factors)

        process = subprocess.Popen(self.command, stdout=out_file,
                                   stderr=err_file)

        # When tsn-listener enables RX timestamping, NIC is reset and ptp4l
        # is disrupted. So, wait for sometime for ptp4l to reconnect.
        sleep(60)

        tai_mono_offset = self._get_tai_offset()
        syslog(
            f'Commencing listener experiment. {factors["PayloadSize"]} '
            f'payload size and {factors["TransmissionInterval"]} '
            f'transmission interval\nTAI-monotonic offset: {tai_mono_offset}')

        self.cmd_socket.send(b'START_TALKER')
        self._start_network_interference()
        data = self.cmd_socket.getmsg()
        while data != b'STOP_LISTENER':
            data = self.cmd_socket.getmsg()

        self._stop_interference()
        self._stop_stress()

        process.send_signal(signal.SIGINT)
        process.wait()

        syslog('Completed listener experiment')

        data = self.cmd_socket.getmsg()
        if data == b'INTERMEDIATE_TSTAMPS_INCOMING':
            intr_data_talker = self._receive_intermediate_tstamps()

        if self.intermediate_latency:
            if tai_mono_offset != self._get_tai_offset():
                print("WARNING: Offset between CLOCK_TAI and CLOCK_MONOTONIC "
                      "has changed. Data might be invalid")

            intr_data_listener = (
                self._process_intermediate_tstamps(
                    self.iface_name,
                    tai_mono_offset,
                    perf_output_name))
            if not self.keep_perf_data:
                os.remove(perf_output_name)

        csv_file_name = (f'{self.results_dir}/results-{factors["PayloadSize"]}'
                         f'-{factors["TransmissionInterval"]}.csv')

        if self.intermediate_latency or intr_data_talker is not None:
            out_file.seek(0)
            dataset = self._read_csv(out_file)

            with open(csv_file_name, 'w') as csv_file:
                self._write_dataset(dataset, intr_data_talker,
                                    intr_data_listener, csv_file)
            out_file.close()
            os.remove(out_file.name)
        else:
            # Without intermediate latency, renaming out_file should be quicker
            out_file.close()
            os.rename(out_file.name, csv_file_name)

        err_file.close()

        self.command = []

        return csv_file_name

    def _start_network_interference(self):
        data = self.cmd_socket.getmsg()
        if data == b'NO_NETWORK_INTERFERENCE':
            return
        elif data == b'START_NETWORK_INTERFERENCE':
            self.interference_process = subprocess.Popen(self.interference_cmd)
        else:
            raise Exception(f'Unexpected listener message {data}')

class AFPacketListenerRunner(ListenerRunner):
    def __init__(self, *args, irq_name):
        super(AFPacketListenerRunner, self).__init__(*args)
        self.irq_name = irq_name

        self._perf_events = [
            ('irq:irq_handler_entry', f"name ~ '{self.irq_name}*'"),
            ('net:napi_gro_receive_entry', 'protocol == 0x22f0'),
            ('net:netif_receive_skb', "name ~ "
                                      "'$phy_name*' && len <= $payload_len"),
            ('syscalls:sys_exit_recvmsg', f"comm == '{self._cmd_name}'")
        ]

    def _process_intermediate_tstamps(self, *args):
        perf_script_name = 'rx-intermediate-perf-script.py'

        return (super(ListenerRunner, self).
                _process_intermediate_tstamps(*args, perf_script_name))

class AFXDPListenerRunner(ListenerRunner):
    def __init__(self, *args, xdp_hw_queue, needs_wakeup, mode, copy_mode):
        super(AFXDPListenerRunner, self).__init__(*args)
        self.xdp_hw_queue = xdp_hw_queue
        self.needs_wakeup = needs_wakeup
        self.xdp_mode = mode
        self.xdp_copy_mode = copy_mode

        if self.intermediate_latency is True:
            NotImplementedError('Intermediate latency metrics not supported on'
                                'AF_XDP Socket.')

    def _insert_run_command(self, *args):
        super(AFXDPListenerRunner, self)._insert_run_command(*args)

        syslog(f'Running XDP socket on queue {self.xdp_hw_queue}, '
               f'mode: {self.xdp_mode} and copy mode: {self.xdp_copy_mode}')
        self.command.extend(['-X', str(self.xdp_hw_queue)])

        if self.needs_wakeup:
            self.command.append('-w')

        if self.xdp_mode == 'Native':
            self.command.append('-N')
        else:
            self.command.append('-S')

        if self.xdp_copy_mode == 'Copy':
            self.command.append('-C')
        else:
            self.command.append('-Z')


from django import forms
from .widgets import CustomClearableFileInput
from .models import Product, Category, Review
from django_summernote.widgets import SummernoteWidget, SummernoteInplaceWidget

class ProductForm(forms.ModelForm):
    """
    Form to add products
    """
    class Meta:
        model = Product
        fields = '__all__'

    image = forms.ImageField(label='Image', required=False,
                             widget=CustomClearableFileInput)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        categories = Category.objects.all()
        friendly_names = [(c.id, c.get_friendly_name()) for c in categories]

        self.fields['category'].choices = friendly_names
        for field_name, field in self.fields.items():
            field.widget.attrs['class'] = 'border-color rounded'

class ReviewForm(forms.ModelForm):
    """ Form for users to add reviews of products"""
    class Meta:
        model = Review
        fields = ('description', 'star_rating')
        widgets = {
            'description': SummernoteWidget,
            'star_rating': forms.NumberInput(attrs={'min': 1, 'max': '5', })
        }


from fastapi import APIRouter, Request
from fastapi.responses import HTMLResponse
from config.db import conn
from fastapi.templating import Jinja2Templates

note = APIRouter()
templates = Jinja2Templates(directory="templates")

@note.get("/", response_class=HTMLResponse)
async def read_item(request: Request): # type: ignore
    docs = conn.notes.notes.find({})

    newDocs = []
    for doc in docs:
        newDocs.append({
            "id": doc["_id"],
            "title": doc["title"],
            "desc": doc["desc"],
            "priority": doc["priority"]
        })
    return templates.TemplateResponse("index.html", {"request": request, "newDocs": newDocs})

@note.post("/")
async def create_item(request: Request):
    form = await request.form()
    formDict = dict(form)
    formDict["priority"] = True if formDict.get("priority") == "on" else False # type: ignore
    note = conn.notes.notes.insert_one(formDict)
    return {"Success": True}


import numpy as np
import pandas as pd
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from xgboost import XGBClassifier, XGBRegressor
import time
from sklearn.metrics import accuracy_score, r2_score
from sklearn.feature_selection import SelectFromModel

#1. 데이터
path = './_data/bike/'
train_set = pd.read_csv(path + 'train.csv')
print(train_set)
print(train_set.shape)   # (10886, 11)
print(train_set.columns) # ['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp',
                         #  'humidity', 'windspeed', 'casual', 'registered', 'count']

print(train_set.info())
print(train_set.describe())
print(train_set.isnull().sum())

test_set = pd.read_csv(path + 'test.csv')
print(test_set)
print(test_set.shape)   # (6493, 8)
print(test_set.columns) # ['season', 'holiday', 'workingday', 'weather', 'temp', 'atemp',
                        #  'humidity', 'windspeed']   ==> casual, registered, count
print(test_set.info())
print(test_set.describe())
print(test_set.isnull().sum())

#### 결측치 처리 ####
from datetime import datetime
train_set['datetime'] = pd.to_datetime(train_set['datetime'])
test_set['datetime'] = pd.to_datetime(test_set['datetime'])

# 날짜 feature 생성
L = ['year', 'month', 'date', 'hour', 'weekday']
train_set = train_set.join(pd.concat([getattr(train_set['datetime'].dt, i).rename(i) for i in L], axis=1))
test_set = test_set.join(pd.concat([getattr(test_set['datetime'].dt, i).rename(i) for i in L], axis=1))

print(train_set)
print(test_set)

# drop_features
drop_features = ['registered', 'datetime', 'year', 'date', 'month']
train_set = train_set.drop(drop_features, axis=1)
test_set = test_set.drop(['datetime', 'year', 'date', 'month'], axis=1)

#### 이상치 처리 ####
# train_set = train_set[train_set['weather'] != 4]  # weather가 4인 데이터는 이상치(폭우, 폭설이 내리는 날 저녁 6시에 대여)

def outliers(df, col):
    out = []
    m = np.mean(df[col])
    sd = np.std(df[col])

    for i in df[col]:
        z = (i-m)/sd
        if np.abs(z) > 3:
            out.append(i)

    print("Outliers:", out)
    print("min",np.median(out))
    return np.median(out)

col = "casual"
medOutlier = outliers(train_set,col)
train_set[train_set[col] >= medOutlier]

col = "weather"
medOutlier = outliers(train_set,col)
train_set[train_set[col] >= medOutlier]

col = "windspeed"
medOutlier = outliers(train_set,col)
train_set[train_set[col] >= medOutlier]

# x, y 데이터
print("train_set.columns :", train_set.columns)
x = train_set.drop(['count'], axis=1)
print(x)
print(x.columns)
# print(x.shape)

y = train_set['count']
print(y)
print(y.shape)

# IterativeImputer() 결측치 처리
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
imputer = IterativeImputer(random_state=72)
imputer.fit(x)
x = imputer.transform(x)

x_train, x_test, y_train, y_test = train_test_split(
    x, y, shuffle=True, random_state=123, train_size=0.8
)

scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)

#2. 모델
from sklearn.svm import SVR
from sklearn.ensemble import BaggingClassifier, BaggingRegressor
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.linear_model import LogisticRegression, LinearRegression
from xgboost import XGBClassifier, XGBRegressor

model = BaggingRegressor(SVR(),
# model = BaggingRegressor(LinearRegression(),
# model = BaggingRegressor(DecisionTreeRegressor(),
# model = BaggingRegressor(XGBRegressor(),
                          n_estimators=100,
                          n_jobs=-1,
                          random_state=123
                          )

#3. 훈련
import time
start = time.time()
model.fit(x_train, y_train)
end = time.time()

#4. 평가, 예측
result = model.score(x_test, y_test)
print('Bagging_LinearRegression 결과 : ', result)
print('걸린 시간 : ', end - start)

#=================================== 결과 =====================================#
# Bagging_SVR 결과 :  0.45923772660779005
# 걸린 시간 :  25.01764988899231
# Bagging_LinearRegression 결과 :  0.5922912575957451
# 걸린 시간 :  1.4196622371673584
# Bagging_DecisionTreeRegressor 결과 :  0.9203885611168507
# 걸린 시간 :  1.7394013404846191
# Bagging_XGBRegressor 결과 :  0.9310516904152373
# 걸린 시간 :  10.2945077419281
#==============================================================================#


from typing import Dict

import torch

# Computes additional embeddings required by the SDXL UNet.
def compute_additional_embeddings(height, width, weight_dtype):
    # Adapted from pipeline.StableDiffusionXLPipeline._get_add_time_ids
    original_size = (height, width)
    target_size = (height, width)
    crops_coords_top_left = (0, 0)
    add_time_ids = list(original_size + crops_coords_top_left + target_size)
    add_time_ids = torch.tensor([add_time_ids])

    add_time_ids = add_time_ids.to(dtype=weight_dtype)
    unet_added_cond_kwargs = {"time_ids": add_time_ids}

    return unet_added_cond_kwargs


import func_timeout
from maze import *
from queue import PriorityQueue
import math
import os
import psutil

class AStar:
    def __init__(self):
        self.iterat = 0
        self.state = 0
        self.mem_st = 0

    def grid(self, rows, cols):
        grid=[]
        maze_map={}
        y=0
        for n in range(cols):
            x = 1
            y = 1+y
            for m in range(rows):
                grid.append((x,y))
                maze_map[x,y]={'right':0,'left':0,'top':0,'bottom':0}
                x = x + 1
        return grid

    def h(self, cell1, cell2):
        x1,y1=cell1
        x2,y2=cell2

        return math.sqrt((x1-x2)**2 + (y1-y2)**2)

    def search(self,m, x, y, start, end):
        return func_timeout.func_timeout(30*60, self._search, args=[m, x, y, start, end])

    def _search(self, m, x, y, start, end):
        self.iterat = 0
        self.state = 0
        self.mem_st = 0

        m_copy = m
        dir_map = self.new_map(m)
        priority = ["bottom", "right", "top", "left"]

        g_score = {cell:float('inf') for cell in self.grid(x, y)}
        g_score[start]=0
        f_score={cell:float('inf') for cell in self.grid(x, y)}
        f_score[start] = self.h(start, end)

        border=PriorityQueue()
        border.put((self.h(start, end), self.h(start, end), start))
        aPath = {}
        states = []
        pathF = False
        while not border.empty():
            if psutil.Process(os.getpid()).memory_info().rss > 1024**3:
                raise MemoryError("1 Gb memory exceeded")
            self.iterat += 1

            if border.qsize() > self.mem_st:
                self.mem_st = border.qsize()

            curr = border.get()[2]

            if curr not in states:
                states.append(curr)

            if curr == end:
                pathF = True
                break

            for d in priority:
                if dir_map[curr][d]==True:
                    if d=='right':
                        nextcell=(curr[0],curr[1]+1)
                    if d=='left':
                        nextcell=(curr[0],curr[1]-1)
                    if d=='top':
                        nextcell=(curr[0]-1,curr[1])
                    if d=='bottom':
                        nextcell=(curr[0]+1,curr[1])
                    temp_g_score=g_score[curr]+1
                    temp_f_score=temp_g_score+self.h(curr,end)

                    if temp_f_score < f_score[nextcell]:
                        g_score[nextcell]= temp_g_score
                        f_score[nextcell]= temp_f_score
                        border.put((temp_f_score,self.h(nextcell,end),nextcell))
                        aPath[nextcell]=curr

                    # if self.iterat > 1 and m_copy[curr[0]][curr[1]] != 'S':
                    #         m_copy[curr[0]][curr[1]] = 'P'
        if (pathF == True):
            fwdPath={}
            cell=end
            while cell!=start:
                fwdPath[aPath[cell]]=cell
                cell=aPath[cell]
            printPath(fwdPath)

            for i in fwdPath:
                if m_copy[i[0]][i[1]] != 'S' and m_copy[i[0]][i[1]] != 'E':
                    m_copy[i[0]][i[1]] = 'P'
            printMaze(m_copy, x, y)
            for i in range(x):
                for j in range(y):
                    if (m[i][j]!="w"):
                        self.state += 1

        else:
            print("NO")

    def new_map(self, m):
        dir_map = {}

        for i in range(len(m)):
            for j in range(len(m[i])):
                x = i
                y = j
                curr = (x, y)
                dir_map[curr] = {"left": self.is_dir(m, i, j-1),
                                 "top": self.is_dir(m, i-1, j),
                                 "right": self.is_dir(m, i, j+1),
                                 "bottom": self.is_dir(m, i+1, j)}

        return dir_map

    def is_dir(self, m, x, y):
        if x>=len(m) or x<0 or y>=len(m[x]) or y<0:
            return False
        return True if m[x][y] == "c" or m[x][y] == "E" else False