import os
import sys
from pytigon_lib.schfs import extractall
import zipfile
from distutils.dir_util import copy_tree
import configparser
from pytigon_lib.schtools.process import py_manage
from pytigon_lib.schtools.cc import make
from pytigon_lib.schtools.process import py_run

def _mkdir(path, ext=None):
    if ext:
        p = os.path.join(path, ext)
    else:
        p = path
    if not os.path.exists(p):
        try:
            os.mkdir(p)
        except:
            pass

def upgrade_test(zip_path, out_path):
    if os.path.exists(zip_path):
        archive = zipfile.ZipFile(zip_path, "r")
        cfg_txt = archive.read("install.ini").decode("utf-8")
        cfg = configparser.ConfigParser()
        cfg.read_string(cfg_txt)
        t1 = cfg["DEFAULT"]["GEN_TIME"]
        ini2 = os.path.join(out_path, "install.ini")
        if os.path.exists(ini2):
            cfg2 = configparser.ConfigParser()
            cfg2.read(ini2)
            t2 = cfg2["DEFAULT"]["GEN_TIME"]
            if t2 < t1:
                return True
    return False

def pip_install(pip_str, prjlib, confirm=False, upgrade=False):
    packages = [x.strip() for x in pip_str.split(" ") if x]
    print("pip install: ", pip_str)
    exit_code, output_tab, err_tab = py_run(
        [
            "-m",
            "pip",
            "--disable-pip-version-check",
            "install",
            f"--target={prjlib}",
        ]
        + (
            [
                "--upgrade",
            ]
            if upgrade
            else []
        )
        + packages
    )
    success = False
    if output_tab:
        for pos in output_tab:
            if pos:
                print("pip info: ", pos)
            if "Successfully installed" in pos:
                success = True
    if err_tab:
        for pos in err_tab:
            if pos:
                print("pip error: ", pos)

    if success and confirm:
        with open(os.path.join(prjlib, "install.txt"), "wt") as f:
            f.write("OK")

def upgrade_local_libs():
    from django.conf import settings

    prjlib = os.path.join(settings.DATA_PATH, settings.PRJ_NAME, "prjlib")
    config_file = os.path.join(settings.PRJ_PATH, settings.PRJ_NAME, "install.ini")
    if os.path.exists(config_file):
        config = configparser.ConfigParser()
        config.read(config_file)
        if "DEFAULT" in config:
            pip_str = config["DEFAULT"].get("PIP", "")
            if pip_str:
                pip_install(pip_str, prjlib, confirm=True, upgrade=True)

def init(prj, root_path, data_path, prj_path, static_app_path, paths=None):
    if prj == "_schall":
        return
    _root_path = os.path.normpath(root_path)
    _data_path = os.path.normpath(data_path)
    _prj_path = os.path.normpath(prj_path)
    _static_app_path = os.path.normpath(static_app_path)
    _base_compiler_path = os.path.join(_data_path, "ext_prg")
    test1 = 0 if os.path.exists(_prj_path) else 1
    test2 = 0 if os.path.exists(_data_path) else 1
    test3 = 0 if os.path.exists(_static_app_path) else 1

    if not test2:
        if upgrade_test(
            os.path.join(os.path.join(_root_path, "install"), ".pytigon.zip"),
            _data_path,
        ):
            test2 = 2
            print("Upgrade data")

    if test2:
        zip_file2 = os.path.join(os.path.join(_root_path, "install"), ".pytigon.zip")
        if not os.path.exists(_data_path):
            os.makedirs(_data_path)
        if os.path.exists(zip_file2):
            if test2 == 2:
                extractall(zipfile.ZipFile(zip_file2), _data_path, exclude=[".*\.db"])
            else:
                extractall(zipfile.ZipFile(zip_file2), _data_path)
        if not os.path.exists(os.path.join(_data_path, "media")):
            media_path = os.path.join(os.path.join(_data_path, "media"))
            os.makedirs(media_path)
            os.makedirs(os.path.join(media_path, "filer_public"))
            os.makedirs(os.path.join(media_path, "filer_private"))
            os.makedirs(os.path.join(media_path, "filer_public_tumbnails"))
            os.makedirs(os.path.join(media_path, "filer_private_thumbnails"))
        if not os.path.exists(os.path.join(_data_path, "doc")):
            doc_path = os.path.join(os.path.join(_data_path, "doc"))
            os.makedirs(doc_path)

        prjs = [ff for ff in os.listdir(_prj_path) if not ff.startswith("_")]

        tmp = os.getcwd()
        for app in prjs:
            path = os.path.join(_prj_path, app)
            if os.path.isdir(path):
                db_path = os.path.join(os.path.join(_data_path, app), f"{app}.db")
                os.chdir(path)
                print("python: pytigon: init: ", path)
                if not os.path.exists(db_path):
                    print("python: pytigon: init: create:", db_path)
                    exit_code, output_tab, err_tab = py_manage(
                        ["makeallmigrations"], False
                    )
                    if err_tab:
                        print(err_tab)
                    exit_code, output_tab, err_tab = py_manage(["migrate"], False)
                    if err_tab:
                        print(err_tab)
                    exit_code, output_tab, err_tab = py_manage(
                        ["createautouser"], False
                    )
                    if err_tab:
                        print(err_tab)
                    if app == "schdevtools":
                        print("python: pytigon: import_projects!")
                        exit_code, output_tab, err_tab = py_manage(
                            ["import_projects"], False
                        )
                        print("python: pytigon: projects imported!")
                        if err_tab:
                            print(err_tab)
        os.chdir(tmp)
    if test2 == 2:
        pass

    if test3:
        p2 = os.path.join(os.path.join(_root_path, "static"), "app")
        if os.path.exists(p2):
            copy_tree(p2, _static_app_path, preserve_mode=0, preserve_times=0)

    _paths = [
        "",
        "cache",
        "plugins_cache",
        "_schall",
        "schdevtools",
        "prj",
        "temp",
        "static",
        prj,
    ]
    for p in _paths:
        _mkdir(_data_path, p)
    if paths:
        for p in paths:
            _mkdir(p)

    prjlib = os.path.join(_data_path, prj, "prjlib")
    if not os.path.exists(prjlib) or not os.path.exists(
        os.path.join(prjlib, "install.txt")
    ):
        if not os.path.exists(prjlib):
            os.mkdir(prjlib)
        config_file = os.path.join(prj_path, prj, "install.ini")
        if os.path.exists(config_file):
            config = configparser.ConfigParser()
            config.read(config_file)
            if "DEFAULT" in config:
                pip_str = config["DEFAULT"].get("PIP", "")
                if pip_str:
                    pip_install(pip_str, prjlib, confirm=True)

    if os.path.exists(prjlib):
        if not prjlib in sys.path:
            sys.path.append(prjlib)
        if test1 or test2 or test3:
            ret = make(_data_path, os.path.join(_prj_path, prj), prj)
            if ret:
                for pos in ret:
                    print(pos)
    syslib = os.path.join(_data_path, prj, "syslib")
    if not os.path.exists(syslib):
        os.makedirs(syslib)
        with open(os.path.join(syslib, "__init__.py"), "wt") as f:
            f.write(" ")


from .curve import JaxCurve, Curve
from math import pi
from jax.ops import index, index_add
import jax.numpy as jnp
import numpy as np
import simsgeopp as sgpp

def stelleratorsymmetriccylindricalfouriercurve_pure(dofs, quadpoints, order, nfp):
    coefficients0 = dofs[:(order+1)]
    coefficients1 = dofs[(order+1):]
    gamma = jnp.zeros((len(quadpoints), 3))
    for i in range(order+1):
        gamma = index_add(gamma, index[:, 0], coefficients0[i] * jnp.cos(nfp * 2 * pi * i * quadpoints) * jnp.cos(2 * pi * quadpoints))
        gamma = index_add(gamma, index[:, 1], coefficients0[i] * jnp.cos(nfp * 2 * pi * i * quadpoints) * jnp.sin(2 * pi * quadpoints))
    for i in range(1, order+1):
        gamma = index_add(gamma, index[:, 2], coefficients1[i-1] * jnp.sin(nfp * 2 * pi * i * quadpoints))
    return gamma

class JaxStelleratorSymmetricCylindricalFourierCurve(JaxCurve):

    """ This class can for example be used to describe a magnetic axis. """

    def __init__(self, quadpoints, order, nfp):
        if isinstance(quadpoints, int):
            quadpoints = np.linspace(0, 1/nfp, quadpoints, endpoint=False)
        pure = lambda dofs, points: stelleratorsymmetriccylindricalfouriercurve_pure(dofs, points, order, nfp)
        super().__init__(quadpoints, pure)
        self.order = order
        self.nfp = nfp
        self.coefficients = [np.zeros((order+1,)), np.zeros((order,))]

    def num_dofs(self):
        return 2*self.order+1

    def get_dofs(self):
        return np.concatenate(self.coefficients)

    def set_dofs_impl(self, dofs):
        counter = 0
        for i in range(self.order+1):
            self.coefficients[0][i] = dofs[i]
        for i in range(self.order):
            self.coefficients[1][i] = dofs[self.order + 1 + i]
        for d in self.dependencies:
            d.invalidate_cache()

class StelleratorSymmetricCylindricalFourierCurve(sgpp.StelleratorSymmetricCylindricalFourierCurve, Curve):

    def __init__(self, quadpoints, order, nfp):
        if isinstance(quadpoints, int):
            quadpoints = list(np.linspace(0, 1./nfp, quadpoints, endpoint=False))
        elif isinstance(quadpoints, np.ndarray):
            quadpoints = list(quadpoints)
        Curve.__init__(self)
        sgpp.StelleratorSymmetricCylindricalFourierCurve.__init__(self, quadpoints, order, nfp)

    def get_dofs(self):
        return np.asarray(sgpp.StelleratorSymmetricCylindricalFourierCurve.get_dofs(self))

    def set_dofs(self, dofs):
        sgpp.StelleratorSymmetricCylindricalFourierCurve.set_dofs(self, dofs)
        for d in self.dependencies:
            d.invalidate_cache()


import numpy as np
import matplotlib.pyplot as plt
import math

W0 = 8
W0i = []
Wbi = []
Wei =[]
it = []

for i in range (1,40):
    W0i.append(W0)
    Wbi.append(1.0417+0.3403*W0i[i-1])
    Wei.append((0.5635*math.pow(W0,-0.0624))*W0)
    it.append(i)
    print(W0i[i-1])
    print(Wbi[i-1])
    print(Wei[i-1])
    k = 1 - (Wei[i-1]/W0i[i-1]) - (Wbi[i-1]/W0i[i-1])
    W0 = .5/k
    print(W0)

plt.plot(it, W0i, color = 'red', label = 'Convergence plot')
plt.ylabel('Maximum takeoff weight (kg)')
plt.xlabel('No. of iterations')
plt.title('MTOW vs No. of iterations')
plt.tight_layout()
plt.legend()
plt.xticks(np.arange(0,41,4))
plt.yticks()
plt.show()


from algorithms import Qlearning, SARSA
import matplotlib.pyplot as plt
import gridworlds
import numpy as np
import gym

env = gym.make('PuddleWorld-v0')

def get_derivables(learner, num_episodes, num_expts, show_policy = False, show_plots = True):
	''' num_episodes is number of episodes '''
	''' num_expts is number of experiments to average the performance over '''
	''' if show_policy is True, an image will be displayed showing optimal policy found by the agent after all episodes '''
	x_axis = np.arange(num_episodes)
	avg_steps = np.zeros(num_episodes)
	avg_reward = np.zeros(num_episodes)
	for i in range(num_expts):
		print ("Experiment number %d" % i)
		steps, rwd = learner.run(num_episodes)
		avg_steps += steps
		avg_reward += rwd
		if show_policy or i == num_expts - 1:
			learner.show_policy()

	avg_steps /= float(num_expts)
	avg_reward /= float(num_expts)

	if (show_plots):
		plt.figure(1)
		plt.semilogy(x_axis, avg_steps)
		plt.ylabel("Average steps to reach the goal (log scale)")
		plt.xlabel("episodes")

		plt.figure(2)
		plt.plot(x_axis, avg_reward)
		plt.ylabel("Average reward per episode")
		plt.xlabel("episodes")

		plt.show()

	return avg_steps, avg_reward

get_derivables(Qlearning(env, verbose = False), 2000, 10, False)
#get_derivables(SARSA(env, Lambda = 0.7, verbose = False), 100, 1, False)

"""
avg_steps = []
avg_reward = []
lambdas = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
for l in lambdas:
	print ("    Lambda = {}".format(l))
	steps, reward = get_derivables(SARSA(env, Lambda = l, verbose = False), 25, 10, show_plots = False)
	avg_steps.append(steps[24])
	avg_reward.append(reward[24])

plt.figure(3)
plt.plot(lambdas, avg_steps)
plt.title(r"Average steps after 25 episodes for different $\lambda$")
plt.ylabel("Average steps to reach the goal (averaged over 10 experiments)")
plt.xlabel(r"$\lambda$")

plt.figure(2)
plt.plot(lambdas, avg_reward)
plt.title(r"Average reward after 25 episodes for different $\lambda$")
plt.ylabel("Average reward per episode(averaged over 10 experiments)")
plt.xlabel(r"$\lambda$")

plt.show()
"""


from collections.abc import Collection
from functools import cache
from typing import Final, NamedTuple, Optional

import networkx as nx
import numpy as np

def _scale(v: float) -> float:
    """Scaling a given value.

    The output is defined by

    .. math::

        {\\rm scale}(v) = \\frac{2}{1 + \\exp(-v)} - 1

    Args:
      v: Input value.

    Returns:
      Output value defined above.
    """
    e = np.exp(-v)
    return float(2.0 / (1.0 + e) - 1.0)

class Node:
    """Abstract class of review graph.

    Args:
      graph: the graph object this node will belong to.
      name: name of this node.
    """

    name: Final[str]
    """Name of this node."""
    _g: Final["ReviewGraph"]
    """Graph object this node belongs to."""

    __slots__ = ("name", "_g")

    def __init__(self, graph: "ReviewGraph", name: Optional[str] = None) -> None:
        self._g = graph
        self.name = name if name else super().__str__()

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, type(self)):
            return False
        return self.name == other.name

    def __hash__(self) -> int:
        return 13 * hash(type(self)) + 17 * hash(self.name)

class Reviewer(Node):
    """A node class representing a reviewer.

    Args:
      graph: Graph object this reviewer belongs to.
      name: Name of this reviewer.
      anomalous: Initial anomalous score (default: None).
    """

    trustiness: float
    """A float value in [0, 1] which represents trustiness of this reviewer."""

    __slots__ = ("trustiness",)

    def __init__(self, graph: "ReviewGraph", name: Optional[str] = None, anomalous: Optional[float] = None) -> None:
        super().__init__(graph, name)

        # If an initial anomalous score is given, use it.
        self.trustiness = 1.0 - anomalous if anomalous else 0.5

    @property
    def anomalous_score(self) -> float:
        """Returns the anomalous score of this reviewer.

        The anomalous score is defined by 1 - trustiness.
        """
        return 1.0 - self.trustiness

    def update_trustiness(self) -> float:
        """Update trustiness of this reviewer.

        The updated trustiness of a reviewer :math:`u` is defined by

        .. math::

           {\\rm trustiness}(u) =
             \\frac{2}{1 + \\exp(-\\sum_{r \\in R(u)} {\\rm honesty(r)} )} - 1

        where :math:`R(u)` is a set of reviews the reviewer :math:`u` posts.

        Returns;
          absolute difference between the old trustiness and updated one.
        """
        sum_h = 0.0
        for re in self._g.retrieve_reviews_by_reviewer(self):
            sum_h += re.honesty
        new = _scale(sum_h)

        diff = abs(self.trustiness - new)
        self.trustiness = new
        return diff

    def __str__(self) -> str:
        return f"{self.name}: {self.anomalous_score}"

class Product(Node):
    """A node class representing a product.

    Args:
      graph: Graph object this product belongs to.
      name: Name of this product.
    """

    reliability: float
    """A float value in [0, 1], which represents reliability of this product."""

    __slots__ = ("reliability",)

    def __init__(self, graph: "ReviewGraph", name: Optional[str] = None):
        super(Product, self).__init__(graph, name)
        self.reliability = 0.5

    @property
    def summary(self) -> float:
        """Summary of reviews.

        This value is same as reliability.
        Original algorithm uses *reliability* but our algorithm uses *summary*.
        For convenience, both properties remain.
        """
        return self.reliability

    def update_reliability(self) -> float:
        """Update product's reliability.

        The new reliability is defined by

        .. math::

          {\\rm reliability}(p) = \\frac{2}{1 + e^{-\\theta}} - 1, \\quad
          \\theta = \\sum_{r \\in R(p)}
                      {\\rm trustiness}(r)({\\rm review}(r, p) - \\hat{s}),

        where :math:`R(p)` is a set of reviewers product *p* receives,
        trustiness is defined in :meth:`Reviewer.trustiness`,
        review(*r*, *p*) is the review score reviewer *r* has given to product *p*,
        and :math:`\\hat{s}` is the median of review scores.

        Returns:
          absolute difference between old reliability and new one.
        """
        res = 0.0

        reviews = self._g.retrieve_reviews_by_product(self)
        s = float(np.median([re.rating for re in reviews]))
        for re in reviews:
            for r in self._g.retrieve_reviewers(re):
                res += r.trustiness * (re.rating - s)

        new = _scale(res)
        diff = abs(self.reliability - new)
        self.reliability = new
        return diff

    def __str__(self) -> str:
        return f"{self.name}: {self.summary}"

class Review:
    """A graph entity representing a review.

    Args:
      graph: Graph object this product belongs to.
      time: When this review is posted.
      rating: Rating of this review.
    """

    rating: Final[float]
    """Rating score of this review."""
    honesty: float
    """Honesty score."""
    agreement: float
    """Agreement score."""
    time: Final[int]
    """Time when this review posted."""

    _g: Final["ReviewGraph"]

    __slots__ = ("rating", "honesty", "agreement", "time", "_g")

    def __init__(self, graph: "ReviewGraph", time: int, rating: float) -> None:
        self._g = graph
        self.time = time
        self.rating = rating

        self.honesty = 0.5
        self.agreement = 0.5

    def update_honesty(self) -> float:
        """Update honesty of this review.

        The updated honesty of this review :math:`r` is defined by

        .. math::

           {\\rm honesty}(r)
             = |{\\rm reliability}(P(r))| \\times {\\rm agreement}(r)

        where :math:`P(r)` is the product this review posted.

        Returns:
          absolute difference between old honesty and new one.
        """
        res = 0.0
        for p in self._g.retrieve_products(self):
            res += abs(p.reliability) * self.agreement

        diff = abs(self.honesty - res)
        self.honesty = res
        return diff

    def update_agreement(self, delta: float) -> float:
        """Update agreement of this review.

        This process considers reviews posted in a close time span of this review.
        More precisely, let :math:`t` be the time when this review posted
        and :math:`\\delta` be the time span,
        only reviews of which posted times are in :math:`[t - \\delta, t+\\delta]`
        are considered.

        The updated agreement of a review :math:`r` will be computed with such
        reviews by

        .. math::

           {\\rm agreement}(r)
             = \\frac{2}{1 + \\exp(
                \\sum_{v \\in R_{+}} {\\rm trustiness}(v)
                    - \\sum_{v \\in R_{-}} {\\rm trustiness}(v)
             )} - 1

        where :math:`R_{+}` is a set of reviews close to the review :math:`r`,
        i.e. the difference between ratings are smaller than or equal to delta,
        :math:`R_{-}` is the other reviews. The trustiness of a review means
        the trustiness of the reviewer who posts the review.

        Args:
          delta: a time span :math:`\\delta`.
                 Only reviews posted in the span will be considered for this update.

        Returns:
          absolute difference between old agreement and new one.
        """
        score_diff = 1.0 / 5.0
        agree, disagree = self._g.retrieve_reviews(self, delta, score_diff)

        res = 0.0
        for re in agree:
            for r in self._g.retrieve_reviewers(re):
                res += r.trustiness
        for re in disagree:
            for r in self._g.retrieve_reviewers(re):
                res -= r.trustiness

        new = _scale(res)
        diff = abs(self.agreement - new)
        self.agreement = new
        return diff

    def __str__(self) -> str:
        return f"Review (time={self.time}, rating={self.rating}, agreement={self.agreement}, honesty={self.honesty})"

class ReviewSet(NamedTuple):
    """Pair of agreed reviews and disagreed reviews."""

    agree: Collection[Review]
    """Collection of agreed reviews."""
    disagree: Collection[Review]
    """Collection of disagreed reviews."""

class ReviewGraph:
    """A bipartite graph of which one set of nodes represent reviewers and the other set of nodes represent products.

    Each edge has a label representing a review.

    Args:
        theta: A parameter for updating.
            See `the paper <https://ieeexplore.ieee.org/document/6137345?arnumber=6137345>`__ for more details.
    """

    graph: Final[nx.DiGraph]
    """Graph object of networkx."""
    reviewers: Final[list[Reviewer]]
    """Collection of reviewers."""
    products: Final[list[Product]]
    """Collection of products."""
    reviews: Final[list[Review]]
    """Collection of reviews."""
    _theta: Final[float]
    """Parameter of the algorithm."""
    _delta: Optional[float]
    """Cached time delta."""

    def __init__(self, theta: float) -> None:
        self.graph = nx.DiGraph()
        self.reviewers = []
        self.products = []
        self.reviews = []
        self._theta = theta
        self._delta = None

    @property
    def delta(self) -> float:
        """Time delta.

        This value is defined by
        :math:`\\delta = (t_{\\rm max} - t_{\\rm min}) \\times \\theta`,
        where :math:`t_{\\rm max}, t_{\\rm min}` are the maximum time,
        minimum time of all reviews, respectively,
        :math:`\\theta` is the given parameter defining time ratio.
        """
        if not self._delta:
            min_time = min([r.time for r in self.reviews])
            max_time = max([r.time for r in self.reviews])
            self._delta = (max_time - min_time) * self._theta
        return self._delta

    def new_reviewer(self, name: Optional[str] = None, anomalous: Optional[float] = None) -> Reviewer:
        """Create a new reviewer.

        Args:
            name: the name of the new reviewer.
            anomalous: the anomalous score of the new reviewer.

        Returns:
            A new reviewer instance.
        """
        n = Reviewer(self, name=name, anomalous=anomalous)
        self.graph.add_node(n)
        self.reviewers.append(n)
        return n

    def new_product(self, name: Optional[str] = None) -> Product:
        """Create a new product.

        Args:
            name: The name of the new product.

        Returns:
            A new product instance.
        """
        n = Product(self, name)
        self.graph.add_node(n)
        self.products.append(n)
        return n

    def add_review(self, reviewer: Reviewer, product: Product, review: float, time: Optional[int] = None) -> Review:
        """Add a new review.

        Args:
          reviewer:  An instance of Reviewer.
          product: An instance of Product.
          review: A real number representing review score.
          time: An integer representing reviewing time. (optional)

        Returns:
          the new review object.
        """
        if not time:
            re = Review(self, len(self.reviews), review)
        else:
            re = Review(self, time, review)
        self.graph.add_node(re)
        self.reviews.append(re)
        self.graph.add_edge(reviewer, re)
        self.graph.add_edge(re, product)
        self._delta = None
        return re

    @cache
    def retrieve_reviewers(self, review: Review) -> Collection[Reviewer]:
        """Find reviewers associated with a review.

        Args:
            review: A review instance.

        Returns:
            A list of reviewers associated with the review.
        """
        return list(self.graph.predecessors(review))

    @cache
    def retrieve_products(self, review: Review) -> Collection[Product]:
        """Find products associated with a review.

        Args:
            review: A review instance.

        Returns:
            A list of products associated with the given review.
        """
        return list(self.graph.successors(review))

    @cache
    def retrieve_reviews_by_reviewer(self, reviewer: Reviewer) -> Collection[Review]:
        """Find reviews given by a reviewer.

        Args:
            reviewer: Reviewer

        Returns:
            A list of reviews given by the reviewer.
        """
        return list(self.graph.successors(reviewer))

    @cache
    def retrieve_reviews_by_product(self, product: Product) -> Collection[Review]:
        """Find reviews to a product.

        Args:
            product: Product

        Returns:
            A list of reviews to the product.
        """
        return list(self.graph.predecessors(product))

    def retrieve_reviews(
        self, review: Review, time_diff: Optional[float] = None, score_diff: float = 0.25
    ) -> ReviewSet:
        """Find agree and disagree reviews.

        This method retrieve two groups of reviews.
        Agree reviews have similar scores to a given review.
        On the other hands disagree reviews have different scores.

        Args:
          review: A review instance.
          time_diff: An integer.
          score_diff: An float value.

        Returns:
          A tuple consists of (a list of agree reviews, a list of disagree reviews)
        """
        if not time_diff:
            time_diff = float("inf")

        agree, disagree = [], []
        for p in self.retrieve_products(review):
            for re in self.retrieve_reviews_by_product(p):
                if re == review:
                    continue
                if abs(re.time - review.time) < time_diff:
                    if abs(re.rating - review.rating) < score_diff:
                        agree.append(re)
                    else:
                        disagree.append(re)
        return ReviewSet(agree, disagree)

    def update(self) -> float:
        """Update reviewers' anomalous scores and products' summaries.

        This update process consists of four steps;

        1. Update honesty of reviews (See also :meth:`Review.update_honesty`),
        2. Update rustiness of reviewers
           (See also :meth:`Reviewer.update_trustiness`),
        3. Update reliability of products
           (See also :meth:`Product.update_reliability`),
        4. Update agreements of reviews
           (See also :meth:`Review.update_agreement`).

        Returns:
          summation of maximum absolute updates for the above four steps.
        """
        diff = max(re.update_honesty() for re in self.reviews)
        diff += max(r.update_trustiness() for r in self.reviewers)
        diff += max(p.update_reliability() for p in self.products)
        diff += max(re.update_agreement(self.delta) for re in self.reviews)
        return diff


# -*- coding: utf-8 -*-
import zajednickeFunkcionalnosti as zf
import pocetak
from colorama import Back
import baza

def test1():
    print(Back.GREEN + "Ulogovani ste kao korisnik")
    print(Back.RESET)

def korisnik_menu(ulogovani):
    global korisnik
    korisnik = ulogovani
    print(Back.BLUE + "\n{}ODABERITE OPCIJU{}".format(">"*5, "<"*5))
    print(Back.RESET)
    print("\n1) Prikaz svih kategorija")
    print("\n2) Pretraga proizvoda po kategoriji")
    print("\n3) Prikaz proizvoda sortiranih po ceni")
    print("\n4) Prikaz proizvoda na akciji")
    print("\n5) Narucivanje proizvoda")
    print("\n6) Prikaz narucenih proizvoda")
    print("\n7) Dodavanje komentara")
    print("\n8) Prikaz proizvoda u nekom opsegu cena")
    print("\n9) Odjava")
    print("\n10) Zavrsi sa radom")
    izbor_opcije()

def izbor_opcije():
    global korisnik
    try:
        opcija = eval(input("Unesite zeljenu opciju: "))
    except:
        opcija = 11

    if opcija == 1:
        zf.sve_kategorije()
    elif opcija == 2:
        zf.pretraga_proizvoda_po_kategoriji()
    elif opcija == 3:
        proizvodi_sortirani_po_ceni()
    elif opcija == 4:
        prikaz_proizvoda_na_akciji()
    elif opcija == 5:
        narucivanje_proizvoda()
    elif opcija == 6:
        prikaz_narudzbina()
    elif opcija == 7:
        dodavanje_komentara()
    elif opcija == 8:
        prikaz_proizvoda_u_opsegu_cena()
    elif opcija == 9:
        zf.odjava()
    elif opcija == 10:
        pocetak.end()
    else:
        print(Back.RED + "Uneli ste pogresnu opciju. Molimo pokusajte ponovo")
        print(Back.RESET)
        korisnik_menu(korisnik)

def proizvodi_sortirani_po_ceni():
      global korisnik
      print(Back.BLUE + "Prikaz proizvoda sortiranih po ceni: ")
      print(Back.RESET)
      print("RBR  | {0:<10} | {1:<10} | {2:<48} | {3:<5} | ".format("Naziv", "Cena", "Opis", "Akcija"))
      print("-"*92)
      rbr = 0
      proizvodi = baza.svi_proizvodi()
      proizvodi.sort(key = lambda x: x[2])
      if proizvodi:
          for p in proizvodi:
               rbr += 1
               print("{4:5}|{0:^12}|{1:^12}|{2:^50}|{3:^8}|".format(p[1], p[2], p[3], p[4],rbr))
               print("-"*92)
      try:
        nazad = eval(input("Unesite 0 za povratak: "))
      except:
        nazad = 2

      if nazad == 0:
        korisnik_menu(korisnik)
      else:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        proizvodi_sortirani_po_ceni()

def prikaz_proizvoda_na_akciji():
    global korisnik
    print(Back.BLUE + "Prikaz proizvodana akciji: ")
    print(Back.RESET)
    print("RBR  | {0:<9} | {1:<10}".format("Naziv", "Cena"))
    print('-'*30)
    rbr = 0
    proizvodi = baza.svi_proizvodi()
    for p in proizvodi:
        if(p[4] == 1):
            rbr += 1
            print("{0:5}|{1:^11}|{2:^12}".format(rbr, p[1], p[2]))
            print("-"*30)

    try:
        nazad = eval(input("Unesite 0 za povratak: "))
    except:
        nazad = 2

    if nazad == 0:
        korisnik_menu(korisnik)
    else:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        prikaz_proizvoda_na_akciji()

def narucivanje_proizvoda():
    global korisnik
    proizvodi = baza.svi_proizvodi()

    print(Back.BLUE + "Proizvodi: \n")
    print(Back.RESET)

    print("{0:<5} | {1:<10} | {2:<10} | {3:<48}".format("Id", "Naziv", "Cena", "Opis"))

    for p in proizvodi:
        print('='*85)
        print('{:^6}| {:<10} | {:<10} | {:^48}'.format(p[0], p[1], p[2], p[3]))

    try:
        id_proizvoda = eval(input("Unesite id proizvoda koji zelite da porucite: "))
    except:
        id_proizvoda = 0

    try:
        baza.narucivanje_proizvoda(id_proizvoda,korisnik[0], postarina = 300)
        print(Back.GREEN + "Uspesno ste porucili proizvod!")
        print(Back.RESET)
    except:
        print(Back.RED + "Nesto nije u redu.Pokusajte ponovo!")
        print(Back.RESET)
        narucivanje_proizvoda()

    try:
        nazad = eval(input("Unesite 0 za povratak: "))
    except:
        nazad

    while not nazad == 0:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        nazad = eval(input("Unesite 0 za povratak: "))

    korisnik_menu(korisnik)

def prikaz_narudzbina():
    global korisnik
    narucene = baza.narucene(korisnik[0])
    if narucene:
        suma_proizvoda = 0
        rbr = 0
        print("\nVasa narudzbina: ")
        for p in narucene:
            proizvod = baza.nadji_proizvod(p[4])
            if proizvod:
                rbr+=1
                print("{:^8}|{:<15}|{}".format(rbr,proizvod[1], proizvod[2]))
                print('-'*40)
                suma_proizvoda += proizvod[2]
        print("{}{}".format("Ukupan iznos svih narucenih proizvoda: ", suma_proizvoda))

    else:
        print(Back.RED + "Niste nista narucili!")
        print(Back.RESET)

    try:
        nazad = eval(input("Unesite 0 za povratak: "))
    except:
        nazad = 2

    if nazad == 0:
        korisnik_menu(korisnik)
    else:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        prikaz_narudzbina()

def dodavanje_komentara():
    global korisnik
    proizvodi = baza.svi_proizvodi()

    print(Back.BLUE + "Proizvodi: \n")
    print(Back.RESET)

    print("{0:<5} | {1:<10} | {2:<10} | {3:<48}".format("Id", "Naziv", "Cena", "Opis"))

    for p in proizvodi:
        print('='*85)
        print('{:^6}| {:<10} | {:<10} | {:^48}'.format(p[0], p[1], p[2], p[3]))

    try:
        id_proizvoda = eval(input("Unesite id proizvoda za koji zelite da dodate komentar: "))
    except:
        id_proizvoda = 0

    tekst = input("Unesite komentar: ")
    try:
        baza.dodavanje_komentara(korisnik[0], id_proizvoda, tekst)
        print(Back.GREEN + "Uspesno ste uneli komentar!")
        print(Back.RESET)
    except:
        print(Back.RED + "Nesto nije u redu.Pokusajte ponovo!")
        print(Back.RESET)
        dodavanje_komentara()

    try:
        nazad = eval(input("Unesite 0 za povratak: "))
    except:
        nazad

    while not nazad == 0:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        nazad = eval(input("Unesite 0 za povratak: "))

    korisnik_menu(korisnik)

def prikaz_proizvoda_u_opsegu_cena():
      minCena = eval(input("Unesite minimalnu cenu proizvoda: "))
      maxCena = eval(input("Unesite maksimalnu cenu proizvoda: "))
      print(Back.BLUE + "\nPrikaz proizvoda u opsegu cena: ")
      print(Back.RESET)
      print("RBR  | {0:<10} | {1:<10} | {2:<48} | {3:<5} | ".format("Naziv", "Cena", "Opis", "Akcija"))
      print("-"*92)
      rbr = 0
      proizvodi = baza.svi_proizvodi()
      proizvodi = (filter(lambda x: x[2]>=minCena and x[2]<=maxCena, proizvodi))
      if proizvodi:
          for p in proizvodi:
               rbr += 1
               print("{4:5}|{0:^12}|{1:^12}|{2:^50}|{3:^8}|".format(p[1], p[2], p[3], p[4],rbr))
               print("-"*92)
      try:
        nazad = eval(input("Unesite 0 za povratak: "))
      except:
        nazad = 2

      if nazad == 0:
        korisnik_menu(korisnik)
      else:
        print(Back.RED + "Pogresno ste uneli opciju")
        print(Back.RESET)
        prikaz_proizvoda_u_opsegu_cena()


from rest_framework.response import Response
from rest_framework.decorators import api_view
from rest_framework import status
from .models import Product
from .serializers import *

@api_view(['GET'])
def productsView(request):
    data    = Product.objects.filter(owner=request.user)
    print(data)

    ser     = ProductSerializer(data, many=True)
    return Response(ser.data, status=status.HTTP_200_OK)

@api_view(['GET', 'PUT', 'DELETE'])
def productView(request, product_id):
    data    = Product.objects.get(pk=product_id)
    return True
@api_view(['GET', 'POST'])
def students_list(request):
    if request.method == 'GET':
        data = Student.objects.all()

        serializer = StudentSerializer(data, context={'request': request}, many=True)

        return Response(serializer.data)

    elif request.method == 'POST':
        serializer = StudentSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(status=status.HTTP_201_CREATED)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['PUT', 'DELETE'])
def students_detail(request, pk):
    try:
        student = Student.objects.get(pk=pk)
    except Student.DoesNotExist:
        return Response(status=status.HTTP_404_NOT_FOUND)

    if request.method == 'PUT':
        serializer = StudentSerializer(student, data=request.data,context={'request': request})
        if serializer.is_valid():
            serializer.save()
            return Response(status=status.HTTP_204_NO_CONTENT)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        student.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)
@api_view(['GET', 'POST'])
def servicesList(request, title=None, description=None):
    if request.method == 'GET':
        data = Services.objects.all()
        serializer = ServicesSerializer(data, context={'request': request}, many=True)
        return Response(serializer.data)
    if request.method == 'POST':
        data = Services()
        data.title = title
        data.description = description
        data.save()

        return Response(status=status.HTTP_201_CREATED)
@api_view(['GET', 'DELETE'])
def service(request, service_title):
    service_title.lower()
    print(service_title)
    try:
        data = Services.objects.get(title=service_title)
        print(data)
    except:
        return Response(status=status.HTTP_204_NO_CONTENT)
    if request.method == 'GET':
        serailizer = ServicesSerializer(data, context={'request': request}, many=True)
        return Response(serailizer.data)

    else :
        if request.method == 'DELETE':
            try:
                data.delete()
            except:
                pass
            return Response(status=status.HTTP_200_OK)


from sklearn.linear_model import LogisticRegression as LR
from sklearn.tree import DecisionTreeClassifier as DR
import xclib.evaluation.xc_metrics as xc_metrics
from xc.libs.utils import pbar
import scipy.sparse as sp
import numpy as np
import joblib

class Fusion(object):
    def __init__(self, reg="DecisionTrees", A=0.55, B=1.5, use_psp=False):
        self.A = A
        self.B = B
        if reg == "DecisionTrees":
            self.regressor = DR(max_depth=7)
        if reg == "LogisticRegression":
            self.regressor = LR(penalty='l2', verbose=1, n_jobs=8)

        self.inv_psp = None
        self.use_psp = use_psp
        self.valid = True
        self.val = np.asarray([])

    def split(self, trn_dset, valid_frac=0.035):
        print("FUSION:Generating SPLITS")
        valid_int = int(np.ceil(len(trn_dset)*valid_frac))
        val = np.random.choice(len(trn_dset), size=valid_int, replace=False)
        trn = np.setdiff1d(np.arange(len(trn_dset)), val)
        self.val = val
        return trn, val

    @property
    def val_indices(self):
        return self.val

    def build_psp(self, trn_y):
        print("FUSION:Building PSP")
        self.inv_psp = xc_metrics.compute_inv_propesity(
            trn_y, A=self.A, B=self.B)

    def prep_descriptor(self, m2, m4):
        rows, cols = m2.nonzero()
        module2_data = np.ravel(m2.tolil()[rows, cols].todense())
        module4_data = np.ravel(m4.tolil()[rows, cols].todense())
        inv_psp = np.ravel(self.inv_psp[cols])
        return np.vstack([inv_psp, module2_data, module4_data]).T, rows, cols

    def fit(self, m2, m4, trn_y):
        self.valid = True
        print("FUSION::SETTING UP PARAMETERS")
        X, rows, cols = self.prep_descriptor(m2, m4)
        y = np.ravel(trn_y.tolil()[rows, cols].todense())
        print("FUSION::TRAINING FUSION LAYER")
        self.regressor.fit(X, y)

    def predict(self, m2, m4, batch_size=200, a_min=1e-1):
        num_docs = m4.shape[0]
        final_score = sp.lil_matrix(m4.shape)
        for start in pbar(np.arange(0, num_docs, batch_size),
                          desc="fusion", write_final=True):
            end = min(num_docs, start+batch_size)
            chotu_m4 = m4[start:end]
            chotu_m2 = m2[start:end]
            X, rows, cols = self.prep_descriptor(chotu_m2, chotu_m4)
            y = self.regressor.predict_proba(X)[:, 1]
            y = np.clip(y, a_min, 1-a_min)
            final_score[rows+start, cols] = y
            del chotu_m2, chotu_m4
        return final_score.tocsr()

    def save(self, model_path):
        joblib.dump({"regressor": self.regressor,
                     "inv_psp": self.inv_psp,
                     "valid": self.valid,
                     "val": self.val}, model_path)

    def load(self, model_path):
        data = joblib.load(model_path)
        self.regressor = data["regressor"]
        self.inv_psp = data["inv_psp"]
        self.valid = data["valid"]
        self.val = data["val"]


import unittest, os

import osaf.pim.tests.TestDomainModel as TestDomainModel
from osaf.pim.contacts import Contact, ContactName

from chandlerdb.util.Path import Path
from i18n.tests import uw

class ContactsTest(TestDomainModel.DomainModelTestCase):
    """ Test Contacts Domain Model """

    def testContacts(self):
        """ Simple test for creating instances of contact related kinds """

        self.loadParcel("osaf.pim.contacts")
        def _verifyContactName(name):
            self.assertEqual(name.firstName, uw('Sylvia'))
            self.assertEqual(name.lastName, uw('Plath'))

        # Test the globals
        contactsPath = Path('//parcels/osaf/pim/contacts')
        view = self.view

        self.assertEqual(Contact.getKind(view),
                         view.find(Path(contactsPath, 'Contact')))
        self.assertEqual(ContactName.getKind(view),
                         view.find(Path(contactsPath, 'ContactName')))

        # Construct sample items
        contactItem = Contact("contactItem", itsView=view)
        contactNameItem = ContactName("contactNameItem", itsView=view)

        # Double check kinds
        self.assertEqual(contactItem.itsKind, Contact.getKind(view))
        self.assertEqual(contactNameItem.itsKind, ContactName.getKind(view))

        # Literal properties
        contactNameItem.firstName = uw("Sylvia")
        contactNameItem.lastName = uw("Plath")

        _verifyContactName(contactNameItem)

        self._reopenRepository()
        view = self.view

        contentItemParent = view.findPath("//userdata")

        contactNameItem = contentItemParent.getItemChild("contactNameItem")
        _verifyContactName(contactNameItem)

if __name__ == "__main__":
    unittest.main()


import numpy as np
import torch
import random
from eval.eval import f1
import pdb

def load_ricap(images, targets, dilates):
    batch_images = torch.empty(images.size())
    batch_targets = torch.empty(targets.size())
    batch_dilates = torch.empty(dilates.size())
    for i in range(images.size()[0]):
        patched_image, patched_target, patched_dilate = ricap(images, targets, dilates)
        batch_images[i] = patched_image
        batch_targets[i] = patched_target
        batch_dilates[i] = patched_dilate

    return batch_images, batch_targets, batch_dilates

def ricap(images, targets, dilates):

    # size of image
    I_x, I_y = images.size()[2:]

    # generate boundary position (w, h)
    w = int(np.round(I_x * np.random.uniform(0.3, 0.7)))
    h = int(np.round(I_y * np.random.uniform(0.3, 0.7)))
    w_ = [w, I_x-w, w, I_x-w]
    h_ = [h, h, I_y-h, I_y-h]

    # select four images
    cropped_images = {}
    cropped_target = {}
    cropped_dilate = {}
    c_ = {}
    W_ = {}

    for k in range(4):
        index = random.choice(torch.randperm(images.size(0)))
        x_k = np.random.randint(0, I_x - w_[k] + 1)
        y_k = np.random.randint(0, I_y - h_[k] + 1)

        cropped_images[k] = images[index, :, x_k:x_k + w_[k], y_k:y_k + h_[k]]
        cropped_target[k] = targets[index, :, x_k:x_k + w_[k], y_k:y_k + h_[k]]
        cropped_dilate[k] = dilates[index, :, x_k:x_k + w_[k], y_k:y_k + h_[k]]
        c_[k] = targets[index]
        W_[k] = (w_[k] * h_[k]) / (I_x * I_y)

    # patch cropped images
    patched_images = torch.cat(
        (torch.cat((cropped_images[0], cropped_images[1]), 1),
         torch.cat((cropped_images[2], cropped_images[3]), 1)),
        2)
    patched_target = torch.cat(
        (torch.cat((cropped_target[0], cropped_target[1]), 1),
         torch.cat((cropped_target[2], cropped_target[3]), 1)),
        2)
    print(patched_target.size())
    patched_dilate = torch.cat(
        (torch.cat((cropped_dilate[0], cropped_dilate[1]), 1),
         torch.cat((cropped_dilate[2], cropped_dilate[3]), 1)),
        2)

    targets = (c_, W_)
    return patched_images, patched_target, patched_dilate

if __name__ == '__main__':
    input_image = torch.rand(4, 3, 256, 256)
    input_target = torch.rand(4, 1, 256, 256)
    input_target = (input_target> 0.5)*1

    image, target, dilate = load_ricap(input_image, input_target, input_target)
    print(target.size())

    acc, _, _, f1_score = f1(target, target, dilate)
    print(acc, f1_score)


import math
import random
from collections import Counter
from unittest import IsolatedAsyncioTestCase

import pytest
from ash_dal import AsyncBaseDAO, AsyncDatabase, AsyncDeferredJoinPaginator, PaginatorPage
from ash_dal.utils import DeferredJoinPaginatorFactory
from faker import Faker
from sqlalchemy import select

from tests.constants import ASYNC_DB_URL
from tests.dao.infrastructure import ExampleEntity, ExampleORMModel

class ExampleAsyncDAO(AsyncBaseDAO[ExampleEntity]):
    __entity__ = ExampleEntity
    __model__ = ExampleORMModel

class ExampleDAOCustomPaginator(ExampleAsyncDAO):
    __paginator_factory__ = DeferredJoinPaginatorFactory[AsyncDeferredJoinPaginator](
        paginator_class=AsyncDeferredJoinPaginator,
        pk_field=ExampleORMModel.id,
    )

class AsyncDAOTestCaseBase(IsolatedAsyncioTestCase):
    async def asyncSetUp(self) -> None:
        self.faker = Faker()
        self.db = AsyncDatabase(db_url=ASYNC_DB_URL)
        await self.db.connect()
        async with self.db.engine.begin() as conn:
            await conn.run_sync(ExampleORMModel.metadata.drop_all)
            await conn.run_sync(ExampleORMModel.metadata.create_all)
        self.dao = ExampleAsyncDAO(database=self.db)

    async def asyncTearDown(self) -> None:
        await self.db.disconnect()

class AsyncDAOFetchingTestCaseBase(AsyncDAOTestCaseBase):
    def _generate_record(
        self, id_: int, first_name: str | None = None, last_name: str | None = None, age: int | None = None
    ):
        return ExampleORMModel(
            id=id_,
            first_name=first_name or self.faker.first_name(),
            last_name=last_name or self.faker.last_name(),
            age=age or self.faker.pyint(min_value=10, max_value=100),
        )

class AsyncDAOTestCase(AsyncDAOFetchingTestCaseBase):
    async def test_get_by_pk(self):
        expected_entity = ExampleEntity(
            id=self.faker.pyint(min_value=1, max_value=10000),
            first_name=self.faker.first_name(),
            last_name=self.faker.last_name(),
            age=self.faker.pyint(min_value=16, max_value=100),
        )

        db_obj = self._generate_record(
            id_=expected_entity.id,
            first_name=expected_entity.first_name,
            last_name=expected_entity.last_name,
            age=expected_entity.age,
        )

        async with self.db.session as session:
            session.add(db_obj)
            await session.commit()

        result = await self.dao.get_by_pk(expected_entity.id)
        assert result
        assert isinstance(result, ExampleEntity)
        assert result.id == expected_entity.id
        assert result.first_name == expected_entity.first_name
        assert result.last_name == expected_entity.last_name
        assert result.age == expected_entity.age

    async def test_get_by_pk__not_found(self):
        result = await self.dao.get_by_pk(123)
        assert not result

class AsyncDAOFetchAllTestCase(AsyncDAOFetchingTestCaseBase):
    async def asyncSetUp(self) -> None:
        await super().asyncSetUp()
        self.records_count = self.faker.pyint(min_value=50, max_value=200)
        await self._create_records(self.records_count)

    async def _create_records(self, count: int):
        records = tuple(self._generate_record(id_=i) for i in range(1, count + 1))
        async with self.db.session as session:
            session.add_all(records)
            await session.commit()

    async def test_all(self):
        results = await self.dao.all()
        assert results
        assert isinstance(results, tuple)
        assert len(results) == self.records_count
        assert isinstance(results[0], ExampleEntity)

    async def test_get_page__default_page_size(self):
        results = await self.dao.get_page()
        assert results
        assert isinstance(results, PaginatorPage)
        assert len(results) == self.dao.__default_page_size__
        assert isinstance(results[0], ExampleEntity)

    async def test_get_page__custom_page_size(self):
        page_size = self.faker.pyint(min_value=2, max_value=20)
        results = await self.dao.get_page(page_size=page_size)
        assert results
        assert len(results) == page_size

    async def test_get_page__defined_page_index(self):
        page_index = self.faker.pyint(min_value=1, max_value=4)
        page_size = 10
        results = await self.dao.get_page(page_index=page_index, page_size=page_size)
        assert results
        assert len(results) == page_size

    async def test_get_page__page_index_out_of_range(self):
        page_size = 10
        page_index = math.ceil(self.records_count / page_size) + 1
        results = await self.dao.get_page(page_index=page_index, page_size=page_size)
        assert not results
        assert isinstance(results, PaginatorPage)

    async def test_paginate__default_page_size(self):
        page_size = self.dao.__default_page_size__
        pages_count = math.ceil(self.records_count / page_size)
        pages_counter = 0
        async for page in self.dao.paginate():
            pages_counter += 1
            assert isinstance(page, PaginatorPage)
            assert isinstance(page[0], ExampleEntity)
            if page.index + 1 < pages_count:
                assert len(page) == page_size
            else:
                assert len(page) <= page_size
        assert pages_count == pages_counter

    async def test_paginate__custom_page_size(self):
        page_size = self.faker.pyint(min_value=2, max_value=20)
        pages_count = math.ceil(self.records_count / page_size)
        pages_counter = 0
        async for page in self.dao.paginate(page_size=page_size):
            pages_counter += 1
            assert isinstance(page, PaginatorPage)
            if page.index + 1 < pages_count:
                assert len(page) == page_size
            else:
                assert len(page) <= page_size
        assert pages_count == pages_counter

class AsyncDAOFetchFilteredTestCase(AsyncDAOFetchingTestCaseBase):
    async def asyncSetUp(self) -> None:
        await super().asyncSetUp()
        self.records_count = self.faker.pyint(min_value=50, max_value=200)
        self.records_counter = Counter()
        await self._create_records(self.records_count)

    async def _create_records(self, count: int):
        records = tuple(self._generate_record(id_=i) for i in range(1, count + 1))
        async with self.db.session as session:
            session.add_all(records)
            await session.commit()

    def _generate_record(
        self, id_: int, first_name: str | None = None, last_name: str | None = None, age: int | None = None
    ):
        record = ExampleORMModel(
            id=id_,
            first_name=first_name or self.faker.first_name(),
            last_name=last_name or self.faker.last_name(),
            age=age or random.choice((20, 30)),
        )
        self.records_counter[str(record.age)] += 1
        return record

    async def test_filter(self):
        results_20_age = await self.dao.filter(specification={"age": 20})
        results_30_age = await self.dao.filter(specification={"age": 30})
        assert len(results_20_age) == self.records_counter["20"]
        assert len(results_30_age) == self.records_counter["30"]

    async def test_filter__not_found(self):
        results = await self.dao.filter(specification={"age": 40})
        assert not results
        assert isinstance(results, tuple)

    async def test_paginate_filtered(self):
        page_size = 3
        pages_count = math.ceil(self.records_counter["30"] / page_size)
        page_counter = 0
        async for page in self.dao.paginate(specification={"age": 30}, page_size=page_size):
            assert page
            assert isinstance(page, PaginatorPage)
            if page.index + 1 < pages_count:
                assert len(page) == page_size
            else:
                assert len(page) <= page_size
            assert isinstance(page[0], ExampleEntity)
            page_counter += 1
        assert page_counter == pages_count

class AsyncDAOCustomPaginatorUseCase(AsyncDAOFetchAllTestCase):
    async def asyncSetUp(self) -> None:
        await super().asyncSetUp()
        self.dao = ExampleDAOCustomPaginator(database=self.db)

class AsyncDAOCreateTestCase(AsyncDAOTestCaseBase):
    async def test_create(self):
        data = {
            "first_name": self.faker.first_name(),
            "last_name": self.faker.last_name(),
            "age": self.faker.pyint(min_value=10, max_value=100),
        }
        entity = await self.dao.create(data=data)
        assert isinstance(entity, ExampleEntity)
        async with self.db.session as session:
            instance = await session.get(ExampleORMModel, entity.id)
            assert instance

    async def test_bulk_create(self):
        items_count = self.faker.pyint(min_value=2, max_value=10)
        data = tuple(
            {
                "first_name": self.faker.first_name(),
                "last_name": self.faker.last_name(),
                "age": self.faker.pyint(min_value=10, max_value=100),
            }
            for _ in range(items_count)
        )
        await self.dao.bulk_create(data=data)
        async with self.db.session as session:
            results = await session.execute(select(ExampleORMModel))
            items = results.all()
            assert len(items) == items_count

class AsyncDAOUpdateTestCase(AsyncDAOTestCaseBase):
    async def _create_record(self):
        data = {
            "first_name": self.faker.first_name(),
            "last_name": self.faker.last_name(),
            "age": self.faker.pyint(min_value=10, max_value=100),
        }
        async with self.db.session as session:
            db_item = ExampleORMModel(**data)
            session.add(db_item)
            await session.commit()
        return {**data, "id": db_item.id}

    async def test_update(self):
        created_record = await self._create_record()
        update_data = {
            "first_name": self.faker.first_name(),
            "last_name": self.faker.last_name(),
            "age": self.faker.pyint(min_value=10, max_value=100),
        }
        record_id = created_record.get("id")
        is_updated = await self.dao.update(specification={"id": record_id}, update_data=update_data)
        assert is_updated
        async with self.db.session as session:
            instance = await session.get(ExampleORMModel, record_id)
            assert instance
            assert instance.first_name == update_data["first_name"]
            assert instance.last_name == update_data["last_name"]
            assert instance.age == update_data["age"]

    async def test_update__empty_specification(self):
        with pytest.raises(ValueError):
            await self.dao.update(specification={}, update_data={})

class AsyncDAODeleteTestCase(AsyncDAOTestCaseBase):
    async def _create_record(self):
        data = {
            "first_name": self.faker.first_name(),
            "last_name": self.faker.last_name(),
            "age": self.faker.pyint(min_value=10, max_value=100),
        }
        async with self.db.session as session:
            db_item = ExampleORMModel(**data)
            session.add(db_item)
            await session.commit()
        return {**data, "id": db_item.id}

    async def test_delete(self):
        created_record = await self._create_record()
        record_id = created_record.get("id")
        is_deleted = await self.dao.delete(specification={"id": record_id})
        assert is_deleted
        async with self.db.session as session:
            instance = await session.get(ExampleORMModel, record_id)
            assert not instance

    async def test_delete__empty_specification(self):
        with pytest.raises(ValueError):
            await self.dao.delete(specification={})


#!/usr/bin/env python

import json,requests,argparse
import os,sys,argparse,time
import numpy as np
import pdb
import heapq

t0 = time.time()

apikey=os.environ.get('SPECTER_API_KEY')
parser = argparse.ArgumentParser()
parser.add_argument("--dir", help="a directory such as $proposed or $specter", required=True)
parser.add_argument("-V", '--verbose', action='store_true')
parser.add_argument("--use_references", help="never|always|when_necessary", default="never")
parser.add_argument("-G", "--graph", help="file (without .X.i and .Y.i)", default=None)
parser.add_argument("-K", "--K", help="The K closest papers in cosine similarity to our desired paper", default=10)
args = parser.parse_args()

def map_int64(fn):
    fn_len = os.path.getsize(fn)
    return np.memmap(fn, dtype=np.int64, shape=(int(fn_len/8)), mode='r')

def map_int32(fn):
    fn_len = os.path.getsize(fn)
    return np.memmap(fn, dtype=np.int32, shape=(int(fn_len/4)), mode='r')

Y = idx = None

if not args.graph is None:
    Y = map_int32(args.graph + '.Y.i')

    if not os.path.exists(args.graph + '.X.i.idx'):
        print('%0.0f sec: computing idx' % (time.time() - t0), file=sys.stderr)
        sys.stderr.flush()
        X = map_int32(args.graph + '.X.i')
        res = np.cumsum(np.bincount(X))
        res.tofile(args.graph + '.X.i.idx')
        print('%0.0f sec: idx computed' % (time.time() - t0), file=sys.stderr)
        sys.stderr.flush()

    idx = map_int64(args.graph + '.X.i.idx')

def extract_row(x):
    # print('extract_row: ' + str(x))
    if x >= len(idx) or x < 0:
        return []
    if x == 0:
        return Y[:idx[0]]
    else:
        return Y[idx[x-1]:idx[x]]

def record_size_from_dir(dir):
    with open(dir + '/record_size', 'r') as fd:
        return int(fd.read().split('\t')[0])

def map_from_dir(dir):
    fn = dir + '/map.old_to_new.i'
    fn_len = os.path.getsize(fn)
    return np.memmap(fn, dtype=np.int32, shape=(int(fn_len/4)), mode='r')

def imap_to_dir(dir):
    fn = dir + '/map.new_to_old.i'
    fn_len = os.path.getsize(fn)
    return np.memmap(fn, dtype=np.int32, shape=(int(fn_len/4)), mode='r')

def embedding_from_dir(dir, K):
    fn = dir + '/embedding.f'
    fn_len = os.path.getsize(fn)
    return np.memmap(fn, dtype=np.float32, shape=(int(fn_len/(4*K)), K), mode='r')

def directory_to_config(dir):
    K = record_size_from_dir(dir)
    return { 'record_size' : K,
             'dir' : dir,
             'map' : map_from_dir(dir),
             'imap': imap_to_dir(dir),
             'embedding' : embedding_from_dir(dir, K)}

t0 = time.time()
config = directory_to_config(args.dir)
print('config: ', config['map'])
print('config contents: ', config['map'][0])
def get_corpusId(ref):
    try:
        return ref['externalIds']['CorpusId']
    except:
        return None

def id_to_references(my_id):
    if not args.graph is None:
        return extract_row(int(my_id))
    cmd = 'https://api.semanticscholar.org/graph/v1/paper/CorpusId:' + str(my_id) + '/?fields=references,references.externalIds'
    j = requests.get(cmd, headers={"x-api-key": apikey}).json()
    if 'references' in j and not j['references'] is None:
        return [ get_corpusId(ref) for ref in j['references']]
    else:
        return []

maxid = config['map'].shape[0]
my_map = config['map'].reshape(-1)
emb = config['embedding']

def centroid(refs):
    mapped_refs = np.array([ my_map[ref] for ref in refs if not ref is None and ref < maxid ], dtype=int)
    mapped_refs = mapped_refs[mapped_refs > 0]
    vectors = emb[mapped_refs,:]
    if args.verbose:
        print('centroid: vectors.shape = ' + str(vectors.shape), file=sys.stderr)
    if len(vectors) > 0:
        return np.mean(vectors, axis=0)
    else:
        return np.zeros(emb.shape[1], dtype=int)

def id_to_centroid(id):
    return centroid(id_to_references(id))

ids = np.array([int(i) for i in sys.stdin.read().split('\n') if len(i) > 0], dtype=int)
if args.verbose:
    print('ids.shape = ' + str(ids.shape), file=sys.stderr)
if args.use_references == 'never' or args.use_references == 'when_necessary':
    mapped_ids = np.zeros(len(ids), dtype=int)
    for e,i in enumerate(ids):
        if i < maxid:
            mapped_ids[e] = my_map[i]
    result = emb[mapped_ids,:]
    result[mapped_ids == 0,:] = 0
elif args.use_references == 'always':
    result = np.array([id_to_centroid(id) for id in ids])
else:
    assert False, 'bad arg: use_references = ' + str(args.use_references)

if args.verbose:
    print('result.shape: ' + str(result.shape), file=sys.stderr)

# instead of saving this output, we want to find the highest K cosine similarities
def cosine_similarity_vector_matrix(vector, matrix):
    dot_product = np.dot(matrix, vector)
    norm_vector = np.linalg.norm(vector)
    norm_matrix = np.linalg.norm(matrix, axis=1)
    denominator = norm_vector * norm_matrix
    denominator[denominator == 0] = 1
    similarity = dot_product / denominator

    return similarity

cosines = cosine_similarity_vector_matrix(np.array(result).reshape(-1), emb)
print("Found cosines in %0.2f seconds." % (time.time() - t0))
# then we find the K largest cosine values

def find_k_largest_with_indices(array, K):
    max_heap = []
    for i, num in enumerate(array):
        if len(max_heap) < K:
            heapq.heappush(max_heap, (num, i))
        else:
            if num > max_heap[0][0]:
                heapq.heappop(max_heap)
                heapq.heappush(max_heap, (num, i))
    result_indices = []
    result_values = []
    while max_heap:
        num, i = heapq.heappop(max_heap)
        result_indices.append(i)
        result_values.append(num)
    result_indices.reverse()
    result_values.reverse()
    return result_values, result_indices

result_values, result_indices = find_k_largest_with_indices(cosines, args.K)
print("Found %0.2f closest values in %0.2f seconds." % (args.K, time.time() - t0))
print("Sorted result_values:", result_values)
print("Sorted result_indices:", result_indices)
print("IDs: ", [config['imap'][index] for index in result_indices])


import requests
import re

title = input("제목 : ")
text = input("내용 : ")

response = requests.post(
    "https://controlc.com/index.php?",
    params={
        "act" : "submit"
    },
    data={
        "subdomain": "",
        "antispam": "1",
        "website": "",
        "paste_title": title,
        "input_text": text,
        "timestamp": "1503c4774cf81847fce13669ff49bc90",
        "paste_password": "",
        "code": "0",
    },
    headers={
        "referer": "https://controlc.com/",
    }
)

print(response.text)

with open(
    "C:/Users/user/Desktop/2022-09-14/test.html",
    'w',
    encoding='UTF-8'
) as f:
    f.write(response.text)


import requests
from bs4 import BeautifulSoup

def get_data(url, keywords, enterprises):
    articles = []
    resource = 'https://fips.ru'

    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'accept-encoding': 'gzip',
        'Accept-Charset': 'utf-8',
        'content-Type': 'charset=utf-8',
        'cache-control': 'max-age=0',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.160 YaBrowser/22.5.3.684 Yowser/2.5 Safari/537.36'
    }

    r = requests.get(
        url=url, headers=headers)

    soup = BeautifulSoup(r.content, "html.parser").find(
        'div', class_='search-page')

    while soup is not None:
        for article in soup.find_all("a"):
            if article.parent.name == 'small' or\
                article.find('div') or \
                article.get_text().strip() in ['Главная', 'О ФИПС', 'Всероссийская патентно-техническая библиотека',
                                               'Новости', 'Архив новостей', 'Сортировать по дате', 'След.', 'Конец',
                                               'fips@rupto.ru', 'Новости', 'Документы', 'Вакансии', 'Глоссарий',
                                               'Ссылки', 'Ответы на вопросы', 'О сайте', 'Карта сайта', 'Как проехать',
                                               'Сообщить о ошибке', 'Начало', 'Пред.', 'Правила приема заявок',
                                               '1', '2', '3', '4', '5', '6', '7', '8', '9', '10',
                                               '11', '12', '13', '14', '15', '16', '17', '18', '19', '20']:
                continue
            articleHeader = article
            siteUrl = resource + articleHeader['href']

            newsDateAttr = article.find_next('small')

            newsDate = newsDateAttr.get_text().strip().split(' ')[
                1] if newsDateAttr else None

            if newsDate is None:
                continue

            [day, month, year] = newsDate.split('.') if newsDate else [
                'None', 'None', 'None']

            newsDate = '-'.join([year, month, day]) if newsDate else 'None'
            articleObject = {
                'enterprises': enterprises,
                # 'resource': resource,
                'resource': 'Федеральный институт промышленной собственности',
                'news': articleHeader.get_text().strip(),
                'date': newsDate,
                'link': siteUrl,
                'keywords': keywords,
            }

            if len(articleObject['news']) < 15 or articleObject['news'].find(' ') == -1:
                continue

            articles.append(articleObject)
        nextPageTag = soup.find(
            "a", text="След.")

        soup = BeautifulSoup(requests.get(
            url=resource + nextPageTag['href'], headers=headers).content, "html.parser").find(
            'div', class_='search-page') if nextPageTag else None

    return articles

def fipsParser(keywords=[], enterprises=[]):
    filter = '+'.join(keywords)+'+'+'+'.join(enterprises)
    searchUrl = "https://fips.ru/search/?q=%s" % (
        filter)

    articles = get_data(searchUrl, keywords, enterprises)
    return articles


#Helper functions
import os
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime # this actually means import datetime.datetime
import seaborn as sns
import emoji

def create_chat_list(path='./data/nacho_gleb_original.txt'):
    with open(path, encoding='utf8') as f:
        chat = f.read()
        chat = chat.split('\n')[:len(chat.split('\n'))-1]
    return chat

# I should add a function that indentifies european vs american calendar dates
# Also another function that identifies 24h format vs am,pm
def clean_chat_list(chat_list):
    """Unifies messages that are split by \n incorrectly"""

    clean_chat_list = []
    clean_counter = -1

## TRY WITH REGEX: re.split(r'\d\d?/\d\d?/\d\d\s\d\d?:\d\d?\s-\s', chat_test[0])
## re.findall(r'\d\d?/\d\d?/\d\d\s\d\d?:\d\d?', chat_test[i])

    for i in chat_list:
        try:
            if (datetime.strptime(i.split(' - ')[0], '%d/%m/%y %H:%M') and \
                i.split(' - ')[1].split(':')[1]):
                    clean_chat_list.append(i)
                    clean_counter += 1
            else:
                clean_chat_list[clean_counter] = \
                clean_chat_list[clean_counter] + ' ' + i
        except:
            clean_chat_list[clean_counter] = \
            clean_chat_list[clean_counter] + ' ' + i

    return clean_chat_list

def chat_to_df(chat):
    """Creates a df with datetime index and additional time columns"""

    times = []
    user = []
    message = []

    try:
        for i in chat:
            times.append(i.split(' - ')[0])
            user.append(i.split(' - ')[1].split(':',1)[0])
            message.append(i.split(' - ')[1].split(':',1)[1])
    except:
        pass

    assert len(times) == len(user) == len(message)

    #times = pd.to_datetime(times)
    times = pd.to_datetime(times)

    dataframe_chat = {
        'id': range(len(times)),
        'date': times,
        'user': user,
        'message': message
    }

    df = pd.DataFrame(dataframe_chat).set_index('date')

    df['year'] = df.index.year
    df['month'] = df.index.month
    df['week'] = df.index.week
    df['day'] = df.index.day
    df['dayweek'] = df.index.dayofweek
    df['hour'] = df.index.hour
    df['minute'] = df.index.minute
    df['emoji_count'] = df.loc[:,'message'].apply(lambda x: emoji_counter(x))
    df['media'] = df.message.apply(lambda x: '<Multimedia omitido>' in x or \
                                   '<Media omitted>' in x).astype(int)
    df['words_count'] = df.message.apply(lambda x: len(x.split(' ')))

    return df

def extract_emojis(s):
    """Extracts emojis from a string"""
    return ''.join(c for c in s if c in emoji.UNICODE_EMOJI)

def emoji_counter(msg):
    if len(emoji.emoji_lis(msg)) != 0:
        return len(extract_emojis(msg))
    else:
        return 0

"""
Code to finish, get the most used emojis:

def emoji_taker(msg):
    if len(emoji.emoji_lis(msg)) != 0:
        return extract_emojis(msg)
    else:
        return 0

def emoji_counter(msg):
    if len(emoji.emoji_lis(msg)) != 0:
        return len(extract_emojis(msg))
    else:
        return 0

df['emoji_count'] = df.loc[:,'message'].apply(lambda x: emoji_counter(x))
df['emoji'] = df.loc[:,'message'].apply(lambda x: emoji_taker(x))

df[df.emoji_count != 0].loc[:, ['emoji']]
"""

def longest_word(msg):

    word = ''

    for i in msg:
        for j in i.split(' '):
            try:
                if len(j) > len(word) and j.isalpha():
                    word = j
            except:
                continue
    return word

# Creating Statistics for first page

def number_msgs(df):
    """ Gives table with the information of the total number of messages and words by user"""

    # Counting total messages

    df1 = pd.DataFrame(
        df.groupby('user').count().iloc[:,0:1],
    )

    total_messages = df1.iloc[:,0].sum()

    df1['% Total Messages'] = round(df1.iloc[:,0]/total_messages*100, 2)

    df1.columns = ['Total Messages', '% Total Messages']

    # Counting total words

    df2 = pd.DataFrame(df.groupby('user').words_count.sum())

    total_words = df2.sum()[0]

    df2['% Total words'] = round(df2.iloc[:,0]/total_words*100, 2)

    df2.columns = ['Total Words', '% Total words']

    return pd.concat([df1, df2], axis=1)

def longest_word_user(df):

    users = df.user.unique()

    df = pd.DataFrame(
     {user : longest_word(df[df.user == user].message) for user in users}.values(),
     df.groupby('user').size().index
    )

    df.columns = ['Longest word']

    return df

def most_words(df):

    users = df.user.unique()

    most_words = {
        user : str(df[df.user == user].resample('d').sum().sort_values('words_count').words_count.\
        iloc[-1]) + ' (' + str(df[df.user == user].resample('d').sum().\
        sort_values('words_count').words_count.index[-1])[:10] + ')' for user in users
    }

    most_messages = {
        user : str(df[df.user == user].resample('d').count().sort_values('message').message.\
        iloc[-1]) + ' (' + str(df[df.user == user].resample('d').count().\
        sort_values('message').index[-1])[:10] + ')' for user in users
    }

    #Creating the dataframes before concatenating
    df1 = pd.DataFrame(most_words.values(), index=users)
    df1.columns = ['Most words']
    df2 = pd.DataFrame(most_messages.values(), index=users)
    df2.columns = ['Most messages']

    df = pd.concat([df1,df2], axis=1)

    return df

def emojis_used(df):
    users = df.user.unique()

    emojis_user = {
         user: df[df.user == user].sum().emoji_count for user in users
    }

    df = pd.DataFrame(emojis_user.values(), index=users)
    df.columns = ['Total emojis used']

    return df

def statistics_users(df):

    return pd.concat(
                        [number_msgs(df),
                         longest_word_user(df),
                         most_words(df),
                         emojis_used(df)],
                         axis=1).transpose()


import requests
from bs4 import BeautifulSoup

def scrape_flipkart_mobile(url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
    response = requests.get(url, headers=headers)

    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')

        # Extract name and brand
        name = soup.find('span', {'class': '_35KyD6'}).get_text(strip=True)
        brand = soup.find('span', {'class': 'B_NuCI'}).get_text(strip=True)

        # Extracting specifications
        specifications = {}
        specs_rows = soup.find_all('tr', {'class': '_1s_Smc row'})
        for row in specs_rows:
            key = row.find('td', {'class': '_1hKmbr col col-3-12'}).get_text(strip=True)
            value = row.find('td', {'class': '_1hKmbr col col-9-12'}).get_text(strip=True)
            specifications[key] = value

        # Print the results
        print(f"Name: {name}")
        print(f"Brand: {brand}")
        print("Specifications:")
        for key, value in specifications.items():
            print(f"{key}: {value}")
    else:
        print(f"Failed to retrieve the page. Status Code: {response.status_code}")

# Example URL of a mobile phone on Flipkart
url = 'https://www.flipkart.com/samsung-galaxy-s21-fe-5g-snapdragon-888-navy-128-gb/p/itmcb8fc8eb2e82b?pid=MOBGTKQG8T9ZHJMM&lid=LSTMOBGTKQG8T9ZHJMMA4D1AC&marketplace=FLIPKART&fm=neo%2Fmerchandising&iid=M_e5c00873-7089-49cb-880c-7f149a3162e8_1_1BUWY8OBA8L9_MC.MOBGTKQG8T9ZHJMM&ppt=browse&ppn=browse&otracker=clp_pmu_v2_Latest%2BSamsung%2Bmobiles%2B_3_1.productCard.PMU_V2_Samsung%2BGalaxy%2BS21%2BFE%2B5G%2Bwith%2BSnapdragon%2B888%2B%2528Navy%252C%2B128%2BGB%2529_samsung-mobile-store_MOBGTKQG8T9ZHJMM_neo%2Fmerchandising_2&otracker1=clp_pmu_v2_PINNED_neo%2Fmerchandising_Latest%2BSamsung%2Bmobiles%2B_LIST_productCard_cc_3_NA_view-all&cid=MOBGTKQG8T9ZHJMM'
scrape_flipkart_mobile(url)


import time
from picraftzero.log import logger
from pprint import pformat
import subprocess
import threading

from picraftzero.config import get_config

# see: https://pymotw.com/2/threading/

class RemoteCommands:

    DEFAULT_COMMAND_TIMEOUT_SECONDS = 30

    def __init__(self):
        self.user = 'pi'
        self.timeout = RemoteCommands.DEFAULT_COMMAND_TIMEOUT_SECONDS

    def run_remote_command(self, host, command, result):
        result[host]['COMMAND'] = command
        logger.debug("run_remote_command: {}".format([self.user, host, command]))
        prog = subprocess.Popen(["/usr/bin/ssh", "{}@{}".format(self.user, host), command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
        out = prog.communicate()
        result[host]['STATUS'] = 'COMPLETED'

        result[host]['RC'] = prog.returncode
        result[host]['STDOUT'] = out[0]
        result[host]['STDERR'] = out[1]

        #logger.error("stderr; {}".format(errdata))

    def run_remote_commands(self, hosts, command):
        results = {}
        threads = []

        for host in hosts:
            results[host] = {}
            results[host]['STATUS'] = 'STARTING'

            logger.debug("Host: {} Cmd: {}".format(host, command))
            t = threading.Thread(target=self.run_remote_command, args=(host, command, results), name=host)
            threads.append(t)
            t.start()

        for t in threads:
            logger.info('Waiting for %s', t.getName())
            t.join(self.timeout)
            logger.info('Final status for {} = {}'.format(t.getName(), t.isAlive()))

        logger.info("Results: {}".format(pformat(results)))
        for host in hosts:
            if host in results:
                if results[host]['STATUS'] != 'COMPLETED':
                    results[host]['STATUS'] = 'TIMEOUT'

                elif results[host]['RC'] == 0:
                    results[host]['STATUS'] = "OK"
                else:
                    results[host]['STATUS'] = "ERROR"
            else:
                logger.error("No results found for {}".format(host))

        return results

    def start_service(self, hosts, service):
        self.run_remote_commands(hosts, "sudo systemctl start {}".format(service))

    def restart_service(self, hosts, service):
        self.run_remote_commands(hosts, "sudo systemctl restart {}".format(service))

    def stop_service(self, hosts, service):
        self.run_remote_commands(hosts, "sudo systemctl stop {}".format(service))

    def disable_service(self, hosts, service):
        self.run_remote_commands(hosts, "sudo systemctl disable {}".format(service))

    def service_status(self, hosts, service):
        self.run_remote_commands(hosts, "sudo systemctl status {}".format(service))

    def reboot(self, hosts):
        self.run_remote_commands(hosts, "sudo reboot")

    def poweroff(self, hosts):
        self.run_remote_commands(hosts, "sudo poweroff")

    def _run_test(self, hosts, cmd):
        results = self.run_remote_commands(hosts, cmd)
        logger.info(pformat(results))

if __name__ == "TEST__main__":

    cmd = RemoteCommands()
    cmd._run_test(['cam0.local'], "sleep {}".format(cmd.timeout+10))
    cmd._run_test(['cam0.local'], "no_such_command")
    cmd._run_test(['no_such_host'], "date")
    cmd._run_test(['cam0.local'], "date")

# ----------------------------------------------------------------------------------------------

class PiCraftServices:

    def __init__(self):
        self.remote_cmds = RemoteCommands()
        self.config = get_config()
        self.camera_services = self.config['services_topology']['services']['camera']
        self.www_services = self.config['services_topology']['services']['www']
        self.topology = self.config['services_topology']['hosts_services_mapping']

    def all_hosts(self):
        return list(self.topology.keys())

    def all_services(self):
        all_services = []
        for host, services in self.topology.items():
            all_services += services
        return set(all_services)

    def hosts_for_service(self, service):
        hosts = []
        for host, services in self.topology.items():
            if service in services:
                hosts.append(host)
        logger.info("Service {} is installed on : {}".format(service, hosts))
        return hosts

    def camera_hosts(self):
        hosts = []
        for host, services in self.topology.items():
            #print(host, services)
            if set(self.camera_services).intersection(set(services)):
                hosts.append(host)
        return hosts

    def www_hosts(self):
        hosts = []
        for host, services in self.topology.items():
            if set(self.www_services).intersection(set(services)):
                hosts.append(host)
        return hosts

    # ---

    def service_status(self):
        for service in self.all_services():
            hosts = self.hosts_for_service(service)
            logger.info("Getting status for service {} on hosts: {}".format(service, hosts))
            self.remote_cmds.service_status(hosts, service)

    def www_services_status(self):
        for service in self.www_services:
            hosts = self.hosts_for_service(service)
            logger.info("Getting status for service {} on hosts: {}".format(service, hosts))
            self.remote_cmds.service_status(hosts, service)

    def camera_services_status(self):
        for service in self.camera_services:
            hosts = self.hosts_for_service(service)
            logger.info("Getting status for service {} on hosts: {}".format(service, hosts))
            self.remote_cmds.service_status(hosts, service)

    def restart_camera_services(self):
        for service in self.camera_services:
            self.remote_cmds.restart_service(self.camera_hosts(), service)

    def restart_www_services(self):
        for service in self.www_services:
            self.remote_cmds.restart_service(self.www_hosts(), service)

    def stop_www_services(self):
        for service in self.www_services:
            self.remote_cmds.stop_service(self.www_hosts(), service)

    def reboot_all(self):
        self.remote_cmds.reboot(self.all_hosts())

    def poweroff_all(self):
        self.remote_cmds.poweroff(self.all_hosts())
        #TODO: expected t fail!
""" 'cam2.local': {'COMMAND': 'sudo poweroff',
                'RC': 255,
                'STATUS': 'COMPLETED',
                'STDERR': 'Connection to cam2.local closed by remote host.\r\n',
                'STDOUT': ''}}"""

if __name__ == "__main__":
    # see: https://mkaz.tech/code/python-argparse-cookbook/
    import argparse

    pcs = PiCraftServices()

    logger.info("All hosts                 : {}".format(pcs.all_hosts()))
    logger.info("All services              : {}".format(pcs.all_services()))
    logger.info("Hosts with camera services: {}".format(pcs.camera_hosts()))
    logger.info("Hosts with www services   : {}".format(pcs.www_hosts()))

    commands = dict(
        service_status=pcs.service_status,
        www_service_restart=pcs.restart_www_services,
        www_service_status=pcs.www_services_status,
        www_service_stop=pcs.stop_www_services,
        camera_service_restart=pcs.restart_camera_services,
        camera_service_status=pcs.camera_services_status,
        reboot_all=pcs.reboot_all,
        poweroff_all=pcs.poweroff_all,
    )

    parser = argparse.ArgumentParser(description='Run admin scripts')

    parser.add_argument('-v', '--verbose', action="store_true", help="verbose output")
    parser.add_argument('-d', '--dryrun', action="store_true", help="dry run, don't execute remote commands")
    parser.add_argument('-l', '--logfile', type=argparse.FileType('w'), help="logfile to be created")
    parser.add_argument('-c', '--command', choices=list(commands.keys()), help="command")

    args = parser.parse_args()

    if args.command:
        commands[args.command]()


import numpy as np
from numpy import random
import pandas as pd
import matplotlib.pyplot as plt
import copy
import datetime
import torch
from engine import sample_z, data_to_z_params

class Game():

    def __init__(self, cfg, dcrnn, action_space, scenario_dict, dataset_idx, is_online = False):
        #super(Game, self).__init__()
        self.SAVE_PATH = cfg.DIR.output_dir
        self.device = torch.device(cfg.TRAIN.device)
        self.action_space = action_space
        self.scenario_dict = scenario_dict
        self.reward_penalty = cfg.TRAIN.reward_penalty
        self.MAX_ACTIONS  = cfg.TRAIN.max_actions
        self.cfg = cfg
        self.dcrnn = dcrnn

        self.num_agents = 1
        self._agent_ids = list(range(self.num_agents))
        self.actions_made = 0

        self.episode_count = 0
        self.gt_reward_arr = None if "gt_rewards" not in scenario_dict.keys() else scenario_dict["gt_rewards"]
        self.is_online = is_online
        self.dataset_idx = dataset_idx
        self.context_pts = self.scenario_dict["context_pts"][self.dataset_idx:self.dataset_idx+1]
        self.target_pts = self.scenario_dict["target_pts"][self.dataset_idx:self.dataset_idx+1]
        self.latent_variable = self.scenario_dict["latent_variable"][self.dataset_idx:self.dataset_idx+1]
        self.valid_pred = self.scenario_dict["valid_pred"][self.dataset_idx:self.dataset_idx+1]
        self.test_pred = self.scenario_dict["test_pred"][self.dataset_idx:self.dataset_idx+1]

    def step(self, action_idx):
        idx = self.dataset_idx
        if(self.gt_reward_arr != None): #load from existing data
            self.gt_reward_arr = self.scenario_dict["gt_rewards"][idx:idx+1].reshape(-1,1)
            self.gt_reward_arr = (self.gt_reward_arr - torch.mean(self.gt_reward_arr))/torch.std(self.gt_reward_arr)
            sorted_idx = np.argsort(self.gt_reward_arr.flatten().numpy().copy())
            self.top_twenty_idx = sorted_idx[-20:]
#             self.reward_rank_arr = torch.ones(self.gt_reward_arr.shape)*-1
#             self.reward_rank_arr[self.top_twenty_idx] = 1
            self.reward_dict = {sorted_idx[i]:(270-i) for i in range(len(sorted_idx))} #store the reward rank given an action_idx

        else: #online training
            x_c, y_c = torch.split(self.context_pts, [2, 100], dim = 2)
            x_t, y_t, y_t_pred = torch.split(self.target_pts, [2, 100, 100], dim = 2) #n_iter x n_pts x pt_dim
            x_train = torch.cat([x_c, x_t], dim = 1)[0]
            y_train = torch.cat([y_c, y_t], dim = 1)[0]
            self.gt_reward_arr = self.calculate_score(self.cfg, self.dcrnn, x_train, y_train, self.action_space)
        if(self.actions_made == 0):
            print("reward stats: mean = {}, median = {}, std = {}, max = {}, min = {}".format(
                np.round(torch.mean(self.gt_reward_arr),3),
                np.round(torch.median(self.gt_reward_arr),3),
                np.round(torch.std(self.gt_reward_arr),3),
                np.round(torch.max(self.gt_reward_arr),3),
                np.round(torch.min(self.gt_reward_arr),3))
            )

        selected_param = self.action_space[action_idx]
        observation = self.dict_to_state(self.context_pts, self.target_pts, self.latent_variable) #convert from dictionary to vector(RL state has to be a flattened vector)
        reward = self.gt_reward_arr[action_idx]*self.reward_penalty**(self.actions_made)
#         reward = torch.tensor([(270-self.reward_dict[action_idx])*self.reward_penalty**(self.actions_made)])
        #reward = self.gt_reward_arr[action_idx]
        self.actions_made +=1
        done = self.actions_made >= self.MAX_ACTIONS
#         if(self.actions_made % 10 ==0):
#             print('data_num = {}, actions_made = {}, reward = {}'.format(self.dataset_idx, self.actions_made, np.round(reward.item(), 3)))
        info = {"selected_param": selected_param, "gt_reward_arr": self.gt_reward_arr.flatten(), "reward_dict":self.reward_dict}

        return observation, reward, done, info

    def get_reward(self, action_idx):
        return self.gt_rewards[action_idx]

    def calculate_score(self, cfg, dcrnn, x_train, y_train, beta_epsilon_all):
        # query z_mu, z_var of the current training data
        with torch.no_grad():
            z_mu, z_logvar = data_to_z_params(dcrnn, x_train.to(self.device),y_train.to(self.device))
            score_list = []
            for i in range(len(beta_epsilon_all)):
                # generate x_search
                x1 = beta_epsilon_all[i:i+1]
                x_search = np.repeat(x1,cfg.SIMULATOR.num_simulations,axis =0)
                x_search = torch.from_numpy(x_search).float()

                # generate y_search based on z_mu, z_var of current training data
                output_list = []
                for j in range (len(x_search)):
                    zsamples = sample_z(z_mu, z_logvar,cfg.MODEL.z_dim)
                    output = dcrnn.decoder(x_search[j:j+1].to(self.device), zsamples).cpu()
                    output_list.append(output.detach().numpy())

                y_search = np.concatenate(output_list)
                y_search = torch.from_numpy(y_search).float()

                x_search_all = torch.cat([x_train,x_search],dim=0)
                y_search_all = torch.cat([y_train,y_search],dim=0)

                # generate z_mu_search, z_var_search
                z_mu_search, z_logvar_search = data_to_z_params(dcrnn, x_search_all.to(self.device),y_search_all.to(self.device))

                # calculate and save kld
                mu_q, var_q, mu_p, var_p = z_mu_search,  0.1+ 0.9*torch.sigmoid(z_logvar_search), z_mu, 0.1+ 0.9*torch.sigmoid(z_logvar)

                std_q = torch.sqrt(var_q)
                std_p = torch.sqrt(var_p)

                p = torch.distributions.Normal(mu_p, std_p)
                q = torch.distributions.Normal(mu_q, std_q)
                score = torch.distributions.kl_divergence(p, q).sum()

                score_list.append(score.item())
            score_array = np.array(score_list)
        return score_array

    def reset(self):
        observation = self.dict_to_state(self.context_pts, self.target_pts, self.latent_variable).to(self.device)
        self.actions_made = 0
        return observation

    def dict_to_state(self, context_pts, target_pts, latent_variable):
        ct_shape = torch.tensor(context_pts.shape[-2:])
        tgt_shape = torch.tensor(target_pts.shape[-2:])
        context_pts = context_pts.flatten()
        context_pts = (context_pts-torch.mean(context_pts))/(torch.std(context_pts))
        target_pts = target_pts.flatten()
        target_pts = (target_pts-torch.mean(target_pts))/(torch.std(target_pts))
        latent_variable = latent_variable.flatten()
        latent_variable = (latent_variable-torch.mean(latent_variable))/(torch.std(latent_variable))

        observation = torch.cat([torch.tensor([self.actions_made]), context_pts,target_pts , latent_variable, ct_shape, tgt_shape], dim = 0).to(self.device)
        return observation

    def update_data_idx(self, new_idx):
        self.dataset_idx = new_idx
        self.context_pts = self.scenario_dict["context_pts"][self.dataset_idx:self.dataset_idx+1]
        self.target_pts = self.scenario_dict["target_pts"][self.dataset_idx:self.dataset_idx+1]
        self.latent_variable = self.scenario_dict["latent_variable"][self.dataset_idx:self.dataset_idx+1]
        self.valid_pred = self.scenario_dict["valid_pred"][self.dataset_idx:self.dataset_idx+1]
        self.test_pred = self.scenario_dict["test_pred"][self.dataset_idx:self.dataset_idx+1]
        self.gt_reward_arr = self.scenario_dict["gt_rewards"][self.dataset_idx:self.dataset_idx+1].reshape(-1,1)
        print('IDX = {}, VAR SHAPE = {}, {}, {}'.format(self.dataset_idx, self.context_pts.shape, self.target_pts.shape, self.latent_variable.shape))

    def render(self, mode):
        pass

    def close(self):
        pass

    def seed(self):
        pass


from typing import Any, Dict, Optional, Union, cast

import httpx

from ...client import Client
from ...models.list_all_geofences_response_200 import ListAllGeofencesResponse200
from ...types import UNSET, Response, Unset

def _get_kwargs(
    id: str,
    *,
    client: Client,
    limit: Union[Unset, None, int] = UNSET,
    offset: Union[Unset, None, int] = UNSET,
) -> Dict[str, Any]:
    url = "{}/assets/{id}/geofences".format(client.base_url, id=id)

    headers: Dict[str, str] = client.get_headers()
    cookies: Dict[str, Any] = client.get_cookies()

    params: Dict[str, Any] = {}
    params["limit"] = limit

    params["offset"] = offset

    params = {k: v for k, v in params.items() if v is not UNSET and v is not None}

    return {
        "method": "get",
        "url": url,
        "headers": headers,
        "cookies": cookies,
        "timeout": client.get_timeout(),
        "params": params,
    }

def _parse_response(*, response: httpx.Response) -> Optional[Union[Any, ListAllGeofencesResponse200]]:
    if response.status_code == 200:
        response_200 = ListAllGeofencesResponse200.from_dict(response.json())

        return response_200
    if response.status_code == 401:
        response_401 = cast(Any, None)
        return response_401
    return None

def _build_response(*, response: httpx.Response) -> Response[Union[Any, ListAllGeofencesResponse200]]:
    return Response(
        status_code=response.status_code,
        content=response.content,
        headers=response.headers,
        parsed=_parse_response(response=response),
    )

def sync_detailed(
    id: str,
    *,
    client: Client,
    limit: Union[Unset, None, int] = UNSET,
    offset: Union[Unset, None, int] = UNSET,
) -> Response[Union[Any, ListAllGeofencesResponse200]]:
    """List All Geofences

    Args:
        id (str): Asset ID Example: 272956057382HD3JBSD24.
        limit (Union[Unset, None, int]):
        offset (Union[Unset, None, int]):

    Returns:
        Response[Union[Any, ListAllGeofencesResponse200]]
    """

    kwargs = _get_kwargs(
        id=id,
        client=client,
        limit=limit,
        offset=offset,
    )

    response = httpx.request(
        verify=client.verify_ssl,
        **kwargs,
    )

    return _build_response(response=response)

def sync(
    id: str,
    *,
    client: Client,
    limit: Union[Unset, None, int] = UNSET,
    offset: Union[Unset, None, int] = UNSET,
) -> Optional[Union[Any, ListAllGeofencesResponse200]]:
    """List All Geofences

    Args:
        id (str): Asset ID Example: 272956057382HD3JBSD24.
        limit (Union[Unset, None, int]):
        offset (Union[Unset, None, int]):

    Returns:
        Response[Union[Any, ListAllGeofencesResponse200]]
    """

    return sync_detailed(
        id=id,
        client=client,
        limit=limit,
        offset=offset,
    ).parsed

async def asyncio_detailed(
    id: str,
    *,
    client: Client,
    limit: Union[Unset, None, int] = UNSET,
    offset: Union[Unset, None, int] = UNSET,
) -> Response[Union[Any, ListAllGeofencesResponse200]]:
    """List All Geofences

    Args:
        id (str): Asset ID Example: 272956057382HD3JBSD24.
        limit (Union[Unset, None, int]):
        offset (Union[Unset, None, int]):

    Returns:
        Response[Union[Any, ListAllGeofencesResponse200]]
    """

    kwargs = _get_kwargs(
        id=id,
        client=client,
        limit=limit,
        offset=offset,
    )

    async with httpx.AsyncClient(verify=client.verify_ssl) as _client:
        response = await _client.request(**kwargs)

    return _build_response(response=response)

async def asyncio(
    id: str,
    *,
    client: Client,
    limit: Union[Unset, None, int] = UNSET,
    offset: Union[Unset, None, int] = UNSET,
) -> Optional[Union[Any, ListAllGeofencesResponse200]]:
    """List All Geofences

    Args:
        id (str): Asset ID Example: 272956057382HD3JBSD24.
        limit (Union[Unset, None, int]):
        offset (Union[Unset, None, int]):

    Returns:
        Response[Union[Any, ListAllGeofencesResponse200]]
    """

    return (
        await asyncio_detailed(
            id=id,
            client=client,
            limit=limit,
            offset=offset,
        )
    ).parsed


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, GdkPixbuf
import os

import w

class IntroductionBox(Gtk.ScrolledWindow):
		def __init__(self):
				Gtk.ScrolledWindow.__init__(self)

				box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)

				path = str(os.path.join(os.path.dirname(os.path.abspath(__file__))))
				pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(path[:-5] + 'install/icon.png', width=200, height=200,
																										 preserve_aspect_ratio=False)
				image = Gtk.Image()
				image.set_from_pixbuf(pixbuf)
				box.pack_start(image, False, False, 0)

				label = Gtk.Label("The Linux Watchful Adaptive Security Profiler is designed to create vulnerabilities on a Linux image and then score cyber security students on finding and fixing those vulnerabilites.  LWASP is designed to work on an unconfigured image, but advanced users can pre-configure their images and allow scoring for more difficult vulnerabilites.")
				label.set_line_wrap(True)
				box.pack_start(label, False, False, 10)

				label2 = Gtk.Label("Explore the tabs at the top to set up scoring on this image, and then press the button at the bottom right to go to the next step.")
				label2.set_line_wrap(True)
				box.pack_start(label2, False, False, 10)

				self.add_with_viewport(box)


import sys
# from PCM_code import *
import queue
from utils import int_to_bit as PCMcode
from utils import bit_to_int as PCMdecode
from utils import AMIcode, AMIdecode, HDB3code, HDB3decode, ASKcode, ASKdecode
from utils import add_noise

import sounddevice as sd

# 配置输入设备参数
input_device = sd.default.device  # 默认输入设备
input_channels = 1  # 输入通道数
input_samplerate = 8000  # 输入采样率

# 配置输出设备参数
output_device = sd.default.device  # 默认输出设备
output_channels = 1  # 输出通道数
output_samplerate = 8000  # 输出采样率

q_cache = queue.Queue()

# 输入回调函数

def input_callback(indata, frames, time, status):
    # x = indata
    x = PCMcode(indata)
    x = HDB3code(x)
    x, plt_t = ASKcode(x)
    x = add_noise(x, 10)
    x = ASKdecode(x)
    x = HDB3decode(x)
    x = PCMdecode(x)
    q_cache.put(x.reshape(x.shape[0], 1))

# 输出回调函数
def output_callback(outdata, frames, time, status):
    outdata[:] = q_cache.get()
    pass

# 打开输入流
input_stream = sd.InputStream(device=input_device, channels=input_channels,
                              samplerate=input_samplerate, callback=input_callback, dtype='int16')
input_stream.start()

# 打开输出流
output_stream = sd.OutputStream(device=output_device, channels=output_channels,
                                samplerate=output_samplerate, callback=output_callback, dtype='int16')
output_stream.start()

# 保持程序运行
while True:

    try:
        sd.sleep(500)
    except KeyboardInterrupt:
        input_stream.stop()
        output_stream.stop()
        # 关闭输入和输出流，释放资源
        input_stream.close()
        output_stream.close()
        raise KeyboardInterrupt

# 关闭流