import gurobipy as gp
from gurobipy import GRB

import numpy as np

import scipy.optimize

from sklearn.linear_model import OrthogonalMatchingPursuit, orthogonal_mp
from time import time

class GurobiInFeasibleException(Exception):
    pass

def restore_cs1_signal(non_zero_features, sdm_signal, transformation, restoration_type="omp",
                       error_handler=print, length=600, **kwargs) -> np.ndarray:
    try:
        if isinstance(non_zero_features, tuple):
            len_non_zero_features = len(non_zero_features[0])
            if len_non_zero_features == 0:
                raise ValueError("No features in array")
        elif isinstance(non_zero_features, int):
            len_non_zero_features = non_zero_features

        set_sdm_signal = set(sdm_signal)
        if set_sdm_signal == {0}:
            cs1_signal = np.zeros((transformation.shape[1],))
        else:
            if restoration_type == "omp":
                matrix = transformation.copy().astype(np.float64)
                arr = np.array(sdm_signal.copy())

                omp = OrthogonalMatchingPursuit(n_nonzero_coefs=len_non_zero_features)
                omp.fit(matrix, arr)

                inds = omp.coef_.argsort()[-len_non_zero_features:][::-1]
                cs1_signal = np.zeros((transformation.shape[1],))
                cs1_signal[inds] = 1
            elif restoration_type == "linprog":
                matrix = np.vstack([transformation.astype(int), np.ones((length,)).astype(int)])
                # arr = np.rint(np.append(sdm_signal, len_non_zero_features)).astype(int)
                arr = np.append(sdm_signal, len_non_zero_features)
                method = kwargs.get("method") or "interior-point"
                solution = scipy.optimize.linprog(c=np.ones((matrix.shape[1],)), A_eq=matrix, b_eq=arr,
                                                  method=method)
                solution_x = solution.x
                if solution_x is None:
                    solution = scipy.optimize.linprog(c=np.ones((matrix.shape[1],)), A_eq=matrix, b_eq=arr)
                    solution_x = solution.x
                inds = np.abs(solution_x).argsort()[-len_non_zero_features:][::-1]
                cs1_signal = np.zeros((length,))
                cs1_signal[inds] = 1
            else:
                raise Exception(f"Unknown restoration type: {restoration_type}")

        return cs1_signal.astype(np.int8)
    except Exception as error:
        if callable(error_handler):
            error_handler(error)
        else:
            print(error)

        return np.zeros((transformation.shape[1],)).astype(np.int8)

def restore_cs1_signal_gurobi(non_zero_features, sdm_signal, transformation, error_handler=print) -> np.ndarray:
    try:
        if isinstance(non_zero_features, tuple):
            len_non_zero_features = len(non_zero_features[0])
            if len_non_zero_features == 0:
                raise ValueError("No features in array")
        elif isinstance(non_zero_features, int):
            len_non_zero_features = non_zero_features

        set_sdm_signal = set(sdm_signal)
        if set_sdm_signal == {0}:
            cs1_signal = np.zeros((transformation.shape[1], ), dtype=np.int8)
        else:
            model = gp.Model("mip1")
            model.setParam('OutputFlag', 0)
            model.setParam('MIPFocus', 3)

            vars = []
            for i in range(transformation.shape[1]):
                x = model.addVar(vtype=GRB.BINARY, name=f"x_{i}")
                vars.append(x)

            ys = []
            for i in range(transformation.shape[0]):
                y = model.addVar(vtype=GRB.INTEGER)
                ys.append(y)

            # sums = []
            for i in range(transformation.shape[0]):
                s = sum([transformation[i][j] * vars[j] for j in range(transformation.shape[1])])
                # sums.append(s)
                # model.addConstr(s == sdm_signal[i], f"c_{i}")
                model.addConstr(s - sdm_signal[i] <= ys[i], f"c0_{i}")
                model.addConstr(sdm_signal[i] - s <= ys[i], f"c1_{i}")

            model.addConstr(sum(vars) == len_non_zero_features, f"c_sum")

            model.setObjective(sum(ys), GRB.MINIMIZE)

            # l = [gp.abs_(sums[i] - sdm_signal[i]) for i in range(transformation.shape[0])]
            # model.setObjective(gp.sum_(l[1:], start=l[0]), GRB.MINIMIZE)

            model.optimize()
            if model.status == GRB.OPTIMAL:
                cs1_signal = np.array([x.x for n, x in enumerate(vars)], dtype=np.int8)
            else:
                raise GurobiInFeasibleException()
                # cs1_signal = np.zeros((transformation.shape[1], ), dtype=np.int8)

        return cs1_signal
    except GurobiInFeasibleException:
        raise
    except Exception as error:
        if callable(error_handler):
            error_handler("err", error)
        else:
            print("err", error)

        return np.zeros((transformation.shape[1], ), dtype=np.int8)

def restore_cs1_signal_cosamp(non_zero_features, sdm_signal, transformation, error_handler=print,
                              tol=1e-10, precision=1e-12, max_iter=1000, transformation_transposed=None):
    """
    @Brief:  "CoSaMP: Iterative signal recovery from incomplete and inaccurate
             samples" by Deanna Needell & Joel Tropp

    @Input:  Phi - Sampling matrix
             u   - Noisy sample vector
             s   - Sparsity vector

    @Return: A s-sparse approximation "a" of the target signal
    """
    if isinstance(non_zero_features, tuple):
        len_non_zero_features = len(non_zero_features[0])
        if len_non_zero_features == 0:
            raise ValueError("No features in array")
    elif isinstance(non_zero_features, int):
        len_non_zero_features = non_zero_features
    try:
        max_iter -= 1  # Correct the while loop
        a = np.zeros(transformation.shape[1])
        v = sdm_signal
        iterations = 0
        halt = False

        # speedup
        if transformation_transposed is None:
            transformation_transposed = np.transpose(transformation)
        sdm_signal_norm = np.linalg.norm(sdm_signal)
        while not halt:
            iterations += 1
            # print("Iteration {}\r".format(iter))

            y = abs(np.dot(transformation_transposed, v))
            # Omega = [i for (i, val) in enumerate(y) if val > np.sort(y)[::-1][2*len_non_zero_features] and val > precision]  # equivalent to below
            # Omega = np.argwhere(y >= np.maximum(np.sort(y)[::-1][2*len_non_zero_features], precision))
            Omega = np.flatnonzero(y >= np.maximum(np.sort(y)[::-1][2*len_non_zero_features], precision))
            T = np.union1d(Omega, a.nonzero()[0])
            # T = np.union1d(Omega, T)
            b = np.dot(np.linalg.pinv(transformation[:, T]), sdm_signal)
            igood = (abs(b) > np.sort(abs(b))[::-1][len_non_zero_features]) & (abs(b) > precision)
            T = T[igood]
            a[T] = b[igood]
            v = sdm_signal - np.dot(transformation[:, T], b[igood])

            halt = np.linalg.norm(v)/sdm_signal_norm < tol or iterations > max_iter

        # print(iterations, end=" ")

        inds = np.abs(a).argsort()[-len_non_zero_features:][::-1]
        cs1_signal = np.zeros((transformation.shape[1],), dtype=np.int8)
        cs1_signal[inds] = 1
        return cs1_signal, iterations
    except Exception as error:
        # if callable(error_handler):
        #     error_handler("err", error)
        # else:
        #     print("err", error)
        cs1_signal = np.zeros((transformation.shape[1], ), dtype=np.int8)

        return cs1_signal, -1


from xml.dom import minidom

from django.contrib.sites.models import Site
from django.core.cache import cache
from django.test import RequestFactory, TestCase, TransactionTestCase

class FeedTestCase(TestCase):
    """
    Borrowing some handy methods from
    https://github.com/django/django/blob/master/tests/syndication_tests/tests.py
    """

    def setUp(self):
        super().setUp()
        # For some reason we need to set this or else, when we run
        # *all* the tests then the domain name is set to the same
        # as the Site object for the dev site in Docker. But not when
        # we only run the feed tests specifically. Weird.
        site = Site.objects.first()
        site.domain = "example.com"
        site.save()
        cache.clear()

    def get_feed_element(self, url):
        response = self.client.get(url)
        doc = minidom.parseString(response.content)

        feed_elem = doc.getElementsByTagName("rss")
        self.assertEqual(len(feed_elem), 1)
        feed = feed_elem[0]

        return feed

    def get_channel_element(self, url):
        """Handy method that returns the 'channel' tag from a feed at url.
        You can then get the items like:

            chan = self.get_channel_element('/blah/')
            items = chan.getElementsByTagName('item')
        """
        feed = self.get_feed_element(url)

        chan_elem = feed.getElementsByTagName("channel")
        self.assertEqual(len(chan_elem), 1)
        chan = chan_elem[0]

        return chan

    def assertChildNodes(self, elem, expected):
        actual = {n.nodeName for n in elem.childNodes}
        expected = set(expected)
        self.assertEqual(actual, expected)

    def assertChildNodeContent(self, elem, expected):
        for k, v in expected.items():
            # It appears that minidom will make a node with no text
            # None, rather than an element. So when we try to get its
            # wholeText, we get an AttributeError.
            # So here we try to work around that...
            first_child = elem.getElementsByTagName(k)[0].firstChild
            try:
                test_value = first_child.wholeText
            except AttributeError:
                test_value = ""

            try:
                self.assertEqual(test_value, v)
            except IndexError as e:
                raise IndexError("{} for '{}' and '{}'".format(e, k, v))

    def assertCategories(self, elem, expected):
        self.assertEqual(
            {
                i.firstChild.wholeText
                for i in elem.childNodes
                if i.nodeName == "category"
            },
            set(expected),
        )

class ViewTestCase(TestCase):
    """
    Parent class to use with all the other view test cases.
    """

    def setUp(self):
        self.factory = RequestFactory()
        # We use '/fake-path/' for all tests because not testing URLs here,
        # and the views don't care what the URL is.
        self.request = self.factory.get("/fake-path/")

class ViewTransactionTestCase(TransactionTestCase):
    """
    Same as ViewTestCase but with a different parent.

    Need to use TransactionTestCase for the SearchView tests because setting the
    search_vectors on objects requires transaction.on_commit() to work, which it
    doesn't with the standard TestCase.
    """

    def setUp(self):
        self.factory = RequestFactory()
        self.request = self.factory.get("/fake-path/")


import os
import sys

from qiskit import QuantumCircuit, QuantumRegister, ClassicalRegister

current_dir = os.path.dirname(os.path.realpath(__file__))
parent_dir = os.path.dirname(current_dir)
sys.path.append(parent_dir)

from methods import test_locally, add_measure_in_base

def get_n2_calibration_circuits():
    qr2 = QuantumRegister(2)
    cr2 = ClassicalRegister(2)

    calibration_base = 'ZZ'

    # 00 State
    c1 = QuantumCircuit(qr2, cr2)
    c1.name = "Calibration_00"

    # 01 B State
    c3 = QuantumCircuit(qr2, cr2)
    c3.x(qr2[0])
    c3.barrier()
    c3.name = "Calibration_01"

    # 10 B State
    c4 = QuantumCircuit(qr2, cr2)
    c4.x(qr2[1])
    c4.barrier()
    c4.name = "Calibration_10"

    # 11 B State
    c2 = QuantumCircuit(qr2, cr2)
    c2.x(qr2[0])
    c2.x(qr2[1])
    c2.barrier()
    c2.name = "Calibration_11"

    circuits_n2 = [c1, c3, c4, c2]
    calibration_circuits = []

    for c in circuits_n2:
        calibration_circuits.append(add_measure_in_base(c.copy(), calibration_base))

    return calibration_circuits

def get_n3_calibration_circuits():
    qr3 = QuantumRegister(3)
    cr3 = ClassicalRegister(3)

    calibration_base = 'ZZZ'

    c0 = QuantumCircuit(qr3, cr3)
    c0.name = "Calibration_000"

    c1 = QuantumCircuit(qr3, cr3)
    c1.x(qr3[0])
    c1.barrier()
    c1.name = "Calibration_001"

    c2 = QuantumCircuit(qr3, cr3)
    c2.x(qr3[1])
    c2.barrier()
    c2.name = "Calibration_010"

    c3 = QuantumCircuit(qr3, cr3)
    c3.x(qr3[0])
    c3.x(qr3[1])
    c3.barrier()
    c3.name = "Calibration_011"

    c4 = QuantumCircuit(qr3, cr3)
    c4.x(qr3[2])
    c4.barrier()
    c4.name = "Calibration_100"

    c5 = QuantumCircuit(qr3, cr3)
    c5.x(qr3[0])
    c5.x(qr3[2])
    c5.barrier()
    c5.name = "Calibration_101"

    c6 = QuantumCircuit(qr3, cr3)
    c6.x(qr3[1])
    c6.x(qr3[2])
    c6.barrier()
    c6.name = "Calibration_110"

    c7 = QuantumCircuit(qr3, cr3)
    c7.x(qr3[0])
    c7.x(qr3[1])
    c7.x(qr3[2])
    c7.barrier()
    c7.name = "Calibration_111"

    circuits_n3 = [c0, c1, c2, c3, c4, c5, c6, c7]

    calibration_circuits = []

    for c in circuits_n3:
        calibration_circuits.append(add_measure_in_base(c.copy(), calibration_base))

    return calibration_circuits

def get_sc_n2_circuits():
    qr2 = QuantumRegister(2)
    cr2 = ClassicalRegister(2)

    bases_n2 = ['XX', 'YY', 'YX', 'XY']

    # 00 State
    c1 = QuantumCircuit(qr2, cr2)
    c1.name = "SC_00"

    # 11 B State
    c2 = QuantumCircuit(qr2, cr2)
    c2.x(qr2[0])
    c2.x(qr2[1])
    c2.barrier()
    c2.name = "SC_11_B"

    circuits_n2 = [c1, c2]

    sc_n2_circuits = []

    for c in circuits_n2:
        for b in bases_n2:
            sc_n2_circuits.append(add_measure_in_base(c.copy(), b))

    return sc_n2_circuits

def get_sc_n3_circuits():
    qr3 = QuantumRegister(3)
    cr3 = ClassicalRegister(3)

    # bases_n3 = ['XXX', 'YYX', 'YXY', 'XYY', 'ZZX', 'ZXZ', 'XZZ']
    bases_n3 = ['XXX', 'YYX', 'YXY', 'XYY']

    c3 = QuantumCircuit(qr3, cr3)
    c3.name = "SC_000"

    c4 = QuantumCircuit(qr3, cr3)
    c4.x(qr3[0])
    c4.x(qr3[1])
    c4.x(qr3[2])
    c4.barrier()
    c4.name = "SC_111_B"

    circuits_n3 = [c3, c4]

    sc_n3_circuits = []

    for c in circuits_n3:
        for b in bases_n3:
            sc_n3_circuits.append(add_measure_in_base(c.copy(), b))

    return sc_n3_circuits

# test_locally(SC_Circuits, use_mapping=True, save_to_file=True, number_of_simulations=100)
# test_locally_with_noise(SC_Circuits)
test_locally(get_sc_n3_circuits())


import ahocorasick
from .string_utils import is_whole_word

def build_trie_from_dict(str2value, to_lower=False):
    trie = ahocorasick.Automaton()
    for k, v in str2value.items():
        if to_lower:
            k = k.lower()
        trie.add_word(k, (k, v))
    trie.make_automaton()
    return trie

def get_values_by_prefix(trie, prefix):
    kvs = trie.values(prefix, '?', ahocorasick.MATCH_AT_LEAST_PREFIX)
    return [v for k, v in kvs]

def get_value_by_key(trie, key, default=None):
    k, v = trie.get(key, (key, default))
    return v

def iter_forms(trie, s, match_lower=False, whole_word=False, return_lower=False):
    if not trie or not s:
        return
    if match_lower:
        s_lower = s.lower()
        it = trie.iter(s_lower)
    else:
        it = trie.iter(s)
    for end, (k, v) in it:
        end = end + 1
        start = end - len(k)
        if whole_word and not is_whole_word(s, start, end):
            continue
        form = s[start: end]
        if return_lower:
            form = form.lower()
        yield form, start, end


from typing import Callable, Match

import re

from ._emoji_codes import EMOJI

_ReStringMatch = Match[str]  # regex match object
_ReSubCallable = Callable[[_ReStringMatch], str]  # Callable invoked by re.sub
_EmojiSubMethod = Callable[[_ReSubCallable, str], str]  # Sub method of a compiled re

def _emoji_replace(
    text: str, _emoji_sub: _EmojiSubMethod = re.compile(r"(:(\S*?):)").sub
) -> str:
    """Replace emoji code in text."""
    get_emoji = EMOJI.get

    def do_replace(match: Match[str]) -> str:
        """Called by re.sub to do the replacement."""
        emoji_code, emoji_name = match.groups()
        return get_emoji(emoji_name.lower(), emoji_code)

    return _emoji_sub(do_replace, text)


from PySide6 import QtCore, QtWidgets, QtGui
from formats.graphics.ani import AniSprite, Animation, AnimationFrame
from typing import List, Callable

class FramesModel(QtCore.QAbstractListModel):
    def __init__(self, update_frame_next: Callable):
        super(FramesModel, self).__init__()
        self.sprite: AniSprite = None
        self.animation: Animation = None
        self.update_frame_next = update_frame_next

    def set_animation(self, sprite: AniSprite, anim_index: int):
        self.layoutAboutToBeChanged.emit()
        self.sprite = sprite
        self.animation = self.sprite.animations[anim_index]
        self.layoutChanged.emit()

    def rowCount(self, parent: QtCore.QModelIndex) -> int:
        return len(self.animation.frames)

    def data(self, index: QtCore.QModelIndex, role: int):
        if not index.isValid():
            return None
        frame_idx = index.row()
        if frame_idx >= len(self.animation.frames):
            return None
        if role == QtCore.Qt.ItemDataRole.DisplayRole or role == QtCore.Qt.ItemDataRole.EditRole:
            return frame_idx
        if role == QtCore.Qt.ItemDataRole.DecorationRole:
            frame = self.animation.frames[frame_idx]
            return QtGui.QIcon(self.sprite.extract_image_qt(frame.image_index))
        return None

    def flags(self, index: QtCore.QModelIndex) -> QtCore.Qt.ItemFlag:
        default_flags = super(FramesModel, self).flags(index)
        if not index.isValid():
            return default_flags | QtCore.Qt.ItemFlag.ItemIsDropEnabled
        return default_flags | QtCore.Qt.ItemFlag.ItemIsDragEnabled

    def supportedDropActions(self) -> QtCore.Qt.DropAction:
        return QtCore.Qt.DropAction.MoveAction

    def mimeData(self, indexes: List[QtCore.QModelIndex]) -> QtCore.QMimeData:
        mime_data = super().mimeData(indexes)
        if not indexes:
            return mime_data
        index = indexes[0].row()
        mime_data.setText(str(index))
        return mime_data

    def dropMimeData(self, data: QtCore.QMimeData, action: QtCore.Qt.DropAction, row: int, column: int,
                     parent: QtCore.QModelIndex) -> bool:
        if row == -1:
            return False
        src_row = int(data.text())
        if src_row < row:
            row -= 1
        if src_row == row:
            return False
        frame = self.animation.frames[src_row]
        self.animation.frames.pop(src_row)
        self.animation.frames.insert(row, frame)
        self.update_frame_next()
        return True

    def append_frame(self):
        self.beginInsertRows(QtCore.QModelIndex(), len(self.animation.frames),
                             len(self.animation.frames))
        self.animation.frames.append(AnimationFrame(next_frame_index=0, image_index=0, duration=0))
        self.update_frame_next()
        self.endInsertRows()

    def remove_frame(self, index: QtCore.QModelIndex):
        self.beginRemoveRows(QtCore.QModelIndex(), index.row(), index.row())
        self.animation.frames.pop(index.row())
        self.update_frame_next()
        self.endInsertRows()


import csv
from datetime import datetime
from scapy.all import *
import time

"""
Run monitor_mode.sh first to set up the network adapter to monitor mode and to
set the interface to the right channel.
To get RSSI values, we need the MAC Address of the connection
of the device sending the packets.
"""

# Variables to be modified
dev_mac = ""  # Assigned transmitter MAC
iface_n = "wlan1"  # Interface for network adapter
duration = 30  # Number of seconds to sniff for
file_name = "rssi.csv"  # Name of CSV file where RSSI values are stored

def create_rssi_file():
    """Create and prepare a file for RSSI values"""
    header = ["date", "time", "dest", "src", "rssi"]
    with open(file_name, "w", encoding="UTF8") as f:
        writer = csv.writer(f)
        writer.writerow(header)

def captured_packet_callback(pkt):
    """Save MAC addresses, time, and RSSI values to CSV file if MAC address of src matches"""
    missed_count = 0  # Number of missed packets while attempting to write to file

    cur_dict = {}
    try:
        cur_dict["mac_1"] = pkt.addr1
        cur_dict["mac_2"] = pkt.addr2
        cur_dict["rssi"] = pkt.dBm_AntSignal
    except AttributeError:
        return  # Packet formatting error

    date_time = datetime.now().strftime("%d/%m/%Y,%H:%M:%S.%f").split(",") #Get current date and time
    date = date_time[0]
    time = date_time[1]

    ################### Your code here ###################

    # Only write the RSSI values of packets that are coming from your assigned transmitter (hint: filter by pkt.addr2, the destination MAC field)
    # Use the 'writerow' method to write the RSSI value and the current timestamp to the CSV file

    ######################################################
    target_mac = "e4:5f:01:d4:9d:2f"
    if pkt.haslayer(Dot11) and cur_dict["mac_2"]==target_mac:
        with open(file_name, mode = "a", newline='') as csv_file:
            csv_writer = csv.writer(csv_file)
            #csv_write.writerow(['Time', 'RSSI'])
           # rssi = packet.dBm_AntSignal if hasattr(pkt, 'dBm_AntSignal') else None
            csv_writer.writerow([date, time, cur_dict['mac_1'], cur_dict['mac_2'],cur_dict['rssi']])

if __name__ == "__main__":
    create_rssi_file()

    t = AsyncSniffer(iface=iface_n, prn=captured_packet_callback, store=0)
    t.daemon = True
    t.start()

    start_date_time = datetime.now().strftime("%d/%m/%Y,%H:%M:%S.%f") #Get current date and time

    time.sleep(duration)
    t.stop()

    print("Start Time: ", start_date_time)


#!/usr/bin/env python3
import argparse
from typing import List, Any
from conllu import parse
from conllu.models import Token
from diaparser.utils import Dataset
from diaparser.parsers import Parser

def main(target_sentence: str, language: str) -> None:
    if not target_sentence:
        return

    model_name: str = 'ja_gsd.mbert' if language == 'ja' else 'en_ewt-electra'
    parser: Parser = Parser.load(model_name)
    dataset: Dataset = parser.predict(target_sentence, text=language)
    sentences: List[Any] = parse(str(dataset.sentences[0]))
    sentence: List[Token] = sentences[0]

    id_token_map: Map[int, Token] = {}
    for token in sentence:
        token: Token
        cur_token_id: int = int(token['id'])
        id_token_map[cur_token_id] = token

    total_distance: int = 0
    for token in sentence:
        token: Token
        cur_token_id: int = int(token['id'])
        head_token_id: int = int(token['head'])
        distance: int = 0 if head_token_id == 0 else abs(
            cur_token_id - head_token_id)
        token.distance: int = distance
        total_distance += token.distance

    print(f'total_distance is {total_distance}.')
    sorted_sentence = sorted(sentence, key=lambda x: x.distance, reverse=True)
    for token in sorted_sentence:
        token: Token
        cur_token_str: str = token["form"]
        head_token_id: int = int(token['head'])
        head_token_str: str = 'root' if head_token_id == 0 else id_token_map[
            head_token_id]["form"]
        print(
            f'"{cur_token_str}" linked to "{head_token_str}". distance is {token.distance}.')

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='')
    parser.add_argument(
        'sentence', help='input target sentence', type=str, default='')
    parser.add_argument('-l', help='selecet sentence language("ja" or "en").',
                        type=str, choices=['ja', 'en'], default='ja')
    args = parser.parse_args()
    main(args.sentence, args.l)


import matplotlib.animation as animation
import matplotlib
import matplotlib.pyplot as plt

def save_video(frames, framerate=30, name_file="video.gif"):
    print(frames[0].shape)
    height, width, _ = frames[0].shape
    dpi = 70
    orig_backend = matplotlib.get_backend()
    # Switch to headless 'Agg' to inhibit figure rendering.
    matplotlib.use('Agg')
    fig, ax = plt.subplots(1, 1, figsize=(width / dpi, height / dpi), dpi=dpi)
    matplotlib.use(orig_backend)  # Switch back to the original backend.
    ax.set_axis_off()
    ax.set_aspect('equal')
    ax.set_position([0, 0, 1, 1])
    im = ax.imshow(frames[0])

    def update(frame):
      im.set_data(frame)
      return [im]
    interval = 1000/framerate
    anim = animation.FuncAnimation(fig=fig, func=update, frames=frames,
                                   interval=interval, blit=True, repeat=False)
    f = f"results/{name_file}"
    #FFwriter = animation.FFMpegWriter(fps=framerate)
    writergif = animation.PillowWriter(fps=framerate)
    anim.save(f, writer=writergif)


from dotenv import load_dotenv
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

import src.api.health as api_health
from src import logger
from src.config.server import ServerConfig
from src.container import Container

def create_app() -> FastAPI:
    load_dotenv()

    container = Container()

    app = FastAPI(title="Template API server")
    add_cors(app=app, cfg=container.server_config())

    app.container = container  # type: ignore[attr-defined]
    app.include_router(api_health.router)

    if container.health().is_healthy():
        logger.info("is healthy!")
    else:
        logger.error("failed health check")
    return app

def add_cors(app, cfg: ServerConfig):
    logger.warning(f"CORS: {cfg.cors}")
    app.add_middleware(
        CORSMiddleware,
        allow_origins=cfg.cors.origins,
        allow_credentials=cfg.cors.allow_credentials,
        allow_methods=cfg.cors.allow_methods,
        allow_headers=cfg.cors.allow_headers,
    )


# -*- coding: utf-8 -*-
# (c) 2020-2022 Martin Wendt and contributors; see https://github.com/mar10/yabs
# Licensed under the MIT license: https://www.opensource.org/licenses/mit-license.php
"""
"""
import os
from typing import TYPE_CHECKING

from git import Repo
from git.exc import GitCommandError

from ..util import check_arg, log_dry, log_response
from .common import TaskContext, WorkflowTask

if TYPE_CHECKING:  # Imported by type checkers, but prevent circular includes
    from yabs.task_runner import TaskInstance

class TagTask(WorkflowTask):
    DEFAULT_OPTS = {
        "name": "v{version}",
        "message": "Version {version}",
    }
    MANDATORY_OPTS = None

    def __init__(self, task_inst: "TaskInstance"):
        super().__init__(task_inst)

        opts = self.opts
        check_arg(opts["name"], str)
        check_arg(opts["message"], str)

    # def to_str(self, context :TaskContext):
    #     add = self.opts["add"] or self.opts["add_known"]
    #     return "{}(add: {}, '{}')".format(
    #         self.__class__.__name__, add, self.opts["message"]
    #     )

    @classmethod
    def register_cli_command(cls, subparsers, parents, run_parser):
        """"""

    @classmethod
    def check_task_def(cls, task_inst: "TaskInstance"):
        return True

    def run(self, context: TaskContext):
        opts = self.opts
        name = opts["name"].format(**vars(context))
        message = opts["message"].format(**vars(context))

        repo_path = os.path.abspath(".")
        repo = Repo(repo_path)
        git = repo.git

        if self.dry_run:
            log_dry("git tag -a {}".format(name))
            context.tag_name = name
            return True
        try:
            res = git.tag(
                name,
                annotate=True,
                message=message,
                dry_run=self.dry_run,
                verbose=self.verbose >= 4,
            )
            log_response("git tag {}".format(name), res, "info", self.dry_run)
            context.tag_name = name
        except GitCommandError as e:
            log_response("git tag", "{}".format(e), "error", self.dry_run)
            return False
        return True


from asyncio.events import AbstractEventLoop
from typing import List
from enum import Enum, auto
import asyncio
from pydantic.main import BaseConfig
from dataclasses import dataclass

from lxdbackup.job import Job

from lxdbackup.zfs import ZfsUtil
from lxdbackup.backup_commands import ArgBuilder, SyncoidArgs, CommandRunner
from lxdbackup.backup_logs import Log
from lxdbackup.copy_params import CopyParams

from config.base_config import BackupConfig

CONFIG_FILENAME = ".config.yml"

def main():

    # Get an instance of Backup
    backup = get_backup()
    # get jobs from Backup
    jobs, _ = get_jobs(backup)

    # job runner
    for job in jobs:
        run_job(job)

# job runner
def run_job(job):
    for container in job["containers"]:
        container_copy_params = get_container_copy_params(job, container)
        # send to backup pipeline
        build_copy_command(container_copy_params)

def build_copy_command(container_copy_params):
    z_src, z_dst = src_dst_creator(container_copy_params)

    args = build_args(z_src, z_dst)
    cmd = ArgBuilder(args=args)
    run = CommandRunner(cmd.arg_string)

    backup(run)

def src_dst_creator(copy_params):
    z_src = setup_source(copy_params)
    z_dst = setup_dest(copy_params)
    return z_src, z_dst

# Job(BackupConfig)
# Factory?
def get_backup():
    return BackupConfig(CONFIG_FILENAME)

def get_jobs(backup):
    job = Job(backup)
    return job.jobs, job.job_containers

# ParamBuilder
# factory?
def get_container_copy_params(job, container):
    copy_params = CopyParams(
        src_container=container["name"],
        src_host=job["src_host"],
        src_host_user=job["src_user"],
        dst_container=container["dst_name"],
        dst_host=job["dst_host"],
        dst_host_user=job["dst_user"],
    )

    return copy_params
    # run backup
    # todo: if host is not local running host, then ssh via paramiko and then run commands locally
    # todo: change backup.yaml file in mounted lxd to suit new storage pools and location
    # todo: lxd unmount
    # todo: lxd import
    # todo: live monitor of process output, threading, asyncio etc
    # todo: run all including syncoid in a bundled docker container or just run from docker container on the source
    # host by default
    # run.backup()
    # print(run.result)
    # logging(run)
    # asyncio

# Runner
def backup(run):
    loop: AbstractEventLoop = asyncio.get_event_loop()
    async_q = asyncio.Queue()
    task = asyncio.gather(run.backup(async_q))

    loop.run_until_complete(task)

    # await async_q.put(run.backup())
    # result = asyncio.run(run.backup())

# logger
def logging(run):
    log = Log()
    log.log(run.result["stdout"], run.result["stderr"])

# param builder
def build_args(z_src, z_dst):
    args = SyncoidArgs(
        src_host=z_src.host,
        src_user=z_src.user,
        dst_host=z_dst.host,
        dst_user=z_dst.user,
        zfs_source_path=z_src.source_container_path,
        zfs_destination_path=z_dst.destination_container_path,
    )

    return args

# job setup
def setup_dest(copy_params):

    z_dst = ZfsUtil(host=copy_params.dst_host, user=copy_params.dst_host_user)

    # run prechecks #todo
    # run_dataset_prechecks(z_dst)

    z_dst.set_destination_container(copy_params.dst_container)
    return z_dst

# job setup
def setup_source(copy_params):
    z_src = ZfsUtil(host=copy_params.src_host, user=copy_params.src_host_user)
    z_src.set_source_container(copy_params.src_container)
    return z_src

def run_dataset_prechecks(z_dst):
    return
    # todo
    lxd_check = DatasetCheck()
    if check_for_existing_dataset(z_dst.datasets):
        # if dataset exists return true
        return
    else:
        # todo
        lxc_dataset_creator = DatasetCreator()

if __name__ == "__main__":
    main()


import numpy as np
import gym
import random
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import scipy.signal

import threading
import multiprocessing
import tensorflow as tf
import tensorflow.contrib.slim as slim
import re
from time import sleep
from time import time
from scipy.misc import imresize

from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file

def process_frame(frame):
    luminance = np.sum(frame, 2).astype('float32')
    luminance = luminance[27:, :]
    luminance = imresize(luminance, (84, 84)).astype('float32')
    luminance = imresize(luminance, (42, 42)).astype('float32')*(1 / 255.0)
    luminance = np.expand_dims(luminance, axis=3)

    return luminance

class A3C_Network:

    def normalized_columns_initializer(self , std = 1.0):
        def _initializer(shape, dtype=None, partition_info=None):
            out = (np.random.randn(*shape).astype(np.float32))
            out *= std / np.sum(np.sqrt(np.square(out)))
            return tf.constant(out)

        return _initializer

    def AIconv2d(self, x, num_filters, name, filter_size=(3, 3), stride=(1, 1), pad="SAME", dtype=tf.float32, collections=None):
        with tf.variable_scope(name):

            stride_shape = [1, stride[0], stride[1], 1]
            filter_shape = [filter_size[0], filter_size[1], int(x.get_shape()[3]), num_filters]

            fan_in = np.prod(filter_shape[:3])

            fan_out = np.prod(filter_shape[:2]) * num_filters
            w_bound = np.sqrt(6. / (fan_in + fan_out))

            w = tf.get_variable("W", filter_shape, dtype, tf.random_uniform_initializer(-w_bound, w_bound),
                                collections=collections)
            b = tf.get_variable("b", [1, 1, 1, num_filters], initializer=tf.constant_initializer(0.0),
                                collections=collections)
            return tf.nn.conv2d(x, w, stride_shape, pad) + b

    def __init__(self, pix_x , pix_y, scope, trainer, act_space= 6):

        with tf.variable_scope(scope):

            strides1 = int(4)
            strides2 = int(2)

            full_c1 = int(pix_x/(strides1*strides2))
            full_c2 = int(pix_y/(strides1*strides2))

            filters2 = 32

            self.input = tf.placeholder(dtype=tf.float32, shape=(None, 42 , 42, 1), name='frame_input')

            self.conv1 = slim.conv2d(activation_fn=tf.nn.elu,
                                     inputs=self.input, num_outputs=32,
                                     kernel_size=[3, 3], stride=[2, 2], padding='SAME')

            self.conv2 = slim.conv2d(activation_fn=tf.nn.elu,
                                     inputs=self.conv1, num_outputs=32,
                                     kernel_size=[3, 3], stride=[2, 2], padding='SAME')
            self.conv3= slim.conv2d(activation_fn=tf.nn.elu,
                                     inputs=self.conv2, num_outputs=32,
                                     kernel_size=[3, 3], stride=[2, 2], padding='SAME')
            self.conv4 = slim.conv2d(activation_fn=tf.nn.elu,
                                     inputs=self.conv3, num_outputs=32,
                                     kernel_size=[3, 3], stride=[2, 2], padding='SAME')
            self.hidden = slim.fully_connected(slim.flatten(self.conv4), 256, activation_fn=tf.nn.elu)

            lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(256, state_is_tuple = True)

            init_cell_state = tf.constant(value = 0, shape = (1, lstm_cell.state_size.c), dtype =  tf.float32)
            init_hidden_state = tf.constant(value = 0, shape = (1, lstm_cell.state_size.h), dtype = tf.float32)

            self.init_cell = [init_cell_state, init_hidden_state]

            self.c_in = tf.placeholder(dtype = tf.float32,shape =  (1, lstm_cell.state_size.c), name = 'c_in')
            self.h_in = tf.placeholder(dtype = tf.float32, shape = (1, lstm_cell.state_size.h), name = 'h_in')

            self.rnn_in = tf.expand_dims(self.hidden, [0])
            step_size = tf.shape(self.input)[:1]
            self.state_in = tf.nn.rnn_cell.LSTMStateTuple(self.c_in, self.h_in)
            self.lstm_outputs, self.lstm_state = tf.nn.dynamic_rnn(
               lstm_cell, self.rnn_in, initial_state=self.state_in,
               time_major=False, sequence_length=step_size)

            self.lstm_outputs = tf.squeeze(self.lstm_outputs, axis = 0)

            condense_to_value = tf.get_variable(dtype=tf.float32, shape = (256), initializer=self.normalized_columns_initializer(std= 1),
                                                name='form_value')

            self.value_output =tf.tensordot(self.lstm_outputs, condense_to_value, [[1], [0]])
            condense_to_actions = tf.get_variable(dtype=tf.float32,
                                                  shape = (256, act_space), initializer=self.normalized_columns_initializer(std= 0.01), name='c_act')
            action_output = tf.tensordot(self.lstm_outputs, condense_to_actions, [[1], [0]], name = 'action1')
            self.norm_actions = tf.nn.softmax(action_output)

            self.test = self.lstm_outputs

            if scope != 'global':
                R = tf.placeholder(dtype=tf.float32, shape=(None), name='perf_reward')
                get_value = tf.placeholder(dtype=tf.float32, shape=(None), name='perf_value')
                get_action = tf.placeholder(dtype=tf.int32, shape=(None), name='perf_action')
                advantage = tf.placeholder(dtype = tf.float32 ,shape = (None) , name = 'advantage')

                self.one_hot_action = tf.one_hot(get_action,act_space)
                self.action_channel1 = tf.multiply(self.norm_actions, self.one_hot_action)
                self.action_channel = tf.reduce_sum(self.action_channel1,1)

                self.clip_action = tf.clip_by_value(self.action_channel,0.000001,9999999)
                self.value_loss = tf.reduce_sum(tf.square(self.value_output-R))
                self.action_loss = tf.reduce_sum(tf.log(self.clip_action)*advantage)
                self.entropy = -tf.reduce_sum(tf.log(self.norm_actions)*self.norm_actions)
                self.full_loss = 0.5*self.value_loss - self.action_loss - 0.01*self.entropy

                local_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope)
                self.gradients = tf.gradients(self.full_loss , local_vars)

                self.var_norms = tf.global_norm(local_vars)
                grads, self.grad_norms = tf.clip_by_global_norm(self.gradients, 40.0)
                global_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,'global')
                self.apply_grads = trainer.apply_gradients(zip(self.gradients, global_vars))

    def create_network(self):
        init = tf.initialize_all_variables()
        return [init, tf.shape(self.norm_actions)]

class Worker(A3C_Network):

    def __init__(self, the_scope, trainer):
        A3C_Network.__init__(self, 42, 42, the_scope, trainer, act_space=3)
        self.gamma = 0.9900
        self.update_local_ops = self.copy_scope('global', the_scope)
        self.the_scope = the_scope
        self.max_episodes = 300
        self.summary_writer = tf.summary.FileWriter("tensorBoard/train_" + the_scope)

    def get_scope(self):
        return self.the_scope

    def copy_scope(self,from_scope,to_scope):
        source_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, from_scope)
        target_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, to_scope)

        op_holder = []
        for source_vars, target_vars in zip(source_vars,target_vars):
            op_holder.append(tf.assign(target_vars,source_vars))
        return op_holder

    def discounted_reward(self , reward_buffer, bsv, clip = True):
        discount_vector = np.ones([1,len(reward_buffer)])*self.gamma

        discount_vector[0,0] = 1
        discount_vector = np.cumproduct(discount_vector)

        L = len(reward_buffer)

        discounted_reward_buffer = []
        for R in enumerate(reward_buffer[:]):
            discounted_reward_buffer.append(bsv*np.power(self.gamma,( L - R[0])-1) + np.sum((discount_vector[0:(L - R[0])]*reward_buffer[R[0]:])))
        if clip == True:
            return discounted_reward_buffer[:-1]
        else:
            return discounted_reward_buffer
    def train_worker(self, discounted_reward_buffer,reward_buffer, state_buffer , action_buffer, value_buffer,rnn_state_buffer_h, rnn_state_buffer_c,  sess):

        advantage = np.asarray(self.reward_buffer[:-1]) + self.gamma * np.asarray(
            self.value_buffer[1:]) - np.asarray(self.value_buffer [:-1])
        advantage = self.discounted_reward(advantage, 0, clip = False)

        feed_dict = {self.the_scope + '/frame_input:0': state_buffer[:-1],
                     self.the_scope + '/c_in:0': rnn_state_buffer_c,
                     self.the_scope + '/h_in:0': rnn_state_buffer_h,
                     self.the_scope + '/advantage:0': advantage,
                     self.the_scope + '/perf_action:0': action_buffer[:-1],
                     self.the_scope + '/perf_reward:0': discounted_reward_buffer,
                     self.the_scope + '/perf_value:0': value_buffer[:-1]}

        sess.run([self.value_output, self.full_loss, self.value_loss,self.action_loss,self.entropy] , feed_dict = feed_dict)
        sess.run(self.apply_grads, feed_dict=feed_dict)
        #print(np.shape(sess.run(self.test, feed_dict=feed_dict)))
        #print(sess.run(self.test, feed_dict=feed_dict))

    def run_atari_episode(self , max_steps, episodes , sess ,coord,  saver):

        env = gym.make('PongDeterministic-v0')

        track_loss = 0
        esp = 0
        with sess.as_default() and sess.graph.as_default():

            while not coord.should_stop():
                if esp % 20 == 0 and self.the_scope == 'local_0':
                    saver.save(sess,"/media/asger/ce7a4008-6b8f-447d-9acc-614945aef109/space_invaders_global.ckpt")
                    print('saving episode ', esp)
                esp += 1
                action = 0
                done = 0
                obs_prev = env.reset()
                obs_prev = process_frame(obs_prev)
                rnn_state = sess.run(self.init_cell)

                self.state_buffer = []
                self.reward_buffer = []
                self.actions_buffer = []
                self.value_buffer = []

                while done == 0:

                    self.rnn_state_buffer_c = rnn_state[0]
                    self.rnn_state_buffer_h = rnn_state[1]

                    for  i in range(max_steps):

                        obs_prev = process_frame(obs_prev)
                        feed_dict = {self.the_scope + '/frame_input:0': [obs_prev], self.the_scope + '/c_in:0': rnn_state[0],
                                          self.the_scope + '/h_in:0': rnn_state[1]}

                        norm_actions, value_est, new_rnn_state = sess.run([self.norm_actions, self.value_output, self.lstm_state], feed_dict=feed_dict)

                        if i != max_steps - 1:
                            rnn_state = new_rnn_state

                        norm_actions = norm_actions[0]
                        a = np.random.choice(norm_actions, p=norm_actions)
                        chosen_action = np.argmax(norm_actions == a)

                        obs_next , reward , done , info = env.step(1+chosen_action)

                        if self.the_scope == 'local_0':
                            env.render()
                            print(chosen_action)
                            print(norm_actions)
                            print(value_est[0])

                        self.state_buffer.append(obs_prev)
                        self.actions_buffer.append(chosen_action)
                        self.value_buffer.append(value_est[0])
                        self.reward_buffer.append(reward)

                        if i == max_steps-1 or done == 1:

                            boot_strap_val = self.value_buffer[len(self.value_buffer)-1]
                            discounted_reward_buffer = self.discounted_reward(self.reward_buffer, boot_strap_val)
                            self.train_worker(discounted_reward_buffer, self.reward_buffer, self.state_buffer, self.actions_buffer, self.value_buffer,self.rnn_state_buffer_h,self.rnn_state_buffer_c, sess)
                            sess.run(self.update_local_ops)
                            if done == 1:
                                break

                            self.state_buffer = [self.state_buffer[len(self.state_buffer) - 1]]
                            self.reward_buffer = [self.reward_buffer[len(self.reward_buffer) - 1]]
                            self.actions_buffer = [self.actions_buffer[len(self.actions_buffer) - 1]]
                            self.value_buffer = [self.value_buffer[len(self.value_buffer) - 1]]
                        obs_prev = obs_next

class DeployModel:

    def display_processed_frame(self):
        env = gym.make('Pong-v0')
        get_frame = env.reset()
        for i in range(46):
            act = env.action_space.sample()
            get_frame,_,_,_ = env.step(act)
        env.render()
        processed_frame = process_frame(get_frame)
        processed_frame = np.tile(processed_frame,3)
        plt.imshow(processed_frame)
        plt.show()

        return processed_frame

    def load(self, episodes, path):

        tf.reset_default_graph()
        saver = tf.train.import_meta_graph(path + '.meta')
        with tf.Session() as sess:

            graph = tf.get_default_graph()
            saver.restore(sess,path)
            print('model restored')
            env = gym.make('SpaceInvaders-v0')
            esp = 0
            print_tensors_in_checkpoint_file(file_name=path , tensor_name='', all_tensors=False, all_tensor_names=True)

            placeholder = graph.get_tensor_by_name("local_0/frame_input:0")
            h_in = graph.get_tensor_by_name("local_0/h_in:0")
            c_in = graph.get_tensor_by_name("local_0/c_in:0")

            init_cell_state = np.asarray(np.zeros([1,256]), dtype=np.float32)
            init_hidden_state = np.asarray(np.zeros([1,256]), dtype=np.float32)

            op_to_restore = graph.get_tensor_by_name("local_0/action2:0")
            while esp < episodes:
                obs = env.reset()
                done = 0
                esp += 1
                while done == 0:
                    obs = process_frame(obs)
                    feed_dict = {placeholder:[obs], h_in:init_hidden_state, c_in: init_cell_state}
                    norm_actions = sess.run(op_to_restore, feed_dict = feed_dict)[0]
                    alt = np.random.choice(norm_actions, p=norm_actions)
                    chosen_action = np.argmax(norm_actions == alt)
                    #print(chosen_action)
                    act = env.action_space.sample()
                    obs, reward, done, info = env.step(act)
                    env.render()
                    sleep(1.0/24)
            sess.close()

    def train(self,threads, episodes, max_steps):
        tf.reset_default_graph()
        with tf.device('/cpu:0'):

            global_network = Worker('global',tf.train.RMSPropOptimizer(learning_rate=0))
            cores = multiprocessing.cpu_count()
            Workers = []
            if threads > cores:
                threads = cores

            for q in range(threads):
                learningrate = 0.0003
                print(learningrate)
                a_worker = Worker('local_' + str(q), tf.train.RMSPropOptimizer(learning_rate=learningrate, epsilon=0.1, decay = 0.99))
                Workers.append(a_worker)
                print('deploy_worker' + str(q))

        saver = tf.train.Saver(var_list= tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,'local_0'))
        with tf.Session() as sess1:
            global buffer
            sess1.run(tf.global_variables_initializer())
            coord = tf.train.Coordinator()
            worker_threads = []
            for w in Workers:
                worker_work = lambda: w.run_atari_episode(max_steps, episodes, sess1,coord, saver)
                t = threading.Thread(target=worker_work)
                t.start()
                sleep(5)
                worker_threads.append(t)

            coord.join(worker_threads)
            sess1.close()
            print('training is complete')

path = '/media/asger/ce7a4008-6b8f-447d-9acc-614945aef109/space_invaders_global.ckpt'

instance = DeployModel()
#instance.load(10,path)
instance.train(100, 100000,5)


import re
import sys
from math import log
from time import time
import numpy as np
import scipy.sparse
import scipy.sparse.linalg

from collections import OrderedDict

GREEN_CLR = '\033[32m'
YELLOW_CLR = '\033[33m'
PURPLE_CLR = '\033[35m'
END_CLR = '\033[0m'

# BM25 parameters
bm25k = 0.75
bm25b = 0.0
lmbda = 0.6

class EvaluateBenchmark:
    """ Class for evaluating a given benchmark. """

    def __init__(self, inv_lists):
        """
        Creates an empty dictionary of queries and their relevant movies ids,
        a list of pair with a movies id and its relevance according to the
        benchmark file and total sums for P@3, P@R and AP.
        """

        self.benchmark_ids = dict()
        self.res_relevance = list()
        self.ii = inv_lists
        self.sum_pa3 = 0
        self.sum_par = 0
        self.sum_ap = 0

    def precision_at_k(self, results_ids, relevant_ids, k):
        """
        Computes the P@k for the given result list and the given set of
        relevant docs.

        (Note: with our approach results_ids and relevant_ids are not needed as
        they were used before to generate the results_relevance list.)
        """
        return len([x for x in self.res_relevance[:k] if x[1] == 1]) / k

    def average_precision(self, results_ids, relevant_ids):
        """
        Computes the AP (average precision) of the given result list and the
        given set of relevant docs.
        """
        sum_p = 0

        r_list = [results_ids.index(x[0]) + 1
                  for x in self.res_relevance if x[1] == 1]

        # Taking into the consideration the note below on the Slide 22
        # (Lecture 2): "for docs not in the result list take P@Ri = 0"
        r_list_length = len(r_list)
        r_list_length += len(relevant_ids) - r_list_length

        for r in r_list:
            sum_p += self.precision_at_k(None, None, r)

        return sum_p / r_list_length

    def evaluate_benchmark(self, file_name):
        """ Evaluates the given benchmark. """

        with open(file_name, 'r', encoding='utf-8') as file:
            for line in file:
                splitted_line = line.replace('\n', '').split('\t')
                self.benchmark_ids[splitted_line[0]] = \
                    [int(x) for x in splitted_line[1].split(' ')]

        print('Benchmark evaluation...')
        st = time()

        for query, relevant_ids in self.benchmark_ids.items():
            # BM25 (VSM)
            res_ids = [x[0] for x in self.ii.process_query_vsm(query)]

            # LSI
            # res_ids = [x[0]
            #            for x in self.ii.process_query_lsi(query,\
            # lmbda, True)]

            # BM25 + LSI
            # res_ids = [x[0] for x in self.ii.process_query_lsi(query, lmbda)]

            self.res_relevance = [[res_id, 1 if res_id in relevant_ids else 0]
                                  for res_id in res_ids]

            self.sum_pa3 += self.precision_at_k(None, None, 3)
            self.sum_par += self.precision_at_k(None, None, len(relevant_ids))
            self.sum_ap += self.average_precision(res_ids, relevant_ids)

        num = len(self.benchmark_ids)
        print('\nMP@3: %s, MP@R: %s, MAP: %s' %
              (round(self.sum_pa3 / num, 2),
               round(self.sum_par / num, 2),
               round(self.sum_ap / num, 2)))
        print('K = %s, B = %s' % (bm25k, bm25b))
        print('\nEvaluation time: %s s' % (round(time() - st, 2)))

class InvertedIndex:
    """ A simple inverted index as explained on the lecture. """

    def __init__(self):
        """ Creates an empty inverted index and additional dicts. """

        self.inverted_lists = dict()
        self.inv_lists_sorted = dict()
        self.records = dict()
        self.record_lengths = dict()

        self.terms = []
        self.num_terms = 0
        self.num_docs = 0

        self.A = None
        self.Uk = None
        self.Sk = None
        self.Vk = None

    def read_from_file(self, file_name):
        """
        Constructs the inverted index from the given file. The format is: one
        record per line.

        >>> ii = InvertedIndex()
        >>> ii.read_from_file('example.txt')
        >>> ii.num_terms, ii.num_docs
        (4, 6)
        >>> ii.terms
        ['internet', 'web', 'surfing', 'beach']
        >>> sorted(ii.inverted_lists.items())
        [('beach', {4: 1, 5: 1, 6: 1}), ('internet', {1: 1, 2: 1, 4: 1}), \
('surfing', {1: 1, 2: 1, 3: 1, 4: 2, 5: 1}), ('web', {1: 1, 3: 1, 4: 1})]

        """

        with open(file_name, 'r', encoding='utf-8') as file:
            doc_id = 0
            for line in file:
                doc_id += 1
                words = re.split("\W+", line)
                self.records[doc_id] = line.replace('\n', '')
                self.record_lengths[doc_id] = len(words)
                for term in words:
                    term = term.lower()
                    if any(term):
                        # If a word is seen for the first time, create an empty
                        # inverted list for it
                        if term not in self.inverted_lists:
                            self.terms.append(term)
                            self.inverted_lists[term] = dict()

                        if doc_id in self.inverted_lists[term].keys():
                            self.inverted_lists[term][doc_id] += 1
                        else:
                            self.inverted_lists[term][doc_id] = 1
            self.num_docs = doc_id
            self.num_terms = len(self.inverted_lists)

    def bm25_score(self, tf, df, N, AVDL, DL):
        return tf * (bm25k + 1) / (bm25k * (1 - bm25b + bm25b * DL / AVDL) +
                                   tf) * log((N / df), 2)

    def preprocessing_vsm(self, k, m):
        """
        Computes the sparse term-document matrix using the (already built)
        inverted index. For LSI it also performs SVD using dimensionality k and
        only the most frequent terms m. Intermediate results are stored as
        members of this class.
        """
        self.inv_lists_sorted = OrderedDict(sorted(self.inverted_lists.items(),
                                                   key=lambda x: len(x[1]),
                                                   reverse=True)[:m])
        N = self.num_docs
        AVDL = sum(self.record_lengths.values()) / float(N)

        term_id = 0
        nz_vals, row_inds, col_inds = [], [], []

        for term, inv_list in self.inv_lists_sorted.items():
            for doc_id, tf in inv_list.items():
                df = len(self.inv_lists_sorted[term])
                DL = self.record_lengths[doc_id]
                self.inv_lists_sorted[term][doc_id] = \
                    self.bm25_score(tf, df, N, AVDL, DL)
            nz_vals += [v for v in self.inv_lists_sorted[term].values()
                        if v != 0]
            row_inds += [term_id] * len(self.inv_lists_sorted[term])
            col_inds += [id - 1 for id, v in
                         self.inv_lists_sorted[term].items() if v != 0]
            term_id += 1
        self.A = scipy.sparse.csr_matrix((nz_vals, (row_inds, col_inds)),
                                         dtype=float)
        # LSI
        self.Uk, Sk, self.Vk = scipy.sparse.linalg.svds(self.A, k)
        self.Sk = np.diag(Sk)

    def process_query_vsm(self, query):
        """
        Executes the query using the (full) term-document matrix in the vsm.
        """
        keywords = [word.lower() for word in re.split("\W+", query)]
        q = scipy.sparse.csr_matrix([1 if term in keywords else 0
                                     for term in self.inv_lists_sorted.keys()])

        scores = q.dot(self.A)
        return sorted(list(zip(scores.indices + 1, scores.data)),
                      key=lambda x: x[1], reverse=True)

    def process_query_lsi(self, query, lmbda, only_lsi=False):
        """
        Executes the query by mapping the query vector to latent space.
        """
        keywords = [word.lower() for word in re.split("\W+", query)]
        q = np.array([1 if term in keywords else 0
                      for term in self.inv_lists_sorted.keys()])
        qk = q.dot(self.Uk).dot(self.Sk)

        scores = qk.dot(self.Vk) if only_lsi \
            else lmbda * np.transpose(q) * self.A + (1 - lmbda) * \
            np.transpose(qk).dot(self.Vk)

        return sorted(list(zip([i + 1 for i in range(0, scores.size)],
                               scores)),
                      key=lambda x: x[1], reverse=True)

    def related_term_pairs(self):
        """
        Computes the term-term association matrix T.
        """

        T = self.Uk.dot(np.transpose(self.Uk))
        term_dict = {idx: term
                     for idx, term in enumerate(self.inv_lists_sorted.keys())}

        added = set()
        np.fill_diagonal(T, 0)
        max_values = sorted([((i, t.argmax()), t[t.argmax()])
                             for i, t in enumerate(T)],
                            key=lambda x: x[1], reverse=True)
        max_values = [item for item in max_values
                      if item[1] not in added and not added.add(item[1])]

        # for pair in max_values[:50]:
        #     print('{0:15} {1:15} {2:2f}'.format(term_dict[pair[0][0]],
        #                                      term_dict[pair[0][1]], pair[1]))

        with open('term_pairs.txt', 'w') as f:
            for pair in max_values[:50]:
                f.write('%s\t%s\t%f\n' % (term_dict[pair[0][0]],
                                          term_dict[pair[0][1]], pair[1]))

    def print_output(self, hits, query):
        for hit in hits:
            record = self.records[hit[0]].split('\t')
            title = record[0]

            print(GREEN_CLR + title + END_CLR)

            keywords = [word.lower() for word in re.split("\W+", query)]
            description = record[0] if len(record) == 1 \
                else record[1]

            # Keywords highlighting
            words = description.split(' ')
            for word in words:
                truncated_word = re.sub(r'[^\w]', '', word)
                if truncated_word.lower() in keywords:
                    wrapped_word = YELLOW_CLR + truncated_word + END_CLR
                    words[words.index(word)] = \
                        word.replace(truncated_word, wrapped_word)
            print(' '.join(words), '\n')

if __name__ == "__main__":
    if len(sys.argv) < 4 or \
            (len(sys.argv) == 5 and sys.argv[4] == '--benchmark') or \
            (len(sys.argv) == 6 and sys.argv[4] != '--benchmark'):
        msg = 'Usage: \n\tpython3 inverted_index.py <file> <k> <m>' + \
              '\n\tpython3 inverted_index.py <file> <k> <m> --benchmark ' + \
              '<benchmark_file>'
        print(msg)
        sys.exit()

    ii = InvertedIndex()
    file_name = sys.argv[1]
    k = int(sys.argv[2])
    m = int(sys.argv[3])
    print('Loading movies file...')
    ii.read_from_file(file_name)
    print('Preprocessing...\n')
    ii.preprocessing_vsm(k, m)

    # ii.related_term_pairs()

    if len(sys.argv) > 5 and sys.argv[4] == '--benchmark':
        eb = EvaluateBenchmark(ii)
        eb.evaluate_benchmark(sys.argv[5])
    else:
        while True:
            msg = PURPLE_CLR + \
                '> Enter the query (type "exit" for quitting): ' + END_CLR
            query = input(msg)
            if query == 'exit':
                break

            print('')

            hits = ii.process_query_vsm(query)
            # hits = ii.process_query_lsi(query, lmbda)
            # hits = ii.process_query_lsi(query, lmbda, only_lsi=True)
            if any(hits):
                ii.print_output(hits[:3], query)
            else:
                print('No hits')


import argparse
from pathlib import Path
import csv
from typing import Dict, Union

import matplotlib.pyplot as plt
import matplotlib.lines as mlines
from matplotlib.legend import Legend
import numpy as np

from eprem import tools, seed

def main(
    spectrum: Union[Path, dict],
    plotpath: Path=None,
    textpath: Path=None,
    free: list=None,
    fixed: dict=None,
    lower: dict=None,
    initial: dict=None,
    upper: dict=None,
):
    """Compute and save a fit to the given fluence spectrum."""
    paths = setup_paths(
        plot=(plotpath, '.png'),
        text=(textpath, '.txt'),
    )
    fit = compute_fit(
        spectrum,
        free=free,
        fixed=fixed,
        lower=lower,
        initial=initial,
        upper=upper,
    )
    print(f"Saving {paths['plot']} ...")
    create_plot(paths['plot'], fit)
    print(f"Saving {paths['text']} ...")
    create_text(paths['text'], fit)

def setup_paths(**paths) -> Dict[str, Path]:
    """Ensure full, read-only paths that exist."""
    return {name: _split(pair) for name, pair in paths.items()}

def _split(pair) -> Path:
    """Helper function for ``setup_path()``."""
    value, suffix = pair if len(pair) == 2 else (pair[0], None)
    try:
        path = Path(value)
    except TypeError:
        path = Path(__file__)
    if suffix is not None:
        path = path.with_suffix(suffix).expanduser().resolve()
    return path

def compute_fit(
    spectrum: Union[Union[str, Path], dict],
    **context
) -> seed.Fitter:
    """Compute the fit and store relevant data in a `Fitter` object."""
    if isinstance(spectrum, (str, Path)):
        data = load_from_file(spectrum)
    elif isinstance(spectrum, dict):
        data = load_from_dict(spectrum)
    else:
        TypeError(spectrum)
    return seed.Fitter(
        energies=data['energies'],
        fluxdata=data['fluences'],
        sigma=data.get('uncertns'),
        **context
    )

def load_from_file(filepath: Union[str, Path]):
    """Load the spectrum from the given file."""
    data = np.loadtxt(filepath, skiprows=6)
    energies = data[:, 0]
    fluences = data[:, 1]
    uncertns = data[:, 2] if data.shape[1] == 3 else None
    return {
        'energies': energies,
        'fluences': fluences,
        'uncertns': uncertns,
    }

def load_from_dict(user: Dict[str, Union[np.ndarray, float]]):
    """Load the spectrum from the given dict."""
    energies = user['E']
    parameters = {
        name: user.get(name, value)
        for name, value in seed.default_values.items()
    }
    fluences = seed.J(energies, **parameters)
    return {
        'energies': energies,
        'fluences': fluences,
    }

def create_plot(path: Path, fit: seed.Fitter):
    """Plot the fluence spectrum and fit."""
    plt.plot(fit.energies, fit.fluxdata, label="data")
    plt.plot(fit.energies, fit.spectrum, label="fit")
    plt.xlim([1e-1, 1e2])
    plt.ylim([1e0, 1e7])
    plt.xscale('log')
    plt.yscale('log')
    plt.xlabel('Energy [MeV]')
    plt.ylabel('Fluence [Counts / (cm² sr MeV)]')
    original_legend = plt.legend(loc='upper right')
    plt.gca().add_artist(parameter_legend(fit))
    plt.gca().add_artist(original_legend)
    plt.savefig(path)

def parameter_legend(fit: seed.Fitter) -> Legend:
    """Build a legend that displays spectrum parameters (fit or not)."""
    labels = fit.get_parameter_labels()
    handles = [
        mlines.Line2D(
            [], [],
            label=rf"{this['string']} = {this['value']}"
        ) for this in labels.values()
    ]
    return plt.legend(
        handles=handles,
        handlelength=0.0,
        loc='lower left',
    )

def create_text(path: Path, fit: seed.Fitter):
    """Write fit results to a text file."""
    with path.open('w', newline='') as fp:
        writer = csv.writer(fp, delimiter=',')
        writer.writerows(csv_header(fit))
        writer.writerow(['Energy (MeV)', 'Data', 'Fit'])
        for row in zip(fit.energies, fit.fluxdata, fit.spectrum.squeeze()):
            writer.writerow(row)

def csv_header(fit: seed.Fitter, comment: str="#") -> list:
    """Build an appropriate header for the text output."""

    def line(name: str, parameter: dict) -> list:
        """Build an individual header line."""
        return [
            f"{comment} {name} = {parameter['value']} = {parameter['status']}"
        ]

    parameters = fit.get_parameter_labels()
    start = [f"{comment} Parameters:"]
    info = [line(*item) for item in parameters.items()]
    end = [f"{comment} "]
    return [start, *info, end]

if __name__ == "__main__":
    p = argparse.ArgumentParser(
        description=main.__doc__,
        formatter_class=argparse.RawTextHelpFormatter,
        epilog=(
            "DEPRECATION WARNING"
            ": Future updates may remove this script. Please use "
            "fit_flux_spectrum.py, which is capable of handling a wider "
            "variety of inputs."
        )
    )
    p.add_argument(
        'spectrum',
        help=(
            "path to the file containing a spectrum to fit"
            ";\nmay be relative and contain wildcards"
            "\n\nnote that the Python interface also accepts a dict that"
            "\nprovides an energy array and parameter values, which may be"
            "\nuseful for testing or debugging"
        )
    )
    p.add_argument(
        '--plotpath',
        help="path to which to save a plot of the fit"
    )
    p.add_argument(
        '--textpath',
        help="path to which to write fit results"
    )
    p.add_argument(
        '--free',
        help="names of free parameters",
        nargs='*',
        choices=tuple(seed.default_values.keys()),
        metavar=("p0", "p1"),
    )
    p.add_argument(
        '--fixed',
        help="key-value pairs of parameters to hold fixed",
        nargs='*',
        value_type=float,
        action=tools.StoreKeyValuePair,
        metavar=("p0=value0", "p1=value1"),
    )
    p.add_argument(
        '--initial',
        help="key-value pairs of initial guesses",
        nargs='*',
        value_type=float,
        action=tools.StoreKeyValuePair,
        metavar=("p0=guess0", "p1=guess1"),
    )
    p.add_argument(
        '--lower',
        help="key-value pairs of lower bounds",
        nargs='*',
        value_type=float,
        action=tools.StoreKeyValuePair,
        metavar=("p0=lower0", "p1=lower1"),
    )
    p.add_argument(
        '--upper',
        help="key-value pairs of upper bounds",
        nargs='*',
        value_type=float,
        action=tools.StoreKeyValuePair,
        metavar=("p0=upper0", "p1=upper1"),
    )
    args = p.parse_args()
    main(**vars(args))


from flask import Flask, request, render_template

app = Flask(__name__)

@app.route("/")
def hello_world():
    return "Hello,World"

# ログイン機能
@app.route("/login", methods=["GET", "POST"])
def login():
    if request.method == "GET":
        return render_template("login.html")
    if request.method == "POST":
        return "Log in!"

if __name__ == '__main__':
    hostname = "127.0.0.1"
    port = 5001
    app.run(port=5001, debug=True)


import datetime
import random
import re

from django.conf import settings
from django.db import models
from django.template.loader import render_to_string
from django.utils.hashcompat import sha_constructor
from django.utils.translation import ugettext_lazy as _
from django.core.mail import EmailMultiAlternatives
from django.core.mail import send_mail

SHA1_RE = re.compile('^[a-f0-9]{40}$')

class RegistrationManager(models.Manager):
    """
    Custom manager for the ``RegistrationProfile`` model.

    The methods defined here provide shortcuts for account creation
    and activation (including generation and emailing of activation
    keys), and for cleaning out expired/already activated profiles.

    """
    def activate_user(self, request, activation_key, callback, **kwargs):
        """
        Validate an activation key and calls ``callback`` in order to activate
        the corresponding user if valid. ``callback`` default value is a
        callable got from importing ``ACTIVATION_METHOD`` settings string.

        If the key is valid, returns ``callback`` result two-tuple: (``User``
        instance, ``None``) on success or (falsy value, errors) on failure.

        If the key is invalid (already activated or expired), returns a
        two-tuple (``False``, 'Your activation key is not valid').

        To prevent reactivation of an account which has been
        deactivated by site administrators, the activation key is
        reset to the string constant ``RegistrationProfile.ACTIVATED``
        after successful activation.

        Args:
            ``activation_key`` SHA1 hash string.
            ``request`` activate view request needed just for passing it to
                ``callback``.
            ``callback`` callable doing the activation process.
            ``kwargs`` extra key arguments for callback.
        Returns:
            Two-tuple containing the new user/account instance and ``None`` on
            success, falsy value and message error on failure.
        """
        # Make sure the key we're trying conforms to the pattern of a
        # SHA1 hash; if it doesn't, no point trying to look it up in
        # the database.
        if SHA1_RE.search(activation_key):
            try:
                profile = self.get(activation_key=activation_key)
            except self.model.DoesNotExist:
                return False, _('Your activation key is not valid')
            if not profile.activation_key_invalid():
                account, errors = callback(request, profile, **kwargs)
                if account:
                    profile.activation_key = self.model.ACTIVATED
                    profile.save()
                return account, errors
        return False, _('Your activation key is not valid')

    def create_profile(self, site, email, send_email=True):
        """
        Create a ``RegistrationProfile`` for a given email, and return the
        ``RegistrationProfile``.

        The activation key for the ``RegistrationProfile`` will be a SHA1 hash,
        generated from a combination of the given email and a random salt.

        Args:
            ``site`` current site object, needed for sending activation email
            ``email`` string represeting email for the new profile
            ``send_email`` boolean value which determines whether email will be
                sent or not. Default value is ``True``.
        Returns:
            The new ``RegistrationProfile`` instance.
        """
        salt = sha_constructor(str(random.random())).hexdigest()[:5]
        if isinstance(email, unicode):
            email = email.encode('utf-8')
        activation_key = sha_constructor(salt+email).hexdigest()
        profile = self.create(email=email, activation_key=activation_key)
        if profile and send_email:
            profile.send_activation_email(site)
        return profile

    @staticmethod
    def delete_expired(queryset=None):
        """
        Deletes expired ``RegistrationProfile`` objects based on settings
        ``ACCOUNT_ACTIVATION_DAYS`` and current date.

        Args:
            ``queryset`` If a queryset is provided then only profiles in the
                given queryset will be tested, if no value is provided then all
                profiles will be tested. Default value is ``None``.
        """
        expiration_date = datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        if queryset is None:
            queryset = RegistrationProfile.objects.all()
        for profile in queryset:
            if (profile.reg_time + expiration_date) <= datetime.datetime.now():
                profile.delete()

    @staticmethod
    def delete_activated(queryset=None):
        """
        Deletes already activated ``RegistrationProfile`` objects based on
        activation key and ``ACTIVATED`` value comparison.

        Args:
            ``queryset`` If a queryset is provided then only profiles in the
                given queryset will be tested, if no value is provided then all
                profiles will be tested. Default value is ``None``.
        """
        if queryset is None:
            queryset = RegistrationProfile.objects.all()
        for profile in queryset:
            if profile.activation_key == RegistrationProfile.ACTIVATED:
                profile.delete()

class RegistrationProfile(models.Model):
    """
    A simple profile which stores an activation key and email for use during
    user account registration.

    Generally, you will not want to interact directly with instances
    of this model; the provided manager includes methods
    for creating and activating new accounts, as well as for cleaning
    invalid profiles.

    While it is possible to use this model as the value of the
    ``AUTH_PROFILE_MODULE`` setting, it's not recommended that you do
    so. This model's sole purpose is to store data temporarily during
    account registration and activation.

    """
    ACTIVATED = u"ALREADY_ACTIVATED"

    email = models.EmailField()
    activation_key = models.CharField(_('activation key'), max_length=40)
    reg_time = models.DateTimeField(_('registration time'), auto_now_add=True)

    objects = RegistrationManager()

    class Meta:
        verbose_name = _('registration profile')
        verbose_name_plural = _('registration profiles')

    def __unicode__(self):
        return u"Registration information for %s" % self.email

    def activation_key_invalid(self):
        """
        Determine whether this ``RegistrationProfile``'s activation
        is invalid.

        Key expiration is determined by a two-step process:

        1. If the user has already activated, the key will have been
           reset to the string constant ``ACTIVATED``. Re-activating
           is not permitted, and so this method returns ``True`` in
           this case.

        2. Otherwise, the date the user signed up is incremented by
           the number of days specified in the setting
           ``ACCOUNT_ACTIVATION_DAYS`` (which should be the number of
           days after signup during which a user is allowed to
           activate their account); if the result is less than or
           equal to the current date, the key has expired and this
           method returns ``True``.

        Returns:
            Boolean value.
        """
        return self.activation_key_already_activated() or \
            self.activation_key_expired() or False
    activation_key_invalid.boolean = True

    def activation_key_already_activated(self):
        """
        Determines whether this ``RegistrationProfile``'s activation key has
        been activated.

        Returns:
            Boolean value.
        """
        return self.activation_key == self.ACTIVATED
    activation_key_already_activated.boolean = True

    def activation_key_expired(self):
        """
        Determines whether this ``RegistrationProfile``'s activation key has
        expired.

        Returns:
            Boolean value.
        """
        expiration_date = datetime.timedelta(days=settings.ACCOUNT_ACTIVATION_DAYS)
        return (self.reg_time + expiration_date) <= datetime.datetime.now()
    activation_key_expired.boolean = True

    def send_activation_email(self, site):
        """
        Send an activation email to the user associated with this
        ``RegistrationProfile``.

        The activation email will make use of two templates:

        ``registration/activation_email_subject.txt``
            This template will be used for the subject line of the
            email. Because it is used as the subject line of an email,
            this template's output **must** be only a single line of
            text; output longer than one line will be forcibly joined
            into only a single line.

        ``registration/activation_email.txt``
            This template will be used for the body of the email.

        These templates will each receive the following context
        variables:

        ``activation_key``
            The activation key for the new account.

        ``expiration_days``
            The number of days remaining during which the account may
            be activated.

        ``site``
            An object representing the site on which the user
            registered; depending on whether ``django.contrib.sites``
            is installed, this may be an instance of either
            ``django.contrib.sites.models.Site`` (if the sites
            application is installed) or
            ``django.contrib.sites.models.RequestSite`` (if
            not). Consult the documentation for the Django sites
            framework for details regarding these objects' interfaces.

        Args:
            ``site`` the above explained ``site``
        """
        email_type = getattr(settings, 'REGISTRATION_EMAIL_TYPE', 'TEXT')
        ctx_dict = getattr(settings, 'REGISTRATION_EMAIL_CTXT', {}).copy()

        ctx_dict.update({'activation_key': self.activation_key,
                         'expiration_days': settings.ACCOUNT_ACTIVATION_DAYS,
                         'site': site})

        subject = render_to_string('registration/activation_email_subject.txt',
                                   ctx_dict)
        # Email subject *must not* contain newlines
        subject = ''.join(subject.splitlines())

        if email_type.upper() == "HTML":
            return self._sent_tml_email(subject, ctx_dict)
        elif email_type.upper() == "MULTI":
            return self._send_multi_email(subject, ctx_dict)

        message = render_to_string('registration/activation_email.txt',
                                   ctx_dict)

        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [self.email])

    def _send_html_email(self, subject, context):
        message = render_to_string('registration/activation_email.html',
                                   context)

        send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [self.email])

    def _send_multi_email(self, subject, context):
        text_content = render_to_string('registration/activation_email.txt',
                                        context)
        html_content = render_to_string('registration/activation_email.html',
                                        context)
        msg = EmailMultiAlternatives(subject,
                                     text_content,
                                     settings.DEFAULT_FROM_EMAIL,
                                     [self.email])
        msg.attach_alternative(html_content, "text/html")
        msg.send()


from dataclasses import dataclass

@dataclass
class Cup:
    size: int
    quantity: int

    def status(self):
        space = self.size - self.quantity
        return space

    # methods within a class can refer to each other
    def fill(self, litres):
        if self.status() > 0:
            self.quantity += litres
        return self.quantity

cup = Cup(100, 50)
print(cup.status())
cup.fill(50)
cup.fill(10)
print(cup.status())


#!/usr/bin/env python3
# coding=utf-8

import socket
import struct
import time
import netifaces
import zlib
import threading
import colorsys
import argparse
import random
import fnmatch

import winreg as wr

def hsv2rgb(h,s,v):
    return tuple(round(i * 255) for i in colorsys.hsv_to_rgb(h/360.,s/100.,v/100.))

def send(msg, port, baddr, saddr):
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # t = threading.Thread(target=recv, args=(sock,))
    # t.daemon = True
    # t.start()

    sock.sendto(msg, (saddr, port))

def get_connection_name_from_guid(iface_guids):
    iface_names = ['(unknown)' for i in range(len(iface_guids))]
    reg = wr.ConnectRegistry(None, wr.HKEY_LOCAL_MACHINE)
    reg_key = wr.OpenKey(reg, r'SYSTEM\\CurrentControlSet\\Control\\Network\\{4d36e972-e325-11ce-bfc1-08002be10318}')
    for i in range(len(iface_guids)):
        try:
            reg_subkey = wr.OpenKey(reg_key, iface_guids[i] + r'\\Connection')
            iface_names[i] = wr.QueryValueEx(reg_subkey, 'Name')[0]
        except FileNotFoundError:
            pass
    return iface_names

def get_driver_name_from_guid(iface_guids):
    iface_names = ['(unknown)' for i in range(len(iface_guids))]
    reg = wr.ConnectRegistry(None, wr.HKEY_LOCAL_MACHINE)
    reg_key = wr.OpenKey(reg, r'SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e972-e325-11ce-bfc1-08002be10318}')
    for i in range(wr.QueryInfoKey(reg_key)[0]):
        subkey_name = wr.EnumKey(reg_key, i)
        try:
            reg_subkey = wr.OpenKey(reg_key, subkey_name)
            guid = wr.QueryValueEx(reg_subkey, 'NetCfgInstanceId')[0]
            try:
                idx = iface_guids.index(guid)
                iface_names[idx] = wr.QueryValueEx(reg_subkey, 'DriverDesc')[0]
            except ValueError:
                pass
        except FileNotFoundError:
            pass
        except PermissionError:
            pass
    return iface_names

def get_addr(pattern):
    ifs = netifaces.interfaces()
    devs = get_driver_name_from_guid(ifs)
    cons = get_connection_name_from_guid(ifs)
    ifs_w = zip(ifs, devs, cons)

    for ifw in ifs_w:
        if_addrs = netifaces.ifaddresses(ifw[0])
        if netifaces.AF_INET in if_addrs:
            for if_addr in if_addrs[netifaces.AF_INET]:
                if len(fnmatch.filter(ifw, pattern)) > 0 or fnmatch.fnmatch(if_addr['addr'], pattern):
                    return if_addr['addr'], if_addr['broadcast'] if 'broadcast' in if_addr else if_addr['addr']

def print_networks():
    ifs = netifaces.interfaces()
    devs = get_driver_name_from_guid(ifs)
    cons = get_connection_name_from_guid(ifs)
    ifs_w = zip(ifs, devs, cons)

    for ifw in ifs_w:
        if_addrs = netifaces.ifaddresses(ifw[0])
        if netifaces.AF_INET in if_addrs:
            print(f"Interface: {ifw[2]}\nUID: {ifw[0]}\nAdapter name: {ifw[1]}")
            for if_addr in if_addrs[netifaces.AF_INET]:
                print(f"   Address: {if_addr['addr']}")
            print("")

def str2col(s):
    if s[0] == "#":
        s = s[1:]
    col_n = int(s, 16)
    col_b = col_n.to_bytes(4, byteorder='big')
    return col_b[1:]

if __name__ == "__main__":
    # screw it, too many arguments, time for argparse
    parser = argparse.ArgumentParser(description='cat artnet packages')
    parser.add_argument('-s', '--shift', help="art net shift", default=0, type=int)
    parser.add_argument('-n', '--number', help="diodes number", default=34, type=int)
    parser.add_argument('-u', '--universe', help="universe", default=1, type=int)
    parser.add_argument('-d', '--delay', help="delay", type=float, default=-1)
    parser.add_argument('-N', '--network', help="IP address beginning of destination network", default="192.*")
    parser.add_argument('-A', '--address', help="IP address of destination", default="")
    parser.add_argument('-P', '--port', help="art-net port", default=6454, type=int)
    parser.add_argument('-t', '--type', help="test type",
     choices=['default', 'full_range', "rainbow", "chain", "chain_reversed"], default="default")
    parser.add_argument('--rainbow_delay', help="delay in the rainbow params, in ms, will be rounded to ticks",
     type=int, default=0x50)
    parser.add_argument('--rainbow_time_step', help="increment in hue in each time step",
     type=int, default=1)
    parser.add_argument('--rainbow_length_step', help="increment in hue in each length step",
     type=int, default=1)
    parser.add_argument('--rainbow_start_color', help="color of 1st pixel at the T=0", default="#880000")
    parser.add_argument('--rainbow_tint_color', help="color of tint", default="#000000")
    parser.add_argument('--rainbow_tint_value', help="tint weight col_final = col (255-tint_value) + tint_col * tint_value",
     default=0, type=int)
    parser.add_argument('--rainbow_tint_hsv', help="use hsv tint equation", action='store_true')
    args = parser.parse_args()

    if args.address != "":
    	addr_out, bc_out = args.address, args.address
    else:
    	addr_out, bc_out = get_addr(args.network)
    print(addr_out, bc_out)

    delay = args.delay
    if delay <= 0:
        if args.type == "rainbow":
            delay = 5
        elif args.type.startswith("chain"):
            delay = .4
        else:
            delay = 1/20.

    rainbow_id = random.randint(1,255)

    i=0
    led_len = args.number
    inc = max(1, 360/led_len)
    while True:
        i+=1
        buf_pl = b"\x00" * (args.shift)
        if args.type == "rainbow":
            #           |RBW|ID |Delay  |T step |L step |start color|tint color |tint value|tint type
            buf_pl += b"\x03"+\
                        rainbow_id.to_bytes(1, byteorder='little')+\
                        args.rainbow_delay.to_bytes(2, byteorder='big')+\
                        args.rainbow_time_step.to_bytes(2, byteorder='big')+\
                        args.rainbow_length_step.to_bytes(2, byteorder='big')+\
                        str2col(args.rainbow_start_color)+\
                        str2col(args.rainbow_tint_color)+\
                        args.rainbow_tint_value.to_bytes(1, byteorder='little')+\
                        (b"\xff" if args.rainbow_tint_hsv else b"\x00")
        elif args.type.startswith("chain"):
            buf_pl += b"\x02" if args.type == "chain_reversed" else b"\x01"
            buf_pl += bytes(hsv2rgb(random.randint(0, 359),100,50))
        else:
            buf_pl += b"\x00"  # Straight DMX
            for l in range(0, led_len):
                buf_pl += bytes(hsv2rgb(
                    (i + (l*inc))%360,
                    100 if args.type == "default" or i%240 > 120 else 0,
                    50 if args.type == "default" else abs(i%120 - 60)/0.6))
        buf_pl += b"\x00" * (512 - len(buf_pl))
        #       |marker    |opcode |proto  |seq|phy|univ   |len    |payload (512 bytes)
        buf = b"Art-Net\x00\x00\x50\x00\x0e\x00\x00"+args.universe.to_bytes(2, byteorder='little')+b"\x02\x00" + \
        buf_pl
        send(buf, args.port, addr_out, bc_out)
        time.sleep(delay)

# buf = b'\xa4l\xf1[\xed\xde\x01\x02\x00\x00\x00\x00\x00\x00' # \x08\xf1\x88l
# buf += socket.htonl(zlib.crc32(buf, 0xCA7ADDED)).to_bytes(4,'little')
# print(buf)


import pandas as pd
from s3_service import get_file
from s3_service import load_data
from datetime import datetime
import io

def update_data():
    column_name: str = f"{datetime.today().date()}"
    rates_data: io.StringIO = get_file("data.csv")
    rate_today: io.StringIO = get_file()

    rates_data_df = None
    rate_today_df: pd.DataFrame = pd.read_csv(rate_today, index_col=0, header=0)

    if rates_data is None:
        rates_data_df = pd.DataFrame(rate_today_df.rename(columns={'rates': column_name}))
    else:
        rates_data_df = pd.read_csv(rates_data, index_col=0, header=0)
        rates_data_df[column_name] = rate_today_df['rates']

    load_data(
        rates_data_df,
        add_index=True,
        key="data.csv"
    )


import socket
import multiprocessing as mp

def handle_connection(connection, address):
    while True:
        data = connection.recv(1024)
        if data:
            print('Data received:', data)
            connection.sendall(data)
        else:
            print('No more data from', address)
            connection.close()
            break

def create_echo_server():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
    sock.bind(('localhost', 8000))
    sock.listen(1)
    try:
        while True:
            connection, address = sock.accept()
            print('Connection from', address)
            process = mp.Process(target=handle_connection, args=(connection, address))
            process.start()

    except KeyboardInterrupt:
        print('Shutting down...')
        sock.close()

if __name__ == '__main__':
    create_echo_server()


'''
################################## client.py #############################
#
################################## client.py #############################
'''
import random
import time
import grpc
import datastore_pb2
import datastore_pb2_grpc
import argparse
import uuid
import rocksdb

PORT = 3000

'''
Request Generator
'''

def make_data(key, data):
  return datastore_pb2.Request(
      key=key,
      data=data)

def generate_request():
    tasks = [
            make_data('100', 'Forrest'),
            make_data('101', 'Chen'),
            make_data('102', 'is'),
            make_data('103', 'a'),
            make_data('104', 'student'),
    ]
    for task in tasks:
        print("Sending key %s and data %s" % (task.key, task.data))
        yield task
        time.sleep(random.uniform(0.5, 1.0))

class DatastoreClient():

    def __init__(self, host='0.0.0.0', port=PORT):
        self.channel = grpc.insecure_channel('%s:%d' % (host, port))
        self.stub = datastore_pb2.DatastoreStub(self.channel)
        self.db = rocksdb.DB("hw2_follower.db", rocksdb.Options(create_if_missing=True))

    '''
    Bidirectional streaming RPC
    Client send streaming data for server to take action.
    '''

    def put(self):
        resps = self.stub.put(generate_request())
        if resps == None:
            print("Empty!")
        else:
            for resp in resps:
                print("Master stored key %s and value %s" % (resp.key, resp.data))

    def delete(self):
        resps = self.stub.delete(generate_request())
        if resps == None:
            print("Empty!")
        else:
            for resp in resps:
                print("Master deleted key %s and value %s" % (resp.key, resp.data))

    '''
    A server-to-client streaming RPC
    Master send sterming data for follower to replicate
    '''

    def replicator(self, action):
        resp = self.stub.replicator(datastore_pb2.PullRequest(action=action))
        if action == 'put':
            for action in resp:
                self.db.put(action.key.encode(), action.data.encode())
                if action.data == self.db.get(action.key.encode()).decode():
                    print("Follower stored key %s and value %s" % (action.key, action.data))
        elif action == 'delete':
            for action in resp:
                self.db.delete(action.key.encode())
                if self.db.get(action.key.encode()) == None:
                    print("Follower deleted key %s and value %s" % (action.key, action.data))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("host", help="display a square of a given number")
    args = parser.parse_args()
    print("Client is connecting to Server at {}:{}...".format(args.host, PORT))
    client = DatastoreClient(host=args.host)

    '''
    Test Case
    '''
    client.put()
    client.replicator("put")
    client.delete()
    client.replicator("delete")

if __name__ == "__main__":
    main()


import datetime
import numbers
import re
from pytest import mark, raises

from wand.version import (MAGICK_VERSION, MAGICK_VERSION_INFO,
                          MAGICK_VERSION_NUMBER, MAGICK_RELEASE_DATE,
                          MAGICK_RELEASE_DATE_STRING, QUANTUM_DEPTH,
                          configure_options, fonts, formats)

def test_version():
    """Test version strings."""
    match = re.match(r'^ImageMagick\s+\d+\.\d+\.\d+(?:-\d+)?', MAGICK_VERSION)
    assert match
    assert isinstance(MAGICK_VERSION_INFO, tuple)
    assert (len(MAGICK_VERSION_INFO) ==
            match.group(0).count('.') + match.group(0).count('-') + 1)
    assert all(isinstance(v, int) for v in MAGICK_VERSION_INFO)
    assert isinstance(MAGICK_VERSION_NUMBER, numbers.Integral)
    assert isinstance(MAGICK_RELEASE_DATE, datetime.date)
    date_strings = (MAGICK_RELEASE_DATE.strftime('%Y-%m-%d'),
                    MAGICK_RELEASE_DATE.strftime('%Y%m%d'))
    assert MAGICK_RELEASE_DATE_STRING in date_strings

def test_quantum_depth():
    """QUANTUM_DEPTH must be one of 8, 16, 32, or 64."""
    assert QUANTUM_DEPTH in (8, 16, 32, 64)

def test_configure_options():
    assert 'RELEASE_DATE' in configure_options('RELEASE_DATE')
    with raises(TypeError):
        configure_options(0xDEADBEEF)

def test_fonts():
    font_list = fonts()
    if not font_list:
        mark.skip('Fonts not configured on system')
    else:
        first_font = font_list[0]
        first_font_part = first_font[1:-1]
        assert first_font in fonts('*{0}*'.format(first_font_part))
    with raises(TypeError):
        fonts(0xDEADBEEF)

def test_formats():
    xc = 'XC'
    assert formats(xc) == [xc]
    with raises(TypeError):
        formats(0xDEADBEEF)


from lxml import etree as et
from functools import lru_cache

from . import model as doc

SEPARATOR = " " + chr(8212) + " "

def _node_to_element_html(node: doc.ContentsTableNode) -> et.ElementBase:
    try:
        display_text = SEPARATOR.join((node.heading.ordinate, node.heading.title))
    except (AttributeError, TypeError):
        display_text = node.heading.ordinate
    li = et.Element("li", {"id": node.locator})
    a = et.SubElement(li, "a")
    a.attrib["href"] = node.href
    if node.children:
        # TODO: change this to
        #  https://stackoverflow.com/questions/10768451/inline-svg-in-css
        et.SubElement(
            a,
            "img",
            attrib={
                "class": "lex-fold-icon",
                "src": "/static/icons/minus-square-o.svg",
                "alt": "toggle",
            },
        ).tail = display_text
        li.attrib["class"] = "toc-node"
        ul = et.SubElement(li, "ul", {"class": "w3-ul"})
        for child in node.children:
            ul.append(_node_to_element_html(child))
    else:
        a.text = display_text
        li.attrib["class"] = "toc-leaf"
    return li

def _node_to_element_xml(node: doc.ContentsTableNode) -> et.ElementBase:
    if node.children:
        e = et.Element("Container")
    else:
        e = et.Element("Leaf")
    e.attrib["Name"] = node.heading.ordinate
    if getattr(node.heading, "title", None) is not None:
        e.attrib["Title"] = node.heading.title
    if node.href != "#":
        e.attrib["URL"] = node.href
    else:
        e.attrib["URL"] = node.locator
    for child in node.children:
        e.append(_node_to_element_xml(child))
    return e

def node_to_element(node: doc.ContentsTableNode, method="html"):
    if method == "html":
        return _node_to_element_html(node)
    assert method == "xml"
    return _node_to_element_xml(node)

class ContentsTable(doc.ContentsTable):
    def __hash__(self):
        return id(self)

    def embed(self, document_domain, document_id, version):
        href_template = f"/{document_domain}/{document_id}/{{}}/{version}"
        for node in self.table:
            if not node.children:
                href = href_template.format(node.locator)
            else:
                href = "#"
            node.href = href

    @property
    @lru_cache()
    def leaf_to_neighbour(self) -> dict:
        # TODO: Unittest this method: Edge-case: only a single leaf
        def node_2_neighbour(n: doc.ContentsTableNode):
            return {"ordinate": n.heading.ordinate, "id": n.locator}

        leaf_to_node = [node for node in self.table if len(node.children) == 0]
        result = {
            leaf_to_node[0].locator: {
                "left": None,
                "right": node_2_neighbour(leaf_to_node[1]),
            },
            leaf_to_node[-1].locator: {
                "left": node_2_neighbour(leaf_to_node[-2]),
                "right": None,
            },
        }
        result.update(
            {
                node.locator: {
                    "left": node_2_neighbour(leaf_to_node[k - 1]),
                    "right": node_2_neighbour(leaf_to_node[k + 1]),
                }
                for k, node in enumerate(leaf_to_node)
                if 0 < k < len(leaf_to_node) - 1
            }
        )
        return result

    def toccordior(self, version, document_id=None, method="html") -> et.ElementBase:
        version = version if version != "latest" else ""
        document_id = document_id or self.abstract.id_local
        self.embed(self.abstract.domain, document_id, version)
        if method == "html":
            toccordion = et.Element(
                "ul", {"id": "toccordion", "class": "w3-ul w3-medium"}
            )
        else:
            assert method == "xml"
            toccordion = et.Element("TableOfContents")
        for ultimate_parent in self.ultimate_parents():
            toccordion.append(node_to_element(ultimate_parent, method))
        return toccordion

    def locator_to_node(self):
        return {node.locator: node for node in self.table}

    def ultimate_parents(self):
        """Note: side-effect: nesting!!"""
        l2n = self.locator_to_node()
        for node in self.table:
            if len(node.children) == 0:
                continue
            if type(node.children[0]) == doc.ContentsTableNode:
                # self is already nested.
                for child in node.children:
                    del l2n[child.locator]
            elif type(node.children[0]) is str:
                node.children = [l2n.pop(locator) for locator in node.children]
        # list with only ultimate parents and children are
        # ContentsTableNode instances
        return l2n.values()