import os

import joblib

from etl.data_loader import DataLoader
from etl.feature_engineer import TitanicFeatureEngineer
from model.dummy_classifier import DummyModel
from model.gradient_boosting import GradientBoostingModel
from model.kfold_cross_validator import KFoldValidator
from model.random_forest import RandomForestModel

class Train:
    def __init__(self, algorithm):
        self.algorithm = algorithm

    def train_model(self, train_file_path, test_file_path):
        if self.algorithm == "Dummy":
            model = DummyModel().get_model()
        elif self.algorithm == "RandomForest":
            model = RandomForestModel().get_model()
        elif self.algorithm == "GradientBoosting":
            model = GradientBoostingModel().get_model()
        else:
            raise ValueError(f"Invalid algorithm name: {self.algorithm}")

        titanic_dl = DataLoader()
        titanic_dl.load_data(train_file_path, test_file_path)
        titanic_fe = TitanicFeatureEngineer(
            titanic_dl.get_train_df(), titanic_dl.get_test_df()
        )
        titanic_fe.feature_engineering()
        titanic_fe.feature_selection("RFE")
        X, y = titanic_fe.get_features_and_target()

        validator = KFoldValidator(model)
        (
            mean_roc_auc,
            std_roc_auc,
            mean_accuracy,
            std_accuracy,
        ) = validator.calculate_metrics(X, y)

        model.fit(X, y)

        joblib.dump(
            model,
            f"{os.getcwd()}/model/models/{str(type(model).__name__).lower()}.joblib",
        )

        print("------------------------")
        print("Training stage completed")
        print("------------------------")
        print("Mean ROC-AUC:", mean_roc_auc)
        print("Standard Deviation of ROC-AUC:", std_roc_auc)
        print("Mean Accuracy:", mean_accuracy)
        print("Standard Deviation of Accuracy:", std_accuracy)


from __future__ import print_function
from past.builtins import cmp
from builtins import range
from builtins import object
import sys,os
os.chdir(os.path.dirname(os.path.abspath(__file__))) # add working directory
sys.path.append(os.path.abspath("../util")) # add relative path

from advutils.threader import api, Designator, IterDecouple

if __name__ == "__main__":

    def worker(q):
        while not q.empty():
            next_job = q.get()
            print('Processing job:', next_job.description)

    class Job(object):
        def __init__(self, priority, description):
            self.priority = priority
            self.description = description
            print('New job:', description)
        def __cmp__(self, other):
            return cmp(self.priority, other.priority)

    #api.spawn = True
    #q = PriorityQueue()
    q = Designator()
    # https://pymotw.com/2/Queue/
    q.put( Job(10, 'Low-level job 1') )
    q.put( Job(1, 'Important job 1') )
    q.put( Job(10, 'Low-level job 2') )
    q.put( None)
    q.put( Job(3, 'Mid-level job') )
    q.put( Job(1, 'Important job 2') )
    q.put( Job(10, 'Low-level job 3') )
    """
    print len(q)
    for i in q:
        print i.description
    print len(q)
    """
    import multiprocessing as m
    import threading as th

    #processes = [m.Process(target=worker,args=(q,)) for i in xrange(3)]
    processes = [th.Thread(target=worker,args=(q,)) for i in range(3)]
    for p in processes:
        p.start()
    for p in processes:
        p.join()
    #print len(q)

if __name__ == '__main__':
    from multiprocessing import freeze_support
    freeze_support()

    def handler_timeout(signum, frame):
        raise Exception("Function taking too long")

    #import signal
    #signal.signal(signal.SIGALRM, handler_timeout)

    def demo_decouple_iter():
        """

        :return:
        """
        import time, random

        class SimulateProcess(object):
            def __init__(self, value, time_processing):
                self.value = value
                self.time_processing = time_processing
                self._creation_time = time.time()

            def process(self):
                time.sleep(self.time_processing)
                return self

            __call__ = process

            def __str__(self):
                return str(self.value)

        def simulate_iterable(max_time=1, no_itmes = 10):
            """
            Simulates iterable that gives data un-evenly in time,
            fast or really slow in some cases.

            :param max_time: maximum random time to simulate process
            :param no_itmes: number of processes to create
            :return: iterator
            """
            for order in range(no_itmes):
                tp = random.random() * max_time
                yield SimulateProcess(order, tp), tp

        def simulate_consumer(myiter, call = False, stop_in= None, test_name = None):
            """
            Simulates iterator consumption. The idea is to simulate a process
            that wants to get data as fast as possible, faster is better.

            :param myiter: iterator of SimulateProcess instances
            :param buffsize: buffer size
            :return: (No_items, time_ideal, time_normal_for, time_test_for,
                    time_spared_normal, time_loosed_ideal)
            """
            if test_name is None:
                test_name = type(myiter)
            name = " ({})".format(test_name)
            print("Testing for loop {}".format(name))

            worker_times = [0] # time of each worker, 0 is added to simulate empty
            times = [] # time for each epoch
            time_test_for = time.time()  # real process time
            time_epoch_start = time.time() # epoch start time
            for i, (task, time_processing) in enumerate(myiter):
                if stop_in is not None and i == stop_in:
                    myiter.close()
                if call:
                    task()
                time_epoch_end = time.time() # epoch finish time
                time_epoch = time_epoch_end - time_epoch_start # epoch time
                worker_times.append(time_processing) # append real worker time
                times.append(time_epoch) # append for loop epoch time
                print("value: {}, epoch time: {}, worker time: {}"
                      "".format(task,time_epoch,time_processing))
                time_epoch_start = time_epoch_end # update epoch start time

            No_items = len(times) # Number of items
            time_test_for = time.time() - time_test_for
            time_ideal = max(worker_times) # ideal process time
            time_normal_for = sum(worker_times) # not decoupled time
            time_spared_normal = time_normal_for - time_test_for # spared time from normal for loop
            time_loosed_ideal = time_test_for - time_ideal # loosed time from ideal
            print("processed items: ", No_items)
            print("ideal process time: ", time_ideal)
            print("normal for loop time: ", time_normal_for)
            print("tested for loop time{}: ".format(name), time_test_for)
            print("time spared (from normal for): ", time_spared_normal)
            print("loosed time (from ideal):", time_loosed_ideal)
            return (No_items, time_ideal, time_normal_for, time_test_for,
                    time_spared_normal, time_loosed_ideal)

        api.spawn = False
        myiter = list(simulate_iterable())
        stop_in = 3
        decoupled = IterDecouple(myiter, buffsize=0)
        decoupled.call_func = lambda x: (x[0](), x[1])
        decoupled.buffsize = 0 # unlimited

        start_before = 2
        print("\ntest: detached for loop is started {} seconds before\n"
              "".format(start_before))
        decoupled.start()
        time.sleep(start_before)
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_ideal < time_test_for < time_normal_for

        processes = 5
        print("\ntest: detached for loop is started in normal for loop with "
              "{} processes\n".format(processes))
        decoupled.processes = processes
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_ideal < time_test_for < time_normal_for

        print("\ntest: detached for loop is started {} seconds before in "
              "normal for loop with {} processes\n".format(start_before, processes))
        decoupled.processes = processes
        decoupled.start()
        time.sleep(start_before)
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)
            assert time_test_for < time_normal_for # the ideal should be 0

        # disable processes
        decoupled.processes = None

        print("\ntest: detached for loop is started in normal for loop\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(decoupled, stop_in = stop_in)
        if stop_in is None:
            assert No_items == len(myiter)

        print("\ntest: simple list\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(myiter, call=True) # tests: simple list
        if stop_in is None:
            assert No_items == len(myiter)

        print("\ntest: empty iteration\n")
        (No_items, time_ideal, time_normal_for, time_test_for,
        time_spared_normal, time_loosed_ideal) = simulate_consumer(IterDecouple([]))

        """
        # tests repeatability and that processes do not block
        i = 0
        while True:
            signal.alarm(5)  # timeout of 5 seconds
            i+=1
            print("Try",i)
            try:
                simulate_consumer([]) # tests: empty iteration
            except Exception as e:
                signal.alarm(0)
                raise e
        """

    demo_decouple_iter()


import cv2
import numpy as np

image_index = 0

def getImage():
    img = cv2.imread('images/image2.jfif')
    return img

def showImage(img):
    global image_index
    cv2.imshow(f'Image#{image_index}', img)
    image_index += 1

def getMask(img):
    mask = np.zeros(
        shape=img.shape[:2],
        dtype='uint8'
    )
    return mask

def getCircleMask(img, mask):
    circle_mask = cv2.circle(
        img=mask,
        center=(img.shape[1]//2, img.shape[0]//2),
        radius=70,
        color=255,
        thickness=-1
    )
    return circle_mask

def getMaskedImage(img, mask):
    masked_image = cv2.bitwise_and(
        src1=img,
        src2=img,
        mask=mask
    )
    return masked_image

def main():
    img = getImage()
    showImage(img)

    mask = getMask(img)
    showImage(mask)

    circle_mask = getCircleMask(img, mask)
    showImage(circle_mask)

    masked_image = getMaskedImage(img, mask)
    showImage(masked_image)

    cv2.waitKey(0)

if __name__ == "__main__":
    main()


import json
import socket
from threading import Thread
import time
from libraries.udp_server_multi_client import UDPServerMultiClient

class Gateway():

    __address: tuple[str, int]
    __client: dict = {
        "socket": socket.socket,
        "address": tuple[str, int],
        "conn": socket.socket,
        "is_connected": False
    }
    __buffer_size: int = 1024
    __drone_server: UDPServerMultiClient
    __is_client_connected: bool = False

    def __init__(self, gateway_ip: str, gateway_port: int, max_drones: int) -> None:
        self.__drone_server = UDPServerMultiClient(
            gateway_ip, gateway_port, max_drones)
        self._setup_gateway_client(gateway_ip, gateway_port)

    def _setup_gateway_client(self, gateway_ip: str, gateway_port: int) -> None:
        """
        Setup the connection between the gateway and the client.
        """
        self.__address = (gateway_ip, gateway_port)
        self.__client["socket"] = socket.socket(
            family=socket.AF_INET, type=socket.SOCK_STREAM)
        self.__client["socket"].bind(self.__address)

    def __connect_client(self) -> None:
        self.__client["socket"].listen()
        self.__client["conn"], self.__client["address"] = self.__client["socket"].accept()
        if (not self.__is_client_connected):
            print(f"Connected to Client: {self.__client['address']}")
            self.__is_client_connected = True

    def main_loop(self) -> None:
        """
        Listen if there are orders to be shipped.
        """
        self.__connect_client()
        drone_server_thread = Thread()
        client_thread = Thread()

        try:
            while True:
                # creating the drone listener thread
                if self.__is_client_connected:
                    if not drone_server_thread.is_alive():
                        drone_server_thread = Thread(
                            target=self.__drone_server.thread_request_handle)
                        drone_server_thread.daemon = True
                        drone_server_thread.start()
                        time.sleep(0.5)

                # handling the client
                if not client_thread.is_alive():
                    client_thread = Thread(self.client_handling())
                    client_thread.daemon = True
                    client_thread.start()

        except ConnectionResetError:
            print(f"Client {self.__client['address']} disconnected!")
            self.__is_client_connected = True

    def client_handling(self) -> None:
        """
        Handles the client
        """
        self.send_drone_information()

        data = self.__client["conn"].recv(self.__buffer_size)
        if not data:
            return
        order = eval(data.decode("utf-8"))
        if not self._check_order(order):
            self._send_message_to_client("Error: wrong order syntax")
        elif not "update" in order:
            print(f"Clinet request: {order}")
            self._process_order(order)

    def send_drone_information(self) -> bool:
        """
        Send the drones id that are free to the client.
        """
        free_drones = self.__drone_server.get_free_drones()
        drones = []
        for i in free_drones:
            drones.append(i["id"])

        try:
            self.__client["conn"].send(str(drones).encode())
            return True
        except:
            print(f"Client {self.__client['address']} disconnected!")
            self.__is_client_connected = False
            return False

    def _send_message_to_client(self, message: str) -> None:
        """
        Send a message to the client.
        """
        self.__client["conn"].send(message.encode())

    def _process_order(self, order: dict) -> None:
        """
        Process the order and send the drone to the order destination.
        """
        self.__drone_server.thread_send_message(
            order["order_destination"], self.__drone_server.get_drone_address_by_id(order["drone_id"]))

    def _check_order(self, order: dict) -> bool:
        """
        Check if the order is valid and if the drone is available.
        """
        if "update" in order:
            return True

        # check if the order syntax is valid
        if not all(keys in order for keys in ("drone_id", "order_destination")):
            print("Wrong order syntax")
            return False

        # check if the drone exists
        if order["drone_id"] not in self.__drone_server.get_drones():
            print("The selected drone does not exist")
            return False

        # check if the drone is free
        drone_id_list = []
        for i in self.__drone_server.get_free_drones():
            drone_id_list.append(i["id"])
        if order["drone_id"] not in drone_id_list:
            print("The selected drone is not free")
            return False

        return True

    def close_client_connection(self) -> None:
        """
        Close the connection between the gateway and the client.
        """
        self.__client["socket"].close()

if __name__ == '__main__':
    with open('setup.json') as json_file:
        data = json.load(json_file)
        gateway = Gateway("", int(data['gateway']["port"]), data["max_drones"])
        gateway.main_loop()

        gateway.close_client_connection()


"""Routines which are useful to create and plot graphs
"""
from typing import Set, Tuple, Optional, List, Any, Text

from numpy import abs, ndarray
from numpy import random as rand
from math import factorial
import matplotlib.pyplot as plt
from seaborn import heatmap

import networkx as nx
from networkx.algorithms import bipartite
from networkx.generators.random_graphs import (
    fast_gnp_random_graph,
    newman_watts_strogatz_graph,
)

def generate_graph(
    n_nodes: int, n_edges: int, n_edge_max: int = 5, seed: Optional[int] = None
) -> Set[Tuple[int]]:
    """Creates random edges for graph.

    Node labels are from 0 to n_nodes - 1

    Arguments:
        n_nodes: Number of nodes.
        n_edges: Number of edges.
        n_edge_max: Maximal number of edges for graph.
        seed: Random seed to set before graph is generated.

    Todo:
        What about connected / unconnected graphs?
        Am I creating the right graph?
        Think about ill input conditions.
    """
    assert n_edge_max > 0
    assert n_nodes * n_edge_max // 2 > n_edges

    if seed:
        rand.seed(42)

    nodes = {n: 0 for n in range(n_nodes)}
    edges = set()

    while len(edges) < n_edges:
        choices = [n for n, count in nodes.items() if count <= n_edge_max]
        e1, e2 = (int(e) for e in rand.choice(choices, size=2, replace=False))
        edge = (e2, e1) if e2 < e1 else (e1, e2)
        if edge not in edges:
            edges.add(edge)
            nodes[e1] += 1
            nodes[e2] += 2

    return edges

def get_plot(
    graph: Set[Tuple[int]],
    color_nodes: Optional[List[int]] = None,
    show_plot: bool = False,
    backend="mpl",
    seed: int = 42,
    directed: bool = False,
):
    """Creates graph plot from edge set.

    Arguments:
        graph: The graph to plot.
        color_nodes: Colors given nodes green, default is white.
        show_plot: Wether or not the plot should be rendered.
        backend: The plotting backend. Either "mpl" or "bokeh".
        directed: Plot graph as a directed graph
    """
    if backend == "mpl":
        fig = get_plot_mpl(
            graph,
            color_nodes=color_nodes,
            show_plot=show_plot,
            seed=seed,
            directed=directed,
        )
    elif backend == "bokeh":
        if directed:
            print("Directed graph not supported for bokeh")
        fig = get_plot_bokeh(graph, color_nodes=color_nodes, show_plot=show_plot)
    else:
        raise KeyError("Only implemented matplotlib ('mpl') or ")

    return fig

def _flatten(obj: Tuple[Any]) -> Tuple[Any]:
    """Flattens nested tuple of tuple to just a tuple.

    Returns iterator.

    Follows:
    https://symbiosisacademy.org/tutorial-index/python-flatten-nested-lists-tuples-sets/
    """
    for item in obj:
        if isinstance(item, tuple):
            yield from _flatten(item)
        else:
            yield item

def _v2str(v: Tuple[int]) -> str:
    """Maps nested tuple of ints to flat string.

    String is joined on '' if v_max < 9 else ','
    """
    if isinstance(v, tuple):
        vflat = list(_flatten(v))
        sep = "," if max(vflat) > 9 else ""
        out = sep.join(map(str, vflat))
    else:
        out = str(v)
    return out

def generate_hamming_graph(
    d: int, q: int, v_as_string: bool = False
) -> Tuple[Set[Tuple[str, str]], Text]:
    """Returns edges for hamming graph (caretesian product of complete graphs).

    See also http://mathworld.wolfram.com/HammingGraph.html

    Arguments:
        d: Dimension of graph (number of copies).
        q: Number of nodes for complete graph used in cartesian product.
        v_as_string: Convert vertices to strings of format "v10v11v12...".
    """
    kq = nx.complete_graph(q)
    graph = kq
    for _ in range(d - 1):
        graph = nx.cartesian_product(graph, kq)

    edges = set((_v2str(v1), _v2str(v2)) for v1, v2 in graph.edges)

    if not v_as_string:
        nodes = dict()
        n_nodes = 0
        int_edges = set()
        for v1, v2 in edges:
            if v1 not in nodes:
                nodes[v1] = n_nodes
                n_nodes += 1
            if v2 not in nodes:
                nodes[v2] = n_nodes
                n_nodes += 1

            int_edges.add((nodes[v1], nodes[v2]))
        edges = int_edges

    return edges, f"Hamming({d},{q})"

def generate_bipartite_graph(p: int, q: int) -> Tuple[Set[Tuple[int, int]], Text]:
    """Returns edges of a complete bipartite graph.

    K(4,4) is the fundamental unit of a Chimera graph.

    Arguments:
        p: Number of vertices in set 1
        q: Number of vertices in set 2
    """

    return set(bipartite.complete_bipartite_graph(p, q).edges), f"K({p},{q})"

def generate_erdos_renyi_graph(n: int, p: float) -> Tuple[Set[Tuple[int, int]], Text]:
    """Return edges of a Erdos Renyi graph.

    A generalized random graph.
    Can be disconnected

    The following statements are true on average

    ``
        np < 1 has O(log(n)) clusters
        np = 1 has O(n^2/3) clusters
        np -> c > 1 will cluster to one large component, sub clusters have O(log(n)) vertices

        p < ln(n)/n the graph will be disconnected
        p > ln(n)/n the graph will be connected
    ``

    G(n,p)

    Arguments:
        n: Number of vertices
        p: Probability of edge creation
    """
    G = fast_gnp_random_graph(n, p)
    return set(G.edges), f"G({n},{p})"

def generate_newman_watts_strogatz_graph(
    n: int, k: int, p: float
) -> Tuple[Set[Tuple[int, int]], Text]:
    """Returns edges of a NWS small-world graph.

    NWS graph is a knn ring graph + random connections.
    This is guaranteed to be connected since edges are not removed for shortcuts.
    The WS graph removes edges from the ring when spawning shortcuts.
    Has applications to modeling social networks.

    Arguments:
        n: Number of vertices
        k: Number of nearest neighbors
        p: Probability of spawning new edge on top of base ring

    """
    return set(newman_watts_strogatz_graph(n, k, p).edges), f"WS({n},{k},{p})"

def generate_nn_graph(v: int) -> Tuple[Set[Tuple[int, int]], Text]:
    """Returns edges of a 1 dimensional nearest neighbor graph.

    Arugments:
        v: Number of vertices
    """
    graph = {(i - 1, i) for i in range(1, v)}
    return graph, f"NN({v})"

def generate_corona_graph(k: int, n: int) -> Tuple[Set[Tuple[int, int]], Text]:
    """Returns edges of a n-shortcut 3k-ring graph with broken rotational symmetry.

    Has unique ground state solution with increased connectivity

    Arguments:
        k: number of 3 vertex segments
        n: number of shortcuts. must be < 3*k - 2 for this definition
    """

    if n >= factorial(3 * k - 2):
        raise ValueError(
            "n >= (3k-2)!: More shortcuts than allowed for graph.\n Choose a smaller number."
        )

    # construct base ring
    ring, _ = generate_nn_graph(3 * k)
    ring.add((3 * k - 1, 0))

    # add antigen
    antigen = ((3 * i + 1, 3 * k + i) for i in range(k))
    ring.update(antigen)

    # add shortcuts
    count = 0
    for nidx in range(k):
        for kidx in range(3 * k):
            if count < n:
                start = kidx
                end = (kidx + nidx + 2) % (3 * k)
                ring.add((start, end))
                count += 1
            else:
                break

    return ring, f"C19({k},{n})"

def generate_limited_corona_graph(k: int, n: int):
    """Returns edges of a n-shortcut 3k-ring graph with broken rotational symmetry.

    Has unique ground state solution with increased connectivity
    For limited corona, the shortcuts are added to keep max connections / vertex the smallest

    Arguments:
        k: number of 3 vertex segments
        n: number of shortcuts.
    """

    if n >= factorial(3 * k - 2):
        raise ValueError(
            "n >= (3k-2)!: More shortcuts than allowed for graph.\n Choose a smaller number."
        )

    # construct base ring
    ring, _ = generate_nn_graph(3 * k)
    ring.add((3 * k - 1, 0))

    # add antigen
    antigen = ((3 * i + 1, 3 * k + i) for i in range(k))
    ring.update(antigen)

    # connectivity
    connections = {idx: 2 for idx in range(3 * k)}
    for i in range(k):
        connections[3 * i + 1] += 1

    # add shortcuts
    shortcuts = [
        (kidx, (kidx + nidx + 2) % (3 * k))
        for nidx in range(k)
        for kidx in range(3 * k)
    ]
    count = 0
    minconnections = min(connections.values())
    while count < n:
        for idx, shortcut in enumerate(shortcuts):
            start = shortcut[0]
            end = shortcut[1]
            if connections[start] > minconnections or connections[end] > minconnections:
                deadend = True
                pass
            else:
                ring.add(shortcut)
                shortcuts.pop(idx)
                connections[start] += 1
                connections[end] += 1
                minconnections = min(connections.values())
                count += 1
                deadend = False
                break
        if idx == len(shortcuts) - 1 and deadend:
            minconnections += 1
    return ring, f"lC19({k},{n})"

def get_plot_mpl(
    graph: Set[Tuple[int]],
    color_nodes: Optional[List[int]] = None,
    show_plot: bool = False,
    ax: Optional["Axes"] = None,
    seed: int = 42,
    directed: bool = False,
) -> Optional["Figure"]:
    """Creates matplotlib graph plot from edge set.

    Arguments:
        graph: The graph to plot.
        color_nodes: Colors given nodes green, default is white.
        show_plot: Wether or not the plot should be rendered.
        ax: The axes to plot in
        directed: Plot graph as a directed graph
    """
    if not ax:
        fig, ax = plt.subplots()
    else:
        fig = None

    G = nx.DiGraph(list(graph)) if directed else nx.Graph(list(graph))
    node_color = (
        ["lightgreen" if node in color_nodes else "white" for node in G.nodes]
        if color_nodes
        else "white"
    )

    if seed is not None:
        pos = nx.spring_layout(G, seed=seed)
        # pos = nx.spectral_layout(G)

    nx.draw(
        G,
        pos,
        with_labels=True,
        node_color=node_color,
        font_size=10,
        node_shape="o",
        linewidths=1.0,
        edgecolors="black",
        ax=ax,
    )

    if show_plot:
        plt.show()

    return fig

def get_plot_bokeh(  # pylint: disable=R0914
    graph: Set[Tuple[int]],
    color_nodes: Optional[List[int]] = None,
    show_plot: bool = False,
) -> "Plot":
    """Creates bokeh graph plot from edge set.

    Arguments:
        graph: The graph to plot.
        color_nodes: Colors given nodes green, default is white.
        show_plot: Wether or not the plot should be rendered.
    """
    from bokeh.io import show
    from bokeh.models import Plot, Range1d, Circle, HoverTool
    from bokeh.models.graphs import from_networkx

    color_nodes = color_nodes or []

    G = nx.Graph(list(graph))
    edge_attrs = {}
    node_attrs = {}

    for start_node, end_node, _ in G.edges(data=True):
        edge_attrs[(start_node, end_node)] = "black"

    for node in G.nodes:
        node_attrs[node] = "green" if node in color_nodes else "white"

    nx.set_edge_attributes(G, edge_attrs, "edge_color")
    nx.set_node_attributes(G, node_attrs, "node_color")

    # Show with Bokeh
    plot = Plot(
        plot_width=400,
        plot_height=400,
        x_range=Range1d(-1.1, 1.1),
        y_range=Range1d(-1.1, 1.1),
    )

    graph_renderer = from_networkx(G, nx.circular_layout, scale=1, center=(0, 0))
    graph_renderer.node_renderer.glyph = Circle(size=15, fill_color="node_color")
    plot.renderers.append(graph_renderer)  # pylint: disable=E1101

    node_hover_tool = HoverTool(tooltips=[("index", "@index")])
    plot.add_tools(node_hover_tool)

    if show_plot:
        show(plot)

    return plot

def plot_qubo(
    qubo, ax: Optional["Axes"] = None, axes: bool = False, fontsize: int = 4, **kwargs
) -> "Axes":
    """Plots qubo as heat map

    Kwargs are fed to heatmap.
    """
    if not ax:
        ax = plt.gca()

    m = qubo.todense() if not isinstance(qubo, ndarray) else qubo
    vmax = abs(m).max()

    data = dict(
        annot=True,
        annot_kws={"fontsize": fontsize},
        cmap="RdBu_r",
        vmin=-vmax,
        vmax=vmax,
        mask=m == 0,
        cbar=False,
    )
    data.update(kwargs)

    heatmap(m, ax=ax, **data)

    if not axes:
        ax.set_yticks([])
        ax.set_xticks([])

    return ax

def main():
    """Creates a random graph (fixed seed) and plots it.
    """

    n_nodes = 5
    n_edges = 5
    n_edge_max = 3

    test_graph = generate_graph(n_nodes, n_edges, n_edge_max=n_edge_max, seed=42)
    get_plot(test_graph, show_plot=True)

if __name__ == "__main__":
    main()


import time

import cv2
import kivy
import numpy as np
from kivy.app import App
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.properties import NumericProperty

# from kivy.uix.boxlayout import BoxLayout
# from kivy.uix.button import Button
from kivy.uix.camera import Camera
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.image import Image

# from kivy.uix.label import Label
from kivy.uix.popup import Popup
from kivy.uix.scatter import Scatter
from kivy.uix.screenmanager import ScreenManager
from kivy.uix.scrollview import ScrollView

# from kivy.uix.slider import Slider
from kivymd.uix.card import MDCard
from kivymd.uix.floatlayout import MDFloatLayout
from kivymd.uix.gridlayout import MDGridLayout
from kivymd.uix.screen import MDScreen as Screen, MDScreen
from kivymd.app import MDApp
from kivymd.uix.label import MDLabel as Label, MDLabel
from kivymd.uix.slider import MDSlider as Slider
from kivymd.uix.button import MDFlatButton as Button, MDRectangleFlatButton
from kivymd.uix.boxlayout import MDBoxLayout as BoxLayout, MDBoxLayout

import transform

class AndroidCamera(Camera):
    camera_resolution = (640 * 2, 480 * 2)
    counter = 0

    def _camera_loaded(self, *largs):
        self.texture = Texture.create(
            size=np.flip(self.camera_resolution), colorfmt="rgb"
        )
        self.texture_size = list(self.texture.size)

    def on_tex(self, *l):
        if self._camera._buffer is None:
            return None
        frame = self.frame_from_buf()

        self.frame_to_screen(frame)
        super(AndroidCamera, self).on_tex(*l)

    def frame_from_buf(self):
        buffer = self._camera._buffer
        if buffer is None:
            return None

        w, h = self.resolution
        frame = np.frombuffer(self._camera._buffer.tostring(), "uint8").reshape(
            (h + h // 2, w)
        )
        frame_bgr = cv2.cvtColor(frame, 93)
        return np.rot90(frame_bgr, 3)

    def frame_to_screen(self, frame):
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        cv2.putText(
            frame_rgb,
            str(self.counter),
            (20, 50),
            cv2.FONT_HERSHEY_SIMPLEX,
            1,
            (0, 255, 0),
            2,
            cv2.LINE_AA,
        )
        self.counter += 1
        flipped = np.flip(frame_rgb, 0)
        buf = flipped.tostring()
        self.texture.blit_buffer(buf, colorfmt="rgb", bufferfmt="ubyte")

class LinuxCamera(Camera):
    camera_resolution = (-1, -1)
    counter = 0

    def frame_from_buf(self):
        image = np.frombuffer(self.texture.pixels, np.uint8).reshape(
            *self.texture.size[::-1], 4
        )
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image

class CameraWidget(Camera):
    index = 0
    allow_stretch = True
    play = True

    def __new__(cls, *args, **kwargs):
        if kivy.platform != "linux":
            return AndroidCamera(
                *args, **kwargs, resolution=AndroidCamera.camera_resolution
            )
        else:
            return LinuxCamera(
                *args, **kwargs, resolution=LinuxCamera.camera_resolution
            )

def numpy_to_image(img, image: Image):
    h, w, _ = img.shape
    # texture = Texture.create(size=(w, h))
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = np.flip(img, 0)
    image.texture = Texture.create(size=(w, h), colorfmt="rgb")
    image.texture.blit_buffer(img.flatten(), colorfmt="rgb", bufferfmt="ubyte")
    # return texture
    # return Image(size=(w, h), texture=texture)

class MyLayout(BoxLayout):
    orientation = "vertical"
    slider_val = NumericProperty(100)

    def __init__(self, **kwargs):

        super().__init__(**kwargs)
        self.data = {}
        layout = self
        self.camera = CameraWidget(allow_stretch=True)
        self.slider = Slider(min=50, max=500, value=100, hint_radius=20)
        self.slider.fbind("value", self.on_slider_val)
        self.label = Label(text=str(self.slider_val), size_hint=(0.2, 1.0))

        self.pictureA = Image(source="assets/template.jpg")
        self.pictureB = Image(source="assets/templateB.jpg")

        buttonA = MDRectangleFlatButton(text="Photo A", size_hint=(1, 1.0))
        buttonB = MDRectangleFlatButton(text="Photo B", size_hint=(1, 1.0))
        buttonMatch = MDRectangleFlatButton(text="Match", size_hint=(1, 1.0))

        layout.add_widget(self.camera)

        hlayout = MDBoxLayout(pos_hint={"center_x": 0.5}, size_hint=(1, 0.25))
        hlayout.add_widget(self.pictureA)
        hlayout.add_widget(self.pictureB)

        layout.add_widget(hlayout)

        hlayout = BoxLayout(orientation="horizontal", size_hint=(1.0, 0.15))
        hlayout.add_widget(self.slider)
        layout.add_widget(hlayout)

        hlayout = MDBoxLayout(size_hint=(1.0, 0.15), spacing=5, padding=5)
        hlayout.add_widget(buttonA)
        hlayout.add_widget(buttonB)
        hlayout.add_widget(buttonMatch)
        layout.add_widget(hlayout)

        buttonA.bind(on_press=self.take_photo_A)
        buttonB.bind(on_press=self.take_photo_B)
        buttonMatch.bind(on_press=self.match)

    def on_slider_val(self, instance, val):
        self.label.text = str(int(val))

    def analyse_photo(self, key, picture):

        image = self.camera.frame_from_buf()

        orb = cv2.SIFT_create()
        gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
        # gray = cv2.resize(gray, (512, 512))
        keypoint, des = orb.detectAndCompute(gray, None)
        img_final = cv2.drawKeypoints(
            image, keypoint, None, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS
        )

        self.data[key] = (keypoint, des, image)
        numpy_to_image(img_final, picture)

    def take_photo_A(self, *args):
        self.analyse_photo("A", self.pictureA)

    def take_photo_B(self, *args):
        self.analyse_photo("B", self.pictureB)

    def match(self, *args):

        # BFMatcher with default params
        if "A" not in self.data:
            return
        if "B" not in self.data:
            return

        kp1, des1, img1 = self.data["A"]
        kp2, des2, img2 = self.data["B"]

        FLANN_INDEX_KDTREE = 0
        index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
        search_params = dict(checks=50)

        bf = cv2.FlannBasedMatcher(index_params, search_params)
        matches = bf.knnMatch(des1, des2, k=2)

        # Apply ratio test
        good = []
        for m, n in matches:
            if m.distance < 0.7 * n.distance:
                good.append(m)

        src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
        dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)

        M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
        matchesMask = mask.ravel().tolist()

        if sum(matchesMask) > self.slider.value:

            h, w, _ = img1.shape
            pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(
                -1, 1, 2
            )
            dst = cv2.perspectiveTransform(pts, M)
            print("Homography", M)
            print("mask", sum(mask))

            print("img2", img2.shape, img2.dtype)
            print("dst", dst)
            color = (255, 0, 0)
            points = np.int32(dst).reshape([-1, 1, 2])

            # img3 = cv2.polylines(img2, pts=[points], isClosed=True, color=color, thickness=3, lineType=cv2.LINE_AA)

            img3, _ = transform.cv_blend_images(img2, img1, np.linalg.inv(M))

        else:
            good = [[p] for p, m in zip(good, matchesMask) if m == 1]

            img3 = cv2.drawMatchesKnn(img1, kp1, img2, kp2, good, flags=2, outImg=None)

        render = Image()
        numpy_to_image(img3, render)

        root = ScrollView(do_scroll_x=False, do_scroll_y=False)

        scatter = Scatter(do_rotation=False)
        # dismiss = Button(text="close", on_press = self.popup.dismiss())
        # image = Image(source='sun.jpg')
        scatter.add_widget(render)
        scatter.scale = 10
        root.add_widget(scatter)

        def play(*args):
            print("Resuming")
            self.camera.play = True

        popup = Popup(
            title=f"Num matches: {sum(matchesMask)}", content=root, size_hint=(1, 0.8),
            on_dismiss = play
        )
        self.camera.play = False

        popup.open()
        popup.on_dismiss()

class MenuScreen(Screen):
    def __init__(self, **kw):
        super().__init__(**kw)
        layout = MyLayout()
        self.add_widget(layout)

class TrackerWidget(BoxLayout):
    orientation = "vertical"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        layout = self
        self.camera = CameraWidget(allow_stretch=True)
        self.image = Image(source="assets/template.jpg")
        layout.add_widget(self.image)

        nn_thresh = 0.7
        max_length = 15
        self.nn_thresh = nn_thresh
        self.min_length = 3
        import tracker

        # This class helps merge consecutive point matches into tracks.
        self.tracker = tracker.PointTracker(max_length, nn_thresh=nn_thresh)

        self.fe = cv2.SIFT_create(
            edgeThreshold=2, contrastThreshold=0.1, nOctaveLayers=2, sigma=0.5
        )
        # self.fe = cv2.ORB_create(1000, scoreType=cv2.ORB_FAST_SCORE)

        self.edgeThreshold = Slider(min=10, max=100, value=20, hint_radius=20)
        self.edgeThreshold.fbind("value", self.on_slider_val)

        self.contrastThreshold = Slider(min=5, max=100, value=10, hint_radius=20)
        self.contrastThreshold.fbind("value", self.on_slider_val)

        self.nOctaveLayers = Slider(min=1, max=8, value=2, hint_radius=20)
        self.nOctaveLayers.fbind("value", self.on_slider_val)

        self.sigma = Slider(min=10, max=200, value=50, hint_radius=20)
        self.sigma.fbind("value", self.on_slider_val)

        self.on_slider_val(None, None)

        slayout = MDBoxLayout(orientation="vertical", size_hint=(1, 0.2), padding=20)
        slayout.add_widget(self.edgeThreshold)
        slayout.add_widget(self.contrastThreshold)
        slayout.add_widget(self.nOctaveLayers)
        slayout.add_widget(self.sigma)
        layout.add_widget(slayout)

        Clock.schedule_interval(self.track_movement, 0.05)

    def on_slider_val(self, instance, val):
        print(
            f"sigma = {self.sigma.value / 100.0} "
            f"nOctaveLayers={int(self.nOctaveLayers.value)} "
            f"contrastThreshold={self.contrastThreshold.value / 1000} "
            f"edgeThreshold={self.edgeThreshold.value / 10}"
        )
        self.fe = cv2.SIFT_create(
            edgeThreshold=self.edgeThreshold.value / 10,
            contrastThreshold=self.contrastThreshold.value / 1000,
            nOctaveLayers=int(self.nOctaveLayers.value),
            sigma=self.sigma.value / 100.0,
        )

    def track_movement(self, dt):

        image = self.camera.frame_from_buf()
        if image is None:
            return

        image = cv2.resize(image, (200, 200), interpolation=cv2.INTER_CUBIC)

        gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
        start1 = time.time()
        pts, desc = self.fe.detectAndCompute(gray, None)

        if desc is None:
            return None

        pts = np.array([[*p.pt, 0.0] for p in pts])
        pts = pts.T
        desc = desc.T.astype(np.float32)

        desc /= np.linalg.norm(desc, axis=0)[np.newaxis, :]
        desc = np.concatenate([pts, desc], axis=0)

        self.tracker.update(pts, desc)

        # Get tracks for points which were match successfully across all frames.
        tracks = self.tracker.get_tracks(self.min_length)

        # Font parameters for visualizaton.
        font = cv2.FONT_HERSHEY_DUPLEX
        font_clr = (255, 255, 255)
        font_pt = (4, 20)
        font_sc = 0.8

        end1 = time.time()
        delta = end1 - start1

        # Primary output - Show point tracks overlayed on top of input image.
        out1 = (np.dstack((gray, gray, gray))).astype('uint8')
        tracks[:, 1] /= float(self.nn_thresh)  # Normalize track scores to [0,1].
        self.tracker.draw_tracks(out1, tracks)

        out1 = cv2.resize(out1, self.camera.texture_size)
        cv2.putText(out1, f'Point Tracks - {delta*1000:.2f} [ms]', font_pt, font, font_sc, font_clr, lineType=16)

        # out2 = (np.dstack((gray, gray, gray))).astype("uint8")
        # for pt in pts.T:
        #     pt1 = (int(round(pt[0])), int(round(pt[1])))
        #     cv2.circle(out2, pt1, 1, (0, 255, 0), -1, lineType=16)
        #
        # out2 = cv2.resize(out2, self.camera.texture_size)
        # cv2.putText(
        #     out2, f"Raw Point Detections - {delta*1000:.2f} [ms]", font_pt, font, font_sc, font_clr, lineType=16
        # )

        numpy_to_image(out1, self.image)

class MenuScreen2(Screen):
    def __init__(self, **kw):
        super().__init__(**kw)
        layout = TrackerWidget()
        self.add_widget(layout)

if kivy.platform == "linux":
    from kivy.core.window import Window

    Window.size = (480, 800)

class MyApp(MDApp):
    def build(self):

        sm = ScreenManager()
        sm.add_widget(MenuScreen(name='menu1'))
        # sm.add_widget(MenuScreen2(name="menu2"))

        return sm

if __name__ == "__main__":
    MyApp().run()


import random
import logging
import numpy as np

import torch
from spacy.lang.zh.stop_words import STOP_WORDS
from sklearn.metrics import f1_score, accuracy_score
from transformers import (
    AdamW,
    get_linear_schedule_with_warmup,
    get_cosine_schedule_with_warmup
)

from category_id_map import lv2id_to_lv1id

def setup_device(args):
    args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
    args.n_gpu = torch.cuda.device_count()

def setup_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)

def setup_logging():
    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S',
                        level=logging.INFO)
    logger = logging.getLogger(__name__)

    return logger

def build_optimizer(args, model):
    # Prepare optimizer and schedule (linear warmup and decay)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
         'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    if args.lr_scheduler == 'cosine':
        scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps,
                                                    num_training_steps=args.max_steps)
    else:
        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps,
                                                    num_training_steps=args.max_steps)

    return optimizer, scheduler

def evaluate(predictions, labels):
    # prediction and labels are all level-2 class ids

    lv1_preds = [lv2id_to_lv1id(lv2id) for lv2id in predictions]
    lv1_labels = [lv2id_to_lv1id(lv2id) for lv2id in labels]

    lv2_f1_micro = f1_score(labels, predictions, average='micro')
    lv2_f1_macro = f1_score(labels, predictions, average='macro')
    lv1_f1_micro = f1_score(lv1_labels, lv1_preds, average='micro')
    lv1_f1_macro = f1_score(lv1_labels, lv1_preds, average='macro')
    mean_f1 = (lv2_f1_macro + lv1_f1_macro + lv1_f1_micro + lv2_f1_micro) / 4.0

    eval_results = {'lv1_acc': accuracy_score(lv1_labels, lv1_preds),
                    'lv2_acc': accuracy_score(labels, predictions),
                    'lv1_f1_micro': lv1_f1_micro,
                    'lv1_f1_macro': lv1_f1_macro,
                    'lv2_f1_micro': lv2_f1_micro,
                    'lv2_f1_macro': lv2_f1_macro,
                    'mean_f1': mean_f1}

    return eval_results

def log_uniform(low, high):
    log_rval = np.random.uniform(np.log(low), np.log(high))
    rval = float(np.exp(log_rval))
    return rval

def remove_stopwords(string, nlp):
    doc = nlp(string)
    new_string = []
    for token in doc:
        if token.text not in STOP_WORDS:
            new_string.append(token.text)
    return ''.join(new_string)


import cv2
import dlib
import numpy as np
import imutils
from skimage import exposure
from PIL import Image
import torch

import poisson
from detect_landmark import detect_landmark
from blazeface import FaceExtractor, BlazeFace

def getPoints(image) :

    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')

    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    rects = detector(gray, 1)

    for rect in rects:

        shape = predictor(gray, rect)
        shape_np = np.zeros((68, 2), dtype="int")
        for i in range(0, 68):
            shape_np[i] = (shape.part(i).x, shape.part(i).y)
        shape = shape_np

    return shape.tolist()

def detect_rec(image):

    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image = Image.fromarray(image)

    device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
    facedet = BlazeFace().to(device)
    facedet.load_weights("./blazeface/blazeface.pth")
    facedet.load_anchors("./blazeface/anchors.npy")
    face_extractor = FaceExtractor(facedet=facedet)
    im_real_faces = face_extractor.process_image(img=image)
    ymin, xmin, ymax, xmax = im_real_faces['detections'][0, 0:4].astype(int)
    return xmin, ymin, xmax, ymax

def applyAffineTransform(src, srcTri, dstTri, size) :

    # Given a pair of triangles, find the affine transform.
    warpMat = cv2.getAffineTransform( np.float32(srcTri), np.float32(dstTri) )

    # Apply the Affine Transform just found to the src image
    dst = cv2.warpAffine( src, warpMat, (size[0], size[1]), None, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101 )

    return dst

# Check if a point is inside a rectangle
def rectContains(rect, point) :
    if point[0] < rect[0] :
        return False
    elif point[1] < rect[1] :
        return False
    elif point[0] > rect[0] + rect[2] :
        return False
    elif point[1] > rect[1] + rect[3] :
        return False
    return True

#calculate delanauy triangle
def calculateDelaunayTriangles(rect, points):
    #create subdiv
    subdiv = cv2.Subdiv2D(rect)

    # Insert points into subdiv

    for p in points:
        subdiv.insert(p)

    triangleList = subdiv.getTriangleList()

    delaunayTri = []

    pt = []

    for t in triangleList:
        pt.append((t[0], t[1]))
        pt.append((t[2], t[3]))
        pt.append((t[4], t[5]))

        pt1 = (t[0], t[1])
        pt2 = (t[2], t[3])
        pt3 = (t[4], t[5])

        if rectContains(rect, pt1) and rectContains(rect, pt2) and rectContains(rect, pt3):
            ind = []
            #Get face-points (from 68 face detector) by coordinates
            for j in range(0, 3):
                for k in range(0, len(points)):
                    if(abs(pt[j][0] - points[k][0]) < 1.0 and abs(pt[j][1] - points[k][1]) < 1.0):
                        ind.append(k)
            # Three points form a triangle. Triangle array corresponds to the file tri.txt in FaceMorph
            if len(ind) == 3:
                delaunayTri.append((ind[0], ind[1], ind[2]))

        pt = []

    return delaunayTri

# Warps and alpha blends triangular regions from img1 and img2 to img
def warpTriangle(img1, img2, t1, t2) :

    # Find bounding rectangle for each triangle
    r1 = cv2.boundingRect(np.float32([t1]))
    r2 = cv2.boundingRect(np.float32([t2]))

    # Offset points by left top corner of the respective rectangles
    t1Rect = []
    t2Rect = []
    t2RectInt = []

    for i in range(0, 3):
        t1Rect.append(((t1[i][0] - r1[0]),(t1[i][1] - r1[1])))
        t2Rect.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))
        t2RectInt.append(((t2[i][0] - r2[0]),(t2[i][1] - r2[1])))

    # Get mask by filling triangle
    mask = np.zeros((r2[3], r2[2], 3), dtype = np.float32)
    cv2.fillConvexPoly(mask, np.int32(t2RectInt), (1.0, 1.0, 1.0), 16, 0)

    # Apply warpImage to small rectangular patches
    img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]
    #img2Rect = np.zeros((r2[3], r2[2]), dtype = img1Rect.dtype)

    size = (r2[2], r2[3])

    img2Rect = applyAffineTransform(img1Rect, t1Rect, t2Rect, size)

    img2Rect = img2Rect * mask

    # Copy triangular region of the rectangular patch to the output image
    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] * ( (1.0, 1.0, 1.0) - mask )

    img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] + img2Rect

def hist_matching(src, dst, mask, rect):
    x, y, w, h = rect
    mask_n = mask.copy().astype(float)
    mask_n[mask_n == 255] = 1.

    matched = exposure.match_histograms((src * mask_n)[y:y+h, x:x+w], (dst * mask_n)[y:y+h, x:x+w], multichannel=True)

    result = src.copy()
    result[y:y+h, x:x+w] = matched
    result = result * mask_n + src * (1 - mask_n)

    return result

def swapface(img1 , img2, SBR=True, hm=True, pe=True):
    img1Warped = np.copy(img2);

    if SBR:
        points1 = detect_landmark(img1).tolist()
        points2 = detect_landmark(img2).tolist()
    else:
        points1 = getPoints(img1)
        points2 = getPoints(img2)

    if ((points1 is None) & (points2 is None)):

        return img1Warped

    else:
        # Find convex hull
        hull1 = []
        hull2 = []

        hullIndex = cv2.convexHull(np.array(points2), returnPoints = False)

        for i in range(0, len(hullIndex)):
            hull1.append(points1[int(hullIndex[i])])
            hull2.append(points2[int(hullIndex[i])])

        # Find delanauy traingulation for convex hull points
        sizeImg2 = img2.shape
        rect = (0, 0, sizeImg2[1], sizeImg2[0])

        dt = calculateDelaunayTriangles(rect, hull2)

        if len(dt) == 0:
            quit()

        # Apply affine transformation to Delaunay triangles
        for i in range(0, len(dt)):
            t1 = []
            t2 = []

            #get points for img1, img2 corresponding to the triangles
            for j in range(0, 3):
                t1.append(hull1[dt[i][j]])
                t2.append(hull2[dt[i][j]])

            warpTriangle(img1, img1Warped, t1, t2)

        # Calculate Mask
        hull8U = []
        for i in range(0, len(hull2)):
            hull8U.append((hull2[i][0], hull2[i][1]))

        mask = np.zeros(img2.shape, dtype = img2.dtype)

        cv2.fillConvexPoly(mask, np.int32(hull8U), (255, 255, 255))

        r = cv2.boundingRect(np.float32([hull2]))

        center = ((r[0]+int(r[2]/2), r[1]+int(r[3]/2)))

        if hm:
            img1Warped = hist_matching(img1Warped, img2, mask, r)

        if pe:
            # mask = mask.astype(float)
            # mask[mask == 255] = 1.
            # output = [poisson.process(img1Warped[:, :, i], img2[:, :, i], mask[:, :, 0]) for i in range(3)]
            # output = np.uint8(cv2.merge(output))
            output = img1Warped

        else:
            output = cv2.seamlessClone(np.uint8(img1Warped), img2, mask, center, cv2.NORMAL_CLONE)

        return np.uint8(output), r


from keras.models import Model
from keras.layers import Input, Lambda
import numpy as np

# 定義 Lambda 要執行的函數
def minus(inputs):
    x, y = inputs
    return (x-y)

if __name__ == "__main__":
    # Step 1. 定義模型
    a = Input(shape=(2,))
    b = Input(shape=(2,))

    # Lambda layer
    minus_layer = Lambda(minus, name='minus')([a, b])

    # Model
    model = Model(inputs=[a, b], outputs=minus_layer)

    # Step 2. 測試模型
    v0 = np.array([8, 4])
    v1 = np.array([5, 2])

    # output: [[3, 2]]
    print(model.predict([v0.reshape(1, 2), v1.reshape(1, 2)]))


import os
import shutil
import filecmp
import sys
from config import origin_folder, destination_folder

def sync_folders(origin_folder, destination_folder):
    # Get the list of files in the origin folder
    origin_files = set(os.listdir(origin_folder))

    # Get the list of files in the destination folder
    destination_files = set(os.listdir(destination_folder))

    # Synchronize only PDF files from origin to destination
    for file in origin_files:
        if file.lower().endswith(".pdf") or file.lower().endswith(".txt") or file.lower().endswith(".ink"):
            origin_file_path = os.path.join(origin_folder, file)
            destination_file_path = os.path.join(destination_folder, file)

            if not os.path.exists(destination_file_path):
                # If the file doesn't exist in destination, copy it from origin
                shutil.copy2(origin_file_path, destination_file_path)
                print(f"Copied: {file}")
            else:
                # If the file exists, compare files to check if update is required
                if not filecmp.cmp(origin_file_path, destination_file_path):
                    # If the files are different, replace the file in destination
                    os.remove(destination_file_path)
                    shutil.copy2(origin_file_path, destination_file_path)
                    print(f"Replaced: {file}")

    # Remove PDF files from destination if they don't exist in origin
    for file in destination_files - origin_files:
        if file.lower().endswith(".pdf") or file.lower().endswith(".txt") or file.lower().endswith(".ink"):
            file_path = os.path.join(destination_folder, file)
            os.remove(file_path)
            print(f"Deleted: {file}")

sync_folders(origin_folder, destination_folder)


import numpy as np
import matplotlib.pyplot as plt

def get_mean_and_standard_deviation_difference_results(results):
    """From a list of lists of agent results it extracts the mean results and the mean results plus or minus
     some multiple of the standard deviation"""

    standard_deviation_results = 1.0

    def get_results_at_a_time_step(results, timestep):
        results_at_a_time_step = [result[timestep] for result in results]
        return results_at_a_time_step

    def get_standard_deviation_at_time_step(results, timestep):
        results_at_a_time_step = [result[timestep] for result in results]
        return np.std(results_at_a_time_step)

    mean_results = [np.mean(get_results_at_a_time_step(results, timestep)) for timestep in range(len(results[0]))]
    mean_minus_x_std = [
        mean_val - standard_deviation_results * get_standard_deviation_at_time_step(results, timestep)
        for
        timestep, mean_val in enumerate(mean_results)]
    mean_plus_x_std = [
        mean_val + standard_deviation_results * get_standard_deviation_at_time_step(results, timestep)
        for
        timestep, mean_val in enumerate(mean_results)]
    return mean_minus_x_std, mean_results, mean_plus_x_std

def get_standard_plot(result, scale=2, agent_name="this_result", color="#800000"):
    fig, ax = plt.subplots(figsize=(24 / scale, 15 / scale), dpi=100 * scale)
    mean_minus_x_std, mean_results, mean_plus_x_std = get_mean_and_standard_deviation_difference_results(result)
    x_vals = list(range(len(mean_results)))

    ax.plot(x_vals, mean_results, label=agent_name, color=color)
    ax.plot(x_vals, mean_plus_x_std, color=color, alpha=0.1)
    ax.plot(x_vals, mean_minus_x_std, color=color, alpha=0.1)
    ax.fill_between(x_vals, y1=mean_minus_x_std, y2=mean_plus_x_std, alpha=0.1, color=color)

    ax.legend(loc='upper left', bbox_to_anchor=(0.5, 0.5), framealpha=0.6, fancybox=True, prop={'size': 30 / scale})
    # plt.show()

if __name__ == '__main__':
    list_to_test = [[j + i for j in range(20)] for i in range(4)]
    get_standard_plot(list_to_test)
    pass


# pylint: disable=import-error,unexpected-keyword-arg
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Roy
"""
# Python standard libararies
import sys
import json

# 3rd party libararies
from turbojpeg import TurboJPEG
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
import uvicorn

app = FastAPI()
jpeg = TurboJPEG()

@app.post("/test/")
async def inference(req: Request):
    data = await req.body()
    img = jpeg.decode(data)
    # print(sys.getsizeof(data))
    # print(sys.getsizeof(img))
    # print(img.shape)
    return JSONResponse(content=json.dumps({"shape": img.shape}))

@app.get("/healthcheck/")
async def check_health():
    return {"Status": 1}

if __name__ == '__main__':  # local dev
    port = int(sys.argv[1])
    print("FastAPI server configuration is done!")
    print('API is running!')
    print('LocalIP: {} , Port: {}'.format("localhost",port))

    print()
    uvicorn.run(app, host="0.0.0.0", port=port)

# uvicorn application:app --reload --port 8000
# curl localhost:8000/health


from django.shortcuts import render
from django.http import HttpResponse
from google.cloud import storage
import os
import posixpath
from PIL import Image

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

def download_blob(bucket_name, background_source_blob_name,  item_source_blob_name, bg_destination_file_name , item_destination_file_name):
    """Downloads a blob from the bucket."""

    storage_client = storage.Client()

    bucket = storage_client.bucket(bucket_name)

    # Construct a client side representation of a blob.
    # Note `Bucket.blob` differs from `Bucket.get_blob` as it doesn't retrieve
    # any content from Google Cloud Storage. As we don't need additional data,
    # using `Bucket.blob` is preferred here.
    BG_blob = bucket.blob(background_source_blob_name)
    BG_blob.download_to_filename(bg_destination_file_name)

    item_blob = bucket.blob(item_source_blob_name)
    item_blob.download_to_filename(item_destination_file_name)

    print(
        "Blob {} downloaded to {}.".format(
            background_source_blob_name, bg_destination_file_name
        )
    )

def upload_blob(bucket_name, source_file_name, destination_blob_name):

    """Uploads a file to the bucket."""
    # The ID of your GCS bucket
    # bucket_name = "your-bucket-name"
    # The path to your file to upload
    # source_file_name = "local/path/to/file"
    # The ID of your GCS object
    # destination_blob_name = "storage-object-name"

    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    blob = bucket.blob(destination_blob_name)

    blob.upload_from_filename(source_file_name)

    print(
        "File {} uploaded to {}.".format(
            source_file_name, destination_blob_name
        )
    )

# create combines the background image (containing faces) with items (mask, glasses etc.)
def create(*args, **kwargs):

    bucket_name = "image-processing-module.appspot.com"
    background_source_blob_name = "input_images/harry_potter.jpg"
    item_source_blob_name = "items/glasses1.png"

    bg_destination_file_name = os.path.join(BASE_DIR, "static_files/face_images/background_image.jpg")
    item_destination_file_name = os.path.join(BASE_DIR, "static_files/items/item1.png")

    final_image_directory = os.path.join(BASE_DIR, "static_files/output_images/final.png")
    destination_blob_name = "output_images/final_image.jpg"

    download_blob(bucket_name, background_source_blob_name,  item_source_blob_name, bg_destination_file_name , item_destination_file_name)

    combine_images(item_destination_file_name , bg_destination_file_name , final_image_directory)

    upload_blob(bucket_name, final_image_directory, destination_blob_name)

    #print(*args)
    #bg_destination_file_name = os.path.join(BASE_DIR, "imageCreator/face_images/downloadedFile.jpg")
    #path = "F:\ImageProcessingModule\ImageProcessingModule\ImageProcessingModule\imageCreator\face_images\downloadedFile.jpg"

    html = "
IMAGE:
 "

    return HttpResponse(html)

def combine_images(item_image_path, bg_image_path, final_image_directory):

    bg = Image.open(bg_image_path)
    item = Image.open(item_image_path)
    new_item_size = (370, 125)
    item = item.resize(new_item_size).rotate(20, expand = True)
    back_im = bg.copy()
    back_im.paste(item, (260,340) , item)
    back_im.save(final_image_directory, quality=95)

    #return null


from collections import Counter
class Solution(object):
    def findTheDifference(self, s, t):
        """
        :type s: str
        :type t: str
        :rtype: str
        """
        # General Idea: Create a counter from data in s and then iterate through letters in t to find the one with count 0 from counter.

        a = Counter(s)
        b = Counter(t)
        for letter in t:
            if b[letter] == 0 or a[letter] != b[letter]:
                return letter


import unittest
import numpy as np
from PIL import Image
from torchvision import transforms

from ops import data_ops

class TestImageConversion(unittest.TestCase):

    def test_tensor2image(self):
        img = Image.open('misc/example.jpg')
        print('Original size:', img.size)

        tsfm = transforms.ToTensor()
        t_array = tsfm(img)
        print('Array shape:', t_array.shape)

        img_converted = data_ops.convert_tensor2image(
            t_array, unnormalize=False)

        print(img_converted.size)
        print(type(img_converted))

        img.save('/tmp/image-original-1.png')
        img_converted.save('/tmp/image-converted-1.png')

        self.assertTrue(np.all(img_converted.size == img.size))

    def test_tensor2image_with_normalize(self):

        img = Image.open('misc/example.jpg')

        tsfm = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(
                mean=(0.5, 0.5, 0.5),
                std=(0.5, 0.5, 0.5))
        ])

        t_array = tsfm(img)
        img_converted = data_ops.convert_tensor2image(
            t_array, unnormalize=True)

        img.save('/tmp/image-original-2.png')
        img_converted.save('/tmp/image-converted-2.png')

        self.assertTrue(np.all(img_converted.size == img.size))


import numpy as np
import pandas as pd
import rasterio as rio

def sample_vals(aoi, x_coords,y_coords,raw_list):
    i = 0
    index_list = []
    filename = []
    if len(aoi) == 17:
        a = 57
    else:
        a = 52

    for raw in raw_list:
        sampled_values = []

        # Sample raster at each combo of x and y coordinates
        with rio.open(raw) as src:
            for val in src.sample(zip(x_coords, y_coords)):
                i = i+1
                # Note that each val is an individual numpy array
                sampled_values.append(val)
            arr = np.array([sampled_values]).squeeze()
            index_list.append([arr])
            filename.append(raw[a:77])

    labels = filename
    index_vals = pd.DataFrame(np.concatenate(index_list))
    index_vals = index_vals.transpose()
    index_vals.columns = labels
    return index_vals


import findspark
findspark.init()

from pyspark import SparkContext, SparkConf
import time
import sys
import os
import random
import math
import json
from itertools import combinations
from copy import deepcopy

startTime = time.time()

conf = SparkConf().setMaster("local[*]")         .set("spark.executor.memory", "4g")         .set("spark.driver.memory", "4g")
sc = SparkContext(conf=conf).getOrCreate()

def summarize(points_cluster_list):
    summary = list()
    for cluster in points_cluster_list:
        cluster_dict = dict() # {N:n, SUM:sum, SUMSQ:sumsq}
        cluster_dict['N'] = len(cluster)
        SUM, SUMSQ = getSUMandSUMSQ(cluster)
        cluster_dict['SUM'] = SUM
        cluster_dict['SUMSQ'] = SUMSQ
        summary.append(cluster_dict)
    return summary

def getSUMandSUMSQ(cluster):
    cluster_points = [pair[0] for pair in cluster]
    for idx, point in enumerate(cluster_points):
        sq_point = tuple([digit**2 for digit in point])
        if idx==0:
            SUM_result = point
            SUMSQ_result = sq_point
        else:
            SUM_result = [sum(x) for x in zip(SUM_result, point)]
            SUMSQ_result = [sum(x) for x in zip(SUMSQ_result, sq_point)]
    return tuple(SUM_result), tuple(SUMSQ_result)

def groupPointsByCluster(k, cluster_result):
    points_cluster_list = list()
    for kth in range(k):
        l = list(filter(lambda pair:pair[1]==kth, cluster_result))
        if len(l)!=0:
            points_cluster_list.append(l)
    points_cluster_list = sorted(points_cluster_list, key=lambda cluster:len(cluster))
    return points_cluster_list

def kmeans(k, points_list, max_iter, initialization, n_init=1):
    best_centroids = dict()
    best_inertia = float('inf')
    for n in range(n_init):
        centroids_dict = dict() # map centroid tuple to unique and index
        if initialization=='random':
            dictinct_points = list(set(points_list))
            selected_cent = set()
            for cluster_idx in range(k):
                random_idx = random.randrange(0, len(dictinct_points))
                while random_idx in selected_cent:
                    random_idx = random.randrange(0, len(dictinct_points))
                selected_cent.add(random_idx)
                centroids_dict[dictinct_points[random_idx]] = cluster_idx
            # centroids is the list of initial centorids
        elif initialization=='farthest':
            #initialize the initial centroids by farthest distance from each other
            curr_sum_distance = 0
            closest_distance = 0
            for kth_centroid in range(k): #first centroid is still randomly picked
                if kth_centroid == 0:
                    random_idx = random.randrange(0, len(points_list))
                    centroids_dict[points_list[random_idx]] = kth_centroid
                else:
                    curr_sum_distance = 0
                    # compute sum of distance to the current centroids to each other point
                    closest_centroid = points_list[0] # in case there is only one point in points_list
                    for point in points_list:
                        if point not in centroids_dict.keys():
                            for curr_cent in centroids_dict.keys():
                                curr_sum_distance += euclideanDist(curr_cent, point)
                            if curr_sum_distance > closest_distance:
                                closest_distance = curr_sum_distance
                                closest_centroid = point
                            curr_sum_distance = 0
                    centroids_dict[closest_centroid] = kth_centroid
        else:
            return 'initialization method not recognized'

        convergence = False
        curr_iter = 0
        while not convergence and curr_iter<max_iter:
            # assign points to clusters
            cluster_list = list() # [(point, centroid_idx),(point, centroid_idx), ...]
            for point in points_list:
                #this point should be assigned to return value centroid
                centroid = findCentroid(point, centroids_dict.keys())
                centroid_idx = centroids_dict[centroid]
                cluster_list.append((point, centroid_idx))

            # update centroids
            centroids_comp = list()
            centroids_dict_items = list(centroids_dict.items())

            for old_centroid, idx in centroids_dict_items:
                idx_cluster = list(filter(lambda pair: pair[1]==idx, cluster_list))
                # a cluster might be empty due to bad initialization
                if len(idx_cluster) == 0:
                    # simply assign a random point to that cluster
                    points = [points_list[random.randrange(0, len(points_list))]]
                else:
                    # list of points in this cluster
                    points = [pair[0] for pair in idx_cluster]

                # calculate new centroid
                new_centroid = calculateNewCentroid(points, len(old_centroid))
                # update centroids_dict
                if new_centroid != old_centroid:
                    # new centroid has same cluster index
                    # add new centroid
                    centroids_dict[new_centroid] = idx
                    # delete old centroid
                    del centroids_dict[old_centroid]

                centroids_comp.append((old_centroid, new_centroid))

            # test convergence
            curr_iter += 1
            convergence = all([euclideanDist(pair[0], pair[1])==0 for pair in centroids_comp])
        # after convergence, calculate interia
        curr_inertia = 0
        for point in points_list:
            #this point should be assigned to return value centroid
            centroid = findCentroid(point, centroids_dict.keys())
            curr_inertia += euclideanDist(point, centroid) ** 2
        if curr_inertia < best_inertia:
            best_inertia = curr_inertia
            best_centroids = deepcopy(centroids_dict)

    cluster_list = list() # [(point, centroid_idx),(point, centroid_idx), ...]
    for point in points_list:
        #this point should be assigned to return value centroid
        centroid = findCentroid(point, best_centroids.keys())
        centroid_idx = best_centroids[centroid]
        cluster_list.append((point, centroid_idx))
    return cluster_list, list(best_centroids.keys())

def calculateNewCentroid(clusterPoints, dimensions):
    coord_list = list()
    for d in range(dimensions):
        coord_list.append([pt[d] for pt in clusterPoints]) # get d-th coordinates of all points
    return tuple([sum(dim)/len(dim) for dim in coord_list])

def euclideanDist(pt, cent):
    sumsq = 0
    for d in range(len(pt)):
        sumsq += (pt[d] - cent[d]) ** 2
    return math.sqrt(sumsq)

def findCentroid(pt, centroids):
    reuslt_list = list()
    for centroid in centroids:
        ED = euclideanDist(pt, centroid) #Euclidean distance
        reuslt_list.append((ED, centroid))
    return sorted(reuslt_list, key=lambda pair:pair[0])[0][1]# return the centroid with min ED to pt

def updateSummary(summary, idx, point):
    summary[idx]['N'] += 1
    summary[idx]['SUM'] = tuple([sum(x) for x in zip(summary[idx]['SUM'], point)])
    sq_point = tuple([digit**2 for digit in point])
    summary[idx]['SUMSQ'] = tuple([sum(x) for x in zip(summary[idx]['SUMSQ'], sq_point)])

def mahalanobisDist(point, summary):
    md = 0
    N = summary['N']
    for d in range(len(point)):
        d_SUM = summary['SUM'][d]
        d_SUMSQ = summary['SUMSQ'][d]
        d_centroid = d_SUM / N
        sd = math.sqrt((d_SUMSQ / N) - (d_SUM / N) ** 2)
        if sd==0:
            normalization = 0
        else:
            normalization = ((point[d] - d_centroid) / sd) ** 2
        md += normalization
    return math.sqrt(md)

def mergeCS(CS_summary):
    merged_CS_idx_set = set()
    new_CS_list = list()

    MD_pair_list = list()
    # generate combinations of CS based on their indices
    CS_idx_comb = list(combinations(range(len(CS_summary)), 2))
    for CS_idx_tuple in CS_idx_comb:
        summary1 = CS_summary[CS_idx_tuple[0]]
        summary2 = CS_summary[CS_idx_tuple[1]]
        MD, thres = getNearness(summary1, summary2)
        if MD < thres:
            MD_pair_list.append((MD, CS_idx_tuple))
    MD_pair_list = sorted(MD_pair_list, key=lambda pair:pair[0])

    for MD_pair in MD_pair_list:
        MD = MD_pair[0]
        CS_idx_tuple = MD_pair[1]
        # if none of these two CS have been merged before
        if CS_idx_tuple[0] not in merged_CS_idx_set and CS_idx_tuple[1] not in merged_CS_idx_set:
            # merge these two CS into one
            summary1 = CS_summary[CS_idx_tuple[0]]
            summary2 = CS_summary[CS_idx_tuple[1]]
            N = summary1['N'] + summary2['N']
            SUM = tuple([sum(zipped) for zipped in zip(summary1['SUM'], summary2['SUM'])])
            SUMSQ = tuple([sum(zipped) for zipped in zip(summary1['SUMSQ'], summary2['SUMSQ'])])
            new_CS = {'N':N, 'SUM':SUM, 'SUMSQ':SUMSQ}
            new_CS_list.append(new_CS) # append merged CS
            merged_CS_idx_set.add(CS_idx_tuple[0])
            merged_CS_idx_set.add(CS_idx_tuple[1])

    # get the list of CS indices that are not merged
    unmerged_CS_idx_list = list(set(range(len(CS_summary))) - merged_CS_idx_set)

    # append the not merged CS back to CS list
    for unmerged_CS_idx in unmerged_CS_idx_list:
        new_CS_list.append(CS_summary[unmerged_CS_idx])

    return new_CS_list

def getNearness(CS1, CS2):
    # nearness is defined as the average MD between a centroid of a cluster to another cluster and vice versa
    cent1 = tuple([d/CS1['N'] for d in CS1['SUM']])
    cent2 = tuple([d/CS2['N'] for d in CS2['SUM']])
    CS1cent2CS2_MD = mahalanobisDist(cent1, CS2)
    CS2cent2CS1_MD = mahalanobisDist(cent2, CS1)
    threshold = alpha_threshold * math.sqrt(len(cent1))
    return (CS1cent2CS2_MD + CS2cent2CS1_MD)/2, threshold

def generateIntermediateResult(round_num, DS_summary, CS_summary, RS_list):
    nof_point_discard = str(sum([DS['N'] for DS in DS_summary]))
    nof_point_compression = str(sum([CS['N'] for CS in CS_summary]))
    line = str(round_num) + ',' + str(len(DS_summary)) + ','            + nof_point_discard + ',' + str(len(CS_summary)) + ','            + nof_point_compression + ',' + str(len(RS_list))
    return line

input_folder_path = r'C:\Users\11921\Downloads\data\test3'
k = 10
output_path_cluster = r'points_assignment.json'
output_path_intermediate_result = r'intermediate_result.csv'

csv_files_list = os.listdir(input_folder_path)
large_num_centroids = 3 * k
max_iterations = 10
very_few = 500
alpha_threshold = 2
sample_ratio = 0.7
kmeans_initial_cent = 'random' #random
generateCSRS = 'complement' #'outliers', method of generating CS and RS, either from complement points or from outliers
cache_datapoints = True
merge_CS = False
all_datapoints = list()

intermediate_result_list = list()
for round_idx, csv in enumerate(sorted(csv_files_list)):
    '''
    Step 1, 7. load data
    '''
    if cache_datapoints:
        full_csv_path = input_folder_path + "\\" + csv
        raw_points = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt:(str(int(pt[0])), tuple(pt[1:])))  # form tuple of idx and coordinates
                        .collect()
                       )
        points_list = [pair[1] for pair in raw_points]
        all_datapoints.extend(raw_points)

    else:
        full_csv_path = input_folder_path + "\\" + csv
        points_list = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt: tuple(pt[1:])) # keep only the point coordinates
                        .collect()
                       )

    if round_idx == 0: # first round, run k-means on a subset to initialize
        # sample the points list to make it smaller
        subset_points_list = random.sample(points_list, int(len(points_list) * sample_ratio))
        comp_points_list = list(set(points_list) - set(subset_points_list))# remaining points of the first load of data

        if generateCSRS == 'outliers':
            '''
            Step 2. run k-means on a subset of data points (to generate outliers and inliers)
            '''
            # cluster_result: [((point1), cluster_index), ((point2), cluster_index), ...]
            # centroids: [(centroid1), (centroid2), ...]
            cluster_result, _ = kmeans(large_num_centroids, subset_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            points_cluster_list = groupPointsByCluster(large_num_centroids, cluster_result)
            del cluster_result

            '''
            Step 3. move clusters with few points to outliers, otherwise to inliers
            '''
            outliers_list = list()
            inliers_list = list()
            for cluster in points_cluster_list:
                if len(cluster) <= very_few:
                    outliers_list.append(cluster)
                else:
                    inliers_list.append(cluster)
            del points_cluster_list
            outlier_points_list = [pair[0] for cluster in outliers_list for pair in cluster]

            '''
            Step 4. run kmeans again on inliers to cluster inlier data to k clusters. Use k clusters as DS and summarize
            '''
            inlier_points_list = [pair[0] for cluster in inliers_list for pair in cluster]
            inliers_points_result, _ = kmeans(k, inlier_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            inliers_cluster_result = groupPointsByCluster(k, inliers_points_result)
            DS_summary = summarize(inliers_cluster_result)
            # discard the points
            del inliers_list # original inliers points with index
            del inlier_points_list # original inliers points
            del inliers_points_result # cluster result of inliers points
            del inliers_cluster_result # cluster result of inliers points grouped by cluster index

            '''
            Step 5. run kmeans on outliers, create CS and RS from the result, keep only the summary of CS
            '''
            outliers_points_result, _ = kmeans(large_num_centroids, outlier_points_list, max_iterations, kmeans_initial_cent)
            outliers_cluster_result = groupPointsByCluster(large_num_centroids, outliers_points_result)
            CS_list, RS_list = list(), list()
            for outliers_cluster in outliers_cluster_result:
                if len(outliers_cluster) > 1:
                    CS_list.append(outliers_cluster)
                else:
                    RS_list.append(outliers_cluster)
            RS_list = [point[0] for cluster in RS_list for point in cluster]
            CS_summary = summarize(CS_list)
            del CS_list, outliers_cluster_result, outlier_points_list

            '''
            Step 6,7,8,9,10. Load the remaining data of the first round, assign each of them to DS, CS or RS
            '''
            for point in comp_points_list:
                min_DS_MD = float('inf')
                min_CS_MD = float('inf')
                found_qualified_DS_point = False
                for curr_DS_idx, DS_cluster in enumerate(DS_summary):
                    if DS_cluster['N'] != 0:
                        curr_DS_MD = mahalanobisDist(point, DS_cluster)
                        if curr_DS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_DS_MD < min_DS_MD:
                                min_DS_MD = curr_DS_MD
                                DS_idx = curr_DS_idx
                                found_qualified_DS_point = True
                if found_qualified_DS_point:
                    updateSummary(DS_summary, DS_idx, point)
                else:
                    found_qualified_CS_point = False
                    for curr_CS_idx, CS_cluster in enumerate(CS_summary):
                        curr_CS_MD = mahalanobisDist(point, CS_cluster)
                        if curr_CS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_CS_MD < min_CS_MD:
                                min_CS_MD = curr_CS_MD
                                CS_idx = curr_CS_idx
                                found_qualified_CS_point = True
                    if found_qualified_CS_point:
                        updateSummary(CS_summary, CS_idx, point)
                    else:
                        RS_list.append(point)
        elif generateCSRS == 'complement':
            #generate DS
            subset_points_result, _ = kmeans(k, subset_points_list, max_iterations, kmeans_initial_cent)

            # put together the points that have been clustered into a same cluster
            subset_points_cluster_result = groupPointsByCluster(k, subset_points_result)
            DS_summary = summarize(subset_points_cluster_result)
            # discard the points
            del subset_points_result
            del subset_points_cluster_result

            #generate CS/RS
            comp_points_result, _ = kmeans(large_num_centroids, comp_points_list, max_iterations, kmeans_initial_cent)
            comp_points_cluster_result = groupPointsByCluster(large_num_centroids, comp_points_result)

            CS_list = list()
            RS_list = list()
            CS_summary = list()
            for cluster in comp_points_cluster_result:
                if len(cluster) > 1:
                    CS_list.append(cluster)
                elif len(cluster) == 1:
                    RS_list.append(cluster)
            CS_summary = summarize(CS_list)
            RS_list = [point[0] for cluster in RS_list for point in cluster]
            del comp_points_result, comp_points_cluster_result, CS_list

    else: # not first round, Repeat step 6 to 12.
        '''
        Step 8,9,10. Assign each of the data points to DS, CS or RS
        '''
        for point in points_list: # go through each point
            min_DS_MD = float('inf')
            min_CS_MD = float('inf')
            found_qualified_DS_point = False
            for curr_DS_idx, DS_cluster in enumerate(DS_summary): # go through each DS cluster
                curr_DS_MD = mahalanobisDist(point, DS_cluster)
                if curr_DS_MD < alpha_threshold * math.sqrt(len(point)):
                    if curr_DS_MD < min_DS_MD:
                        min_DS_MD = curr_DS_MD
                        DS_idx = curr_DS_idx
                        found_qualified_DS_point = True
            if found_qualified_DS_point:
                updateSummary(DS_summary, DS_idx, point)
            else:
                if len(CS_summary) > 0:
                    found_qualified_CS_point = False
                    for curr_CS_idx, CS_cluster in enumerate(CS_summary): # go through each CS cluster
                        curr_CS_MD = mahalanobisDist(point, CS_cluster)
                        if curr_CS_MD < alpha_threshold * math.sqrt(len(point)):
                            if curr_CS_MD < min_CS_MD:
                                min_CS_MD = curr_CS_MD
                                CS_idx = curr_CS_idx
                                found_qualified_CS_point = True
                    if found_qualified_CS_point:
                        updateSummary(CS_summary, CS_idx, point)
                    else:
                        RS_list.append(point)
                else:
                    RS_list.append(point)

    # if CS_summary is empty, run this
    if len(CS_summary) == 0:
        '''
        Step 11. Run kmeans on current RS list, split clusters into CS and RS.
        generate new CS summary and add it to CS summary, keep remaining RS list
        '''
        RS_points_result, _ = kmeans(large_num_centroids, RS_list, max_iterations, kmeans_initial_cent)
        RS_cluster_result = groupPointsByCluster(large_num_centroids, RS_points_result)

        RS_list = list() # delete the old RS_list
        for RS_cluster in RS_cluster_result:
            if len(RS_cluster) > 1:
                new_CS_summary = summarize([RS_cluster])[0] # only one CS cluster in the list
                CS_summary.append(new_CS_summary)
            elif len(RS_cluster) == 1: # RS_cluster might be empty when RS has less points than centroids
                RS_list.append(RS_cluster[0][0]) # keep only the points coordinates

        del RS_points_result # remove points
        del RS_cluster_result

    '''
    Step 12. Merge CS clusters that are close enough. The nearness is calculated based on the average of
    Mahalanobis Distance of one cluster centroid to another CS cluster
    '''
    if merge_CS:
        CS_summary = mergeCS(CS_summary)

    '''
    Last round, merge CS clusters into closest DS cluster
    '''
    if round_idx == len(sorted(csv_files_list)) - 1: #
        if len(CS_summary) > 0:
            for CS in CS_summary:
                min_ED_CSDS = float('inf')
                target_DS_idx = 0
                CS_cent = tuple([d/CS['N'] for d in CS['SUM']])
                for DS_idx, DS in enumerate(DS_summary): # go through each DS, found the cloest one
                    DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                    curr_ED_CSDS = euclideanDist(CS_cent, DS_cent)
                    if curr_ED_CSDS < min_ED_CSDS:
                        min_ED_CSDS = curr_ED_CSDS
                        target_DS_idx = DS_idx
                # merge current CS to DS with min_ED_CSDS
                DS_summary[target_DS_idx]['N'] = DS_summary[target_DS_idx]['N'] + CS['N']
                new_SUM = tuple([sum(zipped) for zipped in zip(DS_summary[target_DS_idx]['SUM'], CS['SUM'])])
                new_SUMSQ = tuple([sum(zipped) for zipped in zip(DS_summary[target_DS_idx]['SUMSQ'], CS['SUMSQ'])])
                DS_summary[target_DS_idx]['SUM'] = new_SUM
                DS_summary[target_DS_idx]['SUMSQ'] = new_SUMSQ
            CS_summary = []

        '''
        for RS_point in RS_list:
            min_ED_RSDS = float('inf')
            target_DS_idx = 0
            for DS_idx, DS in enumerate(DS_summary): # go through each DS, found the cloest one
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED_RSDS = euclideanDist(RS_point, DS_cent)
                if curr_ED_RSDS < min_ED_RSDS:
                    min_ED_RSDS = curr_ED_RSDS
                    target_DS_idx = DS_idx
            # merge current CS to DS with min_ED_CSDS
            updateSummary(DS_summary, target_DS_idx, RS_point)
        RS_list = []
        '''

    '''
    Step 13. Output the intermediate result
    By now, we have DS_summary, CS_summary and RS_list
    '''
    intermediate_result = generateIntermediateResult(round_idx, DS_summary, CS_summary, RS_list)
    intermediate_result_list.append(intermediate_result)
del CS_summary # all points info are now in the DS_summary

'''
Assign each point to corresponding DS cluster
'''
result_dict = dict()
if cache_datapoints:
    for idx_point_pair in all_datapoints:
        min_ED = float('inf')
        pt_idx = str(idx_point_pair[0])
        point = idx_point_pair[1]
        DS_idx = 0
        if point in RS_list:
            result_dict[pt_idx] = -1
        else:
            for curr_DS_idx, DS in enumerate(DS_summary):
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED = euclideanDist(point, DS_cent)
                if curr_ED < min_ED:
                    min_ED = curr_ED
                    DS_idx = curr_DS_idx
            result_dict[pt_idx] = DS_idx
    del RS_list
else:
    for csv in sorted(csv_files_list):
        full_csv_path = input_folder_path + "\\" + csv
        points_list = (sc.textFile(full_csv_path)  # read a data chunk
                        .map(lambda line:line.split(',')) # split the raw data string by comma
                        .map(lambda pt:[float(coord) for coord in pt]) # cast the str coordinates to float
                        .map(lambda pt:(str(int(pt[0])), tuple(pt[1:]))) # form tuple of idx and coordinates
                        .collect()
                       )
        for idx_point_pair in points_list:
            min_ED = float('inf')
            pt_idx = str(idx_point_pair[0])
            point = idx_point_pair[1]
            DS_idx = 0
            for curr_DS_idx, DS in enumerate(DS_summary):
                DS_cent = tuple([d/DS['N'] for d in DS['SUM']])
                curr_ED = euclideanDist(point, DS_cent)
                if curr_ED < min_ED:
                    min_ED = curr_ED
                    DS_idx = curr_DS_idx
            result_dict[pt_idx] = DS_idx

output_cluster = open(output_path_cluster, 'w')
output_cluster.write(json.dumps(result_dict))
output_cluster.close()

output_inter = open(output_path_intermediate_result, 'w')
header = 'round_id,nof_cluster_discard,nof_point_discard,nof_cluster_compression,nof_point_compression,nof_point_retained\n'
output_inter.write(header)
for intermediate_result in intermediate_result_list:
    output_inter.write(intermediate_result + '\n')
output_inter.close()

print('Duration:', str(time.time()-startTime))


import pymongo
import re
from collections import OrderedDict
from datetime import datetime
import os

app_id_event_count_dict  = {}
app_id_app_name_dict = {}
app_id_app_name_list = []

myclient = pymongo.MongoClient("mongodb://localhost:27017", username="", password="")
raw_events_db = myclient["raw_events_db"]
events_collection = raw_events_db["events"]
applications_db = myclient['applications_db']
applications_collection= applications_db['applications']

my_control_type_array = []
automation_id_count_dict = {}
# temp_application_name = ''
control_type_control_name_set_dict = {}
control_type_automation_id_set_dict = {}
control_type_automation_id_count_dict = {}

def application_name_dropdown():

    global myclient
    global raw_events_db
    global events_collection
    global applications_db
    global applications_collection

    global app_id_app_name_dict
    global app_id_app_name_list
    global app_id_event_count_dict

    applications_documents = applications_collection.find({},projection = {"_id":1,"name":1}).sort("name")
    temp_app_id_name_dict = {}
    for x in applications_documents:
        app_id = x['_id'].strip()
        app_name = x['name'].strip()
        app_id_app_name_dict[app_id] = app_name
        temp_app_id_name_dict = {"app_id": app_id, "app_name": app_name}
        app_id_app_name_list.append(temp_app_id_name_dict)
    # print("# Applications: ",len(app_id_app_name_dict))

    #Get events per application
    pipeline= [{"$group": {"_id": "$application_id", "count": {"$sum": 1}}}]
    app_id_event_count_doc = events_collection.aggregate(pipeline)

    total_count = 0
    for x in app_id_event_count_doc:
        app_id = x['_id'].strip()
        event_count = x['count']
        total_count += event_count
        app_id_event_count_dict[app_id] = event_count

    app_id_event_count_dict =  OrderedDict(sorted(app_id_event_count_dict.items(), key=lambda kv:(kv[1], kv[0]), reverse=True))
    # for app_id in app_id_event_count_dict:
    #     print(app_id," - ", app_id_app_name_dict[app_id], " - ", app_id_event_count_dict[app_id])

    # print("Total Event Count: ",total_count)

    # return app_id_app_name_list
    return app_id_app_name_dict

def application_name_table(application_name_selection): # take app id input then show control

    global automation_id_count_dict
    global my_control_type_array
    # global temp_application_name

    my_control_type_array = []

    dirname = os.path.dirname(__file__)
    filename = os.path.join(dirname, '../Schneider/')

    for app_id in app_id_event_count_dict:
        app_id_title_docs = events_collection.find({"application_id": app_id}, projection={'title':1, 'specifications.event_path':1})

        title_set = set({})
        event_path_set = set({})
        for x in app_id_title_docs:
            title_set.add(x['title'])
            event_path_set.add(x['specifications']['event_path'])

        f = open(filename+app_id_app_name_dict[app_id].replace(":","_").replace("-","_")+"_title.txt",'w', encoding="utf-8")
        for title in title_set:
            f.write(title)
            f.write("\n")
        f.close()

        f = open(filename+app_id_app_name_dict[app_id].replace(":","_").replace("-","_")+"_event_path.txt",'w', encoding="utf-8")
        for event_path in event_path_set:
            f.write(event_path)
            f.write("\n")
        f.close()

    input_dir = filename
    application_name = application_name_selection.replace("-","_")

    global control_type_control_name_set_dict
    global control_type_automation_id_set_dict
    global control_type_automation_id_count_dict

    #control_type_control_name_set_dict.clear()

    f = open(input_dir + application_name+"_event_path.txt",'r', encoding='utf-8')
    lines = f.readlines()
    f.close()

    for line in lines:
        event_path = line.strip()
        if len(event_path) == 0:
            continue

        event_path_items = event_path.split("\/")
        for event_path_item in event_path_items:
            event_path_item_elements = event_path_item.split("\|")
            if len(event_path_item_elements) < 4:
                continue
            automation_id = event_path_item_elements[1][3:].strip()
            control_type = event_path_item_elements[2][2:].strip()
            control_name = event_path_item_elements[3][2:].strip()

            if control_type in control_type_control_name_set_dict:
                control_type_control_name_set = control_type_control_name_set_dict[control_type]
            else:
                control_type_control_name_set = set({})
            control_type_control_name_set.add(control_name)
            control_type_control_name_set_dict[control_type] = control_type_control_name_set

            if control_type in control_type_automation_id_set_dict:
                control_type_automation_id_set = control_type_automation_id_set_dict[control_type]

            else:
                control_type_automation_id_set = set({})
            control_type_automation_id_set.add(automation_id)
            control_type_automation_id_set_dict[control_type] = control_type_automation_id_set

            if control_type in control_type_automation_id_count_dict:
                automation_id_count_dict = control_type_automation_id_count_dict[control_type]
            else:
                automation_id_count_dict = {}
            count = 1
            if automation_id in automation_id_count_dict:
                count += automation_id_count_dict[automation_id]
            automation_id_count_dict[automation_id] = count
            control_type_automation_id_count_dict[control_type] = automation_id_count_dict

    control_type_control_name_count_dict = {}
    control_type_control_name_count_dict.clear()
    for control_type in control_type_control_name_set_dict:
        control_type_control_name_count_dict[control_type] = len(control_type_control_name_set_dict[control_type])
    control_type_control_name_count_dict = OrderedDict(sorted(control_type_control_name_count_dict.items(), key=lambda kv:(kv[1], kv[0]), reverse=True))

    for control_type in control_type_automation_id_count_dict:
        automation_id_count = control_type_automation_id_count_dict[control_type]
        automation_id_count = OrderedDict(sorted(automation_id_count.items(), key=lambda kv:(kv[1], kv[0]), reverse=True))
        control_type_automation_id_count_dict[control_type] = automation_id_count

    for control_type in control_type_control_name_count_dict:
        my_control_type_array.append({'control_type': control_type, 'distinct_values': control_type_control_name_count_dict[control_type]})

    return my_control_type_array

# def control_type_dropdown():
#     global my_control_type_array

# def data_type_dropdown():
#     dropdown_options = ['Control Name', 'Automation ID']
#     return dropdown_options

def control_name_table(id, control_type_selected):
    global control_type_control_name_set_dict
    my_control_name_list = []
    for control_name in control_type_control_name_set_dict[control_type_selected]:
        my_control_name_list.append({'control_name':control_name, 'events': 1})
    return my_control_name_list

def automation_id_table(id, control_type_selected):
    global automation_id_count_dict
    global control_type_automation_id_count_dict
    automation_id_list = []
    for automation_id in control_type_automation_id_count_dict[control_type_selected]:
        automation_id_list.append({'automation_id':automation_id, 'events':automation_id_count_dict[automation_id]})
    return automation_id_list

    # def prune_hashes(sentence):
    #     regex = re.compile("#[A-Fa-f0-9]{64,}#", re.IGNORECASE)
    #     modified_sentence = sentence
    #     matches = re.findall(regex, modified_sentence)
    #     for match in matches:
    #         modified_sentence = modified_sentence.replace(match, '')

    #     return modified_sentence.strip()