"""Load html from files, clean up, split, ingest into FAISS."""
import pickle
from typing import Any, List, Optional, Tuple
from langchain.docstore.document import Document
from langchain.document_loaders import WebBaseLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import TokenTextSplitter
from langchain.vectorstores.faiss import FAISS
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import requests
from selenium import webdriver
from selenium.webdriver.firefox.service import Service as FirefoxService
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import contextlib
import lxml.html as LH
import lxml.html.clean as clean
import requests
import os
import re
import tqdm
import time

class APIReferenceLoader(WebBaseLoader):
    """
    Loader that uses Elinks and Selenium to load webpages.
    With customization to scrape the code.

    :param web_path: a string, the path of the website to be scraped
    :param header_template: an optional dictionary, for customizing request headers
    :param strategy: an optional string, specifies the scraping strategy,
                     defaults to "selenium_elinks"
    :param is_visible_scrape: a bool, whether to perform a visible content scraping,
                              defaults to False
    """

    def __init__(self, web_path: str, header_template: Optional[dict] = None, strategy: Optional[str] = "selenium_elinks", is_visible_scrape: bool = False):
        # Initialize the WebBaseLoader
        super().__init__(web_path=web_path, header_template=header_template)

        # Initialize the Firefox driver
        self.driver = self.init_firefox_driver()

        # Set the scraping strategy
        self.strategy = strategy

        # Set whether to perform a visible content scraping
        self.is_visible_scrape = is_visible_scrape

    def _scrape_bs4(self, url: str) -> Any:
        """
        Scrape the webpage using BeautifulSoup4.

        :param url: a string, the URL of the webpage to be scraped
        :return: a BeautifulSoup object
        """
        # Send a GET request to the URL
        html_doc = self.session.get(url)

        # Create a BeautifulSoup object from the HTML text
        soup = BeautifulSoup(html_doc.text, "html.parser")

        return soup

    def load(self) -> List[Document]:
        """
        Load data into document objects.

        :return: a List of Document objects containing the scraped content
        """
        # Implement different scraping strategies
        if self.strategy == "bs4":
            soup = self._scrape_bs4(self.web_path)
            text = soup.get_text()
        elif self.strategy == "selenium_elinks":
            text = self._scrape_SelElinks(self.web_path)
        else:
            raise ValueError("Strategy not supported")

        metadata = {"source": self.web_path}
        return [Document(page_content=text, metadata=metadata)]

    def find_common_words(self, s, t):
        """
        Find the common words between two strings.

        :param s: a string
        :param t: another string
        :return: a List of common words between s and t
        """

        # Split input strings into words
        s_words = s.split()
        t_words = t.split()

        # Find common words
        common_words = [word for word in s_words if word in t_words]

        return common_words

    def insert_missing_words(self, s, t, common_words):
        """
        Insert missing words from visible text into the target text (structured elements).

        :param s: a string, the source string
        :param t: a string, the target string
        :param common_words: a List of common words between s and t
        :return: a string, the target string with missing words inserted
        """
        s_words = s.split()
        t_words = t.split()
        missing_words = []

        for i in range(len(common_words)-1):
            start, end = common_words[i], common_words[i+1]
            start_idx = s_words.index(start)
            end_idx = s_words.index(end)
            missing_words.extend(s_words[start_idx+1:end_idx])

        for word in missing_words:
            if word not in t_words:
                t_words.insert(t_words.index(common_words[-1])+1, word)

        t_new = " ".join(t_words)
        return t_new

    def init_firefox_driver(self):
        """
        Initialize a headless Firefox browser.

        :return: a webdriver.Firefox object, representing the headless browser
        """
        options = Options()
        options.headless = True
        options.binary = FirefoxBinary("/usr/bin/firefox")
        service = FirefoxService(executable_path="geckodriver")
        driver = webdriver.Firefox(service=service, options=options)
        return driver

    def scrape_visible_elements(self, url):
        """
        Scrape the visible elements of the page using a headless browser and Selenium.

        :param url: a string, the URL of the webpage to be scraped
        :return: a string, the scraped visible content of the webpage
        """
        ignore_tags = ('style')
        with contextlib.closing(self.driver) as browser:
            browser.get(url)  # Load page
            time.sleep(10)
            content = browser.page_source
            cleaner = clean.Cleaner()
            content = cleaner.clean_html(content)
            doc = LH.fromstring(content)
            texts = []
            for elt in doc.iterdescendants():
                if elt.tag in ignore_tags:
                    continue
                text = elt.text or ''
                tail = elt.tail or ''
                words = ' '.join((text, tail)).strip()
                if words:
                    texts.append(words)
            return " ".join(texts)

    def scrape_structured_elements(self, url):
        """
        Scrape the structured elements of the page using text-based web browser Elinks.

        :param url: a string, the URL of the webpage to be scraped
        :return: a string, the scraped structured content of the webpage
        """
        response = self.session.get(url)
        with open("/tmp/struct.html", "w") as f:
            f.write(response.text)
        os.system("elinks --dump /tmp/struct.html > /tmp/struct.txt")
        with open("/tmp/struct.txt", "r") as f:
            lines = f.readlines()
        text = "".join(lines)
        return text

    def _scrape_SelElinks(self, url):
        """
        Combine the best from both worlds: visual and structured elements.

        :param url: a string, the URL of the webpage to be scraped
        :return: a string, the combined scraped content of the webpage
        """
        struct_text = self.scrape_structured_elements(url)
        struct_text = self.clean_text(struct_text)

        if self.is_visible_scrape:
            vis_text = self.scrape_visible_elements(url)
            t_joint = self.insert_missing_words(
                vis_text, struct_text, self.find_common_words(vis_text, struct_text))
            with open("/tmp/debug_vis.txt", "w") as f:
                f.write(self.clean_text(t_joint))
            return self.clean_text(t_joint)
        return struct_text

    def clean_text(self, text):
        """
        Clean up the text.

        :param text: a string, the text to be cleaned
        :return: a string, the cleaned text
        """
        delete_str = "Visible links"
        index = text.find(delete_str)
        if index != -1:
            text = text[:index]
        text = re.sub(r'\n\s*\n', '\n', text.strip())
        text = re.sub(r' {2,}', ' ', text)
        text = re.sub(r'-{3,}', '--', text)
        text = re.sub(r'═{3,}', '==', text)
        text = re.sub(r'_', '', text)
        text = re.sub(r'`', '', text)
        text = re.sub(r'Link: \[\d+\]prefetch','', text)
        text = re.sub(r'Link: \[\d+\]preload','', text)
        text = re.sub(r'Link: \[\d+\]preconnect','', text)
        text = re.sub(r'Link: \[\d+\]canonical','', text)
        text = re.sub(r'Link: \[\d+\]alternate','', text)
        text = re.sub(r'\[\d+\]', '', text)
        return text

    def clean_table_content(self, text):
        pass

def hierarchy_links(url_docs: str, recursive_depth: int = 1, current_depth: int = 1) -> List[str]:
    """
    Get all links from a web page up to a specified recursion depth.

    :param url_docs: a string, the URL of the web page
    :param recursive_depth: an optional integer, the maximum recursion depth, defaults to 1
    :param current_depth: an optional integer, the current recursion depth, defaults to 1
    :return: a List of strings, the URLs of documents collected from the web page
    """

    # Check if we have reached the maximum recursion depth
    if current_depth > recursive_depth and recursive_depth != 0:
        return []
    elif recursive_depth == 0:
        return [url_docs]

    # Send a GET request to the provided URL
    reqs = requests.get(url_docs)
    # Create a BeautifulSoup object from the HTML content
    soup = BeautifulSoup(reqs.text, 'html.parser')
    # Initialize the list for collected document links
    docs_link = list()
    # Iterate over all the links in the web page
    for link in soup.find_all('a'):
        # Create an absolute URL by joining the base URL and the href attribute
        ref_link = urljoin(url_docs, link.get('href'))
        # Check if the URL is valid, not equal to the base URL, and not already in the list
        if url_docs in ref_link and ref_link is not None and url_docs != ref_link:
            docs_link.append(ref_link)
            # Recursively collect links if maximum depth is not yet reached
            if current_depth < recursive_depth:
                docs_link.extend(
                    hierarchy_links(ref_link, recursive_depth, current_depth + 1)
                )

    # Return the list of collected document links
    return docs_link

def ingest_docs(url_docs: str, recursive_depth: int = 1, return_summary: bool = True, logger=None) -> Tuple[List, List]:
    """
    Get documents from web pages.

    :param url_docs: a string, the URL of the web page
    :param recursive_depth: an optional integer, the maximum recursion depth for getting links, defaults to 1
    :param return_summary: an optional bool, whether to return a summary of documents, defaults to True
    :param logger: an optional logging object, for logging progress and information, defaults to None
    :return: a Tuple with two Lists,
             first, the list of documents collected from the web pages,
             second, the list of documents used for summary (if return_summary is True)
    """
    embeddings = OpenAIEmbeddings()
    # Get links from the web page, up to the specified recursion depth
    docs_link = set(hierarchy_links(url_docs, recursive_depth))
    # Initialize the lists for collected documents and document summaries
    documents = list()
    docs_for_summary = list()
    logger.info(f"Crawling {docs_link} ...")
    # Iterate over the collected links
    for link in tqdm.tqdm(docs_link):
        # Initialize an APIReferenceLoader with the option to scrape visible content
        loader = APIReferenceLoader(link, is_visible_scrape=True)
        # Load the raw documents
        raw_documents = loader.load()
        # Initialize text splitters for documents and summaries
        text_splitter = TokenTextSplitter(chunk_size=1200, chunk_overlap=150)
        text_splitter_sum = TokenTextSplitter(chunk_size=3100, chunk_overlap=300)
        # Split documents for summary and document lists
        if return_summary:
            docs_for_summary.extend(text_splitter_sum.split_documents(raw_documents))
        documents.extend(text_splitter.split_documents(raw_documents))
    logger.info("Number of documents: {}".format(len(documents)))

    logger.info("Saving vectorstore into assets/vectorstore.pkl")
    vectorstore = FAISS.from_documents(documents, embeddings)
    with open("assets/vectorstore.pkl", "wb") as f:
        pickle.dump(vectorstore, f)

    return documents, docs_for_summary

if __name__ == "__main__":
    import logging
    logger = logging.getLogger(__name__)
    docs, docs_for_summary = ingest_docs("https://developers.notion.com/reference/create-a-token", recursive_depth=0, logger=logger)
    embeddings = OpenAIEmbeddings()
    vectorstore = FAISS.from_documents(docs, embeddings)
    with open("assets/vectorstore_debug.pkl", "wb") as f:
        pickle.dump(vectorstore, f)


import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import sys
import utils
import config as c

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

# Define variational autoencoder class
class VAE(nn.Module):
    def __init__(self):
        super(VAE, self).__init__()
        self.dim = (32, int(c.height / 4), int(c.width / 4))
        self.n_input = c.n_joints * 2

        # Encoder
        self.e_pool = nn.MaxPool2d(kernel_size=2, stride=2)

        self.e_conv1 = nn.Conv2d(3, 32, (3, 3), stride=(1, 1), padding=(1, 1))
        self.e_conv2 = nn.Conv2d(32, 32, (3, 3), stride=(1, 1), padding=(1, 1))

        # Variational latent variable layers
        self.fc_mu = nn.Linear(int(np.prod(self.dim)), self.n_input)
        self.fc_log_var = nn.Linear(int(np.prod(self.dim)), self.n_input)

        # Decoder
        self.d_fc1 = nn.Linear(self.n_input, int(np.prod(self.dim)))

        self.d_upconv3 = nn.ConvTranspose2d(32, 32, (4, 4), stride=(2, 2),
                                            padding=(1, 1))
        self.d_batch3 = nn.BatchNorm2d(32)
        self.d_conv3 = nn.Conv2d(32, 32, (3, 3), stride=(1, 1), padding=(1, 1))

        self.d_upconv4 = nn.ConvTranspose2d(32, 32, (4, 4), stride=(2, 2),
                                            padding=(1, 1))
        self.d_batch4 = nn.BatchNorm2d(32)
        self.d_conv4 = nn.Conv2d(32, 3, (3, 3), stride=(1, 1), padding=(1, 1))

        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()

        self.to(device)

    def encode(self, x):
        """
        Run the encoder
        :param x: input image
        :return: mu and log_var vector
        """
        out = self.relu(self.e_conv1(x))
        out = self.e_pool(out)

        out = self.relu(self.e_conv2(out))
        out = self.e_pool(out)

        out = out.reshape(out.size(0), -1)

        mu = self.fc_mu(out)
        log_var = self.fc_log_var(out)

        return mu, log_var

    @staticmethod
    def reparametrize(mu, log_var):
        """
        Randomly sample z based on mu and log_var vectors
        :param mu: latent mean joint angle vector
        :param log_var: latent log variance joint angle vector
        :return: z
        """
        std = torch.exp(0.5 * log_var)
        eps = torch.randn_like(std)

        return mu + eps * std

    def decode(self, z):
        """
        Run the decoder
        :param z: latent variable vector z
        :return: output image
        """
        out = self.relu(self.d_fc1(z))

        out = out.reshape(-1, *self.dim)

        out = self.relu(self.d_batch3(self.d_upconv3(out)))
        out = self.relu(self.d_conv3(out))

        out = self.relu(self.d_batch4(self.d_upconv4(out)))
        out = self.sigmoid(self.d_conv4(out))

        return out

    def forward(self, x):
        """
        Perform forward pass through the network
        :param x: input image
        :return: output image, mu and log_var vectors
        """
        mu, log_var = self.encode(x)
        z = self.reparametrize(mu, log_var)
        output = self.decode(z)

        return output, mu, log_var

    def predict_visual(self, x):
        """
        Get visual prediction
        :param x: input joint
        :return: output image
        """
        input_ = torch.tensor(x, device=device, dtype=torch.float,
                              requires_grad=True)
        output = self.decode(input_)

        return input_, output

    def predict_joint(self, x):
        """
        Get joint prediction
        :param x: input image
        :return: output joint
        """
        input_ = torch.tensor(x, device=device, dtype=torch.float).unsqueeze(0)
        output, _ = self.encode(input_)

        return output

    def get_grad(self, input_, output, error):
        """
        Get gradient with respect to prediction error
        :param input_: input tensor from the predict function
        :param output: output tensor from the predict function
        :param error: prediction error
        :return: numpy array containing the gradient
        """
        # Set gradient to zero
        input_.grad = torch.zeros(input_.size(), device=device,
                                  dtype=torch.float, requires_grad=False)

        output.backward(torch.tensor(error, dtype=torch.float,
                                     device=device))

        return input_.grad.detach().cpu().numpy()

    @staticmethod
    def train_net(net, x, y):
        """
        Train the neural network
        :param net: network object
        :param x: input samples
        :param y: output samples
        """
        torch.cuda.empty_cache()

        # Split dataset
        train_gen, test_gen = utils.split_dataset(x, y, 0.9)

        # Define optimizer
        optimizer = optim.Adam(net.parameters(), lr=c.learning_rate)
        scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=c.step_size,
                                              gamma=c.gamma)

        train_loss, test_loss, mse_loss = [], [], []

        # Start training
        for epoch in range(c.n_epochs):
            cur_train_loss = 0
            for x_train, y_train in train_gen:
                loss, _ = VAE.run_batch(x_train, y_train, True, net, optimizer)
                cur_train_loss += loss
            train_loss.append(cur_train_loss / len(train_gen))

            scheduler.step()

            # Evaluate network
            if (epoch + 1) % 5 == 0:
                cur_test_loss, cur_mse_loss = 0, 0
                for x_test, y_test in test_gen:
                    loss, mse = VAE.run_batch(x_test, y_test, False, net,
                                              optimizer)
                    cur_test_loss += loss
                    cur_mse_loss += mse
                test_loss.append(cur_test_loss / len(test_gen))
                mse_loss.append(cur_mse_loss / len(test_gen))

                sys.stdout.write('\rEpoch {:3d}  ===> \t Train loss: {:10.5f}'
                                 ' \t Test loss: {:10.5f} \t\t MSE: {:10.5f}'.
                                 format(epoch + 1, train_loss[-1],
                                        test_loss[-1], mse_loss[-1]))
                sys.stdout.flush()

    @staticmethod
    def run_batch(x_target, y_target, train, net, optimizer):
        """
        Execute a training batch
        :param x_target: input samples
        :param y_target: output samples
        :param train: variable for network training
        :param net: network object
        :param optimizer: optimizer
        """
        # Send to GPU
        x_target = x_target.to(device, torch.float32)
        y_target = y_target.to(device, torch.float32)

        # Clear gradient
        optimizer.zero_grad()

        # Forward pass
        y_predict, mu, log_var = net(y_target)

        # Compute loss
        loss, mse = VAE.loss_function(y_predict, y_target, x_target,
                                      mu, log_var)

        if train:
            # Perform optimization step
            loss.backward()
            optimizer.step()

        return loss.item(), mse.item()

    @staticmethod
    def loss_function(y_predict, y_target, x_target, mu, log_var):
        """
        Loss function of the VAE, based on the MSE of the images and
        regularization term based on the Kullback-Leibler divergence
        :param y_predict: predicted image
        :param y_target: target image
        :param x_target: ground truth joint angles
        :param mu: latent mean joint angle vector
        :param log_var: latent log variance joint angle vector
        :return: loss
        """
        criterion_mse = nn.MSELoss()

        mse = criterion_mse(y_predict, y_target)

        var_target = torch.full(log_var.shape, c.variance).to(
            device, torch.float32)
        kld = utils.kl_divergence(x_target, var_target, mu, log_var)

        return mse + kld, mse

    def save(self):
        """
        Save network to file
        """
        torch.save(self.state_dict(), 'network/vae.net')

    def load(self):
        """
        Load network from file
        """
        self.load_state_dict(torch.load('network/vae.net',
                                        map_location=device))
        self.eval()


import yaml
import argparse
import os

parser = argparse.ArgumentParser(description='Change config for evaluation')
parser.add_argument('--config_path', type=str, required=True,
                    help='path to config file')

parser.add_argument('--data_path', type=str, required=True, help='path to your own dataset')
args = parser.parse_args()

config_path = args.config_path.split("/")
def read_yaml_file(files):
    config = None
    with open(files, 'r') as stream:
        try:
           # config = yaml.load(stream, Loader=yaml.BaseLoader)
            config = yaml.load(stream)
            print("READ YAML")
        except yaml.YAMLError as exc:
            print(exc)
    return config

def write_yaml_file(files, data):
    with open(files, 'w') as outfile:
        yaml.dump(data, outfile, default_flow_style=False)

def change_dataset_dir(base_list):
    for idx in range(len(base_list)):
        if("datasets" in base_list[idx]):
            base_list[idx] = args.data_path
    return base_list

config = read_yaml_file(args.config_path)
print(config)
base_config = config['_BASE_']

if(len(base_config)==1):
    root_config_name = base_config[0]
    config_path[-1] = root_config_name
    root_config_path = "/".join(config_path)
    print("Root: ", root_config_path)
    root_config = read_yaml_file(root_config_path)
    root_config_base = root_config['_BASE_']
    root_config_base = change_dataset_dir(root_config_base)
    root_config['_BASE_'] = root_config_base
    write_yaml_file(root_config_path, root_config)
else:
    base_config = change_dataset_dir(base_config)
    config['_BASE_'] = base_config
    print("New config file saved: ", args.config_path)
    write_yaml_file(args.config_path, config)


from paho.mqtt import client as paho_mqtt_client
from datetime import datetime
from config import config
from Classes.db import db

def on_connect(client, userdata, flags, rc):
    if rc == 0:
        print("Connected to MQTT Broker!")
    else:
        print("Failed to connect, return code %d\n", rc)

def on_message(client, userdata, message):
    message_res = "Received message: " + str(message.payload.decode('UTF-8')) + " on topic: " + message.topic + " QoS: " + str(message.qos)
    # print(message_res)
    with open('subscriber_log.txt', 'a') as out_file:
        out_file.write(f'{datetime.now()} {message_res}\n')
    topic_l = str(message.topic).split('/')
    device, sensor = None, None
    if len(topic_l) > 3 and topic_l[0] == 'devices' and topic_l[2] == 'sensors':
        device = topic_l[1]
        sensor = topic_l[3]
        value = float(message.payload.decode('UTF-8'))
        db.add_sensor_value(device_key=device, sensor_key=sensor, value=value)
    else:
        print(f'{datetime.now()} {message_res}')

class MqttClient:
    def __init__(self, config: config):
        """
        """
        self.config = config
        self.client: paho_mqtt_client = None
        self.broker = config.broker
        self.port = config.port
        self.client_id = config.client_id
        self.connected = str(None)

    def connect(self):
        def on_connect(client, userdata, flags, rc):
            if rc == 0:
                print("Connected to MQTT Broker!")
                self.connected = True
            else:
                print("Failed to connect, return code %d\n", rc)
        # Set Connecting Client ID
        self.client = paho_mqtt_client.Client(self.client_id)
        # client.username_pw_set(username, password)
        self.client.on_connect = on_connect
        self.client.connect(self.broker, self.port)

    def subscribe(self, topics: list) -> list:
        result_list = []
        if not self.connected:
            self.connect()
        for topic in topics:
            print(f'Subscribing to {topic}')
            subscribe_result = self.client.subscribe(topic)
            callback_result = self.client.message_callback_add(topic, on_message)
            print(f'Subscribe result: {subscribe_result}, callback_result = {callback_result}')
            result_list.append((topic, subscribe_result))
        self.client.on_message = on_message
        return result_list

    def run_loop(self):
        """
        connect, subscribe and loop mqtt client with config
        """
        self.connect()
        self.subscribe(self.config.topics)
        self.loop()

    def loop(self):
        self.client.loop_forever(timeout=5.0)

if __name__ == '__main__':
    print(config)
    mqtt_client = MqttClient(config)
    mqtt_client.run_loop()
    print(mqtt_client)


from logging import getLogger
from networkx import Graph, connected_components
from MetAromatic.pair import MetAromatic
from MetAromatic.complex_types import TYPE_MA_PARAMS, TYPE_BRIDGE_SPACE

class GetBridgingInteractions:
    log = getLogger("met-aromatic")

    def __init__(self, ma_params: TYPE_MA_PARAMS) -> None:
        self.ma_params = ma_params
        self.f: TYPE_BRIDGE_SPACE

    def get_interacting_pairs(self, code: str) -> bool:
        ma_results = MetAromatic(self.ma_params).get_met_aromatic_interactions(code)

        if not ma_results["OK"]:
            self.log.error(
                "Cannot get bridging interactions as Met-aromatic algorithm failed"
            )

            self.f["OK"] = False
            self.f["status"] = ma_results["status"]
            return False

        if len(ma_results["interactions"]) < 1:
            self.log.info(
                "No Met-aromatic interactions were found therefore cannot find bridges"
            )

            self.f["status"] = "No Met-aromatic interactions were found"
            return False

        for interaction in ma_results["interactions"]:
            pair = (
                f"{interaction['aromatic_residue']}{interaction['aromatic_position']}",
                f"MET{interaction['methionine_position']}",
            )
            self.f["interactions"].add(pair)

        return True

    def isolate_connected_components(self, vertices: int) -> None:
        graph = Graph()
        graph.add_edges_from(self.f["interactions"])

        for bridge in connected_components(graph):
            if len(bridge) == vertices:
                self.f["bridges"].append(bridge)

        # Note that inverse bridges (MET-ARO-MET) not removed!

        num_bridges = len(self.f["bridges"])

        if num_bridges > 0:
            if num_bridges == 1:
                self.log.info("Found 1 bridge")
            else:
                self.log.info("Found %i bridges", num_bridges)

            return

        self.log.info("Found no bridges")
        self.f["status"] = "No bridges"

    def get_bridging_interactions(self, code: str, vertices: int) -> TYPE_BRIDGE_SPACE:
        self.log.info('Locating bridging interactions for entry "%s"', code)

        self.f = {"interactions": set(), "bridges": [], "OK": True, "status": "Success"}

        if not self.get_interacting_pairs(code):
            return self.f

        self.isolate_connected_components(vertices=vertices)

        return self.f


import re
from utils import inputfile_to_array, remove_empty_lines_and_concat
from math import floor, ceil
from statistics import mean

luggage_rules = inputfile_to_array("inputs/input_day_7.txt")

def bags_that_can_contain_directly(luggage_rules, bags_to_search_for):

    bags_that_can_contain_bag_to_search_for = set()
    for luggage_rule in luggage_rules:
        outer_bag = luggage_rule.split("contain")[0].strip().replace("bags","").strip()
        content   = luggage_rule.split("contain")[-1].strip()
        inner_bags_unsanitized = content.split(",")
        inner_bags = []
        for bag in inner_bags_unsanitized:
            inner_bags.append(bag.replace(".","").replace("bags","").replace("bag","").strip())
        for bag_to_search_for in bags_to_search_for:
            for inner_bag in inner_bags:
                if bag_to_search_for in inner_bag:
                    bags_that_can_contain_bag_to_search_for.add(outer_bag)
    return (bags_that_can_contain_bag_to_search_for)

def get_value_of_bag(luggage_rules, input_bag):
    '''
    Value is meant as how many bags it must contain (plus itself). Each individual bag adds 1 in value
    '''
    content_of_bag = get_content_of_bag(luggage_rules, input_bag)
    if content_of_bag:
        value = 1
        for i in content_of_bag:
            number,bag = i
            value +=  int(number) * int(get_value_of_bag(luggage_rules, bag))
        return value
    else:
        return 1

def get_content_of_bag(luggage_rules, input_bag):

    number_and_bag_list = []
    for luggage_rule in luggage_rules:
        outer_bag = luggage_rule.split("contain")[0].strip().replace("bags","").strip()
        if outer_bag == input_bag:
            unsanitized_content = luggage_rule.split("contain")[-1].strip().split(",")
            if "no other" in unsanitized_content[0]:
                return False
            for inner_bag in unsanitized_content:
                inner_bag = inner_bag.strip()
                number_and_bag_list.append((inner_bag.split(" ")[0]," ".join(inner_bag.split(" ")[1:3])))
    return number_and_bag_list

def task_a(luggage_rules):
    bags_to_look_for = {"shiny gold"}
    solution_bags = set()
    while bool(bags_to_look_for):
        (bags_from_this_run)  = bags_that_can_contain_directly(luggage_rules, bags_to_look_for)
        solution_bags = solution_bags | bags_from_this_run
        bags_to_look_for = bags_from_this_run
    return len(solution_bags)

def task_b(luggage_rules):
    bag = "shiny gold"
    return get_value_of_bag(luggage_rules, bag)-1 # Here I have to subtract 1 because the top bag itself is not to be counted

print("Number of bags that can have at least one shiny gold bag in task a:", task_a(luggage_rules))
print("Number of bags included in a shiny gold bag in task b:", task_b(luggage_rules))


import sys
import numpy as np
from PyQt5.QtWidgets import QWidget
from PyQt5.QtWidgets import QApplication
from PyQt5.QtWidgets import QTableWidgetItem
import unittest
from unittest.mock import MagicMock

sys.path.insert(1, 'src/')
from widgets import MplCanvas, LineCanvas, ImageCanvas, Table

class TestLineCanvas(unittest.TestCase):

    def setUp(self):
        self.det = MagicMock()
        self.det.center_mass = np.array([[1, 10, 20], [2, 20, 30], [3, 30, 40]])
        self.canvas = LineCanvas(det_=self.det, current_image_idx=1)

    # Проверяет очистку и обновление графика при отрисовке линейного графика
    def test_draw_line(self):
        self.canvas.axes.clear = MagicMock()
        self.canvas.update_canvas = MagicMock()

        self.canvas.draw_line()

        self.canvas.axes.clear.assert_called_once()
        self.canvas.update_canvas.assert_called_once()

    # Проверяет отрисовку линейного графика с текущей точкой в красном цвете
    def test_draw_line_with_current_point(self):
        self.canvas.axes.clear = MagicMock()
        self.canvas.update_canvas = MagicMock()

        self.canvas.current_image = 2
        self.canvas.draw_line()

        self.canvas.axes.clear.assert_called_once()
        self.canvas.update_canvas.assert_called_once()

class TestImageCanvas(unittest.TestCase):

    def setUp(self):
        self.det = MagicMock()
        self.det.img = np.random.rand(10, 10)
        self.det.img_contour = np.random.rand(10, 10)
        self.canvas = ImageCanvas(det_=self.det)

    # Проверяет загрузку изображения без контура и обновление отображения
    def test_load_image_without_contour(self):
        self.canvas.img = MagicMock()
        self.canvas.axes.draw_artist = MagicMock()
        self.canvas.fig.canvas.blit = MagicMock()

        self.canvas.load_image()

        self.canvas.img.set_data.assert_called_once_with(self.det.img)
        self.canvas.axes.draw_artist.assert_called_once()
        self.canvas.fig.canvas.blit.assert_called_once()

    # Проверяет загрузку изображения с контуром и обновление отображения
    def test_load_image_with_contour(self):
        self.canvas.img = MagicMock()
        self.canvas.axes.draw_artist = MagicMock()
        self.canvas.fig.canvas.blit = MagicMock()
        self.canvas.contour_or_not = 1

        self.canvas.load_image()

        self.canvas.img.set_data.assert_called_once_with(self.det.img_contour)
        self.canvas.axes.draw_artist.assert_called_once()
        self.canvas.fig.canvas.blit.assert_called_once()

    # Проверяет переключение на следующее изображение и его загрузку
    def test_draw_next_image(self):
        self.canvas.current_image_idx = 1
        self.canvas.load_image = MagicMock()

        self.canvas.draw_next_image()

        self.assertEqual(self.canvas.current_image_idx, 2)
        self.canvas.load_image.assert_called_once()

    # Проверяет переключение на предыдущее изображение и его загрузку
    def test_draw_previous_image(self):
        self.canvas.current_image_idx = 1
        self.canvas.load_image = MagicMock()

        self.canvas.draw_previous_image()

        self.assertEqual(self.canvas.current_image_idx, 0)
        self.canvas.load_image.assert_called_once()

    # Проверяет обновление текущего изображения на основе значения слайдера
    def test_update_image_from_slider(self):
        self.canvas.current_image_idx = 0
        self.canvas.load_image = MagicMock()

        self.canvas.update_image_from_slider(200)

        self.assertEqual(self.canvas.current_image_idx, 2)
        self.canvas.load_image.assert_called_once()

class TestTable(unittest.TestCase):

    def setUp(self):
        self.det = MagicMock()
        self.det.center_mass = np.array([[1, 10, 20], [2, 20, 30], [3, 30, 40]])
        self.table = Table(current_image_idx=1, det=self.det)
        self.app = QApplication([])

    def tearDown(self):
        self.app.quit()

    # Проверяет отображение строк таблицы, соответствующих текущему изображению
    def test_show_row(self):
        self.table.numbers_of_rows = MagicMock(return_value=[0, 1, 2])
        self.table.setItem = MagicMock()

        self.table.show_row(1)

        self.table.numbers_of_rows.assert_called_once()
        self.table.setItem.assert_called()

    # Проверяет возвращение индексов строк для первого изображения
    def test_numbers_of_rows_first_image(self):
        self.table.index = 0

        rows = self.table.numbers_of_rows()

        self.assertEqual(rows, [0, 1, 2])

    # Проверяет возвращение индексов строк для последнего изображения
    def test_numbers_of_rows_last_image(self):
        self.table.index = 2

        rows = self.table.numbers_of_rows()

        self.assertEqual(rows, [0, 1, 2])

    # Проверяет возвращение индексов строк для среднего изображения
    def test_numbers_of_rows_middle_image(self):
        self.table.index = 1

        rows = self.table.numbers_of_rows()

        self.assertEqual(rows, [0, 1, 2])

def run_tests():
    unittest.main()

if __name__ == '__main__':
    run_tests()


from PyQt5.QtWidgets import (QApplication, QPushButton,QWidget,QSizePolicy,
                             QMainWindow,QAction,QVBoxLayout,QDockWidget,
                             QLabel,QFileDialog,QTextEdit,QCheckBox)
from PyQt5.QtGui import QFont
from PyQt5.QtCore import Qt,QEvent
import sys
from matplotlib.figure import Figure
from matplotlib.backends.backend_qt5agg import (
        FigureCanvas, NavigationToolbar2QT as NavigationToolbar)

class Interface(QMainWindow):
    current_graph='Scatter Plot'
    plot_type=['Scatter Plot', 'Line Plot','Histogram','Pie Chart',
               'Bar Chart','Double Bar Chart']
    header_labels={'Scatter Plot':'X Data, Y Data',
                   'Line Plot':'X Data, Y Data',
                   'Histogram':'X Data, Y Data',
                   'Pie Chart':'Percent',
                   'Bar Chart':'X Label, Height',
                   'Double Bar Chart':'X Label, Height1, Height2'}
    data_types={'Scatter Plot':['float','float'],
                   'Line Plot':['float','float'],
                   'Histogram':['float','float'],
                   'Pie Chart':['float'],
                   'Bar Chart':['str','float'],
                   'Double Bar Chart':['str','float','float']}
    function_calls={'Scatter Plot':'scatter',
                   'Line Plot':'plot',
                   'Histogram':'hist',
                   'Pie Chart':'pie',
                   'Bar Chart':'bar',
                   'Double Bar Chart':'beast'}
    function_kwargs={'Scatter Plot':None,
                   'Line Plot':None,
                   'Histogram':None,
                   'Pie Chart':"autopct='%1.1f%%'",
                   'Bar Chart':None,
                   'Double Bar Chart':None}
    def __init__(self):
        super().__init__()
        self.setWindowTitle('Live Plotter')
        self.size_policy=QSizePolicy.Expanding
        self.font=QFont()
        self.font.setPointSize(12)
        self.geometry()
        self.menu()
        self.showMaximized()
        self.show()

    def menu(self):
        self.menuType=self.menuBar().addMenu('&Chart Type')
        self.actions={}
        for i in self.plot_type:
            self.actions[i]=self.action_definer(i)
            self.menuType.addAction(self.actions[i])

    def action_definer(self,label):
        action=QAction('&{}'.format(label),self,checkable=True)
        action.setFont(self.font)
        action.triggered.connect(lambda: self.action_changed(label))
        if label==self.plot_type[0]:
            action.setChecked(True)
        return action

    def action_changed(self,label):
        self.text_format.setText(self.header_labels[label])
        for i in self.actions:
            if i!=label:
                self.actions[i].setChecked(False)
            else:
                self.actions[i].setChecked(True)
                self.current_graph=i
                self.update_graph()

    def geometry(self):
        #configure the main graph
        self.plot_window=QWidget()
        layout=QVBoxLayout()
        self.figure=Figure()
        self._canvas=FigureCanvas(self.figure)
        self.toolbar=CustomToolbar(self._canvas,self)
        layout.addWidget(self.toolbar)
        layout.addWidget(self._canvas)
        self.plot_window.setLayout(layout)
        self.setCentralWidget(self.plot_window)
        self.axis = self._canvas.figure.subplots()
        self.figure.tight_layout()
        self.setCentralWidget(self.plot_window)

        #now configure the right side to be a plain text editor
        right_dock=QDockWidget('Data')
        right_widget=QWidget()
        self.text_format=QLabel(self.header_labels['Scatter Plot'])
        self.text_format.setFont(self.font)

        # self.data_editor=QTextEdit(self)
        self.data_editor=Custom_Text_Editor(self)
        self.data_editor.setSizePolicy(self.size_policy, self.size_policy)
        self.data_editor.setFont(self.font)
        self.data_editor.installEventFilter(self)

        self.hold_plot_on=QCheckBox('Reset Graph')
        self.hold_plot_on.setChecked(True)
        self.hold_plot_on.setFont(self.font)

        self.process=QPushButton('Update')
        self.process.setFont(self.font)
        self.process.clicked.connect(self.update_graph)

        layout=QVBoxLayout()
        layout.addWidget(self.text_format)
        layout.addWidget(self.data_editor)
        layout.addWidget(self.hold_plot_on)
        layout.addWidget(self.process)
        right_widget.setLayout(layout)
        right_dock.setWidget(right_widget)
        self.addDockWidget(Qt.LeftDockWidgetArea, right_dock)

    def eventFilter(self, obj, event):
        if event.type() == QEvent.KeyPress and obj is self.data_editor:
            if event.key() == Qt.Key_Return or event.key()==Qt.Key_Enter \
                and self.data_editor.hasFocus():
                self.update_graph()
        return super().eventFilter(obj, event)

    def update_graph(self):
        if self.hold_plot_on.isChecked():
            self.axis.clear()
        #get the data from the text editor first
        data=self.data_import(self.data_editor.toPlainText())
        #now actually plot the data
        d_lists=''
        for i in range(len(data)-1):
            d_lists+='data[{}],'.format(i)
        d_lists+='data[{}]'.format(len(data)-1)
        if self.function_kwargs[self.current_graph]!=None:
            d_lists+=',{}'.format(self.function_kwargs[self.current_graph])
        try:
            exec('{}.{}({})'.format('self.axis',
                                        self.function_calls[self.current_graph],
                                        d_lists))
        except Exception as e:
            print(e)
        #now handle the various options for the pie charts and bar charts
        self._canvas.draw()
        self.figure.tight_layout()

    def data_import(self,text):
        columns=text.split(sep='\n')
        #get the data types to try and convert based on the selected
        #graph style
        d_type=self.data_types[self.current_graph]
        data={}
        for i in range(len(d_type)):
            data[i]=[]

        for i in columns:
            splitr=i.split(sep=',')
            for j in range(len(splitr)):
                value=splitr[j]
                if value!='':
                    try:
                        if d_type[j]!='str':
                            data[j].append(eval('{}({})'.format(d_type[j],value)))
                        else:
                            data[j].append(value)
                    except Exception as e:
                        print(e)
        return data

class Custom_Text_Editor(QTextEdit):
    def __init__(self,parent):
        super().__init__(parent)

    def dragEnterEvent(self, event):
        if event.mimeData().hasText:
            event.accept()
        else:
            event.ignore()

    def dragMoveEvent(self, event):
        if event.mimeData().hasText:
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        if event.mimeData().hasText:
            event.setDropAction(Qt.CopyAction)
            file_path = event.mimeData().urls()[0].toLocalFile()
            self.read_file(file_path)
            event.accept()
        else:
            event.ignore()

    def read_file(self,path):
        self.clear()
        f=open(path,'r')
        data=f.readlines()
        f.close()
        for i in data:
            self.append(i.split(sep='\n')[0])

class CustomToolbar(NavigationToolbar):
    def __init__(self,canvas_,parent_):
        NavigationToolbar.__init__(self,canvas_,parent_)

    def save_figure(self,*args):
        filetypes = self.canvas.get_supported_filetypes_grouped()
        sorted_filetypes = sorted(filetypes.items())
        default_filetype = self.canvas.get_default_filetype()
        des_extens=[]
        for name,exts in sorted_filetypes:
            name_exts=''.join(['*{}'.format(i) for i in exts])
            vals='{} ({})'.format(name, name_exts)
            if default_filetype in name_exts:
                default=vals
            des_extens.append(vals)
        des_extens=';;'.join(des_extens)
        file_name,ok=QFileDialog.getSaveFileName(self,'Image Saving','',
                                                 des_extens,default)
        if ok:
            self.canvas.figure.savefig(file_name,dpi=600)

if __name__ =="__main__":
    app=QApplication(sys.argv)
    ex=Interface()
    sys.exit(app.exec_())


from collections import defaultdict
import sys

if len(sys.argv) == 2:
    with open(sys.argv[1]) as f:
        data = f.readline().rstrip()
        k, L, t = map(int, f.readline().rstrip().split())
elif len(sys.argv) == 5:
    with open(sys.argv[1]) as f:
        data = f.readline().rstrip()
    k, L, t = map(int, sys.argv[2:])

def ClumpFinding(data, k, L, t):
    k_mers = defaultdict(list)

    for i in range(len(data) - k):
        k_mer = data[i:i+k]
        k_mers[k_mer].append(i)

    def has_clump(locs, L, t, k):
        for i in range(len(locs)):
            if len(locs) - i < t:
                break
            if locs[i+t-1] + k - locs[i] <= L:
                return True
        return False

    clump_k_mers = [k_mer for k_mer, locs in k_mers.items()  if has_clump(locs, L, t, k)]
    return clump_k_mers

clump_k_mers = ClumpFinding(data, k, L, t)

print(" ".join(clump_k_mers))
print(len(set(clump_k_mers)))


"""Parse and analyze logging files (:mod:`fluidimage.topologies.log`)
=====================================================================

.. autoclass:: LogTopology
   :members:
   :private-members:

"""

import time
from glob import glob
from pathlib import Path

import matplotlib.pyplot as plt
import numpy as np

from fluiddyn.util import is_run_from_ipython

if is_run_from_ipython():
    plt.ion()

colors = ["r", "b", "y", "g"]

def float_no_valueerror(word):
    try:
        return float(word)
    except ValueError:
        return np.nan

class LogTopology:
    """Parse and analyze logging files."""

    def __init__(self, path):
        path = Path(path)

        if path.is_dir():
            paths = sorted(glob(str(path / "log_*")))
            paths = [path for path in paths if "_multi" not in Path(path).name]
            if len(paths) == 0:
                raise ValueError("No log files found in the current directory.")

            path = Path(paths[-1])
            if path.is_dir():
                path = Path(
                    next(
                        path
                        for path in glob(str(path / "log*"))
                        if "_multi" not in path
                    )
                )

        self.log_dir_path = path.parent
        self.log_file = path.name
        self._title = str(self.log_file)

        self._parse_log(path)

    def _parse_log(self, path):
        self.works = works = []
        self.works_ended = works_ended = []
        self.nb_cpus_allowed = None
        self.nb_max_workers = None
        self.log_files = None
        self.executor_name = None
        self.topology_name = None
        with open(path, "r") as logfile:
            print("Parsing log file: ", path)
            for iline, line in enumerate(logfile):
                if iline % 100 == 0:
                    print(f"\rparse line {iline}", end="", flush=True)

                if line.startswith("ERROR: "):
                    continue

                if self.nb_cpus_allowed is None and line.startswith(
                    "INFO:   nb_cpus_allowed = "
                ):
                    self.nb_cpus_allowed = int(line.split()[3])
                    self._title += f", nb_cpus_allowed = {self.nb_cpus_allowed}"

                if self.nb_max_workers is None and line.startswith(
                    "INFO:   nb_max_workers = "
                ):
                    self.nb_max_workers = int(line.split()[3])
                    self._title += f", nb_max_workers = {self.nb_max_workers}"

                if self.topology_name is None:
                    begin = "INFO:   topology: "
                    if line.startswith(begin):
                        self.topology_name = line.split(begin)[1].strip()

                if self.executor_name is None:
                    begin = "INFO:   executor: "
                    if line.startswith(begin):
                        self.executor_name = line.split(begin)[1].strip()

                if self.log_files is None:
                    begin = "INFO: logging files: "
                    if line.startswith(begin):
                        self.log_files = eval(line.split(begin)[1].strip())

                if line.startswith("INFO: ") and ". mem usage: " in line:
                    line = line[11:]
                    words = line.split()

                    try:
                        mem = float(words[-2])
                    except ValueError:
                        pass

                    if ". Launch work " in line:
                        name = words[4]
                        key = words[5][1:-2]
                        t = float(words[0])
                        works.append(
                            {
                                "name": name,
                                "key": key,
                                "mem_start": mem,
                                "time": t,
                            }
                        )
                    else:
                        date = words[0][:-1]
                        t = time.mktime(
                            time.strptime(date[:-3], "%Y-%m-%d_%H-%M-%S")
                        ) + float(date[-3:])

                    if ": starting execution. mem usage" in line:
                        self.date_start = date
                        self.mem_start = mem
                        time_start = t
                    elif ": end of `compute`. mem usage" in line:
                        self.date_end = date
                        self.duration = t - time_start
                        self.mem_end = mem

                if line.startswith("INFO: work "):
                    words = line.split()
                    name = words[2]
                    key = words[3][1:-1]
                    try:
                        duration = float(words[-2])
                    except ValueError:
                        pass
                    else:
                        works_ended.append(
                            {"name": name, "key": key, "duration": duration}
                        )

        print("\rdone" + 20 * " ")

        if self.log_files is not None:
            path_dir = self.log_dir_path
            for file_name in self.log_files:
                path = path_dir / file_name
                with open(path, "r") as logfile:
                    print("Parsing log file: ", path.name)
                    for iline, line in enumerate(logfile):
                        if iline % 100 == 0:
                            print(f"\rparse line {iline}", end="", flush=True)

                        if line.startswith("ERROR: "):
                            continue

                        if line.startswith("INFO: ") and ". mem usage: " in line:
                            line = line[11:]
                            words = line.split()
                            mem = float_no_valueerror(words[-2])

                            if ". Launch work " in line:
                                name = words[4]
                                key = words[5][1:-2]
                                t = float_no_valueerror(words[0])
                                works.append(
                                    {
                                        "name": name,
                                        "key": key,
                                        "mem_start": mem,
                                        "time": t,
                                    }
                                )
                            else:
                                date = words[0][:-1]
                                t = time.mktime(
                                    time.strptime(date[:-3], "%Y-%m-%d_%H-%M-%S")
                                ) + float_no_valueerror(date[-3:])

                            if ": starting execution. mem usage" in line:
                                self.date_start = date
                                self.mem_start = mem
                                time_start = t
                            elif ": end of `compute`. mem usage" in line:
                                self.date_end = date
                                self.duration = t - time_start
                                self.mem_end = mem

                        if line.startswith("INFO: work "):
                            words = line.split()
                            name = words[2]
                            key = words[3][1:-1]
                            duration = float_no_valueerror(words[-2])
                            works_ended.append(
                                {"name": name, "key": key, "duration": duration}
                            )
                    print("\rdone" + 20 * " ")

        self.names_works = names_works = []
        for work in works:
            if work["name"] not in names_works:
                names_works.append(work["name"])

        self.durations = durations = {}
        self.times = times = {}
        self.keys = keys = {}
        for name in self.names_works:
            times[name] = []
            keys[name] = []
            for work in self.works:
                if work["name"] == name:
                    times[name].append(work["time"])
                    keys[name].append(work["key"])

            works_ended_name = [
                work for work in self.works_ended if work["name"] == name
            ]
            index_vs_keys = {
                work["key"]: index for index, work in enumerate(works_ended_name)
            }

            durations[name] = []
            for key in keys[name]:
                try:
                    index_key = index_vs_keys[key]
                except KeyError:
                    founded = False
                else:
                    founded = True
                    work = works_ended_name[index_key]
                    durations[name].append(work["duration"])

                if not founded:
                    durations[name].append(np.nan)

    def plot_memory(self):
        """Plot the memory usage versus time."""
        plt.figure()
        ax = plt.gca()
        ax.set_xlabel("time (s)")
        ax.set_ylabel("memory (Mo)")
        ax.set_title(self._title, fontdict={"fontsize": 12})

        memories = np.empty(len(self.works))
        times = np.empty(len(self.works))
        for i, work in enumerate(self.works):
            memories[i] = work["mem_start"]
            times[i] = work["time"]

        ax.plot(times, memories, "o-")
        ax.plot(0, self.mem_start, "x")
        if hasattr(self, "duration"):
            ax.plot(self.duration, self.mem_end, "x")
        plt.show()

    def plot_durations(self):
        """Plot the duration of the works."""
        plt.figure()
        ax = plt.gca()
        ax.set_xlabel("time (s)")
        ax.set_ylabel("duration (s)")
        ax.set_title(self._title, fontdict={"fontsize": 12})

        lines = []

        for i, name in enumerate(self.names_works):
            times = np.array(self.times[name])
            durations = self.durations[name]
            (l,) = ax.plot(times, durations, colors[i] + "o")
            lines.append(l)

            for it, t in enumerate(times):
                d = durations[it]
                ax.plot([t, t + d], [d, d], colors[i])

            d = np.nanmean(durations)
            ax.plot(
                [times.min(), times.max()], [d, d], colors[i] + ":", linewidth=2
            )

        ax.legend(lines, self.names_works, loc="center left", fontsize="x-small")

        plt.show()

    def plot_nb_workers(self, str_names=None):
        """Plot the number of workers versus time."""
        if str_names is not None:
            names = [name for name in self.names_works if str_names in name]
        else:
            names = self.names_works

        nb_workers = {}
        times = {}
        for name in names:
            times_start = list(self.times[name])
            times_stop = list(
                np.array(times_start) + np.array(self.durations[name])
            )

            deltas = np.array([1 for t in times_start] + [-1 for t in times_stop])
            times_unsorted = np.array(times_start + times_stop)

            argsort = np.argsort(times_unsorted)

            ts = times_unsorted[argsort]
            nbws = np.cumsum(deltas[argsort])

            ts2 = []
            nbws2 = []
            for i, t in enumerate(ts[:-1]):
                nbw = nbws[i]
                ts2.append(t)
                ts2.append(ts[i + 1])
                nbws2.append(nbw)
                nbws2.append(nbw)

            times[name] = ts2
            nb_workers[name] = nbws2

        plt.figure()
        ax = plt.gca()
        ax.set_xlabel("time (s)")
        ax.set_ylabel("number of workers")
        ax.set_title(self._title, fontdict={"fontsize": 12})

        lines = []

        for i, name in enumerate(self.names_works):
            (l,) = ax.plot(times[name], nb_workers[name], colors[i] + "-")
            lines.append(l)

        ax.legend(lines, names, loc="center left", fontsize="x-small")

        plt.show()


from enum import Enum

class Modal(object):
    class Styles(Enum):
        short = 1
        paragraph = 2

    DEFAULT_STYLE = 1
    COMPONENT_TYPE = 4

    def __init__(self, custom_id, **kwargs) -> None:
        # type is the same for all buttons
        self.type = self.COMPONENT_TYPE

        # mandatory fields
        self.custom_id = custom_id

        # optional fields with defaults
        self.label = kwargs.pop("label", self.custom_id)
        self.style = kwargs.pop("style", self.DEFAULT_STYLE)

        # add other fields if specified
        for k, v in kwargs.items():
            self.__dict__[k] = v

# // this is a modal
# {
#   "title": "My Cool Modal",
#   "custom_id": "cool_modal",
#   "components": [{
#     "type": 1,
#     "components": [{
#       "type": 4,
#       "custom_id": "name",
#       "label": "Name",
#       "style": 1,
#       "min_length": 1,
#       "max_length": 4000,
#       "placeholder": "John",
#       "required": true
#     }]
#   }]
# }


# -*- coding: utf-8 -*-

import jieba

sentence = "接天莲叶无穷碧,映日荷花别样红"

# 精准模式和全模式就是 多项式和伯努利的区别 默认精准模式
# 精准模式
c1 = jieba.cut(sentence=sentence, cut_all=False)
# 全模式
c2 = jieba.cut(sentence, cut_all=True)
# 关闭隐马尔科夫模式
c3 = jieba.cut(sentence, cut_all=False, HMM=False)

cs = jieba.cut_for_search(sentence)

lc = jieba.lcut(sentence)

lcs = jieba.lcut_for_search(sentence=sentence)

# print("/".join(c1))
# print("*".join(c2))
# print("--".join(c3))
# print(" ".join(cs))
#
# print(lc)
# print(c1)

# 词性
import jieba.posseg as pg

pgs = pg.cut(sentence)
print(pgs)
for w in pgs:
    print(w.word, end='')
    print(w.flag)
    print(w)

# 加载自定义 字典分词
# 词典格式和 dict.txt 一样，一个词占一行；每一行分三部分：词语、词频（可省略）、词性（可省略），用空格隔开，顺序不可颠倒。
# file_name 若为路径或二进制方式打开的文件，则文件必须为 UTF-8 编码。
jieba.load_userdict("./dict.txt")

pgss = pg.lcut(sentence)

print(pgss)

# 动态调整字典
# 添加分词
jieba.add_word("无穷碧")
jieba.del_word("")

print('*'*40)
testlist = [
('今天天气不错', ('今天', '天气')),
('如果放到post中将出错。', ('中', '将')),
('我们中出了一个叛徒', ('中', '出')),
]

for sent, seg in testlist:
    print('/'.join(jieba.cut(sent, HMM=False)))
    word = ''.join(seg)
    print('%s Before: %s, After: %s' % (word, jieba.get_FREQ(word), jieba.suggest_freq(seg, True)))
    print('/'.join(jieba.cut(sent, HMM=False)))
    print("-"*40)


from flask import Flask
from flask_sqlalchemy import SQLAlchemy
import os
from flask_login import LoginManager
from flask_socketio import SocketIO

db = SQLAlchemy()

DB_NAME = "boyskaos.db"
UPLOAD_FOLDER = "website/static/uploads"
socketio = SocketIO()

def create_app(debug=False):
    app = Flask(__name__)
    app.debug = debug
    app.config["SECRET_KEY"] = os.environ.get('KEY')
    app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{DB_NAME}'
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
    socketio.init_app(app)

    db.init_app(app)

    from .views import views
    from .auth import auth
    from .challenge_events import challenge_events
    from .battle_events import battle_events

    app.register_blueprint(views, url_prefix="/")
    app.register_blueprint(auth, url_prefix="/")
    app.register_blueprint(challenge_events, url_prefix="/")
    app.register_blueprint(battle_events, url_prefix="/")

    from .models import User, Card, Connection

    create_database(app)

    login_manager = LoginManager()
    login_manager.login_view = 'auth.login'
    login_manager.init_app(app)
    login_manager.login_message = "Por favor inicia sesión para acceder a esta página."
    login_manager.login_message_category = "error"

    @login_manager.user_loader
    def load_user(id):
        return User.query.get(int(id))

    return app

def create_database(app):
    with app.app_context():
        if not os.path.exists('website/' + DB_NAME):
            db.create_all()
            print('Created Database!')


#!/usr/bin/env python
# coding: utf-8

# ### IMAGE CLASSIFICATION USING SVM

# In[1]:

import matplotlib.pyplot as plt
import cv2
import numpy as np

# In[2]:

im=cv2.imread('goku.jpg')               #reads inn bgr format
im=cv2.cvtColor(im,cv2.COLOR_BGR2RGB)   #coverts bgr to rgb
original_shape=im.shape
# print(im.shape)

# In[3]:

plt.imshow(im)                             #shows as RGBB Format
plt.show()

# In[4]:

# Flatten each channel of the image
all_pixels=im.reshape((-1,3))
# print(all_pixels.shape)

# In[5]:

from sklearn.cluster import KMeans
dominant_colors = 4
km=KMeans(n_clusters=dominant_colors)
km.fit(all_pixels)

# In[6]:

centers = km.cluster_centers_
centers = np.array(centers, dtype = 'uint8')
# print(centers)

# ### Plot what all colors are these

# In[7]:

i=1
plt.figure(0, figsize=(4,2))

colors = []
for each_col in centers:
    plt.subplot(1,4,i)
    plt.axis('off')
    i+=1

    colors.append(each_col)

    #color swatch
    a=np.zeros((100,100,3), dtype='uint8')
    a[:,:,:]=each_col
    plt.imshow(a)

plt.show()

# ### Segmenting our original image

# In[8]:

new_img=np.zeros((1385*1628,3),dtype='uint8')
# print(new_img.shape)

# In[9]:

colors

# In[10]:

km.labels_

# In[11]:

for ix in range(new_img.shape[0]):
    new_img[ix]=colors[km.labels_[ix]]

new_img= new_img.reshape((original_shape))
plt.imshow(new_img)
plt.show()


import numpy as np
from os import listdir
from os.path import isfile, isdir, join
import tensorflow as tf
import cv2
from PIL import Image

mypath = 'intput_img/'
outputpath = 'output_img/'
files = listdir(mypath)

model_load = tf.keras.models.load_model('best_img_model')

img_height = 180
img_width = 180
for file in files:
    imgpath = join(mypath, file)
    img = cv2.imread(imgpath)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (img_height, img_width))
    # img = img.astype(np.float32)

    # imgpath = join(mypath, file)
    # img = tf.keras.utils.load_img(
    #     imgpath, target_size=(img_height, img_width)
    # )
    img_array = tf.keras.utils.img_to_array(img)
    img_array = tf.expand_dims(img_array, 0) # tf.keras模型經過優化，可以一次對一批或一組示例進行預測。因此，即使您使用的是單個圖像，也需要將其添加到列表中

    predictions = model_load.predict(img_array)
    score = tf.nn.softmax(predictions[0])
    class_names = ['00','01','02','03','04','05','06','07','08','09']
    # print(
    #     "class : {} ,score : {:.2f} %."
    #     .format(class_names[np.argmax(score)], 100 * np.max(score))
    # )

    cv2.imwrite(outputpath+str(class_names[np.argmax(score)])+'_'+file+'_.jpg',cv2.imread(imgpath))


# si vuole realizzare un sito web per memorizzare le squadre di uno sport a scelta.
# l'utente deve poter inserire il nome della squadra e la data di fondazione e la citta.
# deve inoltre poter effettuare delle ricerche inserendo uno dei valore delle colonne  e ottenendo i dati presenti.
# salvare i dati in un dataframe o lista di dizionari o creare un file csv

from flask import Flask, render_template, request
app = Flask(__name__)
import pandas as pd

@app.route('/', methods=['GET'])
def registration():
  return render_template("1stinterface.html")

@app.route('/inserisci', methods=['GET'])
def inserisci():
  return render_template("inserisci.html")

@app.route('/ricerca', methods=['GET'])
def ricerca():
  return render_template("ricerca1stPart.html")

@app.route('/dati', methods=['GET'])
def dati():
    # inserimento dei dati nel file csv
    # lettura dei dati dal form html
    squadra = request.args['Squadra']
    anno = request.args['Anno']
    citta = request.args['Citta']
    # lettura dei dati daal file nel dataframe
    df1 = pd.read_csv('/workspace/Flask/esercizio5/templates/dati.csv')
    # aggiungiamo i nuovi dati nel dataframe
    nuovi_dati = {'Team Name':squadra,'Foundation Date':anno,'city of foundation':citta}

    df1 = df1.append(nuovi_dati,ignore_index=True)
    # salviamo il dataframe sul file dati.csv
    df1.to_csv('/workspace/Flask/esercizio5/templates/dati.csv', index=False)
    ####### df1.to_html() prende il df e lo converte in html
    return render_template("1stinterface.html")

@app.route('/dataRad', methods=['GET'])
def dataRad():
    Rad_scelto = request.args["sceltaRad"]
    if Rad_scelto == "Team Name":
        return render_template("ricercaTeamName.html")
    elif Rad_scelto == "Foundation Date":
        return render_template("ricercaDateFoundation.html")
    else:
        return render_template("ricercaCityFoundation.html")

@app.route('/dataTN', methods=['GET'])
def ricTN():
    teamName = request.args["TN"]
    dati = pd.read_csv("/workspace/Flask/esercizio5/templates/dati.csv")
    res = dati[dati.Team_Name == teamName]
    return res.to_html()

@app.route('/dataDF', methods=['GET'])
def ricDF():
    FoundationDate = request.args["DF"]
    dati = pd.read_csv("/workspace/Flask/esercizio5/templates/dati.csv")
    # convert column, if not converted ther will be no result because the values in the csv file are int64's and the pc is searching the value as str
    dati["Foundation_Date"] = dati["Foundation_Date"].astype(str)
    res = dati[dati.Foundation_Date == FoundationDate]
    return res.to_html()

@app.route('/dataCF', methods=['GET'])
def ricCF():
    cityoffoundation = request.args["CF"]
    dati = pd.read_csv("/workspace/Flask/esercizio5/templates/dati.csv")
    res = dati[dati.City_of_foundation == cityoffoundation]
    return res.to_html()

if __name__ == '__main__':
  app.run(host='0.0.0.0', port=3245, debug=True)


from collections import deque
dx=[1,-1,0,0];dy=[0,0,1,-1]
def bfs():
    q=deque([[0,0,False]])
    cnt=0
    vstd[0][0][0]=True
    while q:
        for _ in range(len(q)):
            r,c,b=q.popleft()
            if [r,c]==[n-1,m-1]:return cnt+1
            for i in range(4):
                x=r+dx[i];y=c+dy[i]
                if 0<=x<n and 0<=y<m:
                    if not b and not vstd[0][x][y]:
                        if matrix[x][y]=='1':
                            q.append([x,y,True])
                            vstd[1][x][y]=True
                        else:
                            q.append([x,y,False])
                            vstd[0][x][y]=True
                    else:
                        if matrix[x][y]=='0' and not vstd[1][x][y]:
                            q.append([x,y,True])
                            vstd[1][x][y]=True
        cnt+=1
    return -1
n,m=map(int,input().split())
matrix=list(list(input().rstrip()) for _ in range(n))
vstd=[[[False]*m for _ in range(n)] for _ in range(2)]
print(bfs())


#Filename: HW5_skeleton.py
#Author: Christian Knoll
#Edited: May 2020

import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal
from sklearn import datasets

#--------------------------------------------------------------------------------
# Assignment 5
def main():

    scenario = 1

    # Set each to True to see the plots for this classifier
    show_EM = True
    show_kmean = True

    #------------------------
    # 0) Get the input
    ## (a) load the modified iris data
    data, labels, feature_names = load_iris_data()

    ## (b) construct the datasets
    x_2dim = data[:, [0,2]]
    x_4dim = data

    #TODO: implement PCA
    x_2dim_pca, variance = PCA(data,nr_dimensions=2,whitening=False)
    x_2dim_pca_w, variance_w = PCA(data,nr_dimensions=2,whitening=True)

    ## (c) visually inspect the data with the provided function (see example below)
    # plot_iris_data(x_2dim_pca,labels, feature_names[0], feature_names[2], "Iris Dataset with PCA, (variance_explained: "+str(variance)+")")
    # plot_iris_data(x_2dim_pca_w,labels, feature_names[0], feature_names[2], "Iris Dataset with PCA white, (variance_explained: "+str(variance_w)+")")
    # plot_iris_data(x_2dim,labels, feature_names[0], feature_names[2], "Iris Dataset")

    #------------------------
    # 1) Consider a 2-dim slice of the data and evaluate the EM- and the KMeans- Algorithm
    if scenario == 1:
        dim = 2
        nr_components = 3

        max_iter = 100  # maximum iterations for GN

        if show_EM:
            tol = 0.001

            (alpha_0, mean_0, cov_0) = init_EM(dimension = dim, nr_components= nr_components, scenario=scenario, X=x_2dim)
            (alpha_0, mean_0, cov_0, log_likelyhood, labels_2dim) =  EM(x_2dim,nr_components, alpha_0, mean_0, cov_0, max_iter, tol)

            # Plot for EM
            plt.plot(log_likelyhood)
            plt.show()

            plot_iris_data(x_2dim,labels_2dim, feature_names[0], feature_names[2], "Iris Dataset EM 2 Dim")

            for k in range(nr_components):
                plot_gauss_contour(mean_0[k], cov_0[k], 4, 9, 0, 8 ,20)

            plot_iris_data(x_2dim,labels_2dim, feature_names[0], feature_names[2], "Iris Dataset EM 2 Dim")

        if show_kmean:
            tol = 0.0001

            initial_centers = init_k_means(dimension = dim, nr_clusters=nr_components, scenario=scenario, X=x_2dim)
            final_centers, cum_dist, km_labels_2dim = k_means(x_2dim, nr_components, initial_centers, max_iter, tol)

            # Plots for k-means
            plt.plot(cum_dist)
            plt.xlabel("Iterations")
            plt.ylabel("Distance")
            plt.title("k-means cumulative distance")
            plt.show()

            plot_kmeans(x_2dim, km_labels_2dim, feature_names[0], feature_names[2], final_centers, "k-means")

    #------------------------
    # 2) Consider 4-dimensional data and evaluate the EM- and the KMeans- Algorithm
    if scenario == 2:
        dim = 4
        nr_components = 3

        max_iter = 100  # maximum iterations for GN

        if show_EM:
            tol = 0.001

            (alpha_0, mean_0, cov_0) = init_EM(dimension = dim, nr_components= nr_components, scenario=scenario, X=x_4dim)
            (alpha_0, mean_0, cov_0, log_likelyhood, labels_4dim) = EM(x_4dim, nr_components, alpha_0, mean_0, cov_0, max_iter, tol)

            # Plot for EM
            plt.plot(log_likelyhood)
            plt.show()

            plot_iris_data(x_4dim,labels_4dim, feature_names[0], feature_names[2], "Iris Dataset EM 4 Dim")

        if show_kmean:
            tol = 0.0001

            initial_centers = init_k_means(dimension = dim, nr_clusters=nr_components, scenario=scenario, X=x_4dim)
            final_centers, cum_dist, km_labels_4dim = k_means(x_4dim,nr_components, initial_centers, max_iter, tol)

            # Plots for k-means
            plot_kmeans(x_4dim, km_labels_4dim, feature_names[0], feature_names[2], final_centers, "k-means 4 Dimensions")

    #------------------------
    # 3) Perform PCA to reduce the dimension to 2 while preserving most of the variance.
    # Then, evaluate the EM- and the KMeans- Algorithm  on the transformed data
    if scenario == 3:
        dim = 2
        nr_components = 3

        max_iter = 100  # maximum iterations for GN
        nr_components = 3 #n number of components

        #TODO: implement
        if show_EM:
            tol = 0.001

            (alpha_0, mean_0, cov_0) = init_EM(dimension = dim, nr_components= nr_components, scenario=scenario, X=x_2dim_pca)
            (alpha_0, mean_0, cov_0, log_likelyhood, labels_pca) = EM(x_2dim_pca, nr_components, alpha_0, mean_0, cov_0, max_iter, tol)

            # Plot for EM
            plt.plot(log_likelyhood)
            plt.show()

            plot_iris_data(x_2dim_pca,labels_pca, feature_names[0], feature_names[2], "Iris Dataset EM PCA")

            for k in range(nr_components):
                plot_gauss_contour(mean_0[k], cov_0[k], 2, 9, 0, 8 ,20)

            plot_iris_data(x_2dim_pca,labels_pca, feature_names[0], feature_names[2], "Iris Dataset EM PCA")

        if show_kmean:
            tol = 0.0001

            initial_centers = init_k_means(dimension = dim, nr_clusters=nr_components, scenario=scenario, X=x_2dim_pca)
            final_centers, cum_dist, km_labels_pca = k_means(x_2dim_pca ,nr_components, initial_centers, max_iter, tol)

            # Plots for k-means
            plt.plot(cum_dist)
            plt.xlabel("Iterations")
            plt.ylabel("Distance")
            plt.title("k-means cumulative distance")
            plt.show()

            plot_kmeans(x_2dim_pca, km_labels_pca, feature_names[0], feature_names[2], final_centers, "k-means PCA")

        #TODO: visualize your results
        #TODO: compare PCA as pre-processing (3.) to PCA as post-processing (after 2.)

#--------------------------------------------------------------------------------
#--------------------------------------------------------------------------------
def init_EM(dimension=2,nr_components=3, scenario=None, X=None):
    """ initializes the EM algorithm
    Input:
        dimension... dimension D of the dataset, scalar
        nr_components...scalar
        scenario... (optional) parameter that allows to further specify the settings, scalar
        X... (optional) samples that may be used for proper inititalization, nr_samples x dimension(D)
    Returns:
        alpha_0... initial weight of each component, 1 x nr_components
        mean_0 ... initial mean values, D x nr_components
        cov_0 ...  initial covariance for each component, D x D x nr_components"""

    alpha_0 = np.ones((1, nr_components))/nr_components
    mean_0 = np.ones((dimension, nr_components))
    cov_0 = np.ones((dimension, dimension, nr_components))

    if X is not None:
        mean = np.mean(X)
        summe = 0
        for _, x_n in enumerate(X):
            diff = x_n-mean
            diff = diff.reshape((dimension,1))
            summe += np.matmul(diff, diff.T)

        cov = (summe/nr_components)
        cov_0 = np.tile(cov,(nr_components,1,1)).T

        mean_0 = X[np.random.choice(X.shape[0], nr_components, replace=False)].T

    #Best values for plot
    if False:
        mean_0[0][0] = 5.00600066
        mean_0[0][1] = 5.96810282
        mean_0[0][2] = 6.535967

        mean_0[1][0] = 1.46199854
        mean_0[1][1] = 4.01009197
        mean_0[1][2] = 5.49963309

        cov_0 = [[[0.12176427, 0.28003145, 0.42380503],[0.01602834, 0.20985688, 0.3422579 ]],
                [[0.01602834, 0.20985688, 0.3422579 ],[0.02955546, 0.23795915, 0.35377924]]]

        cov_0 = np.array(cov_0)

    return (alpha_0, mean_0, cov_0)
#--------------------------------------------------------------------------------
def EM(X,K,alpha_0,mean_0,cov_0, max_iter, tol):
    """ perform the EM-algorithm in order to optimize the parameters of a GMM
    with K components
    Input:
        X... samples, nr_samples x dimension (D)
        K... nr of components, scalar
        alpha_0... initial weight of each component, 1 x K
        mean_0 ... initial mean values, D x K
        cov_0 ...  initial covariance for each component, D x D x K
    Returns:
        alpha... final weight of each component, 1 x K
        mean...  final mean values, D x K
        cov...   final covariance for ech component, D x D x K
        log_likelihood... log-likelihood over all iterations, nr_iterations x 1
        labels... class labels after performing soft classification, nr_samples x 1"""
    # compute the dimension
    D = X.shape[1]
    assert D == mean_0.shape[0]
    #TODO: iteratively compute the posterior and update the parameters

    mean_0 = mean_0.T
    cov_0 = cov_0.T
    alpha_0 = alpha_0.T

    r = np.zeros((K, X.shape[0]))

    log_likelihood = []

    N = X.shape[0]

    for i in range(max_iter):

        r = em_expectation(N,K,alpha_0, X,mean_0, cov_0)

        em_maximization(N,K,alpha_0, X,mean_0, cov_0, r)

        #calc log_likelihood per iteration
        log_likelihood_it = em_likelyhood_calc(N,K,alpha_0, X,mean_0, cov_0)

        log_likelihood.append(log_likelihood_it)
        if len(log_likelihood) > 1:
            pass
        if len(log_likelihood) > 1 and np.abs(log_likelihood[-1] - log_likelihood[-2]) < tol:
            break

    labels = np.zeros(N, dtype=np.int)
    for n in range(N):
        value = np.zeros(K)
        for k in range(K):
            value[k] = alpha_0[k] * likelihood_multivariate_normal(X[n], mean_0[k], cov_0[k])
        labels[n] = np.argmax(value)

    return alpha_0, mean_0, cov_0, log_likelihood, labels

#--------------------------------------------------------------------------------
def em_expectation(N, K, alpha_0, X, mean_0, cov_0):
    r = np.zeros((K, X.shape[0]))
    #calc r
    for n in range(N):
        for k in range(K):
            r_nenner = 0
            for k_ in range(K):
                r_nenner += alpha_0[k_] * likelihood_multivariate_normal(X[n], mean_0[k_], cov_0[k_])

            r[k][n] = alpha_0[k] * likelihood_multivariate_normal(X[n], mean_0[k], cov_0[k]) / r_nenner
    return r

#--------------------------------------------------------------------------------
def em_maximization(N, K, alpha_0, X, mean_0, cov_0, r):
    #calc new alpha, mean and cov
    for k in range(K):
        #calc mean_0 new
        mean_temp = 0
        N_k = 0

        for n in range(N):
            mean_temp += r[k][n]*X[n]

            #calc N_k and N for later use
            N_k += r[k][n]

        mean_0[k] = mean_temp / N_k

        #calc cov_0 new
        cov_temp = 0
        for n in range(N):
            cov_temp += r[k][n] *  np.multiply.outer((X[n] - mean_0[k]), (X[n] - mean_0[k]))

        cov_0[k] = cov_temp / N_k

        #calc alpha_0 new
        alpha_0[k] = N_k / N

#--------------------------------------------------------------------------------
def em_likelyhood_calc(N, K, alpha_0, X, mean_0, cov_0):
    #calc log_likelihood per iteration
    log_likelihood_it = 0

    for n in range(N):
        temp = 0
        for k in range(K):
            temp += alpha_0[k] * likelihood_multivariate_normal(X[n], mean_0[k], cov_0[k],log=False)
        log_likelihood_it += np.log(temp)
    return log_likelihood_it

#--------------------------------------------------------------------------------
def init_k_means(dimension=None, nr_clusters=None, scenario=None, X=None):
    """ initializes the k_means algorithm
    Input:
        dimension... dimension D of the dataset, scalar
        nr_clusters...scalar
        scenario... (optional) parameter that allows to further specify the settings, scalar
        X... (optional) samples that may be used for proper inititalization, nr_samples x dimension(D)
    Returns:
        initial_centers... initial cluster centers,  D x nr_clusters"""
    #TODO: choose suitable inital values for each scenario
    return X[np.random.choice(X.shape[0], nr_clusters, replace=False)].T

#--------------------------------------------------------------------------------
def k_means(X,K, centers_0, max_iter, tol):
    """ perform the KMeans-algorithm in order to cluster the data into K clusters
    Input:
        X... samples, nr_samples x dimension (D)
        K... nr of clusters, scalar
        centers_0... initial cluster centers,  D x nr_clusters
    Returns:
        centers... final centers, D x nr_clusters
        cumulative_distance... cumulative distance over all iterations, nr_iterations x 1
        labels... class labels after performing hard classification, nr_samples x 1"""
    D = X.shape[1]
    assert D == centers_0.shape[0]
    #TODO: iteratively update the cluster centers

    #indices of closest centers for each point
    nearest_centers = np.zeros(X.shape[0], dtype=np.int)
    centers = centers_0.T
    cumulative_distance = np.ndarray((0,1))

    for i in range(max_iter):
        distance_sum = 0

        for x_index, x in enumerate(X):
            # find closest center
            min_dist = np.inf
            for center_index, center in enumerate(centers):
                dist = np.linalg.norm(x-center)
                if dist < min_dist:
                    min_dist = dist
                    distance_sum+=dist
                    nearest_centers[x_index] = center_index

        if i == 0 or np.abs(distance_sum - cumulative_distance[i-1]) > tol:
            cumulative_distance = np.append(cumulative_distance, distance_sum)
            # set new centers
            centers = np.zeros((K, X.shape[1]))
            for x_index, center_index in enumerate(nearest_centers):
                centers[center_index] += 1/len([x for x in nearest_centers if x == center_index]) * X[x_index]
        else:
            break

    return centers.T, cumulative_distance, nearest_centers

    #TODO: classify all samples after convergence

#--------------------------------------------------------------------------------
def PCA(data,nr_dimensions=None, whitening=False):
    """ perform PCA and reduce the dimension of the data (D) to nr_dimensions
    Input:
        data... samples, nr_samples x D
        nr_dimensions... dimension after the transformation, scalar
        whitening... False -> standard PCA, True -> PCA with whitening

    Returns:
        transformed data... nr_samples x nr_dimensions
        variance_explained... amount of variance explained by the the first nr_dimensions principal components, scalar"""
    if nr_dimensions is not None:
        dim = nr_dimensions
    else:
        dim = 2

    if whitening:
        X_centered = data - np.mean(data, axis=0)
        Sigma = np.dot(X_centered.T, X_centered) / X_centered.shape[0]
        U, L, _ = np.linalg.svd(Sigma)
        W = np.dot(np.diag(1.0 / np.sqrt(L + 1e-5)), U.T)
        data = np.dot(X_centered, W.T)

    data = data.T
    var_bef = np.var(data)

    cov_matrix = np.cov([data[0, :], data[1, :], data[2, :], data[3, :]])

    eig_values, eig_vector = np.linalg.eig(cov_matrix)

    eigs = [(np.abs(eig_values[i]), eig_vector[:, i]) for i in range(len(eig_values))]
    eigs.sort(key=lambda x: x[0], reverse=True)

    transform_mat = np.hstack((eigs[0][1].reshape(4, 1), -eigs[1][1].reshape(4, 1))).T
    transformed = transform_mat.dot(data)

    var_aft = np.var(transformed)

    variance_explained = (eig_values[0] + eig_values[1]) / np.sum(eig_values)

    return transformed.T, variance_explained

#--------------------------------------------------------------------------------
def plot_kmeans(data, labels, x_axis, y_axis, centers, title):
    #reassign labels for kmeans
    new_labels = reassign_class_labels(labels)
    reshuffled_labels =np.zeros_like(labels)
    reshuffled_labels[labels==0] = new_labels[0]
    reshuffled_labels[labels==1] = new_labels[1]
    reshuffled_labels[labels==2] = new_labels[2]

    # print kmeans plot
    plt.scatter(data[reshuffled_labels==0,0], data[reshuffled_labels==0,1], label='Iris-Setosa')
    plt.scatter(data[reshuffled_labels==1,0], data[reshuffled_labels==1,1], label='Iris-Versicolor')
    plt.scatter(data[reshuffled_labels==2,0], data[reshuffled_labels==2,1], label='Iris-Virgnica')
    plt.scatter(centers[0], centers[1], label='Centers', marker="x", color="black")
    plt.xlabel(x_axis)
    plt.ylabel(y_axis)
    plt.title(title)
    plt.legend()
    plt.show()
#--------------------------------------------------------------------------------
#--------------------------------------------------------------------------------
# Helper Functions
#--------------------------------------------------------------------------------
def load_iris_data():
    """ loads and modifies the iris data-set
    Input:
    Returns:
        X... samples, 150x4
        Y... labels, 150x1
        feature_names... name of the data columns"""
    iris = datasets.load_iris()
    X = iris.data
    X[50:100,2] =  iris.data[50:100,2]-0.25
    Y = iris.target
    return X,Y, iris.feature_names
#--------------------------------------------------------------------------------
def plot_iris_data(data, labels, x_axis, y_axis, title):
    """ plots a 2-dim slice according to the specified labels
    Input:
        data...  samples, 150x2
        labels...labels, 150x1
        x_axis... label for the x_axis
        y_axis... label for the y_axis
        title...  title of the plot"""

    plt.scatter(data[labels==0,0], data[labels==0,1], label='Iris-Setosa')
    plt.scatter(data[labels==1,0], data[labels==1,1], label='Iris-Versicolor')
    plt.scatter(data[labels==2,0], data[labels==2,1], label='Iris-Virgnica')
    #plt.scatter(data[labels==0,0], data[labels==0,1], label='nr_component 1')
    #plt.scatter(data[labels==1,0], data[labels==1,1], label='nr_component 2')
    #plt.scatter(data[labels==2,0], data[labels==2,1], label='nr_component 3')
    #plt.scatter(data[labels==3,0], data[labels==3,1], label='nr_component 4')
    plt.xlabel(x_axis)
    plt.ylabel(y_axis)
    plt.title(title)
    plt.legend()
    plt.show()
#--------------------------------------------------------------------------------
def likelihood_multivariate_normal(X, mean, cov, log=False):
   """Returns the likelihood of X for multivariate (d-dimensional) Gaussian
   specified with mu and cov.

   X  ... vector to be evaluated -- np.array([[x_00, x_01,...x_0d], ..., [x_n0, x_n1, ...x_nd]])
   mean ... mean -- [mu_1, mu_2,...,mu_d]
   cov ... covariance matrix -- np.array with (d x d)
   log ... False for likelihood, true for log-likelihood
   """

   dist = multivariate_normal(mean, cov)
   if log is False:
       P = dist.pdf(X)
   elif log is True:
       P = dist.logpdf(X)
   return P

#--------------------------------------------------------------------------------
def plot_gauss_contour(mu,cov,xmin,xmax,ymin,ymax,nr_points,title="Title"):
    """ creates a contour plot for a bivariate gaussian distribution with specified parameters

    Input:
      mu... mean vector, 2x1
      cov...covariance matrix, 2x2
      xmin,xmax... minimum and maximum value for width of plot-area, scalar
      ymin,ymax....minimum and maximum value for height of plot-area, scalar
      nr_points...specifies the resolution along both axis
      title... title of the plot (optional), string"""

	#npts = 100
    delta_x = float(xmax-xmin) / float(nr_points)
    delta_y = float(ymax-ymin) / float(nr_points)
    x = np.arange(xmin, xmax, delta_x)
    y = np.arange(ymin, ymax, delta_y)

    X, Y = np.meshgrid(x, y)
    pos = np.dstack((X, Y))

    Z = multivariate_normal(mu, cov).pdf(pos)
    plt.plot([mu[0]],[mu[1]],'r+') # plot the mean as a single point
    CS = plt.contour(X, Y, Z)
    plt.clabel(CS, inline=1, fontsize=10)
    #plt.show()
    return
#--------------------------------------------------------------------------------
def sample_discrete_pmf(X, PM, N):
    """Draw N samples for the discrete probability mass function PM that is defined over
    the support X.

    X ... Support of RV -- np.array([...])
    PM ... P(X) -- np.array([...])
    N ... number of samples -- scalar
    """
    assert np.isclose(np.sum(PM), 1.0)
    assert all(0.0 <= p <= 1.0 for p in PM)

    y = np.zeros(N)
    cumulativePM = np.cumsum(PM) # build CDF based on PMF
    offsetRand = np.random.uniform(0, 1) * (1 / N) # offset to circumvent numerical issues with cumulativePM
    comb = np.arange(offsetRand, 1 + offsetRand, 1 / N) # new axis with N values in the range ]0,1[

    j = 0
    for i in range(0, N):
        while comb[i] >= cumulativePM[j]: # map the linear distributed values comb according to the CDF
            j += 1
        y[i] = X[j]

    return np.random.permutation(y) # permutation of all samples
#--------------------------------------------------------------------------------
def reassign_class_labels(labels):
    """ reassi    #TODO calc covariancegns the class labels in order to make the result comparable.
    new_labels contains the labels that can be compared to the provided data,
    i.e., new_labels[i] = j means that i corresponds to j.
    Input:
        labels... estimated labels, 150x1
    Returns:
        new_labels... 3x1"""
    class_assignments = np.array([[np.sum(labels[0:50]==0)   ,  np.sum(labels[0:50]==1)   , np.sum(labels[0:50]==2)   ],
                                  [np.sum(labels[50:100]==0) ,  np.sum(labels[50:100]==1) , np.sum(labels[50:100]==2) ],
                                  [np.sum(labels[100:150]==0),  np.sum(labels[100:150]==1), np.sum(labels[100:150]==2)]])
    new_labels = np.array([np.argmax(class_assignments[:,0]),
                           np.argmax(class_assignments[:,1]),
                           np.argmax(class_assignments[:,2])])
    return new_labels
#--------------------------------------------------------------------------------
def sanity_checks():
    # likelihood_multivariate_normal
    mu =  [0.0, 0.0]
    cov = [[1, 0.2],[0.2, 0.5]]
    x = np.array([[0.9, 1.2], [0.8, 0.8], [0.1, 1.0]])
    P = likelihood_multivariate_normal(x, mu, cov)
    print(P)

    plot_gauss_contour(mu, cov, -2, 2, -2, 2,100, 'Gaussian')

    # sample_discrete_pmf
    PM = np.array([0.2, 0.5, 0.2, 0.1])
    N = 1000
    X = np.array([1, 2, 3, 4])
    Y = sample_discrete_pmf(X, PM, N)

    print('Nr_1:', np.sum(Y == 1),
          'Nr_2:', np.sum(Y == 2),
          'Nr_3:', np.sum(Y == 3),
          'Nr_4:', np.sum(Y == 4))

    # re-assign labels
    class_labels_unordererd = np.array([2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
       2, 2, 2, 2, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,
       0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0,
       0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0,
       0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0,
       0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0])
    new_labels = reassign_class_labels(class_labels_unordererd)
    reshuffled_labels =np.zeros_like(class_labels_unordererd)
    reshuffled_labels[class_labels_unordererd==0] = new_labels[0]
    reshuffled_labels[class_labels_unordererd==1] = new_labels[1]
    reshuffled_labels[class_labels_unordererd==2] = new_labels[2]

#--------------------------------------------------------------------------------
#--------------------------------------------------------------------------------
if __name__ == '__main__':

    #sanity_checks()
    main()


#!/usr/bin/env python
import subprocess
import logging
import sys
import os
import json
from os.path import join

MAIN_DIR = os.path.expanduser('~/.dakis')
JOBS_DIR = join(MAIN_DIR, 'jobs')

def prepare_environment():
    '''Prepares ~/.dakis directory and installs Python packages'''
    subprocess.Popen('mkdir -p %s && mv %s %s' % (JOBS_DIR, sys.argv[0], MAIN_DIR), shell=True)
    cmds = [
        'cd %s' % MAIN_DIR,
        'wget https://bootstrap.pypa.io/get-pip.py',
        'python get-pip.py --user',
        'rm get-pip.py',
        'pip install --user requests argparse || ~/.local/bin/pip install --user requests argparse',
    ]
    subprocess.Popen(' && '.join(cmds), shell=True)
    p1 = subprocess.Popen(' && '.join(cmds), shell=True)
    p1.communicate()    # Wait till command is finished
    print("\n == Successfully prepared Dakis environment == \n")

def parse_json(value):
    if type(value) == str or type(value) == unicode:
        return json.loads(value.replace("'", '"'))
    return value

def get_job_filename(exp_id, resp_json):
    input_values = dict(parse_json(resp_json['input_values']))
    if input_values.get('func_cls') and input_values.get('func_id'):
        job_filename = join(JOBS_DIR, 'e%dc%df%d.sh' % (exp_id, input_values['func_cls'], input_values['func_id']))
    else:
        job_filename = join(JOBS_DIR, 'e%dt%d.sh' % (exp_id, resp_json.get('task_id') or resp_json.get('id')))
    return job_filename

def was_called_in_supercomputer():
    is_qsub_installed = subprocess.Popen('which qsub', stdout=subprocess.PIPE, shell=True)
    return bool(is_qsub_installed.communicate()[0])

def run_next_task(exp_id):
    '''Get task data, prepare executable, check if supercomputer, run_task.'''
    resp = requests.get('http://dakis.lt/api/exp/%d/next-task/' % exp_id)
    # logging.info('Getting next task: exp_id=%d, status_code=%d, resp=%s' % (exp_id, resp.status_code, resp.json()))
    print('Trying to run next task')
    if resp.status_code == 200 and resp.json():
        resp_json = resp.json()

        executable = prepare_executable(resp_json['algorithm_id'], resp_json['repository'], resp_json['branch'], resp_json['executable'])

        cmd = [
            '%s' % executable,       # Should pass path as it will be called  .strip('./')
            '--task_id=%s' % resp_json['task_id'],
            '--callback=%s' % join(MAIN_DIR, 'worker.py'),
        ]

        max_duration = 43200   # 12 hours
        for name, value in parse_json(resp_json['input_values']):
            name = ''.join(str(name).split())   # Note: this is protection for injection attack
            value = ''.join(str(value).split())
            cmd.append('--%s=%s' % (name, value))
            if name == 'max_duration' and int(value) > max_duration:
                max_duration = int(value)
        max_duration = '%d:%.2d:%.2d' % (max_duration / 3600, (max_duration % 3600) / 60, max_duration % 60)

        logging.info('Handling cmd: ' + ' '.join(cmd))

        try:
            if was_called_in_supercomputer():
                job_filename = get_job_filename(exp_id, resp_json)
                job_file = open(job_filename, 'w')
                job_file.write('#!/bin/bash\n#$ -j y\n#$ -l h_rt=%s\n#$ -S /bin/bash\n#$ -cwd\nmpirun -np 1 ' % max_duration)
                job_file.write(' '.join(cmd) + '\n')
                job_file.close()
                logging.info('Created job file: %s' % job_filename)
                add_to_queue_cmd = 'qsub -pe orte 1 -o {0}.o -e {0}.e {0}'.format(job_filename)
                # Note: should use separate files for stdout and stderr. Remove them only if they are empty or reported.
                p1 = subprocess.Popen(add_to_queue_cmd, shell=True, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE)  # , close_fds=True)
                logging.info('Called command: %s' % add_to_queue_cmd)
            else:
                # logging.info('Calling: cmd=%s' % ' '.join(cmd))
                subprocess.Popen(' '.join(cmd), shell=True, stdin=None, stdout=None, stderr=None)
        except Exception as e:
            logging.error('Got error while calling:  %s' % str(e))

def send_task_results(args, unknown):
    url = 'http://dakis.lt/api/tasks/%d/' % args.task_id
    output_values = []
    for p in unknown:
        name, value = p.split('=')
        name = name.lstrip('-')
        output_values.append([name, value])
    # logging.info('Sending: url=%s, data=%s' % (url, data))
    resp = requests.put(url, {'output_values': str(output_values), 'status': args.status})
    # logging.info('Response from sending: status_code=%s, data=%s' % (resp.status_code, resp.json()))
    return resp.json()

def prepare_executable(alg_id, repository, branch, exe_file):    # Note: Should get algorithm id, not experiment id
    '''Clone and compile experiments code. Uses REST API if not all data provided in parameters.'''
    exe_dir = join(MAIN_DIR, 'alg_%d' % alg_id)
    executable = join(exe_dir, exe_file)

    # Note: commit head should also be saved to the task.
    if not os.path.exists(executable):  # Note: code version will be cached, new commits won't be pulled
        if not os.path.exists(exe_dir):
            if 'hg@' not in repository:
                cmd = 'git clone {0} {1} && cd {1} && git fetch origin {2} && git checkout {2} && git pull -r && make compile'.format(repository, exe_dir, branch)
            else:
                cmd = 'hg clone {0} {1} && cd {1} && make compile'.format(repository, exe_dir, branch)
            proc = subprocess.Popen(cmd, shell=True)
            proc.communicate()
        else:
            proc = subprocess.Popen('cd %s && git pull -r && make compile' % exe_dir, shell=True)  # Use ``make compile`` instead of ``make``
            proc.communicate()
    return executable

def request_to_run_next_task(exp_id):
    url = 'http://dakis.lt/api/exp/%d/run/' % exp_id
    resp = requests.get(url)
    return

def get_argparser():
    import argparse
    parser = argparse.ArgumentParser(description='Schedules tasks')
    parser.add_argument('-exp', '--exp_id', type=int, help='Experiment ID', nargs='?', default=None)

    parser.add_argument('-task', '--task_id', type=int, help='Task ID', nargs=None, default=None)
    parser.add_argument('-st', '--status', type=str, help='Status of the task. D - done, S - suspended.', nargs=None, default='D')

    # Prepare environment arguments
    parser.add_argument('-env', '--prepare_environment', help='Create ~/.dakis dir and prepare environment', nargs='?', const=True)
    return parser

def main(args, unknown):
    global requests
    import requests

    if args.exp_id:
        run_next_task(args.exp_id)
    elif args.task_id:
        # Send results
        resp_json = send_task_results(args, unknown)
        # Remove job file if it exists
        exp_id = int(resp_json['experiment'].split('experiments')[-1].strip('/'))
        job_filename = get_job_filename(exp_id, resp_json)
        if os.path.isfile(job_filename):
            os.remove(job_filename)
        for file in os.listdir(JOBS_DIR):
            if file.endswith('.o') or file.endswith('.e'):  # Note: should split output and error streams
                os.remove(join(JOBS_DIR, file))
        ## Run next task
        if was_called_in_supercomputer():
            request_to_run_next_task(exp_id)   # Note: cannot start new job from inside of job
        else:
            run_next_task(exp_id)

if __name__ == '__main__':
    logging.basicConfig(
        level=logging.INFO,
        filename=os.path.expanduser(join(MAIN_DIR, 'worker.log')),
        format='%(asctime)s - %(message)s',
    )
    logging.info('Called with argv: %s' % sys.argv)

    # Parse arguments
    args, unknown = get_argparser().parse_known_args()

    # Prepare environment
    if args.prepare_environment or not os.path.exists(MAIN_DIR) or not os.path.exists(JOBS_DIR):
        prepare_environment()

    # Get and run task
    main(args, unknown)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('article', '0015_auto_20150303_1558'),
    ]

    operations = [
        migrations.AddField(
            model_name='issue',
            name='pdf',
            field=models.FileField(default=None, null=True, upload_to=b'issues'),
            preserve_default=True,
        ),
    ]