import sys, pathlib, re, string, pprint

class Bookmark:
    def __init__(self):
        self.source = SourceData()
        self.pageData = PageData()
        self.html = Html()
        self.css = Css()

    def make(self, sourcePath, dest):
        lines = self.source.get(sourcePath)
        pageData = self.pageData.get(lines)
        html = self.html.generate(pageData)
        self.html.write(html, sourcePath, dest)
        self.css.setup(dest)

class SourceData:
    def get(self, sourcePath):
        with open(sourcePath, encoding='utf-8') as f:
            lines = f.readlines()
        cleaned = self.clean(lines)
        return cleaned

    def clean(self, lines):
        striped = [self.strip(l) for l in lines]
        proper = [l for l in striped if not self.is_ignore(l)]
        return proper

    def strip(self, line):
        withoutComment = re.sub(';;.*', '', line)
        striped = withoutComment.strip()
        return striped

    def is_ignore(self, line):
        if line.startswith('title:'):
            return False
        if re.match('>>>.+', line):
            return False
        if '---' in line:
            return False
        return True

class PageData:
    def get(self, lines):
        title = self.getTitle(lines[0])
        groups = self.getGroups(lines[1:]) # タイトルは除く

        return {
            'title': title,
            'groups': groups
        }

    def getTitle(self, text):
        title = 'Bookmark'
        if text.startswith('title:'):
            title = text.replace('title:','').strip()
        return title

    def getGroups(self, lines):
        separated = self.separateByGroup(lines)
        groups = []
        for groupChank in separated:
            groups.append(self.getGroupData(groupChank))
        return groups

    def separateByGroup(self, lines):
        groups = []
        group = []
        for l in lines:
            if l.startswith('>>>'):
                groups.append(group)
                group = []
                group.append(l)
                continue
            group.append(l)
        groups.append(group)

        if not groups[0]:  # 通常、先頭は空リスト（[]）
            groups = groups[1:]
        return groups

    def getGroupData(self, groupChank):
        title = 'No Title'
        if groupChank[0].startswith('>>>'):
            title = groupChank[0].replace('>>>', '').strip()
            groupChank = groupChank[1:]

        return {
            'title': title,
            'links': groupChank
        }

class Html:
    def generate(self, pageData):
        pageTitle = pageData['title']
        groupsHtml = '\n'.join(self.groupsHtml(pageData['groups']))
        d = dict(title=pageTitle, groups=groupsHtml)
        s = string.Template(Template.page)
        html = s.safe_substitute(d)
        return html

    def groupsHtml(self, groups):
        html = []
        for g in groups:
            html.append('{}
'.format(self.indent(2)))
            html.append('{}
{}
'.format(self.indent(3), g['title']))
            html.extend(self.groupHtml(g['links']))
            html.append('{}
'.format(self.indent(2)))
        return html

    def groupHtml(self, links):
        html = []
        html.append('{}
'.format(self.indent(3)))
        for l in links:
            html.append('{}
{}
'.format(self.indent(4), self.linkHtml(l)))
        html.append('{}
'.format(self.indent(3)))
        return html

    def linkHtml(self, link):
        # link: 'title---url'
        title, url = link.split('---')
        html = '{}'.format(url, title)
        return html

    def indent(self, depth):
        return '  ' * depth

    def write(self, html, sourcePath, dest):
        htmlPath = sourcePath.with_suffix('.html')
        htmlPath = dest.joinpath(htmlPath.name)
        if not dest.exists():
            dest.mkdir(parents=True)
        with open(htmlPath, 'w', encoding='utf-8') as f:
            f.write(html)

class Template:
    page = '''<!doctype html>

<title>${title}</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link href="css/style.css" rel="stylesheet" type="text/css">
<link href="https://use.fontawesome.com/releases/v6.1.1/css/all.css" rel="stylesheet">

${title}

${groups}

'''

class Css:
    code = '''
* {
	margin: 0;
	padding:0;
}

body {
    background: rgb(202, 201, 194);
    color:rgb(129, 129, 129);
}

#contents {
	padding: 1em 1em;
}

h1 {
    color: rgb(105, 42, 105);
    font-weight: normal;
    margin: 0.25em 0em 0.5em 0em;
    font-size: 160%;
    letter-spacing: 0.16em;
}

h2 {
	padding: 0.5em;
    border-top: solid 2px #e000e0;
	background: #535;
    letter-spacing: 0.16em;
	font-size: medium;
	font-width: normal;
	font-weight: normal;
	color: #E1B8ED;
}

.group {
	padding-left: 1px;
	font-size: normal;
	letter-spacing: 0.05em;
}

li {
    position: relative; /* font awsome 用 */
	list-style-type: none;
    border-bottom: dotted 1px #767;
    line-height: 2em;
	color: #4c0e5f;
}

li::before {
    font-family: "Font Awesome 6 Free";
    font-weight: 900;
    content: '\\f138';
    position: absolute;
    left: 0.5em;
    top: 0.5em;
    /*color:#e000e0;*/
}

a {
    display: block;
    background: url("../img/linkmark.png") 1em 1em no-repeat;
    padding: 0.5em 1em 0.5em 2.25em;
    color:rgb(76, 0, 92);
	text-decoration: none;

}

a:hover {
    background-color: rgb(182, 180, 169);
    color:rgb(61, 61, 61)
}
    '''
    def setup(self, dest):
        cssDir = dest.joinpath('css')
        cssFile = dest.joinpath('css', 'style.css')
        if cssFile.exists():
            return
        if not cssDir.exists():
            cssDir.mkdir(parents=True)
        cssFile.write_text(self.code, encoding='utf-8')

Usage = '\nUsage: python bookmark.py source [destination]\n'

def main(args):
    if len(args) < 2:
        print(Usage)
        return
    sourcePath = pathlib.Path(args[1])
    dest = pathlib.Path(args[2]) if len(args) > 2 else sourcePath.parent # 無ければsourcePathと同じディレクトリ
    Bookmark().make(sourcePath, dest)

if __name__ == '__main__':
    main(sys.argv)


import tornado
from tornado.web import RequestHandler
from tornado.web import asynchronous
from tornado.websocket import WebSocketHandler
from tornado.httpclient import AsyncHTTPClient
import json

class StudentsHandler(RequestHandler):
    def on_response(self, response):
        if response.error:
            self.send_error(500)
        else:
            data = json.loads(response.body)
            self.write(data)
        self.finish()

    @asynchronous # 不关闭通信通道
    def get(self, *args, **kwargs):
        url = "http://127.0.0.1:8888/jsontest"
        # 创建客户端
        client = AsyncHTTPClient()
        client.fetch(url, self.on_response)

# 协程异步
class Students2Handler(RequestHandler):

    @tornado.gen.coroutine
    def get(self, *args, **kwargs):
        url = "http://127.0.0.1:8888/jsontest"
        client = AsyncHTTPClient()
        res = yield client.fetch(url)
        if res.error:
            self.send_error(500)
        else:
            data = json.loads(res.body)
            self.write(data)

class HomeHandler(RequestHandler):
    def get(self, *args, **kwargs):
        self.render("home.html")

class ChatHandler(WebSocketHandler):
    users = []

    def open(self, *args, **kwargs):
        self.users.append(self)
        for user in self.users:
            print(user)
            user.write_message(u"[%s]登陆了" % self.request.remote_ip)

    def on_message(self, message):
        for user in self.users:
            user.write_message(u"[%s]说：%s" % (self.request.remote_ip, message))

    def on_close(self):
        self.users.remove(self)
        for user in self.users:
            print(user)
            user.write_message(u"[%s]退出了" % self.request.remote_ip)

    def check_origin(self, origin):
        return True


import time
from datetime import datetime

import requests
from selenium import webdriver, common

class LinkedinScraper:
    """
    Scrapes the job postings from the Linkedin job board
    """

    def __init__(self):
        self.driver = webdriver.Chrome()
        self.driver.set_page_load_timeout(10)
        self.logged_in = False

    def search(self, keywords, location, start):
        def build_link():
            link = 'https://www.linkedin.com/jobs/search/?f_TP=1&'
            link += 'keywords={}'.format('%20'.join(keywords.split()))
            link += '&location={}'.format('%20'.join(location.split()))
            link += '&start={}'.format(start)
            link += '&trk=jobs_jserp_posted_one_day'
            return link

        self.keywords = keywords
        self.location = location

        link = build_link()
        self.driver.get(link)
        time.sleep(2)

    def login(self, user, pw):
        login_page_link = self.driver.get('https://www.linkedin.com/uas/login')
        self.driver.find_element_by_css_selector('input#username').send_keys(user)
        self.driver.find_element_by_css_selector('input#password').send_keys(pw)
        self.driver.find_element_by_css_selector('button').click()
        self.logged_in = True

    def scrape_jobs(self):
        job_postings = []
        for elem in self.driver.find_elements_by_css_selector('li'):
            if 'ember' in elem.get_attribute('id') and 'occludable-update' in elem.get_attribute('class'):
                job_postings.append(elem)

        job_data = []

        for elem in job_postings:
            elem.click()
            print('sleeping; waiting for job posting to load...')
            time.sleep(1)
            buttons = self.driver.find_elements_by_css_selector('button')

            job_information = {
                    'title': '',
                    'description': '',
                    'link': '',
                    'date': datetime.now().date(),
                    'keywords': self.keywords,
                    'location': self.location
            }

            # get title
            for h1 in self.driver.find_elements_by_css_selector('h1'):
                if 'jobs-details-top-card__job-title' in h1.get_attribute('class'):
                    print('got title')
                    job_information['title'] = h1.text

            # get article description
            for div in self.driver.find_elements_by_css_selector('div'):
                try:
                    if 'jobs-box__html-content' in div.get_attribute('class'):
                        print('got description')
                        job_information['description'] = div.text
                except common.exceptions.StaleElementReferenceException:
                    print('stale element; ignoring and continuing...')

            for button in buttons:
                if 'ember' in button.get_attribute('id') and 'jobs-apply-button' in button.get_attribute('class'):
                    if 'Easy Apply' in button.text:
                        # have to implement easy apply handler
                        break
                    button.click()
                    self.driver.switch_to.window(self.driver.window_handles[1])
                    time.sleep(3)
                    print('sleeping; waiting for page to load...')
                    try:
                        job_information['link'] = self.driver.current_url
                    except common.exceptions.TimeoutException:
                        print('timed out; moving on...')
                    self.driver.close()
                    self.driver.switch_to.window(self.driver.window_handles[0])
                    break
            job_data.append(job_information)

        return job_data


from datetime import datetime

from airflow.decorators import dag

from datarobot_provider.operators.datarobot import CreateProjectOperator
from datarobot_provider.operators.datarobot import DeployRecommendedModelOperator
from datarobot_provider.operators.datarobot import GetFeatureDriftOperator
from datarobot_provider.operators.datarobot import GetTargetDriftOperator
from datarobot_provider.operators.datarobot import ScorePredictionsOperator
from datarobot_provider.operators.datarobot import TrainModelsOperator
from datarobot_provider.sensors.datarobot import AutopilotCompleteSensor
from datarobot_provider.sensors.datarobot import ScoringCompleteSensor

@dag(
    schedule=None,
    start_date=datetime(2022, 1, 1),
    tags=['example'],
)
def datarobot_pipeline():
    create_project_op = CreateProjectOperator(task_id='create_project')

    train_models_op = TrainModelsOperator(
        task_id="train_models",
        project_id=create_project_op.output,
    )

    autopilot_complete_sensor = AutopilotCompleteSensor(
        task_id="check_autopilot_complete",
        project_id=create_project_op.output,
    )

    deploy_model_op = DeployRecommendedModelOperator(
        task_id="deploy_recommended_model",
        project_id=create_project_op.output,
    )

    score_predictions_op = ScorePredictionsOperator(
        task_id="score_predictions",
        deployment_id=deploy_model_op.output,
    )

    scoring_complete_sensor = ScoringCompleteSensor(
        task_id="check_scoring_complete",
        job_id=score_predictions_op.output,
    )

    target_drift_op = GetTargetDriftOperator(
        task_id="target_drift",
        deployment_id=deploy_model_op.output,
    )

    feature_drift_op = GetFeatureDriftOperator(
        task_id="feature_drift",
        deployment_id=deploy_model_op.output,
    )

    (
        create_project_op
        >> train_models_op
        >> autopilot_complete_sensor
        >> deploy_model_op
        >> score_predictions_op
        >> scoring_complete_sensor
    )

    scoring_complete_sensor >> target_drift_op
    scoring_complete_sensor >> feature_drift_op

datarobot_pipeline_dag = datarobot_pipeline()


from typing import Tuple
import numpy as np
import random
import json
import csv
import os

def readData(path:str, ext=['.mp4']) -> list:
    assert len(path) > 0, "path string is empty"

    _data_path = []

    for _path, _dir, _files in os.walk(path):
        for _filename in _files:
            _ext = os.path.splitext(_filename)[-1]

            if _ext not in ext:
                continue

            _data_path.append(os.path.join(_path, _filename))

    return _data_path

def readJson(path:str) -> dict:
    with open(path) as json_file:
        json_data = json.load(json_file)

    return json_data

def splitData(path_list:list, train:float=0.8, valid:float=0.1) -> Tuple[list, list, list]:

    """
        split path data to train, valid, test

        if train+valid less than 1, other data is test data
    """
    for i in range(5):
        random.shuffle(path_list)

    assert (train + valid) <= 1, "please sum of train, valid is less than 1"
    _path_len = len(path_list)

    _train_path_list = path_list[:int(_path_len*train)]
    _valid_path_list = path_list[int(_path_len*train):int(_path_len*(train+valid))]

    if train + valid == 1:
        _test_path_list = None
    else:
        _test_path_list = path_list[int(_path_len*(train+valid)):]

    return _train_path_list, _valid_path_list, _test_path_list

def saveToCSV(data_list:list, filename:str, header:list=None):
    assert np.array(header).ndim == 1 or header is None, "header dimention is only 1 or None"

    with open(filename, 'w', encoding='utf-8') as f:
        _wr = csv.writer(f)

        if header is not None:
            _wr.writerow(header)

        for _data_row in data_list:
            _wr.writerow([_data_row])

def readCSV(csv_path:str):
    _csv_list = []

    with open(csv_path, 'r', encoding='utf-8') as f:
        _wr = csv.reader(f)

        for _row in _wr:
            _csv_list.append(_row)

    return _csv_list


# Import libraries
import openmdao.api as om
import numpy as np
import dymos as dm
import matplotlib.pyplot as plt
from paropt.paropt_sparse_driver import ParOptSparseDriver

class ODESystem1(om.ExplicitComponent):
    def initialize(self):
        # Define obstacles
        self.x0 = 8
        self.y0 = 6
        self.x1 = 5
        self.y1 = 5
        self.x2 = 8
        self.y2 = 8

        # Define penalty
        self.p = 45.0

        # Declare options
        self.options.declare("num_nodes", types=int)

    def setup(self):
        nn = self.options["num_nodes"]
        # Inputs
        self.add_input("x", val=np.zeros(nn), desc="Horizontal Position", units="m")
        self.add_input("y", val=np.zeros(nn), desc="Vertical Position", units="m")
        self.add_input("theta", val=np.zeros(nn), desc="angle of wire", units="rad")

        # outputs
        self.add_output(
            "xdot", val=np.zeros(nn), desc="horizontal velocity", units="m/s"
        )
        self.add_output("ydot", val=np.zeros(nn), desc="vertical velocity", units="m/s")
        self.add_output(
            "cflag", val=np.zeros(nn), desc="collision detection", units="m"
        )

        # Setup partials
        arange = np.arange(self.options["num_nodes"], dtype=int)
        self.declare_partials(of="xdot", wrt="theta", rows=arange, cols=arange)
        self.declare_partials(of="ydot", wrt="theta", rows=arange, cols=arange)
        self.declare_partials(of="xdot", wrt="x", rows=arange, cols=arange)
        self.declare_partials(of="ydot", wrt="x", rows=arange, cols=arange)
        self.declare_partials(of="xdot", wrt="y", rows=arange, cols=arange)
        self.declare_partials(of="ydot", wrt="y", rows=arange, cols=arange)
        self.declare_partials(of="cflag", wrt="x", rows=arange, cols=arange)
        self.declare_partials(of="cflag", wrt="y", rows=arange, cols=arange)

    def compute(self, inputs, outputs):
        theta = inputs["theta"]
        x = inputs["x"]
        y = inputs["y"]
        cos_theta = np.cos(theta)
        sin_theta = np.sin(theta)
        r0 = np.sqrt((x - self.x0) ** 2 + (y - self.y0) ** 2) + 0.01
        r1 = np.sqrt((x - self.x1) ** 2 + (y - self.y1) ** 2) + 0.01
        r2 = np.sqrt((x - self.x2) ** 2 + (y - self.y2) ** 2) + 0.01

        cflag = (
            1.0
            / (1.0 + np.exp(-self.p * (r0 - 1.0)))
            * 1.0
            / (1.0 + np.exp(-self.p * (r1 - 1.0)))
            * 1.0
            / (1.0 + np.exp(-self.p * (r2 - 1.0)))
        )
        outputs["xdot"] = 5.0 * cflag * cos_theta
        outputs["ydot"] = 5.0 * cflag * sin_theta
        outputs["cflag"] = cflag

    def compute_partials(self, inputs, jacobian):
        theta = inputs["theta"]
        x = inputs["x"]
        y = inputs["y"]
        cos_theta = np.cos(theta)
        sin_theta = np.sin(theta)

        r0 = np.sqrt((x - self.x0) ** 2 + (y - self.y0) ** 2) + 0.01
        r1 = np.sqrt((x - self.x1) ** 2 + (y - self.y1) ** 2) + 0.01
        r2 = np.sqrt((x - self.x2) ** 2 + (y - self.y2) ** 2) + 0.01

        cflag = (
            1.0
            / (1.0 + np.exp(-self.p * (r0 - 1.0)))
            * 1.0
            / (1.0 + np.exp(-self.p * (r1 - 1.0)))
            * 1.0
            / (1.0 + np.exp(-self.p * (r2 - 1.0)))
        )

        jacobian["xdot", "theta"] = -5 * cflag * sin_theta
        jacobian["ydot", "theta"] = 5 * cflag * cos_theta

        cflag__r0 = (
            cflag
            / (1.0 + np.exp(-self.p * (r0 - 1.0)))
            * (self.p)
            * np.exp(-self.p * (r0 - 1.0))
        )
        cflag__r1 = (
            cflag
            / (1.0 + np.exp(-self.p * (r1 - 1.0)))
            * (self.p)
            * np.exp(-self.p * (r1 - 1.0))
        )
        cflag__r2 = (
            cflag
            / (1.0 + np.exp(-self.p * (r2 - 1.0)))
            * (self.p)
            * np.exp(-self.p * (r2 - 1.0))
        )
        r0__x = (x - self.x0) / r0
        r0__y = (y - self.y0) / r0
        r1__x = (x - self.x1) / r1
        r1__y = (y - self.y1) / r1
        r2__x = (x - self.x2) / r2
        r2__y = (y - self.y2) / r2

        jacobian["cflag", "x"] = (
            cflag__r0 * r0__x + +cflag__r1 * r1__x + +cflag__r2 * r2__x
        )
        jacobian["cflag", "y"] = (
            cflag__r0 * r0__y + +cflag__r1 * r1__y + +cflag__r2 * r2__y
        )

        xdot__cflag = 5 * cos_theta
        ydot__cflag = 5 * sin_theta

        jacobian["xdot", "x"] = xdot__cflag * jacobian["cflag", "x"]
        jacobian["xdot", "y"] = xdot__cflag * jacobian["cflag", "y"]

        jacobian["ydot", "x"] = ydot__cflag * jacobian["cflag", "x"]
        jacobian["ydot", "y"] = ydot__cflag * jacobian["cflag", "y"]

def pen_f(r0, r1, r2):
    p = 45
    return (
        1
        / (1 + np.exp(-p * (r0 - 1)))
        * 1
        / (1 + np.exp(-p * (r1 - 1)))
        * 1
        / (1 + np.exp(-p * (r2 - 1)))
    )

# Define the OpenMDAO problem
p = om.Problem(model=om.Group())

# Define a Trajectory object
traj = p.model.add_subsystem("traj", subsys=dm.Trajectory())

# Define a Dymos Phase object with GaussLobatto Transcription
n_segments = 20
transcript = dm.GaussLobatto(num_segments=n_segments, order=3)
phase = dm.Phase(ode_class=ODESystem1, transcription=transcript)
traj.add_phase(name="phase0", phase=phase)

# Set the time options
# Time has no targets in our ODE.
# We fix the initial time so that the it is not a design variable in the optimization.
# The duration of the phase is allowed to be optimized, but is bounded on [0.5, 10].
phase.set_time_options(
    initial_bounds=(0, 0), duration_bounds=(0.5, 10.0), units="s"  # fix_initial = True,
)

# Set the time options
# Initial values of positions and velocity are all fixed.
# The final value of position are fixed, but the final velocity is a free variable.
# The equations of motion are not functions of position, so 'x' and 'y' have no targets.
# The rate source points to the output in the ODE which provides the time derivative of the given state.
phase.add_state(
    "x", rate_source="xdot", units="m", fix_initial=True, fix_final=True, targets=["x"]
)

phase.add_state(
    "y", rate_source="ydot", units="m", fix_initial=True, fix_final=True, targets=["y"]
)

# Define theta as a control.
phase.add_control(
    name="theta", targets=["theta"], units="rad", lower=-np.pi / 2, upper=np.pi / 2
)

# Define constraints
phase.add_timeseries_output("cflag", output_name="cflag", units="m")

# Minimize final time.
phase.add_objective("time", loc="final")

# Setup the problem
p.setup(check=True)

p.set_val(
    "traj.phase0.states:x",
    phase.interpolate(xs=[0, 1, 2], ys=[0, 5, 10], kind="linear", nodes="state_input"),
    units="m",
)

# initialize 0: left, 1: middle, 2: right
init = 1
if init == 0:
    p.set_val(
        "traj.phase0.states:y",
        phase.interpolate(
            xs=[0, 1, 2], ys=[0, 8, 10], kind="quadratic", nodes="state_input"
        ),
        units="m",
    )
elif init == 1:
    p.set_val(
        "traj.phase0.states:y",
        phase.interpolate(ys=[0, 10], nodes="state_input"),
        units="m",
    )
elif init == 2:
    p.set_val(
        "traj.phase0.states:y",
        phase.interpolate(
            xs=[0, 1, 2], ys=[0, 2, 10], kind="quadratic", nodes="state_input"
        ),
        units="m",
    )

p.set_val(
    "traj.phase0.controls:theta",
    phase.interpolate(ys=[-5, 5], nodes="control_input"),
    units="deg",
)

p["traj.phase0.t_duration"] = 3.0
p["traj.phase0.t_initial"] = 0.0

# Set up the optimization driver
p.driver = ParOptSparseDriver()

options = {
    "algorithm": "ip",
    "norm_type": "infinity",
    "qn_type": "bfgs",
    "qn_subspace_size": 10,
    "starting_point_strategy": "least_squares_multipliers",
    "qn_update_type": "damped_update",
    "abs_res_tol": 1e-6,
    "barrier_strategy": "monotone",
    "armijo_constant": 1e-5,
    "penalty_gamma": 100.0,
    "max_major_iters": 500,
}

for key in options:
    p.driver.options[key] = options[key]

# Allow OpenMDAO to automatically determine our sparsity pattern.
# Doing so can significantly speed up the execution of Dymos.
p.driver.declare_coloring(show_summary=True, show_sparsity=True)

# Run the driver to solve the problem
p.run_driver()
print(p.get_val("traj.phase0.timeseries.time")[-1])

# Check the validity of our results by using scipy.integrate.solve_ivp to integrate the solution.
sim_out = traj.simulate()

# Plot the results
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4.5))
axes[0].plot(
    p.get_val("traj.phase0.timeseries.states:x"),
    p.get_val("traj.phase0.timeseries.states:y"),
    "ro",
    label="Solution",
)
axes[0].plot(
    sim_out.get_val("traj.phase0.timeseries.states:x"),
    sim_out.get_val("traj.phase0.timeseries.states:y"),
    "b-",
    label="Simulation",
)
axes[0].set_xlabel("x (m)")
axes[0].set_ylabel("y (m)")
axes[0].legend()
axes[0].grid()

axes[1].plot(
    p.get_val("traj.phase0.timeseries.time"),
    p.get_val("traj.phase0.timeseries.cflag", units="m"),
    "ro",
    label="Solution",
)
axes[1].plot(
    sim_out.get_val("traj.phase0.timeseries.time"),
    sim_out.get_val("traj.phase0.timeseries.cflag", units="m"),
    "b-",
    label="Simulation",
)
axes[1].set_xlabel("time (s)")
axes[1].set_ylabel("$v$ (m/s)")
axes[1].legend()
axes[1].grid()

ode = ODESystem1()
ode.initialize()

# Add circles
circle0 = plt.Circle((ode.x0, ode.y0), 1, color="k", fill=False, hatch="///")
circle1 = plt.Circle((ode.x1, ode.y1), 1, color="k", fill=False, hatch="///")
circle2 = plt.Circle((ode.x2, ode.y2), 1, color="k", fill=False, hatch="///")
axes[0].add_artist(circle0)
axes[0].add_artist(circle1)
axes[0].add_artist(circle2)

# Additional plots
x_v = np.linspace(0, 10, 50)
y_v = np.linspace(0, 10, 50)
x_m, y_m = np.meshgrid(x_v, y_v)
r0_m = np.sqrt((x_m - ode.x0) ** 2 + (y_m - ode.y0) ** 2)
r1_m = np.sqrt((x_m - ode.x1) ** 2 + (y_m - ode.y1) ** 2)
r2_m = np.sqrt((x_m - ode.x2) ** 2 + (y_m - ode.y2) ** 2)
eta_m = pen_f(r0_m, r1_m, r2_m)
z_m = 5 * eta_m
fig, ax = plt.subplots(1, 1)
cplot = ax.contourf(x_m, y_m, z_m)
cbar = fig.colorbar(cplot)  # Add a colorbar to a plot
cbar.set_label("velocity (m/s)")
ax.set_xlabel("x position (m)")
ax.set_ylabel("y position (m)")
ax.grid()

# Add circles
circle0 = plt.Circle((ode.x0, ode.y0), 1, color="k", fill=False)
circle1 = plt.Circle((ode.x1, ode.y1), 1, color="k", fill=False)
circle2 = plt.Circle((ode.x2, ode.y2), 1, color="k", fill=False)
ax.add_artist(circle0)
ax.add_artist(circle1)
ax.add_artist(circle2)

plt.show()


#!/usr/bin/env python

import argparse
import json
import os
import shutil

from pathlib import Path

def process_source(content, snipfolder, fpath, fnum, ext):
	if not ext:
		return()
	fname = os.path.basename(fpath)
	outname = snipfolder + "/" + fname.replace(".", "_") + "_snip" + str(fnum) + ext
	outfile = open(outname, "wt")
	outfile.write(content)
	outfile.close

def process_file(filename, snipfolder):
	code = False
	filenum = 1
	source = ""
	ext = ""
	f = open(filename, "rt")
	for line in f:
		if code:
			source = source + line.replace("
", "").replace("", "").replace("","")
		if line.find("<language>") >= 0:
			if line.find("C++") >= 0:
				ext = ".cpp"
			code = True
		if line.find("") >= 0:
			process_source(source, snipfolder, filename, filenum, ext)
			filenum = filenum + 1
			code = False
			source = ""
	f.close()
	print("  - {} code segments extracted".format(filenum -1))

parser = argparse.ArgumentParser(description='Extract source embedded within specified XML format files into .snippet_scan folder (or -o folder) ready for snippet analysis using Detect script.', prog='extract_xml_snippets.py')
parser.add_argument("-s", "--sourcepath", type=str, default='.', help='Path to scan', required=True)
parser.add_argument("-e", "--extension", type=str, help='File extension(s) to scan for (multiple extensions can be specified)', action='append', nargs=1)
parser.add_argument("-d", "--deletesnippetfolder", action='store_true', help='Delete .snippet_scan folder if it exists')
parser.add_argument("-o", "--outputfolder", type=str, default='.snippet_scan', help='Specify output folder (default .snippet_scan)')

args = parser.parse_args()

snipfolder = args.sourcepath + '/' + args.outputfolder
if os.path.isdir(snipfolder):
	if args.deletesnippetfolder:
		shutil.rmtree(snipfolder)
		#os.rmdir(snipfolder)
		os.mkdir(snipfolder)
	else:
		print("{} folder exists already - use --deletesnippetfolder option to delete".format(snipfolder))
		exit()
else:
	os.mkdir(snipfolder)

for ext in args.extension:
    if (ext[0][0] != '.'):
        myext = '.' + ext[0]
    else:
    	myext = ext[0]

    for filename in Path(args.sourcepath).rglob("*" + myext):
        print("Processing file: {}".format(filename))
        process_file(filename, snipfolder)


from kafka import KafkaProducer
import json
import requests
import time

topic = "randomuser_data"
kafka_config = {
    "bootstrap_servers": "localhost:9092",  # Change this to your Kafka server address
}

producer = KafkaProducer(bootstrap_servers=kafka_config["bootstrap_servers"])

while True:
    try:
        # Fetch user data from the RandomUser.me API
        response = requests.get('https://randomuser.me/api/')
        response.raise_for_status()  # Raise an exception if the response is not successful

        # Check if the response is not empty
        if response.text.strip():
            # Attempt to parse the response as JSON
            user_data = response.json()

            # Serialize user data to JSON
            user_data_json = json.dumps(user_data)

            # Print the fetched data
            print("Fetched user data:")
            print(user_data_json)

            # Send the data to Kafka
            producer.send(topic, value=user_data_json.encode("utf-8"))
            producer.flush()
        else:
            print("Error: Empty response from the API")

    except requests.exceptions.RequestException as e:
        print(f"Error: An HTTP request error occurred - {e}")

    except json.JSONDecodeError as e:
        print(f"Error: Invalid JSON response from the API - {e}")

    except Exception as e:
        print(f"An unexpected error occurred - {e}")

    # Add a delay of 6 seconds before the next data collection
    time.sleep(6)


import numpy as np
from skimage import io
import matplotlib.pyplot as plt
from matplotlib.patches import Ellipse

## Q1

# Function to compute mean. Inputs:( image, xOrder(column), yOrder(row))
def computeSpatialMoment(img, p, q):
	moment = 0.0
	for r in range(0, len(img)):
		for c in range(0, len(img[0])):
			moment += (pow(r, q) * pow(c, p) * img[r][c])

	return moment

# Function to compute similitude moments. Inputs:(image, xOrder(column), yOrder(row))
def computeSimilitudeMoment(img, i, j):
	moment = 0.0

	# Zeroth order moments
	m00 = computeSpatialMoment(boxIm, 0, 0)

	#First order moments
	m10 = computeSpatialMoment(boxIm, 1, 0)
	m01 = computeSpatialMoment(boxIm, 0, 1)

	centroidC = m10 / m00
	centroidR = m01 / m00

	for r in range(0, len(img)):
		for c in range(0, len(img[0])):
			moment += (pow(r - centroidR, j) * pow(c - centroidC, i) * img[r][c])

	moment = moment / pow(m00, 0.5 * (i+j) + 1)
	return moment

def similitudeMoments(boxIm):
	# Similitude Moments
	N=[]
	# Use the condition 2<= i+j <= 3 to compute values of i and j
	for i in range(0,4):
		jLower = max(0, 2-i)
		jUpper = 3 - i
		for j in range(jLower, jUpper+1):
			N.append(computeSimilitudeMoment(boxIm, i, j))
	return N

numfiles = 4
for filenum in range (0,numfiles):
	filename = 'boxIm' + str(filenum+1) + '.bmp'
	boxIm = io.imread(filename)
	Nvals = similitudeMoments(boxIm)
	print(Nvals)

## Q2
# Load the data
X = np.loadtxt('eigdata.txt')
plt.subplot(2,1,1)
plt.plot(X[:,0], X[:,1],'b.')
plt.axis('equal')

# mean subtract data
m = np.mean(X, axis=0)
Y = X - (np.ones((len(X),1))*m)
plt.subplot(2,1,2)
plt.plot(Y[:,0], Y[:,1], 'r.')
plt.axis('equal')
plt.show()

##Q3

N = len(Y)

# Compute covariance matrix
K = (1.0/(N-1)) * np.matmul(np.transpose(Y), Y)
fig, ax = plt.subplots()
plt.plot(Y[:,0], Y[:,1], 'r.')
plt.axis('equal')

# Compute eigenvalues(V) and eigenvectors(U) from covariance matrix
[V,U] = np.linalg.eig(K)
print(K)
print(V)
print(U)
C = 9.0
for i in range(0,len(V)):
	# Compute half length
	halfLength = np.sqrt(V[i] * C)
	# the columns of U are the axes
	plt.plot([-halfLength*U[0,i],halfLength*U[0,i]], [-halfLength*U[1,i],halfLength*U[1,i]],'g', lw=1)

# plot ellipse
ang=np.arctan(U[1,0]/U[0,0]) / np.pi *180
ax.add_patch(Ellipse(xy=(0,0), width=2*np.sqrt(V[0] * C),height=2*np.sqrt(V[1] * C), angle=ang, fc='none', ec='b'))
plt.show()

## Q4
#Rotate Y using eigenvectors

# Projection of Y on eigenvectors
YRotated = np.matmul(Y, np.transpose(U))

# plot results
plt.plot(YRotated[:,0], YRotated[:,1], 'r.')
plt.axis('equal')
plt.show()

# plot histogram after projecting to y-axis
nbins = int(max(YRotated[:,1]) - min(YRotated[:,1]))
plt.hist(YRotated[:,1], nbins,orientation=u'horizontal')
plt.show()


import asyncio
import logging
import socket

CHUNK_SIZE_HTTP = 1024
QUEUE_SOCKET = 8
TIMEOUT_CLIENT = 5

async def get_data(client):
    loop = asyncio.get_event_loop()
    data = b''
    while True:
        chunk = await loop.sock_recv(client, CHUNK_SIZE_HTTP)
        data += chunk
        if len(chunk) < CHUNK_SIZE_HTTP:
            break
    return data

def get_socket(host, port):
    server_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    server_sock.bind((host, port))
    server_sock.listen(QUEUE_SOCKET)
    server_sock.setblocking(False)
    logging.info('Start on {}'.format(server_sock.getsockname()))
    return server_sock

async def handle_client(client, core_engine, loop):
    data = await get_data(client)

    data_generator = await core_engine(data)

    for chunk in data_generator:
        await loop.sock_sendall(client, chunk)

    client.close()

async def run(loop, server_sock, core_engine):
    while True:
        client, _ = await loop.sock_accept(server_sock)
        client.settimeout(TIMEOUT_CLIENT)
        loop.create_task(handle_client(client, core_engine, loop))


# -*- coding: utf-8 -*-
"""
Test cases for rendering exporters
"""
from io import BytesIO
from hashlib import sha256
from unittest import SkipTest
import numpy as np

from holoviews.plotting.mpl.renderer import MPLRenderer
from holoviews import HoloMap, Image, ItemTable
from holoviews.element.comparison import ComparisonTestCase

from nose.plugins.attrib import attr

from .testwidgets import normalize

try:
    # Standardize backend due to random inconsistencies
    from matplotlib import pyplot
    pyplot.switch_backend('agg')
except:
    pyplot = None

def digest_data(data):
    hashfn = sha256()
    hashfn.update(data)
    return hashfn.hexdigest()

@attr(optional=1)
class MPLRendererTest(ComparisonTestCase):
    """
    Note if not possible to compare the hashes of SVG and WebM formats
    as the hashes are not stable across exports.
    """

    def setUp(self):
        if pyplot is None:
            raise SkipTest("Matplotlib required to test widgets")

        self.basename = 'no-file'
        self.image1 = Image(np.array([[0,1],[2,3]]), label='Image1')
        self.image2 = Image(np.array([[1,0],[4,-2]]), label='Image2')
        self.map1 = HoloMap({1:self.image1, 2:self.image2}, label='TestMap')

        self.unicode_table = ItemTable([('β','Δ1'), ('°C', '3×4')],
                                       label='Poincaré', group='α Festkörperphysik')

        self.renderer = MPLRenderer.instance()

    def test_simple_export_gif(self):
        data = self.renderer(self.map1, fmt='gif')[0]
        self.assertEqual(digest_data(data),
                         '95258c17d10620f20604c9cbd17e6b65e886a6163c96d6574f3eb812e0f149c2')

    def test_simple_export_gif_double_size(self):
        data = self.renderer.instance(size=200)(self.map1, fmt='gif')[0]
        self.assertEqual(digest_data(data),
                         'fbe6d753df1471315cbd83d370379591af0fdea114601c5ce1a615777749ca91')

    def test_simple_export_gif_half_fps(self):
        data = self.renderer.instance(fps=5)(self.map1, fmt='gif', )[0]
        self.assertEqual(digest_data(data),
                         'add756aa3caeb4c5f2396cdd5bd0122128c6a1275de9d3a44a0c21a734c4d5f4')

    def test_simple_export_png1(self):
        data = self.renderer(self.image1, fmt='png')[0]
        self.assertEqual(digest_data(data),
                         '08b7d97e79f715f9d8593416b1f6561ebe3e75bb038172ffd6048286ab09e671')

    def test_simple_export_png1_double_size(self):
        data = self.renderer.instance(size=200)(self.image1, fmt='png')[0]
        self.assertEqual(digest_data(data),
                         '0b48a3e366b300fcfea3cba47d6a07631a8fcc02a96860f1291233ef2c976764')

    def test_simple_export_png2(self):
        data = self.renderer(self.image2, fmt='png')[0]
        self.assertEqual(digest_data(data),
                         '2b5f1639584cd4c18c01cfb3f26d26dfa582fff39b869223269c0d941f17cc8b')

    def test_simple_export_png2_double_size(self):
        data = self.renderer.instance(size=200)(self.image2, fmt='png')[0]
        self.assertEqual(digest_data(data),
                         'ef8e2df9c3d3a27e738ae1ca9f5b0704b6467cb44265f7933a3c137ce8a8a519')

    def test_simple_export_unicode_table_png(self):
        "Test that unicode support and rendering is working"
        data = self.renderer.instance(size=200)(self.unicode_table, fmt='png')[0]
        self.assertEqual(digest_data(data),
                         'a3dd68a888de14064cb621c14be5b175d96781cdbc932a3f778def34beaee1ff')

    def test_static_html_scrubber(self):
        data = normalize(self.renderer.static_html(self.map1, fmt='scrubber'))
        self.assertEqual(digest_data(data),
                         '631f32e5be35211e49e1dcd13a7ea117331deddafb97fc4815000ca1ed80397f')

    def test_static_html_widgets(self):
        data = normalize(self.renderer.static_html(self.map1, fmt='widgets'))
        self.assertEqual(digest_data(data),
                         '9c4ac8fc5e5689c4f671b8483b06a7d6042559539b224adf82a3ed4946c8eae6')

    def test_static_html_gif(self):
        data = self.renderer.static_html(self.map1, fmt='gif')
        self.assertEqual(digest_data(data),
                         '9d43822e0f368f3c673b19aaf66d22252849947b7dc4a157306c610c42d319b5')
    def test_export_widgets(self):
        bytesio = BytesIO()
        self.renderer.export_widgets(self.map1, bytesio, fmt='widgets')
        data = normalize(bytesio.read())
        self.assertEqual(digest_data(data),
                         '91bbc7b4efebd07b1ee595b902d9899b27f2c7e353dfc87c57c2dfd5d0404301')


import torch
import argparse
from timeit import default_timer as time
from PIL import Image
import yaml
import numpy as np
from denoising_pipeline.model_estimator.model import Model
from denoising_pipeline.architectures.configure_model import \
    build_model_from_config

def parse_args():
    parser = argparse.ArgumentParser(description='Video denoising train script')
    parser.add_argument('--config', required=False, type=str,
                          default='../configuration/example_train_config.yml',
                          help='Path to configuration yml file.'
                        )
    parser.add_argument('--model-weights', required=True, type=str,
                        help='Path to model weights')
    parser.add_argument('--input-image', required=True, type=str,
                        help='Path to image file')
    parser.add_argument('--output-image', required=True, type=str,
                        help='Path to result image file')
    parser.add_argument('--batch-size', required=False, type=int, default=16,
                        help='Inference model batch size')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    return parser.parse_args()

def main():
    args = parse_args()

    use_cuda = not args.no_cuda and torch.cuda.is_available()
    device = 'cuda' if use_cuda else 'cpu'

    with open(args.config, 'r') as f:
        config = yaml.safe_load(f)

    window_size = config['model']['window_size']

    series_size = 1 \
        if config['model']['architecture'] != 'sequential' else \
        config['model'][config['model']['architecture']]['series_size']

    denoising_model = build_model_from_config(config)

    model = Model(
        denoising_model,
        device
    )

    print(
        'Count of model trainable parameters: {}'.format(
            model.get_parameters_count()
        )
    )

    model.load(args.model_weights)

    print('Model out shape: {}'.format(
        model.model.inference(*tuple([
            torch.FloatTensor(
                np.zeros((1, 3, window_size, window_size))
            ).to(device)
        ] * series_size)).shape
    ))

    inp_image = np.array(Image.open(args.input_image).convert('RGB'))
    start_time = time()
    out_image = model.predict(
        image=inp_image,
        window_size=window_size,
        batch_size=args.batch_size,
        verbose=True
    )
    finish_time = time()
    Image.fromarray(out_image).save(args.output_image)

    print(
        'Inference time: {:.2f} sec'.format(
            finish_time - start_time
        )
    )

if __name__ == '__main__':
    main()


from app.api import bp
from flasgger import swag_from
from flask import jsonify, request
from marshmallow import ValidationError
from app.schemas import ZoneSchema, ZoneEntrySchema
from app.api.zones.service import zone_list, zone_history, deactivate
from app.api.errors import bad_request
from app.swagger.zones_specs import get_spec, post_spec, put_spec, get_history_spec, post_deactive_spec
from app import db
from app.models import Zone
from app.api.users.service import token_auth
from app.utils.helpers import build_page

@bp.route('/zones', methods=['GET'])
@swag_from(get_spec)
def get_zones():
    zones = zone_list()
    zone_schema = ZoneSchema(many=True)
    return jsonify(zone_schema.dump(zones))

@bp.route('/zones', methods=['POST'])
@swag_from(post_spec)
@token_auth.login_required
def create_zone():
    data = request.get_json()
    zone_schema = ZoneSchema()

    try:
        zone = zone_schema.load(
            data, session=db.session)
        print(zone)
        db.session.add(zone)
        db.session.commit()

    except ValidationError as err:
        print("error")
        return bad_request(err.messages)

    return jsonify(zone_schema.dump(zone))

@bp.route('/zones/<id>', methods=['PUT'])
@swag_from(put_spec)
@token_auth.login_required
def edit_zone(id):
    data = request.get_json()
    zone_schema = ZoneSchema()
    zone = Zone.query.get(id)
    if(zone):
        data['id'] = zone.id
    else:
        bad_request(f'Zone with id={id} does not exist')

    try:
        zone = zone_schema.load(
            data, session=db.session)
        db.session.add(zone)
        db.session.commit()

    except ValidationError as err:
        print("error")
        return bad_request(err.messages)

    return jsonify(zone_schema.dump(zone))

@bp.route('/zones/<id>', methods=['GET'])
@swag_from(get_history_spec)
def get_history(id):
    zone = Zone.query.get(id)
    if(not zone):
        return bad_request(f'Zone with id={id} does not exist')

    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 10, type=int)
    order_by = request.args.get('order_by', "")
    order = request.args.get('order', "asc")

    history = zone_history(id, order_by, order).paginate(page, per_page, False)
    zone_entry_schema = ZoneEntrySchema(many=True)

    return jsonify(build_page(zone_entry_schema, history, order_by, order))

@bp.route('/zones/deactivate/<id>', methods=['POST'])
@swag_from(post_deactive_spec)
@token_auth.login_required
def deactivate_zone(id):
    zone = deactivate(id)
    if(not zone):
        return bad_request(f'Zone with id={id} does not exist')
    zone_schema = ZoneSchema()
    return jsonify(zone_schema.dump(zone))


from itertools import permutations

with open('input.txt') as f:
    data = f.read()

routes = {}

for line in data.splitlines():
    split = line.split()
    routes[frozenset([split[0], split[2]])] = int(split[-1])

all_destinations = set()
for route in routes:
    all_destinations.update(route)

possible_paths = list(permutations(all_destinations, len(all_destinations)))

costs = []
for path in possible_paths:
    cost = 0
    for i in range(len(path) - 1):
        cost += routes[frozenset([path[i], path[i+1]])]
    costs.append(cost)

print("Part 1:", min(costs))
print("Part 2:", max(costs))


from data_utils import *
from collections import Counter
from tqdm import tqdm
import utils
from sklearn import svm
from sklearn.linear_model import SGDClassifier
from torch.nn import functional as F
from torch import nn
from torch import optim
import train_utils

class KNNModel(object):
    def __init__(self, h=2, k=5):
        self.train_x = None
        self.train_y = None
        self.h = h
        self.k = k

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.train_x = train_x
        self.train_y = train_y if isinstance(train_y, np.ndarray) else np.array(train_y)

    @utils.logging_time_wrapper
    def predict(self, X):
        dis = self.distance(self.train_x, X)
        neighbor_k = np.argsort(dis, axis=1)[:, :self.k]
        neighbor_labels = self.train_y[neighbor_k]
        labels = [Counter(neighbor_labels[i, :]).most_common(1)[0][0] for i in range(neighbor_labels.shape[0])]
        return np.array(labels)

    def distance(self, X1, X2):
        assert X1.shape[1] == X2.shape[1]
        dis = [np.sum(np.abs(X1 - X2[i]) ** self.h, axis=1) ** (1 / self.h) for i in tqdm(range(X2.shape[0]), desc='Distance :')]
        dis = np.array(dis)
        return dis

class TwoLayerNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(TwoLayerNN, self).__init__()
        self.hidden_layer = nn.Linear(input_size, hidden_size)
        self.output_layer = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        x = self.hidden_layer(x)
        x = F.relu(x)
        output = self.output_layer(x)
        output = F.log_softmax(output)
        return output

class SVMModel(object):
    def __init__(self):
        self.model = svm.LinearSVC()

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.model.fit(train_x, train_y)

    @utils.logging_time_wrapper
    def predict(self, test_x):
        return self.model.predict(test_x)

class LRModel(object):
    def __init__(self):
        self.model = SGDClassifier()

    @utils.logging_time_wrapper
    def train(self, train_x, train_y):
        self.model.fit(train_x, train_y)

    @utils.logging_time_wrapper
    def predict(self, test_x):
        return self.model.predict(test_x)

if __name__ == '__main__':
    train, test, train_loader, test_loader, classes = cifar10_datasets(train_sample_num=5000, test_sample_num=1000, flat_pix=False)
    model = TwoLayerNN(3072, 256, 10)
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(params=model.parameters(), lr=0.001)
    train_utils.train(train_loader, model, criterion, optimizer, epoch_size=1)
    # report = utils.evaluate_report(test.targets, labels, target_names=classes)
    # print(report)


import sys
from cx_Freeze import setup, Executable

# Dependencies are automatically detected, but it might need to be fine-tuned.
build_exe_options = {
    "excludes": ['PyQt5'],
    'packages': ['imagej', 'PySimpleGUI', 'caiman', 'ipyparallel', 'skimage', 'seaborn'],
    'include_files': ['config.ini', 'cache'],
    'silent_level': ['1']
}

msi_data = {
    "Icon": [
        ("IconId", "icon.ico"),
    ],
}

bdist_msi_options = {}

# base="Win32GUI" should be used only for Windows GUI app
base = "Win32GUI" if sys.platform == "win32" else None

executables = [
    Executable(
        'main_GUI.py',
        base=base
    )
]

setup(
    name="pipeline",
    version="0.1",
    description="Test on build for incomplete GUI application",
    options={
        "build_exe": build_exe_options,
        "build_msi": bdist_msi_options,
    },
    executables=executables,
)


from rest_framework import serializers
from rest_framework.exceptions import ValidationError

from swp.models import Monitor, Publication, Thinktank
from .validators import css_selector

class ThinktankField(serializers.ModelSerializer):

    class Meta:
        model = Thinktank
        fields = ['id', 'name']
        read_only_fields = ['id']

class MonitorField(serializers.ModelSerializer):
    class Meta:
        model = Monitor
        fields = ['id', 'name']
        read_only_fields = ['id']

class PublicationField(serializers.ModelSerializer):
    class Meta:
        model = Publication
        fields = ['id', 'url', 'title']
        read_only_fields = ['id']

class CSSSelectorField(serializers.CharField):
    default_validators = [css_selector]

class RecipientsField(serializers.ListField):
    child = serializers.EmailField()

    def run_child_validation(self, data):
        result = []
        errors = []

        for idx, item in enumerate(data):
            try:
                result.append(self.child.run_validation(item))
            except ValidationError as e:
                errors.extend(e.detail)

        if not errors:
            return result

        raise ValidationError(errors)


import kfp
from kfp import dsl

CONTAINER_REGISTRY="<Todo_Insert_container_registry_here>"
CONTAINER_TAG="latest"
MINIO_SERVER='minio.cloudtraining-mavencode.com:9000'
MINIO_ACCESS_KEY='some_access_key'
MINIO_SECRET_KEY='some_secret_key'

def datadownload_op(minio_server, data_dir):
    return dsl.ContainerOp(
        name='Data Download',
        image=f'{CONTAINER_REGISTRY}/datadownloader:{CONTAINER_TAG}',
        arguments=[
            '--minio_server', minio_server,
            '--data_dir', data_dir
        ],
        command=["python3", "datadownloader.py"]
    )

def data_preprocess_op(data_dir, clean_data_dir):
    return dsl.ContainerOp(
        name='Data Preprocessing',
        image=f'{CONTAINER_REGISTRY}/preprocess_dataset:{CONTAINER_TAG}',
        arguments=[
            '--data_dir', data_dir,
            '--clean_data_dir', clean_data_dir
        ],
        command=["python3", "preprocess_dataset.py"]
    )

def pca_analysis_op(clean_data_dir, pca_dir):
    return dsl.ContainerOp(
        name='PCA Analysis',
        image=f'{CONTAINER_REGISTRY}/pca_analysis:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--pca_dir', pca_dir
        ],
        command=["python3", "pca_analysis.py"]
    )

def train_op(clean_data_dir, model_dir):
    return dsl.ContainerOp(
        name='Model Training',
        image=f'{CONTAINER_REGISTRY}/train_model:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--model_dir', model_dir
        ],
        command=["python3", "train.py"]
    )

def model_test_op(clean_data_dir, model_dir, metrics_path):
    return dsl.ContainerOp(
        name='Model Testing',
        image=f'{CONTAINER_REGISTRY}/test_model:{CONTAINER_TAG}',
        arguments=[
            '--clean_data_dir', clean_data_dir,
            '--model_dir', model_dir,
            '--metrics_path', metrics_path
        ],
        command=["python3", "test_model.py"]
    )

def model_export_op(model_dir, metrics_path, export_bucket, model_name, model_version, minio_server,
                    minio_access_key, minio_secret_key):
    return dsl.ContainerOp(
        name='Model Export',
        image=f'{CONTAINER_REGISTRY}/export_models:{CONTAINER_TAG}',
        arguments=[
            '--model_dir', model_dir,
            '--metrics_path', metrics_path,
            '--export_bucket', export_bucket,
            '--model_name', model_name,
            '--model_version', model_version,
            '--minio_server', minio_server,
            '--minio_access_key', minio_access_key,
            '--minio_secret_key', minio_secret_key
        ],
        command=["python3", "export_models.py"]
    )

def print_results_op(pca_dir, metrics_path):
    return dsl.ContainerOp(
        name='Result Printing',
        image=f'{CONTAINER_REGISTRY}/print_results:{CONTAINER_TAG}',
        arguments=[
            '--pca_dir', pca_dir,
            '--metrics_path', metrics_path
        ],
        command=["python3", "print_results.py"]
    )

def op_transformer(op):
    op.add_pod_annotation(name="sidecar.istio.io/inject", value="false")
    return op

@dsl.pipeline(
   name='End-to-End Industrial IoT Pipeline',
   description='A sample pipeline to demonstrate multi-step model training, evaluation and export'
)
def monitoring_pipeline(
    model_dir: str = "/train/model",
    data_dir: str = "/train/data",
    clean_data_dir: str = "/train/data",
    metrics_path: str ="/train/metrics",
    pca_dir: str ="/train/metrics",
    export_bucket: str = "monitoring",
    model_name: str = "monitoring",
    model_version: int = 1,
    minio_server: str=MINIO_SERVER,
    minio_access_key: str=MINIO_ACCESS_KEY,
    minio_secret_key: str=MINIO_SECRET_KEY
):

    pvc_op = dsl.VolumeOp(name='Persistent Storage',
            resource_name='data-volume',
            size='2Gi',
            modes=dsl.VOLUME_MODE_RWO)

    datadownload_task = datadownload_op(MINIO_SERVER, data_dir).add_pvolumes({"/train": pvc_op.volume})

    data_preprocess_task = data_preprocess_op(data_dir=data_dir, clean_data_dir=clean_data_dir).add_pvolumes({"/train": pvc_op.volume}).after(datadownload_task)

    pca_analysis_task = pca_analysis_op(clean_data_dir=clean_data_dir, pca_dir=pca_dir).add_pvolumes({"/train": pvc_op.volume}).after(data_preprocess_task)

    train_task = train_op(clean_data_dir=clean_data_dir, model_dir=model_dir).add_pvolumes({"/train": pvc_op.volume}).after(data_preprocess_task)

    test_task = model_test_op(clean_data_dir=clean_data_dir, model_dir=model_dir, metrics_path=metrics_path).add_pvolumes({"/train": pvc_op.volume}).after(train_task)

    model_export_task = model_export_op(
                            model_dir=model_dir,
                            metrics_path=metrics_path,
                            export_bucket=export_bucket,
                            model_name=model_name,
                            model_version=model_version,
                            minio_server=minio_server,
                            minio_access_key=minio_access_key,
                            minio_secret_key=minio_secret_key).add_pvolumes({"/train": pvc_op.volume}).after(test_task)

    results_task = print_results_op(pca_dir=pca_dir, metrics_path=metrics_path).add_pvolumes({"/train": pvc_op.volume}).after(test_task, pca_analysis_task)

    dsl.get_pipeline_conf().add_op_transformer(op_transformer)

if __name__ == '__main__':
    kfp.compiler.Compiler().compile(monitoring_pipeline, 'monitoring_pipeline.yaml')
    # client = kfp.Client(host='pipelines-api.kubeflow.svc.cluster.local:8888')


import pytest

from timeaftertime.core import Coord, Block, Board

# Coord tests
def test_coord_initialization():
    coord = Coord(0,0)
    assert (coord.x == 0) & (coord.y == 0)

def test_coord_eq():
    coord1 = Coord(1,2)
    coord2 = Coord(3,3)
    assert coord1 == coord1
    assert coord1 != coord2

def test_coord_iterable():
    coord = Coord(1,2)
    assert [x for x in coord] == [1,2]

def test_coord_get_neighbors():
    coord = Coord(0,0)
    neighbors = coord.get_neighbors()
    assert neighbors == [Coord(-1,0),
                         Coord(1,0),
                         Coord(0,-1),
                         Coord(0,1)]

# Block tests

def test_block_empty_initialization():
    block = Block()
    assert ((block.coords == []) &
             block.color == 0)

def test_block_initialization():
    block = Block()
    block = Block([Coord(0,0)], 1)
    assert ((block.coords == [Coord(0,0)]) &
             block.color == 1)

def test_block_add_coord():
    block = Block([Coord(0,0)])
    block.add_coord(Coord(0,1))
    assert block.coords == [Coord(0,0), Coord(0,1)]

def test_block_add_duplicate_coord():
    block = Block([Coord(0,0)])
    with pytest.raises(AssertionError):
        block.add_coord(Coord(0,0))

def test_block_add_non_neighboring_coord_far_away():
    block = Block([Coord(0,0)])
    with pytest.raises(AssertionError):
        block.add_coord(Coord(3,3))

def test_block_add_non_neighboring_coord_diagonal():
    block = Block([Coord(3,3)])
    # all diagonal positions
    with pytest.raises(AssertionError):
        block.add_coord(Coord(4,4))
    with pytest.raises(AssertionError):
        block.add_coord(Coord(2,2))
    with pytest.raises(AssertionError):
        block.add_coord(Coord(2,4))
    with pytest.raises(AssertionError):
        block.add_coord(Coord(4,2))

def test_block_add_coords():
    block = Block([Coord(0,0)])
    block.add_coords([Coord(0,1), Coord(0,2)])
    assert block.coords == [Coord(0,0), Coord(0,1), Coord(0,2)]

def test_block_add_duplicate_coords():
    block = Block([Coord(0,0)])
    # all duplicate coords
    with pytest.raises(AssertionError):
        block.add_coords([Coord(0,0), Coord(0,0)])
    # some duplicate coords (but not all)
    with pytest.raises(AssertionError):
        block.add_coords([Coord(0,0), Coord(0,0), Coord(0,1)])

def test_block_add_non_neighboring_coords():
    block = Block([Coord(0,0)])
    # no neighboring coords
    with pytest.raises(AssertionError):
        block.add_coords([Coord(0,3), Coord(1,5), Coord(4,2)])
    # some neighboring coords (but not all)
    with pytest.raises(AssertionError):
        block.add_coords([Coord(0,1), Coord(0,2), Coord(4,2)])

def test_block_remove_coord():
    block = Block([Coord(0,0)])
    block.remove_coord(Coord(0,0))
    assert block.coords == []

def test_block_remove_coord_not_in_coords():
    block = Block([Coord(0,0)])
    block.remove_coord(Coord(0,1))
    assert block.coords == [Coord(0,0)]

def test_block_remove_coords():
    block = Block([Coord(0,0), Coord(0,1)])
    block.remove_coords([Coord(0,0), Coord(0,1)])
    assert block.coords == []

def test_block_remove_coords_some_in_coords():
    block = Block([Coord(0,0), Coord(0,1)])
    block.remove_coords([Coord(0,0), Coord(0,2)])
    assert block.coords == [Coord(0,1)]

def test_block_set_color():
    block = Block([Coord(0,0)], 1)
    assert block.color == 1
    block.set_color(3)
    assert block.color == 3

def test_block_overlaps():
    # some overlap
    block1 = Block([Coord(0,0), Coord(0,1)])
    block2 = Block([Coord(0,0), Coord(1,0)])
    assert block1.overlaps(block2)
    # no overlap
    block3 = Block([Coord(0,2), Coord(0,3)])
    assert not block1.overlaps(block3)

def test_block_overlaps_itself():
    block = Block([Coord(0,0), Coord(0,1)])
    assert block.overlaps(block)

def test_block_not_overlaps_with_empty():
    block = Block([Coord(0,0), Coord(0,1)])
    assert not block.overlaps(Block())

def test_block_neighbors():
    # no overlap
    block1 = Block([Coord(0,0), Coord(0,1)])
    block2 = Block([Coord(2,0), Coord(1,0)])
    assert block1.neighbors(block2)
    # some overlap (not allowed)
    block3 = Block([Coord(0,1), Coord(0,2)])
    assert not block1.neighbors(block3)
    # some overlap (allowed)
    assert block1.neighbors(block3, overlap_allowed=True)

def test_block_not_neighbors_far_away():
    block1 = Block([Coord(0,0), Coord(0,1)])
    block2 = Block([Coord(4,4), Coord(4,5)])
    assert not block1.neighbors(block2)

def test_block_not_neighbors_itself():
    block = Block([Coord(0,0), Coord(0,1)])
    assert not block.neighbors(block)
    assert not block.neighbors(block, overlap_allowed=True)

def test_block_not_neighbors_with_empty():
    block = Block([Coord(0,0), Coord(0,1)])
    assert not block.neighbors(Block())

# Board tests

def test_board_initialization():
    board = Board(15,7)
    assert ((board.height == 15) &
            (board.width == 7) &
            (board.blocks == []))

def test_board_add_block():
    board = Board(15,7)
    block = Block([Coord(0,0), Coord(0,1)], 1)
    board.add_block(block)
    assert len(board.blocks) == 1
    assert board.blocks[0].coords == [Coord(0,0), Coord(0,1)]
    assert board.blocks[0].color == 1

def test_board_add_blocks():
    board = Board(15,7)
    block1 = Block([Coord(0,0)], 1)
    board.add_block(block1)
    block2 = Block([Coord(2,2), Coord(3,2)], 2)
    board.add_block(block2)
    assert len(board.blocks) == 2
    assert board.blocks[1].coords == [Coord(2,2), Coord(3,2)]
    assert board.blocks[1].color == 2

def test_board_add_block_overlap():
    board = Board(15,7)
    block1 = Block([Coord(0,0)], 1)
    board.add_block(block1)
    block2 = Block([Coord(0,0), Coord(0,1)], 2)
    with pytest.raises(AssertionError):
        board.add_block(block2)

def test_board_add_block_same_color_neighbor():
    board = Board(15,7)
    block1 = Block([Coord(0,0)], 1)
    board.add_block(block1)
    block2 = Block([Coord(0,1), Coord(0,2)], 1)
    with pytest.raises(AssertionError):
        board.add_block(block2)

def test_board_add_block_outside_bounds():
    board = Board(15,7)
    block1 = Block([Coord(0,0)], 1)
    board.add_block(block1)
    block2 = Block([Coord(20,20), Coord(20,21)], 1)
    with pytest.raises(AssertionError):
        board.add_block(block2)

def test_board_remove_block():
    board = Board(15,7)
    block = Block([Coord(0,0)], 1)
    board.add_block(block)
    board.remove_block(block)
    assert len(board.blocks) == 0

def test_board_remove_blocks():
    board = Board(15,7)
    block1 = Block([Coord(0,0)], 1)
    board.add_block(block1)
    block2 = Block([Coord(2,2), Coord(3,2)], 2)
    board.add_block(block2)
    board.remove_block(block2)
    assert len(board.blocks) == 1
    board.remove_block(block1)
    assert len(board.blocks) == 0

def test_board_is_empty():
    # empty
    board = Board(2,2)
    assert board.is_empty()
    # partially filled
    block1 = Block([Coord(0,0), Coord(0,1)], 1)
    board.add_block(block1)
    assert not board.is_empty()
    # full
    block2 = Block([Coord(1,0), Coord(1,1)], 2)
    board.add_block(block2)
    assert not board.is_empty()

def test_board_is_full():
    # empty
    board = Board(2,2)
    assert not board.is_full()
    # partially filled
    block1 = Block([Coord(0,0), Coord(0,1)], 1)
    board.add_block(block1)
    assert not board.is_full()
    # full
    block2 = Block([Coord(1,0), Coord(1,1)], 2)
    board.add_block(block2)
    assert board.is_full()

def test_board_coords_available():
    board = Board(2,2)
    block = Block([Coord(0,0)], 1)
    board.add_block(block)
    available_coords = board.coords_available()
    assert len(available_coords) == 2*2-1
    assert Coord(0,0) not in available_coords

def test_board_coords_available_color_same_color():
    board = Board(2,2)
    block = Block([Coord(0,0)], 1)
    board.add_block(block)
    available_coords1 = board.coords_available_color(1)
    assert len(available_coords1) == 2*2-3
    assert available_coords1 == [Coord(1,1)]

def test_board_coords_available_color_different_color():
    board = Board(2,2)
    block = Block([Coord(0,0)], 1)
    board.add_block(block)
    available_coords2 = board.coords_available_color(2)
    assert len(available_coords2) == 2*2-1
    assert Coord(0,0) not in available_coords2
    available_coords = board.coords_available()
    assert available_coords == available_coords2


import sys
import math
import time
import random
import numpy as np
import scipy.stats as stats
from itertools import product
sys.path.append('../')
from common import call_prolog
from common import parmap

num_trials = 10
num_tasks = 50
trials = list(range(1,num_trials+1))
mrules = ['u-2-s','u-2-e','u-2-d','u-2-dm']
tasks = list(range(0,num_tasks))
jobs = list(product(mrules,tasks,trials))
max_time = 600

def get_train_file(task, trial):
    return f'data/train/{task}-{trial}.pl'

def get_test_file(task, trial):
    return f'data/test/{task}-{trial}.pl'

def get_res_file(ms, task, trial):
    return f'results/{ms}/{task}-{trial}.txt'

def get_program_file(ms, task, trial):
    return f'programs/{ms}/{task}-{trial}.pl'

def load_probs(nowrites=True):
    dic = {}
    with open('probs.txt', 'r') as f:
        for line in f:
            if line.startswith('# -*- coding: utf-8 -*-'):
                continue
            xs=line.strip().split('=>')
            if len(xs)==1 and len(xs[0])>0:
                try:
                    problem=int(xs[0][1:])
                    if prob_num > 273:
                        return dic
                except:
                    continue
            elif len(xs)>1:
                if problem not in dic:
                    dic[problem] = []
                dic[problem].append(xs)
    return dic

def gen_data_(dic, trial):
    for problem, examples in dic.items():
        if problem not in tasks:
            continue
        # if len(examples) != 10:
            # continue
        random.shuffle(examples)
        train = examples[:5]
        test = examples[5:]
        filename = get_train_file(problem, trial)
        with open(filename, 'w') as f:
            for x in train:
                a = list(x[0].strip())
                b = list(x[1].strip())
                atom = f'pos({a},{b}).'
                f.write(atom + '\n')
        filename = get_test_file(problem, trial)
        with open(filename, 'w') as f:
            for x in test:
                a = list(x[0].strip())
                b = list(x[1].strip())
                atom = f'pos({a},{b}).'
                f.write(atom + '\n')

def gen_data():
    dic = load_probs()
    for trial in trials:
        gen_data_(dic, trial)

def learn_(args):
    (m, task, k) = args
    data_file = get_train_file(task, k)
    data = f'data/{k}.pl'
    mf = f'../mrules/{m}.pl'
    program_file = get_program_file(m, task, k)
    with open(program_file, 'w') as f:
        a = time.time()
        load_files = ['experiment', data_file, mf]
        prog = call_prolog('learn', load_files, max_time)
        b = time.time()
        if prog == '':
            prog = 'f(_,_):-false.'
        f.write(prog + '\n')
        f.write(f'%time,{b-a}')

def learn():
    list(parmap(learn_,jobs))

def test_(args):
    (m, task, k) = args
    data_file = get_test_file(task, k )
    program_file = get_program_file(m, task, k)
    results_file = get_res_file(m, task, k)
    with open(results_file, 'w') as f:
        f.write(call_prolog('do_test', ['experiment', data_file, program_file]))

def test():
    list(parmap(test_,jobs))

def get_times(ms, task, trial):
    filename = get_program_file(ms, task, trial)
    with open(filename) as f:
        for line in f:
            xs = line.strip().split(',')
            if xs[0]!='%time':
              continue
            return float(xs[1])
    return max_time

def get_accuracies(metarules, task, trial):
    results_file = get_res_file(metarules, task, trial)
    accs = []
    with open(results_file, 'r') as f:
        for line in f:
            line = line.strip()
            if line == '1':
                accs.append(1)
            if line == '0':
                accs.append(0)
    return accs

def acc_results():
    all_accs = {ms:[] for ms in mrules}
    for ms in mrules:
        for trial in trials:
            accs = []
            for task in tasks:
                k_accs = get_accuracies(ms, task, trial)
                accs.append(np.mean(k_accs))
            mu_acc = int(np.mean(accs)*100)
            sem_acc = int(stats.sem(accs)*100)
            all_accs[ms].append(mu_acc)
    out = ''
    for m in mrules:
        mu_acc = int(np.mean(all_accs[m]))
        sem_acc = int(stats.sem(all_accs[m]))
        out += f' & {mu_acc} $\pm$ {sem_acc}'
    print(out)

def time_results():
    all_times = {ms:[] for ms in mrules}
    for ms in mrules:
        for trial in trials:
            times = []
            for task in tasks:
                k_times = get_times(ms, task, trial)
                times.append(np.mean(k_times))
            mu = int(np.mean(times))
            sem = int(stats.sem(times))
            all_times[ms].append(mu)

    out = ''
    # out+='mean'
    for m in mrules:
        mu = int(np.mean(all_times[m]))
        sem = int(stats.sem(all_times[m]))
        out += f' & {mu} $\pm$ {sem}'
    print(out)

def results():
    print('accuracies')
    acc_results()
    print('times')
    time_results()

arg = sys.argv[1]

if arg == 'gen':
    gen_data()
if arg == 'learn':
    learn()
if arg == 'test':
    test()
if arg == 'results':
    results()


import pandas as pd
import numpy as np
from analysis import blocks
from numpy import nan
from sklearn.multioutput import MultiOutputRegressor
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDRegressor
from sklearn import svm
from sklearn import preprocessing
import pickle
from math import floor

MODELS_PATH = '/Users/apple/Desktop/mag/dane/DANE_PO_MODERNIZACJI_VRM/modele/regresja/'

COL_NAMES = ['in', 'control', 'out']

def load_data(block_name):
    df = pd.read_csv(blocks.LOAD_PATH + block_name + '.csv')
    print('data loaded')
    return df

def load_block_vars():
    df = pd.read_excel(blocks.BLOCK_VARS_PATH_XLSX, sheetname=None)
    print('blocks vars loaded')
    return df

def linear_regresion(x, y):
    r = LinearRegression()
    r.fit(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    return r

def logistic_regresion(x, y):
    r = LogisticRegression()
    lab_enc = preprocessing.LabelEncoder()
    y_enc = lab_enc.fit_transform(y)
    # matrix = np.asmatrix(y_enc)
    # transpose = np.transpose(matrix)
    r.fit(np.transpose(np.matrix(x)), y_enc)
    return r

def single_output_regression(x, y):
    print('Single output regression')
    models = []
    for output in y:
        print('new model')
        r = linear_regresion(x, output)
        models.append(r)
    return models

def single_output_logistic_regression(x, y):
    print('Single output log regression')
    models = []
    for output in y:
        print('new model')
        r = logistic_regresion(x, output)
        models.append(r)
    return models

def multi_output_regression(x, y, regressor = svm.SVR()):
    print('multi output regression')
    r = MultiOutputRegressor(regressor)

    r.fit(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    return r

def evaluate(model, x, y):
    y_pred = model.predict(np.transpose(np.matrix(x)))
    r2 = model.score(np.transpose(np.matrix(x)), np.transpose(np.matrix(y)))
    mse = mean_squared_error(y, y_pred)
    return r2, mse

def save_model(model, file_name):
    pickle.dump(model, open(file_name, 'wb'))

def main():
    blocks_vars = load_block_vars()
    for block_name in blocks.names:
        df = load_data(block_name)
        vars = blocks_vars[block_name][COL_NAMES]
        vars_in = [x for x in vars['in'] if not x is nan and not '#' in x]
        vars_out = [x for x in vars['out'] if not x is nan and not '#' in x]
        input_data = [df[x].tolist() for x in vars_in]
        output_data = [df[x].tolist() for x in vars_out]
        last_train_idx = floor(len(input_data[0]) * 0.875)
        x_train = [x[:last_train_idx] for x in input_data]
        y_train = [x[:last_train_idx] for x in output_data]
        x_test = [x[last_train_idx:len(input_data[0])-1] for x in input_data]
        y_test = [x[last_train_idx:len(output_data[0])-1] for x in output_data]

        r_list = single_output_regression(x_train, y_train)
        for i, model in enumerate(r_list):
            r2, mse = evaluate(model, x_test, y_test[i])
            print('r^2: {0}\n mse: {1}'.format(r2, mse))
            save_model(model, MODELS_PATH  + block_name + '/' + 'single_linear_{0}'.format(i) + '.sav')

        r_list = single_output_logistic_regression(x_train, y_train)
        for i, model in enumerate(r_list):
            r2, mse = evaluate(model, x_test, y_test[i])
            print('r^2: {0}\n mse: {1}'.format(r2, mse))
            save_model(model, MODELS_PATH + block_name + '/' + 'single_logistic_{0}'.format(i) + '.sav')

        r = multi_output_regression(x_train, y_train)
        r2, mse = evaluate(r, x_test, y_test)
        print('r^2: {0}\n mse: {1}'.format(r2, mse))
        save_model(model, MODELS_PATH + block_name + '/' + 'multi_SVR' + '.sav')

        r = multi_output_regression(x_train, y_train, SGDRegressor())
        r2, mse = evaluate(r, x_test, y_test)
        print('r^2: {0}\n mse: {1}'.format(r2, mse))
        save_model(model, MODELS_PATH + block_name + '/' + 'multi_SGD' + '.sav')

if __name__ == '__main__':
    main()


from http.server import HTTPServer, BaseHTTPRequestHandler
import json
import os
from app.db import add_item, get_items, delete_item, ItemJSONEncoder, Item, update_item
from app.readers import get_item_price_with_retries
from typing import Union

def fetch_price_and_add_item(name: str, url: str):
    price = get_item_price_with_retries(url)
    add_item(Item(name, url, price))

class SimpleWebServer(BaseHTTPRequestHandler):

    def __send_response(self, response_code: int, response: Union[str, bytes]):
        self.send_response(200)
        self.end_headers()
        if isinstance(response, str):
            response = response.encode('utf-8')
        self.wfile.write(response)

    def do_GET(self):
        try:
            path = self.path.strip('/')
            if path == 'items':
                data_to_send = json.dumps(get_items(), cls=ItemJSONEncoder)
                self.__send_response(200, data_to_send)
                return
            file_path = os.path.join(os.path.dirname(__file__), path or 'index.html')
            with open(file_path, 'rb') as fp:
                self.__send_response(200, fp.read())
        except FileNotFoundError:
            self.__send_response(404, "File not found!")

    def do_PUT(self):
        content_length = int(self.headers['Content-Length'])
        put_data = self.rfile.read(content_length)

        data = json.loads(put_data)

        item_index = int(self.path.split('/')[-1])
        items = get_items()
        item = items[item_index]
        item.name = data['name']
        item.url = data['url']
        update_item(item_index, item)

        self.__send_response(200, "Success")

    def do_POST(self):
        content_length = int(self.headers['Content-Length'])
        post_data = self.rfile.read(content_length)

        data = json.loads(post_data)
        fetch_price_and_add_item(data['name'], data['url'])
        self.__send_response(200, "Success")

    def do_DELETE(self):
        item_index = int(self.path.split('/')[-1])
        delete_item(item_index)
        self.__send_response(200, "Success")

def run(server_class=HTTPServer, handler_class=SimpleWebServer, port=8700):
    server_address = ('', port)
    httpd = server_class(server_address, handler_class)
    print(f'Starting httpd server on port {port}')
    httpd.serve_forever()

run()


from flask import Flask, request, jsonify
import joblib

app = Flask(__name__)

#訓練済みモデルの読み込み
clf = joblib.load('clf_wine_quality.pkl')

@app.route('/', methods=['POST'])
def predict():
    x = request.json['x']
    y = clf.predict([x])[0]   #[0]はリストの"中身"を取り出している
    ret = {"y" : int(y)}
    return jsonify(ret)

if __name__ == '__main__':
    app.run(debug=True)


from unittest.mock import AsyncMock, MagicMock, PropertyMock

import pytest

from envoy.distribution import repo

def test_runner_constructor():
    runner = repo.RepoBuildingRunner()
    assert isinstance(runner, repo.ARepoBuildingRunner)

@pytest.mark.parametrize("archive", [None, "ARCHIVE"])
def test_runner_archive(patches, archive):
    runner = repo.RepoBuildingRunner()
    patched = patches(
        "pathlib",
        ("RepoBuildingRunner.args",
         dict(new_callable=PropertyMock)),
        prefix="envoy.distribution.repo.runner")

    with patched as (m_plib, m_args):
        m_args.return_value.archive = archive
        assert (
            runner.archive
            == (m_plib.Path.return_value
                if archive
                else None))
    if not archive:
        assert not m_plib.Path.called
    else:
        assert (
            m_plib.Path.call_args
            == [(m_args.return_value.archive, ), {}])
    assert "archive" not in runner.__dict__

def test_runner_packages(patches):
    runner = repo.RepoBuildingRunner()
    patched = patches(
        "pathlib",
        ("RepoBuildingRunner.args",
         dict(new_callable=PropertyMock)),
        prefix="envoy.distribution.repo.runner")
    packages = [f"PACKAGE{i}" for i in range(0, 5)]

    with patched as (m_plib, m_args):
        m_args.return_value.packages = packages
        assert (
            runner.packages
            == tuple(m_plib.Path.return_value
                     for i in range(0, 5)))

    assert (
        m_plib.Path.call_args_list
        == [[(package, ), {}] for package in packages])

    assert "packages" not in runner.__dict__

def test_runner_add_arguments(patches):
    runner = repo.RepoBuildingRunner()
    parser = MagicMock()
    patched = patches(
        "ARepoBuildingRunner.add_arguments",
        prefix="envoy.distribution.repo.runner")

    with patched as (m_super, ):
        assert not runner.add_arguments(parser)

    assert (
        m_super.call_args
        == [(parser, ), {}])
    assert (
        parser.add_argument.call_args_list
        == [[('--packages',),
             {'nargs': '*'}],
            [('--archive',),
             {'nargs': '?'}]])

@pytest.mark.parametrize("archive", [True, False])
def test_runner_create_archive(patches, archive):
    runner = repo.RepoBuildingRunner()
    patched = patches(
        "tarfile",
        ("RepoBuildingRunner.archive",
         dict(new_callable=PropertyMock)),
        ("RepoBuildingRunner.log",
         dict(new_callable=PropertyMock)),
        prefix="envoy.distribution.repo.runner")
    paths = [f"PATH{i}" for i in range(0, 5)]

    with patched as (m_tar, m_archive, m_log):
        if not archive:
            m_archive.return_value = None
        assert not runner.create_archive(*paths)

    if not archive:
        assert not m_tar.open.called
        assert (
            m_log.return_value.warning.call_args
            == [("No `--archive` argument provided, dry run only", ), {}])
        return

    assert not m_log.called
    assert (
        m_tar.open.call_args
        == [(m_archive.return_value, "w"), {}])
    tfile = m_tar.open.return_value.__enter__.return_value
    assert (
        tfile.add.call_args_list
        == [[(path, ), dict(arcname=".")]
            for path in paths])

def test_runner_extract_packages(patches):
    runner = repo.RepoBuildingRunner()
    patched = patches(
        "utils",
        ("RepoBuildingRunner.packages",
         dict(new_callable=PropertyMock)),
        ("RepoBuildingRunner.path",
         dict(new_callable=PropertyMock)),
        prefix="envoy.distribution.repo.runner")

    with patched as (m_utils, m_packages, m_path):
        m_packages.return_value = [
            f"PACKAGES{i}" for i in range(0, 5)]
        assert not runner.extract_packages()

    assert (
        m_utils.extract.call_args_list
        == [[(m_path.return_value, p), {}]
            for p in m_packages.return_value])

async def test_runner_run(patches):
    runner = repo.RepoBuildingRunner()
    patched = patches(
        "utils",
        "RepoBuildingRunner.create_archive",
        "RepoBuildingRunner.extract_packages",
        ("RepoBuildingRunner.published_repos",
         dict(new_callable=PropertyMock)),
        prefix="envoy.distribution.repo.runner")
    create_args = [f"ARG{i}" for i in range(0, 3)]

    with patched as (m_utils, m_create, m_extract, m_repos):
        m_utils.async_list = AsyncMock(return_value=create_args)
        assert not await runner.run()

    assert (
        m_extract.call_args
        == [(), {}])
    assert (
        m_create.call_args
        == [tuple(create_args), {}])
    filter = m_utils.async_list.call_args.kwargs["filter"]
    assert (
        m_utils.async_list.call_args
        == [(m_repos.return_value, ), dict(filter=filter)])
    assert filter("XYZ") == "XYZ"
    assert filter(None) is None


import os
import pathlib
import shutil
from os.path import realpath, dirname

from src.create_similarity_features.lexical_similarity import tokenize_and_filter_out_stop_words
from src.create_similarity_features.referential_similarity import get_named_entities_of_sentence, get_text_synonyms
from src.create_similarity_features.string_similarity import match_sequences, levenshtein_sim
from src.get_data import DATA_PATH
import pandas as pd

from src.utils import get_queries, get_targets

filepath = realpath(__file__)
dir_of_file = dirname(filepath)
parent_dir_of_file = dirname(dir_of_file)
parent_parent_dir_of_file = dirname(parent_dir_of_file)
parents_parent_parent_dir_of_file = dirname(parent_parent_dir_of_file)
repo_path = parents_parent_parent_dir_of_file

def group_dataset(dataset_name):
    gold_path = DATA_PATH + dataset_name + "/gold.tsv"
    query_path = DATA_PATH + dataset_name + "/queries.tsv"
    target_path = DATA_PATH + dataset_name + "/corpus"

    gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
    query_df = pd.read_csv(query_path, sep='\t', dtype=str, names=["query_id", "query_text"])
    target_df_orig = pd.read_csv(target_path, sep='\t', dtype=str, names=["target_id", "target_text_1"])#, "target_text_2"])
    #print(target_df_orig)
    target_df = pd.DataFrame(columns=['target_id', 'target_text'])
    target_df["target_id"] = target_df_orig["target_id"]
    target_df["target_text"] = target_df_orig["target_text_1"]#target_df_orig.target_text_1.str.cat(target_df_orig.target_text_2, sep=" ")
    gold_df = gold_df.drop_duplicates()
    gold_text_df = gold_df[["query_id", "target_id"]].merge(query_df, on="query_id", how="left").merge(target_df, on="target_id", how="left")
    queries = gold_text_df["query_text"].tolist()
    targets = list(map(str, gold_text_df["target_text"].tolist()))
    target_ids = gold_text_df["target_id"].tolist()

    #gold_text_df.loc[gold_text_df["target_text"]]

    query_tokens = [set(tokenize_and_filter_out_stop_words(query)) for query in queries]
    target_tokens = [set(tokenize_and_filter_out_stop_words(target)) for target in targets]
    lexical_overlaps = [len(query_tokens[i].intersection(target_tokens[i])) for i in range(len(gold_text_df))]
    gold_text_df["lexical_similarity"] = lexical_overlaps

    gold_text_df = gold_text_df.sort_values("lexical_similarity")
    scores = [-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 100]
    for idx, score in enumerate(scores[:-1]):
        group_df = gold_text_df[(gold_text_df["lexical_similarity"] <= scores[idx + 1]) & (gold_text_df["lexical_similarity"] > score)].copy()
        group_df["0"] = str(0)
        group_df["score"] = str(1)
        group_gold_df = group_df[["query_id", "0", "target_id", "score"]]
        group_query_df = group_df[["query_id", "query_text"]]
        group_query_df = group_query_df.drop_duplicates()
        lexical_output_path = DATA_PATH + "lexical/" + dataset_name + "/" + str(score)
        pathlib.Path(lexical_output_path).mkdir(parents=True, exist_ok=True)
        lexical_gold_path = lexical_output_path + "/gold.tsv"
        lexical_query_path = lexical_output_path + "/queries.tsv"
        group_gold_df.to_csv(lexical_gold_path, sep='\t', header=False, index=False)
        group_query_df.to_csv(lexical_query_path, sep='\t', header=False, index=False)

def analyse_subsets(dataset_name):
    similarity_features = ["ne"]#["levenshtein", "sequence_matching", "synonym", "lexical"]
    features_dict = {}
    for feature in similarity_features:
        output_path = DATA_PATH + feature + "/" + dataset_name
        n_queries = []
        for file in os.listdir(output_path):
            gold_path = output_path + "/" + file + "/gold.tsv"
            gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
            n_queries.append(len(gold_df))
        features_dict[feature] = n_queries
    return features_dict

def analyse_subsets_lexical(dataset_name):
    features_dict = {}
    output_path = DATA_PATH + "lexical/" + dataset_name
    for file in os.listdir(output_path):
        gold_path = output_path + "/" + file + "/gold.tsv"
        gold_df = pd.read_csv(gold_path, sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        features_dict[file] = len(gold_df)
    return features_dict

def print_subset_examples_lexical():

    dataset_names = ["clef_2020_checkthat_2_english",
                     "clef_2021_checkthat_2a_english",
                     "clef_2022_checkthat_2a_english",
                     "clef_2021_checkthat_2b_english",
                     "clef_2022_checkthat_2b_english"]

    input_claims_highest_lexical = {}
    verified_claims_highest_lexical = {}

    input_claims_lowest_lexical = {}
    verified_claims_lowest_lexical = {}

    for dataset_name in dataset_names:

        query_path = DATA_PATH + dataset_name + "/queries.tsv"
        target_path = DATA_PATH + dataset_name + "/corpus"

        lexical_output_path = DATA_PATH + "lexical/" + dataset_name
        highest_lexical_output_path = lexical_output_path + "/8"
        lowest_lexical_output_path = lexical_output_path + "/-1"

        highest_lexical_gold_df = pd.read_csv(highest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        highest_lexical_gold_sample = highest_lexical_gold_df.sample(1, random_state=2).values[0]
        highest_lex_query_id = highest_lexical_gold_sample[0]
        highest_lex_target_id = highest_lexical_gold_sample[2]
        highest_lex_query_text = get_queries(query_path)[highest_lex_query_id]
        highest_lex_target_text = get_targets(target_path)[highest_lex_target_id]
        input_claims_highest_lexical[dataset_name] = highest_lex_query_text
        verified_claims_highest_lexical[dataset_name] = highest_lex_target_text

        lowest_lexical_gold_df = pd.read_csv(lowest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str, names=["query_id", "0", "target_id", "score"])
        lowest_lexical_gold_sample = lowest_lexical_gold_df.sample(1, random_state=2).values[0]
        lowest_lex_query_id = lowest_lexical_gold_sample[0]
        lowest_lex_target_id = lowest_lexical_gold_sample[2]
        lowest_lex_query_text = get_queries(query_path)[lowest_lex_query_id]
        lowest_lex_target_text = get_targets(target_path)[lowest_lex_target_id]
        input_claims_lowest_lexical[dataset_name] = lowest_lex_query_text
        verified_claims_lowest_lexical[dataset_name] = lowest_lex_target_text

    print("\\begin{figure}[hbtp!]")
    print("\\begin{quote}")

    print("\\textbf{Input Claim ct 2020 2a:} \\textit{"+input_claims_highest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2020 2a:} \\textit{"+verified_claims_highest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2a:} \\textit{"+input_claims_highest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2a:} \\textit{"+verified_claims_highest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2a:} \\textit{"+input_claims_highest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2a:} \\textit{"+verified_claims_highest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2b:} \\textit{"+input_claims_highest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2b:} \\textit{"+verified_claims_highest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2b:} \\textit{"+input_claims_highest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2b:} \\textit{"+verified_claims_highest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\end{quote}")
    print("\caption{Example of true pairs with high lexical overlap.}")
    print("\end{figure}")

    print()

    print("\\begin{figure}[hbtp!]")
    print("\\begin{quote}")

    print("\\textbf{Input Claim ct 2020 2a:} \\textit{"+input_claims_lowest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2020 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2020_checkthat_2_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2a:} \\textit{"+input_claims_lowest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2021_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2a:} \\textit{"+input_claims_lowest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2a:} \\textit{"+verified_claims_lowest_lexical["clef_2022_checkthat_2a_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2021 2b:} \\textit{"+input_claims_lowest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2021 2b:} \\textit{"+verified_claims_lowest_lexical["clef_2021_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\\textbf{Input Claim ct 2022 2b:} \\textit{"+input_claims_lowest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\\textbf{Verified Claim ct 2022 2b:} \\textit{"+verified_claims_lowest_lexical["clef_2022_checkthat_2b_english"]+"}\\\\")
    print("\hline")

    print("\end{quote}")
    print("\caption{Example of true pairs with no lexical overlap.}")
    print("\end{figure}")

def print_lowest_lexical_subset():

    dataset_names = ["clef_2020_checkthat_2_english",
                     "clef_2021_checkthat_2a_english",
                     "clef_2022_checkthat_2a_english",
                     "clef_2021_checkthat_2b_english",
                     "clef_2022_checkthat_2b_english"]

    input_claims_lowest_lexical = {}
    verified_claims_lowest_lexical = {}

    for dataset_name in dataset_names:
        query_path = DATA_PATH + dataset_name + "/queries.tsv"
        target_path = DATA_PATH + dataset_name + "/corpus"

        lexical_output_path = DATA_PATH + "lexical/" + dataset_name
        lowest_lexical_output_path = lexical_output_path + "/-1"

        lowest_lexical_gold_df = pd.read_csv(lowest_lexical_output_path + "/gold.tsv", sep='\t', dtype=str,
                                             names=["query_id", "0", "target_id", "score"])
        lowest_lex_query_ids = lowest_lexical_gold_df["query_id"].values
        lowest_lex_target_ids = lowest_lexical_gold_df["target_id"].values
        queries = get_queries(query_path)
        targets = get_targets(target_path)
        lowest_lex_query_texts = [queries[query_id] for query_id in lowest_lex_query_ids]
        lowest_lex_target_texts = [targets[target_id] for target_id in lowest_lex_target_ids]
        input_claims_lowest_lexical[dataset_name] = lowest_lex_query_texts
        verified_claims_lowest_lexical[dataset_name] = lowest_lex_target_texts
        for idx, query in enumerate(lowest_lex_query_texts):
            print(query)
            print(lowest_lex_target_texts[idx])
            print("---------------")

#print_lowest_lexical_subset()

#print_subset_examples_lexical()

# dataset_names = ["clef_2020_checkthat_2_english",
#                  "clef_2021_checkthat_2a_english",
#                  "clef_2022_checkthat_2a_english",
#                  "clef_2021_checkthat_2b_english",
#                  "clef_2022_checkthat_2b_english"]

dataset_names = ["climate-fever", "fever", "scifact"]

# shutil.rmtree(DATA_PATH + "levenshtein")
# shutil.rmtree(DATA_PATH + "sequence_matching")
# shutil.rmtree(DATA_PATH + "synonym")
# shutil.rmtree(DATA_PATH + "lexical")
# shutil.rmtree(DATA_PATH + "ne")
for dataset_name in dataset_names:
    group_dataset(dataset_name)
#
# for dataset_name in dataset_names:
#     print(analyse_subsets(dataset_name))