import numpy as np
import matplotlib.pyplot as plt
from IPython import display
import torch
import torch.nn as nn
import torch.nn.functional as nnF
import torch.optim as optim
from RobustDNN_module import Linear
from ECG_Dataset import get_dataloader, get_dataloader_bba
from Evaluate import test_adv, test_rand
from Evaluate_bba1 import test_adv as test_adv_bba1
from Evaluate_bba1 import train_sub_model
from Evaluate_bba_spsa import test_adv as test_adv_spsa
#%% Net: fully connected feedforward Neural Network
class Net(nn.Module):
    def __init__(self, num, bias):
        super().__init__()

        self.E = nn.ModuleList([Linear(187, num),
                                Linear(num, num),
                                Linear(num, num),
                                Linear(num, 32)])

        self.C = nn.ModuleList([Linear(32, 5)])

        self.G = nn.ModuleList([nn.ReLU(inplace=True),
                                nn.ReLU(inplace=True),
                                nn.ReLU(inplace=True)])
        if bias == False:
            self.disable_bias()

    def disable_bias(self):
        for n in range(0, len(self.E)):
            self.E[n].disable_bias()
        self.C[0].disable_bias()

    def normalize_kernel(self):
        for n in range(0, len(self.E)):
            self.E[n].normalize_kernel()

    def zero_WoW(self):
        for n in range(0, len(self.E)):
            self.E[n].zero_WoW()

    def update_WoW(self):
        for n in range(0, len(self.E)):
            self.E[n].update_WoW()

    def initialize_dead_kernel(self):
        for n in range(0, len(self.E)):
            counter=self.E[n].initialize_dead_kernel()
            print('initialize_dead_kernel in E, counter=', counter)

    def forward(self, x):
        #self.feature_in=[]
        #self.feature_out=[]
        for n in range(0, len(self.E)):
            #self.feature_in.append(x)
            x=self.E[n](x)
            #self.feature_out.append(x)
            if n < len(self.E)-1:
                x=self.G[n](x)
        z=self.C[0](x)
        return z
#%%
class SubstituteNet(nn.Module):
    def __init__(self, num, bias):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(187, num, bias=bias),
                                 nn.ReLU(inplace=True),
                                 nn.BatchNorm1d(num, affine=False),
                                 nn.Linear(num, num, bias=bias),
                                 nn.ReLU(inplace=True),
                                 nn.BatchNorm1d(num, affine=False),
                                 nn.Linear(num, num, bias=bias),
                                 nn.ReLU(inplace=True),
                                 nn.BatchNorm1d(num, affine=False),
                                 nn.Linear(num, num, bias=bias),
                                 nn.ReLU(inplace=True),
                                 nn.BatchNorm1d(num, affine=False),
                                 nn.Linear(num, 5, bias=bias))
    def forward(self, x):
        z=self.net(x)
        return z
#%%
def save_checkpoint(filename, model, optimizer, result, epoch):
    torch.save({'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'result':result},
               filename)
    print('saved:', filename)
#%%
def plot_result(loss_train_list, acc_train_list,
                loss_val_list, acc_val_list,
                loss_test_list, acc_test_list,
                adv_acc_val_list, adv_acc_test_list):
    fig, ax = plt.subplots(1, 3, figsize=(9,3))
    ax[0].set_title('loss v.s. epoch')
    ax[0].plot(loss_train_list, '-b', label='training loss')
    ax[0].plot(loss_val_list, '-g', label='validation loss')
    ax[0].plot(loss_test_list, '-r', label='testing loss')
    ax[0].set_xlabel('epoch')
    #ax[0].set_xticks(np.arange(len(loss_train_list)))
    #ax[0].legend()
    ax[0].grid(True)
    ax[1].set_title('accuracy v.s. epoch')
    ax[1].plot(acc_train_list, '-b', label='training acc')
    ax[1].plot(acc_val_list, '-g', label='validation acc')
    ax[1].plot(acc_test_list, '-r', label='testing acc')
    ax[1].set_xlabel('epoch')
    #ax[1].legend()
    ax[1].grid(True)
    ax[2].set_title('accuracy v.s. epoch')
    ax[2].plot(adv_acc_val_list, '-c', label='adv val acc')
    ax[2].plot(adv_acc_test_list, '-m', label='adv test acc')
    ax[2].set_xlabel('epoch')
    #ax[2].legend()
    ax[2].grid(True)
    return fig, ax
#%%
def get_filename(num, bias, loss_name, train_arg, epoch=None):
    filename='result/ECG_MLP1_'+str(num)+str(bias)+'_'+loss_name

    if 'jacob1' in train_arg:
        filename =filename+'_Jacob1_'
    elif 'jacob2' in train_arg:
        filename =filename+'_Jacob2_'
    if epoch is not None:
        filename =filename+'_epoch'+str(epoch)
    print ("filename is ",filename)
    return filename
#%%
def main(batch_size, num, bias, loss_name, epoch_start, epoch_end, train, train_arg, evaluate_model):
    main_train(batch_size,num, bias, loss_name, epoch_start, epoch_end, train, train_arg)
    if evaluate_model == True:
        epoch_save=epoch_end-1
        device=train_arg['device']
        loader_train, loader_val, loader_test = get_dataloader(batch_size)
        del loader_train, loader_val
        print ("only bba")
        #main_evaluate_wba(train_arg, num, bias, loss_name, epoch_save, device, loader_test)
        #main_evaluate_bba(train_arg, num, bias, loss_name, epoch_save, device, loader_test)
        #main_evaluate_rand(train_arg,num, bias, loss_name, epoch_save, device, loader_test)
        main_evaluate_SAP(train_arg,num, bias, loss_name, epoch_save, device, loader_test)
        #-------------------------------------
        #loader_bba = get_dataloader_bba()
        #main_evaluate_bba_spsa(train_arg,num, bias, loss_name, epoch_save, device, loader_bba)
#%%
def main_train(batch_size,num, bias, loss_name, epoch_start, epoch_end, train, train_arg):
#%%
    filename=get_filename(num, bias, loss_name, train_arg)
    print('train model: '+filename)
    lr=train_arg['lr']
    device=train_arg['device']
#%%
    loader_train, loader_val, loader_test = get_dataloader(batch_size)
#%%
    model=Net(num, bias)
    model.to(device)
    #%%
    x=loader_train.dataset[0][0]
    x=x.view(1,187).to(device)
    z=model(x)
#%%
    optimizer = optim.Adamax(model.parameters(), lr=lr)
#%%
    loss_train_list=[]
    acc_train_list=[]
    loss_val_list=[]
    acc_val_list=[]
    loss_test_list=[]
    acc_test_list=[]
    adv_acc_val_list=[]
    adv_acc_test_list=[]
    epoch_save=epoch_start-1
#%%
    if epoch_start>0:
        if epoch_save >= 0:
            checkpoint=torch.load(filename+'_epoch'+str(epoch_save)+'.pt', map_location=device)
            model=Net(num, bias)
            model.load_state_dict(checkpoint['model_state_dict'])
            model.to(device)
            model.eval()
            (loss_train_list, acc_train_list,
             loss_val_list, acc_val_list,
             loss_test_list, acc_test_list,
             adv_acc_val_list,adv_acc_test_list)=checkpoint['result']
            optimizer = optim.Adamax(model.parameters(), lr=lr)
            optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
#%%
    for epoch in range(epoch_save+1, epoch_end):
        #-------- training --------------------------------
        loss_train, acc_train =train(model, device, optimizer, loader_train, epoch, train_arg)
        loss_train_list.append(loss_train)
        acc_train_list.append(acc_train)
        print('epoch', epoch, 'training loss:', loss_train, 'acc:', acc_train)
        #-------- validation --------------------------------
        result_val = test_adv(model, device, loader_val, 5, 0.1)
        #loss_val_list.append(result_val['loss_ce'])
        acc_val_list.append(result_val['acc_clean'])
        adv_acc_val_list.append(result_val['acc_noisy'])
        #-------- testing --------------------------------
        result_test = test_adv(model, device, loader_test, 5, 0.1)
        #loss_test_list.append(result_test['loss_ce'])
        acc_test_list.append(result_test['acc_clean'])
        adv_acc_test_list.append(result_test['acc_noisy'])
        #--------save model-------------------------
        result = (loss_train_list, acc_train_list,
                  loss_val_list, acc_val_list,
                  loss_test_list, acc_test_list,
                  adv_acc_val_list, adv_acc_test_list)
        save_checkpoint(filename+'_epoch'+str(epoch)+'.pt', model, optimizer, result, epoch)
        epoch_save=epoch
        #------- show result ----------------------
        display.clear_output(wait=False)
        plt.close('all')
        fig, ax = plot_result(loss_train_list, acc_train_list,
                              loss_val_list, acc_val_list,
                              loss_test_list, acc_test_list,
                              adv_acc_val_list, adv_acc_test_list)
        display.display(fig)
        fig.savefig(filename+'_epoch'+str(epoch)+'.png')
#%%
def main_evaluate_wba(train_arg, num, bias, loss_name, epoch, device, loader):
#%%
    filename=get_filename(num, bias, loss_name, train_arg, epoch)
    checkpoint=torch.load(filename+'.pt', map_location=device)
    model=Net(num, bias)
    model.load_state_dict(checkpoint['model_state_dict'])
    model.to(device)
    model.eval()
    print('evaluate_wba model in '+filename+'.pt')

#%% ifgsm and stop_if_success
    """
    result_ifgsm=[]
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3]:
        result_ifgsm.append(test_adv(model, device, loader, 5, noise_norm))
    fig, ax = plt.subplots()
    noise=[0]
    acc=[result_ifgsm[0]['acc_clean']]
    for k in range(0, len(result_ifgsm)):
        noise.append(result_ifgsm[k]['noise_norm'])
        acc.append(result_ifgsm[k]['acc_noisy'])
        ax.plot(noise, acc, '.-b')
        ax.set_ylim(0, 1)
        ax.set_yticks(np.arange(0, 1.05, step=0.05))
        ax.grid(True)
        ax.set_title('wba_ifgsm')
        ax.set_xlabel(filename)
        display.display(fig)
        fig.savefig(filename+'_wba_ifgsm.png')
    """
#%% 10pgd and stop_if_success
    """
    result_10pgd=[]
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.2, 0.3]:
        result_10pgd.append(test_adv(model, device, loader, 5, noise_norm, 10, 0.01, method='pgd'))
    fig, ax = plt.subplots()
    noise=[0]
    acc=[result_10pgd[0]['acc_clean']]
    for k in range(0, len(result_10pgd)):
        noise.append(result_10pgd[k]['noise_norm'])
        acc.append(result_10pgd[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('wba_10pgd')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_wba_10pgd.png')
#%% 20pgd and stop_if_success
    result_20pgd=[]
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3]:
        result_20pgd.append(test_adv(model, device, loader, 5, noise_norm, 20, 0.01, method='pgd'))
    fig, ax = plt.subplots()
    noise=[0]
    acc=[result_20pgd[0]['acc_clean']]
    for k in range(0, len(result_20pgd)):
        noise.append(result_20pgd[k]['noise_norm'])
        acc.append(result_20pgd[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('wba_20pgd')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_wba_20pgd.png')
    """
#%% 100pgd and stop_if_success
    result_100pgd=[]
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.2, 0.3]:
        result_100pgd.append(test_adv(model, device, loader, 5, noise_norm, 100, 0.01, method='pgd'))
    fig, ax = plt.subplots()
    noise=[0]
    acc=[result_100pgd[0]['acc_clean']]
    for k in range(0, len(result_100pgd)):
        noise.append(result_100pgd[k]['noise_norm'])
        acc.append(result_100pgd[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('wba_100pgd')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_wba_100pgd.png')
#%%
    torch.save({'result_100pgd':result_100pgd},
               filename+'_result_wba.pt')
#%%
def main_evaluate_SAP(train_arg, num, bias, loss_name, epoch, device, loader):
    #https://www.nature.com/articles/s41591-020-0791-x
    filename=get_filename(num, bias, loss_name, train_arg, epoch)
    checkpoint=torch.load(filename+'.pt', map_location=device)
    model=Net(num, bias)
    model.load_state_dict(checkpoint['model_state_dict'])
    model.to(device)
    model.eval()
    print('evaluate_wba model in '+filename+'.pt')

    result_100pgd=[]
    max_iter = 100
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.2, 0.3]:
        result_100pgd.append(test_adv(model, device, loader, 5, noise_norm, max_iter, 0.01, method='SAP'))
    fig, ax = plt.subplots()
    noise=[0]
    acc=[result_100pgd[0]['acc_clean']]
    for k in range(0, len(result_100pgd)):
        noise.append(result_100pgd[k]['noise_norm'])
        acc.append(result_100pgd[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('wba_'+str(max_iter)+'sap')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_wba_'+str(max_iter)+'sap.png')

    torch.save({'result_'+str(max_iter)+'sap':result_100pgd},
               filename+'_result_'+str(max_iter)+'sap.pt')
#%%
def main_evaluate_bba_spsa(train_arg,num, bias, loss_name, epoch, device, loader):

    filename=get_filename(num, bias, loss_name, train_arg, epoch)
    checkpoint=torch.load(filename+'.pt', map_location=device)
    model=Net(num, bias)
    model.load_state_dict(checkpoint['model_state_dict'])
    model.to(device)
    model.eval()
    print('evaluate_bba_spsa model in '+filename+'.pt')
#%%
    result_spsa=[]
    for noise_norm in [0.01, 0.03, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3]:
        result_spsa.append(test_adv_spsa(model, device, loader, 5, noise_norm, method='spsa_attack',spsa_samples=2048))
        fig, ax = plt.subplots()
    noise=[0]
    acc=[result_spsa[0]['acc_clean']]
    for k in range(0, len(result_spsa)):
        noise.append(result_spsa[k]['noise_norm'])
        acc.append(result_spsa[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('bba_spsa')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_bba_spsa.png')
    #------------------------------------
    torch.save({'result_spsa':result_spsa},filename+'_result_bba_spsa.pt')
def main_evaluate_rand(train_arg,num, bias, loss_name, epoch, device, loader):
#%%
    filename=get_filename(num, bias, loss_name, train_arg, epoch)
    checkpoint=torch.load(filename+'.pt', map_location=device)
    model=Net(num, bias)
    model.load_state_dict(checkpoint['model_state_dict'])
    model.to(device)
    model.eval()
    print('evaluate_rand model in '+filename+'.pt')
#%%
    result_rand=[]
    for noise_norm in [0, 0.01, 0.03, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3]:
        result_rand.append(test_rand(model, device, loader, 5, noise_norm))
        fig, ax = plt.subplots()
    noise=[]
    acc=[]
    for k in range(0, len(result_rand)):
        noise.append(result_rand[k]['noise_norm'])
        acc.append(result_rand[k]['acc_noisy'])
    ax.plot(noise, acc, '.-b')
    ax.set_ylim(0, 1)
    ax.set_yticks(np.arange(0, 1.05, step=0.05))
    ax.grid(True)
    ax.set_title('rand')
    ax.set_xlabel(filename)
    display.display(fig)
    fig.savefig(filename+'_rand.png')
    #------------------------------------
    torch.save({'result_rand':result_rand},filename+'_result_rand.pt')


#!/usr/bin/env python

import curses

def simple_curses_menu(prompt, choices):
  def impl(stdscr, prompt, choices):
    curses.use_default_colors()
    selected = 0
    ch = curses.KEY_RESIZE
    while ch != ord('\n'):
      if ch == curses.KEY_DOWN:
        selected = min(selected + 1, len(choices) - 1)
      elif ch == curses.KEY_UP:
        selected = max(0, selected - 1)
      (height, width) = stdscr.getmaxyx()
      stdscr.addnstr(0, 0, prompt, width, curses.A_BOLD)
      for (i, choice) in enumerate(choices):
        if i + 1 < height:
          stdscr.addnstr(1 + i, 0, choice, width)
          stdscr.clrtoeol()
      number_string = "({0}/{1})".format(selected + 1, len(choices))
      stdscr.addnstr(min(height-1, selected+1), 0, choices[selected], width - len(number_string), curses.A_REVERSE)
      stdscr.addstr(min(height-1, selected+1), width - len(number_string), number_string)
      ch = stdscr.getch()
    stdscr.clear()
    return selected
  return curses.wrapper(impl, prompt, choices)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, sys, re
import click
from html import parser
from urllib import request, response

def get_reviry_list(url, separator="
", match="""a href="/index.php/14-mimopstruhove-reviry/"""):
    output = []
    urlparser = request.urlparse(url)
    with request.urlopen(url) as basepage:
        htmlpage = basepage.read().decode()
        for item in htmlpage.split(separator):
            if match in item:
                rematch = re.search('href="([^"]+)"', item)
                if rematch:
                    output.append(f"{urlparser.scheme}://{urlparser.netloc}{rematch.groups()[0]}")
    return output

def parser_revir_page(url):
    with request.urlopen(url) as page:
        readed = page.read().decode()
        sep = """
"""
        sep2 = """
"""
        tmp = readed.split(sep)[1].split(sep2)[0]
        tmp = re.sub(r"
", "\n", tmp)
        tmp = re.sub(r"", " ", tmp)
        tmp = re.sub(r"<.?br[^>]+>", "\n", tmp)
        tmp = re.sub(r"<[^>]+>", " ", tmp)
        tmp = re.sub(r"&[^;]+;", "", tmp)
        return tmp

def strip_list(inp):
    output = []
    for item in inp:
        if item is not None:
            output.append(item.strip())

def revir_data(text):
    jmeno = ""
    jmeno_add = ""
    GPS = {}
    for line in text.split("\n"):
        #print(">>:", line, file=sys.stderr)
        stripped_line = line.strip()
        if not jmeno:
            jmeno = stripped_line
            jmeno_add = jmeno
            print("REVIR:", jmeno, file=sys.stderr)
            continue

        # podrevir
        search = re.search('^\d+\..*ha', stripped_line)
        if search:
            jmeno_add = stripped_line
            print("NAME:", jmeno_add, file=sys.stderr)
            continue

        # reka
        search = re.search('^GPS.*Z:(.+)K:(.+)', stripped_line)
        if search:
            zacatek = search.groups()[0].split("N")
            konec = search.groups()[1].split("N")
            GPS[jmeno_add] = (convert(zacatek[0]), convert(zacatek[1]), convert(konec[0]), convert(konec[1]))
            print("adding reka", zacatek, konec, file=sys.stderr)
            continue

        # rybnik
        search = re.search('^GPS(.*)', stripped_line)
        if search:
            zacatek = search.group(0).split("N")
            if len(zacatek) < 2:
                raise Exception(f"BAD GPS {zacatek}")
            GPS[jmeno_add] = (convert(zacatek[0]), convert(zacatek[1]))
            print("adding rybnik", zacatek)
            continue

        # misto ci zakaz
        search_all = re.findall(r'GPS\S*\s+([^N]+N\S*\s+[^E]+E)', line)
        for search in search_all:
            #print(">>>>>>>>>>>", search)
            zacatek = search.split("N")

            GPS[f"ZAKAZ ci RYBNIK: {search}"] = (convert(zacatek[0]), convert(zacatek[1]))
            print("ZAKAZ ci RYBNIK", zacatek, file=sys.stderr)
            continue
    return jmeno, GPS

def convert(coor):
    output = 0
    found = re.search(r"(\d+)\D+(\d+)\D+([0-9.]+)", coor)
    if found:
        output = float(found.group(1)) + (float(found.group(2)) / 60.0) + (float(found.group(3))) / 3600.0
    else:
        found = re.search(r"(\d+\.\d+)", coor)
        if found:
            output = float(found.group(1))
        else:
            print(sys.stderr, "ERROR convert coordinates:", coor, file=sys.stderr)
    return output

def output_to_file(filename, reviry):
    with open(filename, mode="w") as outfile:
        intro="""<?xml version="1.0" encoding="UTF-8"?>
                 <kml xmlns="http://www.opengis.net/kml/2.2">
                <Document>
    """
        modnum=1
        colors=["ff0000", "ffff00", "ff00ff", "990000", "99ff00", "9900ff", "ff9900", "ff0099", "550000", "55ff00", "5500ff"]
        for foo in range(0, modnum):
            intro +="""

                    """.format(foo, colors[foo])

        outfile.write(intro)

        count=0
        for revir_name, revir_data in reviry.items():
            count += 1
            for k, v in revir_data['GPS'].items():
                outfile.write('<Placemark>\n')
                outfile.write(f'<name>{revir_name}</name>')
                description = revir_data["data"]
                outfile.write(f'<description>\nJmeno: {k}\n\n{description}</description>')
                if "ZAKAZ" in k:
                    outfile.write('\n'.format(count % modnum))
                    outfile.write('<Point> <coordinates>{0},{1}\n</coordinates> </Point>\n'.format(v[1], v[0]))
                elif len(v) == 2:
                    outfile.write('\n'.format(count%modnum))
                    outfile.write('<Point> <coordinates>{0},{1}\n</coordinates> </Point>\n'.format(v[1], v[0]))
                elif len(v) == 4:
                    outfile.write('\n'.format(count%modnum))
                    outfile.write('<LineString><tessellate>1</tessellate><coordinates>{0},{1}\n{2},{3}\n</coordinates> </LineString>\n'.format(v[1], v[0], v[3], v[2]))
                outfile.write('</Placemark>')
        outfile.write('</Document>\n')
        outfile.write('</kml>')

@click.command()
@click.option("--url", "-u", default="http://mrsbrno.cz/index.php/33-seznam-reviru/142-seznam-rybarskych-reviru-mimopstruhovych")
@click.option("--outputfile", "-o", default="outfile.kml")
@click.option("--quiet", "-q", is_flag=True, default=False)
def parser(url, outputfile, quiet):
    reviry = {}
    for item in get_reviry_list(url):
        text = parser_revir_page(item)
        jmeno, GPS = revir_data(text)
        reviry[jmeno] = {'GPS': GPS, 'data': text}
    output_to_file(outputfile, reviry)

if __name__ == '__main__':
    parser()


#!/usr/bin/env python
import cv2
import numpy as np
import os
import glob
if __name__ == '__main__':
    filepath = 'video.flv'

    npzfile = np.load('calibr_parameters.npz')
    mtxstr, diststr = npzfile.files
    mtx = npzfile[mtxstr]
    dist = npzfile[diststr]

    cap = cv2.VideoCapture(filepath)
    if not cap.isOpened():
        print("VideoCapture failed...")
    else:
        print("Video opened! ...")
    fps = cap.get(cv2.CAP_PROP_FPS)
    print("FPS:", fps)

    ret, frame = cap.read()         # Read first frame to get info about the resolution
    h,w = frame.shape[:2]
    newcameramtx, roi = cv2.getOptimalNewCameraMatrix(mtx,dist,(w,h),0,(w,h))
    x,y,w,h = roi
    out = cv2.VideoWriter('undistorted.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 15, (w, h))

    dst = cv2.undistort(frame, mtx, dist, None, newcameramtx)
    dst = dst[y:y+h, x:x+w]
    out.write(dst)

    while cap.isOpened():
        ret, frame = cap.read()
        if np.shape(frame) == ():
            continue

        dst = cv2.undistort(frame, mtx, dist, None, newcameramtx)
        dst = dst[y:y+h, x:x+w]
        out.write(dst)
        #cv2.imshow("video", dst)
        #k = cv2.waitKey(30) & 0xff                   # if 'Esc' is pressed then quit
        #if k == 27:
        #    break

    cv2.destroyAllWindows()
    out.release()
    cap.release()


import frida
import base64
import time
import os
import sys
import time
import signal

output_folder = sys.argv[1]
if not os.path.exists(output_folder):
    os.mkdir(output_folder)

pid = (sys.argv[2])

mypid = os.getpid()

class QEntry(object):
    def __init__(self):
        self.filename = ""
        self.size = 0
        self.num = 0
        self.was_fuzzed = False
        self.exec_us = 0
        self.time = 0
        self.new_cov = False
        self.next = None

class Queue(object):
    def __init__(self):
        self.size = 0
        self.start = None
        self.cur = None
        self.top = None

    def add(self, buf, exec_us, new_cov, stage):
        q = QEntry()
        q.filename = os.path.join(output_folder, "id_%d_%s" % (self.size, stage))
        if new_cov:
            q.filename += "_cov"
        q.num = self.size
        q.exec_us = exec_us
        q.new_cov = new_cov
        q.time = int(time.time())
        q.size = len(buf)
        with open(q.filename, "wb") as f:
            f.write(buf)
        self.size += 1
        if self.top:
            self.top.next = q
            self.top = q
        else:
            self.start = q
            self.top = q

    def get(self):
        if self.cur is None:
            self.cur = self.start
        elif self.cur.next is None:
            self.cur = self.start
        else:
            q = self.cur.next
            self.cur = q
        return self.cur

queue = Queue()
start_time = 0
last_path = 0

def readable_time(t):
    h = t // 60 // 60
    m = t // 60 - h * 60
    s = t - m * 60
    return "%dh-%dm-%ds" % (h, m, s)

def status_screen(status):
    global queue, pid
    t = time.time()
    TERM_HOME = "\x1b[H"
    TERM_CLEAR = TERM_HOME + "\x1b[2J"
    print (TERM_CLEAR)
    print (" |=------------=[ frida-fuzz ]=------------|")
    print ("   target app       :", pid)
    print ("   output folder    :", output_folder)
    print ("   uptime           :", readable_time(t - start_time))
    print ("   last path        :", readable_time(last_path - start_time))
    print ("   queue size       :", queue.size)
    print ("   last stage       :", status["stage"])
    print ("   current testcase :", "<init>" if queue.cur is None else os.path.basename(queue.cur.filename))
    print ("   total executions :", status["total_execs"])
    print ("   execution speed  : %d/sec" % (status["total_execs"] / (t - start_time)))
    print (" |=----------------------------------------|\n")

with open("frida-fuzz-agent.js") as f:
    code = f.read()

device = frida.get_usb_device()
session = device.attach(pid)
#session = frida.attach(pid)
session.enable_jit()

script = session.create_script(code)

def on_interesting(message, data):
    global queue, last_path
    exec_us = message["exec_us"]
    new_cov = message["new_cov"]
    stage = message["stage"]
    last_path = time.time()
    queue.add(data, exec_us, new_cov, stage)

def on_next(message, data):
    global queue
    q = queue.get()
    with open(q.filename, "rb") as f:
        buf = f.read()
    script.post({
      "type": "input",
      "buf": buf.hex(),
      "was_fuzzed": q.was_fuzzed,
    })

def on_crash(message, data):
    global queue, script, session
    print ("\n"*2 + " ************** CRASH FOUND! **************")
    print ("    type:", message["err"]["type"])
    if "memory" in message["err"]:
        print ("    %s at:" % message["err"]["memory"]["operation"], message["err"]["memory"]["address"])
    print ("")
    t = int(time.time())
    name = os.path.join(output_folder, "crash_%s_%s_%d" % (message["stage"], message["err"]["type"], t))
    #name = os.path.join(output_folder, "crash_%d" % t)
    print (" >> Saving at %s" % name)
    with open(name, "wb") as f:
        f.write(data)
    print (" >> Press Control-C to exit...")
    script.unload()
    session.detach()

def on_stats(message, data):
    status_screen(message)

def on_message(message, data):
    if message["type"] == "error":
        print (" ************** FUZZER ERROR! **************")
        print ("  line %d: %s" % (message["lineNumber"], message["description"]))
        print ("  JS stacktrace:\n")
        print (message["stack"])
        print ("")
        print (" >> Press Control-C to exit...")
        script.unload()
        session.detach()
    msg = message["payload"]
    if msg["event"] == "interesting":
        on_interesting(msg, data)
        on_stats(msg, data)
    elif msg["event"] == "next":
        on_next(msg, data)
        on_stats(msg, data)
    elif msg["event"] == "crash":
        on_crash(msg, data)
    elif msg["event"] == "stats":
        on_stats(msg, data)

script.on('message', on_message)

script.load()

def signal_handler(sig, frame):
    print (" >> Exiting...")
    try:
        script.unload()
        session.detach()
    except: pass
    os._exit (0)
signal.signal(signal.SIGINT, signal_handler)

start_time = int(time.time())

last_path = start_time
queue.add(b"0000", 0, True, "init")

try:
    script.exports.loop()
except frida.InvalidOperationError as e:
    print (e)
    exit (1)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

"""
Основная логика
"""

import sys

import logging
import logging.config

from time import sleep

import telebot
from telebot.types import Update

import cherrypy
from cherrypy import request

from database import get_all_poems
from database import get_coming_notifications
from database import add_manual_repetition
from database import delete_all_manual_notifications
from database import insert_author
from database import insert_poem
from database import InsertAuthorException, InsertPoemException, InsertReminder

from config import SETTINGS
import constants

logging.config.fileConfig('log_config.ini')
logger = logging.getLogger('myLogger')

TG_TOKEN = SETTINGS['telegram_token']
telebot.logger.setLevel(logging.INFO)
bot = telebot.TeleBot(TG_TOKEN)

@bot.message_handler(commands=['start'])
def start_msg(message):
    bot.send_message(
        message.chat.id,
        constants.MSG_START_COMMAND
    )

    logger.info('Обработана команда /start')

@bot.message_handler(commands=['list'])
def list_all_poems(message):
    """Все стихотворения"""

    poems = [
        '{}. {}'.format(idx + 1, poem.author_poem_name_str())
        for idx, poem in enumerate(get_all_poems())
    ]
    bot.send_message(
        message.chat.id,
        '\n'.join(poems)
    )

    logger.info('Обработана команда /list')

@bot.message_handler(commands=['detlist'])
def list_all_poems_detailed(message):
    """Все стихотворения с детальной информацией"""

    poems = [poem.author_poem_year_link_date_str() for poem in get_all_poems()]
    bot.send_message(
        message.chat.id,
        '\n\n'.join(poems),
        disable_web_page_preview=True
    )

    logger.info('Обработана команда /detlist')

@bot.message_handler(commands=['nextmonth'])
def list_next_30_days_notifications(message):
    poems = [str(poem) for poem in get_coming_notifications(days=30)]

    if poems:
        bot.send_message(message.chat.id, '\n\n'.join(poems))
    else:
        bot.send_message(message.chat.id, 'Уведомлений на ближайшие 30 дней нет')

    logger.info('Обработана команда /nextmonth')

@bot.message_handler(commands=['addauthor'])
def add_author(message):
    """Добавление автора в БД"""

    msg_text = message.text[len('/addauthor'):]
    try:
        author, date_of_birthday, date_of_death = msg_text.split(';')
        insert_author(author.strip(), date_of_birthday.strip(), date_of_death.strip())
        bot.send_message(message.chat.id, constants.MSG_SUCCESS_AUTHOR_INSERT)
        logger.info('Успешно выполнена команда /addauthor')
    except ValueError:
        bot.send_message(
            message.chat.id,
            constants.ERROR_WRONG_ADD_AUTHOR_FORMAT
        )

        logger.info(
            'При выполнении команды /addauthor был несоблюден формат '
            'ввода данных. Текст сообщения: {}'.format(constants.ERROR_WRONG_ADD_AUTHOR_FORMAT, message.text)
        )
    except InsertAuthorException as e:
        bot.send_message(message.chat.id, e)

        logger.error('При выполнении команды /addauthor возникла ошибка: {}'.format(e))

@bot.message_handler(commands=['addpoem'])
def add_poem(message):
    """Добавление стихотворения в БД"""

    msg_text = message.text[len('/addpoem'):]
    try:
        surname, caption, year, link = msg_text.split(';')
        insert_poem(surname.strip(), caption.strip(), year.strip(), link.strip())
        bot.send_message(message.chat.id, constants.MSG_SUCCESS_POEM_INSERT)
        logger.info('Успешно выполнена команда /addpoem')
    except ValueError:
        bot.send_message(
            message.chat.id,
            constants.ERROR_WRONG_ADD_POEM_FORMAT
        )

        logger.info(
            'При выполнении команды /addpoem был несоблюден формат '
            'ввода данных. Текст сообщения: {}'.format(constants.ERROR_WRONG_ADD_POEM_FORMAT, message.text)
        )
    except InsertPoemException as e:
        bot.send_message(message.chat.id, e)
        logger.error('При выполнении команды /addauthor возникла ошибка: {}'.format(e))

@bot.message_handler(commands=['del_extra_reps'])
def delete_manual_repetitions(message):
    """Удаление всех дополнительных повторений"""
    delete_all_manual_notifications()
    bot.send_message(
        message.chat.id,
        "Все дополнительные повторения удалены"
    )

@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    """Обработка inline-кнопок"""

    poem_id = int(call.data.split('_')[1])

    if call.data.startswith('done'):
        bot.send_message(call.message.chat.id, 'Спасибо, что повторили стихотворение #{}'.format(poem_id))
        logger.info('Стихотворение ID:{} повторено!'.format(poem_id))
        # TODO: хранить статистику в БД
    elif call.data.startswith('rep'):
        try:
            add_manual_repetition(poem_id)
            bot.send_message(
                call.message.chat.id,
                'Стихотворение #{} сохранено для дополнительного повторения'.format(poem_id)
            )
            logger.info('Стихотворение ID:{} сохранено для дополнительных повторений!'.format(poem_id))
        except InsertReminder as ex1:
            bot.send_message(call.message.chat.id, ex1)
            logger.info('Стихотворение ID:{}. {}'.format(poem_id, ex1))
        except Exception as ex2:
            bot.send_message(call.message.chat.id, "Что-то пошло не так")
            logger.error('Стихотворение ID:{}. {}'.format(poem_id, ex2))

def notify_admin(message):
    """Уведомления админа"""

    bot.send_message(SETTINGS['telegram_admin_id'], message)

class WebhookServer:
    """Класс для CherryPy приложения"""

    @cherrypy.expose
    def index(self):
        if request.headers.get('content-type', None) == 'application/json':
            length = int(request.headers['content-length'])
            json_string = request.body.read(length).decode("utf-8")
            update = Update.de_json(json_string)
            bot.process_new_updates([update])
            return ''
        else:
            raise cherrypy.HTTPError(403)

def start_webhook_server(WEBHOOK_URL_PATH, attempt=0):
    """Запуск и перезапуск сервера при необходимости"""

    try:
        if attempt <= 5:
            # Запуск сервера
            cherrypy.quickstart(WebhookServer(), WEBHOOK_URL_PATH, {'/': {}})

            # Сообщение о том, что бот запущен
            msg = 'запущен' if not attempt else 'перезапущен в {} раз'.format(attempt)
            notify_admin("Бот {} (webhook)".format(msg))

            logger.info('Бот запущен. Режим: webhook. Попытка {}'.format(attempt))
        else:
            msg = '❌ Бот прекратил работу'
            notify_admin(msg)
            logger.info(msg)
    except:
        sleep(2)
        start_webhook_server(WEBHOOK_URL_PATH, attempt=attempt + 1)
    finally:
        final_msg = 'Файл bot.py прекратил исполнение'
        logger.error(final_msg)
        notify_admin("❌ Бот выключен")

def start_bot():
    """Запуск бота"""

    # Запуск на тесте бота через polling
    # Запуск на проде бота через polling, если вебхук отключен
    if int(SETTINGS['test_mode']) or not int(SETTINGS['webhook']):
        # Работа на пуллинге локально
        bot.remove_webhook()

        # Сообщение о том, что бот запущен
        notify_admin('Бот запущен')
        logger.info('Бот запущен. Режим: polling')

        while True:
            try:
                bot.polling(none_stop=True, interval=0, timeout=0)
            except Exception as ex:
                logger.error(ex)
                sleep(10)
            finally:
                final_msg = 'Файл bot.py прекратил исполнение'
                logger.error(final_msg)
                notify_admin("❌ Бот выключен")
                sys.exit(0)
    else:
        WEBHOOK_HOST = SETTINGS['webhook_host']
        WEBHOOK_PORT = int(SETTINGS['webhook_port'])
        WEBHOOK_LISTEN = SETTINGS['webhook_listen']

        WEBHOOK_SSL_CERT = './webhook_cert.pem'
        WEBHOOK_SSL_PRIV = './webhook_pkey.pem'

        WEBHOOK_URL_BASE = 'https://{}:{}'.format(WEBHOOK_HOST, WEBHOOK_PORT)
        WEBHOOK_URL_PATH = '/{}/'.format(TG_TOKEN)

        # Удаление и создание вебхука
        bot.remove_webhook()
        bot.set_webhook(
            url=WEBHOOK_URL_BASE + WEBHOOK_URL_PATH,
            certificate=open(WEBHOOK_SSL_CERT, 'r')
        )

        logger.info('Вебхук успешно установлен: {}'.format(WEBHOOK_URL_BASE + WEBHOOK_URL_PATH))

        # Настройки сервера CherryPy
        cherrypy.config.update({
            'server.socket_host': WEBHOOK_LISTEN,
            'server.socket_port': WEBHOOK_PORT,
            'server.ssl_module': 'builtin',
            'server.ssl_certificate': WEBHOOK_SSL_CERT,
            'server.ssl_private_key': WEBHOOK_SSL_PRIV
        })

        logger.info('Конфигурация CherryPy обновлена')

        start_webhook_server(WEBHOOK_URL_BASE + WEBHOOK_URL_PATH)


from tkinter import *
from tkinter import filedialog
from tkinter.messagebox import showinfo
from tkinter.ttk import *
from openpyxl import *
from openpyxl.utils import get_column_letter
from os import walk
import os
import xlrd

class GUI:
    def __init__(self):
        self.window = Tk()

        self.window.title("Μετατροπή αρχείων xls σε xlsx")
        self.window.resizable(False, False)
        self.create_widgets()

    def getFilesDirName(self):
        dName = filedialog.askdirectory(initialdir="./data/", title="Επιλέξτε τον φάκελο με τα αρχεία προς μετατροπή")

        if dName == "":
            return

        self.filesDirName.set(dName)
        self.btnRun.configure(state='normal')

    def setColsWidth(self, ws):
        column_widths = []
        for row in ws.iter_rows():
            for i, cell in enumerate(row):
                try:
                    column_widths[i] = max(column_widths[i], len(str(cell.value)))
                except IndexError:
                    column_widths.append(len(str(cell.value)))

        for i, column_width in enumerate(column_widths):
            ws.column_dimensions[get_column_letter(i + 1)].width = column_width * 1.23

    def convertXls(self):
        inputDirectory = self.filesDirName.get()

        for path, dirs, files in walk(inputDirectory):
            for file in files:
                if file[-4:] == ".xls":
                    book_xls = xlrd.open_workbook(os.path.join(path, file))
                    book_xlsx = Workbook()

                    sheet_names = book_xls.sheet_names()
                    for sheet_index, sheet_name in enumerate(sheet_names):
                        sheet_xls = book_xls.sheet_by_name(sheet_name)
                        if sheet_index == 0:
                            sheet_xlsx = book_xlsx.active
                            sheet_xlsx.title = sheet_name
                        else:
                            sheet_xlsx = book_xlsx.create_sheet(title=sheet_name)

                        for row in range(0, sheet_xls.nrows):
                            for col in range(0, sheet_xls.ncols):
                                sheet_xlsx.cell(row=row + 1, column=col + 1).value = sheet_xls.cell_value(row, col)

                        self.setColsWidth(sheet_xlsx)

                    book_xlsx.save(os.path.join(path, file + 'x'))

    def run(self):
        self.btnRun.configure(state='disabled')
        self.convertXls()
        showinfo(title="Ολοκλήρωση εκτέλεσης", message="Η μετροπή των αρχείων ολοκληρώθηκε.")

    def create_widgets(self):
        self.fData = Frame(self.window)

        self.lData = Label(self.fData, text="Φάκελος με αρχεία\nπρος μετατροπή:")
        self.lData.grid(column=0, row=0, padx=10, pady=10, sticky=E)

        self.filesDirName = StringVar()
        self.ntrFilesDirName = Entry(self.fData, width=100, state='readonly', textvariable=self.filesDirName)
        self.ntrFilesDirName.grid(column=1, row=0, padx=10, pady=10, sticky=W)

        self.btnOpenData = Button(self.fData, text="Επιλέξτε φάκελο...", command=self.getFilesDirName)
        self.btnOpenData.grid(column=2, row=0, padx=10, pady=10)

        self.btnRun = Button(self.fData, text="Εκτέλεση", command=self.run, state='disabled')
        self.btnRun.grid(column=1, row=10, padx=10, pady=10)

        self.fData.pack()

gui = GUI()
gui.window.mainloop()


import asyncio
import itertools as it
import os
import random
import time

async def makeitem(size: int = 5) -> str:
    return os.urandom(size).hex()

async def randsleep(caller=None) -> None:
    """Sleep for a random amount of time between 0 and 2 seconds.

    Example::

        asyncio.run(randsleep(caller="Producer 1"))
    """
    i = random.randint(0, 2)
    if caller:
        print(f"{caller} sleeping for {i} seconds.")
    await asyncio.sleep(i)

async def produce(name: int, q: asyncio.Queue) -> None:
    """Produce random data and put it into the queue.

    Example::

        async def main():
            q = asyncio.Queue()
            # have 2 producers
            await asyncio.gather(produce(1, q), produce(2, q))
            #print out q current size
            print(f"q size: {q.qsize()}")

        asyncio.run(main())
    """
    n = random.randint(0, 10)
    print(f"Producer {name} will produce {n} items.")
    for _ in it.repeat(None, n):  # Synchronous loop for each single producer
        await randsleep(caller=f"Producer {name}")
        i = await makeitem()
        t = time.perf_counter()
        await q.put((i, t))
        print(f"Producer {name} added <{i}> to queue.")

async def consume(name: int, q: asyncio.Queue) -> None:
    """Produce random data and put it into the queue.

    Example::

        async def main():
            q = asyncio.Queue()
            # put 2 items into the queue
            await q.put((1, time.perf_counter()))
            await q.put((2, time.perf_counter()))
            # 2 consumer as task
            consumers = [asyncio.create_task(consume(1, q)),
                            asyncio.create_task(consume(2, q))]
            # wait for the queue to be empty
            await q.join()
            # cancel the consumers
            for c in consumers:
                c.cancel()
            #print out q current size
            print(f"q size: {q.qsize()}")

        asyncio.run(main())
    """
    while True:
        await randsleep(caller=f"Consumer {name}")
        i, t = await q.get()
        now = time.perf_counter()
        print(f"Consumer {name} got element <{i}>" f" in {now-t:0.5f} seconds.")
        print(f"q size: {q.qsize()}")
        q.task_done()

async def main(nprod: int, ncon: int):
    q = asyncio.Queue()
    producers = [asyncio.create_task(produce(n, q)) for n in range(nprod)]
    consumers = [asyncio.create_task(consume(n, q)) for n in range(ncon)]
    await asyncio.gather(*producers)
    await q.join()  # Implicitly awaits consumers, too
    for c in consumers:
        c.cancel()

if __name__ == "__main__":
    import argparse

    random.seed(444)
    parser = argparse.ArgumentParser()
    parser.add_argument("-p", "--nprod", type=int, default=3)
    parser.add_argument("-c", "--ncon", type=int, default=4)
    ns = parser.parse_args()
    start = time.perf_counter()
    asyncio.run(main(**ns.__dict__))
    elapsed = time.perf_counter() - start
    print(f"Program completed in {elapsed:0.5f} seconds.")


#!/usr/bin/python3
#
# sheetmetal.py - Generator for images resembling a sheet metal part
#
# Frank Blankenburg, Mar. 2017
#

import argparse
import os.path
import random
import numpy as np

import common.utils as utils
import generator.background
import generator.generator
import generator.fixture
import generator.tools

from common.geometry import Point2d, Size2d, Rect2d, Ellipse2d, Polygon2d

#----------------------------------------------------------------------------------------------------------------------
# CLASS SheetMetalGenerator
#
# This class will generate an image containing a simulated sheet metal like part inclucing some features like drilled
# or punched holes together with a mask marking the location of the pixels belonging to the sheet metal part.
#
class SheetMetalGenerator (generator.generator.Generator):

    TYPE = 'sheetmetal'

    #--------------------------------------------------------------------------
    # Configuration
    #

    #
    # Spacing used for distance to border, distance of the generated features, ...
    #
    spacing = Size2d (32, 32)

    #--------------------------------------------------------------------------
    # Constructor
    #
    def __init__ (self, args):
        super ().__init__ (args)
        self.size = Size2d (args.width, args.height)

        self.texture_creator = generator.tools.MetalTextureCreator (width=self.width, height=self.height, color=0.5, shine=0.1)

    #--------------------------------------------------------------------------
    # Return if this generator creates an active layer which must be detected as a separate image segmentation class
    #
    def is_active_layer (self):
        return True

    #--------------------------------------------------------------------------
    # Generate image
    #
    def generate (self):

        #
        # Mask marking the area covered by the sheet
        #
        mask = np.zeros ((self.height, self.width), dtype=np.float64)

        #
        # Step 1: Compute area used for the specimen border
        #
        # The border will be rectangular in principle but in later steps segments of the
        # specimen are cut out to simulate a more random layout.
        #
        outer_border_limit = [Point2d (SheetMetalGenerator.spacing), Point2d (self.size - SheetMetalGenerator.spacing)]
        inner_border_limit = [Point2d (2 * SheetMetalGenerator.spacing), Point2d (self.size - 2 * SheetMetalGenerator.spacing)]

        border_rect = Rect2d (Point2d (random.randint (outer_border_limit[0].x,
                                                       inner_border_limit[0].x),
                                       random.randint (outer_border_limit[0].y,
                                                       inner_border_limit[0].y)),
                              Point2d (random.randint (inner_border_limit[1].x,
                                                       outer_border_limit[1].x),
                                       random.randint (inner_border_limit[1].y,
                                                       outer_border_limit[1].y)))

        #
        # Compute segments of the specimen used. There are 3x3 segments available.
        #
        available = [[True, True, True], [True, True, True], [True, True, True]]
        border = []

        segment_mode = random.randint (0, 2)

        #
        # Case 1.1: Corner segments might be missing
        #
        if segment_mode == 0:
            available[0][0] = random.randint (0, 9) > 5
            available[2][0] = random.randint (0, 9) > 5
            available[0][2] = random.randint (0, 9) > 5
            available[2][2] = random.randint (0, 9) > 5

            segment = self.get_segment_rect (border_rect, (0, 0))
            used = available[0][0]

            border.append (segment.p3)
            border.append (segment.p0 if used else segment.p2)
            border.append (segment.p1)

            segment = self.get_segment_rect (border_rect, (2, 0))
            used = available[2][0]

            border.append (segment.p0)
            border.append (segment.p1 if used else segment.p3)
            border.append (segment.p2)

            segment = self.get_segment_rect (border_rect, (2, 2))
            used = available[2][2]

            border.append (segment.p1)
            border.append (segment.p2 if used else segment.p0)
            border.append (segment.p3)

            segment = self.get_segment_rect (border_rect, (0, 2))
            used = available[0][2]

            border.append (segment.p2)
            border.append (segment.p3 if used else segment.p1)
            border.append (segment.p0)

        #
        # Case 1.2: Top/down edge segments might be missing
        #
        elif segment_mode == 1:
            available[1][0] = random.randint (0, 9) > 5
            available[1][2] = random.randint (0, 9) > 5

            segment = self.get_segment_rect (border_rect, (0, 0))
            border.append (segment.p0)

            segment = self.get_segment_rect (border_rect, (1, 0))
            used = available[1][0]

            border.append (segment.p0)
            if not used:
                border.append (segment.p3)
                border.append (segment.p2)
            border.append (segment.p1)

            segment = self.get_segment_rect (border_rect, (2, 0))
            border.append (segment.p1)

            segment = self.get_segment_rect (border_rect, (2, 2))
            border.append (segment.p2)

            segment = self.get_segment_rect (border_rect, (1, 2))
            used = available[1][2]

            border.append (segment.p2)
            if not used:
                border.append (segment.p1)
                border.append (segment.p0)
            border.append (segment.p3)

            segment = self.get_segment_rect (border_rect, (0, 2))
            border.append (segment.p3)

        #
        # Case 1.3: Left/right edge segments might be missing
        #
        elif segment_mode == 2:
            available[0][1] = random.randint (0, 9) > 5
            available[2][1] = random.randint (0, 9) > 5

            segment = self.get_segment_rect (border_rect, (0, 0))
            border.append (segment.p0)

            segment = self.get_segment_rect (border_rect, (2, 0))
            border.append (segment.p1)

            segment = self.get_segment_rect (border_rect, (2, 1))
            used = available[2][1]

            border.append (segment.p1)
            if not used:
                border.append (segment.p0)
                border.append (segment.p3)
            border.append (segment.p2)

            segment = self.get_segment_rect (border_rect, (2, 2))
            border.append (segment.p2)

            segment = self.get_segment_rect (border_rect, (0, 2))
            border.append (segment.p3)

            segment = self.get_segment_rect (border_rect, (0, 1))
            used = available[0][1]

            border.append (segment.p3)
            if not used:
                border.append (segment.p2)
                border.append (segment.p1)
            border.append (segment.p0)

        #
        # Step 2: Draw border
        #
        border = Polygon2d (border)
        border.draw (mask, 1.0, fill=True)

        specimen_image = self.texture_creator.create ()

        copy_mask = np.dstack ((mask, mask, mask))
        specimen = copy_mask * specimen_image

        #
        # Step 2: Add some features to the available areas
        #
        for y in range (0, 3):
            for x in range (0, 3):
                if available[x][y] and random.randint (0, 9) < 7:
                    area = self.get_segment_rect (border_rect, (x, y))
                    available[x][y] = False

                    if x < 2 and available[x + 1][y] and random.randint (0, 9) < 5:
                        area = area.expanded (self.get_segment_rect (border_rect, (x + 1, y)))
                        available[x + 1][y] = False

                    elif y < 2 and available[x][y + 1] and random.randint (0, 9) < 5:
                        area = area.expanded (self.get_segment_rect (border_rect, (x, y + 1)))
                        available[x][y + 1] = False

                    self.create_feature_set (specimen, mask, area)

        #
        # Step 3: Apply some transformations to the specimen
        #
        scale = (random.uniform (0.5, 0.8), random.uniform (0.5, 0.8))
        shear = random.uniform (0.0, 0.2)
        rotation = random.uniform (-1.0, 1.0)

        specimen = utils.transform (specimen, self.size, scale=scale, shear=shear, rotation=rotation)
        mask = utils.transform (mask, self.size, scale=scale, shear=shear, rotation=rotation)

        return specimen, mask

    #--------------------------------------------------------------------------
    # Generate feature set in the given area
    #
    # @param area Area the feature may occupy
    #
    def create_feature_set (self, specimen, mask, area):

        split_scenarios = []
        split_scenarios.append ([area])
        split_scenarios.append ([area])
        split_scenarios.append ([area])
        split_scenarios.append ([area])
        split_scenarios.append ([area])
        split_scenarios.append ([area.split ((0, 1, 3), (0, 1, 3)),
                                 area.split ((2, 2, 3), (0, 1, 3)),
                                 area.split ((0, 2, 3), (2, 2, 3))])
        split_scenarios.append ([area.split ((0, 2, 3), (0, 0, 3)),
                                 area.split ((0, 2, 3), (1, 2, 3))])
        split_scenarios.append ([area.split ((0, 2, 3), (0, 0, 3)),
                                 area.split ((0, 2, 3), (1, 2, 3))])
        split_scenarios.append ([area.split ((0, 2, 3), (0, 1, 3)),
                                 area.split ((0, 2, 3), (2, 2, 3))])
        split_scenarios.append ([area.split ((0, 1, 3), (0, 2, 3)),
                                 area.split ((2, 2, 3), (0, 2, 3))])
        split_scenarios.append ([area.split ((0, 1, 3), (0, 1, 3)),
                                 area.split ((2, 2, 3), (0, 1, 3)),
                                 area.split ((0, 1, 3), (2, 2, 3)),
                                 area.split ((2, 2, 3), (2, 2, 3))])
        split_scenarios.append ([area.split ((0, 0, 3), (0, 0, 3)),
                                 area.split ((1, 1, 3), (0, 0, 3)),
                                 area.split ((2, 2, 3), (0, 0, 3)),
                                 area.split ((0, 0, 3), (1, 1, 3)),
                                 area.split ((1, 1, 3), (1, 1, 3)),
                                 area.split ((2, 2, 3), (1, 1, 3)),
                                 area.split ((0, 0, 3), (2, 2, 3)),
                                 area.split ((1, 1, 3), (2, 2, 3)),
                                 area.split ((2, 2, 3), (2, 2, 3))])

        split = split_scenarios[random.randint (0, len (split_scenarios) - 1)]

        for area in split:
            self.create_feature (specimen, mask, area)

    #--------------------------------------------------------------------------
    # Generate random feature
    #
    # @param image Image to draw into
    # @param area  Area the feature may occupy
    #
    def create_feature (self, specimen, mask, area):
        inner_rect = Rect2d (area.p0 + SheetMetalGenerator.spacing / 2, area.p2 - SheetMetalGenerator.spacing / 2)

        if inner_rect.size ().width > SheetMetalGenerator.spacing.width and inner_rect.size ().height > SheetMetalGenerator.spacing.height:

            offset = Size2d (random.randint (0, int (inner_rect.size ().width - SheetMetalGenerator.spacing.width)),
                             random.randint (0, int (inner_rect.size ().height - SheetMetalGenerator.spacing.height)))

            feature_rect = Rect2d (inner_rect.p0 + offset / 2, inner_rect.p2 - offset / 2)
            feature_type = random.randint (0, 1)

            #
            # Feature type 1: Rectangle
            #
            if feature_type == 0:
                feature_rect.draw (specimen, 0.0, fill=True)
                feature_rect.draw (mask, 0.0, fill=True)

            #
            # Feature type 2: Circle
            #
            elif feature_type == 1:
                circle = Ellipse2d (feature_rect).to_circle ()
                circle.draw (specimen, 0.0, fill=True)
                circle.draw (mask, 0.0, fill=True)

    #--------------------------------------------------------------------------
    # Return the segment rect in the (3, 3) raster of an image rect
    #
    # @param rect    Complete image rect
    # @param segment Segment (x, y) identifier in (3, 3) raster
    # @return Rectangle of the matching segment part
    #
    @staticmethod
    def get_segment_rect (rect, segment):
        x = segment[0]
        y = segment[1]

        assert x >= 0 and x <= 2
        assert y >= 0 and y <= 2

        return Rect2d (Point2d (rect.p0.x + x * rect.size ().width / 3,
                                rect.p0.y + y * rect.size ().height / 3),
                       Point2d (rect.p0.x + (x + 1) * rect.size ().width / 3,
                                rect.p0.y + (y + 1) * rect.size ().height / 3))

#--------------------------------------------------------------------------
# Local functions
#
def get_file_name (args, index):
    return os.path.join (args.save, 'dataset_{index:0>8}.h5'.format (index=index))

#--------------------------------------------------------------------------
# MAIN
#
if __name__ == '__main__':

    random.seed ()

    #
    # Parse command line arguments
    #
    parser = argparse.ArgumentParser ()

    parser.add_argument ('-x', '--width',   type=int, default=640,  help='Width of the generated images')
    parser.add_argument ('-y', '--height',  type=int, default=480,  help='Height of the generated images')

    args = parser.parse_args ()

    generator = SheetMetalGenerator (args)
    image, mask = generator.generate ()

    assert len (image.shape) == 3
    assert image.shape[0] == args.height
    assert image.shape[1] == args.width
    assert image.shape[2] == 3
    assert image.dtype == np.float64

    assert len (mask.shape) == 2
    assert mask.shape[0] == args.height
    assert mask.shape[1] == args.width
    assert mask.dtype == np.float64

    utils.show_image ([image, 'Sheet metal'],
                      [mask,  'Specimen mask'])


"""
This file reads the CSV file containing the lyrics data by chunks.
The path to the CSV file should be ./Data/song_lyrics.csv
For each row the data is processed.
This file saves the processed data chunks in the folders: Data/InputData & Data/OutputData

Content of the saved files:
    Data/InputData -> npy files containing a matrix of the input data.
    Data/InputDataNotNorm -> npy files containing a matrix of the non normalized input data.
    Data/OutputData -> npy files containing a matrix of the output data.
"""

import nltk, re, string, logging
import numpy as np
import pandas as pd
import gensim.downloader as api
from nltk.corpus import stopwords
import time
import threading
import os

START_FROM = int(input("Start from: "))

word2vec_model = api.load("word2vec-google-news-300")
print("---- Loading of Word2Vec model completed")

def data_clean(text):
    """
    Cleans the input text by removing metadata, stopwords, and punctuation.

    Args:
    text (str): The input text to be cleaned.

    Returns:
    list: A list of cleaned words.
    """

    text = re.sub(r'\[.*?\]', '', text) # Remove metadata from text using regular expressions
    text.replace('\n',' ')

    words = nltk.tokenize.wordpunct_tokenize(text)

    text_result = []
    for word in words: # for each word in the tokenized list of words from the sentence
        if word.lower() not in stopwords.words('english') and word[0] not in string.punctuation and word.lower() not in text_result: # if the word not a stop word, word not already in vocab and word not punctuation
            text_result.append(word.lower())

    return text_result

def process_data(start_row=START_FROM, chunk_size=250000, verbose=True):
    empty_songs_counter = 0
    if verbose:
        print(f"---- Chunk size = {chunk_size}")

    # Define the vectors for "rap", "pop" and others
    pop_vector =        np.array([1,0,0,0,0,0])
    rap_vector =        np.array([0,1,0,0,0,0])
    rock_vector =       np.array([0,0,1,0,0,0])
    rb_vector =         np.array([0,0,0,1,0,0])
    country_vector =    np.array([0,0,0,0,1,0])
    other_vector =      np.array([0,0,0,0,0,1])

    # Define the counter for the number of songs processed
    count = start_row
    progress = start_row
    if verbose:
        print(f"---- Starting from {count}")

    timed = time.time()

    # Iterate through the CSV file in chunks
    for chunk in pd.read_csv('./Data/song_lyrics.csv', skiprows=range(1, start_row), chunksize=chunk_size):

        # declare arrays of input and output
        input_data = []
        input_data_non_normalized = []
        output_data = []

        # update counters
        count+= chunk_size

        # for each row (song)
        for index, row in chunk.iterrows():

            # Show progress
            progress+=1
            if progress%500==0:
                if verbose:
                    print(f"Progress -> {progress} || Time taken -> {round(time.time()-timed,2)}")
                    timed = time.time()

            # Process each row

            if row['language']=="en": # if the lyrics are in english

                ##_________________Input processing_________________
                sum_word_vecs = np.zeros(300) # initialise sum of word vectors
                total_number_words = 0 # number of words transformed to vectors

                for word_to_vectorize in data_clean(row['lyrics']): # for each word in the cleaned, tokenized list from the lyrics
                    # print(word_to_vectorize)
                    try: # vectorise the word
                        sum_word_vecs+= word2vec_model[word_to_vectorize]
                        total_number_words+=1
                    except Exception as e:
                        # print(f"failed on: {word_to_vectorize} error: {e}")
                        pass

                if total_number_words>=10: # if the lyrics contain more than 10 words

                    tot_vec = sum_word_vecs/total_number_words # mean of all the vectors
                    input_data_non_normalized.append(tot_vec)
                    input_data.append(tot_vec/np.linalg.norm(tot_vec)) # append the normalized vector to the inpu_data list
                    ##____________________________________________________________________

                    ##_________________Output processing_________________
                    # process output data
                    tag = row['tag']
                    if tag == 'rap':
                        output_data.append(rap_vector)
                    elif tag == 'pop':
                        output_data.append(pop_vector)
                    elif tag == 'rock':
                        output_data.append(rock_vector)
                    elif tag == 'rb':
                        output_data.append(rb_vector)
                    elif tag == 'country':
                        output_data.append(country_vector)
                    else:
                        output_data.append(other_vector)
                    ##____________________________________________________________________
                else:
                    print(f"Not enough words: {progress}, found only {total_number_words}")
                    empty_songs_counter +=1

        np.save(f'{save_directories[0]}/I_data_chunk_{count}.npy', np.array(input_data)) # save input data matrix
        np.save(f'{save_directories[1]}/I_data_chunk_{count}.npy', np.array(input_data_non_normalized)) # save input data matrix
        np.save(f'{save_directories[2]}/O_data_chunk_{count}.npy', np.array(output_data)) # save output data matrix

        print(f"Number of rows processed: from {start_row} to {count} = {(count-start_row)}") # print how many rows have been processed
        print(f"Number of empty songs encountered: {empty_songs_counter}")

    print("DONE. CHUNK COMPLETED!!!")

# Helper function to calculate the starting indexes for multithreading the whole set
def get_starting_indexes(goal=5000000, chunksize=250000, threads=5):
    starting_indexes = []

    adder = goal/threads
    i=0

    while (threads*i) < goal:
        k = i
        indexes = []
        while k < goal:
            indexes.append(int(k))
            k += adder
        i += chunksize
        starting_indexes.append(indexes)

    return starting_indexes

# ______________________________EXECUTION OF THE FUNCTIONS________________________________________________
# Directories where all the output will be saved
save_directories = ['Data/InputData_Complete',
                   'Data/InputData_Complete_NotNorm',
                   'Data/OutputData_Complete']
for directory in save_directories:
    os.makedirs(directory, exist_ok=True)

# Call function wich will process only the output and start at the row given as global argument
# process_data(chunk_size=2, verbose=True)

# ________________________________________________________________________________________________________
# SINGLE THREAD, MULTIPLE TERMINALS:
chunksize = 50000
threads = 4

starting_2d = get_starting_indexes(chunksize=chunksize, threads=threads)
starting_2d = np.array(starting_2d)
starting_2d = np.transpose(starting_2d)

for starting_index in starting_2d[START_FROM]:
    process_data(start_row=starting_index, chunk_size=chunksize, verbose=True)
# ________________________________________________________________________________________________________

# ________________________________________________________________________________________________________
# Multi threaded Approach
# chunksize = 50000
# threads = 8

# starting_2d = get_starting_indexes(chunksize=chunksize, threads=threads)

# # for serie in starting_2d:
# def process_data_threaded(start_row):
#     process_data(start_row=start_row, chunk_size=chunksize, verbose=False)

# threads = []

# for starting_index in serie:
#     thread = threading.Thread(target=process_data_threaded, args=(starting_index,))
#     threads.append(thread)
#     thread.start()

# # Wait for all threads to finish
# for thread in threads:
#     thread.join()

# print("\n\n!!!DONE!!! \n\n", serie, "\n\n\n")
# ________________________________________________________________________________________________________


import logging

from automation_infra.utils.waiter import wait_for_predicate, wait_for_predicate_nothrow
from automation_infra.plugins.ssh_direct import SshDirect, SSHCalledProcessError
from pytest_automation_infra.helpers import hardware_config
from devops_automation_infra.plugins.power import Power
from devops_automation_infra.plugins.gravity import Gravity
from devops_automation_infra.plugins.k8s import K8s
from devops_automation_infra.utils.k8s_utils import create_deployment_with_replicas
from devops_automation_infra.utils.health_check import host_is_active

def all_deployments_pods_alive(host, deployment_name):
    deployment_pod_ips = host.K8s.get_deployments_pod_internal_ips(deployment_name)
    for pod_ip in deployment_pod_ips:
        res = pod_is_live(host, ip=pod_ip, port=8080)
        return res == "Hello World!"

def pod_is_live(host, ip, port, timeout=10):
    try:
        res = host.SshDirect.execute(f"curl --max-time {timeout} http://{ip}:{port}")
        return res
    except SSHCalledProcessError as e:
        logging.error(e.output)

@hardware_config(hardware={"host1": {}, "host2": {}})
def test_cluster_network_master_restart(base_config,
                                        clean_up_all_deployments_and_svcs,
                                        amount_of_replicas=100,
                                        docker_image_name='gcr.io/hello-minikube-zero-install/hello-node',
                                        deployment_name="test"):
    # Clean up before test starts
    base_config.hosts.host1.SshDirect.connect(timeout=60)
    create_deployment_with_replicas(base_config.hosts.host1, deployment_name, docker_image_name, amount_of_replicas)
    base_config.hosts.host1.Power.reboot()
    # Check host has started again
    wait_for_predicate_nothrow(lambda: host_is_active(base_config.hosts.host1.ip), timeout=60)
    base_config.hosts.host1.SshDirect.connect(timeout=60)
    wait_for_predicate(lambda: base_config.hosts.host1.Gravity.is_cluster_healthy(),
                       timeout=120,
                       interval=5)
    wait_for_predicate_nothrow(lambda: all_deployments_pods_alive(base_config.hosts.host1, deployment_name),
                       timeout=300,
                       interval=10)


import sys
import os
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtMultimediaWidgets import QVideoWidget

import resources

class Ui_NewProject(object):
    def setupUi(self, NewProject):
        NewProject.setObjectName("NewProject")
        NewProject.resize(736, 625)
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/plugins/VideoGis/icon.png"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        NewProject.setWindowIcon(icon)
        self.gridLayout_2 = QtWidgets.QGridLayout(NewProject)
        self.gridLayout_2.setObjectName("gridLayout_2")
        self.gridLayout = QtWidgets.QGridLayout()
        self.gridLayout.setObjectName("gridLayout")
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.video_frame_2 = QVideoWidget(NewProject)
        p = self.video_frame_2.palette()
        p.setColor(QtGui.QPalette.Window, QtCore.Qt.black)
        self.video_frame_2.setPalette(p)
        self.video_frame_2.setAttribute(QtCore.Qt.WA_OpaquePaintEvent)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.video_frame_2.sizePolicy().hasHeightForWidth())
        self.video_frame_2.setSizePolicy(sizePolicy)
        self.video_frame_2.setStyleSheet("background-color: rgb(0, 0, 10);")
        self.video_frame_2.setObjectName("video_frame_2")

        # self.scroll = QtWidgets.QScrollArea()#+
        # self.scroll.setWidget(self.video_frame_2)#+
        # self.video_frame_2.setGeometry(0,0,736,625)
        # self.horizontalLayout.addWidget(self.scroll)

        self.horizontalLayout.addWidget(self.video_frame_2)

        self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 16)
        self.horizontalSlider = QtWidgets.QSlider(NewProject)
        self.horizontalSlider.setOrientation(QtCore.Qt.Horizontal)
        self.horizontalSlider.setObjectName("horizontalSlider")
        self.gridLayout.addWidget(self.horizontalSlider, 1, 0, 1, 16)
        self.replayPlay_pushButton = QtWidgets.QPushButton(NewProject)
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Preferred)
        sizePolicy.setHorizontalStretch(0)
        sizePolicy.setVerticalStretch(0)
        sizePolicy.setHeightForWidth(self.replayPlay_pushButton.sizePolicy().hasHeightForWidth())
        self.replayPlay_pushButton.setSizePolicy(sizePolicy)
        self.replayPlay_pushButton.setCheckable(False)
        self.replayPlay_pushButton.setChecked(False)
        self.replayPlay_pushButton.setObjectName("replayPlay_pushButton")
        self.gridLayout.addWidget(self.replayPlay_pushButton, 3, 1, 1, 1)
        self.replayPosition_label = QtWidgets.QLabel(NewProject)
        self.replayPosition_label.setObjectName("replayPosition_label")
        self.gridLayout.addWidget(self.replayPosition_label, 3, 4, 1, 1)
        self.muteButton = QtWidgets.QToolButton(NewProject)
        self.muteButton.setText("")
        self.muteButton.setObjectName("muteButton")
        self.gridLayout.addWidget(self.muteButton, 3, 2, 1, 1)
        self.comboBox = QtWidgets.QComboBox(NewProject)
        self.comboBox.setObjectName("comboBox")
        self.gridLayout.addWidget(self.comboBox, 3, 15, 1, 1)
        self.pushButton_2 = QtWidgets.QPushButton(NewProject)
        self.pushButton_2.setObjectName("pushButton_2")
        self.gridLayout.addWidget(self.pushButton_2, 3, 13, 1, 1)
        self.toolButton_3 = QtWidgets.QToolButton(NewProject)
        icon1 = QtGui.QIcon()
        icon1.addPixmap(QtGui.QPixmap(":/plugins/VideoGis/mIconFormSelect.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_3.setIcon(icon1)
        self.toolButton_3.setObjectName("toolButton_3")
        self.gridLayout.addWidget(self.toolButton_3, 3, 11, 1, 1)

        # <<
        self.toolButton = QtWidgets.QToolButton(NewProject)
        icon5 = QtGui.QIcon()
        icon5.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasPrev.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton.setIcon(icon5)
        self.toolButton.setObjectName("toolButton")
        self.gridLayout.addWidget(self.toolButton, 3, 6, 1, 1)

        # >>
        self.toolButton_2 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_2.setIcon(icon2)
        self.toolButton_2.setObjectName("toolButton_2")
        self.gridLayout.addWidget(self.toolButton_2, 3, 9, 1, 1)

        # <<<
        self.toolButton_6 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_6.setIcon(icon2)
        self.toolButton_6.setObjectName("toolButton_6")
        self.gridLayout.addWidget(self.toolButton_6, 3, 5, 1, 1)

        # >>>
        self.toolButton_5 = QtWidgets.QToolButton(NewProject)
        icon2 = QtGui.QIcon()
        icon2.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionAtlasNext.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.toolButton_5.setIcon(icon2)
        self.toolButton_5.setObjectName("toolButton_5")
        self.gridLayout.addWidget(self.toolButton_5, 3, 10, 1, 1)

        # >
        self.SkipFortoolButton_8 = QtWidgets.QToolButton(NewProject)
        self.SkipFortoolButton_8.setStyleSheet("")
        icon3 = QtGui.QIcon()
        icon3.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionArrowRight.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.SkipFortoolButton_8.setIcon(icon3)
        self.SkipFortoolButton_8.setObjectName("SkipFortoolButton_8")
        self.gridLayout.addWidget(self.SkipFortoolButton_8, 3, 8, 1, 1)

        self.SkipBacktoolButton_7 = QtWidgets.QToolButton(NewProject)
        self.SkipBacktoolButton_7.setStyleSheet("")
        icon4 = QtGui.QIcon()
        icon4.addPixmap(QtGui.QPixmap(":/VgisIcon/mActionArrowLeft.svg"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
        self.SkipBacktoolButton_7.setIcon(icon4)
        self.SkipBacktoolButton_7.setObjectName("SkipBacktoolButton_7")
        self.gridLayout.addWidget(self.SkipBacktoolButton_7, 3, 7, 1, 1)

        self.pushButton = QtWidgets.QPushButton(NewProject)
        self.pushButton.setObjectName("pushButton")
        self.gridLayout.addWidget(self.pushButton, 3, 0, 1, 1)
        self.toolButton_4 = QtWidgets.QToolButton(NewProject)
        self.toolButton_4.setObjectName("toolButton_4")
        self.gridLayout.addWidget(self.toolButton_4, 3, 12, 1, 1)
        self.checkBox = QtWidgets.QCheckBox(NewProject)
        self.checkBox.setObjectName("checkBox")
        self.gridLayout.addWidget(self.checkBox, 3, 14, 1, 1)
        self.gridLayout_2.addLayout(self.gridLayout, 0, 0, 1, 1)

        self.retranslateUi(NewProject)
        QtCore.QMetaObject.connectSlotsByName(NewProject)

    def retranslateUi(self, NewProject):
        _translate = QtCore.QCoreApplication.translate
        NewProject.setWindowTitle(_translate("NewProject", "Video UAV Tracker - New Project"))
        self.replayPlay_pushButton.setText(_translate("NewProject", "Play/Pause"))
        self.replayPosition_label.setText(_translate("NewProject", "-:- / -:-"))
        self.pushButton_2.setToolTip(_translate("NewProject", "
Synchronize actual video frame with selected GPS time
"))
        self.comboBox.setToolTip(_translate("NewProject", "
 GPS time
"))
        self.pushButton_2.setText(_translate("NewProject", "Synchronize!"))
        self.toolButton_3.setToolTip(_translate("NewProject", "
Add point shape database to project
"))
        #self.toolButton_3.setText(_translate("NewProject", "DB"))

        self.toolButton_2.setToolTip(_translate("NewProject", "
Next second
"))
        self.toolButton_2.setText(_translate("NewProject", ">>"))

        self.toolButton_6.setToolTip(_translate("NewProject", "
Previous 1min
"))
        self.toolButton_6.setText(_translate("NewProject", "<<<"))

        self.toolButton_5.setToolTip(_translate("NewProject", "
Next 1min
"))
        self.toolButton_5.setText(_translate("NewProject", ">>>"))

        self.SkipFortoolButton_8.setToolTip(_translate("NewProject", "
Next frame
"))
        self.SkipFortoolButton_8.setText(_translate("NewProject", ">"))
        self.SkipBacktoolButton_7.setToolTip(_translate("NewProject", "
Previous frame
"))
        self.SkipBacktoolButton_7.setText(_translate("NewProject", "<"))
        self.toolButton.setToolTip(_translate("NewProject", "
Previous second
"))
        self.toolButton.setText(_translate("NewProject", "<<"))
        self.pushButton.setToolTip(_translate("NewProject", "
Select video and relative gpx
"))
        self.pushButton.setText(_translate("NewProject", "Select Video and GPX"))
        self.toolButton_4.setText(_translate("NewProject", "3D"))
        self.checkBox.setToolTip(_translate("NewProject","
Activate Pixel value conversion and display (see README)
"))


# import statsapi as mlbstats
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pybaseball as pbb
from datetime import date
from matplotlib import gridspec
from matplotlib import gridspec

df_sch = pd.read_csv('mets_2023.csv')
df_sch.pop('Unnamed: 0') #artifact from saving csv in other code

other_team = [] # list of other teams played

for index, row in df_sch.iterrows():

    if row['home_id'] == 121:
        other_team.append(row['away_name'])
    else:
        other_team.append(row['home_name'])

df_sch['other_team'] = other_team

team_set = set(other_team) #only want unique values of other teams

df_teams = pd.DataFrame({'team': [t for t in team_set]})

df_teams['won'] = np.zeros(len(team_set))
df_teams['lost'] = np.zeros(len(team_set))
df_teams['incomplete'] = np.zeros(len(team_set))
df_teams['total'] = np.zeros(len(team_set))
df_teams['inc_home'] = np.zeros(len(team_set))
df_teams['inc_away'] = np.zeros(len(team_set))

### Not optimal method, should optimize to list comprehension in future version
for index, row in df_sch.iterrows():
    if row['status'] == 'Scheduled':
        df_teams.loc[df_teams['team']==row['other_team'], ['incomplete']] += 1
        if row['other_team'] == row['away_name']:
            df_teams.loc[df_teams['team']==row['other_team'], ['inc_home']] += 1
        else:
            df_teams.loc[df_teams['team']==row['other_team'], ['inc_away']] += 1

    elif row['winning_team'] == 'New York Mets':
        df_teams.loc[df_teams['team']==row['other_team'], ['won']] += 1

    else:
        df_teams.loc[df_teams['team']==row['other_team'], ['lost']] += 1

    df_teams.loc[df_teams['team']==row['other_team'], ['total']] += 1

### Preseason messes up pybaseball.standings:
try:
    standings = pd.concat(pbb.standings(), ignore_index=True)
    team_rec = [f"{standings.loc[standings['Tm']==team, ['W']].values[0][0]}-{standings.loc[standings['Tm']==team, ['L']].values[0][0]}" for team in df_teams['team']]
except:
    team_rec = ["0-0" for team in df_teams['team']]

nicknames = [] ## Adding city names crowds up the labels too much
for i,t in enumerate(df_teams['team']):
    if t.split()[-1] == "Sox": #Accounting for Boston/Chicago Sox
        nicknames.append(f"{t.split()[-2]+ ' ' + t.split()[-1] } ({team_rec[i]})")
    else:
        nicknames.append(f"{t.split()[-1]} ({team_rec[i]})")
df_teams['team'] = nicknames

### Order for the teams to be plotted in, consistent throughout the season
df_teams.sort_values(['total', 'team'], ascending=[False, True], inplace=True, ignore_index=True)

_, won, lost, incomplete, total, inc_home, inc_away = df_teams.sum().values

won = round(won)
lost = round(lost)
incomplete = round(incomplete)
incomplete_home = round(inc_home)
incomplete_away = round(inc_away)

### Trying new way due to 50% more teams:
fig = plt.figure(tight_layout=True)
gs = gridspec.GridSpec(2, 2, height_ratios=[1,12])

w = 0.9 #main bar width
x = [np.arange(15), np.arange(14)]

### Total season progress bar:
ax = fig.add_subplot(gs[0, :])

if won != 0:
    wins   = ax.barh(0, won, w, label='won',
                        color="cornflowerblue", edgecolor='k')
    ax.text(won/2, 0, round(won) ,weight='bold', va='center', ha='center',
                    size=18, fontname='serif')
if lost != 0:
    losses = ax.barh(0, lost, w, left=won,
                        label='lost', color="darkorange", edgecolor='k')
    ax.text(won+lost/2, 0, round(lost), weight='bold', va='center', ha='center',
                    size=18, fontname='serif')
if inc_home != 0:
    home   = ax.barh(0, inc_home, w, left=won+lost,
                        label="incomplete home", color='silver', edgecolor='k')
    ax.text(won+lost+inc_home/2, 0, round(inc_home),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

if inc_away != 0:
    away   = ax.barh(0, inc_away, w, left=won+lost+inc_home,
                        label="incomplete away", color='k', edgecolor='k')
    ax.text(won+lost+inc_home+inc_away/2, 0, round(inc_away),
                    weight='bold', va='center', ha='center', color='w',
                    size=18, fontname='serif')

ax.axis('off')
ax.set_title('Total Season:', size=20, weight='bold',
             fontname='serif')

ax.set_xlim(0, 162)

### Individual team progress bars, broken into two plots
for i in range(2):
    ax = fig.add_subplot(gs[1, i])
    wins       = ax.barh(x[i], df_teams['won'][i::2], w, label='won',
                        color="cornflowerblue", edgecolor='k')

    losses     = ax.barh(x[i], df_teams['lost'][i::2], w, left=df_teams['won'][i::2],
                        label='lost', color="darkorange", edgecolor='k')

    home       = ax.barh(x[i], df_teams['inc_home'][i::2], w,
                          left=df_teams['won'][i::2]+df_teams['lost'][i::2],
                        label="incomplete home", color='silver', edgecolor='k')

    away       = ax.barh(x[i], df_teams['inc_away'][i::2], w,
                         left=df_teams['won'][i::2]+df_teams['lost'][i::2] + df_teams['inc_home'][i::2],
                        label="incomplete away", color='k', edgecolor='k')

    for j in x[i]:
        if wins.datavalues[j] != 0: #Ignore values that are currently 0
            ax.text(wins.datavalues[j]/2, x[i][j], int(wins.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if losses.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[i]/2, x[i][j], int(losses.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if home.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[j] + home.datavalues[j]/2, x[i][j], int(home.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif')

        if away.datavalues[j] != 0:
            ax.text(wins.datavalues[j] + losses.datavalues[j] + home.datavalues[j] + away.datavalues[j]/2, x[i][j], int(away.datavalues[j]),
                    weight='bold', va='center', ha='center',
                    size=18, fontname='serif', color='w')

    ax.set_ylim(min(x[0])-w/2, max(x[0])+w/2)
    ax.invert_yaxis()
    ax.set_xticks(range(0, 21, 5), fontfamily='serif', minor=False)
    ax.set_xticks(range(0, 21, 1), fontfamily='serif', minor=True)
    ax.tick_params('x', labelsize='large')
    ax.set_yticks(ticks = x[i], labels=df_teams['team'][i::2], size=18, fontfamily='serif')
    ax.set_xlim(0, df_teams['total'].max())

handles, labels = ax.get_legend_handles_labels()
fig.legend(handles, labels, loc='upper center', ncol=4, bbox_to_anchor=(0.5, 0.05),
           prop={'family': "serif", 'size':'x-large', 'weight': 'bold'}, framealpha=0)

fig.suptitle(f"Mets Current Season Series Progress\n{date.today()} ({won}-{lost}, {incomplete} GR)"
             , size=24, weight='bold', fontname='serif', y=0.9, va='bottom')

plt.show()


from flask import render_template, request
from app import app
import MySQLdb
from a_Model import ModelIt

import networkx as nx
import matplotlib.pylab as plt
from geopy.distance import great_circle
from geopy.geocoders import Nominatim
from setup_functions2 import *

@app.route('/')

@app.route('/index')
def index():
	return render_template("index.html",
		title = 'Home', user = { 'nickname': 'Missy' },
		)

@app.route('/db')
def bkmap_page():
	db = MySQLdb.connect(user="root", host="localhost", db="bk_map",  charset='utf8')
	with db:
		cur = db.cursor()
		cur.execute("SELECT * FROM bk_weights LIMIT 15;")
		query_results = cur.fetchall()
	edges = ""
	for result in query_results:
		edges += result[0]
		edges += "<br>"
		return edges

@app.route("/db_fancy")
def bkmap_page_fancy():
	db = MySQLdb.connect(user="root", host="localhost", db="bk_map",  charset='utf8')
	with db:
		cur = db.cursor()
		cur.execute("SELECT edgeid, node1, node2, weight FROM bk_weights LIMIT 15;")

		query_results = cur.fetchall()
	edges = []
	for result in query_results:
		edges.append(dict(edgeid=result[0], node1=result[1], node2=result[2], weight = result[3]))
	return render_template('cities.html', edges = edges)

@app.route('/input')
def crawl_input():
	return render_template("input.html")

@app.route('/input_2')
def crawl_input2():
	return render_template("input_2.html")

@app.route('/output')
def crawl_output():
	start = request.args.get("START_LOC")
	bar1 = request.args.get("BAR1")
	bar2 = request.args.get("BAR2")
	bar3 = request.args.get("BAR3")
	end = request.args.get("END_LOC")
	bar_input_list = [bar1, bar2, bar3]
	bar_address = get_bar_address(bar1, bar2, bar3)

	start_tuple, end_tuple = get_start_end(start,end)
	bar_tuple_list = get_bar_location(bar1,bar2,bar3)

	node_sql = ReadNodefromSQL(cur)
	edge_sql = ReadEdgefromSQL(cur)

	node_dic = NodeSQLtoDic(node_sql)
	edge_dic = EdgeDQLtoDic(edge_sql)

	G = gen_network(node_dic, edge_dic)

	path, line_path = crawl_path(G, start_tuple, bar_tuple_list, end_tuple, node_dic, edge_dic)
	bar_address = reorder_beer_list(bar_address, path)
	bar_tuple_list = reorder_beer_list(bar_tuple_list, path)
	single_path = []
	for line in line_path:
		single_path.extend(line)
	bar_list = tuplebarlist_floatlist(bar_tuple_list)
	bar_add = tuplebarlist_stringlist(bar_address)
	bar_dic = make_bar_dic(bar_add, bar_list)
	return render_template('output.html',  start = [start, start_tuple[0],start_tuple[1]],  bar_add =bar_add, bar_dic = bar_dic, end = [end,end_tuple[0],end_tuple[1]], path = path, path_list = single_path)


from math import log
from preProcess import preProcess
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix
from collections import Counter
from testResults import testResults
import numpy as np

class node:
    def __init__(self, attribute=None, depth=None):
        # Add a new node with a particular attribute value and depth
        self.depth = depth
        self.children = [None] * 10
        self.attribute = attribute
        self.definiteLabel = None

    def assign_child(self, attribute_value, child):
        # Assigning children at particular attribute value
        self.children[attribute_value] = child

    def assign_label(self, label):
        # Assigning a definite label when the node is a leaf node
        self.definiteLabel = label

class decisionTree:
    def __init__(self, criterion='entropy', max_depth=None, min_samples_split=2):
        # Initialize a decision tree with parameters
        # max_depth refers to maximum depth of the tree
        # min_samples_built refers to the minimum samples needed to build a subtree
        self.criterion = criterion
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.tree_built = False

    def find_entropy(self, dataset):
        # finds the entropy in a particular dataset using shannonEntropy measure
        numEntries = len(dataset)
        labelCounts = Counter()
        for i in range(numEntries):
            labelCounts[dataset[i][-1]] += 1
        shannonEntropy = 0
        for label in labelCounts:
            prob = float(labelCounts[label])/numEntries
            shannonEntropy -= prob*log(prob, 2)
        return shannonEntropy

    def perform_split(self, dataset, axis, value):
        # performs split at a particular axis of the dataset for a particular value and returns newDataset
        newDataset = []
        listDataset = list(dataset)
        for sample in listDataset:
            if sample[axis] == value:
                reducedFeatureVector = list(sample[:axis])
                reducedFeatureVector.extend(list(sample[axis+1:]))
                newDataset.append(reducedFeatureVector)
        return np.array(newDataset)

    def chooseBestFeatureToSplit(self, dataset):
        # finds the best attribute to split at based on the entropy value
        numFeatures = len(dataset[0])-1
        baseEntropy = self.find_entropy(dataset)
        bestInfoGain = 0
        bestFeature = -1
        for i in range(numFeatures):
            entropyVal = 0
            for value in range(1, 11):
                subDataset = self.perform_split(dataset, i, value)
                prob = len(subDataset) / float(len(dataset))
                entropyVal += prob*self.find_entropy(subDataset)
            infoGain = baseEntropy - entropyVal
            if infoGain > bestInfoGain:
                bestInfoGain = infoGain
                bestFeature = i
        return bestFeature

    def returnClassProbabilities(self, dataset):
        # returns the probabilites of 2 classes of the dataset
        sampleCount = len(dataset)
        labelCounts = Counter()
        for i in range(sampleCount):
            if dataset[i][-1] == 2:
                labelCounts[2] += 1
            else:
                labelCounts[4] += 1
        return [labelCounts[2]/float(sampleCount), labelCounts[4]/float(sampleCount)]

    def recursiveBuild(self, dataNode, dataset):
        # recursively builds the nodes
        probs = self.returnClassProbabilities(dataset)
        probs = sorted(self.returnClassProbabilities(dataset), reverse=True)
        # for the following conditions, the node stops building further and assigns class with highest count as definitiveLabel
        if probs[0] > 90 or dataNode.depth == self.max_depth or len(dataset)<self.min_samples_split:
            dataNode.assign_label(probs[0])
            return

        dataNode.labelSamples = [2,4][np.argmax(probs)]
        for i in range(1, 11):
            # performing n-way split
            splitDataset = self.perform_split(dataset, dataNode.attribute, i)
            if len(splitDataset):
                best_feature = self.chooseBestFeatureToSplit(splitDataset)
                # if bestFeature isn't found, we assign that node as a leaf node
                if best_feature==-1:
                    subProbs = self.returnClassProbabilities(splitDataset)
                    dataNode.children[i-1] = node(depth = dataNode.depth+1)
                    dataNode.children[i-1].assign_label([2,4][np.argmax(subProbs)])
                # else we assign the best feature as the attribute to the child node and recursively build it
                else:
                    dataNode.children[i-1] = node(attribute=best_feature, depth = dataNode.depth+1)
                    self.recursiveBuild(dataNode.children[i-1], splitDataset)

    def fit(self, dataset):
        # fits the dataset to the model and recursively starts building the decision tree
        self.tree_built = True
        self.root = node(attribute = self.chooseBestFeatureToSplit(dataset), depth=0)
        self.recursiveBuild(self.root, dataset)

    def predict(self, groups):
        # predicts for group of samples
        results = []
        for group in groups:
            results.append(self.predict_single(group))
        return np.array(results)

    def predict_single(self, x):
        # predicts for single sample
        available_features = list(x)
        if self.tree_built:
            pres_node = self.root
            while pres_node.definiteLabel == None:
                attribute = pres_node.attribute
                if pres_node.children[available_features[attribute]-1]:
                    pres_node = pres_node.children[available_features[attribute]-1]
                    available_features.pop(attribute)
                else: return pres_node.labelSamples
        return pres_node.definiteLabel

if __name__ == '__main__':
    preProcessData = preProcess()
    preProcessData.handle_missing_values()
    preProcessData.handle_highly_correlated_features()
    df = preProcessData.return_df()
    DT = decisionTree()
    kFold = KFold(6, True, 1)
    values = df.values
    for train, test in kFold.split(values):
        print("Taking %d train datapoints" % len(train))
        train_x = values[train]
        test_x, test_y = values[test][:,:-1], values[test][:,-1]
        DT.fit(train_x)
        pred_y = DT.predict(test_x)
        results = testResults(pred_y, test_y)
        print("Accuracy of model is ", results.return_accuracy())
        print('F Score of model is ', results.return_fscore())
        print("Confusion Matrix for the model :\n", confusion_matrix(test_y, pred_y))
        print()
        # print(DT.predict_single(test_x[0]), test_y[0])


import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow import keras
from keras.layers import Dense, Conv2D, Flatten, MaxPooling2D, BatchNormalization, Dropout
from keras.models import Sequential
from keras.callbacks import ReduceLROnPlateau
from keras.utils import to_categorical
import seaborn as sns
from sklearn.model_selection import train_test_split

## importing data
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

type(train)
train.head()
#shape of data imported
train.shape, test.shape

train.iloc[:,0].head()

#converting to the data to array
train_data = train.iloc[:,1:]
train_labels = train.iloc[:,0]

#one hot encoding for train_labels
train_labels = to_categorical(train_labels, num_classes = 10)
train_labels.shape

train_data = np.array(train_data)
test = np.array(test)

train_data = train_data.astype('float32')
train_labels = train_labels.astype('int32')
test = test.astype('float32')
train_data = train_data / 255.0
test = test / 255.0

# plotting an image
img = train_data[999].reshape(28,28)
plt.imshow(img, cmap = 'bone')

#plotting graphs for the train_labels
sns.countplot(train['label'])

#train_labels data
train_labels

#splitting data to training and validation data
x_train, x_val, y_train, y_val = train_test_split(train_data,train_labels, random_state = 0, shuffle = True)
x_train.shape, x_val.shape

#converting the data to keras required dimensions
x_train = x_train.reshape(x_train.shape[0], img_rows,img_cols, 1)
y_train = np.array(y_train)
y_train = to_categorical(y_train)

x_train.shape, y_train.shape

#validation data
x_val = x_val.reshape(x_val.shape[0],img_rows, img_cols, 1)
y_val = np.array(y_val)
y_val = to_categorical(y_val, num_classes = 10)

x_val.shape, y_val.shape

# Conv2D MODEL
img_rows, img_cols = 28,28
img_shape = (img_rows, img_cols, 1)

model = Sequential()

model.add(Conv2D(128, (3,3), activation = 'relu', kernel_initializer = 'he_normal', input_shape = img_shape))
model.add(Conv2D(128, (3,3), activation = 'relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.3))

model.add(Conv2D(256, (3,3), activation = 'relu'))
model.add(BatchNormalization())
model.add(Conv2D(256, (3,3), activation = 'relu'))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.3))

model.add(Flatten())

model.add(Dense(512, activation = 'relu'))
model.add(Dense(10, activation = 'softmax'))

#COMPILING MODEL
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])

# Fitting the MODEL

model.fit(x_train,y_train, epochs = 1, batch_size = 120, validation_data =(x_val,y_val), verbose = 1)


from cassandra.cluster import Cluster

hostname = '127.0.0.1'
keyspace = 'db1'
column_family = 'postInfo'
nodes = []
nodes.append(hostname)
cluster = Cluster(nodes)
session = cluster.connect(keyspace)

def insertInPostInfoCF(domainName, postinfo_map):
    '''

    Inserts a row into Post_Info ColumnFamily with key as domainName

    :param domainName:
    :param Postinfo_map:
    :return:
    '''
    executeBQ = session.prepare(
        "insert into postInfo (domain, totalQuestions, unansweredQuestions, trendingTags, averageAnswersCount, mostViewedPosts, mostScoredPosts, averageTimeToAnswer) values (?, ?, ?, ?, ?, ?, ?);")
    session.execute(executeBQ,
                    (domainName, postinfo_map['totalQuestions'], postinfo_map['unansweredQuestions'], postinfo_map['trendingTags'], postinfo_map['averageAnswersCount'],
                    postinfo_map['mostViewedPosts'], postinfo_map['mostScoredPosts'], postinfo_map['averageTimeToAnswer']))