import logging
import time
import requests
from requests.auth import HTTPDigestAuth
import os
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

LOGIN_FLAG = False
SHUTDOWN_FLAG = False
RESTART_FLAG = False
RESET_FLAG = False

def logging_start():
    logging.basicConfig(filename="/var/log/aim-power/result.log",
                        format="%(asctime)s:%(levelname)s:%(message)s",
                        level=logging.INFO)
    logging.info("ADDER: ==== Started Logging ====")

def logging_stop():
    logging.info("ADDER: ==== Stopped Logging ====")
    time.sleep(1)
    logging.shutdown()

def send_power_on():
    logging.info("ADDER: Power On")
    r = requests.get("http://10.10.10.254/hidden.htm?M0:O1=On", auth=HTTPDigestAuth("api", "api"))
    assert(r.status_code == 200)

def send_power_off():
    logging.info("ADDER: Power Off")
    r = requests.get("http://10.10.10.254/hidden.htm?M0:O1=Off", auth=HTTPDigestAuth("api", "api"))
    assert(r.status_code == 200)

def send_power_restart():
    logging.info("ADDER: Power Restart")
    r = requests.get("http://10.10.10.254/hidden.htm?M0:O1=Restart", auth=HTTPDigestAuth("api", "api"))
    assert(r.status_code == 200)

def start_driver():
    driver = webdriver.Firefox()
    wait = WebDriverWait(driver, 30)
    return driver, wait

def aim_login():
    global LOGIN_FLAG
    LOGIN_FLAG = False
    logging.info("ADDER: Login Aim")
    try:
        driver, wait = start_driver()
        driver.navigate.to("http://10.10.10.10/")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#username"))).send_keys("admin")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#password"))).send_keys("password")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#login"))).click()
        LOGIN_FLAG = wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
    except Exception as e:
        logging.info("ADDER: Problem with Aim Login - Stacktrace: %s" %e)
    finally:
        driver.quit()
    # ret = os.system("ruby aim_login.rb")
    # if ret == 0:
    #     LOGIN_FLAG = True
    # else:
    #     logging.info("ADDER: Problem with Aim Login")

def aim_shutdown():
    global SHUTDOWN_FLAG
    SHUTDOWN_FLAG = False
    logging.info("ADDER: Login and Shutdown Aim")
    try:
        driver, wait = start_driver()
        driver.navigate.to("http://10.10.10.10/")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#username"))).send_keys("admin")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#password"))).send_keys("password")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#login"))).click()
        wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
        driver.navigate.to("http://10.10.10.10/admin/process_power.php?mode=shutdown")
        time.sleep(5)
        driver.refresh()
        found = driver.find_elements((By.LINK_TEXT, "DASHBOARD"))
        if len(found) == 0:
            SHUTDOWN_FLAG = True
    except Exception as e:
        logging.info("ADDER: Problem with Aim Shutdown - Stacktrace: %s" %e)
    finally:
        driver.quit()
    # ret = os.system("ruby aim_shutdown.rb")
    # if ret == 0:
    #     SHUTDOWN_FLAG = True
    # else:
    #     logging.info("ADDER: Problem with Aim Shutdown")

def aim_restart():
    global RESTART_FLAG
    RESTART_FLAG = False
    logging.info("ADDER: Login and Restart Aim")
    try:
        driver, wait = start_driver()
        driver.navigate.to("http://10.10.10.10/")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#username"))).send_keys("admin")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#password"))).send_keys("password")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#login"))).click()
        wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
        driver.navigate.to("http://10.10.10.10/admin/process_power.php?mode=restart")
        time.sleep(20)
        driver.refresh()
        RESTART_FLAG = wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
    except Exception as e:
        logging.info("ADDER: Problem with Aim Restart - Stacktrace: %s" %e)
    finally:
        driver.quit()
    # ret = os.system("ruby aim_restart.rb")
    # if ret == 0:
    #     RESTART_FLAG = True
    # else:
    #     logging.info("ADDER: Problem with Aim Restart")

def aim_reset():
    global RESET_FLAG
    RESET_FLAG = False
    logging.info("ADDER: Login and Reset Aim")
    try:
        driver, wait = start_driver()
        driver.navigate.to("http://10.10.10.10/")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#username"))).send_keys("admin")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#password"))).send_keys("password")
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#login"))).click()
        wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
        wait.until(EC.presense_of_element_located((By.LINK_TEXT, "UPDATES"))).click()
        wait.until(EC.presense_of_element_located((By.LINK_TEXT, "Reset AIM Configuration"))).click()
        wait.until(EC.presense_of_element_located((By.CSS_SELECTOR, "#confirm_reset_link"))).click()
        time.sleep(120)
        RESET_FLAG = wait.until(EC.presense_of_element_located((By.LINK_TEXT, "DASHBOARD"))).is_displayed()
    except Exception as e:
        logging.info("ADDER: Problem with Aim Reset - Stacktrace: %s" %e)
    finally:
        driver.quit()
    # ret = os.system("ruby aim_reset.rb")
    # if ret == 0:
    #     RESET_FLAG = True
    # else:
    #     logging.info("ADDER: Problem with Aim Reset")

if __name__ == "__main__":
    logging_start()
    execution = 0
    passes = 0
    fails = 0
    while True:
        try:
            execution += 1
            send_power_on()
            time.sleep(60)

            aim_login()
            time.sleep(60)

            send_power_off()
            time.sleep(60)

            send_power_on()
            time.sleep(60)

            aim_login()
            time.sleep(60)

            send_power_restart()
            time.sleep(60)

            aim_shutdown()
            time.sleep(60)

            send_power_restart()
            time.sleep(60)

            aim_restart()
            time.sleep(60)

            aim_reset()
            time.sleep(60)

            aim_login()
            time.sleep(60)

            send_power_off()
            time.sleep(60)

            if LOGIN_FLAG and SHUTDOWN_FLAG and RESET_FLAG and RESTART_FLAG:
                passes += 1
            else:
                fails += 1
            logging.info("ADDER: Exes: %d Passes: %d Fails: %d" %(execution, passes, fails))
        except KeyboardInterrupt:
            logging_stop()
            break


from setuptools import setup, find_packages
import textwrap
from yardstick.openstack.common import setup as common_setup

install_requires = common_setup.parse_requirements(['tools/pip-requires'])
install_options = common_setup.parse_requirements(['tools/pip-options'])
tests_require = common_setup.parse_requirements(['tools/test-requires'])
setup_require = common_setup.parse_requirements(['tools/setup-requires'])
dependency_links = common_setup.parse_dependency_links([
    'tools/pip-requires',
    'tools/pip-options',
    'tools/test-requires',
    'tools/setup-requires'
])

setup(
    name='yardstick',
    version=common_setup.get_version('yardstick'),
    description='DNS as a Service',
    author='Kiall Mac Innes',
    author_email='kiall@hp.com',
    url='https://launchpad.net/yardstick',
    packages=find_packages(exclude=['bin']),
    include_package_data=True,
    test_suite='nose.collector',
    setup_requires=setup_require,
    install_requires=install_requires,
    tests_require=tests_require,
    extras_require={
        'test': tests_require,
        'optional': install_options,
    },
    dependency_links=dependency_links,
    scripts=[
        'bin/yardstick',
    ],
    cmdclass=common_setup.get_cmdclass(),
    # entry_points=textwrap.dedent("""
    #     [yardstick.storage]
    #     sqlalchemy = yardstick.storage.impl_sqlalchemy:SQLAlchemyStorage
    # """),
)


import os
import numpy as np

import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

from torchvision import transforms

## 네트워크 저장하기
def save(ckpt_dir, net, optim, epoch):
    if not os.path.exists(ckpt_dir):
        os.makedirs(ckpt_dir)

    torch.save({'net': net.state_dict(), 'optim': optim.state_dict()},
               "%s/model_epoch%d.pth" % (ckpt_dir, epoch))

## 네트워크 불러오기
def load(ckpt_dir, net, optim):
    if not os.path.exists(ckpt_dir):
        epoch = 0
        return net, optim, epoch

    ckpt_lst = os.listdir(ckpt_dir)
    ckpt_lst.sort(key=lambda f: int(''.join(filter(str.isdigit, f))))

    dict_model = torch.load('%s/%s' % (ckpt_dir, ckpt_lst[-1]))

    net.load_state_dict(dict_model['net'])
    optim.load_state_dict(dict_model['optim'])
    epoch = int(ckpt_lst[-1].split('epoch')[1].split('.pth')[0])

    return net, optim, epoch

# 훈련 파라미터 설정하기
from init import dir_data

lr = 1e-3
batch_size = 4
num_epoch = 3

base_dir = './drive/MyDrive/DACrew/unet'
data_dir = dir_data
ckpt_dir = os.path.join(base_dir, "checkpoint")
log_dir = os.path.join(base_dir, "log")

# 훈련을 위한 Transform과 DataLoader
from loader import Dataset
from transform import ToTensor, Normalization, RandomFlip

transform = transforms.Compose([Normalization(mean=0.5, std=0.5), RandomFlip(), ToTensor()])

dataset_train = Dataset(data_dir=os.path.join(data_dir, 'train'), transform=transform)
loader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)#, num_workers=8)

dataset_val = Dataset(data_dir=os.path.join(data_dir, 'val'), transform=transform)
loader_val = DataLoader(dataset_val, batch_size=batch_size, shuffle=False)#, num_workers=8)

# 네트워크 생성하기
from Unet import UNet

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
net = UNet().to(device)

# 손실함수 정의하기
fn_loss = nn.BCEWithLogitsLoss().to(device)

# Optimizer 설정하기
optim = torch.optim.Adam(net.parameters(), lr=lr)

# 그밖에 부수적인 variables 설정하기
num_data_train = len(dataset_train)
num_data_val = len(dataset_val)

num_batch_train = np.ceil(num_data_train / batch_size)
num_batch_val = np.ceil(num_data_val / batch_size)

# 그 밖에 부수적인 functions 설정하기
fn_tonumpy = lambda x: x.to('cpu').detach().numpy().transpose(0, 2, 3, 1)
fn_denorm = lambda x, mean, std: (x * std) + mean
fn_class = lambda x: 1.0 * (x > 0.5)

# Tensorboard 를 사용하기 위한 SummaryWriter 설정
writer_train = SummaryWriter(log_dir=os.path.join(log_dir, 'train'))
writer_val = SummaryWriter(log_dir=os.path.join(log_dir, 'val'))

# 네트워크 학습시키기
st_epoch = 0
# 학습한 모델이 있을 경우 모델 로드하기
net, optim, st_epoch = load(ckpt_dir=ckpt_dir, net=net, optim=optim)

for epoch in range(st_epoch + 1, num_epoch + 1):
        net.train()
        loss_arr = []

        for batch, data in enumerate(loader_train, 1):
            # forward pass
            label = data['label'].to(device)
            input = data['input'].to(device)

            output = net(input)

            # backward pass
            optim.zero_grad()

            loss = fn_loss(output, label)
            loss.backward()

            optim.step()

            # 손실함수 계산
            loss_arr += [loss.item()]

            print("TRAIN: EPOCH %04d / %04d | BATCH %04d / %04d | LOSS %.4f" %
                  (epoch, num_epoch, batch, num_batch_train, np.mean(loss_arr)))

            # Tensorboard 저장하기
            label = fn_tonumpy(label)
            input = fn_tonumpy(fn_denorm(input, mean=0.5, std=0.5))
            output = fn_tonumpy(fn_class(output))

            writer_train.add_image('label', label, num_batch_train * (epoch - 1) + batch, dataformats='NHWC')
            writer_train.add_image('input', input, num_batch_train * (epoch - 1) + batch, dataformats='NHWC')
            writer_train.add_image('output', output, num_batch_train * (epoch - 1) + batch, dataformats='NHWC')

        writer_train.add_scalar('loss', np.mean(loss_arr), epoch)

        with torch.no_grad():
            net.eval()
            loss_arr = []

            for batch, data in enumerate(loader_val, 1):
                # forward pass
                label = data['label'].to(device)
                input = data['input'].to(device)

                output = net(input)

                # 손실함수 계산하기
                loss = fn_loss(output, label)

                loss_arr += [loss.item()]

                print("VALID: EPOCH %04d / %04d | BATCH %04d / %04d | LOSS %.4f" %
                      (epoch, num_epoch, batch, num_batch_val, np.mean(loss_arr)))

                # Tensorboard 저장하기
                label = fn_tonumpy(label)
                input = fn_tonumpy(fn_denorm(input, mean=0.5, std=0.5))
                output = fn_tonumpy(fn_class(output))

                writer_val.add_image('label', label, num_batch_val * (epoch - 1) + batch, dataformats='NHWC')
                writer_val.add_image('input', input, num_batch_val * (epoch - 1) + batch, dataformats='NHWC')
                writer_val.add_image('output', output, num_batch_val * (epoch - 1) + batch, dataformats='NHWC')

        writer_val.add_scalar('loss', np.mean(loss_arr), epoch)

        # epoch 50마다 모델 저장하기
        # if epoch % 50 == 0:
        #     save(ckpt_dir=ckpt_dir, net=net, optim=optim, epoch=epoch)
        save(ckpt_dir=ckpt_dir, net=net, optim=optim, epoch=epoch)

        writer_train.close()
        writer_val.close()

# 훈련 모델 테스트
import matplotlib.pyplot as plt

transform = transforms.Compose([Normalization(mean=0.5, std=0.5), ToTensor()])

dataset_test = Dataset(data_dir=os.path.join(data_dir, 'test'), transform=transform)
loader_test = DataLoader(dataset_test, batch_size=batch_size, shuffle=False)#, num_workers=8)

# 그밖에 부수적인 variables 설정하기
num_data_test = len(dataset_test)
num_batch_test = np.ceil(num_data_test / batch_size)

# 결과 디렉토리 생성하기
result_dir = os.path.join(base_dir, 'result')
if not os.path.exists(result_dir):
    os.makedirs(os.path.join(result_dir, 'png'))
    os.makedirs(os.path.join(result_dir, 'numpy'))

net, optim, st_epoch = load(ckpt_dir=ckpt_dir, net=net, optim=optim)

with torch.no_grad():
      net.eval()
      loss_arr = []

      for batch, data in enumerate(loader_test, 1):
          # forward pass
          label = data['label'].to(device)
          input = data['input'].to(device)

          output = net(input)

          # 손실함수 계산하기
          loss = fn_loss(output, label)

          loss_arr += [loss.item()]

          print("TEST: BATCH %04d / %04d | LOSS %.4f" %
                (batch, num_batch_test, np.mean(loss_arr)))

          # Tensorboard 저장하기
          label = fn_tonumpy(label)
          input = fn_tonumpy(fn_denorm(input, mean=0.5, std=0.5))
          output = fn_tonumpy(fn_class(output))

          # 테스트 결과 저장하기
          for j in range(label.shape[0]):
              id = num_batch_test * (batch - 1) + j

              plt.imsave(os.path.join(result_dir, 'png', 'label_%04d.png' % id), label[j].squeeze(), cmap='gray')
              plt.imsave(os.path.join(result_dir, 'png', 'input_%04d.png' % id), input[j].squeeze(), cmap='gray')
              plt.imsave(os.path.join(result_dir, 'png', 'output_%04d.png' % id), output[j].squeeze(), cmap='gray')

              np.save(os.path.join(result_dir, 'numpy', 'label_%04d.npy' % id), label[j].squeeze())
              np.save(os.path.join(result_dir, 'numpy', 'input_%04d.npy' % id), input[j].squeeze())
              np.save(os.path.join(result_dir, 'numpy', 'output_%04d.npy' % id), output[j].squeeze())

print("AVERAGE TEST: BATCH %04d / %04d | LOSS %.4f" %
        (batch, num_batch_test, np.mean(loss_arr)))

# 시각화

##
lst_data = os.listdir(os.path.join(result_dir, 'numpy'))

lst_label = [f for f in lst_data if f.startswith('label')]
lst_input = [f for f in lst_data if f.startswith('input')]
lst_output = [f for f in lst_data if f.startswith('output')]

lst_label.sort()
lst_input.sort()
lst_output.sort()

##
id = 0

label = np.load(os.path.join(result_dir,"numpy", lst_label[id]))
input = np.load(os.path.join(result_dir,"numpy", lst_input[id]))
output = np.load(os.path.join(result_dir,"numpy", lst_output[id]))

## 플롯 그리기
plt.figure(figsize=(8,6))
plt.subplot(131)
plt.imshow(input, cmap='gray')
plt.title('input')

plt.subplot(132)
plt.imshow(label, cmap='gray')
plt.title('label')

plt.subplot(133)
plt.imshow(output, cmap='gray')
plt.title('output')

plt.show()


import json
l=[i for i in range(1,20) if i%2==0]
lJson=json.dumps(l)
print(lJson)

d={"name":"Jack","age":20,"score":(90,80,89)}
dj=json.dumps(d,sort_keys=True)
print(dj)

import pickle
f=open('1.txt','wb')
pickle.dump(d,f)
f.close()


""" References Views """

# Django rest framework
from rest_framework import viewsets, mixins, status
from rest_framework.decorators import action
from rest_framework.response import Response

# Reference models
from reference.models import Reference

# Reference serializers
from reference.serializers import ReferenceModelSerializer

class ReferenceViewSet(viewsets.GenericViewSet,
                       mixins.ListModelMixin,
                       mixins.RetrieveModelMixin):
    """ The viewset of references """

    queryset = Reference.objects.all()
    serializer_class = ReferenceModelSerializer
    lookup_field = 'id'

    def retrieve(self, request, *args, **kwargs):
        """
            Retrieves the model of the reference and if it
            asks for availability, returns a boolean
            :param request: The request donde by the user
            :param args: Some arguments carried on the
            :param kwargs: Some keyword arguments carried on the request
            :return: The retrieved object
        """
        shoe = Reference.objects.get(id=kwargs['id'])
        data = ReferenceModelSerializer(shoe).data
        if request.META.get('QUERY_STRING'):
            quantity = request.META['QUERY_STRING'].split('=')[1]
            if shoe.stock < int(quantity):
                data = {
                    'available': 'no'
                }
            else:
                data = {
                    'available': 'yes'
                }
        return Response(data, status=status.HTTP_200_OK)

    @action(detail=True, methods=['get'])
    def quantity(self, request, *args, **kwargs):
        """
            Retrieves the availability of the reference
            :param request: The request done by the user
            :param args: Some arguments carried on the request
            :param kwargs: Some keyword arguments carried on the request
            :return: A message saying if the object is available or not
        """
        shoe = Reference.objects.get(id=kwargs['id'])
        quantity = request.data['quantity']
        if shoe.stock < int(quantity):
            data = {
                'message': '0'
            }
        else:
            data = {
                'message': '1'
            }
        return Response(data, status=status.HTTP_200_OK)


import requests
from framework.utils.logger import Logger
from framework.constants.methods import Methods

class API():

    def __init__(self, method: Methods, url, data=None):
        self.send_request(method, url, data)

    def send_request(self, method: Methods, url, data=None):
        Logger.info('Отправляю запрос "' + method.value + '" на ' + url)
        self.__url = url
        self.__data = data
        if method == Methods.GET:
            self.__response = requests.get(url=self.__url, data=self.__data)
        if method == Methods.POST:
            self.__response = requests.post(url=self.__url, data=self.__data)
        if method == Methods.PUT:
           self.__response = requests.put(url=self.__url, data=self.__data)
        if method == Methods.DELETE:
            self.__response = requests.put(url=self.__url, data=self.__data)
        if method == Methods.HEAD:
            self.__response = requests.head(url=self.__url, data=self.__data)
        if method == Methods.OPTIONS:
            self.__response = requests.options(url=self.__url, data=self.__data)

    def get_status(self):
        Logger.info('Получаю статус запроса')
        return self.__response.status_code

    def is_json(self):
        Logger.info('Проверка, являются ли полученные данные json')
        res = self.get_headers().get('content-type')
        return 'application/json' in res

    def get_json(self):
        Logger.info('Получаю данные в формате json')
        return self.__response.json()

    def get_headers(self):
        Logger.info('Получение заголовков')
        return self.__response.headers


from bs4 import BeautifulSoup
import urllib.request

url = 'http://planet.mu/releases/'
req = urllib.request.Request(url)
response = urllib.request.urlopen(req)
html = response.read()
soup = BeautifulSoup(html, "lxml")

artist = []
title = []
url = []

for bbr in soup.find_all(class_="box-inner"):
    ra = bbr.find_all(class_="release-artist")
    rt = bbr.find_all(class_="release-title")

    for art in ra:
     artist.append(art.text)
    for tit in rt:
     title.append(tit.text)
     for link in tit.find_all("a"):
         url.append(link['href'])


""" Unit tests """

import json
import unittest
from unittest.mock import patch
from test.unit import mock_response
from requests.auth import HTTPBasicAuth
from requests import Session

from xcputils.ingestion.http import HttpIngestor, HttpMethod, HttpRequest

class TestHttpIngestor(unittest.TestCase):
    """ Test xcputils.ingest.http """

    def __init__(self, *args, **kwargs):
        unittest.TestCase.__init__(self, *args, **kwargs)

    @patch.object(Session, 'get')
    def test_get(self, mock_get):
        """ Test xcputils.ingest.get """

        mock_get.return_value = mock_response(json_data={"ip": "1.2.3.4"})
        result = HttpIngestor(
            http_request=HttpRequest(url="https://mock.com/ip")) \
            .write_to_string()
        self.assertTrue("ip" in result, f"key 'ip' not in: {result}")

    @patch.object(Session, 'get')
    def test_get_html(self, mock_get):
        """ Test xcputils.ingest.get HTML """

        mock_get.return_value = mock_response(content="<!DOCTYPE html>")
        result = HttpIngestor(
            http_request=HttpRequest(url="https://mock.com")) \
            .write_to_string()
        self.assertEqual(result[0:15], "<!DOCTYPE html>")

    @patch.object(Session, 'get')
    def test_auth(self, mock_get):
        """ Test xcputils.ingest.get authentication """

        mock_get.return_value = mock_response(json_data={"authenticated": True})
        result = HttpIngestor(
            http_request=HttpRequest(
                url="https://mock.com/basic-auth",
                auth=HTTPBasicAuth('postman', 'password'))) \
            .write_to_string()
        result = json.loads(result)
        self.assertEqual(result["authenticated"], True, result)

    @patch.object(Session, 'post')
    def test_post(self, mock_post):
        """ Test xcputils.ingest.post """

        data = {"test_key": "test_value"}

        mock_post.return_value = mock_response(json_data={"data" : data})
        result = HttpIngestor(
            http_request=HttpRequest(
                url="https://mock.com/post",
                method=HttpMethod.POST,
                body=data)
            ) \
            .write_to_string()
        result = json.loads(result)
        self.assertEqual(
            result["data"],
            data,
            f"Expected to contain data: {data}: {result}")

if __name__ == '__main__':
    unittest.main()


#!/bin/env python3

import sys
import os
import os.path
import re
import io

from typing import Tuple

LOW_REGISTERS = ["R0", "R1", "R2", "R3", "R4", "R5", "R6", "R7"]
HIGH_REGISTERS = ["R8", "R9", "R10", "R11", "R12"]
# TODO: Only allow high registers if we run out of low registers
ALL_REGISTERS = LOW_REGISTERS
RESERVED_REGISTERS = ALL_REGISTERS[0:1]

RE_PLONK_BEGIN = re.compile("^!\s+PLONK\(([0-9]+)\):")
RE_PLONK_NAMED = re.compile("\s+([a-zA-Z_][a-zA-Z_0-9]*)\s+=\s+([^,\s]+)(,|$)")
RE_PLONK_END = re.compile("^!!$")
RE_PLONK_INSERT = re.compile("^!\s+INSERT\s+(.+)$")

RE_ASM_COMMENT = re.compile("[;@].*$")

RE_ADDRESS_TARGET = re.compile("\[([^]]+)\]")
def match_address_target(value: str) -> Tuple[bool, str]:
	"""Matches [[^]]+]"""
	match = RE_ADDRESS_TARGET.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_ADDRESS_CONSTANT = re.compile("=([a-zA-Z_0-9]+)")
def match_address_constant(value: str) -> Tuple[bool, str]:
	"""Matches =ADDR"""
	match = RE_ADDRESS_CONSTANT.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_NAMED_REGISTER = re.compile("\$([a-zA-Z_][a-zA-Z_0-9]+)")
def match_named_register(value: str) -> Tuple[bool, str]:
	"""Matches $[a-zA-Z_][a-zA-Z_0-9]+"""
	match = RE_NAMED_REGISTER.match(value)

	if match is not None:
		return True, match.group(1)

	return False, None

RE_ARGUMENT_REGISTER = re.compile("\$A([0-9]+)")
def match_argument_register(value: str) -> Tuple[bool, int]:
	"""Matches $A[0-9]+"""
	match = RE_ARGUMENT_REGISTER.match(value)

	if match is not None:
		return True, int(match.group(1))

	return False, None

RE_VARIABLE_REGISTER = re.compile("\$([0-9]+)")
def match_variable_register(value: str) -> Tuple[bool, int]:
	"""Matches $[0-9]+"""
	match = RE_VARIABLE_REGISTER.match(value)

	if match is not None:
		return True, int(match.group(1))

	return False, None

RE_RAW_REGISTER = re.compile("R([0-9]|1[0-2])")
def match_raw_register(value: str) -> Tuple[bool, str]:
	"""Matches R[0-9]|1[0-2]"""
	match = RE_RAW_REGISTER.match(value)

	if match is not None:
		return True, match.group(0)

	return False, None

RE_DEC_NUMBER_LITERAL = re.compile("([0-9_]+)")
RE_HEX_NUMBER_LITERAL = re.compile("(0x[a-fA-F0-9_]+)")
RE_BINARY_NUMBER_LITERAL = re.compile("(0b[0-1_]+)")
def match_number(value: str) -> Tuple[bool, int]:
	match = RE_BINARY_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 2)

		return True, number

	match = RE_HEX_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 16)

		return True, number

	match = RE_DEC_NUMBER_LITERAL.match(value)
	if match is not None:
		number = int(match.group(1), 10)

		return True, number

	return False, None

RE_STRING_ADDRESS = re.compile("""^("([^"]+)"|'([^']+)')""")
def match_string(value: str) -> Tuple[bool, str]:
	match = RE_STRING_ADDRESS.match(value)
	if match is not None:
		return True, match.group(2)

	return False, None

def code_line(value: str) -> str:
	return f"\t{value}\n"

class PlonkVariable:
	"""Represents a variable with optional initialization code and an assigned register."""
	name: str
	register: str
	code: str

	needs_free: bool

	def __init__(self, name, register, code = None, needs_free = False):
		self.name = name
		self.register = register
		self.code = code

		self.needs_free = needs_free

	def __str__(self):
		return f"{self.name}@{self.register}"

	def __repr__(self):
		return f"{self.__str__()} at 0x{id(self):X}"

class PlonkRegisterContext:
	"""Represents registers available throughout one PLONK context."""

	# Registers with assigned names
	named_registers: dict
	# Registers used for arguments
	argument_registers: list
	# Registers free to be allocated
	variable_registers: dict

	def __init__(
		self,
		arguments = 0,
		named_registers = {},
		available_registers = ALL_REGISTERS
	):
		self.named_registers = {}
		self.argument_registers = []
		self.variable_registers = {}

		registers = available_registers.copy()

		for x in range(arguments):
			self.argument_registers.append(
				PlonkVariable(
					f"$A{x}",
					registers.pop(0)
				)
			)

		for key in named_registers:
			reg = registers.pop()
			val = named_registers[key]

			code = ""
			if type(val) is str:
				code += code_line(f"LDR {reg}, ={val}")
			elif type(val) is int:
				code += code_line(f"LDR {reg}, =0x{val:X}")
			else:
				raise RuntimeError(f"Invalid constant value type {val}")

			self.named_registers[key] = PlonkVariable(
				key,
				reg,
				code
			)

		for register in registers:
			if register not in RESERVED_REGISTERS:
				self.variable_registers[register] = { "free": True }

	def __str__(self):
		return f"PlonkRegisterContext(named = {self.named_registers}, args = {self.argument_registers}, variables = {self.variable_registers})"

	def generate_initialization_code(self):
		code = ""
		for key in self.named_registers:
			code += self.named_registers[key].code

		return code

	def get_named(self, name: str) -> PlonkVariable:
		if not name in self.named_registers:
			return None

		return self.named_registers[name]

	def get_argument(self, index: int) -> PlonkVariable:
		if len(self.argument_registers) <= index:
			return None

		return self.argument_registers[index]

	def allocate_variable(self) -> PlonkVariable:
		for register in self.variable_registers:
			if self.variable_registers[register]["free"]:
				self.variable_registers[register]["free"] = False

				return PlonkVariable(None, register, None, needs_free = True)

		raise RuntimeError(f"Could not allocate register: {self}")

	def check_register(self, register: str):
		if register in RESERVED_REGISTERS:
			return True

		if not register in self.variable_registers:
			return False

		return self.variable_registers[register]["free"]

	def free_variable(self, variable: PlonkVariable):
		if not variable.register in self.variable_registers:
			raise RuntimeError(f"Attempted to free non-existent register {variable}")

		if self.variable_registers[variable.register]["free"]:
			raise RuntimeError(f"Attempted to free already free register {variable}")

		self.variable_registers[variable.register]["free"] = True

class PlonkRegisterStack:
	"""Represents a stack of PlonkVariables that can be pushed and poped with freeing when necessarry."""
	stack: list

	def __init__(self):
		self.stack = []

	def __str__(self):
		return f"{self.stack}"

	def push(self, variable: PlonkVariable):
		self.stack.append(variable)

	def get(self, index: int) -> PlonkVariable:
		if index < len(self.stack):
			return self.stack[index]
		return None

	def get_named(self, name: str) -> PlonkVariable:
		for var in self.stack:
			if var.name == name:
				return var

		return None

	def pop(self, context: PlonkRegisterContext) -> PlonkVariable:
		variable = self.stack.pop()

		if variable.needs_free:
			context.free_variable(variable)

		return variable

class PlonkFileState:
	"""Represents the Plonk state throughout a source file."""

	plonking: bool = False
	context: PlonkRegisterContext = None

	# variables addressable with $[0-9]+ syntax
	scoped_variables: PlonkRegisterStack
	# variables allocated but not addresable
	scoped_internal: PlonkRegisterStack

	def __init__(self):
		self.scoped_variables = PlonkRegisterStack()
		self.scoped_internal = PlonkRegisterStack()

	def process_line(self, line: str):
		# check for plonk beginning
		begin_match = RE_PLONK_BEGIN.match(line)
		if begin_match is not None:
			if self.plonking:
				raise RuntimeError("Can only plonk in one dimension at once")

			argc = int(begin_match.group(1))
			named_registers = {}
			for named_match in RE_PLONK_NAMED.finditer(
				line,
				begin_match.span()[1]
			):
				key = named_match.group(1)
				value = named_match.group(2)

				matches, address = match_address_constant(value)
				if not matches:
					matches, number = match_number(value)
					if not matches:
						raise RuntimeError(f"Could not parse {value} as number nor address constant")
					else:
						value = number
				else:
					value = address

				named_registers[key] = value

			self.plonking = True
			self.context = PlonkRegisterContext(
				arguments = argc,
				named_registers = named_registers
			)

			return self.context.generate_initialization_code()

		# check for plonk insert
		insert_match = RE_PLONK_INSERT.match(line)
		if insert_match is not None:
			path = insert_match.group(1)
			print(f"Inserting \"{path}\"", file = sys.stderr)

			outstream = io.StringIO("")
			with open(path, "r") as file:
				process_file(file, outstream)

			return outstream.getvalue()

		# check for plonk ending
		end_match = RE_PLONK_END.match(line)
		if end_match is not None:
			if not self.plonking:
				raise RuntimeError("Cannot plonk in negative")

			self.plonking = False
			self.context = None

			return None

		# process the line specially when plonking
		if self.plonking:
			return self.process_line_plonking(line)

		# if not plonking then just return the current line
		return line

	def process_line_plonking(self, line: str):
		# strip comments and whitespace
		stripped = RE_ASM_COMMENT.sub("", line).strip()
		# ignore empty lines
		if stripped == '':
			return line

		first_split = stripped.split(" ", maxsplit = 1)
		word = first_split[0].lower()
		rest = None
		if len(first_split) > 1:
			rest = first_split[1]

		# self-contained
		if word == "store":
			return self.process_store(rest)

		if word == "call":
			return self.process_call(rest)
		elif word == "calln":
			return self.process_call(rest, nested = True)

		if word == "ifeq":
			return self.process_if(rest)
		elif word == "ifneq":
			return self.process_if(rest, branch = "BNE")
		elif word == "ifgt":
			return self.process_if(rest, branch = "BGT")
		elif word == "iftestone":
			return self.process_if(rest, test = "TST", branch = "BNE")
		elif word == "iftestzero":
			return self.process_if(rest, test = "TST", branch = "BEQ")

		# scoping
		if word == "read":
			return self.process_read(rest)
		elif word == "write":
			return self.process_write()
		elif word == "discard":
			return self.process_discard()

		if word == "alloc":
			return self.process_alloc(rest)
		elif word == "free":
			return self.process_free(rest)

		# other

		return self.process_other(line)

	def end(self):
		return None

	## MATCHERS ##

	def match_and_scope(self, value: str, *matchers) -> PlonkVariable:
		"""Returns the first match and scopes it into `self.scoped_internal`."""
		for matcher in matchers:
			match = matcher(value)
			if match is not None:
				self.scoped_internal.push(match)
				return match

		return None

	def match_and_code(self, value: str, target: str, *matchers) -> str:
		"""Returns the first match and creates a code line moving the value of the match into the target register"""
		for matcher in matchers:
			match = matcher(value)
			if match is not None:
				if match.needs_free:
					self.context.free_variable(match)

				if match.code is not None:
					return match.code.replace(match.register, target, 1)
				else:
					return code_line(f"MOV {target}, {match.register}")

		return None

	def match_address_constant(self, value: str) -> PlonkVariable:
		matches, address = match_address_constant(value)
		if matches:
			var = self.context.allocate_variable()
			return PlonkVariable(
				f"={address}",
				var.register,
				code_line(f"LDR {var.register}, ={address}"),
				needs_free = True
			)

		return None

	def match_argument_register(self, value: str) -> PlonkVariable:
		matches, index = match_argument_register(value)
		if matches:
			arg = self.context.get_argument(index)
			if arg is None:
				raise RuntimeError(f"Argument $A{index} does not exist")
			return PlonkVariable(f"$A{index}", arg.register)

		return None

	def match_variable_register(self, value: str) -> PlonkVariable:
		matches, index = match_variable_register(value)
		if matches:
			var = self.scoped_variables.get(index)
			if var is None:
				raise RuntimeError(f"Variable ${index} does not exist: {self.scoped_variables}")
			return PlonkVariable(f"${index}", var.register)

		return None

	def match_named_register(self, value: str) -> PlonkVariable:
		matches, name = match_named_register(value)
		if matches:
			named = self.scoped_variables.get_named(name)
			if named is None:
				named = self.context.get_named(name)
				if named is None:
					raise RuntimeError(f"Named register ${name} does not exist")
			return PlonkVariable(name, named.register)

		return None

	def match_raw_register(self, value: str) -> PlonkVariable:
		matches, register = match_raw_register(value)
		if matches:
			return PlonkVariable(register, register)

		return None

	def match_number(self, value: str) -> PlonkVariable:
		matches, number = match_number(value)
		if matches:
			var = self.context.allocate_variable()
			return PlonkVariable(
				f"#0x{number:X}",
				var.register,
				code_line(f"LDR {var.register}, =0x{number:X}"),
				needs_free = True
			)

		return None

	## SELF-CONTAINED ##

	def process_store(self, rest: str):
		"""
		store [=ADDR|$N|$AN|$NAMED|RAW] NUM|$N|$AN|$NAMED|RAW

		store [=ADDR] _ ->
			LDR RI, =ADDR
			STR _ ,[RI]

		store [$N|$AN|$NAMED|RAW] _ ->
			STR _, [$N|$AN|$NAMED|RAW]

		store _ NUM ->
			MOV RI, #NUM
			STR RI, _

		store _ $N|$AN|$NAMED|RAW ->
			STR $N|$AN|$NAMED|RAW, _
		"""
		split = rest.split(" ")
		target_argument_matches, target_argument = match_address_target(split[0])
		if not target_argument_matches:
			raise RuntimeError(f"First argument of store must match pattern `[[^]]+]`: {split[0]}")
		source_argument = split[1]

		code = ""

		target = self.match_and_scope(
			target_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if target.code is not None:
			code += target.code

		source = self.match_and_scope(
			source_argument,
			self.match_number,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if source.code is not None:
			code += source.code

		code += code_line(f"STR {source.register}, [{target.register}]")

		# pop both source and target
		self.scoped_internal.pop(self.context)
		self.scoped_internal.pop(self.context)

		return code

	def process_call(self, rest: str, nested = False):
		"""
		call|calln LABEL =ADDR|NUMBER|$N|$AN|$NAMED|RAW ...

		call _ =ADDR ->
			LDR R0, =ADDR
			BL _

		call _ NUMBER ->
			MOV R0, #NUMBER
			BL _

		call _ $N|$AN|$NAMED ->
			MOV R0, $N|$AN|$NAMED|RAW
			BL _

		calln _ _ ->
			_
			PUSH { LR }
			BL _
			POP { LR }
		"""
		split = rest.split(" ")
		label = split[0]
		arguments = split[1:]

		overlapping_registers = []
		if len(arguments) > 0:
			# overlapping with the current context arguments
			overlapping_registers = ALL_REGISTERS[
				:min(len(arguments), len(self.context.argument_registers))
			]

			# overlapping with the allocated ones or the constant ones
			remaining_registers = ALL_REGISTERS[len(overlapping_registers) : len(arguments)]
			for reg in remaining_registers:
				if not self.context.check_register(reg):
					overlapping_registers.append(reg)

		code = ""

		if len(overlapping_registers) > 0:
			regs = ", ".join(overlapping_registers)
			code += code_line(f"PUSH {{ {regs} }}")

		for x in range(len(arguments)):
			code += self.match_and_code(
				arguments[x],
				f"R{x}",
				self.match_number,
				self.match_address_constant,
				self.match_argument_register,
				self.match_variable_register,
				self.match_named_register,
				self.match_raw_register
			)

		if nested:
			code += code_line("PUSH { LR }")
		code += code_line(f"BL {label}")
		if nested:
			code += code_line("POP { LR }")

		if len(overlapping_registers) > 0:
			regs = ", ".join(reversed(overlapping_registers))
			code += code_line(f"POP {{ {regs} }}")

		return code

	def process_if(self, rest: str, test = "CMP", branch = "BEQ"):
		"""
		ifeq|ifneq|ifgt|iftestone|iftestzero =ADDR|$N|$AN|$NAMED|RAW =ADDR|NUM|$N|$AN|$NAMED|RAW LABEL

		ifeq =ADDR _ _ ->
			LDR RI, =ADDR
			CMP RI, _
			BEQ _

		ifeq $N|$AN|$NAMED|RAW _ _ ->
			CMP $N|$AN|$NAMED|RAW _
			BEQ _

		ifeq _ =ADDR _ ->
			LDR RI, =ADDR
			CMP _, RI
			BEQ _

		ifeq _ NUM _ ->
			CMP _, #NUM
			BEQ _

		ifeq _ $N|$AN|$NAMED|RAW _ ->
			CMP _, $N|$AN|$NAMED|RAW
			BEQ _

		ifeq _, _, LABEL ->
			CMP _, _
			BEQ LABEL

		ifneq _, _, LABEL ->
			CMP _, _
			BNE LABEL

		ifgt _, _, LABEL ->
			CMP _, _
			BGT LABEL

		iftestone _, _, LABEL ->
			TST _, _
			BNZ LABEL

		iftestzero _, _, LABEL ->
			TST _, _
			BEQ LABEL
		"""
		split = rest.split(" ")
		left_argument = split[0]
		right_argument = split[1]
		label = split[2]

		code = ""

		left = self.match_and_scope(
			left_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if left.code is not None:
			code += left.code

		right_variable = self.match_and_scope(
			right_argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if right_variable is not None:
			if right_variable.code is not None:
				code += right_variable.code

			code += code_line(f"{test} {left.register}, {right_variable.register}")

			self.scoped_internal.pop(self.context)
		else:
			matches, number = match_number(right_argument)
			if not matches:
				raise RuntimeError(f"Argument {right_argument} does not match any known right side")

			code += code_line(f"{test} {left.register}, #0x{number:X}")

		code += code_line(f"{branch} {label}")

		# pop left
		self.scoped_internal.pop(self.context)

		return code

	## SCOPED ##

	def process_alloc(self, rest: str):
		"""
		alloc NAME =ADDR|NUM|$N|$AN|$NAMED|RAW

		Allocates a variable addressable with `$[0-9]+` and with an optional name syntax.
		"""
		name = None
		value_argument = None
		if rest is not None:
			split = rest.split(" ")
			if RE_NAMED_REGISTER.match(f"${split[0]}") is not None:
				name = split[0]

			if len(split) > 1:
				value_argument = split[1]

		var = self.context.allocate_variable()
		var.name = name
		self.scoped_variables.push(var)

		code = None
		if value_argument is not None:
			code = self.match_and_code(
				value_argument,
				var.register,
				self.match_address_constant,
				self.match_number,
				self.match_argument_register,
				self.match_variable_register,
				self.match_named_register,
				self.match_raw_register
			)

		return code

	def process_free(self, rest: str):
		"""
		free N

		Frees the last `N` addressable variables.
		"""
		count = int(rest or 1)

		for x in range(count):
			self.scoped_variables.pop(self.context)

	def process_read(self, rest: str):
		"""
		Allocates a new addressable variable and loads the value of `[=ADDR|$N|$AN|$NAMED|RAW]` into it.

		read [=ADDR|$N|$AN|$NAMED|RAW] NAME

		read [=ADDR] ->
			LDR RI, =ADDR
			LDR RV, [RI]

		read [$N|$AN|$NAMED|RAW]
			LDR RV, [$N|$AN|$NAMED|RAW]
		"""
		split = rest.split(" ")
		argument_matches, argument = match_address_target(split[0])
		if not argument_matches:
			raise RuntimeError(f"Argument of read must match pattern `[[^]]+]`: {rest}")

		name = None
		if len(split) > 1 and RE_NAMED_REGISTER.match(f"${split[1]}") is not None:
			name = split[1]

		code = ""

		address = self.match_and_scope(
			argument,
			self.match_address_constant,
			self.match_argument_register,
			self.match_variable_register,
			self.match_named_register,
			self.match_raw_register
		)
		if address.code is not None:
			code += address.code

		variable = self.context.allocate_variable()
		variable.name = name
		self.scoped_variables.push(variable)

		code += code_line(f"LDR {variable.register}, [{address.register}]")

		return code

	def process_write(self):
		"""
		Frees the last allocated addressable variabled and writes its value into the address specified by previous `read`.

		write

		write -> ; with previous `read =ADDR`
			STR RV, [RI]

		write -> ; with previous `read $N|$AN|$NAMED|RAW`
			STR RV, [$N|$AN|$NAMED|RAW]
		"""
		reg = self.scoped_variables.pop(self.context)
		addr = self.scoped_internal.pop(self.context)

		return code_line(f"STR {reg.register}, [{addr.register}]")

	def process_discard(self):
		"""
		Frees the last allocated addressable variable and discards its value. This can be used to end a `read` block without emitting a store.

		discard
		"""
		reg = self.scoped_variables.pop(self.context)
		addr = self.scoped_internal.pop(self.context)

		return None

	## OTHER ##

	def process_other(self, line: str):
		def transform_with(line: str, regex, callback):
			transformed_line = ""
			last_pos = 0

			for match in regex.finditer(line):
				span = match.span()

				transformed_line += line[last_pos:span[0]]
				transformed_line += callback(match)

				last_pos = span[1]

			transformed_line += line[last_pos:]

			return transformed_line

		line = transform_with(
			line,
			RE_BINARY_NUMBER_LITERAL,
			lambda match: f"0x{int(match.group(1), 2):X}"
		)

		# $0 -> RN
		line = transform_with(
			line,
			RE_VARIABLE_REGISTER,
			lambda match: self.scoped_variables.get(int(match.group(1))).register
		)

		# $A0 -> RN
		line = transform_with(
			line,
			RE_ARGUMENT_REGISTER,
			lambda match: self.context.argument_registers[int(match.group(1))].register
		)

		# $NAMED -> RN
		line = transform_with(
			line,
			RE_NAMED_REGISTER,
			lambda match: (
				self.scoped_variables.get_named(match.group(1))
				or
				self.context.get_named(match.group(1))
			).register
		)

		return line

def process_file(stream, outstream):
	context = PlonkFileState()
	line_number = 0

	for line in stream:
		line_number += 1
		try:
			output = context.process_line(line)
		except BaseException as e:
			raise RuntimeError(
				f""" Cannot process line #{line_number}
	{line.rstrip()}

	Reason: {e}
				"""
			)
		if output is not None:
			print(output, file = outstream, end = "")

	final_output = context.end()
	if final_output is not None:
		print(final_output, file = outstream, end = "")

def process_folder(folder):
	files = os.listdir(folder)

	for file in files:
		if os.path.isfile(file):
			base, ext = os.path.splitext(file)
			if ext in [".plonk"]:
				out_path = os.path.join(
					os.path.dirname(folder),
					base + ".s"
				)

				print(f"## Processing {file} > {out_path}")

				with open(file, "r") as in_file:
					with open(out_path, "w") as out_file:
						process_file(in_file, out_file)

def main():
	if len(sys.argv) > 1:
		path = sys.argv[1]
		if os.path.isdir(path):
			process_folder(path)
		else:
			with open(path, "r") as file:
				process_file(file, sys.stdout)
	else:
		process_file(sys.stdin, sys.stdout)

main()


# -*- coding: utf-8 -*-
# File   : prepare.py
# Author : Kai Ao
# Email  : capino627@163.com
# Date   : 2020/12/12 11:11
#
# This file is part of Rotation-Decoupled Detector.
# https://github.com/Capino512/pytorch-rotation-decoupled-detector
# Distributed under MIT License.

import sys

sys.path.append('.')

import os
import json
import numpy as np

from bs4 import BeautifulSoup as bs
from utils.box.bbox_np import xywha2xy4

def xml2json(dir_xml, dir_json):
    os.makedirs(dir_json, exist_ok=True)
    for xml in os.listdir(dir_xml):
        objs = []
        name = os.path.splitext(xml)[0]
        for obj in bs(open(os.path.join(dir_xml, xml)), "html.parser").findAll('hrsc_object'):
            xywha = []
            xywha.append(float(obj.select_one('mbox_cx').text))
            xywha.append(float(obj.select_one('mbox_cy').text))
            xywha.append(float(obj.select_one('mbox_w').text))
            xywha.append(float(obj.select_one('mbox_h').text))
            xywha.append(np.rad2deg(float(obj.select_one('mbox_ang').text)))
            obj = dict()
            obj['name'] = 'ship'
            obj['bbox'] = xywha2xy4(xywha).tolist()
            objs.append(obj)
        if objs:
            json.dump(objs, open(os.path.join(dir_json, name + '.json'), 'wt'), indent=2)

def main():
    # (1)
    dir_xml = os.path.join(dir_dataset, 'labelXml')
    out_dir_json = os.path.join(dir_dataset, 'annotations')
    xml2json(dir_xml, out_dir_json)

    # (2)
    out_dir = os.path.join(dir_dataset, 'image-sets')
    os.makedirs(out_dir, exist_ok=True)
    for image_set in ['trainval', 'test']:
        pairs = []
        with open(os.path.join(dir_dataset, 'ImageSets', f'{image_set}.txt')) as f:
            for line in f.readlines():
                line = line.strip()
                img = os.path.join('images', f'{line}.bmp')
                anno = os.path.join('annotations', f'{line}.json')
                if not os.path.exists(os.path.join(dir_dataset, anno)):
                    anno = None
                pairs.append([img, anno])
        json.dump(pairs, open(os.path.join(out_dir, f'{image_set}.json'), 'wt'), indent=2)

if __name__ == '__main__':

    # directory hierarchy

    # root/images/100000001.bmp
    # ----/images/...

    # root/labelXml/100000001.xml
    # ----/labelXml/...

    # (1) convert annotation files
    # (2) generate image-set files

    dir_dataset = '<replace with your local path>'

    main()


#coding:utf-8

import os
import sys
import logging
import argparse
import tensorflow as tf

from dataset import Dataset
from multi_gpus import MultiGpusCRNN
import utils

## logging config
logging.basicConfig(
  level = logging.INFO,
  format = '[%(levelname)-8s %(asctime)-11s L%(lineno)-4d] %(message)s',
  datefmt = '%m-%d %H:%M')

def save(saver, sess, logdir, step):
    model_name = 'model.ckpt'
    checkpoint_path = os.path.join(logdir, model_name)
    logging.info('Storing checkpoint {} to {}.'.format(step, logdir))
    sys.stdout.flush()
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    saver.save(sess, checkpoint_path, global_step=step)

def main(args):
  ## build model
  optimizer = tf.train.AdamOptimizer(1e-4)
  mg_crnn = MultiGpusCRNN(args.n_gpus, optimizer,
      batch_size=args.batch_size/args.n_gpus,
      with_spatial_transform=args.with_spatial_transform)
  objective = mg_crnn.objective
  loss = mg_crnn.loss
  error = mg_crnn.error
  predict = mg_crnn.predict
  probability = mg_crnn.log_prob

  ## build dataset
  dataset = Dataset(args.img_path_prefix, args.img_list_file, args.label_list_file,
      args.lexicon_file, val_ratio=args.val_ratio, test_ratio=args.test_ratio)
  n_train_batches = dataset.n_train_samples / args.batch_size
  n_val_batches = dataset.n_val_samples / args.batch_size
  n_test_batches = dataset.n_test_samples / args.batch_size

  ## start session, to fit the model
  config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
  with tf.Session(config=config) as sess:
    ## init parameters
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver(var_list=tf.trainable_variables(),
        max_to_keep=args.max_checkpoints)

    for i in xrange(n_train_batches * args.n_epochs):
      # {m0.image, m0.label, m1.image, m1.label, ...}
      batches = [dataset.next_train_batch(args.batch_size/args.n_gpus)
          for j in range(args.n_gpus)]
      feed_dict = mg_crnn.feed_dict(batches)
      result = sess.run([objective, loss, error], feed_dict=feed_dict)

      if (i+1) % args.validate_every == 0:
        logging.info('step {}, epoch {}, loss {}, train err {}'.format(
            i+1, batches[-1].epoch_id, result[1], result[2]))
        ## evaluate on dev set
        if n_val_batches > 0:
          val_batches = [dataset.next_val_batch(args.batch_size/args.n_gpus)
              for j in range(args.n_gpus)]
          feed_dict = mg_crnn.feed_dict(val_batches)
          val_result = sess.run([predict, error], feed_dict=feed_dict)
          ## val_result[pred/err][gpu][0]
          pred = [sample for j in range(args.n_gpus)
              for sample in utils.sparse2dense((val_result[0][j].indices,
              val_result[0][j].values, val_result[0][j].dense_shape)) ]
          label = [sample for j in range(args.n_gpus)
              for sample in utils.sparse2dense(val_batches[j].labels) ]
          n_exact_matches = sum([1 for p, t in zip(pred, label) if p == t])
          accuracy = n_exact_matches / float(args.batch_size)
          logging.info('val err {}, acc {}'.format(val_result[1], accuracy))
          logging.info('val last sample pred:  {}'.format(pred[-1]))
          logging.info('val last sample label: {}'.format(label[-1]))

      if (i+1) % args.checkpoint_every == 0:
        save(saver, sess, args.checkpoint_dir, i)

    ## evaluate on test set
    if n_test_batches > 0:
      test_err = 0
      for i in xrange(n_test_batches):
        batches = [dataset.next_test_batch(args.batch_size/arg.n_gpus)
            for j in range(args.n_gpus)]
        feed_dict = mg_crnn.feed_dict(batches)
        batch_err = sess.run([error], feed_dict=feed_dict)
        test_err += batch_err[0]
      test_err /= n_test_batches
      logging.info('Test error {}'.format(test_err))

if __name__ == '__main__':
  def _get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--img_path_prefix', default='./data/image',
        help='Prefix of image path')
    parser.add_argument('--img_list_file', default='./data/image_list.txt',
        help="Contain images' path")
    parser.add_argument('--label_list_file', default='./data/label_list.txt',
        help="ontain images' label")
    parser.add_argument('--lexicon_file', default='./data/lexicon.txt',
        help='Vocabulary, one word per line')
    parser.add_argument('--with_spatial_transform', type=bool, default=False)
    parser.add_argument('--n_gpus', type=int, default=1)
    parser.add_argument('--batch_size', type=int, default=128)
    parser.add_argument('--n_epochs', type=int, default=20)
    parser.add_argument('--val_ratio', type=float, default=0.2)
    parser.add_argument('--test_ratio', type=float, default=0.1)
    parser.add_argument('--validate_every', type=int, default=100)
    parser.add_argument('--checkpoint_every', type=int, default=2000)
    parser.add_argument('--checkpoint_dir', default='./checkpoint')
    parser.add_argument('--max_checkpoints', type=int, default=10)
    args = parser.parse_args()
    return args
  args = _get_args()

  main(args)


from typing import (
    Any,
    Dict,
    List,
    Optional,
)

from easypost.easypost_object import convert_to_easypost_object
from easypost.models import Tracker
from easypost.requestor import (
    RequestMethod,
    Requestor,
)
from easypost.services.base_service import BaseService

class TrackerService(BaseService):
    def __init__(self, client):
        self._client = client
        self._model_class = Tracker.__name__

    def create(self, **params) -> Tracker:
        """Create a Tracker."""
        return self._create_resource(self._model_class, **params)

    def all(self, **params) -> Dict[str, Any]:
        """Retrieve a list of Trackers."""
        url = self._class_url(self._model_class)

        response = Requestor(self._client).request(method=RequestMethod.GET, url=url, params=params)
        response["tracking_code"] = params.get("tracking_code")
        response["carrier"] = params.get("carrier")

        return convert_to_easypost_object(response=response)

    def retrieve(self, id: str) -> Tracker:
        """Retrieve a Tracker."""
        return self._retrieve_resource(self._model_class, id)

    def get_next_page(
        self,
        trackers: Dict[str, Any],
        page_size: int,
        optional_params: Optional[Dict[str, Any]] = None,
    ) -> Dict[str, Any]:
        """Retrieve the next page of the list Tracker response."""
        optional_params = {
            "tracking_code": trackers.get("tracking_code"),
            "carrier": trackers.get("carrier"),
        }

        return self._get_next_page_resources(self._model_class, trackers, page_size, optional_params)

    def create_list(self, trackers: List[Dict[str, Any]]) -> None:
        """Create a list of Trackers."""
        url = f"{self._class_url(self._model_class)}/create_list"
        wrapped_params = {"trackers": trackers}

        Requestor(self._client).request(method=RequestMethod.POST, url=url, params=wrapped_params)


# -*- coding: utf-8 -*-
"""Console script for flipdot_controller."""
import logging
import sys
import time

import click
from serial import Serial

from flipdot_controller.config import ConfigParser
from flipdot_controller.controller import FlipdotController
from flipdot_controller.server import Server

@click.command()
@click.option('--config',
              type=click.Path(exists=True),
              required=True,
              help="TOML config file")
def main(config):
    """Console script for flipdot_controller."""
    logging.basicConfig(level=logging.DEBUG)
    # Read the config
    parser = ConfigParser.create(config)
    # Create controller from config
    with Serial(parser.basic_config['serial_port']) as ser, FlipdotController(
            port=ser, signs=parser.signs_config,
            pins=parser.pin_config) as controller:
        server = Server(controller, port=parser.basic_config['grpc_port'])
        try:
            server.start()
            while True:
                time.sleep(1)
        except KeyboardInterrupt:
            server.stop()

if __name__ == "__main__":
    sys.exit(main())  # pylint: disable=E1120


import os
import os.path as osp

import random
import numpy as np
from PIL import Image, ImageFilter

import torch
from torchvision import transforms
import torchvision.transforms.functional as transforms_f

import cv2
import matplotlib.pyplot as plt

#Ref: https://github.com/rwightman/pytorch-image-models/blob/master/timm/data/transforms.py
try:
    from torchvision.transforms.functional import InterpolationMode
    map_nearest = InterpolationMode.NEAREST
    map_bilinear = InterpolationMode.BILINEAR
    map_bicubic = InterpolationMode.BICUBIC
except ImportError:
    map_nearest = Image.NEAREST
    map_bilinear = Image.BILINEAR
    map_bicubic = Image.BICUBIC

# #Ref: https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py
def CutOut(image, label, pseudo_label = None, n_holes = 1, sz_holes = 50, img_pad = 0, label_pad = 255):
        image = np.asarray(image)
        label = np.asarray(label)

        if pseudo_label is not None:
            pseudo_label = np.asarray(pseudo_label)

        w = image.shape[0]
        h = image.shape[1]

        for n in range(n_holes):
            y = np.random.randint(h)
            x = np.random.randint(w)

            x1 = np.clip(y - sz_holes // 2, 0, h)
            x2 = np.clip(y + sz_holes // 2, 0, h)
            y1 = np.clip(x - sz_holes // 2, 0, w)
            y2 = np.clip(x + sz_holes // 2, 0, w)

            image[y1: y2, x1: x2] = img_pad
            label[y1: y2, x1: x2] = label_pad
            if pseudo_label is not None:
                pseudo_label[y1: y2, x1: x2] = img_pad

        image = Image.fromarray(image.astype(np.uint8))
        label = Image.fromarray(label.astype(np.uint8))
        if pseudo_label is not None:
            pseudo_label = Image.fromarray(pseudo_label.astype(np.uint8))

        if pseudo_label is not None:
            return image, label, pseudo_label
        else:
            return image, label, None

def do_transforms(image,
                  label,
                  pseudo_label = None,
                  ratio_range = (1.0, 1.0),
                  crop_size = (224, 224),
                  use_augmentation_weak=True,
                  use_augmentation_strong=True,
                  use_augmentation_cutout=True):

    og_w, og_h = image.size

    scale_ratio = random.uniform(ratio_range[0], ratio_range[1])

    resized_image_size = (int(og_h * scale_ratio), int(og_w * scale_ratio))
    image = transforms_f.resize(image, resized_image_size, map_bilinear)
    label = transforms_f.resize(label, resized_image_size, map_nearest)

    if pseudo_label is not None:
        pseudo_label  = transforms_f.resize(pseudo_label, resized_image_size, map_nearest)

    if crop_size[0] > resized_image_size[0] or crop_size[1] > resized_image_size[1]:

        pad_lr = max(crop_size[1] - resized_image_size[1], 0)
        pad_hw = max(crop_size[0] - resized_image_size[0], 0)
        right_pad, bottom_pad = int(pad_lr * 0.5), int(pad_hw * 0.5)

        image = transforms_f.pad(image, padding=(pad_lr - right_pad, pad_hw - bottom_pad, right_pad, bottom_pad), fill=0)
        label = transforms_f.pad(label, padding=(pad_lr - right_pad, pad_hw - bottom_pad, right_pad, bottom_pad), fill=255)

        if pseudo_label is not None:
            pseudo_label = transforms_f.pad(pseudo_label, padding=(pad_lr - right_pad, pad_hw - bottom_pad, right_pad, bottom_pad), fill=255)

    if ratio_range[0] != 1.0:
        # Random Cropping
        i, j, h, w = transforms.RandomCrop.get_params(image, output_size=crop_size)
        image = transforms_f.crop(image, i, j, h, w)
        label = transforms_f.crop(label, i, j, h, w)
        if pseudo_label is not None:
            pseudo_label = transforms_f.crop(pseudo_label, i, j, h, w)
    else:
        image = transforms_f.center_crop(image, crop_size)
        label = transforms_f.center_crop(label, crop_size)
        if pseudo_label is not None:
            pseudo_label = transforms_f.center_crop(pseudo_label, crop_size)

    if use_augmentation_weak:
        if torch.rand(1) > 0.5:
            image = transforms_f.hflip(image)
            label = transforms_f.hflip(label)
            if pseudo_label is not None:
                pseudo_label = transforms_f.hflip(pseudo_label)

    if use_augmentation_strong:
        if torch.rand(1) > 0.5:
            sigma = random.uniform(0.25, 1.25)
            image = image.filter(ImageFilter.GaussianBlur(radius=sigma))

        elif torch.rand(1) > 0.5:
            color_transforms_params = transforms.ColorJitter(brightness=0.8,
                                                             contrast=0.8,
                                                             saturation=0.8,
                                                             hue=0.2
                                                             )
            image = color_transforms_params(image)

            if torch.rand(1) > 0.25:
                image = transforms.Grayscale(num_output_channels=3)(image)

    if use_augmentation_cutout:
        if torch.rand(1) > 0.5:
            image, label, pseudo_label = CutOut(image, label, pseudo_label)

    return image, label, pseudo_label


from asyncio.windows_events import NULL
import csv
import cantools
import struct
import json

dbcMessages = []
dbcSignals = []

class PacketManager:
    def createPacketDict(packets_file):
        if(packets_file.endswith('.csv')):
            packet_dict = {}
            packets = []
            fieldNames = ['Timestamp', 'Type', 'ID', 'Data']
            with open(packets_file, newline='\n') as csvfile:
                reader = csv.DictReader(csvfile, fieldnames=fieldNames, delimiter=';')
                i = 0
                for row in reader:
                    #print(row)
                    packets.append(row)
            #print(packets)
            return packets
        elif(packets_file.endswith('.log')):
            packet_dict = {}

    def loadDBCFile(filename):
        print("DBC loading")
        db = cantools.database.load_file(filename)
        i = 0
        nodeList = {}
        for x in db.messages:
            print("")
            print("Message ",i,": ", x)
            print("Signals ",i,": ", x.signals)

            splitMessage = str(x).split(",")
            print("split messages: ", splitMessage)

            annotation = splitMessage[0].split("'")[1::2]
            annotation = str(annotation).strip("[']")
            nodeName = splitMessage[4].split("'")[1::2]
            nodeName = str(nodeName).strip("[']")
            nodeID = splitMessage[1][1:]
            node = {
                "Node_Name": nodeName,
                "Project_ID": '1.1.0',
                "Node_ID": nodeID,
                "Annotation": annotation,
                "Icon_Path": "",
                "xyPosition": ""
            }
            nodeList[i] = node
            i+=1
        return db, nodeList

    def translatePackets(packetID, packetData, timestamp, db):

        reformatedID = int(packetID, 16)
        reformatedData = b''.join([struct.pack('B', int(''.join(x),16)) for x in zip(packetData[0::2], packetData[1::2])])
        try:
            #example for 0x18fef1fe, when translated into Decimal
            #  0x18fef1fe(hex) == 419361278(decimal)
            #  0xcf00203 == 217,055,747
            decoded_message = db.decode_message(reformatedID, reformatedData)
            print(decoded_message)
            print("")
            canID = ''
            if("STATUS" in decoded_message):
                canID = '0x190'

            if("CMD" in decoded_message):
                canID = '0x65'

            if("SENSOR" in decoded_message):
                canID = '0xc8'

            if("IO" in decoded_message):
                canID = '0x1f4'

            if("HEARTBEAT" in decoded_message):
                canID = '0x64'
            if("Engine" in decoded_message):
                canID = '0Xcf004fe'
            if("Wheel" in decoded_message):
                canID = '0Xcf004fe'
            # endFrame = timestamp+=1

            packetJ = {
                "PacketID": packetID,
                "NodeID": "1",
                "Frame_Start": timestamp,
                "CAN_ID" : canID,
                "Control_Field": "1",
                "Data_Field": decoded_message,
                "CRC_Field": "",
                "ACK": "1",
                "Frame_End": timestamp,
            }

            packetJson = {
                "timestamp": timestamp,
                "packet_id": packetID,
                "packet_data": packetData,
                "decoded_message": decoded_message,
                "decoded_id": reformatedID,
            }
            return packetJ
        except:
            pass

            #print('No Message found')

db, nodeList = PacketManager.loadDBCFile("C:\\Users\\chris\\OneDrive\\Documents\\SCHOOL\\CAN-Bus Visualizer\\packetID\\CSS-Electronics-SAE-J1939-DEMO.dbc")
# db = PacketManager.loadDBCFile('j1939.dbc')
# db, nodeList = PacketManager.loadDBCFile('practiceDBC.dbc')

print(json.dumps(nodeList, indent=4))
#PacketManager.translatePackets(id, data, db)
packetCSV = PacketManager.createPacketDict('packets1.csv')
packetCSV.pop(0)   #removes first line
print("")
packetList = {}
i = 0
for x in packetCSV:
    id = (x['ID'])
    data = (x['Data'])
    timestamp = (x['Timestamp'])
    packetInfo = PacketManager.translatePackets(id, data, timestamp, db)
    if(packetInfo):
        #print(packetInfo)
        packetList[i] = (packetInfo)
        i+=1
# out : {'Timestamp;Type;ID;Data': '11T082557132;1;cf00203;cde0270004b827ff'}
# out : {'Timestamp;Type;ID;Data': '11T082557132;1;18fe592f;ffffffffe37cdd7c'}
print(json.dumps(packetList, indent =4 ))


from urllib.request import urlopen as req
from bs4 import BeautifulSoup as soup

def GoldPrice():
	#ปป.แค่ symbol = GPSC
	url = f'https://www.goldtraders.or.th/'
	webopen = req(url)    #เปิดเว็ปโดยไม่เปิด Web Browser urllib.request ทำการดึง(request)
	page_html = webopen.read() # สั่ง read เป็น text , Code html ไปใช้งานต่อ
	webopen.close() # ปิดการ request
	data = soup(page_html,'html.parser') # soupทำการอ่านข้อมูลอัตโนมัติ #แปลงเป็น soup

	allclass = ['DetailPlace_uc_goldprices1_lblBLSell',
				'DetailPlace_uc_goldprices1_lblBLBuy',
				'DetailPlace_uc_goldprices1_lblOMSell',
				'DetailPlace_uc_goldprices1_lblOMBuy']

	allprice = []

	# class = col-xs-6
	for al in allclass:
		rawdata = data.find_all('span',{'id':al}) # print('COUNT',len(rawdata))
		allprice.append(float(rawdata[0].text.replace(',','')))
	# print(allprice)
	header = ['ราคาทองคำแท่ง-ขายออก', 'ราคาทองคำแท่ง-รับซื้อ', 'ทองคำรูปพรรณ-ขายออก', 'ทองคำรูปพรรณ-รับซื้อ']

	result = {}

	for h,p in zip(header,allprice):  #แปลงจาก list เป็น Dictionary
		result[h] = p

	# print(result)
	return result

price = GoldPrice()
# print(price['ทองคำรูปพรรณ-รับซื้อ'])

for h,p in price.items():  #แปลงจาก dict มาคล้าย list
	# print(h,p)
	print(f'{h} ราคา {p} บาท')

# GoldPrice()
# print(GoldPrice())


from __future__ import annotations
from datetime import datetime

from models.base import Base

class Report(Base):
    _CLASS: str = "report"
    _properties: dict[str, object] = {
        "user": str,
        "experiment": str,
        "module": str,
        "timestamp": datetime.fromisoformat,
        "running": bool,
        "datapoints": int,
    }

    user: str = None
    experiment: str = None
    module: str = None
    timestamp: datetime = None
    running: bool = None
    datapoints: int = None

    def toDict(self) -> dict:
        return {
            "user": self.user,
            "experiment": self.experiment,
            "module": self.module,
            "timestamp": self.timestamp.isoformat(),
            "running": self.running,
            "datapoints": self.datapoints
        }

    @classmethod
    def fromDict(cls: Report, data: dict) -> Report | None:
        if not cls.isValid(data):
            return None

        model: Report = Report(f'{data["user"]}-{data["experiment"]}', cls._CLASS)

        model.user = cls._properties["user"](data["user"])
        model.experiment = cls._properties["experiment"](data["experiment"])
        model.module = cls._properties["module"](data["module"])
        model.timestamp = cls._properties["timestamp"](data["timestamp"])
        model.running = cls._properties["running"](data["running"])
        model.datapoints = cls._properties["datapoints"](data["datapoints"])

        return model


import os
import logging

from more_executors import Executors

from .base import Phase

LOG = logging.getLogger("pubtools.pulp")

CHECKSUM_THREADS = int(os.getenv("PUBTOOLS_PULP_CHECKSUM_THREADS") or "4")

class LoadChecksums(Phase):
    """Phase for loading/calculating checksums of push items.

    Push items may be initially discovered either with checksum information
    (e.g. from Errata Tool) or without (e.g. files in a staging directory).

    In cases where a push item checksum is currently unknown, this phase
    will read the push item's file in order to calculate it. This is handled
    by a dedicated phase since it can be very slow for large content accessed
    via NFS.

    Input queue:
    - items which may or may not have checksum info.

    Output queue:
    - items which have checksums (where the push item type supports that).

    Side-effects:
    - sends updated item states to the Collect phase as soon as checksums
      are known.
    """

    # Outputs of this phase should update push items since this is the first
    # point during a push where we know what the items actually are (in terms
    # of the bytes being pushed...)
    UPDATES_PUSH_ITEMS = True

    def __init__(self, context, in_queue, **kwargs):
        super(LoadChecksums, self).__init__(
            context, in_queue=in_queue, name="Calculate checksums", **kwargs
        )

    def run(self):
        with Executors.thread_pool(
            max_workers=CHECKSUM_THREADS, name="checksummer"
        ) as exc:
            for item in self.iter_input():
                # Use a heuristic to try to hand off the item onto the next
                # phase as quickly as possible.
                #
                # - in general we use a thread pool to do calculations in parallel.
                #
                # - but if we probably already have sums, we don't want to put
                #   that item onto the back of a potentially long queue where it
                #   may have to wait a long time, when it could be passsed on
                #   immediately.
                #
                # Hence we handle some items synchronously and others not.

                LOG.debug(
                    "Calculating checksums (blocking: %s): %s",
                    item.blocking_checksums,
                    item.pushsource_item.name,
                )

                if not item.blocking_checksums:
                    # with_checksums (probably) won't block so just do
                    # it immediately, thus letting the next phase get hold
                    # of the item more quickly.
                    self.put_output(item.with_checksums())

                else:
                    # with_checksums (probably) will block so put it onto
                    # the thread pool's queue.
                    f = exc.submit(item.with_checksums)
                    self.put_future_output(f)


from collections import deque

N, K = map(int,input().split())

# 제거를 N번 pop, enque로 구현.

vector = deque(list(range(1,N+1)))

myword = '<'

for i in range(N-1):
    for j in range(K-1):
        temp = vector.popleft()
        vector.append(temp)
    temp = vector.popleft()
    myword += str(temp) +', '

myword += str(vector.popleft()) + '>'

print(myword)


# -*- coding: utf-8 -*-
from argparse import ArgumentParser

from tqdm import tqdm
import wandb
import numpy as np
import random, os

import torch
from torch.utils.data import DataLoader
from torch.nn.parallel import DistributedDataParallel as DDP
import torch.distributed as dist
from torch.utils.data.distributed import DistributedSampler
from torch.optim.lr_scheduler import ReduceLROnPlateau
import torch.multiprocessing as mp

from model import DecoderTTS
from load_dataset import PrepareDataset, collate_fn
import trainer
import hparams as hp
import writer
SEED=44

def init_process(rank, size, backend='nccl'):
    """ Initialize the distributed environment. """

    dist.init_process_group(
        backend=backend,
        init_method='env://',
        world_size=size,
        rank=rank
    )

def get_model():
    return DecoderTTS(idim=hp.hidden_dim)

def get_dataloader(rank, world_size):

    dataset = PrepareDataset(hp.meta_file, hp.dumpdir)

    train_size = int(0.9 * len(dataset))
    val_size = len(dataset) - train_size
    train_set, val_set = torch.utils.data.random_split(dataset, [train_size, val_size])

    sampler = DistributedSampler(train_set, rank=rank, num_replicas=world_size)

    batch_size = hp.batch_size // world_size

    train_loader = DataLoader(train_set,
                            batch_size=batch_size,
                            collate_fn=collate_fn,
                            num_workers=8,
                            sampler=sampler)
    val_loader = DataLoader(val_set,
                            batch_size=batch_size,
                            shuffle=False,
                            collate_fn=collate_fn,
                            num_workers=8)

    return train_loader, val_loader

def train(rank, world_size):
    init_process(rank, world_size)
    print(f"Rank {rank}/{world_size} training process initialized.\n")
    torch.cuda.set_device(rank)
    # master process gets data
    if rank == 0:
        get_dataloader(rank, world_size)
        get_model()
    dist.barrier()

    m = get_model()
    m.cuda()
    m = DDP(m, device_ids=[rank])
    train_loader, val_loader = get_dataloader(rank, world_size)

    optimizer = torch.optim.Adam(m.parameters(), lr=0.0001  * world_size)
    l1_loss = torch.nn.L1Loss()
    bce_loss = torch.nn.BCEWithLogitsLoss(pos_weight=torch.tensor(hp.bce_weights))

    n_epochs = hp.n_epochs

    if rank == 0: writer.init_wandb(m)

    dist.barrier()

    best_loss = 1e10
    for epoch in range(n_epochs):
        train_loss = trainer.train_fn(m, train_loader, optimizer, l1_loss, bce_loss)
        val_loss, spec_fig, gate_fig, alignment_fig = trainer.eval_fn(m, val_loader, l1_loss, bce_loss)
        print(f'TRAIN LOSS = {sum(train_loss)} | VAL LOSS = {sum(val_loss)} | \n')

        # Log the loss and accuracy values at the end of each epoch
        if rank == 0: writer.log_wandb(epoch, train_loss, val_loss, spec_fig, gate_fig, alignment_fig)

        if best_loss > sum(val_loss):
            best_loss = sum(val_loss)
            torch.save(m.state_dict(), 'checkpoints/model'+str(epoch)+'.pth')
    dist.destroy_process_group()

WORLD_SIZE = torch.cuda.device_count()
def main():
    try:
        mp.spawn(train, args=(WORLD_SIZE,),
                nprocs=WORLD_SIZE, join=True)
    except KeyboardInterrupt:
        print('Interrupted')
        try:
            dist.destroy_process_group()
        except KeyboardInterrupt:
            os.system("kill $(ps aux | grep multiprocessing.spawn | grep -v grep | awk '{print $2}') ")

if __name__ == "__main__":
    main()


import json
import sys

import pm4py
from pm4py.util import xes_constants as xes

from constants import PATH

def get_all_event_attributes_from_log(log):
    """
    Get all events attributes from the log

    Parameters
    -------------
    log
        Log

    Returns
    -------------
    all_attributes
        All trace attributes from the log
    """
    all_attributes = set()
    for trace in log:
        for event in trace:
            all_attributes = all_attributes.union(set(event.keys()))
    if xes.DEFAULT_TRANSITION_KEY in all_attributes:
        all_attributes.remove(xes.DEFAULT_TRANSITION_KEY)
    return all_attributes

def get_all_event_attributes_values_from_log(log, attrName):
    """
    Get all events attributes from the log

    Parameters
    -------------
    log
        Log

    Returns
    -------------
    all_attributes
        All trace attributes from the log
    """
    all_attributes = set()
    for trace in log:
        for event in trace:
            sett = event.get(attrName)
            if sett:
                all_attributes = all_attributes.union([event[attrName]])
    if xes.DEFAULT_TRANSITION_KEY in all_attributes:
        all_attributes.remove(xes.DEFAULT_TRANSITION_KEY)
    return all_attributes

if __name__ == "__main__":
    file_name = sys.argv[1]
    log = pm4py.read_xes(PATH + "logs/" + file_name)

    event_attributes = get_all_event_attributes_from_log(log)
    filtered = [x for x in event_attributes if x != 'concept:name' and x != 'time:timestamp']

    res = {}
    for attr_name in filtered:
        values = get_all_event_attributes_values_from_log(log, attr_name)
        if any(isinstance(value, str) for value in values):
            res[attr_name] = 'literal'
        elif any(isinstance(value, int) or isinstance(value, float) for value in values):
            res[attr_name] = 'number'
        elif any(isinstance(value, bool) for value in values):
            res[attr_name] = 'boolean'

    print(json.dumps(res))


import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# Load the movie data
movies_df = pd.read_csv("movies.csv")

# Create a TF-IDF matrix for the movie descriptions
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(movies_df["plot"].values)

# Calculate the cosine similarity between all movies
cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)

# Get the recommendations for a particular movie
movie_title = "The Shawshank Redemption"
movie_index = movies_df[movies_df["title"] == movie_title].index[0]

# Get the top 10 most similar movies
recommendations = []
for i in range(10):
    similar_movie_index = cosine_sim[movie_index].argsort()[-i - 1]
    similar_movie_title = movies_df.iloc[similar_movie_index]["title"]
    recommendations.append(similar_movie_title)

print("Recommendations for " + movie_title + ":")
print(recommendations)


import torch
import torch.nn as nn
import gpytorch
import numpy as np

class SpectralMixtureGPModel(gpytorch.models.ExactGP):
    """
    Gaussian process model with Spectral Mixture Kernel from Wilson et al. (2013).

    Used for univariate data. In our case, we (train_x, train_y) is a univariate
    time series.

    params:
    ------
        train_x (torch.Tensor): Tensor with the index of the time series
        train_y (torch.Tensor): Tensor with the values of the time series
        num_mixtures (int): Number of mixtures for the kernel.
        likelihood (gpytorch.likelihoods): Likelihood for the problem. For a
        real-valued time series, we use GaussianLikelihood().
    """
    def __init__(self, train_x, train_y, num_mixtures, likelihood):
        super(SpectralMixtureGPModel, self).__init__(train_x, train_y, likelihood)
        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.SpectralMixtureKernel(num_mixtures=num_mixtures)
        self.covar_module.initialize_from_data_empspect(train_x, train_y)

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return gpytorch.distributions.MultivariateNormal(mean_x, covar_x)

class GRUFeatureExtractor(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers):
        super(GRUFeatureExtractor, self).__init__()
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.gru = nn.GRU(input_dim, hidden_dim, num_layers)

    def forward(self, x):
        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size)
        out, _ = self.gru(x, h0)
        out = out.reshape(out.shape[0], -1)
        return out

class LSTMFeatureExtractor(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers):
        super(LSTMFeatureExtractor, self).__init__()
        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, bidirectional=False)

    def forward(self, x):
        h0 = torch.zeros(self.num_layers, x.size(1), self.hidden_dim)
        c0 = torch.zeros(self.num_layers, x.size(1), self.hidden_dim)
        out, _ = self.lstm(x, (h0, c0))
        out = out.reshape(out.shape[0], -1)
        return out

def create_sequences(x, sequence_length):
    """
    Creates a transformed dataset for the LSTM with a new sequence length.
    """
    x_seq, y_seq = [], []
    for i in range(len(x) -sequence_length):
        x_seq.append(x[i: i + sequence_length])
        y_seq.append(x[i+1: i + sequence_length + 1])
    x_seq = np.array(x_seq).reshape(-1, sequence_length)
    y_seq = np.array(y_seq).reshape(-1, sequence_length)
    return x_seq, y_seq

if __name__ == '__main__':
    # generate univariate data
    y = np.random.normal(0, 1, 120)
    sequence_length = 5

    test_x, test_y = create_sequences(y, sequence_length=5)
    print(test_x)
    print(test_y.shape)

    # convert to Tensor and reshape to (seq, batch, dim)
    y = torch.Tensor(torch.from_numpy(test_y).float())
    y = y.view([len(y), -1, 5])
    print(y.size())

    # test forward method
    model = LSTMFeatureExtractor(input_dim=5, hidden_dim=16, num_layers=2)
    yhat = model(y)
    print(yhat)


import numpy as np
import matplotlib.pyplot as plt

import ahocorasick

# NOT FOR USE WITHIN DATALOADER THREADS (sic!)

class SeqEncoder:
    def __init__(self, alphabet=("A", "G", "C", "T")):
        self.alphabet = np.array(alphabet)
        self.code = {n: i for n, i in zip(self.alphabet, range(len(self.alphabet)))}
        self.code_r = {v: k for k, v in self.code.items()}

    def encode(self, sequence_str):
        sequence_labels = [self.code[i] for i in sequence_str]
        return np.eye(self.alphabet.__len__())[sequence_labels].T

    def decode(self, sequence_array):
        sequence_labels = sequence_array.argmax(axis=0)
        sequence_string = "".join(self.alphabet[sequence_labels])
        return sequence_string

class MotifsAutomaton:
    def __init__(self, motifs: list, check=True, alphabet=("A", "G", "C", "T")):
        self.enc = SeqEncoder(alphabet=alphabet)
        if check:
            motif_dtypes = tuple(set([type(m) for m in motifs]))
            if len(motif_dtypes) > 1:
                raise TypeError("Passed motifs must be of the same type")
            if motif_dtypes[0].__module__ == np.__name__:
                motifs = [self.enc.decode(m) for m in motifs]
            elif motif_dtypes[0] is str:
                pass
            else:
                raise TypeError(f"Cannot parse motifs of type {motif_dtypes[0]}")
        self.motifs = motifs
        lengths = set([len(m) for m in motifs])
        self.motif_length = None if len(lengths) != 1 else next(iter(lengths))
        self.enc_motifs = [self.enc.encode(m) for m in motifs]
        self.make_ahocorasick()

    def make_ahocorasick(self):
        self.aho = ahocorasick.Automaton()
        for idx, m in enumerate(self.motifs):
            self.aho.add_word(m, (idx, m))
        self.aho.make_automaton()

    def findings_exact(self, sequence_text_str):
        # O(N) algorithm
        findings = np.zeros(len(sequence_text_str))
        for end_index, (_, motif) in self.aho.iter(sequence_text_str):
            start_index = end_index - len(motif) + 1
            findings[start_index:start_index + len(motif)] += 1
        return findings

    def similarity_inexact(self, sequence_text_array):
        # O(M**2 * N * |A|) algorithm
        similarities = list()
        for m_array in self.enc_motifs:
            motif_length = m_array.shape[-1]
            sim = self.similarity_arrays(sequence_text_array, m_array)
            if self.motif_length is None:
                sim = self.similarity_array_highlightmax(sim, motif_length)
            similarities.append(sim)
        sim_stacked = np.stack(similarities).max(axis=0)
        if self.motif_length is not None:
            sim_stacked = self.similarity_array_highlightmax(sim_stacked, self.motif_length)
        return sim_stacked

    def similarity_arrays(self, sequence_text_array, sequence_motif_array):
        seq_text_flat = sequence_text_array.ravel(order="F")
        seq_motif_flat = sequence_motif_array.ravel(order="F")
        # convolve without flipping (cross-correlate):
        similarity = np.correlate(seq_text_flat, seq_motif_flat)
        # remove incorrect shifts (due to 2D structure):
        similarity = similarity[::4]
        # pad with zeros to save shape:
        pad_length = sequence_text_array.shape[1] - similarity.shape[0]
        sim_padded = np.concatenate((similarity,
                                     np.zeros(pad_length),
                                    ))
        sim = sim_padded / sim_padded.max()
        return sim

    def similarity_array_highlightmax(self, sim, highlight_length):
        sims = [sim]
        rolls = highlight_length
        for i in range(1, rolls):
            sims.append(np.roll(sim, i))
        sim_max = np.stack(sims).max(axis=0)
        return sim_max

class MotifsTracer(MotifsAutomaton):
    def __init__(self, exact=True, inexact=True, **kwargs):
        super().__init__(**kwargs)
        self.exact = exact
        self.inexact = inexact

    def get_vis_arrays(self, sequence_text):
        if isinstance(sequence_text, str):
            sequence_text_str = sequence_text
            sequence_text_array = self.enc.encode(sequence_text)
        else:
            sequence_text_array = sequence_text
            sequence_text_str = self.enc.decode(sequence_text)
        out_arr = sequence_text_array
        if self.exact:
            exact_arr = self.findings_exact(sequence_text_str)
            out_arr = self.add_channel(out_arr, exact_arr, intensify_func=lambda x: np.log2(x + 1))
        if self.inexact:
            inexact_arr = self.similarity_inexact(sequence_text_array)
            out_arr = self.add_channel(out_arr, inexact_arr, intensify_func=lambda x: 1.2 ** x)
        mask_all = np.ones_like(out_arr, dtype=bool)
        if self.exact and self.inexact:
            mask_all[-2:,:] = False
            mask_exact = np.zeros_like(mask_all, dtype=bool)
            mask_exact[-2] = True
            mask_inexact = np.zeros_like(mask_all, dtype=bool)
            mask_inexact[-1] = True
        elif self.exact and not self.inexact:
            mask_all[-1:,:] = False
            mask_exact = np.zeros_like(mask_all, dtype=bool)
            mask_exact[-1] = True
            mask_inexact = np.zeros_like(mask_all, dtype=bool)
        elif not self.exact and self.inexact:
            mask_all[-1:,:] = False
            mask_exact = np.zeros_like(mask_all, dtype=bool)
            mask_inexact = np.zeros_like(mask_all, dtype=bool)
            mask_inexact[-1] = True
        else:
            mask_exact = np.zeros_like(mask_all, dtype=bool)
            mask_inexact = np.zeros_like(mask_all, dtype=bool)
        return out_arr, mask_all, mask_exact, mask_inexact

    @staticmethod
    def add_channel(src_array, added_array, intensify_func=None):
        if intensify_func is not None:
            added_array = intensify_func(added_array)
        added_array = added_array[np.newaxis,:]
        extended_src_array = np.concatenate((src_array, added_array), axis=0)
        return extended_src_array

    def visualize_similarity(
        self,
        sequence_text: str,
        ax=None,
        cmap1="Greys_r",
        cmap2="Wistia",
        cmap3="Wistia",
        **kwargs
    ):
        kwargs.pop("cmap", None)

        out_arr, mask_all, mask_exact, mask_inexact = self.get_vis_arrays(sequence_text)
        # mask for visualization
        if ax is None:
            _, ax = plt.subplots(1, 1)
        labels = list(self.enc.alphabet)
        ax.imshow(np.ma.masked_array(out_arr, ~mask_all), cmap=cmap1, **kwargs)
        if self.exact:
            ax.imshow(np.ma.masked_array(out_arr, ~mask_exact), cmap=cmap2, **kwargs)
            labels.append("Exact")
        if self.inexact:
            ax.imshow(np.ma.masked_array(out_arr, ~mask_inexact), cmap=cmap3, **kwargs)
            labels.append("~Sim")
        ax.set_yticks(np.arange(len(labels)))
        ax.set_yticklabels(labels)

        # set spines invisible
        # ax.spines['top'].set_visible(False)
        # ax.spines['right'].set_visible(False)
        # ax.spines['bottom'].set_visible(False)
        # ax.spines['left'].set_visible(False)
        return None

if __name__ == "__main__":
    seq1 = "AAAAGTAGCTACGTAGCTAGTCGATCGTAGCTGCATCGTAGTTTATCGTAGCGCGATCGGGA"
    seq2 = "AAAAGTAGGTACGAAGCTAGTCGATCATTGCTGCATCGTAGTTTAACGTAGCGCGATCGGGA"
    motifs = ["GTAGCTA", "CGTAGCTGCAT", "TGCATCGTAGT", "TTTTT"]

    fig, ax = plt.subplots(2, 1, figsize=(20, 5))
    MotifsTracer(motifs=motifs).visualize_similarity(seq1, ax=ax[0])
    MotifsTracer(motifs=motifs).visualize_similarity(seq2, ax=ax[1])
    plt.show()


from functools import reduce
from random import randrange
from itertools import repeat

# parametros

k = 3
maxit = 10

# declaracion de funciones

def deuclidea(x, y):
   return sum(map(lambda a, b: (float(a) - float(b)) ** 2,
                  x, y)) ** 0.5

def contar(l):
   p = {}
   for x in l:
      p.setdefault(x, 0)
      p[x] += 1
   return p

def classify(t):
   ds = [deuclidea(centroides[i][1], t)
         for i in range(len(centroides))]
   return min([(ds[i], centroides[i][0])
               for i in range(len(centroides))],
              key=lambda x: x[0])[1]

def nextCentr(l):
   if (l == []):
      return None
   else:
      if len(l)==1:
         return l[0][2]
      else:
         tot2 = list(zip(*[x[2] for x in l]))
         num = len(tot2[0])
         return list(map(lambda l: reduce(lambda a, b:
                                       float(a) + float(b), l)
                      / num, list(tot2)))

def kmeans(clase, k, maxit):
   filt = list(filter(lambda x: x[0] == x[1],
                    [(clasesTrain[i], clase, train[i])
                     for i in range(len(train))]))
   conj = [filt[i][2] for i in range(len(filt))]
   centr = [conj[randrange(len(conj))] for i in range(k)]
   anteriores = None
   for it in range(maxit):
      cercanos = [min(zip(*[list(map(deuclidea,
                                     repeat(ej, k),
                                     centr)), range(k)]))[1]
                  for ej in conj]
      centr2 = [nextCentr(list(filter(
         lambda x: x[0] == x[1],
         zip(*[list(repeat(c, len(conj))),
               cercanos, conj]))))
                for c in range(k)]
      if None in centr2: break
      if cercanos == anteriores: break
      anteriores = cercanos

   return [(clase, x) for x in filter(lambda x: x!=None, centr2)]

# carga del archivo
l = list(map(lambda l: (l.strip()).split(','),
             open('iris.data.txt', 'r').readlines()))

# construccion del training: 2 de cada 3
train = list(map(lambda x: x[1],
                 filter(lambda v: v[0] % 3 != 0, enumerate(l))))
nc = len(train[0])
clasesTrain = list(map(lambda x: x.pop(nc - 1), train))

# construccion del test: 1 de cada 3
test = list(map(lambda x: x[1],
                filter(lambda v: v[0] % 3 == 0, enumerate(l))))
clasesTest = list(map(lambda x: x.pop(nc - 1), test))

# Entrenamiento
clases = contar(clasesTrain)
centroides = reduce(lambda x, y: x + y,
                    [kmeans(c, k, maxit) for c in clases.keys()])

# Clasificacion
predicciones = list(map(classify, test))

# Numero de correctos
print('Prec.:',
      len(list(filter(lambda x: x[0] == x[1],
                      zip(*[predicciones, clasesTest]))))
      / len(test) * 100, '%')