from sys import stdin

def intTOBin16bit(Val):
    rawBin = bin(Val)[2::]
    length = len(rawBin)
    binary = '0' * (16 - length) + rawBin
    return binary

def intToBin8bit(Val):
    rawBin = bin(Val)[2::]
    length = len(rawBin)
    binary = '0' * (8 - length) + rawBin
    return binary

def bin8bitToInt(binaryVal):
    return int(binaryVal, 2)

def isOverflow(Val):
    if(Val > (2**16 - 1)):
        return True
    else:
        return False

class Reg:

    flagRegr = "0000000000000000"
    Reg = {"000": 0, "001": 0, "010": 0, "011": 0, "100": 0, "101": 0, "110": 0,}
                  #R1       #R2       #R3       #R4       #R5       #R6       #R7

    def resetflagRegr(self):
        self.flagRegr = "0000000000000000"

    def setsOverflowFlag(self):
        self.flagRegr = "0000000000001000"

    def setsLessThanFlag(self):
        self.flagRegr = "0000000000000100"

    def setsGreaterThanFlag(self):
        self.flagRegr = "0000000000000010"

    def setsEqualsFlag(self):
        self.flagRegr = "0000000000000001"

    def printFlag(self):
        print(self.flagRegr, end=" ")

    def setReg(self, Reg_Addr, Val):
        if(not isOverflow(Val)):
            self.Reg[Reg_Addr] = Val
        else:
            rawBin = bin(Val)[2::]
            self.Reg[Reg_Addr] = int(rawBin[len(rawBin)-16::], 2)

    def getReg(self, Reg_Addr, Bin_or_Dec):
        if(Bin_or_Dec):
            if(Reg_Addr == "111"):
                return self.flagRegr
            rawBin = bin(self.Reg[Reg_Addr])[2::]
            if(len(rawBin)>16):
                return rawBin[len(rawBin)-16::]
            else:
                return intTOBin16bit(self.Reg[Reg_Addr])
        else:
            if(Reg_Addr == "111"):
                return int(self.flagRegr, 2)
            return self.Reg[Reg_Addr]

    def dump(self):
        for key in self.Reg.keys():
            print(intTOBin16bit(self.Reg[key]), end = " ")
        print(self.flagRegr)

RF = Reg()

class ProgramCounter:
    cur_cntr = 0

    def initialize(self):
        self.cur_cntr = 0

    def run(self):
        return self.cur_cntr

    def modify(self, new_cntr):

        self.cur_cntr = new_cntr

    def dump(self):

        print(intToBin8bit(self.cur_cntr), end = " ")

PC = ProgramCounter()
\
class Memory:

    mem = []
    X_axis = []
    Y_axis = []
    cycle = 0

    def initialize(self):
        for line in stdin:
            self.mem.append(line[0:16:])
        if(len(self.mem)<256):
            lineDifference = 256 - len(self.mem)
            while(lineDifference):
                self.mem.append("0000000000000000")
                lineDifference -= 1

    def getData(self, currentPC):
        return self.mem[currentPC]

    def dump(self):
        for ins in self.mem:
            print(ins)

    def runFromAddress(self, Mem_Addr):
        return int(self.mem[bin8bitToInt(Mem_Addr)], 2)

    def setValOfAddress(self, Mem_Addr, intVal):
        self.mem[bin8bitToInt(Mem_Addr)] = intTOBin16bit(intVal)

    def plotMemoryAccessTrace(self):
        import matplotlib.pyplot as plt
        plt.style.use('dark_background')
        plt.scatter(self.X_axis, self.Y_axis, c="blue")
        plt.ylabel("Memory Address")
        plt.xlabel("Cycle Number")
        plt.title("Memory Access Trace Graph")
        plt.show()
        plt.savefig('Memory_Access_Trace.png')

MEM = Memory()

class ExecutionEngine:

    def run(self, Inst):
        opcode = Inst[:5:]

        MEM.X_axis.append(MEM.cycle)
        MEM.Y_axis.append(PC.run())

        if(opcode == "10000"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2, False) + RF.getReg(reg3, False)
            if(isOverflow(res)):
                RF.setsOverflowFlag()
            else:
                RF.resetflagRegr()
            RF.setReg(reg1, res)
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10001"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2,False) - RF.getReg(reg3,False)
            if (res < 0):
                RF.setsOverflowFlag()
                RF.setReg(reg1,0)
            else:
                RF.setReg(reg1,res)
                RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10010"):
            reg1 = Inst[5:8:]
            Val = Inst[8::]
            Val = bin8bitToInt(Val)
            RF.setReg(reg1, Val)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10011"):
            reg1 = Inst[10:13:]
            reg2 = Inst[13::]
            RF.setReg(reg1, RF.getReg(reg2, False))
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10100"):
            reg1 = Inst[5:8:]
            Mem_Addr = Inst[8::]
            MEM.X_axis.append(MEM.cycle)
            MEM.Y_axis.append(bin8bitToInt(Mem_Addr))
            ValAtMemory = MEM.runFromAddress(Mem_Addr)
            RF.setReg(reg1, ValAtMemory)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10101"):
            # st reg1 mem_addr
            # 5  3    8
            reg1 = Inst[5:8:]
            Mem_Addr = Inst[8::]
            MEM.X_axis.append(MEM.cycle)
            MEM.Y_axis.append(bin8bitToInt(Mem_Addr))
            MEM.setValOfAddress(Mem_Addr, RF.getReg(reg1, False))
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10110"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2, False) * RF.getReg(reg3, False)
            if(isOverflow(res)):
                RF.setsOverflowFlag()
            else:
                RF.resetflagRegr()
            RF.setReg(reg1, res)
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "10111"):
            reg3 = Inst[10:13:]
            reg4 = Inst[13::]
            remainder = RF.getReg(reg3,False) % RF.getReg(reg4,False)
            quotient = RF.getReg(reg3,False) // RF.getReg(reg3,False)
            RF.setReg("000",quotient)
            RF.setReg("111",remainder)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11000"):
            reg1 = Inst[5:8:]
            immediateVal = bin8bitToInt(Inst[8::])
            shiftedString = '0' * immediateVal + RF.getReg(reg1, True)[:len(RF.getReg(reg1, True)) - immediateVal:]
            RF.setReg(reg1, int(shiftedString, 2))
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11001"):
            reg1 = Inst[5:8:]
            immediateVal = bin8bitToInt(Inst[8::])
            shiftedString = RF.getReg(reg1, True)[immediateVal::] + '0' * immediateVal
            RF.setReg(reg1, int(shiftedString, 2))
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11010"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2,False) ^ RF.getReg(reg3,False)
            RF.setReg(reg1,res)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11011"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2,False) | RF.getReg(reg3,False)
            RF.setReg(reg1,res)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11100"):
            reg1 = Inst[ 7:10:]
            reg2 = Inst[10:13:]
            reg3 = Inst[13:16:]
            res = RF.getReg(reg2,False) & RF.getReg(reg3,False)
            RF.setReg(reg1,res)
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11101"):
            reg1 = Inst[10:13:]
            reg2 = Inst[13::]
            inverted = ""
            for bit in reg2:
                if bit=='1':
                    inverted += '0'
                else:
                    inverted += '1'
            RF.setReg(reg1, int(inverted, 2))
            RF.resetflagRegr()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11110"):
            reg1 = Inst[10:13:]
            reg2 = Inst[13::]
            if RF.getReg(reg1, False) < RF.getReg(reg2, False):
                RF.setsLessThanFlag()
            elif RF.getReg(reg1, False) > RF.getReg(reg2, False):
                RF.setsGreaterThanFlag()
            else:
                RF.setsEqualsFlag()
            (halt, newPC) = (False, PC.run() + 1)

        elif(opcode == "11111"):
            Mem_Addr = Inst[8::]
            (halt, newPC) = (False, bin8bitToInt(Mem_Addr))
            RF.resetflagRegr()

        elif(opcode == "01100"):
            if RF.flagRegr == "0000000000000100":
                Mem_Addr = Inst[8::]
                (halt, newPC) = (False, bin8bitToInt(Mem_Addr))
            else:
                (halt, newPC) = (False, PC.run() + 1)
            RF.resetflagRegr()

        elif(opcode == "01101"):
            if RF.flagRegr == "0000000000000010":
                Mem_Addr = Inst[8::]
                (halt, newPC) = (False, bin8bitToInt(Mem_Addr))
            else:
                (halt, newPC) = (False, PC.run() + 1)
            RF.resetflagRegr()

        elif(opcode == "01111"):
            if RF.flagRegr == "0000000000000001":
                Mem_Addr = Inst[8::]
                (halt, newPC) = (False, bin8bitToInt(Mem_Addr))
            else:
                (halt, newPC) = (False, PC.run() + 1)
            RF.resetflagRegr()

        elif(opcode == "01010"):
            RF.resetflagRegr()
            (halt, newPC) = (True, PC.run() + 1)

        MEM.cycle += 1
        return (halt, newPC)

EE = ExecutionEngine()

MEM.initialize()
PC.initialize()
halted = False

while(not halted):
    Inst = MEM.getData(PC.run())
    halted, newPC = EE.run(Inst)
    PC.dump()
    RF.dump()
    PC.modify(newPC)

MEM.dump()
MEM.plotMemoryAccessTrace()


#!/usr/bin/env python3
"""Count the number of unique power a ** b where 2 <= a <= 100 and
<= b <= 100"""

from itertools import product
from math import log

lower = 2  # lowest possible value of the exponents and base
higher = 100  # higest one

base = range(lower, higher + 1)
power = range(lower, higher + 1)

def power_tuple(tup):
    """Find the power from a tuple of numbers of length two. The first
    item is the base and the last is the exponent.

    Examples:
    >>> power_tuple((3, 2))
    9
    >>> power_tuple((9, 0.5))
    3.0
    """

    if len(tup) != 2:
        raise ValueError(
            'tuple must be of length 2; given {}'.format(len(tup)))

    return pow(*tup)

def count_results():
    # get all the posible combinations of base and power
    pos_combi = product(base, power)
    result = set(map(power_tuple, pos_combi))

    # get the length of the result
    return len(result)

def count_results2():
    prod = set(x ** y for x in base for y in power)
    return len(prod)

def main():
    print(count_results2())

if __name__ == '__main__':
    from timeit import Timer
    stmt = 'from __main__ import {0}; {0}()'.format('main')
    print('time =', Timer(stmt=stmt).timeit(1))


import os
import requests
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status

class YoutubeVideos(APIView):
    def get(self, request):
        try:
            api_key = os.getenv("YOUTUBE_API_KEY")
            url = f"https://youtube.googleapis.com/youtube/v3/search?part=snippet&channelId=UCCU2H8fnVx20POKCzFm-G5Q&maxResults=50&type=video&videoDuration=medium&order=date&videoEmbeddable=true&key={api_key}"

            response = requests.get(url)
            items = response.json().get("items")

            if items is None:
                error_message = response.json().get("error").get("message")
                raise Exception(error_message)

            # next_page_token = response.json().get("nextPageToken")

            # while next_page_token:
            #     response = requests.get(url + "&pageToken=" + next_page_token)
            #     items += response.json().get("items")
            #     next_page_token = response.json().get("nextPageToken")

            return Response(
                status=status.HTTP_200_OK,
                data=items,
            )

        except Exception as e:
            print(e)
            return Response(
                status=status.HTTP_400_BAD_REQUEST,
                data={"message": "뭔가가 잘못됐다. 지금 당장 개발자 불러!!"},
            )


import pyNN.utility.plotting as plot
import matplotlib.pyplot as plt
import pyNN.spiNNaker as sim

n_neurons = 100
simtime = 5000

sim.setup(timestep=1.0)

pre_pop = sim.Population(n_neurons, sim.IF_curr_exp(), label="Pre")
post_pop = sim.Population(n_neurons, sim.IF_curr_exp(), label="Post")
pre_noise = sim.Population(
    n_neurons, sim.SpikeSourcePoisson(rate=10.0), label="Noise_Pre")
post_noise = sim.Population(
    n_neurons, sim.SpikeSourcePoisson(rate=10.0), label="Noise_Post")

pre_pop.record("spikes")
post_pop.record("spikes")

training = sim.Population(
    n_neurons,
    sim.SpikeSourcePoisson(rate=10.0, start=1500.0, duration=1500.0),
    label="Training")

sim.Projection(pre_noise,  pre_pop,  sim.OneToOneConnector(),
               synapse_type=sim.StaticSynapse(weight=2.0))
sim.Projection(post_noise, post_pop, sim.OneToOneConnector(),
               synapse_type=sim.StaticSynapse(weight=2.0))

sim.Projection(training, pre_pop,  sim.OneToOneConnector(),
               synapse_type=sim.StaticSynapse(weight=5.0, delay=1.0))
sim.Projection(training, post_pop, sim.OneToOneConnector(),
               synapse_type=sim.StaticSynapse(weight=5.0, delay=10.0))

timing_rule = sim.SpikePairRule(tau_plus=20.0, tau_minus=20.0,
                                A_plus=0.5, A_minus=0.5)
weight_rule = sim.AdditiveWeightDependence(w_max=5.0, w_min=0.0)

stdp_model = sim.STDPMechanism(timing_dependence=timing_rule,
                               weight_dependence=weight_rule,
                               weight=0.0, delay=5.0)

stdp_projection = sim.Projection(pre_pop, post_pop, sim.OneToOneConnector(),
                                 synapse_type=stdp_model)

sim.run(simtime)

pre_neo = pre_pop.get_data(variables=["spikes"])
pre_spikes = pre_neo.segments[0].spiketrains

post_neo = post_pop.get_data(variables=["spikes"])
post_spikes = post_neo.segments[0].spiketrains

print(stdp_projection.get('weight', format="list"))

sim.end()

line_properties = [{'color': 'red', 'markersize': 5},
                   {'color': 'blue', 'markersize': 2}]

plot.Figure(
    # plot spikes
    plot.Panel(pre_spikes, post_spikes, yticks=True, xlim=(0, simtime),
               line_properties=line_properties),
    title="STDP Network Example",
    annotations="Simulated with {}".format(sim.name())
)
plt.show()


import os
from setuptools import setup, find_packages

def read(fname):
    # read the contents of a text file
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

# allow setup.py to be run from any path
os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))

setup(
    name='djangocms-portfolio',
    version='0.8',
    packages=find_packages(),
    include_package_data=True,
    license='GNU General Public License',
    description='Portfolio small app',
    long_description=read('README.md'),
    url='https://github.com/Pyc0kw4k/djangocms-portfolio',
    author='Lozano Joaquim',
    install_requires=["django-taggit==0.22.1"],
    author_email='joaquimlozano@gmail.com',
)


# mesh_files = ['/Users/xx/thesis/mesh/c2018.txt', '/Users/xx/thesis/mesh/d2018.txt', '/Users/xx/thesis/mesh/q2018.txt']
import codecs
import json
from collections import defaultdict

import requests
from lxml.html import fromstring

id_to_term = {}
term_to_id = {}
trec_mesh_terms = []

def parse_mesh_file(file='/Users/xx/thesis/mesh/cdq1997.txt'):
    def get_value(l):
        _, value = l.split('=')
        return value.strip()

    with codecs.open(file, encoding='ISO-8859-1') as f:
        for line in f:
            line = line.strip()
            if line == '*NEWRECORD':
                uid = ''
                term = ''
            elif line.startswith('MH =') or line.startswith('SH =') or line.startswith('NM ='):
                term = get_value(line).lower()
            elif line.startswith('UI ='):
                uid = get_value(line)
                term_to_id[term] = uid
                id_to_term[uid] = term

def parse_trec_mesh(file='/Users/xx/thesis/ir-datasets/ohsu-trec/trec9-train/query.mesh.1-4904'):
    with open(file) as f:
        for line in f:
            if line.startswith('<title>'):
                term = line[len('<title>'):].strip().lower()
                trec_mesh_terms.append(term)

def df_ohsu_trec(file='/Users/xx/thesis/ir-standard/ohsu-trec/ohsu-trec-documents.txt'):
    dfs = defaultdict(int)
    with open(file) as f:
        for line in f:
            if not line or line.startswith('.'):
                continue
            words = line.split()
            for word in words:
                dfs[word] += 1
    return sorted(dfs, key=dfs.get, reverse=True)

def generate_dictionary(words):
    for word in words:
        if word in term_to_id:
            swedish_name = get_swedish_name(term_to_id[word])
            if swedish_name and ' ' not in swedish_name:
                print(word, swedish_name.lower())

umls_ticket_generating_ticket_uri = 'https://utslogin.nlm.nih.gov'
umls_auth_endpoint = '/cas/v1/api-key'
umls_ticket_uri = 'http://umlsks.nlm.nih.gov'
umls_api_uri = 'https://uts-ws.nlm.nih.gov'
umls_mesh_swe_endpoint = '/rest/content/current/source/MSHSWE/'

def get_umls_ticket_generating_ticket(api_key):
    params = {'apikey': api_key}
    header = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", "User-Agent": "python"}
    response = requests.post(umls_ticket_generating_ticket_uri + umls_auth_endpoint, data=params, headers=header)
    response_text = fromstring(response.text)
    return response_text.xpath('//form/@action')[0]

def get_umls_ticket(ticket_generating_ticket_uri):
    params = {'service': umls_ticket_uri}
    header = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", "User-Agent": "python"}
    response = requests.post(ticket_generating_ticket_uri, data=params, headers=header)
    return response.text

def get_swedish_name(uid):
    try:
        params = {'ticket': get_umls_ticket(ticket_generating_ticket)}
        response = requests.get(umls_api_uri + umls_mesh_swe_endpoint + uid, params=params)
        response_json = json.loads(response.text)['result']
        return response_json['name']
    except:
        return None

parse_mesh_file()
# parse_trec_mesh()

ticket_generating_ticket = get_umls_ticket_generating_ticket('<INSERT_API_KEY_HERE>')

generate_dictionary(df_ohsu_trec())


# 문제 푸는 날짜: 220313
# BOJ 2504번 괄호의 값
# 분류:
# 난이도: 실버 2
# https://www.acmicpc.net/problem/2504
# 문제 풀이 핵심:

from collections import defaultdict
import sys

sys.setrecursionlimit( 10 ** 6 )

def anal_str(string):
    if string == '()': return 2
    elif string == '[]': return 3

    # 분석
    ret = 0; part_list = [0]
    for i, v in enumerate(string):
        if v == '(': brace_dict[2] += 1
        elif v == ')': brace_dict[2] -= 1
        elif v == '[': brace_dict[3] += 1
        elif v == ']': brace_dict[3] -= 1

        if set(brace_dict.values()) == {0}: part_list.append(i+1)

    # 올바르지 않은 문자열일 경우 - 0 출력하고 프로그램 종료
    if set(brace_dict.values()) != {0}:
        print(0)
        exit()

    # 2개 이상의 덩어리로 구성된 경우
    if len(part_list) > 2:
        for i in range(len(part_list) - 1):
            ret += anal_str(string[part_list[i]:part_list[i+1]])

        return ret

    # 1개 덩어리 경우 1
    elif string[0] == '(' and string[-1] == ')':
        return 2 * anal_str(string[1:-1])

    # 1개 덩어리 경우 2
    elif string[0] == '[' and string[-1] == ']':
        return 3 * anal_str(string[1:-1])

    else:
        print(0)
        exit()

brace_dict = defaultdict(int)
str_input = input()

print(anal_str(str_input))


import xml.etree.ElementTree as ET
directory = "/var/www/html/OCR++/myproject/media/documents/"
f = open(directory + "input_Allmails.txt",'r')
out = open(directory + 'eval_emails.txt','w')
xml = '<root>' + f.read() + '</root>'
tree = ET.ElementTree(ET.fromstring(xml))
root = tree.getroot()
for mail in root.findall('email'):
    out.write("<<Email>> \n" + mail.text.strip() + "\n")
f.close()
out.close()


from django.shortcuts import render, redirect
from django.urls import reverse_lazy, reverse
from django.views.generic import *
from .models import *
from .forms import *
from django.http import JsonResponse
from django.conf import settings
from django.contrib.messages.views import SuccessMessageMixin

# HANDLERS
# HANDLERS
# HANDLERS
# HANDLERS

def handle404(request, exception=None):
    return render(request, '404.html', {
        # 'orginfo': OrgnizationalInformation.objects.first(),
        # 'webreq': WebsiteRequirement.objects.first(),
        # 'allcategories': Category.objects.filter(is_active=True),
        # 'clientsubscriberform': ClientSubscriberForm
    })

def handle500(request, exception=None):
    return render(request, '500.html', {
        # 'orginfo': OrgnizationalInformation.objects.first(),
        # 'webreq': WebsiteRequirement.objects.first(),
        # 'allcategories': Category.objects.filter(is_active=True),
        # 'clientsubscriberform': ClientSubscriberForm
    })

# Client Page
class ClientMixin(SuccessMessageMixin, object):
    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['user'] = Admin.objects.first()
        context['settings_debug'] = settings.DEBUG

        return context

class ClientHomeView(ClientMixin, TemplateView):
    template_name = "clienttemplates/clienthome.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['allsliders'] = Gallery.objects.get(slug='slider')

        return context

class ClientProfileView(ClientMixin, TemplateView):
    template_name = "clienttemplates/clientprofile.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)

        return context

class ClientContactView(ClientMixin, CreateView):
    template_name = "clienttemplates/clientcontact.html"
    form_class = CustomerMessageForm
    success_url = reverse_lazy('profileapp:clienthome')
    success_message = "Message Sent Successfully"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['gallerylist'] = Gallery.objects.all().order_by('title')

        context['formset'] = MessageSpecificationFormSet()

        return context

    def form_valid(self, form):
        data = form.cleaned_data
        # print(data)
        gallery = form.save()
        try:
            formset = MessageSpecificationFormSet(data=self.request.POST)
            # formset = MessageSpecificationFormSet(
                # self.request.POST, self.request.FILES)
            # print(formset)
            for index, specform in enumerate(formset):
                print(specform)
                if specform.is_valid():
                    # print('valid +++++++++++++++++')
                    # print(specform)
                    value = specform.cleaned_data['value']
                    specification = specform.cleaned_data['specification']
                    spec = Specification.objects.get(id = specification)
                    MessageSpecificationComment.objects.get_or_create(
                        message_obj = gallery,
                        comment = value,
                        specification=spec)
                    # print(value)

            # Check if submitted forms are valid
            # if formset.is_valid():
            #     for data in formset.cleaned_data:
            #             print(data['value'])
            #             print(data['specification'])
            # if formset.is_valid():
            #     print(formset)
            #     print('success formset........................................')
        except:
            pass
        return super().form_valid(form)

    def form_invalid(self, form):
        print('failed form')
        return super().form_invalid(form)

# Axios Request Views
class AxiosSpecificationListView(ClientMixin, View):
    def get(self, request, *args, **kwargs):
        from django.core import serializers
        cat_id = self.request.GET.get('keyword', None)
        if cat_id:
            gallery_obj = Gallery.objects.get(id = cat_id)
            gallery_specifications = Specification.objects.filter(gallery = gallery_obj)
            print(gallery_specifications)
            list_spec = []
            for spec in gallery_specifications:
                list_spec.append({'id': spec.id, 'title': spec.title})
            serialized_gallery = serializers.serialize("json", gallery_specifications)
            return JsonResponse({
                'message': 'success',
                'cat_id': cat_id,
                'gallery_obj': serialized_gallery,
                'list_spec': list_spec
                })
        else:
            return JsonResponse({"error": "No search keyword"})


import sys
import getopt
import warnings
import rpyc
import json
import yaml
from PIL import Image, ImageTk
import PySimpleGUI as sg

from PC import PC
from II import II
from MAR import MAR
from REG import REG
from RAM_STACK import STACK
from MEMS import MEMS

from parse_vars import *
from print_call_graph import *

# TODO: dead code analysis
# TODO: call graph to include call/ret/jumps. add subgraph for subroutines
# TODO: extended memory

def main():

  FILE_NAME = 'bin/01_test_a_reg.bin' # default file
  EXIT_ON_HALT = False
  SIM = True
  GUI = True

  # process flags

  OPTIONS = "f:" #TODO: no abbreviated options.
  OPTIONS_LONG = ['file =', 'exit-on-halt', 'no-gui', 'no-sim']

  try:
    # grab flags
    args, vals = getopt.getopt(sys.argv[1:], OPTIONS, OPTIONS_LONG)
    for arg_, val_ in args:
      arg_ = arg_.strip()

      if arg_ in ("--file"):
        FILE_NAME = val_
      elif arg_ in ("--exit-on-halt"):
        EXIT_ON_HALT = True
      elif arg_ in ("--no-gui"):
        GUI = False
        EXIT_ON_HALT = True
      elif arg_ in ("--no-sim"):
        SIM = False

  except getopt.error as err:
    # output error, and return with an error code
    print (str(err))

  if SIM:
    try:
      conn = rpyc.connect("localhost", 18812, config={'allow_all_attrs': True})
    except:
      conn = None
      warnings.warn("Warning. Sim not connected!")
  else:
    conn = None

  print(f"running ${FILE_NAME}")

  pc = PC()
  mar = MAR()
  ii = II()
  mems = MEMS(sim=conn)
  A = REG(8)
  B = REG(8)
  X = REG(8)
  Y = REG(8)
  HL = REG(16)
  J = REG(16)
  K = REG(16)
  D = REG(16)
  stack = STACK(mems)

  clk_counter = 0

  UPDATE_RATE = 1000#10000

  IMG_HEI = 80
  IMG_WID = 101

  addr = 0
  data = 0
  UCC = 0

  flags = {
    'ZF' : 0,
    'CF' : 0,
    'NF' : 0
  }

  ctrl = {
    # outs
    'PO' : 0, # Program counter out
    'AO' : 0, # A out
    'BO' : 0, # B out
    'MO' : 0, # Memory out
    'NO' : 0, # Stack pointer out
    'SO' : 0, # Stack out
    'HO' : 0, # HL register out
    'JO' : 0, # J Scratch register out
    'KO' : 0, # K Scratch register out
    'DO' : 0, # D Scratch register out
    'HT' : 0, # Halt
    # ins
    'PI' : 0, # Program counter in
    'II' : 0, # Instruction register in
    'XI' : 0, # Alu X Register in
    'YI' : 0, # Alu X Register in
    'AI' : 0, # A in
    'BI' : 0, # Bi in
    'HI' : 0, # HL register in
    'MA' : 0, # MAR in
    'MI' : 0, # Memory in
    'NI' : 0, # Stack Pointer in
    'SI' : 0, # Stack in
    'JI' : 0, # J Scratch register in
    'KI' : 0, # K Scratch register in
    'DI' : 0, # D Scratch register in
    # alu
    'ADD' : 0, # Add
    'SUB' : 0, # Subtract
    'AND' : 0, # Logical AND
    'OR'  : 0, # Logical OR
    'XOR' : 0, # Logical XOR
    'SHL' : 0, # Logical Shift Left
    'SHR' : 0, # Logical Shift Right
    'ONES': 0, # Outputs 0xFF
    'ZERO': 0, # Outputs 0x00
    'ONE' : 0, # Outputs 0x01
    #misc
    'CE' : 0, # Program counter increment
    'FI' : 0, # CPU Flag refresh
    'MC' : 0, # MAR increment
    'IS' : 0, # increment stack pointer
    'DS' : 0, # decrement stack pointer
    'LM' : 0, # bus byte indictor
    'RU' : 0  # reset ucode counter
  }

  breakpt = False
  finish_inst = False

  DEFAULT_REG_LAYOUT = {
    'size':(6,1),
    'justification':'left',
    'text_color':'black',
    'background_color':'white'
  }

  layout_regs = [
    [sg.T('Regs')],
    [sg.T('PC  '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_PC_')],
    [sg.T('MAR '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_MAR_')],
    [sg.T('INST'), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_INST_')],
    [sg.T('UCC '), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_UCC_')],
    [sg.T('A   '), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_A_')],
    [sg.T('B   '), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_B_')],
    [sg.T('HL  '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_HL_')],
    [sg.T('X   '), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_X_')],
    [sg.T('Y   '), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_Y_')],
    [sg.T('J   '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_J_')],
    [sg.T('K   '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_K_')],
    [sg.T('D   '), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_D_')],
    [sg.T('DATA'), sg.T('0x00  ', **DEFAULT_REG_LAYOUT, key='_DATA_')],
    [sg.T('ADDR'), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_ADDR_')],
    [sg.T('STPR'), sg.T('0x0000', **DEFAULT_REG_LAYOUT, key='_STPR_')],
  ]

  DEFAULT_INDICATOR_LAYOUT = {
    'text':'\u2B24',
    'size':(2,1),
    'justification':'left',
    'text_color':'gray',
  }

  layout_flags = [
    [sg.T('Flags')],
    [sg.T('ZF  '), sg.T(**DEFAULT_INDICATOR_LAYOUT, key='_ZF_')],
    [sg.T('CF  '), sg.T(**DEFAULT_INDICATOR_LAYOUT, key='_CF_')],
    [sg.T('NF  '), sg.T(**DEFAULT_INDICATOR_LAYOUT, key='_NF_')],
    [],
    [sg.Button('uSTEP'), sg.Button('STEP Inst.')],
    [sg.Button('Resume')],
    [sg.T('CLK  '), sg.T(
      text='',
      size=(12,1),
      justification='left',
      text_color='black',
      background_color='white',
      key='_CNT_'
    )],
    [sg.T('INST '), sg.T(
      text='inst??',
      size=(21,1),
      justification='left',
      text_color='black',
      background_color='white',
      font=('courier new',7),
      key='_INST_NAME_')
    ],
    [sg.Image(key="-IMAGE-", size=(IMG_WID,IMG_HEI))],
    [sg.Image(key="-MAP-", size=(128,128))],
    [sg.T('UART')],
    [
      sg.Multiline(
        size=(30,24),
        autoscroll=True,
        font=('courier new',7),
        justification='left',
        text_color='black',
        background_color='white',
        key='_UART_'
      )
    ]
  ]

  layout_ctrl = [
    [sg.T('Ctrl')]
  ]

  for ctrl_sig,val in ctrl.items():
    layout_ctrl.append([
      sg.T(text=f"{ctrl_sig:<5}"),
      sg.T(**DEFAULT_INDICATOR_LAYOUT, key=f'_{ctrl_sig}_')
    ])

  layout_sram = [[
    sg.T(
      text='',
      size=(128*3+16,128),
      font=('courier new',6),
      justification='left',
      text_color='black',
      background_color='white',
      key='_SRAM_'
    )
  ]]
  layout_stack = [[
    sg.Column([
      [
        sg.Multiline(
          size=(12,70),
          autoscroll=True,
          font=('courier new',8),
          justification='left',
          text_color='black',
          background_color='white',
          write_only = True,
          disabled = True,
          key='_STACK_'
        )
      ]
    ], vertical_alignment='t' ),
    sg.Column([
      [
        sg.T('Current Usage   '),
        sg.T(
          text='',
          size=(12,1),
          justification='left',
          text_color='black',
          background_color='white',
          key='_STACK_USAGE_'
        )
      ],
      [
        sg.T('Max Stack Usage '),
        sg.T(
          text='',
          size=(12,1),
          justification='left',
          text_color='black',
          background_color='white',
          key='_STACK_MAX_USAGE_'
        )
      ]
    ], vertical_alignment='t' )
  ]]
  layout_vars = [[
    sg.Multiline(
      size=(100,70),
      font=('courier new',8),
      justification='left',
      text_color='black',
      background_color='white',
      write_only = True,
      disabled = True,
      key='_VARS_'
    )
  ]]
  layout_ext_rom = [
    [
      sg.T('EXT ROM Addr Ptr   '),
      sg.T(
        text='',
        size=(6,1),
        justification='left',
        text_color='black',
        background_color='white',
        key='_EXT_ROM_ADDR_'
      )
    ],
    [
      sg.T('EXT ROM Current Val'),
      sg.T(
        text='',
        size=(4,1),
        justification='left',
        text_color='black',
        background_color='white',
        key='_EXT_ROM_VAL_'
      )
    ],
    [
      sg.T(
        text='',
        size=(32*3+10,65),
        font=('courier new',6),
        justification='left',
        text_color='black',
        background_color='white',
        key='_EXT_ROM_'
      )
    ]
  ]
  layout_time = [[
    sg.Multiline(
      size=(100,70),
      font=('courier new',8),
      justification='left',
      text_color='black',
      background_color='white',
      write_only = True,
      disabled = True,
      key='_TIME_ANALYSIS_'
    )
  ]]

  tabs_layout = [[
    sg.TabGroup([[
      sg.Tab('SRAM', layout_sram),
      sg.Tab('STACK', layout_stack),
      sg.Tab('VARS', layout_vars),
      sg.Tab('EXT_ROM', layout_ext_rom),
      sg.Tab('TIME', layout_time),
      ]])
    ]]

  layout = [[
    sg.Column(layout_regs,  vertical_alignment='t'),
    sg.Column(layout_ctrl,  vertical_alignment='t'),
    sg.Column(layout_flags, vertical_alignment='t'),
    sg.Column(tabs_layout,  vertical_alignment='t')
  ]]

  if GUI:
    window = sg.Window('8B', layout, font=('courier new',11))
  else:
    window = None

  def update_uart(char):
    if GUI:
      window['_UART_'].update(char, append=True)
    else:
      pass
  mems.uart.callback = update_uart

  INDC_COLOR = ['gray', 'green']

  file = open(FILE_NAME, 'rb')
  eep_ptr = 0
  while 1:
    byte = file.read(1)
    if not byte:
      break
    mems.eeprom.value[eep_ptr] = ord(byte) & 0xFF
    # TODO: load real ext_rom bin
    mems.ext_eeprom.value[eep_ptr] = ord(byte) & 0xFF
    eep_ptr+=1

  file.close()

  vars, symbols, code = parse_vars(FILE_NAME)
  max_var_width = max([len(list(var.keys())[0]) for var in vars])

  if GUI:
    event, values = window.Read(timeout=0)
  else:
    event = None
    values = None

  inst = []
  inst_names = {}
  inst_stats = {}
  call_graph = [0x0000]
  previous_pc = 0x0000

  with open('instruction_set.yaml', 'r') as stream:
    try:
      IS = yaml.safe_load(stream)
      inst = [IS['instructions']['default']['ucode']]*int(0x80)
      for key, value in IS['instructions'].items():
        if key != 'default':
          inst[int(value['opcode'])] = value['ucode']
          inst_names[int(value['opcode'])] = key
          inst_stats[key] = {
            'calls': 0,
          }

      while True:
        if GUI:
          if clk_counter % UPDATE_RATE == 0 or ctrl['HT'] or breakpt:
            event, values = window.Read(timeout=0)

          if breakpt and not finish_inst:
            event, values = window.Read()
            if event == "STEP Inst.":
              finish_inst = True
            if event == "Resume":
              finish_inst = False
              breakpt = False
            if event == "uSTEP":
              pass

        clk_counter += 1

        ## rising edge
        # outputs
        #if ctrl['PO']+ctrl['MO']+ctrl['AO']+(ctrl['Mn']>0)+ctrl['PP'] > 1:
        #  raise Exception("more than one bus output enabled")
        if ctrl['PO']:
          data = pc.get(ctrl['LM'])
        elif ctrl['MO']:
          data = mems.get(addr)
        elif ctrl['NO']:
          data = stack.get(ctrl['LM'])
        elif ctrl['AO']:
          data = A.get()
        elif ctrl['BO']:
          data = B.get()
        elif ctrl['HO']:
          data = HL.get(ctrl['LM'])
        elif ctrl['JO']:
          data = J.get(ctrl['LM'])
        elif ctrl['KO']:
          data = K.get(ctrl['LM'])
        elif ctrl['DO']:
          data = D.get(ctrl['LM'])
        elif ctrl['SO']:
          data = stack.pop()
        elif ctrl['ADD']:
          # Add
          data = (X.value+Y.value) & 0xFF
        elif ctrl['SUB']:
          # Subtract
          data = (X.value+~(Y.value)+1) & 0xFF
        elif ctrl['AND']:
          # And
          data = (X.value & Y.value) & 0xFF
        elif ctrl['OR']:
          # Or
          data = (X.value | Y.value) & 0xFF
        elif ctrl['XOR']:
          # Xor
          data = (X.value ^ Y.value) & 0xFF
        elif ctrl['SHL']:
          # Left Shift 1
          data = (X.value << 1) & 0xFF
        elif ctrl['SHR']:
          # Right Shift 1
          data = (X.value >> 1) & 0xFF
        elif ctrl['ZERO']:
          data = 0x00
        elif ctrl['ONES']:
          data = 0xFF
        elif ctrl['ONE']:
          data = 0x01

        # FLAGS
        if ctrl['FI']:
          flags['ZF'] = ( (data & 0xFF) == 0 )
          flags['NF'] = ( data & 0x80) > 0
          if ctrl['ADD']:
            # Add
            flags['CF'] = ((X.value+Y.value) > 0xFF) or ((X.value+Y.value) < 0x00)
          elif ctrl['SUB']:
            # Subtract
            flags['CF'] = ((X.value+~(Y.value)+1) > 0xFF) or ((X.value+~(Y.value)+1) < 0x00)
          elif ctrl['AND']:
            flags['CF'] = 0
          elif ctrl['OR']:
            flags['CF'] = 0
          elif ctrl['XOR']:
            flags['CF'] = 0
          elif ctrl['SHL']:
            # Left Shift 1
            flags['CF'] = (X.value << 1) > 0xFF
          elif ctrl['SHR']:
            # Right Shift 1
            flags['CF'] = X.value & 0x01
          else:
            raise Exception("should only update CF on ALU operation")

        # CLK
        if ctrl['HT']:
          if EXIT_ON_HALT:
            break
          #if GUI:
          #  event, values = window.Read()
          breakpt = True

        # PC
        if (ctrl['PO'] or ctrl['PI']) and ctrl['CE']:
          raise Exception("CE asserted during PC access")
        if ctrl['PI']:
          pc.set(ctrl['LM'], data)
        if ctrl['CE']:
          pc.inc()

        # MAR
        if ctrl['MA'] and ctrl['MC']:
          raise Exception("CE asserted during PC access")
        if ctrl['MA']:
          mar.set(ctrl['LM'], data)
          addr = mar.value
        if ctrl['MC']:
          mar.inc()
          addr = mar.value

        # MEM
        if ctrl['MI']:
          mems.set(addr, data)

        # INST
        if ctrl['II']:
          ii.set(data)
          inst_stats[inst_names[ii.value]]['calls'] += 1
          # print(pc.value)
          # store dead code info
          list_addrs = [c['addr'] for c in code]
          indices = [i for i, x in enumerate(list_addrs) if x == pc.value-1]
          for i in indices:
            code[i]['execs'] += 1
            code[i]['dead'] = False

        # A REG
        if ctrl['AI']:
          A.set(data)
        # B REG
        if ctrl['BI']:
          B.set(data)
        # HL REG
        if ctrl['HI']:
          HL.set(data,ctrl['LM'])
        # X ALU REG
        if ctrl['XI']:
          X.set(data)
        # Y ALU REG
        if ctrl['YI']:
          Y.set(data)
        # J REG
        if ctrl['JI']:
          J.set(data,ctrl['LM'])
        # K REG
        if ctrl['KI']:
          K.set(data,ctrl['LM'])
        # D REG
        if ctrl['DI']:
          D.set(data,ctrl['LM'])

        # STACK
        if ctrl['SI']:
          stack.push(data)
        elif ctrl['IS']:
          stack.inc()
        elif ctrl['DS']:
          stack.dec()
        elif ctrl['NI']:
          stack.set(data,ctrl['LM'])

        # check if this is a jmp/call/ret instruction
        if ctrl['PI'] and ctrl['RU']:
          # if call_graph[-1] != previous_pc:
          #   call_graph.append(previous_pc)
          call_graph.append(pc.value)

        # U CODE
        UCC = (UCC + 1) & 0x1F
        if ctrl['RU']:
          UCC = 0

        if UCC == 0:
          previous_pc = pc.value

        ## falling edge
        # handle ucode
        ctrl = dict.fromkeys(ctrl, 0)
        if isinstance(inst[ii.value], dict):
          # branch/conditional instruction
          condition = True
          for c in inst[ii.value]['conditions']:
            if not flags[c]:
              condition = False
          if UCC < len(inst[ii.value][condition]):
            for u in inst[ii.value][condition][UCC]:
              ctrl[u] = 1

        # normal sequential instruction
        elif  UCC < len(inst[ii.value]):
          for u in inst[ii.value][UCC]:
            ctrl[u] = 1

          # handle the assert instructions. Only do anything on the last cycle
          if ctrl['RU']:
            if ii.value == int(IS['instructions']['assert_a']['opcode']):
              if mems.get(addr) == A.value:
                print(f"pass, a = 0x{A.value:02X}")
              else:
                raise Exception(f"test case failed, a = 0x{A.value:02X}, expected 0x{mems.get(addr):02X}\nPC=0x{pc.value:04X}")
            elif ii.value == int(IS['instructions']['assert_b']['opcode']):
              if mems.get(addr) == B.value:
                print(f"pass, b = 0x{B.value:02X}")
              else:
                raise Exception(f"test case failed, b = 0x{B.value:02X}, expected 0x{mems.get(addr):02X}\nPC=0x{pc.value:04X}")
            elif ii.value == int(IS['instructions']['assert_hl']['opcode']):
              if (mems.get(addr-1)<<8)+(mems.get(addr)) == HL.value:
                print(f"pass, HL = 0x{HL.value:04X}")
              else:
                raise Exception(f"test case failed, HL = 0x{HL.value:04X}, expected 0x{((mems.get(addr-1)<<8)+(mems.get(addr))):04X}\nPC=0x{pc.value:04X}")
            elif ii.value == int(IS['instructions']['assert_zf']['opcode']):
              if mems.get(addr) == flags['ZF']:
                print(f"pass, ZF = {flags['ZF']}")
              else:
                raise Exception(f"test case failed, ZF = {flags['ZF']}, expected {mems.get(addr)}\nPC=0x{pc.value:04X}")

        if GUI:
          if finish_inst:
            if UCC == 0:
              finish_inst = False

          if event is None or event == 'Exit':
            print()
            break

          if clk_counter % UPDATE_RATE == 0 or breakpt:
            window['_PC_'].update(f"0x{pc.value:04X}")
            window['_MAR_'].update(f"0x{mar.value:04X}")
            window['_INST_'].update(f"0x{ii.value:02X}")
            window['_UCC_'].update(f"0x{UCC:02X}")
            window['_A_'].update(f"0x{A.value:02X}")
            window['_B_'].update(f"0x{B.value:02X}")
            window['_HL_'].update(f"0x{HL.value:04X}")
            window['_X_'].update(f"0x{X.value:02X}")
            window['_Y_'].update(f"0x{Y.value:02X}")
            window['_J_'].update(f"0x{J.value:04X}")
            window['_K_'].update(f"0x{K.value:04X}")
            window['_D_'].update(f"0x{D.value:04X}")
            window['_DATA_'].update(f"0x{data:02X}")
            window['_ADDR_'].update(f"0x{addr:04X}")

            window['_ZF_'].update(text_color=INDC_COLOR[flags['ZF']])
            window['_CF_'].update(text_color=INDC_COLOR[flags['CF']])
            window['_NF_'].update(text_color=INDC_COLOR[flags['NF']])

            for ctrl_sig,val in ctrl.items():
              window[f'_{ctrl_sig}_'].update(text_color=INDC_COLOR[val>0])

            window['_STPR_'].update(f"0x{stack.get_pointer():04X}")

            window['_CNT_'].update(f"{clk_counter:,}")

            sram_values = ""
            for n in range(2**14):
              if mems.sram.value[n] is None:
                sram_values += "--"
              else:
                sram_values += f"{mems.sram.value[n]:02X}"
              # TODO: sram address is hard coded...
              if (n == stack.get_pointer()-0x8000-1):
                sram_values += "*"
              else:
                sram_values += " "
              if (n+1)%8 == 0:
                sram_values += " "

            window['_SRAM_'].update(sram_values)

            stack_values = ""
            for n in range(stack.starting_addr,stack.starting_addr-stack.pointer,-1):
              stack_values += f"0x{n:04X}:  {mems.get(n):02X}\n"

            window['_STACK_'].update(stack_values)
            window['_STACK_USAGE_'].update(stack.pointer)
            window['_STACK_MAX_USAGE_'].update(stack.max_used)

            var_values = ""
            for var in vars:
              # print(var, max_var_width)
              var_name = list(var.keys())[0]
              var_addr = list(var.values())[0]
              var_bytes = [None]*4
              for n in range(len(var_bytes)):
                var_bytes[n] = mems.get(var_addr+n,ignore_uninit=True)

              var_values += f"0x{var_addr:04X} {var_name:<{max_var_width}} "
              if var_bytes[0] is not None:
                var_values += f"0x{var_bytes[0]:02X} "
              else:
                var_values += "?"
              if not (None in var_bytes[0:2]):
                var_values += f"0x{var_bytes[0]:02X}{var_bytes[1]:02X} "
              if not (None in var_bytes):
                var_values += f"0x{var_bytes[0]:02X}{var_bytes[1]:02X}{var_bytes[2]:02X}{var_bytes[3]:02X} "
              var_values += "\n"
            window['_VARS_'].update(var_values)

            if  ii.value in inst_names:
              window['_INST_NAME_'].update(inst_names[ii.value])
            else:
              window['_INST_NAME_'].update("?")

            window['_EXT_ROM_ADDR_'].update(f"0x{mems.ext_eeprom.addr_reg:04X}")
            window['_EXT_ROM_VAL_'].update(f"0x{mems.ext_eeprom.get(0x00):02X}")

            ext_rom_values = ""
            ext_rom_base_addr = mems.ext_eeprom.addr_reg & (~(2**11-1))
            for n in range(2**11):
              rom_addr = ext_rom_base_addr + n
              if n%32 == 0:
                ext_rom_values += f"\n {rom_addr:04X} "
              elif(n)%8 == 0:
                ext_rom_values += " "
              if (rom_addr == mems.ext_eeprom.addr_reg):
                ext_rom_values += "*"
              else:
                ext_rom_values += " "
              if mems.ext_eeprom.value[rom_addr] is None:
                ext_rom_values += "--"
              else:
                ext_rom_values += f"{mems.ext_eeprom.value[rom_addr]:02X}"

            window['_EXT_ROM_'].update(ext_rom_values)

            inst_stats_str = ""
            for x in sorted(inst_stats.items(), key=lambda item: item[1]['calls'], reverse=True):
              name = x[0]
              stats = x[1]
              inst_stats_str += f"{name:<20} {stats}\n"
            window['_TIME_ANALYSIS_'].update(inst_stats_str)

            img = Image.new('L', [IMG_WID,IMG_HEI], 255)
            pixels = img.load()

            for row in range(IMG_HEI):
              for col in range(IMG_WID):
                dr_addr = int(row/8)+col*10
                v = 100
                if mems.dpram.value[dr_addr] is not None:
                  v = ((mems.dpram.value[dr_addr] >> (row%8)) & 0x01) * 255
                pixels[col,row] = v
            window["-IMAGE-"].update(data=ImageTk.PhotoImage(image=img))

            mp = Image.new('L', [128,128], 255)
            mp_pixels = mp.load()

            for row in range(128):
              for col in range(128):
                dr_addr = int(col/4)+row*int(128/4)
                v = 100
                if mems.sram.value[(dr_addr+0xA000)&(mems.sram.size-1)] is not None:
                  v = ((mems.sram.value[(dr_addr+0xA000)&(mems.sram.size-1)] >> (6-((col%4)*2)) ) & 0b11) * 85
                  #print(row,col,dr_addr,(col%4)*2,v)
                mp_pixels[col,row] = v
            window["-MAP-"].update(data=ImageTk.PhotoImage(image=mp))

      print(f"max stack usage {stack.max_used} bytes")
      print(f"clk cycles {clk_counter}")

      # call graph
      # print_call_graph(call_graph, symbols)

      # dead code
      # for l in code:
      #   print(l)

      if GUI:
        window.close()

    except yaml.YAMLError as exc:
      print(exc)

if __name__ == "__main__":
  main()


from urllib.parse import quote
import urllib.request
import requests as req
import json
from bs4 import BeautifulSoup as bs

def get_news(url):
  headers = {"User-Agent":"Mozilla/5.0"}
  news_req = req.get(url, headers=headers)
  news_bs = bs(news_req.content, 'html.parser')

  _text = news_bs.select('#articleBodyContents')[0].get_text().replace('\n', " ")
  btext = _text.replace("// flash 오류를 우회하기 위한 함수 추가 function _flash_removeCallback() {}", "")
  return btext.strip()

#print(get_news("https://news.naver.com/main/read.nhn?mode=LSD&mid=sec&sid1=104&oid=001&aid=0012021683"))

def bscrawling(keywords, max_news=10):
  search_query = " ".join(s for s in keywords)
  search_query = quote(search_query, safe=':/?-=')

  url = 'https://openapi.naver.com/v1/search/'+'news'+'?query='+ search_query + "&display=" + str(max_news) + "&sort=sim"
  client_id = 'CVbcjoXB1ftbVlrp7T2C'
  client_secret = 'XpEq9FDjlm'
  request = urllib.request.Request(url)
  request.add_header("X-Naver-client-Id", client_id)
  request.add_header("X-Naver-Client-Secret", client_secret)
  response = urllib.request.urlopen(request)
  rescode = response.getcode()

  if rescode == 200:
    response_body = response.read()
  else:
    print("Error Code:" + rescode)
    return "Error Occured"

  data = json.loads(response_body)
  urllist= []
  for single_data in data["items"]:
    if single_data["link"].startswith("https://news.naver.com/"):
      urllist.append(single_data["link"])

  body_list = []
  for single_url in urllist:
    body_list.append(get_news(single_url))
  return body_list

#print(bscrawling(["트럼프", "바이든", "투표", "선거"]))


import cv2
from PIL import Image
import os
import numpy as np
from cvzone.SelfiSegmentationModule import SelfiSegmentation

def arrayToImg(arr, path, file_name):
    """
    Converts a numpy array to an image and saves it at the provided path.

    Keyword Arguments:
    arr: the numpy array to be converted.
    path: the file path to save the image.
    file_name: name of the image.
    """
    arr = cv2.cvtColor(arr, cv2.COLOR_BGR2RGB) #Conversion from OpenCV's default BGR to RGB
    img = Image.fromarray(arr)
    img.save(f'{path}{file_name}.png')

def rescaleImg(img, dim = (300, 300)):
    """
    Rescales the OpenCV image to be 300 x 300 pixels.

    Keyword Arguments
    img: the image to be transformed.
    dim: the (m x n) pixel dimensions to be converted to.

    Returns the scaled image.
    """
    return cv2.resize(img, dim)

def removeBackground(img):
    """
    Removes the background of an image in OpenCV.

    Keyword Arguments
    img: the image to remove the background from.

    Returns the image with the background removed.
    """
    segmentor = SelfiSegmentation()
    img = segmentor.removeBG(img, (255, 255, 255), threshold = 0.55)

    return img

def createDirectories():
    """
    Creates the necessary folders for storing rock, paper, and scissors examples.
    """

    directories = [
        "data",
        "data/rps-train",
        "data/rps-test",
        "data/rps-val",
        "data/rps-train/rock",
        "data/rps-train/paper",
        "data/rps-train/scissors",
        "data/rps-val/rock",
        "data/rps-val/paper",
        "data/rps-val/scissors",
        "data/rps-test/rock",
        "data/rps-test/paper",
        "data/rps-test/scissors"
    ]

    try:
        for k in range(len(directories)):
            os.makedirs(directories[k])
    except:
        raise Exception("Training folders already exist.")

def recordExamples(record_type, path, n_examples):
    """
    Given the path to the appropriate directory, saves webcam example images.

    Keyword Arguments:
    record_type: a string that can be one of "rock" "paper" or "scissors."
    path: a string representing the path to which the examples images are saved.
    n_examples: an int representing the number of examples to record.
    """

    camera = cv2.VideoCapture(0)
    coordinates, fontScale, thickness, font, color = (5, 18), 0.45, 2, cv2.FONT_HERSHEY_SIMPLEX, (0, 255, 0)
    all_matrices = []

    result = input(f"Type 'y' to begin recording {record_type} examples: ")

    if result == "y":
        count = 0
        while count < n_examples:
            success, img = camera.read()
            img = rescaleImg(img)
            all_matrices.append(img)

            img = cv2.putText(img, f"n_examples: {count + 1}", coordinates, font, fontScale, color, thickness)
            cv2.imshow("Recording Window", img)

            count += 1
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        camera.release()
        cv2.destroyAllWindows()

        print("Saving images and adding augmentation...")
        for idx, image in enumerate(all_matrices):
            image = removeBackground(image)
            arrayToImg(image, path, str(idx))

def createDataSet(flag, n_imgs_per_class = {"train": 500, "val": 150, "test": 150}):
    """
    Automatically creates a dataset of images per class, using the user's
    images as examples.

    Keyword Arguments
    flag: a string that can be one of "train," "val," or "test" depending on which dataset is being generated.
    n_imgs_per_class: hashmap that can be used to customize train-test split if specified.
    """

    try:
        n_examples = n_imgs_per_class[flag]
        paths = {
            "rock": f"data/rps-{flag}/rock/",
            "paper": f"data/rps-{flag}/paper/",
            "scissors": f"data/rps-{flag}/scissors/"
        }
    except:
        raise Exception("Flag can only be one of \"train\" \"test\" or \"val\"")

    if flag == "train":
        createDirectories()

    for key, value in paths.items():
        recordExamples(key, value, n_examples)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jan 28 15:35:52 2021

@author: zachz
"""

#%% Imports

import numpy as np
import matplotlib.pyplot as plt

# Load in .spydata - there is no way to do this with a command

plt.style.use('seaborn-notebook')

#%% Start building plots

n,x,_ = plt.hist(ofc_pre_taus, bins = 10**np.arange(0,4,0.1), histtype=u'step' ,weights=np.zeros_like(ofc_pre_taus) + 1/len(ofc_pre_taus))
bin_centers = 0.5*(x[1:]+x[:-1])
plt.plot(bin_centers,n,'o-',label='OFC pre')
plt.axvline(x=ofc_pre_mean,color='tab:orange')

n,x,_ = plt.hist(acc_pre_taus, bins = 10**np.arange(0,4,0.1), histtype=u'step' ,weights=np.zeros_like(acc_pre_taus) + 1/len(acc_pre_taus))
bin_centers = 0.5*(x[1:]+x[:-1])
plt.plot(bin_centers,n,'o-',label='ACC pre')
plt.axvline(x=acc_pre_mean,color='tab:red')

n,x,_ = plt.hist(ofc_post_taus, bins = 10**np.arange(0,4,0.1), histtype=u'step' ,weights=np.zeros_like(ofc_post_taus) + 1/len(ofc_post_taus))
bin_centers = 0.5*(x[1:]+x[:-1])
plt.plot(bin_centers,n,'o-',label='OFC post')
plt.axvline(x=ofc_post_mean,color='tab:brown')

n,x,_ = plt.hist(acc_post_taus, bins = 10**np.arange(0,4,0.1), histtype=u'step' ,weights=np.zeros_like(acc_post_taus) + 1/len(acc_post_taus))
bin_centers = 0.5*(x[1:]+x[:-1])
plt.plot(bin_centers,n,'o-',label='ACC post')
plt.axvline(x=acc_post_mean,color='tab:grey')

plt.xlabel('tau (ms)')
plt.ylabel('proportion')
plt.xscale('log')
plt.title('FROOT')
plt.legend()
plt.show()

Across Species Comparison Figures/FROOT_tau_histograms.py


import argparse
import pickle
import statistics

parser = argparse.ArgumentParser()
parser.add_argument("--dataset", type=str, help="The dataset", default="Cora")
args = parser.parse_args()

with open(f"./data/{args.dataset}/test_labels.pkl", "rb") as f:
    y_test = pickle.load(f)
bucket_sizes = []
for i in range(3):
    bucket_sizes.append(len(y_test[i][0]) + len(y_test[i][1]))

exps = [0, 17, 18, 19]
data = [[] for _ in range(len(exps))]
imparity = [[] for _ in range(len(exps))]
imparity_sd = [[] for _ in range(len(exps))]
top_imparity = [0 for _ in range(len(exps))]
accuracies = [0 for _ in range(len(exps))]

TOP_K = [0.2, 0.5, 1]

for indx, exp in enumerate(exps):
    with open(f"res/{args.dataset}/EXP{exp}/imparity.txt", "r") as f:
        temp = sorted([float(line) for line in f.readlines()])
        data[indx] = temp
    for k in TOP_K:
        imparity[indx].append(
            sum(data[indx][: max(1, int(k * len(data[indx])))])
            / max(1, int(k * len(data[indx])))
        )
        imparity_sd[indx].append(
            statistics.pstdev(data[indx][: max(1, int(k * len(data[indx])))])
        )
        top_imparity[indx] = data[indx][0]

    acc = 0
    for i in range(3):
        with open(f"res/{args.dataset}/EXP{exp}/cls_acc/{i}.txt", "r") as f:
            temp = [float(line) for line in f.readlines()]
            acc += (sum(temp) / len(temp)) * bucket_sizes[i]
    acc /= sum(bucket_sizes)
    accuracies[indx] = acc

for indx, exp in enumerate(exps):
    for i, k in enumerate(TOP_K):
        print(
            f"Improvement for exp {exp} averaged over the top {k*100}% of runs: {(1 - (imparity[indx][i]/imparity[0][i]))*100}% \
              \nCoefficient of Variance: {100*imparity_sd[indx][i]/imparity[indx][i]}%"
        )
    print(
        f"Improvement for exp {exp} for the best run:  {(1 - (top_imparity[indx]/top_imparity[0]))*100}%"
    )
    print(f"Accuracy for exp {exp}: {accuracies[indx]*100}%")
    print(f"Accuracy change for exp {exp}: {(accuracies[indx] - accuracies[0])*100}%")
    print()


from bson import ObjectId
from config.config import DB, CONF
from fastapi import APIRouter, Depends, HTTPException
from typing import List
from datetime import datetime, date, timedelta
import logging
import random
import string

from .model_user import UserBase, UserOnDb, Gender, Hobby
from .router_graph import get_user_pie_chart_jenis_kelamin, get_user_garis_chart_new_user_15_hari

router_summary = APIRouter()

def validate_object_id(id_: str):
    try:
        _id = ObjectId(id_)
    except Exception:
        if CONF["fastapi"].get("debug", False):
            logging.warning("Invalid Object ID")
        raise HTTPException(status_code=400)
    return _id

async def _get_user_or_404(id_: str):
    _id = validate_object_id(id_)
    user = await DB.tbl_user.find_one({"_id": _id})
    if user:
        return fix_id(user)
    else:
        raise HTTPException(status_code=404, detail="User not found")

def fix_id(user):
    user["id_"] = str(user["_id"])
    return user

def randomString(stringLength=6):
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(stringLength))

# =================================================================================

@router_summary.get("/summary_user", response_model=dict)
async def get_summary():
    total = await DB.tbl_user.count_documents({})
    laki = await DB.tbl_user.count_documents({"jenisKelamin":"laki-laki"})
    perempuan = await DB.tbl_user.count_documents({"jenisKelamin":"perempuan"})
    start_date = datetime.utcnow() - timedelta(14)
    print(start_date)
    newUser = await DB.tbl_user.count_documents({
        "createTime": {"$gte": start_date}
    })
    return {
        "totalUser":total,
        "laki-laki":laki,
        "perempuan":perempuan,
        "newUser":newUser
    }

@router_summary.get("/dashboard", response_model=dict)
async def get_dashboard():
    data = await get_summary()
    graph1 = await get_user_pie_chart_jenis_kelamin()
    graph2 = await get_user_garis_chart_new_user_15_hari()
    return {"summary":data,"graph1":graph1,"graph2":graph2}


from flask import Flask, redirect, render_template, request
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from io import BytesIO
from flask import Response, make_response, session
from alch_conn import increment_data, fetch_data, show_graph, decrement_data

app = Flask(__name__)

@app.route("/")
def hello_world():
    return render_template("index.html")

@app.route("/data/", methods=["POST"])
def update_data():
    past_choices = get_past(request.cookies)
    if past_choices == None:
        past_choices = []
    choices = form_data(request.form.to_dict(), past_choices)
    increment_data(choices)

    resp = make_response(redirect("/"))
    for val in choices:
        resp.set_cookie(str(val), 'true')
    return resp

@app.route("/remove/", methods=["POST"])
def remove_data():
    past_choices = get_past(request.cookies)
    choices = form_data(request.form.to_dict(), eval(past_choices))
    decrement_data(choices)

    resp = make_response(redirect("/"))
    for val in choices:
        resp.set_cookie(str(val), 'true')
    return resp

@app.route("/graph" )
def get_graph():
    fig = show_graph()
    output = BytesIO()
    FigureCanvas(fig).print_png(output)
    return Response(output.getvalue(), mimetype='image/png')

@app.route("/current")
def show_data():
    return fetch_data()

# Helper function to translate form data into list if the choice is not in the past choices
def form_data(form_dict, past_choices):
    choices = []
    if form_dict.get('date1') == '1' and 'date1' not in past_choices:
        choices.append(0)
    if form_dict.get('date2') == '1' and 'date2' not in past_choices:
        choices.append(1)
    if form_dict.get('date3') == '1' and 'date3' not in past_choices:
        choices.append(2)
    return choices

def set_true(choices):
    past = {0: False, 1: False, 2: False}
    for val in choices:
        past[val] = True
    return str(past)

def set_false(choices):
    past = {0: True, 1: True, 2: True}
    for val in choices:
        past[val] = False
    return str(past)

def get_past(cookie_dict):
    past = []
    if 'date1' in cookie_dict.keys():
        past.append('date1')
    if 'date2' in cookie_dict.keys():
        past.append('date2')
    if 'date3' in cookie_dict.keys():
        past.append('date3')


from flask import g, jsonify

from grocerylistapp import db
from grocerylistapp.errors.exceptions import InvalidUsage
from grocerylistapp.models import recipe_list_associations, user_list_associations
from grocerylistapp.associations.schemas import ListRecipeAssociationSchema

list_recipes_associations_schema = ListRecipeAssociationSchema(many=True)
list_recipes_association_schema = ListRecipeAssociationSchema()

def get_associaton_by_params(param):
    if param.get("recipe") and param.get("list"):
        print(param.get("recipe"), param.get("list"))
        association = db.session.query(recipe_list_associations)\
            .filter(recipe_list_associations.c.recipe == param.get("recipe"))\
            .filter(recipe_list_associations.c.grocery_list == param.get("list")).first()
        print(association)
        if not association:
            raise InvalidUsage("No association exists between the specified list and recipe", 404)
        return list_recipes_association_schema.dump(association)
    if param.get("recipe"):
        associations = db.session.query(recipe_list_associations).filter(recipe_list_associations.c.recipe == param.get("recipe")).all()
    elif param.get("list"):
        associations = db.session.query(recipe_list_associations).filter(recipe_list_associations.c.grocery_list == param.get("list")).all()
    else:
        associations = db.session.query(recipe_list_associations).all()

    return list_recipes_associations_schema.dump(associations)

def load_list_and_check_permissions(association_to_modify, association_schema):
    list_to_modify, resource_to_change = association_schema.load(association_to_modify)
    if g.user not in list_to_modify.editors and g.user.id != list_to_modify.creator_id:
        raise InvalidUsage("You don't have permission to modify this list.", 401)

    return list_to_modify, resource_to_change

def get_association_or_404(id_, table):
    association = db.session.query(table).filter_by(id=id_).first()
    if not association:
        raise InvalidUsage("The association you're looking for can't be found.", 404)
    return association

def add_association(new_resource_for_list, list_to_add_to):
    if new_resource_for_list not in list_to_add_to:
        list_to_add_to.append(new_resource_for_list)
        db.session.commit()


from operator import truediv
from django import template

from ..models import Consulta
register = template.Library()

@register.filter(name="remover")
def remover (texto, r):
    return texto.replace(r, "")

@register.filter(name="substituir")
def substituir(string, encontrar, subs):
    return string.replace(encontrar, subs)

@register.filter(name="verificardddpr")
def verificardddpr(telefone):
    ddd = telefone[0:4]
    if(ddd == "44"):
        return True
    else:
        return False

@register.filter(name="esta_no_grupo")
def esta_no_grupo (usuario, nome_do_grupo):
    if(usuario.groups.filter(name=nome_do_grupo)):
        return True
    else:
        return False

@register.filter(name="verificarStatus")
def verificarStatus(status):
    if(status == 'Pendente'):
        return 'bg-pendente'
    elif(status == 'Realizada'):
        return 'bg-realizada'
    elif(status == 'Cancelada'):
        return 'bg-cancelada'
    elif(status == 'Agendada'):
        return 'bg-agendada'
    else:
        return 'bg-light'

# @register.filter(name="textStatus")
# def textStatus(status):
#     if(status == 'Pendente' or status == 'Realizada' or status == 'Cancelada' or status == 'Agendada'):
#         return 'text-white'
#     else:
#         return 'text-dark'

@register.simple_tag(name="total_consultas")
def total_consultas():
      return Consulta.objects.count()


"""
Instruction-tuning with LoRA on the Alpaca dataset.

Note: If you run into a CUDA error "Expected is_sm80 to be true, but got false", uncomment the line
`torch.backends.cuda.enable_flash_sdp(False)` in the script below (see https://github.com/Lightning-AI/lit-llama/issues/101).
"""
import random
import sys
from pathlib import Path
import os
import time

import lightning as L
import numpy as np
import torch

# support running without installing as a package
from tqdm import tqdm
from collections import OrderedDict

wd = Path(__file__).parent.parent.resolve()
sys.path.append(str(wd))

from lora_pill import mark_only_lora_pill_as_trainable, lora_pill, lora_pill_state_dict
from model_extra import LLaMA_extra, LLaMAExtraConfig

instruction_tuning = True
eval_interval = 100
save_interval = 100
eval_iters = 100
log_interval = 1

# Hyperparameters
learning_rate = 3e-4
batch_size = 128
micro_batch_size = 4
gradient_accumulation_iters = batch_size // micro_batch_size
assert gradient_accumulation_iters > 0
max_iters = 20000 * 3 // micro_batch_size
weight_decay = 0.0
max_seq_length = 256  # see scripts/prepare_alpaca.py
lora_r = 8
lora_alpha = 16
lora_dropout = 0.05
warmup_iters = 100

def main(
        data_dir: str = "../data/spatial_dataset",
        pretrained_path: str = "../checkpoints/lit-llama/7B/lit-llama.pth",
        tokenizer_path: str = "../checkpoints/lit-llama/tokenizer.model",
        out_dir: str = "../out/pill/poi",
):
    fabric = L.Fabric(accelerator="cuda", devices=3, precision="bf16-true")
    fabric.launch()
    fabric.seed_everything(1337 + fabric.global_rank)

    if fabric.global_rank == 0:
        os.makedirs(out_dir, exist_ok=True)

    train_data, test_data = load_datasets(data_dir=data_dir, cal_max_len=False)

    config = LLaMAExtraConfig.from_name("7B")
    config.block_size = max_seq_length

    checkpoint = torch.load(pretrained_path)

    with fabric.init_module(), lora_pill(r=lora_r, alpha=lora_alpha, dropout=lora_dropout, enabled=True):
        model = LLaMA_extra(config)
        # strict=False because missing keys due to LoRA weights not contained in checkpoint state
        model.load_state_dict(checkpoint, strict=False)

    mark_only_lora_pill_as_trainable(model)

    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
    model, optimizer = fabric.setup(model, optimizer)
    train(fabric, model, optimizer, train_data, test_data, tokenizer_path, out_dir)

    # Save the final LoRA checkpoint at the end of training
    checkpoint = lora_pill_state_dict(model)
    fabric.save(os.path.join(out_dir, "llama-7b-lora-pill-finetuned.pth"), checkpoint)

def train(
        fabric: L.Fabric,
        model: torch.nn.Module,
        optimizer: torch.optim.Optimizer,
        train_data: list,
        val_data: list,
        tokenizer_path: str,
        out_dir: str,
) -> None:
    """The training loop.

    Loosely based on the nanoGPT implementation: https://github.com/karpathy/nanoGPT.
    """
    step_count = 0
    BH = BatchHandler(train_data)
    pbar = tqdm(total=max_iters, ncols=100)
    for iter_num in range(max_iters):

        if step_count <= warmup_iters:
            # linear warmup
            lr = learning_rate * step_count / warmup_iters
            for param_group in optimizer.param_groups:
                param_group['lr'] = lr

        t0 = time.time()

        input_ids, targets, spatial_addition = BH.get_batch(fabric)
        with fabric.no_backward_sync(model, enabled=((iter_num + 1) % gradient_accumulation_iters != 0)):
            logits, space = model(input_ids, spatial_addition=spatial_addition)
            loss_1 = loss_fn(logits, targets)
            loss_2 = loss_sp(space, spatial_addition)
            loss = loss_1 + loss_2
            fabric.backward(loss / gradient_accumulation_iters)

        if (iter_num + 1) % gradient_accumulation_iters == 0:
            optimizer.step()
            optimizer.zero_grad()
            step_count += 1

            if step_count % eval_interval == 0:
                val_loss = validate(fabric, model, val_data, tokenizer_path)
                fabric.print(f"step {iter_num}: val loss {val_loss:.4f}")
                fabric.barrier()

            if step_count % save_interval == 0:
                print(f"Saving LoRA weights to {out_dir}")
                # We are only saving the LoRA weights
                # TODO: Provide a function/script to merge the LoRA weights with pretrained weights
                checkpoint = lora_pill_state_dict(model)
                fabric.save(os.path.join(out_dir, f"iter-{iter_num:06d}-ckpt.pth"), checkpoint)

        dt = time.time() - t0
        postfix = OrderedDict([
            ('loss', f'{loss.item():.4f}'),
            ('time', f'{dt * 1000:.2f}ms'),
        ])
        pbar.set_postfix(postfix)
        pbar.update(1)
        # if iter_num % log_interval == 0:
        #     fabric.print(f"iter {iter_num}: loss {loss.item():.4f}, time: {dt * 1000:.2f}ms")
    pbar.close()

@torch.no_grad()
def validate(fabric: L.Fabric, model: torch.nn.Module, val_data: list, tokenizer_path: str) -> torch.Tensor:
    fabric.print("Validating ...")
    model.eval()
    losses = torch.zeros(eval_iters)
    BH = BatchHandler(val_data)
    for k in range(eval_iters):
        input_ids, targets, spatial_addition = BH.get_batch(fabric)
        logits, space = model(input_ids, spatial_addition=spatial_addition)
        loss_1 = loss_fn(logits, targets)
        loss_2 = loss_sp(space, spatial_addition)
        loss = loss_1 + loss_2
        losses[k] = loss.item()
    out = losses.mean()

    # produce an example:
    # instruction = "predict which place the user will visit: "
    #
    # output = generate_response(model, instruction, tokenizer_path)
    # fabric.print(instruction)
    # fabric.print(output)

    model.train()
    return out.item()

def loss_fn(logits, targets):
    # shift the targets such that output n predicts token n+1
    logits = logits[..., :-1, :].contiguous()
    targets = targets[..., 1:].contiguous()
    loss = torch.nn.functional.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
    return loss

def loss_sp(coord, targets):
    # shift the targets such that output n predicts token n+1
    coord = coord[..., :-1, :].contiguous()
    targets = targets[..., 1:, :].contiguous()
    loss = torch.nn.functional.mse_loss(coord, targets)
    return loss

class BatchHandler:
    def __init__(self, data: list):
        self.index = list(range(len(data)))
        self.data = data

    def get_batch(self, fabric: L.Fabric):
        if len(self.index) < micro_batch_size:
            self.__init_ix()
        ix = [self.__get_ix() for _ in range(micro_batch_size)]

        input_ids = [self.data[i]["sentence_token"].type(torch.int64) for i in ix]
        labels = [self.data[i]["sentence_token"].type(torch.int64) for i in ix]
        spatial_addition = [self.data[i]["spatial_addition"] for i in ix]

        max_len = max(len(s) for s in input_ids)

        x = torch.stack([self.pad_right(x, pad_id=0, max_len=max_len) for x in input_ids])
        y = torch.stack([self.pad_right(x, pad_id=-1, max_len=max_len) for x in labels])
        s = torch.stack([self.space_pad_right(s, pad_id=0, max_len=max_len) for s in spatial_addition])
        x, y, s = fabric.to_device((x.pin_memory(), y.pin_memory(), s.pin_memory()))
        return x, y, s

    @staticmethod
    def pad_right(x, pad_id, max_len):
        # pad right based on the longest sequence
        n = max_len - len(x)
        return torch.cat((x, torch.full((n,), pad_id, dtype=x.dtype)))

    @staticmethod
    def space_pad_right(s, pad_id, max_len):
        # pad right based on the longest sequence
        s = torch.stack(s)
        n = max_len - len(s)
        return torch.cat((s, torch.full((n, s.shape[-1],), pad_id, dtype=s.dtype)))

    def __get_ix(self):
        i = random.randint(0, len(self.index)-1)
        ix = self.index[i]
        self.index.remove(ix)
        return ix

    def __init_ix(self):
        self.index = list(range(len(self.data)))

def load_datasets(data_dir, cal_max_len=False):
    train_data = torch.load(os.path.join(data_dir, "poi_train.pt"))
    test_data = torch.load(os.path.join(data_dir, "poi_test.pt"))
    if cal_max_len:
        l1 = calMaxLength(train_data)
        l2 = calMaxLength(test_data)
        global max_seq_length
        max_seq_length = l1 if l1 > l2 else l2
        print(max_seq_length)
    return train_data, test_data

def calMaxLength(data):
    max_length = 0
    for tuple in data:
        length = len(tuple["sentence_token"])
        if length > max_length:
            max_length = length
    return max_length

if __name__ == "__main__":
    # Uncomment this line if you see an error: "Expected is_sm80 to be true, but got false"
    # torch.backends.cuda.enable_flash_sdp(False)
    torch.set_float32_matmul_precision("high")

    from jsonargparse.cli import CLI

    CLI(main)


__DESCRIPTION__="""
By M.Maris - 2014 Nov 20 -
Converts a couple of XY profiles into a patch
"""

class profiles2patch :
  """converts a couple of XY profiles into a patch"""
  def __init__(self,xylow,xyup) :
    """xylow, xyup = two rows columns forl ower and upper profile"""
    import numpy as np
    x=[]
    y=[]
    for k in range(len(xyup[0])) :
      x.append(xyup[0][k])
      y.append(xyup[1][k])
    for k in range(len(xylow[0])-1,-1,-1) :
      x.append(xylow[0][k])
      y.append(xylow[1][k])
    x.append(xyup[0][0])
    y.append(xyup[1][0])
    self.xy = np.array([x,y]).transpose()
  def __call__(self,*arg,**karg) :
    """creates a patch, keyowrds from patches"""
    from matplotlib import patches
    return patches.Polygon(self.xy,**karg)