import tkinter as tk
from datetime import datetime
from PIL import Image, ImageTk
from io import BytesIO
from tkhtmlview import HTMLLabel
from urllib.request import Request, urlopen
import requests
from flask import request
from six.moves import urllib
import urllib
from time import sleep

import tkinter as tk
from pandas import DataFrame
import matplotlib.pyplot as plt

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

import refinitiv
import unpri
import unep_fi
import summary
import icma
import ungc
import iigcc
import topic_model
import company_details

## FIREBASE
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.realpath("group18---natwest-firebase.json")

import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore

cred = credentials.Certificate("group18---natwest-firebase.json")
firebase_admin.initialize_app(cred)

db = firestore.client()

from pathlib import Path

# from tkinter import *
# Explicit imports to satisfy Flake8
from tkinter import Tk, Canvas, Entry, Text, Button, PhotoImage, Label

OUTPUT_PATH = Path(__file__).parent
ASSETS_PATH = OUTPUT_PATH / Path("./assets")

def relative_to_assets(path: str) -> Path:
    return ASSETS_PATH / Path(path)

window = Tk()

window.title("ESG Dashboard")
window.geometry("1371x800")
window.configure(bg = "#FFFFFF")

##################
## FUNCTIONS
##################

def unavailable_refinitiv():
    canvas.create_rectangle(359.0,166.0, 797.0, 380.0, fill="black",outline="")
    canvas.create_text(370.0, 310.0,anchor="nw",text='Sorry, the ESG Rating is unavailable',fill="white",
                           font=("Raleway SemiBold", 18 * -1))

def show_esg_rating(input_company) :
    esg_rating = refinitiv.RefinitivList(input_company)

    #title
    canvas.create_text(360.0,125.0,anchor="nw",text="ESG Rating",
                       fill="#192159",font=("Raleway Bold", 18 * -1))
    #bigbox
    #update
    canvas.create_rectangle(359.0,166.0, 797.0, 380.0, fill="#FFFFFF",outline="")

    #Placeholder boxes for ESG Rating
    canvas.create_rectangle(359.0,166.0,567.0,274.0, fill="#401564",outline="")
    canvas.create_rectangle(589.0,166.0,797.0,274.0, fill="#401564",outline="")
    canvas.create_rectangle(359.0,287.0,491.0,380.0,fill="#F5F4FD",outline="")
    canvas.create_rectangle(512.0,287.0,644.0,380.0,fill="#F5F4FD",outline="")
    canvas.create_rectangle(665.0,287.0,797.0,380.0,fill="#F5F4FD",outline="")

    #Label for ESG Rating
    canvas.create_text(400.0,175.0,anchor="nw",text="Overall Rating",fill="#FFFFFF",
                       font=("Raleway Bold", 18 * -1))
    canvas.create_text(633.0,175.0,anchor="nw",text="Industry Rank",fill="#FFFFFF",
                       font=("Raleway Bold", 18 * -1))
    canvas.create_text(380.0,303.0,anchor="nw",text="Environment",fill="#192159",
                       font=("Raleway Bold", 14 * -1))
    canvas.create_text(556.0,303.0,anchor="nw",text="Social",fill="#192159",
                       font=("Raleway Bold", 14 * -1))
    canvas.create_text(689.0,303.0,anchor="nw",text="Governance",fill="#192159",
                       font=("Raleway Bold", 14 * -1))

    if (esg_rating == 'NA'):
        unavailable_refinitiv()
    else:
        esg_score = esg_rating[1][-2:]
        env_score = esg_rating[2][-2:]
        social_score = esg_rating[3][-2:]
        gov_score = esg_rating[4][-2:]
        esg_rank = esg_rating[5].split(':')[1]
        actual_rank = int(esg_rank.split('/')[0][1:])
        industry = int(esg_rank.split('/')[1])

        #Determine the color for company by the industry percentage
        def what_color():
            per30 = int(industry * 3 / 10)
            per70 = int(industry * 7 / 10)
            if (actual_rank < per30):
                return "#90D27F"
            elif (actual_rank < per70):
                return "#FFAD71"
            else:
                return "#FF8080"

        #The values of the numbers
        canvas.create_text(434.0, 196.0, anchor="nw",text=esg_score, fill=what_color(),
                           font=("Raleway SemiBold", 48 * -1))
        canvas.create_text(596.0, 196.0, anchor="nw",text=esg_rank, fill=what_color(),
                           font=("Raleway SemiBold", 48 * -1))
        canvas.create_text(406.0,323.0,anchor="nw",text=env_score,fill="#192159",
                           font=("Raleway SemiBold", 30 * -1))
        canvas.create_text(563.0,323.0,anchor="nw",text=social_score,fill="#192159",
                           font=("323 SemiBold", 30 * -1))
        canvas.create_text(713.0, 323.0,anchor="nw", text=gov_score, fill="#192159",
                           font=("Raleway SemiBold", 30 * -1))

def show_company_profile(input_company):
    canvas.create_rectangle(0.0,150.0,295.0,800.0,
                            fill='#F4F4FC',outline="")
    canvas.create_text(122.0,175.0,anchor="nw",text=input_company.upper(),
                       fill="#192159",font=("Raleway SemiBold", 24 * -1))
    logo_list = company_details.get_logo(input_company)
    found = False
    for x in logo_list:
        try:
            URL = Request(x, headers={'User-Agent': 'Mozilla/5.0'})
            raw_data = urlopen(URL).read()
            im = Image.open(BytesIO(raw_data))
            photo = ImageTk.PhotoImage(im)
            h = photo.height()
            w = photo.width()
            ratio = min(200/w, 100/h)
            resized = im.resize((int(w*ratio),int(h*ratio)), Image.ANTIALIAS)
            photo = ImageTk.PhotoImage(resized)
            #label = tk.Label(image=photo)
            #label = Label(canvas, image=photo)
            bw = (295 - w*ratio)//2
            #label = Label(canvas, image=photo, borderwidth = bw, relief = "solid")
            label = Label(canvas, image=photo,
                          highlightthickness=bw,
                          highlightbackground="#F4F4FC",
                          highlightcolor="#F4F4FC")
            label.image = photo
            label.place(x=0, y = 300)
        except:
            continue
        else:
            found = True
            break

    if not found:
        canvas.create_rectangle(90.0,220.0,204.0,334.0,
                                fill="#AE99C0",outline="")
    continent_country = company_details.get_country_continent(input_company)
    canvas.create_text(95.0, 240.0,anchor="nw",text="Continent: "+continent_country[0],
                       fill="#000000",font=("Raleway Regular", 14 * -1))
    canvas.create_text(95.0,220.0,anchor="nw",text="Country: "+continent_country[1],
                       fill="#000000",font=("Raleway Regular", 14 * -1))

def show_summary(input_company):
    result = topic_model.get_summary(input_company)
    if not result:
        result = summary.get_summary(input_company)
    canvas.create_rectangle(833.0,123.0, 1371.0, 650.0, fill="white",outline="")
    canvas.create_text(836.0, 123.0, anchor="nw",text="ESG Article Summarization",fill="#192159",
                       font=("Raleway Bold", 18 * -1))
    if (len(result) < 1000) :
        counter = (1000 - len(result))//100
        result += "\n "*counter
    sum_label = tk.Label(canvas, text=result, font=("Raleway Bold", 14 * -1),
                 wraplength=500, justify="left", fg="#000000")
    sum_label.place(x=833, y=164)

def show_keywords(input_company):
    canvas.create_text(359.0,440.0,anchor="nw",text="Sustainability Report Topic Modelling",
                       fill="#192159",font=("Raleway Bold", 18 * -1))

def show_membership(input_company):
    canvas.create_rectangle(835.0,480.0,1257.0,602.0,
                            fill="#F4F4FC",outline="")
    canvas.create_rectangle(876.0,480.0,876.0,710.0,
                            fill="#FFFFFF",outline="")
    canvas.create_rectangle(1126.0,480.0,1126.0,710.0,
                            fill="#FFFFFF",outline="")
    canvas.create_rectangle(835.0,480.0,1257.0,710.0,
                            fill="#F5F4FD",outline="")
    canvas.create_text(836.0,440.0,anchor="nw",text="Membership Table",
                       fill="#192159",font=("Raleway Bold", 18 * -1))
    canvas.create_text(846.0,483.0,anchor="nw",text="No.",
                       fill="#AE99C0",font=("Raleway SemiBold", 12 * -1))
    canvas.create_text(897.0,483.0,anchor="nw",text="Membership Name",
                       fill="#AE99C0",font=("Raleway SemiBold", 12 * -1))
    canvas.create_text(1141.0,483.0,anchor="nw",text="Status",
                       fill="#AE99C0",font=("Raleway SemiBold", 12 * -1))

    get_unpri(input_company)
    get_unep_fi(input_company)
    get_icma(input_company)
    get_ungc(input_company)
    get_iigcc(input_company)

def get_unpri(input_company):
    canvas.create_text(852.0,517.0,anchor="nw",text="1",fill="#000000",
                       font=("Raleway Regular", 14 * -1))
    canvas.create_text(897.0,519.0,anchor="nw",text="UNPRI",fill="#000000", font=("Raleway Regular", 12 * -1))
    if (unpri.is_member(input_company) is True):
        logo = Image.open("image_8.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=510)
    else:
        logo = Image.open("image_9.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=510)

def get_unep_fi(input_company):
    canvas.create_text(852.0,545.0,anchor="nw",text="2",fill="#000000",
                       font=("Raleway Regular", 14 * -1))

    canvas.create_text(897.0,548.0,anchor="nw",text="UNEP FI",fill="#000000",
                       font=("Raleway Regular", 12 * -1))

    if (unep_fi.is_member_unepfi(input_company) is True):
        logo = Image.open("image_8.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=540)
    else:
        logo = Image.open("image_9.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=540)

def get_icma(input_company):
    canvas.create_text(852.0,578.0,anchor="nw",text="3",fill="#000000",
                       font=("Raleway Regular", 14 * -1))

    canvas.create_text(897.0,580.0,anchor="nw",text="ICMA",fill="#000000",
                       font=("Raleway Regular", 12 * -1))

    if (icma.is_member_icma(input_company) is True):
        logo = Image.open("image_8.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=570)
    else:
        logo = Image.open("image_9.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=570)

def get_iigcc(input_company):
    canvas.create_text(852.0,607.0,anchor="nw",text="4",fill="#000000",
                       font=("Raleway Regular", 14 * -1))

    canvas.create_text(897.0,609.0,anchor="nw",text="IIGCC",fill="#000000",
                       font=("Raleway Regular", 12 * -1))

    if (iigcc.is_member_iigcc(input_company) is True):
        logo = Image.open("image_8.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=600)
    else:
        logo = Image.open("image_9.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=600)

def get_ungc(input_company):
    canvas.create_text(852.0,638.0,anchor="nw",text="5",fill="#000000",
                       font=("Raleway Regular", 14 * -1))

    canvas.create_text(897.0,640.0,anchor="nw",text="UNGC",fill="#000000",
                       font=("Raleway Regular", 12 * -1))

    if (ungc.is_member_ungc(input_company) is True):
        logo = Image.open("image_8.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=630)
    else:
        logo = Image.open("image_9.png")
        logo = ImageTk.PhotoImage(logo)
        logo_label = tk.Label(image=logo, bg="#F5F4FD")
        logo_label.image = logo
        logo_label.place(x=1130, y=630)

def show_topic_model(input_company):
    canvas.create_rectangle(359.0,480.0,760.0,710.0,fill="#F4F4FC",outline="")
    topic_dist = topic_model.get_topic_model(input_company)
    canvas.create_text(532.0, 714.0,anchor="nw",text="Topics",
                       fill="#000000",font=("Raleway Bold", 14 * -1))
    canvas.create_text(332.0,620.0,anchor="nw",text="Proportion",
                       fill="#000000",font=("Raleway Bold", 14 * -1), angle=90)

    canvas.create_text(410, 700, anchor=tk.SW, text='Environment')
    canvas.create_text(537, 700, anchor=tk.SW, text='Social')
    canvas.create_text(633, 700, anchor=tk.SW, text='Governance')
    data = list(topic_dist)

    canvas.create_rectangle(388.0,680.0,740.0,682.0,fill="black",outline="") #Horizontal

    canvas.create_rectangle(388.0,500.0,390.0,680.0,fill="black",outline="") #Vertical

##    label = list(map(lambda x:"Topic " + str(x[0]),topic_dist))

    canvas.pack()

    c_width = 600
    c_height = 600

    # the variables below size the bar graph
    # experiment with them to fit your needs
    # highest y = max_data_value * y_stretch
    y_stretch = 190

    # gap between lower canvas edge and x axis
    y_gap = 30

    # stretch enough to get all data items in
    x_stretch = 40
    x_width = 70

    # gap between left canvas edge and y axis
    x_gap = 40

    # x place
    x_place = 370
    y_place = 110

    counter = 0

    for x, y in enumerate(data):
        # calculate reactangle coordinates (integers) for each bar
        x0 = x_place + x * x_stretch + x * x_width + x_gap
        y0 = y_place + c_height - (y * y_stretch + y_gap)
        x1 = x_place + x * x_stretch + x * x_width + x_width + x_gap
        y1 = y_place + c_height - y_gap
        # draw the bar
        canvas.create_rectangle(x0, y0, x1, y1, fill="#401564")
        # put the y value above each bar
        canvas.create_text(x0+15, y0, anchor=tk.SW, text=str(round(y,3)))
        #name_list = ['Environment', 'Social', 'Governance']
        #canvas.create_text(x0+3, y0-13, anchor=tk.SW, text=name_list[counter])
        #counter+=1

def click():
    root = tk.Tk()
    label = tk.Label(root, text="Please wait for task to finish.",
                     font=("Raleway Bold", 18 * -1))
    label.pack()

    def task():
        # The window will stay open until this function call ends.

        company_name = variable.get().lower()
        show_company_profile(company_name) # Company Profile
        show_summary(company_name) # Summary
        show_keywords(company_name) #Keywords
        show_membership(company_name) #Membership
        show_esg_rating(company_name) #Ratings
        show_topic_model(company_name)

        #image_image_10 = PhotoImage(file=relative_to_assets("image_10.png"))
        #image_10 = canvas.create_image(515.0,675.0,image=image_image_10)
        root.destroy()

    window.after(200, task)
    label.after(1000 , lambda: label.destroy())

#Header Left
canvas = Canvas(window,bg = "#FFFFFF",height = 814,width = 1371,
                bd = 0,highlightthickness = 0,relief = "ridge")
canvas.place(x = 0, y = 0)
canvas.create_rectangle(0.0,0.0,295.0,900.0,fill="#F4F4FC",outline="")

sum_label = tk.Label(canvas, text="", font=("Raleway Bold", 14 * -1),
                     fg="#000000")
sum_label.place(x=355, y=153)

#info Color
green = PhotoImage(file=relative_to_assets("green.png"))
green_pic = canvas.create_image(438.0,405.0,image=green)
orange = PhotoImage(file=relative_to_assets("orange.png"))
orange_pic = canvas.create_image(570.0,405.0,image=orange)
red = PhotoImage(file=relative_to_assets("red.png"))
red_pic= canvas.create_image(711.0,405.0,image=red)

#Right - Title

canvas.create_text(359.0,45.0,anchor="nw",
                   text="Companies’ ESG Performance",fill="#192159",
                   font=("Raleway Bold", 36 * -1))

canvas.create_text(359.0,91.0,anchor="nw",
                   text="Performance based on Ratings,  ESG Keywords, and Membership",
                   fill="#BCBCBC",font=("Raleway Bold", 14 * -1))

#Left - Title

canvas.create_text(102.0,42.0,anchor="nw",text="NatWest Markets",
                   fill="#192159",font=("Raleway Bold", 18 * -1))
image_image_1 = PhotoImage(file=relative_to_assets("image_1.png"))
image_1 = canvas.create_image(51.0,52.0,image=image_image_1)

#Search Bar
variable = tk.StringVar(window)

entry_1 = Entry(bd=0,bg="#FFFFFF",textvariable=variable,
                highlightthickness=0)
entry_1.place(x=24.0,y=106.0,width=257.73095703125,
              height=36.19720458984375)
button_image_1 = PhotoImage(file=relative_to_assets("button_1.png"))
button_1 = Button(image=button_image_1,borderwidth=0,
                  highlightthickness=0,command=click,relief="flat")
button_1.place(x=245.0,y=112.0,width=28.0,height=28.0)

window.resizable(False, False)
window.mainloop()


import itertools
class Solution(object):
    def validSquare(self, p1, p2, p3, p4):
        """
        :type p1: List[int]
        :type p2: List[int]
        :type p3: List[int]
        :type p4: List[int]
        :rtype: bool
        """
        ls = [p1,p2,p3,p4]
        fd = False
        def dis(p1,p2):
            return (p1[0]-p2[0])**2 + (p1[1] -p2[1])**2
        for comb in itertools.permutations(ls):
            p1,p2,p3,p4 = comb
            if dis(p1,p2) == dis(p2,p3) ==dis(p3,p4) == dis(p4,p1) and dis(p1,p3) ==dis(p2,p4):
                fd = True
        return fd

re = Solution().validSquare(p1 = [0,0], p2 = [1,1], p3 = [1,0], p4 = [0,1])
print(re)


# -*- coding: utf-8 -*-
import codecs
import frida
import sys
import threading

#device = frida.get_remote_device()
device = frida.get_device_manager().enumerate_devices()[-1]
print(device)

pending = []
sessions = []
scripts = []
event = threading.Event()

jscode = """
Java.perform(function () {
    var MainActivity = Java.use('com.l.testprocess.MainActivity1');
    var clazz = Java.use('java.lang.Class');
	MainActivity.onCreate.implementation = function(savedInstanceState){
		send('MainActivity')
		this.onCreate(savedInstanceState);
	}

});
"""

def on_spawned(spawn):
	print('on_spawned:', spawn)
	pending.append(spawn)
	event.set()

def spawn_added(spawn):
	print('spawn_added:', spawn)
	event.set()
	if(spawn.identifier.startswith('com.l.testprocess')):
		session = device.attach(spawn.pid)
		script = session.create_script(jscode)
		script.on('message', on_message)
		script.load()
		device.resume(spawn.pid)

def spawn_removed(spawn):
	print('spawn_added:', spawn)
	event.set()

def on_message(spawn, message, data):
	print('on_message:', spawn, message, data)

def on_message(message, data):
	if message['type'] == 'send':
		print("[*] {0}".format(message['payload']))
	else:
		print(message)

device.on('spawn-added', spawn_added)
device.on('spawn-removed', spawn_removed)
device.on('child-added', on_spawned)
device.on('child-removed', on_spawned)
device.on('process-crashed', on_spawned)
device.on('output', on_spawned)
device.on('uninjected', on_spawned)
device.on('lost', on_spawned)
device.enable_spawn_gating()
event = threading.Event()
print('Enabled spawn gating')

pid = device.spawn(["com.l.testprocess"])

session = device.attach(pid)
print("[*] Attach Application id:",pid)
device.resume(pid)
# print("[*] Application onResume")
# script = session.create_script(jscode)
# script.on('message', on_message)
# print('[*] Running CTF')
# script.load()
sys.stdin.read()


from keras.datasets import mnist
import numpy as np
import tensorflow
from matplotlib import pyplot
from sklearn.cluster import KMeans
from sklearn.metrics import mean_squared_error

# Load the data to be used for the Plot and Kmeans clustering
(train_X, train_y), (test_X, test_y) = mnist.load_data()
train_data = train_X
train_labels = train_y

# Print the shape of the dataset
print('X_train: ' + str(train_X.shape))
print('Y_train: ' + str(train_y.shape))
print('X_test:  ' + str(test_X.shape))
print('Y_test:  ' + str(test_y.shape))

# Assign classes that fit datapoints with labels 0-9
label_0 = train_data[train_labels==0]
label_1 = train_data[train_labels==1]
label_2 = train_data[train_labels==2]
label_3 = train_data[train_labels==3]
label_4 = train_data[train_labels==4]
label_5 = train_data[train_labels==5]
label_6 = train_data[train_labels==6]
label_7 = train_data[train_labels==7]
label_8 = train_data[train_labels==8]
label_9 = train_data[train_labels==9]

#Apply KMeans clustering onto classes of all labels, and calculate the centroid out of 10 clusters.
kmeans_0 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_0)
sample_0 = kmeans_0.cluster_centers_
kmeans_1 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_1)
sample_1 = kmeans_1.cluster_centers_
kmeans_2 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_2)
sample_2 = kmeans_2.cluster_centers_
kmeans_3 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_3)
sample_3 = kmeans_3.cluster_centers_
kmeans_4 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_4)
sample_4 = kmeans_4.cluster_centers_
kmeans_5 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_5)
sample_5 = kmeans_5.cluster_centers_
kmeans_6 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_6)
sample_6 = kmeans_6.cluster_centers_
kmeans_7 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_7)
sample_7 = kmeans_7.cluster_centers_
kmeans_8 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_8)
sample_8 = kmeans_8.cluster_centers_
kmeans_9 = KMeans(n_clusters=10, init='k-means++',random_state=0).fit(label_9)
sample_9 = kmeans_9.cluster_centers_

#Concatenate centroids into a single array, as prototypes
proto_sample = np.concatenate((sample_0,sample_1,
                               sample_2,sample_3,
                               sample_4,sample_5,
                               sample_6,sample_7,
                               sample_8,sample_9
                              ), axis = 0
                             )

# Generate labels for the prototypes of the sample.
sample_0_labels = np.full((10,), 0)
sample_1_labels = np.full((10,), 1)
sample_2_labels = np.full((10,), 2)
sample_3_labels = np.full((10,), 3)
sample_4_labels = np.full((10,), 4)
sample_5_labels = np.full((10,), 5)
sample_6_labels = np.full((10,), 6)
sample_7_labels = np.full((10,), 7)
sample_8_labels = np.full((10,), 8)
sample_9_labels = np.full((10,), 9)

#Contatenate the labels of the sample into a single array
proto_labels = np.concatenate((sample_0_labels,sample_1_labels,
                               sample_2_labels,sample_3_labels,
                               sample_4_labels,sample_5_labels,
                               sample_6_labels,sample_7_labels,
                               sample_8_labels,sample_9_labels),
                              axis = 0
                             )

# Define a function that randomly chooses an M number of training points as prototypes for the KNN classification
#fof MNIST
def rand_prototypes(M):
    """
    rand_prototypes() takes in a parameter M and returns M. M is an int.
    M is the number of data points to be samples.
    The function rand_prototypes returns numpy arrays train_data and train_labels of M randomly chosen
    datapoints and labels.
    """

    indices = np.random.choice(len(train_labels), M, replace=False)
    return train_data[indices, :], train_labels[indices]

#Define a function "comparison()" that compares the output of the random selection classification with the
#Kmeans classification
#Initiailize the function
@interact_manual(M=(10000, 6000, 1000), rounds=(1, 10))
def comparison(M, rounds):
    """
    comparison() takes in a value M and rounds and returns the mean-squared error of both randomly selected prototypes
    and KMeans clustered prototypes.
    Calculations are done #rounds times to improve randomness in the study.
    Both input parameters are ints.
    """
    print("Comparing your prototype selection method to random prototype selection...")
    rand_err, rand_std = mean_squared_error(rand_prototypes, M, rounds)
    my_err, my_std = mean_squared_error(my_prototypes, M, rounds)

    print;
    print("Number of prototypes:", M)
    print("Number of trials:", rounds)
    print("Error for random prototypes:", rand_err)
    print("Error for your prototypes:", my_err);
    print
    if rand_err < my_err:
        print("The randomly selected prototypes have lower error.")
    else:
        print("The KMeans clustered prototypes have lower error.")

# Plotting the images from the MNIST dataset
for i in range(9):
    pyplot.subplot(330 + 1 + i)
pyplot.imshow(train_X[i], cmap=pyplot.get_cmap('gray'))
pyplot.show()


from __future__ import annotations

import json
from enum import Enum
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import Request, urlopen

class TranslateType(str, Enum):
    FSA_NT = "fsaNT"
    SHORT_SNT = "shortSnt"
    VOICETRA_NT = "voicetraNT"
    PAT_ETC = "patETC"
    LAWS_NT = "lawsNT"
    GENERAL_PT = "generalPT"
    PATENT_NT = "patentNT"
    SCIENCE = "science"
    FSA_GO_JP = "fsaGoJP"
    MINNA_PE = "minnaPE"
    GENERAL_NT = "generalNT"

def load_support_translate() -> list[tuple[str, str, str]]:
    path = Path(__file__).parent / "assets/support_translate.json"
    with open(path, encoding="utf-8") as f:
        support_languages = json.load(f)
    return [tuple(support_language) for support_language in support_languages]

def send_request(url: str, data: dict[str, str], timeout=10) -> dict:
    request_object = Request(url, urlencode(data).encode("ascii"), method="POST")

    with urlopen(request_object, timeout=timeout) as response:
        response_body = response.read().decode("utf-8")
    return json.loads(response_body)

class Translator:
    __DOMAIN = "https://mt-auto-minhon-mlt.ucri.jgn-x.jp"
    __SUPPORT_TRANSLATE = load_support_translate()

    def __init__(self, client_id: str, client_secret: str, user_name: str) -> None:
        self.__client_id = client_id
        self.__user_name = user_name
        self.__access_token = self.__get_access_token(client_id, client_secret)

    @classmethod
    def __get_access_token(cls, client_id: str, client_secret: str) -> str:
        url = f"{cls.__DOMAIN}/oauth2/token.php"
        request_data = {
            "grant_type": "client_credentials",
            "client_id": client_id,
            "client_secret": client_secret,
        }
        response_data = send_request(url, request_data)
        return response_data["access_token"]

    def translate_text(
        self,
        text: str,
        source_lang: str,
        target_lang: str,
        translate_type: TranslateType = TranslateType.GENERAL_NT,
        split: int = None,
        history: int = None,
        xml: int = None,
        term_id: str | list[str] | int = None,
        bilingual_id: str | list[str] | int = None,
        log_use: int = None,
        data: str | int = None,
    ) -> str:
        """
        Translate text.

        :param: text: text to translate.
        :param: source_lang: source language.
        :param: target_lang: target language.
        :param: translate_type: translate type.
        :param: split: split text.
        :param: history: history.
        :param: xml: xml.
        :param: term_id: term id.
        :param: bilingual_id: bilingual id.
        :param: log_use: log use.
        :param: data: data.

        :return: translated text.
        """
        self.__check(text, translate_type, source_lang, target_lang)

        url = f"{self.__DOMAIN}/api/mt/{translate_type.value}_{source_lang}_{target_lang}/"
        request_data = {
            "access_token": self.__access_token,
            "key": self.__client_id,
            "name": self.__user_name,
            "type": "json",
            "text": text,
            "split": split,
            "history": history,
            "xml": xml,
            "term_id": term_id,
            "bilingual_id": bilingual_id,
            "log_use": log_use,
            "data": data,
        }
        response_data = send_request(url, request_data)

        if response_data["resultset"]["code"] != 0:
            raise ValueError(
                f"code: {response_data['resultset']['code']}, message: \"{response_data['resultset']['message']}\""
            )
        return response_data["resultset"]["result"]["text"]

    def __check(self, text: str, translate_type: TranslateType, source_lang: str, target_lang: str) -> None:
        if not self.__is_support_translate(translate_type, source_lang, target_lang):
            raise ValueError(
                f"doesn't support. translate_type: {translate_type}, source_lang: {source_lang}, "
                "target_lang: {target_lang}"
            )
        if not text:
            raise ValueError("doesn't exist text")

    @classmethod
    def __is_support_translate(cls, translate_type: TranslateType, src_lang: str, target_lang: str) -> bool:
        return (translate_type.value, src_lang, target_lang) in cls.__SUPPORT_TRANSLATE


#!/usr/bin/env python
from cosc343world import Creature, World
import numpy as np
import time
import random
import matplotlib.pyplot as plt
import math

# To keep track for the plot
avgFitnessOverTime = []
avgTurnsOverTime = []
numWinnersOverTime = []
lastPopulation = []
oldAvgFitness = 0
numGenerationsSinceSwitch = 0

# You can change this number to specify how many generations creatures are going to evolve over...
numGenerations = 500

# You can change this number to specify how many turns in simulation of the world for given generation
numTurns = 100

# You can change this number to change the percept format.  You have three choice - format 1, 2 and 3 (described in
# the assignment 2 pdf document)
perceptFormat=1

# You can change this number to chnage the world size
gridSize=24

# You can set this mode to True to have same initial conditions for each simulation in each generation.  Good
# for development, when you want to have some determinism in how the world runs from generatin to generation.
repeatableMode=False

class Chromosome:
    def __init__(self):
        self.enemyScore = random.randint(-100, 100)
        self.eatEnemyScore = random.randint(-100, 100)
        self.friendScore = random.randint(-100, 100)
        self.eatFriendScore = random.randint(-100, 100)
        self.goToFoodScore = random.randint(-100, 100)
        self.awayFromFoodScore = random.randint(-100, 100)
        self.eatFoodScore = random.randint(-100, 100)
        self.randomScore = random.randint(-100, 100)

    def printAll(self):
        print("     Enemy Score    : %d" % self.enemyScore)
        print("    Friend Score    : %d" % self.friendScore)
        print("   Go To Food Score : %d" % self.goToFoodScore)
        print("Away From Food Score: %d" % self.awayFromFoodScore)
        print("     Eat Score      : %d" % self.eatFoodScore)
        print("     Random Score   : %d" % self.randomScore)

# This is a class implementing you creature a.k.a MyCreature.  It extends the basic Creature, which provides the
# basic functionality of the creature for the world simulation.  Your job is to implement the AgentFunction
# that controls creature's behavoiur by producing actions in respons to percepts.
class MyCreature(Creature):

    # Initialisation function.  This is where you creature
    # should be initialised with a chromosome in random state.  You need to decide the format of your
    # chromosome and the model that it's going to give rise to
    #
    # Input: numPercepts - the size of percepts list that creature will receive in each turn
    #        numActions - the size of actions list that creature must create on each turn
    def __init__(self, numPercepts, numActions):
        self.numP = numPercepts
        self.numA = numActions

        # Place your initialisation code here.  Ideally this should set up the creature's chromosome
        # and set it to some random state.
        self.chromosome = Chromosome()
        #self.chromosome.printAll()
        # Do not remove this line at the end.  It calls constructors
        # of the parent classes.
        Creature.__init__(self)

    # This is the implementation of the agent function that is called on every turn, giving your
    # creature a chance to perform an action.  You need to implement a model here, that takes its parameters
    # from the chromosome and it produces a set of actions from provided percepts
    #
    # Input: percepts - a list of percepts
    #        numAction - the size of the actions list that needs to be returned
    def AgentFunction(self, percepts, numActions):
        actions = [0] * numActions
        actions[10] = self.chromosome.randomScore

        # The farther I am from an enemy, the better
        for i in range(0, 9):
            if percepts[i] != 0:
                if i == 4:
                    actions[9] += self.chromosome.eatEnemyScore * (100 - self.getEnergy())
                for j in range(0, 9):
                    actions[j] += Distance(j % 3, j / 3, i % 3, i / 3) * self.chromosome.enemyScore

        # The farther I am from a friend, the better, because then we won't fight over food.
        for i in range(9, 18):
            if percepts[i] != 0:
                if i == 13:
                    actions[9] += self.chromosome.eatFriendScore * (100 - self.getEnergy()) * percepts[i]
                for j in range(0, 9):
                    actions[j] += Distance(j % 3, j / 3, i % 3, (i - 9) / 3) * self.chromosome.friendScore

        # The closer I am to food, the better, because then I can eat.
        for i in range(18, 27):
            if percepts[i] != 0:
                if i == 22:
                    actions[9] += self.chromosome.eatFoodScore * (100 - self.getEnergy()) * percepts[i]
                for j in range(0, 9):
                    if Distance(j % 3, j / 3, i % 3, (i - 18) / 3) == 0:
                        actions[j] += self.chromosome.goToFoodScore * (100 - self.getEnergy()) * percepts[i]
                    else:
                        actions[j] -= Distance(j % 3, j / 3, i % 3, (i - 18) / 3) * self.chromosome.awayFromFoodScore * percepts[i]

        return actions

def checkAndSwapGenerations(newGen, avgFitness, oldPopulation, oldAvgFitness, numGenerationsSinceSwitch):
    if oldPopulation and oldAvgFitness > avgFitness and (numGenerationsSinceSwitch <= 5 or avgFitness * 4 / 3 < oldAvgFitness):
        return oldPopulation, oldAvgFitness, numGenerationsSinceSwitch + 1
    return newGen, avgFitness, 0

def Distance(a1, a2, b1, b2):
    return math.sqrt((a1 - b1) * (a1 - b1) + (a2 - b2) * (a2 - b2))

def mateTwoParents(a, b):
    c = MyCreature(a.numP, a.numA)
    mutationChance = 0#.001
    c.chromosome = a.chromosome
    if random.uniform(0,1) <= 0.5:
        c.chromosome.enemyScore = b.chromosome.enemyScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.enemyScore = random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.eatEnemyScore = b.chromosome.eatEnemyScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.eatEnemyScore = random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.friendScore = b.chromosome.friendScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.friendScore += random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.eatFriendScore = b.chromosome.eatFriendScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.eatFriendScore += random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.goToFoodScore = b.chromosome.goToFoodScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.goToFoodScore = random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.awayFromFoodScore = b.chromosome.awayFromFoodScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.awayFromFoodScore = random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.eatFoodScore = b.chromosome.eatFoodScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.eatFoodScore = random.randint(-100, 100)
    if random.uniform(0,1) <= 0.5:
        c.chromosome.randomScore = b.chromosome.randomScore
    if random.uniform(0,1) <= mutationChance:
        c.chromosome.randomScore = random.randint(-100, 100)
    return c

def rouletteWheelSelection(oldPopulation):
    newPopulation = []
    totalFitness = 0
    # Keep the survivors from last time
    oldPopulation.sort(key=lambda x: x.fitness, reverse=True)
    for i in range(0, len(oldPopulation)):
        if oldPopulation[i].fitness >= 1000:
            c = MyCreature(oldPopulation[i].numP, oldPopulation[i].numA)
            c.chromosome = oldPopulation[i].chromosome
            newPopulation.append(c)
        totalFitness += oldPopulation[i].fitness
    for i in range(len(newPopulation), len(oldPopulation)):
        randA = random.randint(0, int(totalFitness))
        randB = random.randint(0, int(totalFitness))
        A = None
        B = None
        currTotal = 0
        for i in range(0, len(oldPopulation)):
            currTotal += oldPopulation[i].fitness
            if currTotal >= randA:
                A = oldPopulation[i]
            if currTotal >= randB:
                B = oldPopulation[i]
            if A and B:
                newPopulation.append(mateTwoParents(A, B))
                break
    return newPopulation

def tournamentSelection(oldPopulation):
    newPopulation = []
    # Keep the survivors from last time
    oldPopulation.sort(key=lambda x: x.fitness, reverse=True)
    i = 0
    while(oldPopulation[i].fitness >= 1000):
        c = MyCreature(oldPopulation[i].numP, oldPopulation[i].numA)
        c.chromosome = oldPopulation[i].chromosome
        newPopulation.append(c)
        i+=1
    for i in range(len(newPopulation), len(oldPopulation)):
        # Randomly choose 50% of the old population
        randomPopulation = random.sample(oldPopulation, int(len(oldPopulation) / 2))
        randomPopulation.sort(key=lambda x: x.fitness, reverse=True)
        newPopulation.append(mateTwoParents(randomPopulation[0], randomPopulation[1]))
    return newPopulation

# This function is called after every simulation, passing a list of the old population of creatures, whose fitness
# you need to evaluate and whose chromosomes you can use to create new creatures.
#
# Input: old_population - list of objects of MyCreature type that participated in the last simulation.  You
#                         can query the state of the creatures by using some built-in methods as well as any methods
#                         you decide to add to MyCreature class.  The length of the list is the size of
#                         the population.  You need to generate a new population of the same size.  Creatures from
#                         old population can be used in the new population - simulation will reset them to starting
#                         state.
#
# Returns: a list of MyCreature objects of the same length as the old_population.
def newPopulation(old_population):
    global numTurns, oldAvgFitness, lastPopulation, numGenerationsSinceSwitch

    nSurvivors = 0
    avgLifeTime = 0
    fitnessScoreList = []

    # For each individual you can extract the following information left over
    # from evaluation to let you figure out how well individual did in the
    # simulation of the world: whether the creature is dead or not, how much
    # energy did the creature have a the end of simualation (0 if dead), tick number
    # of creature's death (if dead).  You should use this information to build
    # a fitness function, score for how the individual did
    for individual in old_population:
        fitnessScore = 1000
        # You can read the creature's energy at the end of the simulation.  It will be 0 if creature is dead
        energy = individual.getEnergy()

        # This method tells you if the creature died during the simulation
        dead = individual.isDead()

        # If the creature is dead, you can get its time of death (in turns)
        if dead:
            timeOfDeath = individual.timeOfDeath()
            fitnessScore = energy / 4 + timeOfDeath / 2
            avgLifeTime += timeOfDeath
        else:
            nSurvivors += 1
            avgLifeTime += numTurns
            fitnessScore += energy
            #individual.chromosome.printAll()
        fitnessScoreList.append(fitnessScore)
        individual.fitness = fitnessScore

    # Here are some statistics, which you may or may not find useful
    avgLifeTime = float(avgLifeTime)/float(len(population))
    avgFitness = float(sum(fitnessScoreList)) / float(len(population))
    print("Simulation stats:")
    print("  Survivors    : %d out of %d" % (nSurvivors, len(population)))
    print("  Avg life time: %.1f turns" % avgLifeTime)
    print("  Avg fitness  : %.1f points" % avgFitness)
    print("oldAvg fitness : %.1f points" % oldAvgFitness)

    avgFitnessOverTime.append(avgFitness)
    avgTurnsOverTime.append(avgLifeTime)
    numWinnersOverTime.append(nSurvivors)

    # The information gathered above should allow you to build a fitness function that evaluates fitness of
    # every creature.  You should show the average fitness, but also use the fitness for selecting parents and
    # creating new creatures.

    # Here we decide if we want to make babies of the newer gen, or the older and (maybe) better gen
    lastPopulation, oldAvgFitness, numGenerationsSinceSwitch = checkAndSwapGenerations(old_population, avgFitness, lastPopulation, oldAvgFitness, numGenerationsSinceSwitch)

    return tournamentSelection(lastPopulation)
    # return rouletteWheelSelection(lastPopulation)

# Create the world.  Representaiton type choses the type of percept representation (there are three types to chose from);
# gridSize specifies the size of the world, repeatable parameter allows you to run the simulation in exactly same way.
w = World(representationType=perceptFormat, gridSize=gridSize, repeatable=repeatableMode)

#Get the number of creatures in the world
numCreatures = w.maxNumCreatures()

#Get the number of creature percepts
numCreaturePercepts = w.numCreaturePercepts()

#Get the number of creature actions
numCreatureActions = w.numCreatureActions()

# Create a list of initial creatures - instantiations of the MyCreature class that you implemented
population = list()
for i in range(numCreatures):
   c = MyCreature(numCreaturePercepts, numCreatureActions)
   population.append(c)

# Pass the first population to the world simulator
w.setNextGeneration(population)

# Runs the simulation to evalute the first population
w.evaluate(numTurns)

# Show visualisation of initial creature behaviour
w.show_simulation(titleStr='Initial population', speed='fast')

for i in range(numGenerations):
    print("\nGeneration %d:" % (i+1))

    # Create a new population from the old one
    population = newPopulation(population)

    # Pass the new population to the world simulator
    w.setNextGeneration(population)

    # Run the simulation again to evalute the next population
    w.evaluate(numTurns)

    # Show visualisation of final generation
    if i==numGenerations-1:
        w.show_simulation(titleStr='Final population', speed='fast')

plt.plot(avgFitnessOverTime)
plt.show()


from dataclasses import dataclass, field

import numpy as np
import torch

from torchsim.core.graph.slot_container import MemoryBlocks, Inputs
from torchsim.core.graph.unit import Unit
from torchsim.core.graph.worker_node_base import WorkerNodeBase
from torchsim.core.memory.tensor_creator import TensorCreator
from torchsim.core.models.expert_params import ParamsBase
from torch import FloatTensor

from torchsim.gui.observables import ObserverPropertiesItem, disable_on_runtime, enable_on_runtime
from torchsim.gui.validators import *

@dataclass
class WeightedAvgNodeParams(ParamsBase):
    num_inputs: int = 2  # number of inputs, also expected size of 3rd dimension of the input
    weights: List[float] = field(default_factory=list)
    weights_on_input: bool = False  # if True, the weights are read from the second input of the Node

class WeightedAvgNodeUnit(Unit):
    _creator: TensorCreator
    _params: WeightedAvgNodeParams

    def __init__(self, creator: TensorCreator,
                 output_height: int,
                 output_width: int,
                 params: WeightedAvgNodeParams,
                 num_weights: int = 0):

        super().__init__(creator.device)

        self._params = params
        self._creator = creator

        if self._params.weights_on_input:
            self.num_weights = num_weights

            weights = np.ones((output_height, output_width, self.num_weights))
        else:
            assert len(self._params.weights) != 0, 'Parametric weights not set!'
            assert self._params.num_inputs == len(self._params.weights)
            self.num_weights = len(self._params.weights)

            weights = np.ones((output_height, output_width, self.num_weights))
            for i in range(self.num_weights):
                weights[i] = np.dot(weights[i], params.weights[i])

        self.node_weights = creator.tensor(weights, dtype=creator.float, device=creator.device)
        self.last_output = creator.zeros([output_width, output_height], dtype=creator.float, device=creator.device)

    def step(self, inputs: FloatTensor, input_weights: FloatTensor = None):
        if self._params.weights_on_input:
            for i in range(self.num_weights):
                self.node_weights[i] *= input_weights[i]

        self.last_output.copy_(torch.sum(inputs * self.node_weights, dim=0))

class WeightedAvgNodeInputs(Inputs):
    def __init__(self, owner):
        super().__init__(owner)
        self.node_inputs = self.create("Inputs")
        self.input_weights = self.create("Input weights")

class WeightedAvgNodeOutputs(MemoryBlocks):
    def __init__(self, owner):
        super().__init__(owner)
        self.output = self.create("Output")

    def prepare_slots(self, unit: WeightedAvgNodeUnit):
        self.output.tensor = unit.last_output

class WeightedAvgNode(WorkerNodeBase):
    """ Node that computes weighted average of the input of shape [Y, Xm, n_inputs] across the last dimension.

    The `n_inputs` has to correspond to `params.num_inputs`.
    The given list of weights defines weight with which the input dimensions are weighted.

    """
    _unit: WeightedAvgNodeUnit
    _params: WeightedAvgNodeParams
    inputs: WeightedAvgNodeInputs
    outputs: WeightedAvgNodeOutputs

    @property
    def num_inputs(self) -> int:
        return self._params.num_inputs

    @num_inputs.setter
    def num_inputs(self, value: int):
        element = 1.0 / value

        self._params.num_inputs = value
        self._params.weights = [element] * value

    @property
    def weights_on_input(self) -> bool:
        return self._params.weights_on_input

    @weights_on_input.setter
    def weights_on_input(self, value: bool):
        self._params.weights_on_input = value

    @property
    def input_weights(self) -> List[float]:
        return self._params.weights

    @input_weights.setter
    def input_weights(self, value: List[int]):
        self._params.num_inputs = len(value)
        validate_list_of_size(value=value, size=self._params.num_inputs)

        self._params.weights = value

    def __init__(self, name="WeightedAvgNode"):
        super().__init__(name=name,
                         inputs=WeightedAvgNodeInputs(self),
                         outputs=WeightedAvgNodeOutputs(self))

        self._params = WeightedAvgNodeParams()

    def _create_unit(self, creator: TensorCreator) -> Unit:
        height, width, num_inputs = self.inputs.node_inputs.tensor.shape

        if self._params.weights_on_input:
            num_weights = self.inputs.input_weights.tensor.shape[0]
            return WeightedAvgNodeUnit(creator=creator,
                                       output_height=height,
                                       output_width=width,
                                       params=self._params,
                                       num_weights=num_weights)
        else:
            return WeightedAvgNodeUnit(creator=creator,
                                       output_height=height,
                                       output_width=width,
                                       params=self._params)

    def _step(self):
        if self._params.weights_on_input:
            self._unit.step(self.inputs.node_inputs.tensor, self.inputs.input_weights.tensor)
        else:
            self._unit.step(self.inputs.node_inputs.tensor)

    def validate(self):
        """Checks that the input has correct dimensions."""
        super().validate()

        node_inputs = self.inputs.node_inputs.tensor

        validate_predicate(lambda: node_inputs.dim() == 3,
                           f"The input should be 3D (y, x, input_idx) but has shape {node_inputs.shape}")

        if self._params.weights_on_input:
            validate_predicate(lambda: self.inputs.input_weights.tensor is not None,
                               f"if the weights_on_input is enabled, the input_weights has to be connected")

    def get_properties(self) -> List[ObserverPropertiesItem]:
        """Define which properties can be changed from GUI and how."""

        return [self._prop_builder.auto('Number of inputs', type(self).num_inputs, edit_strategy=disable_on_runtime),
                self._prop_builder.auto('Input weights', type(self).input_weights, edit_strategy=enable_on_runtime),
                self._prop_builder.auto('Weights on input', type(self).weights_on_input,
                                        edit_strategy=disable_on_runtime)]


import logging

logging.basicConfig(filename="list.log",level = logging.DEBUG,format=("%(name)s %(levelname)s %(asctime)s %(message)s"))

class list:
    def __init__(self, l):
        self.l = l

    def reverse_list(self,l):
        """this function helps in reversing the whole list"""
        try:
            val = self.l[::-1]
            logging.info(val)
            return val
        except Exception as e:
            logging.error(e)

    def accesing_234(self,l):
       """this func helps in accesing 234"""
       output_list=[]
       try:
           for i  in self.l:
               if type(i)==list or type(i)==tuple:
                   for j in i:
                       if type(j)==int:
                           if j == 234:
                               output_list.append(j)
               if type(i)==dict:
                   for k,v in i.items():
                       if k == int:
                           if k == 234:
                               output_list.append(k)
           logging.info(output_list)
       except Exception as e:
           logging.error(e)

    def access_456_in_data(self,l):
       '''This function access 456'''
       dummy_list = []
       try:
           for i in self.l:
               if type(i) == list or type(i) == tuple:
                   for j in i:
                       if type(j) == int:
                           if j == 456:
                               dummy_list.append(j)
               if type(i) == dict:
                   for k, v in i.items():
                       if type(k) == int:
                           if k == 456:
                               dummy_list.append(k)
           logging.info(dummy_list)
       except Exception as e:
            logging.exception(e)

    def extracting_sudh(self,l):
        """helps in extracting sudh from list entity"""
        dummy_list=[]
        try:
            for i in self.l:
                if type(i)==list or type(i)==tuple:
                    for j in i :
                        if type(j)!= int :
                            if j == "sudh":
                                dummy_list.append(j)
                if type(i)==dict:
                    for k,v in i.items():
                        if type(k) != int or type(v) != int:
                            if k == "sudh" or v == "sudh":
                                dummy_list.append(v)
            logging.info(dummy_list)
        except Exception as e:
            logging.error(e)

    def extracting_values(self,l):
        """extracting values from dict entity"""
        dummy_list=[]
        try:
            for i in self.l:
                if type(i)==dict:
                    for k,v in i.items():
                        dummy_list.append(v)
            logging.info(dummy_list)
        except Exception as e:
            logging.error(e)

    def extracting_keys(self,l):
        """extracting keys frm dict entity"""
        dummy_list=[]
        try:
            for i in self.l:
                if type(i)==dict:
                    for k , v in i.items():
                        dummy_list.append(k)
            logging.info(dummy_list)
        except Exception as e:
            logging.error(e)

logging and exception tasks/list task/list.py


# Makes an array of images for the Arduino sketch lightshow the persistence of vision wand.
# Each image is an array of unsigned LONGs that record on/off of each pixel in a COLUMN

import numpy as np
import cv2
import numpy as np
import os
#Get a list of all the BMP files in the current directory
def file_list(directory='filestoconvert', file_ext='.bmp'):
    return [x for x in os.listdir(directory) if x.lower().endswith(file_ext.lower())]

image_size = 32 #number pixels high and wide of each image
directory = 'filestoconvert'
bmps = file_list()
f = open('my_images.h', "w")

f.write('int numOfImages = {};\n'.format(len(bmps)))
f.write('int sizeOfImages = {};\n'.format(image_size))
f.write('// Array of images to display. Each image is made up of 16 longs; one long per column. C0R1 is most sig. bit. \n')
f.write('unsigned long images[{}][{}] = {{\n'.format(len(bmps), image_size))

for bmp in bmps:
    #Read the image from the file.
    img = cv2.imread(directory + '\\' + bmp, 0)
    img = np.transpose(img) # get read the image a column at a time
    f.write('// {}\n'.format(bmp)) # output the name of the image as a comment
    f.write('{')
    for col in img:
        pixels_as_num = 0;
        for pixel in col:
            pixels_as_num = pixels_as_num << 1;
            if pixel == 0:  # 0 means black in CV2
                pixels_as_num += 1
        f.write('{}, '.format(pixels_as_num))
    f.write('}, \n') # End of this image
f.write('};\n') # End of array of images
f.close()


from flask import Flask,render_template,request
from flask_mysqldb import MySQL
from random import randint

# import json
app = Flask(__name__)#interface between webserver and web application

app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'aman1'
app.config['MYSQL_PASSWORD'] = '1234'
app.config['MYSQL_DB'] = 'aman'

mysql = MySQL(app)

firstname = " "
add = 0
result = " "
age = ""
smoke = ""
alcohol = ""
measurement = ""
physical = ""
disease_family_hist = ""
patient_id = " "

@app.route("/" , methods = ['GET','POST'])
def home():
    return render_template("index.html")

@app.route("/templates/about.html")
def about():
    return render_template("about.html")

@app.route("/templates/search.html")
def search():
    return render_template("search.html")

@app.route("/templates/contact.html")
def contact():
    return render_template("contact.html")

@app.route("/templates/services.html")
def services():
    return render_template("services.html")

@app.route("/templates/registration.html")
def registration1():
    return render_template("registration.html")

# route to get data from html form and insert data into database
@app.route('/registration', methods=["GET", "POST"])
def registration():
    global patient_id
    global firstname
    middlename = " "
    lastname = " "
    email = " "
    gender = " "
    birthday = " "
    pincode = " "

    patient_id = randint(10000000000000,99999999999999)

    if request.method == "POST":

        firstname = request.form['fname']
        middlename = request.form['mname']
        lastname = request.form['lname']
        email =  request.form['email']
        gender = request.form['gender']
        birthday = request.form['birthday']
        pincode = request.form["pincode"]

        cursor = mysql.connection.cursor()

        cursor.execute(''' INSERT INTO patient_info VALUES(%s ,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''',(patient_id,firstname,middlename,lastname,email,gender,birthday,pincode,age,smoke,alcohol,measurement,physical,disease_family_hist,add,result))
        mysql.connection.commit()

    return render_template("ncd1.html",patientid = patient_id , fname = firstname,mname= middlename,lname =  lastname,email1 = email,gender1 = gender,birthday1 = birthday,pincode1 = pincode)

@app.route('/res',methods=['GET',"POST"])
def res():

    return render_template('result1.html')

@app.route('/login/', methods=['GET',"POST"])
def login():

    return render_template('ncd1.html')

@app.route('/result1',methods=['GET',"POST"])
def result1():
    if request.method == "POST":
        count = 0

# getting the value for age

        while True :
            global age
            global smoke
            global alcohol
            global measurement
            global physical
            global disease_family_hist

            age = request.form['age']

            smoke = request.form['smoke']

            alcohol =request.form['alcohol']

            measurement = request.form['measurement']

            physical =  request.form['physical']

            disease_family_hist =  request.form['history']

            count = int(age)+int(smoke)+int(alcohol)+int(measurement)+int(physical)+int(disease_family_hist)

            global add
            add = count
            print(add)
            global result
            if count>4:
                result="you need screening"

            else:
                result="No screening needed"

            cursor = mysql.connection.cursor()

            query = 'UPDATE patient_info SET age  = %s,smoke = %s,alcohol = %s,measurement = %s,physical = %s,disease_family_hist = %s,count = %s,result = %s WHERE firstname = %s'
            value = (age,smoke,alcohol,measurement,physical,disease_family_hist,add,result,firstname)
            cursor.execute(query,value)
            mysql.connection.commit()
            cursor.close()

            return render_template('result1.html', add1=add,prescription=result)
    return render_template('result1.html', add1="result not found in session.")

@app.route('/back',methods=['GET',"POST"])
def back():
    if request.method == 'POST':
        return render_template("ncd1.html");

#back 1 is used for contact.html
@app.route('/back1',methods=['GET',"POST"])
def back1():
    if request.method == 'POST':
        return render_template("index.html");

@app.route("/searching1", methods=['GET', 'POST'])
def searching():
    if request.method == 'POST':
        x = request.form['patient if']

        query =  "SELECT * FROM patient_info WHERE patientid LIKE '%"+x+"%'OR firstname LIKE '%"+x+"%'"
        cursor = mysql.connection.cursor()
        cursor.execute(query)
        data = cursor.fetchall()

        print(data)
        return render_template("search.html",data = data)

if __name__ == '__main__':

    app.run(debug=True)


from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Conv1D, MaxPool1D, \
    Flatten, concatenate
from __layers__ import *
from __logger__ import LOGGER_NAME
import logging
from __paths__ import path_to_lm, path_to_language_models
import json

logger = logging.getLogger(LOGGER_NAME)

with open(path_to_lm.joinpath('params.json'), 'r') as f:
    params = json.load(f)

class LanguageModel(object):
    def __init__(self,
                 word_embedding_size=None,
                 char_embedding_size=None,
                 word_inp_mask_val=None,
                 word_vocab_size=None,
                 char_vocab_size=None,
                 max_seq_len=None,
                 max_word_len=None,
                 char_cnn_filters=None,
                 char_cnn_ker_size=None,
                 char_cnn_pool_size=None,
                 dropout=0.0
                 ):
        self._model = None
        self._max_seq_len = max_seq_len
        self._max_word_len = max_word_len
        self._word_emb_size = word_embedding_size
        self._char_emb_size = char_embedding_size
        self._word_inp_mask = word_inp_mask_val
        self._word_vocab_size = word_vocab_size
        self._char_vocab_size = char_vocab_size
        self._char_cnn_filters = char_cnn_filters
        self._char_cnn_ker_size = char_cnn_ker_size
        self._char_cnn_pool_size = char_cnn_pool_size
        self._dropout = dropout

    def _word_embedding_layer(self, word_in, train_wts=True):
        """
        --> get word embeddings
        :param word_in: tensor of word idxs (2d)
        :param train_wts: (boolean) if True then wts will be trained
        :return: a sequence of word vectors
        """
        masked_word_in = Mask(mask_value=self._word_inp_mask, name='masked_word_inputs')(word_in)
        emb_word = Embedding(input_dim=self._word_vocab_size,
                             output_dim=self._word_emb_size,
                             input_length=self._max_seq_len,
                             name='word_embeddings', trainable=train_wts)(masked_word_in)
        return emb_word

    def _char_embedding_layer(self, char_in, train_wts=True):
        """
        --> get character level encodings
        :param char_in: a tensor of char idxs (3d)
        :param train_wts: (boolean) if True then wts will be trained
        :return: a sequence of character encodings
        """
        emb_char = TimeDistributed(Embedding(input_dim=self._char_vocab_size, output_dim=self._char_emb_size,
                                             input_length=self._max_word_len,
                                             name='char_embeddings'),
                                   name='time_distributed_char_emb',
                                   trainable=train_wts)(char_in)
        emb_char = TimeDistributed(Conv1D(filters=self._char_cnn_filters,
                                          kernel_size=self._char_cnn_ker_size, activation='relu', name='char_cnn',
                                          trainable=train_wts),
                                   name='time_distributed_char_cnn',
                                   trainable=train_wts)(emb_char)
        emb_char = TimeDistributed(Dropout(self._dropout), name='timedistributed_char_cnn_dropout')(emb_char)
        emb_char = TimeDistributed(MaxPool1D(pool_size=self._char_cnn_pool_size), name='char_cnn_pooling')(emb_char)
        emb_char = TimeDistributed(Flatten(name='char_cnn_flatten'), name='time_dist_char_cnn_flatten')(emb_char)

        return emb_char

    def _embedding_layer(self, word_in, char_in, train_wts=True):
        """
        --> final embedding layer
        --> merge all embeddings
        :param word_in: tensor of word idxs (2d)
        :param char_in: tensor of char idxs (3d)
        :param train_wts: (boolean) if True then wts will be trained
        :return: final merged embeddings
        """
        word_emb = self._word_embedding_layer(word_in, train_wts=train_wts)
        char_emb = self._char_embedding_layer(char_in, train_wts=train_wts)

        merged_embedding = concatenate([word_emb, char_emb], name='merged_embeddings')
        final_emb = PosEmb(self._max_seq_len, merged_embedding.shape[-1], name='final_embedding')(merged_embedding)

        return final_emb

    def _elmo_lstm_out(self, inp, name='', rev=False, train_wts=True):
        """
        --> elmo style architecture
        --> input is fed to 1st rnn
        --> input merged with the output of first rnn is fed to the second rnn
        :param inp: (tensor) merged embeddings
        :param name: (str) name to be given to layers
        :param rev: (boolean) rnns will be reversed if True
        :param train_wts: (boolean) if True then wts will be trained
        :return: tuple of tensors (lstm1_out, lstm2_out)
        """
        _lstm1 = LSTM(units=inp.get_shape().as_list()[-1] // 2,
                      return_sequences=True,
                      recurrent_dropout=self._dropout, go_backwards=rev,
                      name='{}_1'.format(name), trainable=train_wts)(inp)
        merged_inp = concatenate([inp, _lstm1], name='{}_merge_lstm1_embed'.format(name))
        _lstm2 = LSTM(units=inp.get_shape().as_list()[-1] // 2,
                      return_sequences=True,
                      recurrent_dropout=self._dropout,
                      name='{}_2'.format(name), trainable=train_wts)(merged_inp)
        return _lstm1, _lstm2

    def get_model(self):
        """
        --> build model
        :return: self._model
        """
        if self._model:
            return self._model

        word_in = Input(shape=(self._max_seq_len,), name='token_inputs', dtype=params.get('dtype_int'))
        char_in = Input(shape=(self._max_seq_len, self._max_word_len,), name='char_inputs',
                        dtype=params.get('dtype_int'))

        embedding_layer = self._embedding_layer(word_in, char_in, train_wts=True)

        _, l2r_lstm = self._elmo_lstm_out(embedding_layer, name='l2r_lstm', rev=False, train_wts=True)
        _, r2l_lstm = self._elmo_lstm_out(embedding_layer, name='r2l_lstm', rev=True, train_wts=True)

        # merge left2right and right2left lstm outs
        merged = concatenate([l2r_lstm, r2l_lstm], name='merged_lstm1_lstm2')

        model = TimeDistributed(Dense(self._word_vocab_size, name='token_prediction', activation='softmax'),
                                name='time_dist_token_pred_layer')(merged)
        model = Model(inputs=[word_in, char_in], outputs=[model], name='LanguageModel')
        self._model = model
        return self._model

    def save(self, name):
        """
        --> saves model params to "LanguageModel/saved_models/name/model_params.json"
        --> saves model wts to "LanguageModel/saved_models/name/wts.h5"
        :param name (str): name for saving the model
        :return: None
        """
        path_to_vocab = path_to_language_models.joinpath(name)
        path_to_vocab.mkdir(parents=True, exist_ok=True)

        with open(path_to_vocab.joinpath("model_params.json"), 'w', encoding='utf-8') as f:
            all_items = {k: v for k, v in self.__dict__.items() if k not in ['_model']}
            json.dump(all_items, f)

        self._model.save_weights(path_to_vocab.joinpath('wts.h5'))

        logger.info('language model saved successfully at {}'.format(path_to_vocab))

    def load(self, name):
        """
        --> loads model params from "LanguageModel/saved_models/name/model_params.json"
        --> loads model wts from "LanguageModel/saved_models/name/wts.h5"
        :param name (str): name for loading the model
        :return: None
        """
        path_to_vocab = path_to_language_models.joinpath(name)
        if not path_to_vocab.is_dir():
            logger.error('language model could not be loaded since "" is not a valid directory'.format(path_to_vocab))
            exit(1)

        if not path_to_vocab.joinpath("model_params.json").is_file():
            logger.error(
                'language model could not be loaded since params do not exist at {}'.format(path_to_vocab))

        with open(path_to_vocab.joinpath("model_params.json"), 'r', encoding='utf-8') as f:
            obj = json.load(f)
            for k, v in obj.items():
                self.__setattr__(k, v)

        # build model
        self.get_model()

        wts_path = sorted(path_to_vocab.glob('*.h5'), key=lambda p: p.lstat().st_mtime)[-1]
        self._model.load_weights(wts_path)
        logger.info('name of loaded wts for language model {} = {}'.format(name, wts_path.name))

        logger.info('language model loaded successfully from {}'.format(path_to_vocab))


import os, sys
import logging
import argparse

data_path = r"http://livecell-dataset.s3.eu-central-1.amazonaws.com/LIVECell_dataset_2021/images.zip"

def parse_opt():
    parser = argparse.ArgumentParser()
    parser.add_argument('--save_path', type=str, default="data/coco", help='path to save downloaded data')
    return parser.parse_args()

def plot_stage(stage: str):
    stage_len = len(stage)
    full_row_len = 6 * stage_len
    mid_row_len = full_row_len - stage_len - 2

    lrow = mid_row_len//2
    rrow = mid_row_len - lrow

    print("#" * full_row_len)
    print("#" * lrow + " " + stage + " " + "#"* rrow)
    print("#" * full_row_len)
    print()

if __name__ == "__main__":

    is_wrutable = os.access('./', os.R_OK & os.W_OK & os.X_OK) # Check if we can write and read from current dir and execute files
    if not is_wrutable:
        raise PermissionError(f"You do not have enough permissions for directory {os.path.dirname(os.path.abspath(__file__))}.")

    logging.basicConfig(level=logging.DEBUG, filename='loadlog.log', format='%(asctime)s %(levelname)s:%(message)s')
    logging.info(f"Download data from {data_path}") ###Log

    opt = parse_opt()
    save_path = opt.save_path
    data_tmp_name = os.path.join(save_path, "data_tmp" + os.path.splitext(data_path)[-1])

    if not os.path.exists(save_path):
        os.makedirs(save_path)
        logging.info(f"Dir '{save_path}' created.") ###Log
    else:
        logging.info(f"Dir '{save_path}' already exists.") ###Log

    logging.info("Data download:start")
    command = f"wget -O {data_tmp_name} {data_path}"
    logging.info(f"terminat: ${command}")
    os.system(command)
    logging.info(f"Data download:done")

    logging.info("Data unzip:start")
    logging.info(f"terminat: ${command}")
    command = f"unzip -q {data_tmp_name} -d {save_path}; rm {data_tmp_name}"
    os.system(command)
    logging.info("Data unzip:done")

    logging.info("Data move:start")
    IMAGE_PATH = os.path.join(save_path, "images")
    move_path_from, move_path_to = IMAGE_PATH + f"{os.sep}**{os.sep}*.tif", IMAGE_PATH
    drop_dir_list = [os.path.join(move_path_to, i) for i in os.listdir(move_path_to)]
    drop_dir_list = [i for i in drop_dir_list if os.path.isdir(i)]
    logging.info(f"terminat: ${command}")
    command = f"mv {move_path_from} {move_path_to}; rm -d " + " ".join(drop_dir_list)
    os.system(command)
    logging.info("Data move:done")

    logging.info(f"Download data finished") ###Log


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os
import pprint
import random
import shutil
import sys
import warnings
import logging
import time
import threading

import torch
import torch.npu
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.nn as nn
import torch.nn.parallel
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
from apex import amp
from tensorboardX import SummaryWriter

import _init_paths
import models

from config import cfg
from config import update_config
from core.loss import MultiLossFactory
from dataset import make_dataloader
from utils.utils import create_logger
from utils.utils import get_optimizer
from utils.utils import save_checkpoint
from utils.utils import setup_logger
from utils.utils import AverageMeter
from utils.vis import save_debug_images

def parse_args():
    parser = argparse.ArgumentParser(description='Train keypoints network')
    # general
    parser.add_argument('--cfg',
                        help='experiment configure file name',
                        required=True,
                        type=str)

    parser.add_argument('opts',
                        help="Modify config options using the command-line",
                        default=None,
                        nargs=argparse.REMAINDER)

    # distributed training
    parser.add_argument('--gpu',
                        help='gpu id for multiprocessing training',
                        type=str)
    parser.add_argument('--world-size',
                        default=1,
                        type=int,
                        help='number of nodes for distributed training')
    parser.add_argument('--dist-url',
                        default='tcp://127.0.0.1:23456',
                        type=str,
                        help='url used to set up distributed training')
    parser.add_argument('--rank',
                        default=0,
                        type=int,
                        help='node rank for distributed training')

    parser.add_argument('--device', default='npu', type=str, help='npu or gpu')
    parser.add_argument('--device_num', default=-1, type=int,
                        help='device_num')
    parser.add_argument('--device-list', default='', type=str, help='device id list')
    parser.add_argument('--addr', default='10.136.181.115', type=str, help='master addr')
    parser.add_argument('--seed', default=None, type=int,
                        help='seed for initializing training. ')
    # apex
    parser.add_argument('--amp', default=False, action='store_true',
                        help='use amp to train the model')
    parser.add_argument('--loss-scale', default=None, type=float,
                        help='loss scale using in amp, default -1 means dynamic')
    parser.add_argument('--opt-level', default='O1', type=str,
                        help='loss scale using in amp, default -1 means dynamic')
    args = parser.parse_args()

    return args

def main():
    args = parse_args()
    update_config(cfg, args)

    cfg.defrost()
    cfg.RANK = args.rank
    cfg.freeze()

    logger, final_output_dir, tb_log_dir = create_logger(
        cfg, args.cfg, 'train'
    )

    if args.seed is not None:
        random.seed(args.seed)
        torch.manual_seed(args.seed)
        cudnn.deterministic = True
        warnings.warn('You have chosen to seed training. '
                      'This will turn on the CUDNN deterministic setting, '
                      'which can slow down your training considerably! '
                      'You may see unexpected behavior when restarting '
                      'from checkpoints.')

    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '29688'

    logger.info(pprint.pformat(args))
    logger.info(cfg)
    os.environ['LOCAL_DEVICE_ID'] = str(0)
    print("+++++++++++++++++++++++++++LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID'])
    if args.gpu is not None:
        warnings.warn('You have chosen a specific GPU. This will completely '
                      'disable data parallelism.')

    if args.dist_url == "env://" and args.world_size == -1:
        args.world_size = int(os.environ["WORLD_SIZE"])

    args.distributed = args.world_size > 1 or cfg.MULTIPROCESSING_DISTRIBUTED
    # print("world_size",args.world_size)
    ngpus_per_node = int(os.getenv("RANK_SIZE"))
    if cfg.MULTIPROCESSING_DISTRIBUTED:
        # Since we have ngpus_per_node processes per node, the total world_size
        # needs to be adjusted accordingly
        args.world_size = ngpus_per_node * args.world_size
        # Use torch.multiprocessing.spawn to launch distributed processes: the
        # main_worker process function
        mp.spawn(
            main_worker,
            nprocs=ngpus_per_node,
            args=(ngpus_per_node, args, final_output_dir, tb_log_dir)
        )
    else:
        # Simply call main_worker function
        main_worker(
            ','.join([str(i) for i in cfg.GPUS]),
            ngpus_per_node,
            args,
            final_output_dir,
            tb_log_dir
        )

def main_worker(
        gpu, ngpus_per_node, args, final_output_dir, tb_log_dir
):

    if args.device_list != '':
        args.gpu = int(args.device_list.split(',')[gpu])
    else:
        args.gpu = gpu

    print("[npu id:", args.gpu, "]", "++++++++++++++++ before set LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID'])
    os.environ['LOCAL_DEVICE_ID'] = str(args.gpu)
    print("[npu id:", args.gpu, "]", "++++++++++++++++ LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID'])

    if args.gpu is not None:
        print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu))

    if args.distributed:
        if args.dist_url == "env://" and args.rank == -1:
            args.rank = int(os.environ["RANK"])
        if cfg.MULTIPROCESSING_DISTRIBUTED:
            # For multiprocessing distributed training, rank needs to be the
            # global rank among all the processes
            args.rank = args.rank * ngpus_per_node + gpu
        print('Init process group: dist_url: {}, world_size: {}, rank: {}'.
              format(args.dist_url, args.world_size, args.rank))
        if args.device == 'npu':
            dist.init_process_group(backend=cfg.DIST_BACKEND,# init_method=args.dist_url,
                                    world_size=args.world_size, rank=args.rank)
        else:
            dist.init_process_group(backend=cfg.DIST_BACKEND, init_method=args.dist_url,
                                    world_size=args.world_size, rank=args.rank)
    update_config(cfg, args)

    # setup logger
    logger, _ = setup_logger(final_output_dir, args.rank, 'train')
    model = eval('models.'+cfg.MODEL.NAME+'.get_pose_net')(
        cfg, is_train=True
    )
    # copy model file
    if not cfg.MULTIPROCESSING_DISTRIBUTED or (
            cfg.MULTIPROCESSING_DISTRIBUTED
            and args.rank % ngpus_per_node == 0
    ):
        this_dir = os.path.dirname(__file__)
        shutil.copy2(
            os.path.join(this_dir, '../lib/models', cfg.MODEL.NAME + '.py'),
            final_output_dir
        )

    writer_dict = {
        'writer': SummaryWriter(log_dir=tb_log_dir),
        'train_global_steps': 0,
        'valid_global_steps': 0,
    }

    if not cfg.MULTIPROCESSING_DISTRIBUTED or (
            cfg.MULTIPROCESSING_DISTRIBUTED
            and args.rank % ngpus_per_node == 0
    ):
        dump_input = torch.rand(
            (1, 3, cfg.DATASET.INPUT_SIZE, cfg.DATASET.INPUT_SIZE)
        )

    loc = 'npu:{}'.format(args.gpu)
    torch.npu.set_device(loc)
    model = model.to(loc)
    # define loss function (criterion) and optimizer
    loss_factory = MultiLossFactory(cfg).to(loc)

    best_perf = -1
    best_model = False
    last_epoch = -1
    optimizer = get_optimizer(cfg, model)

    if args.amp:
        amp.register_half_function(torch, 'bmm')
        model, optimizer = amp.initialize(model, optimizer, opt_level=args.opt_level, loss_scale=args.loss_scale, combine_grad=True)
    if args.distributed:
        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False)
    train_loader,train_loader_sampler= make_dataloader(
        cfg, is_train=True, distributed=args.distributed
    )

    begin_epoch = cfg.TRAIN.BEGIN_EPOCH
    checkpoint_file = os.path.join(
        final_output_dir, 'checkpoint.pth.tar')
    if cfg.AUTO_RESUME and os.path.exists(checkpoint_file):
        logger.info("=> loading checkpoint '{}'".format(checkpoint_file))
        checkpoint = torch.load(checkpoint_file)
        begin_epoch = checkpoint['epoch']
        best_perf = checkpoint['perf']
        last_epoch = checkpoint['epoch']
        model.load_state_dict(checkpoint['state_dict'], strict=False)

        optimizer.load_state_dict(checkpoint['optimizer'])
        logger.info("=> loaded checkpoint '{}' (epoch {})".format(
            checkpoint_file, checkpoint['epoch']))

    lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
        optimizer, cfg.TRAIN.LR_STEP, cfg.TRAIN.LR_FACTOR,
        last_epoch=last_epoch)

    for epoch in range(begin_epoch, cfg.TRAIN.END_EPOCH):
        # train one epoch
        if train_loader_sampler is not None:
            train_loader_sampler.set_epoch(epoch)
        do_train(cfg, args, model, train_loader, loss_factory, optimizer, epoch,
                 final_output_dir, tb_log_dir, writer_dict)
        lr_scheduler.step()

        perf_indicator = epoch
        if perf_indicator >= best_perf:
            best_perf = perf_indicator
            best_model = True
        else:
            best_model = False

        if not cfg.MULTIPROCESSING_DISTRIBUTED or (
                cfg.MULTIPROCESSING_DISTRIBUTED
                and args.rank == 0
        ):
            logger.info('=> saving checkpoint to {}'.format(final_output_dir))
            if not cfg.MULTIPROCESSING_DISTRIBUTED:
                save_checkpoint({
                    'epoch': epoch + 1,
                    'model': cfg.MODEL.NAME,
                    'state_dict': model.state_dict(),
                    'best_state_dict': model.state_dict(),
                    'perf': perf_indicator,
                    'optimizer': optimizer.state_dict(),
                }, best_model, final_output_dir)
            else:
                save_checkpoint({
                    'epoch': epoch + 1,
                    'model': cfg.MODEL.NAME,
                    'state_dict': model.state_dict(),
                    'best_state_dict': model.module.state_dict(),
                    'perf': perf_indicator,
                    'optimizer': optimizer.state_dict(),
                }, best_model, final_output_dir)

    final_model_state_file = os.path.join(
        final_output_dir, 'final_state{}.pth.tar'.format(gpu)
    )

    logger.info('saving final model state to {}'.format(
        final_model_state_file))
    torch.save(model.state_dict(), final_model_state_file)
    writer_dict['writer'].close()

def do_train(cfg, args, model, data_loader, loss_factory, optimizer, epoch,
             output_dir, tb_log_dir, writer_dict):
    logger = logging.getLogger("Training")

    batch_time = AverageMeter()
    data_time = AverageMeter()

    heatmaps_loss_meter = [AverageMeter() for _ in range(cfg.LOSS.NUM_STAGES)]
    push_loss_meter = [AverageMeter() for _ in range(cfg.LOSS.NUM_STAGES)]
    pull_loss_meter = [AverageMeter() for _ in range(cfg.LOSS.NUM_STAGES)]

    loc = 'npu:{}'.format(args.gpu)
    # switch to train mode
    model.train()

    end = time.time()
    for i, (images, heatmaps, masks, joints) in enumerate(data_loader):
        # measure data loading time
        data_time.update(time.time() - end)
#        with torch.autograd.profiler.profile(use_npu=True) as prof:
        images = images.to(loc, non_blocking=True)
        heatmaps = list(map(lambda x: x.to(loc, non_blocking=True), heatmaps))
        masks = list(map(lambda x: x.to(loc, non_blocking=True), masks))
        joints = list(map(lambda x: x.to(loc, non_blocking=True), joints))

        # compute output
        outputs = model(images)

        # loss = loss_factory(outputs, heatmaps, masks)
        heatmaps_losses, push_losses, pull_losses = \
            loss_factory(outputs, heatmaps, masks, joints)

        loss = 0
        for idx in range(cfg.LOSS.NUM_STAGES):
            if heatmaps_losses[idx] is not None:
                heatmaps_loss = heatmaps_losses[idx].mean(dim=0)
                heatmaps_loss_meter[idx].update(
                    heatmaps_loss.item(), images.size(0)
                )
                loss = loss + heatmaps_loss
                if push_losses[idx] is not None:
                    push_loss = push_losses[idx].mean(dim=0)
                    push_loss_meter[idx].update(
                        push_loss.item(), images.size(0)
                    )
                    loss = loss + push_loss
                if pull_losses[idx] is not None:
                    pull_loss = pull_losses[idx].mean(dim=0)
                    pull_loss_meter[idx].update(
                        pull_loss.item(), images.size(0)
                    )
                    loss = loss + pull_loss

        # compute gradient and do update step
        optimizer.zero_grad()
        if args.amp:
            with amp.scale_loss(loss, optimizer) as scaled_loss:
                scaled_loss.backward()
        else:
            loss.backward()
        optimizer.step()
        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        if i % cfg.PRINT_FREQ == 0 and cfg.RANK == 0:
            msg = 'Epoch: [{0}][{1}/{2}]\t' \
                  'Time: {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\t' \
                  'Speed: {speed:.1f} samples/s\t' \
                  'Data: {data_time.val:.3f}s ({data_time.avg:.3f}s)\t' \
                  '{heatmaps_loss}{push_loss}{pull_loss}'.format(
                      epoch, i, len(data_loader),
                      batch_time=batch_time,
                      speed=images.size(0)/batch_time.val,
                      data_time=data_time,
                      heatmaps_loss=_get_loss_info(heatmaps_loss_meter, 'heatmaps'),
                      push_loss=_get_loss_info(push_loss_meter, 'push'),
                      pull_loss=_get_loss_info(pull_loss_meter, 'pull')
                  )
            logger.info(msg)

            writer = writer_dict['writer']
            global_steps = writer_dict['train_global_steps']
            for idx in range(cfg.LOSS.NUM_STAGES):
                writer.add_scalar(
                    'train_stage{}_heatmaps_loss'.format(i),
                    heatmaps_loss_meter[idx].val,
                    global_steps
                )
                writer.add_scalar(
                    'train_stage{}_push_loss'.format(idx),
                    push_loss_meter[idx].val,
                    global_steps
                )
                writer.add_scalar(
                    'train_stage{}_pull_loss'.format(idx),
                    pull_loss_meter[idx].val,
                    global_steps
                )
            writer_dict['train_global_steps'] = global_steps + 1

            prefix = '{}_{}'.format(os.path.join(output_dir, 'train'), i)
            for scale_idx in range(len(outputs)):
                prefix_scale = prefix + '_output_{}'.format(
                    cfg.DATASET.OUTPUT_SIZE[scale_idx]
                )
                save_debug_images(
                    cfg, images, heatmaps[scale_idx], masks[scale_idx],
                    outputs[scale_idx], prefix_scale
                )

def _get_loss_info(loss_meters, loss_name):
    msg = ''
    for i, meter in enumerate(loss_meters):
        msg += 'Stage{i}-{name}: {meter.val:.3e} ({meter.avg:.3e})\t'.format(
            i=i, name=loss_name, meter=meter
        )

    return msg

if __name__ == '__main__':
    main()


import torch
import numpy as np
from sklearn.datasets import make_blobs
from torch.autograd import Variable
import matplotlib.pyplot as plt

class Feedforward(torch.nn.Module):
    def __init__(self, input_size, hidden_size):
        super(Feedforward, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size)
        self.relu = torch.nn.ReLU()
        self.fc2 = torch.nn.Linear(self.hidden_size, 1)
        self.sigmoid = torch.nn.Sigmoid()

    def forward(self, x):
        hidden = self.fc1(x)
        relu = self.relu(hidden)
        output = self.fc2(relu)
        output = self.sigmoid(output)
        return output

def blob_label(y, label, loc):
    target = np.copy(y)
    for l in loc:
        target[y == l] = label
    return target

x_train, y_train = make_blobs(n_samples=40, n_features=2, centers= 2, cluster_std=10, shuffle=True)
x_train = torch.FloatTensor(x_train)
y_train = torch.FloatTensor(y_train)

x_test, y_test = make_blobs(n_samples=10, n_features=2, centers= 2, cluster_std=1.5, shuffle=True)
x_test = torch.FloatTensor(x_test)
y_test = torch.FloatTensor(y_test)

model = Feedforward(2, 10)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr = 0.01)

#To see how the model is improving, we can check the test loss before the model training and
#compare it with the test loss after the training.

model.cuda()
model.eval()
x_test = Variable(x_test.cuda())
y_test = Variable(y_test.cuda())
y_pred = model(x_test)
before_train = criterion(y_pred.squeeze(), y_test)
print('Test loss before traing', before_train.item())

x_train = Variable(x_train.cuda())
y_train = Variable(y_train.cuda())

model.train()
epoch = 20
# epoch = 1000
for epoch in range(epoch):
    optimizer.zero_grad()
    y_pred = model(x_train) #Forward pass
    loss = criterion(y_pred.squeeze(), y_train) # Compute Loss

    print('Epock -->', epoch, ' train loss--> ',  loss.item())

    #Bacward pass
    loss.backward()
    optimizer.step()

model.eval()
y_pred = model(x_test)
after_train = criterion(y_pred.squeeze(), y_test)
print('Test loss after Training', after_train.item())

# print(y_test)
# print(y_pred.squeeze())
# ====================================================
x_testCPU = Variable(x_test).cpu().numpy()
y_testCPU = Variable(y_test).cpu().numpy()
y_predCPU = Variable(y_pred).cpu().numpy()
# print(y_predCPU)

plt.clf()
plt.plot(x_testCPU, y_testCPU, 'go', label='True data', alpha = 0.5)
plt.plot(x_testCPU, y_predCPU, '--', label="Predictions", alpha = 0.5)
plt.legend(loc='best')
plt.show()


import cv2
import numpy as np
from keras.models import model_from_json
from keras.preprocessing.image import img_to_array
import _thread

arquivo = open('modelo_01_expressoes.json', 'r')
estrutura_rede = arquivo.read()
arquivo.close()

modelo = model_from_json(estrutura_rede)
modelo.load_weights('model_weights.h5')

cascade_faces = 'haarcascade_frontalface_default.xml'
face_detection = cv2.CascadeClassifier(cascade_faces)
expressoes = ['Raiva', 'Nojo', 'Feliz', 'Triste', 'Surpreso', 'Neutro']
cap = cv2.VideoCapture(0)
expressao = expressoes[5]
identificando = False

def identificar_expressao(modelo, face, cinza):
    global expressao
    global identificando
    identificando = True
    # Extraindo o ponto de interesse
    roi = cinza[face[1]:face[1] + face[3], face[0]:face[0] + face[2]]
    # Redimensionando a imagem para o tamanho necessário no treinamento do modelo
    roi = cv2.resize(roi, (48, 48))
    # Passando os valores RGB da imagem para a escala de números decimais de 0 a 1
    roi = roi / 255
    roi = img_to_array(roi)
    roi = np.expand_dims(roi, axis=0)
    # Utilizando o modelo para detectar a emoção
    preds = modelo.predict(roi)[0]
    modelo.predict(roi)
    expressao = expressoes[preds.argmax() - 1]
    identificando = False

while True:
    faces_qtd = 0
    _, frame = cap.read()  # OBTENDO A CAPTURA DA WEBCAM
    original = frame.copy()
    faces = face_detection.detectMultiScale(original, scaleFactor=1.1, minNeighbors=3, minSize=(20, 20))
    cinza = cv2.cvtColor(original, cv2.COLOR_BGR2GRAY)
    for face in faces:
        if not identificando:
            _thread.start_new_thread(identificar_expressao, (modelo, face, cinza))
        # Escrevendo a emoção resultante na imagem original
        cv2.putText(original, expressao, (face[0], face[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.65,
                    (0, 0, 255), 2, cv2.LINE_AA)
        # Desenhando o retângulo ao redor do rosto que está sendo analisado na imagem original
        cv2.rectangle(original, (face[0], face[1]), (face[0] + face[2], face[1] + face[3]), (0, 0, 255), 2)

    cv2.imshow("Frame", original)
    key = cv2.waitKey(1)
    if key == 27:
        break


import requests
import time
from functools import lru_cache

# Dictionary to keep track of URL access count
url_access_count = {}

def update_access_count(url):
    """Helper function to update URL access count."""
    url_access_count[url] = url_access_count.get(url, 0) + 1

def get_page(url):
    """Fetches the HTML content of a URL and caches the result for 10 seconds."""
    update_access_count(url)

    # Use lru_cache to cache the result with an expiration time of 10 seconds
    @lru_cache(maxsize=None, typed=False)
    def fetch_page(url):
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        else:
            raise Exception(f"Failed to fetch URL: {url}")

    return fetch_page(url)


import os
import json
import codecs
import collections
import argparse
import glob
import re

def sentence2words(s):
  # Turn words to lowercase
  s = s.lower()
  # Turn '-'  to space in sentence
  s = re.sub("[-']", ' ', s)
  # Split sentence to words
  words = s.split(' ')
  # Strip )(;"?!.,
  words = [ re.sub('[)(;"?!.,]', '', w) for w in words]
  return words

def load_embedding(embedding_path):
  vocab = []
  embedding = []
  with codecs.open(embedding_path,'r',encoding='utf8') as file:
    for line in file.readlines():
      line = line.strip()
      vocab.append(line.split(' ')[0])
      embedding.append(line)
  return vocab, embedding

def write_embedding(embedding_path,embedding):
  embedding_dir = os.path.dirname(embedding_path)
  embedding_file = os.path.basename(embedding_path)
  trimmed_embedding_path = embedding_dir+"/"+"customized_"+embedding_file
  # Write embedding
  with codecs.open(trimmed_embedding_path,'w',encoding='utf8') as file:
    embedding = '\n'.join(embedding)
    file.write(embedding)

def collect_data_vocab(data_path):
  words = []
  with open(data_path,'r') as file:
    for line in file.readlines():
      data = json.loads(line.strip())
      sentence1 = sentence2words(data['sentence1'])
      sentence2 = sentence2words(data['sentence2'])
      words += sentence1+sentence2
  return words

if __name__ == '__main__':
  # Create Parser
  parser = argparse.ArgumentParser()
  parser.add_argument(
      "--data_dir",
      type=str,
      default="./data",
      help="Directory to load data"
  )
  parser.add_argument(
      "--embedding_path",
      type=str,
      default="./vocab",
      help="Path to load pre-trained embedding and write back trimmed embedding"
  )
  FLAGS, _ = parser.parse_known_args()
  # Load embeding
  print("Loading pre-trained embedding")
  vocab, embedding = load_embedding(FLAGS.embedding_path)
  print("Loaded pre-trained embedding")
  # Find all files
  file_names = glob.glob(FLAGS.data_dir)
  # Get all unique words in data
  words = []
  for file_name in file_names:
    words += collect_data_vocab(file_name)
  words = [w[0] for w in collections.Counter(words).most_common()]
  print("Num of unique words: %d" % (len(words)))
  # Keep the part of embedding which we need
  print("Start to chop embedding")

  trimmed_embedding = []
  word2id = {}
  for v in vocab:
    word2id[v]=len(word2id)
  for word in words:
    if word in vocab:
      trimmed_embedding.append(embedding[word2id[word]])

  print("num of OOV words: %d" % (len(words)-len(trimmed_embedding)))
  print("Finished chopping embedding")
  # Write back trimmed embedding
  write_embedding(FLAGS.embedding_path,trimmed_embedding)


import csv
import xml.etree.ElementTree as ET

tree = ET.parse('sb_sortiment.xml')
root = tree.getroot()

print('year,name,name2,price,price_per_liter,size')
for i in root.findall("./artikel/[Ursprung='Champagne']"):

	year = i.find('Argang').text or "NV"
	name = i.find('Namn').text or ""
	name2 = i.find('Namn2').text
	if name2:
		name = f"{name} {name2}"
	packaging = i.find('Forpackning').text
	price = i.find('Prisinklmoms').text
	price_per_liter = i.find('PrisPerLiter').text
	size = i.find('Volymiml').text

	# I'm just interested in standard and Magnum bottles for comparison
	# Note: need to check both packaging and size,
	# there are sets of two bottles in a case adding up to 1500 ml
	if packaging in ('Flaska', 'Magnum') and size in ('750.00', '1500.00'):
		print(f"{year},{name},{price},{price_per_liter},{size}")


import streamlit as st
import pathlib
from PIL import Image
from io import BytesIO, StringIO

from fastbook import *
from fastai.vision.widgets import *
from fastai.vision.all import *

@st.experimental_memo
def get_categories(_model):
    return _model.dls.vocab

def showcase():
    st.header('Thomas More - Big Data Team Assigment: Image Classification')
    st.subheader('War Tanks Classification')

    model_name='tanks.pkl'

    plt = platform.system()
    if plt == 'Windows': pathlib.PosixPath = pathlib.WindowsPath
    model = load_learner(pathlib.Path()/model_name)

    image = st.file_uploader("Upload file", type=["png", "jpg", "jpeg", "webp"])
    show_image = st.empty()

    if not image:
        show_image.info("Choose a file to upload, only type: png, jpg, jpeg, webp ")
        return
    else:
        image = PILImage.create((image))
        show_image.image(image.to_thumb(500, 500), caption='Uploaded Image')

    if st.button('Classify'):
        pred, pred_idx, probs = model.predict(image)
        st.write(f'Prediction: {pred}; Probability: {probs[pred_idx]*100:.02f}%')

        chart_data = pd.DataFrame({'Probability (%)': probs*100}, index=get_categories(model))
        st.bar_chart(chart_data)
    else:
        st.write(f'Click the button to classify')

    image.close()


#!/usr/bin/env python
try:
    import apt
except ImportError:
    pass

from powerline.lib.threaded import ThreadedSegment, with_docstring

class AptUpgradesSegment(ThreadedSegment):
    interval = 300

    def update(self, oldthing):
        upgrades = 0
        for pkg in apt.Cache():
            if pkg.is_upgradable:
                upgrades += 1
        return upgrades
    def render(self, upgrades, **kwargs):
        if upgrades > 0:
            return [{
                "contents": "%i packages can be upgraded" % upgrades,
                "highlight_groups": ["packages:upgrades", "packages"]
                }]
        else:
            return None

AptUpgrades = with_docstring(AptUpgradesSegment(),
    """Displays the number of upgradable packages""")