#!/usr/bin/python3
"""

@@@@@@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@             @@@.    @@@    @@@.     @@@@    @@@@        @@@@@@@@@        @@@@@@@@@@@@@.     .@@@@@@@@@@@@@        @@@@@@@@@
@@@@@%%@@@%%@@@%%@@@@@             @@@@    @@@    @@@@     @@@@    @@@@       @@@@@@@@@@        @@@@@@@@@@@@@@     @@@@@@@@@@@@@@        @@@@@@@@@
@@@@@  @@@  @@@  @@@@@             @@@@    @@@    @@@@     @@@@    @@@@       @@@@   @@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@   @@@@
@@@@@            @@@@@             @@@@   @@@@    @@@@     @@@@@@@@@@@@       @@@@   @@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@   @@@@
@@@@@   @    @   @@@@@             @@@@   @@@@    @@@@     @@@@@@@@@@@@       @@@@@@@@@@@        @@@@@   @@@@@       @@@@@   @@@@       @@@@@@@@@@@
@@@@@            @@@@@             @@@@###@@@@@##@@@@@     @@@@    @@@@      @@@@@@@@@@@@       #@@@@@###@@@@@     ##@@@@@###@@@@       @@@@@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@             @@@@@@@@@@@@@@@@@@@     @@@@    @@@@      @@@@    @@@@@      @@@@@@@@@@@@@@     @@@@@@@@@@@@@@      @@@@     @@@@
@@@@@@@@@@@@@@@@@@@@@@

Project name: Whadda you see? (Raspberry pi edition)
Colour sensor Raspberry Pi Demo

Description:
  This project uses a colour sensor to determine the main colour of an object.
  The Hue color parameter (a number that represents the "pure colour") is calculated based on the RGB-values that are read from the sensor.
  From the Hue parameter it is possible to determine which basic color is presented to the sensor.
  If the button (connected to GPIO pin 20) is pressed, the corresponding colour is printed on the terminal and can be heard using the on-board audio (using a Text-To-Speech program)

  More information on the working principles can be found on whadda.com

Level of difficulty: Intermediate

Parts required:
  - Raspberry Pi 3B/B+ or 4B or equivalent (e.g. Whadda PI4SET)
  - 0.96" or 1.3" OLED screen (e.g. Whadda INSERT WHADDA CODE/VMA437 1.3 INCH OLED SCREEN FOR ARDUINO or INSERT WHADDA CODE/VMA438 0.96 INCH OLED SCREEN WITH I2C FOR ARDUINO)
  - TCS3200 color sensor module (e.g. Whadda INSERT WHADDA CODE/VMA325 COLOR SENSOR TCS3200 MODULE)

See illustrated wiring diagram for the necessary connections, available on whadda.com

For more information, check out whadda.com

Required Python modules:
    - RPi.GPIO (should be installed by default, if not use folowing command to install: sudo apt install python-rpi.gpio python3-rpi.gpio)
    - Pillow (command to install: pip3 install Pillow)
    - adafruit_ssd1306 (command to install: pip3 install adafruit-circuitpython-ssd1306)

Code inspired by how to Mechatronics (https://www.electronicshub.org/raspberry-pi-color-sensor-tutorial/)

Author: Midas Gossye
(c) 2020 Whadda, premium makers brand by Velleman
"""

# Import necessary modules
import RPi.GPIO as GPIO
from time import sleep
import time
import colorsys
import os
from PIL import Image, ImageDraw, ImageFont
import argparse

# Parse the command arguments
parser = argparse.ArgumentParser(prog='whadda_you_see', description='Determine and display colour from TCS3200 colour sensor')
parser.add_argument('--OLED', help='Add if OLED screen is installed', action='store_true', default=False)

args = parser.parse_args()
OLED_SCREEN_INSTALLED = args.OLED
###############################################################################################################################

# Import the OLED modules if an OLED screen is installed
if OLED_SCREEN_INSTALLED:
    import adafruit_ssd1306
    from board import SCL, SDA
    import busio
########################################################

GPIO.setmode(GPIO.BCM) # Use processor pin numbering system

NUM_CYCLES = 10 # number of measurements to perform to determine colour

# pin connections
S0 = 5      # S0 colour sensor
S1 = 6      # S1 colour sensor
S2 = 13     # 20 colour sensor
S3 = 19     # S3 colour sensor
Out = 26    # Out colour sensor
Button = 20 # Button
###############################

if OLED_SCREEN_INSTALLED:
    i2c = busio.I2C(SCL, SDA) # Initialize the I2C bus for the OLED screen

    oled = adafruit_ssd1306.SSD1306_I2C(128, 64, i2c) # Initialize OLED screen (128x64 pixels)
    oled.fill(0) # fill the screen with 0's (clear screen)

# setup GPIO pins (inputs/outputs)
GPIO.setup(S0, GPIO.OUT)
GPIO.setup(S1, GPIO.OUT)
GPIO.setup(S2, GPIO.OUT)
GPIO.setup(S3, GPIO.OUT)
GPIO.setup(Out, GPIO.IN, pull_up_down=GPIO.PUD_UP)
GPIO.setup(Button, GPIO.IN, pull_up_down=GPIO.PUD_UP)
#####################################################

# pin reading sequence for Red, Green, Blue measurements [S2, S3]
pin_reading_seq = [[GPIO.LOW, GPIO.LOW], [GPIO.HIGH, GPIO.HIGH], [GPIO.LOW, GPIO.HIGH], [GPIO.HIGH, GPIO.LOW]]
##############################################################################################################

RGBW = [0, 0, 0, 0] # Initialize empty array to hold Red, Green, Blue & White colour sensor readings

# Set colour frequency scaling to 2% / 12 kHz
GPIO.output(S0, GPIO.LOW)
GPIO.output(S1, GPIO.HIGH)

colour_names = ['Red', 'Green', 'Blue', 'White']

if OLED_SCREEN_INSTALLED:
    # Show Raspberry Pi Logo on OLED screen for 1 sec
    image = Image.open("RPI.bmp")
    oled.image(image)
    oled.show()
    sleep(1)
    ##################################################

    oled.fill(0) # fill the screen with 0's (clear screen)

    # Show Whadda Logo on OLED screen for 2 secs
    image = Image.open("WHADDA.bmp")
    oled.image(image)
    oled.show()
    sleep(2)
    ##################################################

    # Open a new blank image and draw object to later draw tex and shapes on
    image = Image.new("1", (oled.width, oled.height))
    draw = ImageDraw.Draw(image)
    #########################################################################

    oled.fill(0) # fill the screen with 0's (clear screen)

    # Load a font in 2 different sizes.
    font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 10)
    font2 = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 22)

    # Draw the text that doesn't need updating
    draw.text((0, 0), "Red: ", font=font, fill=255, align="right")
    draw.text((0, 10), "Green: ", font=font, fill=255, align="right")
    draw.text((0, 20), "Blue: ", font=font, fill=255, align="right")

    draw.text((70, 0), "Hue: ", font=font, fill=255, align="right")
    draw.text((70, 10), "Sat.: ", font=font, fill=255, align="right")
    draw.text((70, 20), "Value: ", font=font, fill=255, align="right")
    ##################################################################

    draw.line((64, 0, 64, 32), width=1, fill=255)  # draw vertical seperation line
    draw.line((0, 32, 127, 32), width=1, fill=255) # draw horizontal seperation line

    # Update image on OLED
    oled.image(image)
    oled.show()
    ######################

"""
Function descrition:
    Reads the Red, Green, Blue & White colour components from the TCS3200 colour sensor,
    converts the RGB colour into the HSV colour space, determines the colour name from the Hue value,
    and displays all of the above information on the OLED screen.

Arguments:
    None

Returns:
    -color_str (String with the colour name)
    -HSV_Val (list with Hue, Saturation and Value components respectively)
"""
def read_color():
    for idx, pin_states in enumerate(pin_reading_seq):
        # Select colour filter
        GPIO.output(S2, pin_states[0])
        GPIO.output(S3, pin_states[1])
        ##############################
        time.sleep(0.01)

        start = time.time() # start timing measurement to time pulse length

        # Wait until the NUM_cycles pulses have been counted
        for impulse_count in range(NUM_CYCLES):
            GPIO.wait_for_edge(Out, GPIO.FALLING)
        ####################################################

        duration = time.time() - start # calculate duration of pulses
        RGBW[idx] = ((NUM_CYCLES / duration) / 10000) *255 # compute Red, Green and White colour components and scale to 0-255

    # Scale Red, Green & Blue components with respect to the White/Brightness reading
    RGBW[0] = (RGBW[0] /RGBW[3])
    RGBW[1] = (RGBW[1] / RGBW[3])
    RGBW[2] = (RGBW[2] / RGBW[3])
    #################################################################################

    HSV_val = colorsys.rgb_to_hsv(RGBW[0], RGBW[1], RGBW[2]) # Convert RGB colour readings into HSV colour space

    color_str = ""

    # Convert Red, Green & Blue values to rounded percentages
    R_perc = round(RGBW[0]*100)
    G_perc = round(RGBW[1]*100)
    B_perc = round(RGBW[2]*100)
    #########################################################

    hue_deg = round(HSV_val[0]*360) # Convert Hue to rounded degrees

    # Convert value and saturation to rounded percentages
    val_perc = round(HSV_val[1]*100)
    sat_perc = round(HSV_val[2]*100)
    #####################################################

    if OLED_SCREEN_INSTALLED:
        # draw "black" rectangle to clear screen space for updated numbers
        draw.rectangle((34, 0, 63, 30), fill=0)
        draw.rectangle((100, 0, 127, 30), fill=0)
        ###################################################################

        # Put updated Red, Green and Blue percentages on screen
        draw.text((30, 0), "{:3d} %".format(R_perc), font=font, fill=255, align="right")
        draw.text((30, 10), "{:3d} %".format(G_perc), font=font, fill=255, align="right")
        draw.text((30, 20), "{:3d} %".format(B_perc), font=font, fill=255, align="right")
        #################################################################################

        # Put updated Hue, value and saturation numbers on screen
        draw.text((100, 0), "{:3d} °".format(hue_deg), font=font, fill=255, align="right")
        draw.text((100, 10), "{:3d} %".format(val_perc), font=font, fill=255, align="right")
        draw.text((100, 20), "{:3d} %".format(sat_perc), font=font, fill=255, align="right")
        ####################################################################################

    # Determine colour based on hue parameter
    if sat_perc < 35:
        color_str = "Black/Dark"
    elif hue_deg > 357 or hue_deg < 10:
        color_str = "ORANGE"
    elif hue_deg > 10 and hue_deg < 40:
        color_str = "YELLOW"
    elif hue_deg > 45 and hue_deg < 160:
        color_str = "GREEN"
    elif hue_deg > 210 and hue_deg < 270:
        color_str = "BLUE"
    elif hue_deg > 270 and hue_deg < 330:
        color_str ="PURPLE"
    elif hue_deg > 340 and hue_deg < 356:
        color_str = "RED"
    else:
        color_str = "No valid color detected"
    #########################################

    if OLED_SCREEN_INSTALLED:
        draw.rectangle((0, 34, 128, 60), fill=0) # clear bottom part of screen by drawing "black" rectangle

        # if a valid color is detected...
        if color_str != "No valid color detected" and color_str != "Black/Dark":
            draw.rectangle((0, 34, 128, 60), fill=255)                  # Draw a filled rectangle as background
            w, h = draw.textsize(color_str, font=font2)                 # Calculate pixel width and height of color name
            draw.text(((128-w)/2, 33), color_str, font=font2, fill=0)   # Put color name on screen with inversed text

        # Update OLED screen with new image
        oled.image(image)
        oled.show()
        ###################################

    return color_str, HSV_val

print("Whadda you see?")
print("===============")
print("\nPush button to get colour info\n")

while True:
    color_str, HSV_val = read_color() # Read and display colour name and values

    # If button is pressed...
    if GPIO.input(Button) == GPIO.LOW:
        # Print colour details on terminal
        print("Detected color: {}".format(color_str))
        print("Hue:", HSV_val[0]*360)
        print("\n")
        ##################################

        # Using the Pico Text-To-Speech program SAY the colour
        os.system('pico2wave -w tts.wav "{}" && aplay tts.wav'.format(color_str))

        print("=================")
        print("\n")


#!/usr/bin/env python

"""
   reads sqllite databases created by readlist.py  and readdu.py and writes out
    file information to a pandas dataframe

   example:  python ~/repos/pythonlibs/diskinventory/all_write.py files_pha.db scm16_pha.h5
"""
from __future__ import print_function

from sqlalchemy import and_
from sqlalchemy.orm import sessionmaker
import dataset
from pandas import DataFrame, Series
import pandas as pd
import argparse, textwrap
import datetime
import dateutil
import site,os
home_dir=os.getenv('HOME')
site.addsitedir('{}/repos/'.format(home_dir))
from pythonlibs.pyutils import check_md5 as c5

def get_frame_from_query(the_query,colnames):
    """make a dataframe from an sqlalchemy query"""
    df=DataFrame.from_records(list(the_query),columns=colnames)
    return df

linebreaks=argparse.RawTextHelpFormatter
descrip=textwrap.dedent(globals()['__doc__'])
parser = argparse.ArgumentParser(formatter_class=linebreaks,description=descrip)
parser.add_argument('file_db',type=str,help='file list db  generated by read_cron.py')
parser.add_argument('out_h5',type=str,help='HDFstore output file')
parser.add_argument('--size',nargs='?',default=0, type=float,help='float smallest size in bytes (default 0)')
args=parser.parse_args()

dbstring='sqlite:///{:s}'.format(args.file_db)
db = dataset.connect(dbstring)
session=sessionmaker(bind=db.engine)
thesession=session()
file_table=db.metadata.tables['files']
direc_table=db.metadata.tables['direcs']

large_query=thesession.query(file_table).filter(file_table.c.size > args.size).with_labels()
colnames=[item.name for item in list(db.metadata.tables['files'].columns)]

df_files=get_frame_from_query(large_query,colnames)

## test=df_files.loc[:]
## ## records=test.to_dict('records')
## ## print(records)

buf_length=int(1.e5)

def fullnames(row):
    the_file='{}/{}'.format(row['directory'],row['name'])
    return the_file

def calc_hash(row):
    the_file='{}/{}'.format(row['directory'],row['name'])
    size,start,mid,stop=c5.check_md5(the_file,buf_length)
    return (size,'{}{}{}'.format(start,mid,stop))

file_count=0
save_names=[]
save_fullhash=[]

save_names=df_files.apply(fullnames,axis=1)
save_fulltup=df_files.apply(calc_hash,axis=1)
pure_size=[item[0] for item in save_fulltup]
pure_hash=[item[1] for item in save_fulltup]

saveit=pd.DataFrame()
saveit['filenames']=save_names
saveit['fullhash']=pure_hash
saveit['fullsize']=pure_size
with pd.HDFStore(args.out_h5,'w') as store:
    store.put('files',saveit,format='table')


import cv2 as cv
import numpy as np

#Aufgabe 9_01

points = []
img = cv.imread("CVBVP-Daten/OpenCV-09/Img09a.jpg")

height, width, channels = img.shape

# Cut the image in half
width_cutoff = width // 2
s1 = img[:, :width_cutoff]
s2 = img[:, width_cutoff:]

def definePoints(action, x, y, flags, param):
    if len(points) == 8 and action == cv.EVENT_LBUTTONDOWN:
        print("Points 0 bis 3: ", points[0:3])
        print("Points 4 bis 7: ", points[4:7])
        M, mask = cv.findHomography(np.float32(points[4:8]), np.float32(points[0:4]))
        # M = cv.findHomography(points[0:3], points[4:7], cv.RANSAC, 3.0)
        warpedImage = cv.warpPerspective(s1, M, (width_cutoff, height))
        cv.imshow("Warped Image", warpedImage)
    elif action == cv.EVENT_LBUTTONDOWN:
        if len(points) < 4:
            cv.circle(s1, (x, y), 5, (0, 0, 255), -1)
            cv.imshow("Image1: ", s1)
        else:
            cv.circle(s2, (x, y), 5, (0, 0, 255), -1)
            cv.imshow("Image2: ", s2)
        print((x, y))
        points.append((x, y))
        print(len(points))

cv.namedWindow("Image1: ")
cv.namedWindow("Image2: ")
cv.setMouseCallback("Image1: ", definePoints)
cv.setMouseCallback("Image2: ", definePoints)

k = 0

while k != 113:
    cv.imshow("Image1: ", s1)
    cv.imshow("Image2: ", s2)

    k = cv.waitKey(0)
cv.destroyAllWindows()


import numpy as np
import matplotlib.pyplot as plt
from scipy.integrate import quad

def unprecize(step,p1,p2,sigma,epsil):
    r = np.linspace(p1,p2,int((p2-p1)/step))
    f = (1/r**2)*(sigma/r)**6*(1-2*(sigma/r)**6)*24*epsil*r
    I = 0
    for i in range(len(f)-1):
        I += 0.5*(f[i+1]+f[i])*step
    return I

def trapz(x,y):
    I = 0
    for i in range(len(x)-1):
        I += 0.5*(y[i+1]+y[i])*(x[i+1]-x[i-1])
    return I

def flj(r,sigma, epsil):
    return (1/r**2)*(sigma/r)**6*(1-2*(sigma/r)**6)*24*epsil*r

print("O que fazer?\n1 - Integrar LJ trapezoidal.\n2 - Distribuição Maxwell-Boltzmann\n3 - Interagir duas partículas.\n")
tarefa = input("Entre 1, 2 ou 3: ")

if tarefa == "1":

    v_max = 5
    n_points = 100
    dt = np.logspace(-5,-1,n_points)
    step = v_max*dt
    sigma = 1
    epsil = 1
    m = 1
    p1,p2 = sigma*.98, 3*sigma
    desv = np.zeros(n_points)

    r = np.linspace(p1,p2,10000)
    f = flj(r,sigma,epsil)
    v = 4*epsil*((sigma/r)**6*((sigma/r)**6)-1)

    E_precisa = quad(flj,p1,p2,args=(sigma,epsil))

    for i in range(n_points):
        desv[i] = (unprecize(step[i],p1,p2,sigma,epsil)-E_precisa[0])/E_precisa[0]

    fig, ax1 = plt.subplots()
    ax2  = ax1.twinx()

    dts = [0.00001, 0.0001, 0.001, 0.01, 0.02]

    lns2 = ax2.plot(r,0.00001*f/m,label='v after dt = 0.00001')
    lns3 = ax2.plot(r,0.0001*f/m,label='v after dt = 0.0001')
    lns4 = ax2.plot(r,0.001*f/m,label='v after dt = 0.001')
    lns5 = ax2.plot(r,0.01*f/m,label='v after dt = 0.01')
    lns6 = ax2.plot(r,0.01*f/m,label='v after dt = 0.02')
    #lns7 = ax2.plot(r,0.05*f/m,label='v after dt = 0.05')
    ax2.set_ylabel('speed')

    lns1 = ax1.plot(r,f,'-k',label='force')
    ax1.set_xlabel('r')
    ax1.set_ylabel('force')

    leg = lns2 + lns3 + lns4 + lns5 + lns6 #+ lns7
    labs = [l.get_label() for l in leg]
    ax1.legend(leg, labs, loc=0)

    plt.show()

    plt.semilogx(dt,desv,label='desvio')
    plt.legend()

    plt.show()
elif tarefa == "2":

# pdf maxwell

    plt.figure(7)
    ans = "y"
    while (ans == "y"):

        prop = input("Enter Temperature and mass and v_max for the graph: ").split()
        T = float(prop[0])
        m = float(prop[1])

        print("mean v = {} at T + {}".format(np.sqrt(2*T),T))

        v = np.linspace(0,float(prop[2]),1000)

        pdf = (m/(2*np.pi*T))**(3/2)*(4*np.pi*v**2)*np.exp(-(m*v**2/2*T))

        lab = "T = " + prop[0] + " m = " + prop[1]
        plt.plot(v,pdf,label=lab)
        plt.xlabel("Velocity")
        plt.ylabel("Probabilty density")
        ans = input("Draw one more curve? [y/n]")
    plt.legend()
    plt.show()

elif tarefa == 3:

# caso para duas partículas muito rápidas se aproximando

    t_fim  = 2
    print("tfim = {}\n".format(t_fim))
    vini = -float(input('V inicial >0 ? '))

    p1 =  3*sigma/2
    p2 =  -3*sigma/2
    ch = 'c'
    w = np.zeros(len(dts))
    j = 0
    while ch != 'cq':
        for dt1 in dts:

            v1, v2 = np.zeros(int(t_fim/dt1)), np.zeros(int(t_fim/dt1))
            x1, x2 = np.zeros(int(t_fim/dt1)), np.zeros(int(t_fim/dt1))
            F = np.zeros(int(t_fim/dt1))
            x1[0] = p1
            x2[0] = p2
            v1[0] = vini
            v2[0] = -vini
            for i in range(len(v1)-1):
                f1 = -flj(x1[i]-x2[i],sigma,epsil)

                x1[i+1] = x1[i] + dt1*v1[i] + f1*dt1**2/(2*m)
                x2[i+1] = x2[i] + dt1*v2[i] + (-f1)*dt1**2/(2*m)

                f2 = -flj(x1[i+1]-x2[i+1],sigma,epsil)
                F[i] = f2

                v1[i+1] = v1[i] + (f1+f2)*dt1/(2*m)
                v2[i+1] = v2[i] -(f1+f2)*dt1/(2*m)

            temp = np.linspace(0,t_fim, int(t_fim/dt1))
            plt.figure(3)
            plt.plot(temp,v1,label='V1 dt = '+str(dt1))
            #plt.plot(temp,v2,label='V2 dt = '+str(dt1))
            plt.figure(4)
            plt.plot(temp,x1,label='X1 dt = '+str(dt1))
            plt.figure(5)
            plt.plot(temp,F,label='F1 dt '+str(dt1))
            w[j] = trapz(x1,F)
            j += 1

        plt.figure(4)
        plt.xlabel('Time')
        plt.ylabel('Distance')
        plt.legend()

        plt.figure(3)
        plt.xlabel('Time')
        plt.ylabel('Speed')
        plt.legend()

        plt.figure(5)
        plt.xlabel('temp')
        plt.ylabel('force')
        plt.legend()

        plt.figure(6)
        plt.semilogx(dts,w)
        plt.xlabel('dt')
        plt.xlabel('work on one particle')

        plt.show()

        ch = input("Change something? \nEnter vini for initial velocity;\nEnter p1 for initial position,\nEnter tfim for final time.\n> Enter c to continue or cq to quit.\n")
        while (ch[0] != 'c'):
            if ch == 'vini':
                vini = -float(input('-V inicial?  '))
            elif ch == 'p1':
                vini = float(input('p1 inicial? '))
            elif ch == 'tfim':
                t_fim = float(input('tfim ? '))
            ch = input("Change something? \nEnter vini for initial velocity;\nEnter p1 for initial position,\nEnter tfim for final time.\n> Enter c to continue or cq to quit.\n")

    #plt.plot(temp,x1,label='X1 dt = '+str(dt1))
    #plt.plot(temp,x2,label='X2 dt = '+str(dt1))


from rest_framework.serializers import ModelSerializer

from ..models import User

class UserSerializer(ModelSerializer):
    class Meta:
        model = User
        fields = ('username', 'email', 'is_active', 'date_joined',
                  'orders', 'orders_amount')
        read_only_fields = ('date_joined', 'orders')


import warnings

import torch
import torch.nn.functional as F
from torch import nn

from antgo.framework.helper.models.classification.losses.accuracy import Accuracy
from antgo.framework.helper.models.classification.losses.cross_entropy_loss import *
from ...builder import *

@HEADS.register_module()
class ClsHead(nn.Module):
    """classification head.

    Args:
        loss (dict): Config of classification loss.
        topk (int | tuple): Top-k accuracy.
        cal_acc (bool): Whether to calculate accuracy during training.
            If you use Mixup/CutMix or something like that during training,
            it is not reasonable to calculate accuracy. Defaults to False.
    """

    def __init__(self,
                 loss=dict(type='CrossEntropyLoss', loss_weight=1.0, class_weight=None),
                 init_cfg=dict(use_softmax=True)):
        super().__init__()

        assert isinstance(loss, dict)
        self.compute_loss = CrossEntropyLoss(loss_weight=loss['loss_weight'], class_weight=loss['class_weight'])
        self.use_softmax = init_cfg.get('use_softmax', True)

    def loss(self, cls_score, gt_label, **kwargs):
        num_samples = len(cls_score)
        losses = dict()
        # compute loss
        loss = self.compute_loss(
            cls_score, gt_label, avg_factor=num_samples, **kwargs)
        losses['loss'] = loss
        return losses

    def forward_train(self, cls_score, gt_label, **kwargs):
        if isinstance(cls_score, tuple):
            cls_score = cls_score[-1]

        losses = self.loss(cls_score, gt_label.view(-1), **kwargs)
        return losses

    def simple_test(self, x, **kwargs):
        """Inference without augmentation.

    Args:
            x (tuple[Tensor]): The input features.
                Multi-stage inputs are acceptable but only the last stage will
                be used to classify. The shape of every item should be
                ``(num_samples, in_channels)``.
            softmax (bool): Whether to softmax the classification score.
            post_process (bool): Whether to do post processing the
                inference results. It will convert the output to a list.

        Returns:
            Tensor | list: The inference results.

                - If no post processing, the output is a tensor with shape
                  ``(num_samples, num_classes)``.
                - If post processing, the output is a multi-dimentional list of
                  float and the dimensions are ``(num_samples, num_classes)``.
        """
        cls_score = self.pre_logits(x)

        if self.use_softmax:
            pred = (
                F.softmax(cls_score, dim=1) if cls_score is not None else None)
        else:
            pred = cls_score
        return pred

    def pre_logits(self, x):
        if isinstance(x, tuple):
            x = x[-1]
        return x


from django.urls import path

from apps.comments import views

urlpatterns = [

    path('ajax/deleteComment/',     views.delete_comment,   name='deleteComment'),

    path('ajax/likeComment/',       views.like_comment,     name='likeComment'),
    path('ajax/unlikeComment/',     views.unlike_comment,   name='unlikeComment'),

    path('ajax/comment/',           views.post_comment,     name='comment'),
]


from typing import List

url = "https://leetcode.com/problems/rotate-function/description/"

class Solution:
    def maxRotateFunction(self, nums: List[int]) -> int:
        nums_length = len(nums)
        total_sum = sum(nums)
        current_sum = sum(i * nums[i] for i in range(nums_length))
        result = current_sum
        for i in range(1, nums_length):
            current_sum += total_sum - nums_length * nums[nums_length - i]
            result = max(result, current_sum)
        return result


import torch
import torch.nn as nn
from torch.utils.data import DataLoader
import logging
import os
from utils.utils import AverageMeter
from utils.config import ConfigParser
from model.rl.actor_critic import A2C
from definitions import *

logger = logging.getLogger(name=__name__)

# Classes
class RLTrainer:

    def __init__(self,
                 module: A2C,
                 optimizer: torch.optim.Optimizer,
                 train_loader: DataLoader,
                 valid_loader: DataLoader = None,
                 lr_scheduler=None,
                 config: ConfigParser = None,
                 device='cuda'
                 ):
        self.device = device
        self.module = module.to(device)
        self.critic_loss_fn = nn.MSELoss()
        self.optimizer = optimizer
        self.train_loader = train_loader
        self.valid_loader = valid_loader
        self.lr_scheduler = lr_scheduler
        self.config = config
        self.current_epoch = 0
        self.current_batch_idx = 0
        self.best_reward = 0
        os.makedirs(os.path.join(self.config['save_dir'], 'rl'), exist_ok=True)

    def train(self, resume=False):
        with torch.autograd.set_detect_anomaly(True):
            if resume:
                self._load_model()
                logger.info(f"------> Resume training from epoch {self.current_epoch}, "
                            f"batch_idx {self.current_batch_idx}...")
            else:
                logger.info(f"------> Begin training...")
            for epoch in range(self.current_epoch, self.config['rl']['num_epochs']):
                logger.info(f"------> Current epoch: {epoch}")
                train_result = self._train_epoch(epoch)

                if self.valid_loader is not None:
                    valid_result = self._eval_epoch(epoch)
                    valid_loss = valid_result['loss']
                    valid_reward = valid_result['reward']
                    valid_result['state_dict'] = self.module.model.state_dict()
                    if not self.module.freeze_critic:
                        valid_result['critic_model_state_dict'] = self.module.critic_model.state_dict()
                    valid_result['critic_head_state_dict'] = self.module.critic_head.state_dict()
                    valid_result['optimizer'] = self.optimizer.state_dict()
                    valid_result['lr_scheduler'] = self.lr_scheduler.state_dict() if self.lr_scheduler else None
                    valid_result['epoch'] = epoch + 1
                    valid_result['batch_idx'] = 0
                    valid_result['train_sampler'] = self.train_loader.sampler.get_state()
                    if valid_reward > self.best_reward:
                        self.best_reward = valid_reward
                        logger.info(f"New best valid reward found: {valid_reward}. Saving checkpoint...")
                        valid_result['best_reward'] = self.best_reward
                        torch.save(valid_result, os.path.join(self.config['save_dir'], 'rl', 'checkpoint_best.pt'))
                    logger.info(f"------> Saving checkpoint...")
                    valid_result['best_reward'] = self.best_reward
                    torch.save(valid_result, os.path.join(self.config['save_dir'], 'rl', 'checkpoint_last.pt'))
                    logger.info("------> Done.")
        return

    def _train_epoch(self, epoch):
        result = None
        self.module.train()
        reward_meter = AverageMeter()
        loss_meter = AverageMeter()
        num_batches_per_print = len(self.train_loader) // self.config['rl']['num_prints']
        for batch_idx, batch in enumerate(self.train_loader):
            rewards, critic_vals, action_lp_vals, entropy, total_rewards = self.module.train_env_episode(batch)
            loss = self.module.compute_loss(action_p_vals=action_lp_vals,
                                            G=rewards,
                                            V=critic_vals,
                                            entropy=entropy,
                                            critic_loss_fn=self.critic_loss_fn)
            self.optimizer.zero_grad()
            loss.backward()
            if self.config['rl']['grad_clip'] is not None:
                nn.utils.clip_grad_norm_(self.module.parameters(), self.config['rl']['grad_clip'])
            self.optimizer.step()
            lr = self.optimizer.param_groups[0]['lr']
            if self.lr_scheduler:
                self.lr_scheduler.step()
            loss_meter.update(loss.item())
            reward_meter.update(total_rewards.mean().item())
            result = {"loss": loss_meter.average(),
                      "reward": reward_meter.average(),
                      "lr": lr}
            if (batch_idx + 1) % num_batches_per_print == 0:
                logger.info(f"Training "
                            f"[{(self.current_batch_idx + batch_idx) * self.train_loader.batch_size}/{len(self.train_loader.dataset)} "
                            f"({(100. * (self.current_batch_idx + batch_idx) / len(self.train_loader)):.0f}%)] "
                            f"| Loss: {loss_meter.average():.5f} "
                            f"| Reward: {reward_meter.average():.4f}"
                            f"| Learning Rate: {lr}")
                # Save checkpoint
                logger.info(f"------> Saving checkpoint...")
                result['state_dict'] = self.module.model.state_dict()
                result['critic_model_state_dict'] = self.module.critic_model.state_dict()
                result['critic_head_state_dict'] = self.module.critic_head.state_dict()
                result['optimizer'] = self.optimizer.state_dict()
                result['lr_scheduler'] = self.lr_scheduler.state_dict() if self.lr_scheduler else None
                result['epoch'] = epoch
                result['batch_idx'] = self.current_batch_idx + batch_idx
                result['best_reward'] = self.best_reward
                result['train_sampler'] = self.train_loader.sampler.get_state()
                torch.save(result, os.path.join(self.config['save_dir'], 'rl', 'checkpoint_last.pt'))
                logger.info("------> Done.")
        logger.info(f"Finished training epoch {epoch} "
                    f"| Loss: {loss_meter.average():.5f} "
                    f"| Reward: {reward_meter.average():.4f}"
                    f"| Learning Rate: {self.optimizer.param_groups[0]['lr']}"
                    )
        self.current_batch_idx = 0
        return result

    def _eval_epoch(self, epoch):
        self.module.eval()
        result = None
        loss_meter = AverageMeter()
        reward_meter = AverageMeter()
        num_batches_per_print = len(self.valid_loader) // self.config['rl']['num_prints']
        with torch.no_grad():
            for batch_idx, batch in enumerate(self.valid_loader):
                rewards, critic_vals, action_lp_vals, entropy, total_rewards = self.module.train_env_episode(batch)
                loss = self.module.compute_loss(action_p_vals=action_lp_vals,
                                                G=rewards,
                                                V=critic_vals,
                                                entropy=entropy,
                                                critic_loss_fn=self.critic_loss_fn)
                loss_meter.update(loss.item())
                reward_meter.update(total_rewards.mean())
                result = {"loss": loss_meter.average(),
                          "reward": reward_meter.average()
                          }
                if (batch_idx + 1) % num_batches_per_print == 0:
                    logger.info(f"Evaluating "
                                f"[{batch_idx * self.valid_loader.batch_size}/{len(self.valid_loader.dataset)} "
                                f"({(100. * batch_idx / len(self.valid_loader)):.0f}%)] "
                                f"| Loss: {loss_meter.average():.5f} "
                                f"| Reward: {reward_meter.average():.4f}"
                                )
        logger.info(f"Finished training epoch {epoch} "
                    f"| Loss: {loss_meter.average():.5f} "
                    f"| Reward: {reward_meter.average():.4f}"
                    )
        return result

    def _load_model(self):
        checkpoint_path = os.path.join(self.config['save_dir'], 'rl', 'checkpoint_last.pt')
        checkpoint = torch.load(checkpoint_path, map_location=self.device)
        self.module.model.load_state_dict(checkpoint['state_dict'])
        if not self.module.freeze_critic:
            self.module.critic_model.load_state_dict(checkpoint['critic_model_state_dict'])
        self.module.critic_head.load_state_dict(checkpoint['critic_head_state_dict'])
        self.optimizer.load_state_dict(checkpoint['optimizer'])
        self.optimizer.param_groups[0]['capturable'] = True
        if self.lr_scheduler is not None:
            self.lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
        self.best_reward = checkpoint['best_reward']
        self.current_epoch = checkpoint['epoch']
        self.current_batch_idx = checkpoint['batch_idx']
        self.train_loader.sampler.set_state(checkpoint['train_sampler'])
        logger.info(f"------> Loaded checkpoint from {checkpoint_path}")


import matplotlib.pyplot as plt
import pandas as pd

# load the csv file (Monthly average air temperatures of the city of Barcelona since 1780) into a pandas data frame
df_temperature = pd.read_csv('bar1.csv', encoding='utf-8')

print(df_temperature.info())
df_temperature.set_index('Any', inplace=True)

df_temperature.index.name = 'year'

# calculate the yearly average air temperature
df_temperature['average_temperature'] = df_temperature.mean(axis=1)

# drop columns containing monthly values
df_temperature = df_temperature[['average_temperature']]

# visualize the first 5 columns
df_temperature.head()

plt.style.use('seaborn')

# line plot - the yearly average air temperature in Barcelona
df_temperature.plot(color='green', linewidth=3, figsize=(12,6))

# modify ticks size
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.legend('')

# title and labels
plt.title('The  average Count ', fontsize=20)
plt.xlabel('Days', fontsize=16)
plt.ylabel('Count x 10', fontsize=16)


# -*- coding: utf-8 -*-
"""
Created on Sun Sep 27 18:31:48 2020
"""
import matplotlib
import matplotlib.pyplot as plt
import torch
import numpy as np
import numpy

'''
The generalization gap is estimated by the gap between the training error and the test error.

In particular, we save the model which achieves the smallest training error and test it over the test tasks and compute the error as the test error.

As the training procedure of MAML is unstable and the test variance is big, we compute the average of the last five gaps (see get_gap).
'''

path='./regression/'
path_loo='./regression_loo/'
font = {'size': 20}
matplotlib.rc('font', **font)

def get_val(model,i):
    return model['val_loss'][list.index(model['train_loss'], get_train(model,i))]
def get_train(model,i):
    return sorted(model['train_loss'])[i]
def get_gap(model):
    gap_list=[]
    for i in range(5):
        if abs((get_val(model,i)-get_train(model,i)))<1:
            gap_list.append(get_val(model,i)-get_train(model,i))
    return np.mean(np.array(gap_list))

task_num=1000
print_loovssq_shot=True
print_high=True

if print_loovssq_shot:
    gap_sq=[]
    gap_loo=[]
    gap_sq_upper=[]
    gap_sq_lower=[]
    gap_loo_upper=[]
    gap_loo_lower=[]
    shot_list=[3,4,5,6,7,8,9,10]
    for shot in shot_list:
        gap_list=[]
        for run_time in range(20):
            model=torch.load(path+'ReMaml_1000_'+str(shot)+'shot_15query'+'_'+str(run_time)+'/trlog')
            if get_gap(model)<1:
                gap_list.append(abs(get_gap(model)))
        gap_sq.append(np.mean(np.array(gap_list)))
        gap_sq_upper.append(np.mean(np.array(gap_list))+np.std(np.array(gap_list)))
        gap_sq_lower.append(np.mean(np.array(gap_list))-np.std(np.array(gap_list)))
    for shot in shot_list:
        gap_list=[]
        for run_time in range(20):
            model=torch.load(path_loo+'ReMaml_1000_'+str(shot)+'shot'+'_'+str(run_time)+'/trlog')
            if get_gap(model)<1:
                gap_list.append(abs(get_gap(model)))
        gap_loo.append(np.mean(np.array(gap_list)))
        gap_loo_upper.append(np.mean(np.array(gap_list))+np.std(np.array(gap_list)))
        gap_loo_lower.append(np.mean(np.array(gap_list))-np.std(np.array(gap_list)))

    shot=shot_list
    plt.plot(shot, gap_sq, linewidth=5, label='$|\mathcal{R}-\hat{\mathcal{R}}_{s/q}|$')
    plt.plot(shot, gap_sq, 'v', ms=14, color='red')
    plt.fill_between(shot, gap_sq_upper,gap_sq_lower,alpha=0.3)
    plt.plot(shot, gap_loo, linewidth=5, color='orange', label='$|\mathcal{R}-\hat{\mathcal{R}}_{loo}|$')
    plt.plot(shot, gap_loo, 'v', ms=14,color='g')
    plt.fill_between(shot, gap_loo_upper,gap_loo_lower,color='orange',alpha=0.3)
    plt.ylabel('Error gap', fontsize=24)
    plt.xlabel('# of shots', fontsize=24)
    plt.grid(True)
    plt.tight_layout()
    plt.legend()
    #plt.savefig('./maml_gap_loo_vs_sq_shot.pdf')
    plt.show()

if print_high:
    gap_sq=[]
    gap_sq_upper=[]
    gap_sq_lower=[]
    shot_list=[2,20,200,2000,20000]
    for shot in shot_list:
        gap_list=[]
        for run_time in range(20):
            model=torch.load(path+'ReMaml_1000_'+str(shot)+'shot_15query'+'_'+str(run_time)+'/trlog')
            if get_gap(model)<1:
                gap_list.append(abs(get_gap(model)))
        gap_sq.append(np.mean(np.array(gap_list)))
        gap_sq_upper.append(np.mean(np.array(gap_list))+np.std(np.array(gap_list)))
        gap_sq_lower.append(np.mean(np.array(gap_list))-np.std(np.array(gap_list)))

    shot=[1,2,3,4,5]
    plt.fill_between(shot, gap_sq_upper,gap_sq_lower,alpha=0.5)
    plt.plot(shot, gap_sq, linewidth=5, label='$\mathcal{R}-\hat{\mathcal{R}}_{s/q}$')
    plt.plot(shot, gap_sq, 'v', ms=14)
    plt.ylabel('Error gap', fontsize=24)
    plt.xlabel('# of shots', fontsize=24)
    plt.xticks(shot,shot_list)
    plt.grid(True)
    plt.ylim(-0.5,0.5)
    plt.tight_layout()
    plt.legend()
    #plt.savefig('./figure/ours_gap_loo_vs_sq_shot.pdf')
    plt.show()


import json

class User:

    def to_json(self):
        return json.dumps(self, default=lambda o: o.__dict__)

    def __init__(self, user_id, first_name, last_name, creation_time, update_time, existing_skills,
                 desired_skills, courses):

        self.user_id = str(user_id)
        self.first_name = first_name
        self.last_name = last_name
        self.creation_time = str(creation_time)
        self.update_time = str(update_time)
        self.existing_skills = existing_skills
        self.desired_skills = desired_skills
        self.courses = courses


# -*- coding: utf-8 -*-

# Standard imports
import argparse as ap
from pathlib import Path
import os
import pickle

# External imports
import pandas as pd
from sklearn import metrics
from sklearn.model_selection import cross_validate, train_test_split
from sklearn.model_selection import GridSearchCV, KFold
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.ensemble import VotingRegressor
from sklearn.neighbors import KNeighborsClassifier
from matplotlib import pyplot as plt
from matplotlib import cm
from matplotlib.colors import Normalize
import numpy as np
from scipy.spatial.distance import cdist

import warnings
warnings.filterwarnings("ignore")

# Script information
__author__ = "Marti Municoy"
__license__ = "GPL"
__version__ = "1.0.1"
__maintainer__ = "Marti Municoy"
__email__ = "marti.municoy@bsc.es"

def parse_args():
    parser = ap.ArgumentParser()
    parser.add_argument("descriptors_path", metavar="PATH", type=str,
                        help="Path to descriptors")
    parser.add_argument("labels_path", metavar="PATH", type=str,
                        help="Path to labels")
    parser.add_argument("-o", "--output", metavar="PATH", type=str,
                        default="model_out", help="Output path")

    args = parser.parse_args()

    return args.descriptors_path, args.labels_path, args.output

def generate_logistic_model(X_train, y_train, X_test, y_test):
    print(' - Generating logistic regression model')
    log_reg = LogisticRegression(random_state=1993)
    cv = KFold(n_splits=5)
    results = cross_validate(log_reg, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='balanced_accuracy')

    best_model = None
    best_scorer = 0
    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', np.mean(results['test_score']))

    y_pred = best_model.predict(X_test)

    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_forest_model(X_train, y_train, X_test, y_test):
    print(' - Generating random forest model')

    for_cla = RandomForestClassifier(class_weight='balanced',
                                     random_state=1993)
    cv = KFold(n_splits=5)
    params_rf = {'n_estimators': [50, 100], 'max_depth': [2, 3, 4]}
    for_cla_gs = GridSearchCV(for_cla, params_rf, cv=cv,
                              scoring='balanced_accuracy')
    for_cla_gs.fit(X_train, y_train)

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', for_cla_gs.best_score_)

    best_model = for_cla_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][0]))
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][1]))
    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_neighbors_model(X_train, y_train, X_test, y_test):
    print(' - Generating k-nearest neighbors model')
    for_cla = KNeighborsClassifier()
    cv = KFold(n_splits=5)
    params_rf = {'n_neighbors': [1, 2, 3, 4, 5]}
    for_cla_gs = GridSearchCV(for_cla, params_rf, cv=cv,
                              scoring='balanced_accuracy')
    for_cla_gs.fit(X_train, y_train)

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', for_cla_gs.best_score_)

    best_model = for_cla_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(for_cla_gs.best_params_.items())][0]))
    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_ensemble_classifier(models_to_combine,
                                 X_train, y_train, X_test, y_test):
    print(' - Generating ensemble model')

    ensemble = VotingClassifier(estimators=models_to_combine, voting='soft')
    cv = KFold(n_splits=5)
    results = cross_validate(ensemble, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='balanced_accuracy')

    best_model = None
    best_scorer = 0
    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print('   - Cross-validation results:')
    print('     - Balanced accuracy:', np.mean(results['test_score']))

    y_pred = best_model.predict(X_test)

    print("   - Test set results:")
    print("     - Confusion matrix:")
    confusion_matrix = metrics.confusion_matrix(y_test, y_pred)
    print("        | {} | {} |".format(*confusion_matrix[0]))
    print("        | {} | {} |".format(*confusion_matrix[1]))
    print("     - Accuracy:", metrics.accuracy_score(y_test, y_pred))
    print("     - Balanced accuracy:",
          metrics.balanced_accuracy_score(y_test, y_pred))
    print("     - Precision:", metrics.precision_score(y_test, y_pred))
    print("     - Recall:", metrics.recall_score(y_test, y_pred))

    return best_model

def generate_regression_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear regression model')
    lin_reg = LinearRegression()
    cv = KFold(n_splits=5)
    results = cross_validate(lin_reg, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    print(X_test)

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_lasso_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear lasso model')
    lin_las = Lasso()
    cv = KFold(n_splits=5)
    params_rf = {'alpha': np.arange(0, 1, 0.05)}
    lin_las_gs = GridSearchCV(lin_las, params_rf, cv=cv,
                              scoring='r2')
    lin_las_gs.fit(X_train, y_train)

    best_model = lin_las_gs.best_estimator_

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', lin_las_gs.best_score_)
    print("   - Best parameters:")
    print("     - {}: {}".format(
        *[(i, j) for i, j in list(lin_las_gs.best_params_.items())][0]))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_ridge_model(X_train, y_train, X_test, y_test):
    print(' - Generating linear ridge model')
    lin_rid = Ridge()
    cv = KFold(n_splits=5)
    results = cross_validate(lin_rid, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Coefficients:")
    print("     - be / atoms : {}".format(best_model.coef_[0]))
    print("     - rotamers : {}".format(best_model.coef_[1]))
    print("     - internal : {}".format(best_model.coef_[2]))
    print("   - Intercept:")
    print("     - {}".format(best_model.intercept_))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def generate_ensemble_regressor(models_to_combine,
                                X_train, y_train, X_test, y_test):
    print(' - Generating ensemble model')

    ensemble = VotingRegressor(estimators=models_to_combine)
    cv = KFold(n_splits=5)
    results = cross_validate(ensemble, X_train, y_train, cv=cv,
                             return_estimator=True,
                             scoring='r2')

    best_model = None
    best_scorer = 0

    for m, s in zip(results['estimator'], results['test_score']):
        if (best_model is None or best_scorer < s):
            best_scorer = s
            best_model = m

    y_pred = best_model.predict(X_test)

    print('   - Cross-validation results:')
    print('     - r2:', np.max(results['test_score']))
    print("   - Test set results:")
    print("     - r2:", metrics.r2_score(y_test, y_pred))

    return best_model

def plot_classification(activity_classifier_1, X_test, y_test,
                        title, output_path):
    disp = metrics. plot_confusion_matrix(activity_classifier_1,
                                          X_test, y_test,
                                          cmap=plt.cm.Blues)

    disp.ax_.set_title(title)

    plt.savefig(output_path)
    plt.clf()
    plt.cla()

def plot_regressor(model, X_test, y_test, X_train, y_train, title,
                   output_path, nn=None, min_nn=1):
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 6))
    plt.subplots_adjust(left=0.15, bottom=0.3, right=0.95, top=0.9)
    fig.suptitle(title, fontsize=16)

    y_pred = model.predict(X_train)
    ax1.plot(((min(y_train), max(y_train))), ((min(y_pred), max(y_pred))),
             'k--', linewidth=1)
    ax1.scatter(y_train, y_pred, color='red')
    ax1.set_title("Cross-validation results")
    ax1.set_ylabel("Predicted -pIC50")
    ax1.set_xlabel("Experimental -pIC50")

    y_pred = model.predict(X_test)
    ax2.plot(((min(y_test), max(y_test))), ((min(y_pred), max(y_pred))),
             'k--', linewidth=1)
    if (nn is None):
        ax2.scatter(y_test, y_pred, color='red', c=nn)
    else:
        colormap = cm.get_cmap('Wistia', max(nn) - (min_nn - 1))
        norm = Normalize(0, max(nn) - (min_nn - 1))
        colormap.set_under('grey')
        for x, y, n in zip(y_test, y_pred, nn):
            ax2.scatter(x, y, c=[colormap(n - min_nn)], label=n)
        fig.colorbar(cm.ScalarMappable(cmap=colormap, norm=norm), ax=ax2,
                     extend='min')
        X_test, y_test = filter_entries_by_nn(X_test, y_test,
                                              nn,
                                              min_nn=min_nn)
    ax2.set_title("Test set results")
    ax2.set_xlabel("Experimental -pIC50")

    try:
        props = dict(boxstyle='round', facecolor='orange', alpha=0.5)
        text = 'Coefficients:\n'
        text += ' • be / atoms = {:.2f}\n'.format(model.coef_[0])
        text += ' • rotamers = {:.2f}\n'.format(model.coef_[1])
        text += ' • internal = {:.2f}'.format(model.coef_[2])
        # place a text box in upper left in axes coords
        plt.text(0.1, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Intercept:\n'
        text += ' • {:.2f}'.format(model.intercept_)
        plt.text(0.35, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Cross-validation r2:\n'
        text += ' • {:.2f}'.format(model.score(X_train, y_train))
        plt.text(0.55, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Test set r2:\n'
        text += ' • {:.2f}'.format(model.score(X_test, y_test))
        plt.text(0.8, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

    except AttributeError:
        text = 'Cross-validation r2:\n'
        text += ' • {:.2f}'.format(model.score(X_train, y_train))
        plt.text(0.30, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

        text = 'Test set r2:\n'
        text += ' • {:.2f}'.format(model.score(X_test, y_test))
        plt.text(0.75, 0.1, text, fontsize=12,
                 verticalalignment='center', bbox=props,
                 transform=fig.transFigure)

    plt.savefig(output_path)
    plt.clf()
    plt.cla()

def domain_analysis(train_domain):
    print(" - Computing applicability domains")

    # Matrix of descriptor distances between entries
    distances = np.array([cdist([x], train_domain) for x in train_domain])
    distances_sorted = [np.sort(d[0]) for d in distances]

    # Discard 0s, which are distances to the same entry (diagonal
    # elements in the matrix)
    distances_matrix = [d[1:] for d in distances_sorted]

    # Empirical calculation of the smoothing parameter k
    k = int(round(pow(len(train_domain), 1 / 3)))

    # Calculate the mean of first k distances for each sample
    d_means = [np.mean(d[:k][0]) for d in distances_matrix]

    # Calculate quartiles
    Q1 = np.quantile(d_means, .25)
    Q3 = np.quantile(d_means, .75)

    # Calculate reference value
    d_ref = Q3 + 1.5 * (Q3 - Q1)

    # Calculate thresholds
    Dijs = []
    Kis = []
    for distances in distances_matrix:
        allowed_distances = [d for d in distances if d <= d_ref]
        Dijs.append(allowed_distances)
        Kis.append(len(allowed_distances))

    thresholds = []
    for Dij, Ki in zip(Dijs, Kis):
        if (Ki == 0):
            thresholds.append(0)
        else:
            thresholds.append(np.sum(Dij) / Ki)

    thresholds = np.array(thresholds)

    # Convert 0 to minimum value not zero
    thresholds[thresholds == 0] = min(thresholds[thresholds != 0])

    return thresholds

def check_applicability_domain(train_domain, test_domain, thresholds):
    print(" - Checking applicability domains")

    # Matrix of descriptor distances between entries
    distances = np.array([cdist([x], train_domain) for x in test_domain])

    results = []
    for Dijs in distances:
        # get nearest neighbors
        NN = len([j for j, Dij in enumerate(Dijs[0]) if Dij <= thresholds[j]])
        results.append(NN)

    return results

def filter_entries_by_nn(X, y, nn, min_nn):
    f_X = X[[i for i, _ in enumerate(X) if nn[i] >= min_nn]]
    f_y = y[[i for i, _ in enumerate(y) if nn[i] >= min_nn]]

    return f_X, f_y

def main():
    descriptors_path, labels_path, output_path = parse_args()

    descriptors_path = Path(descriptors_path)
    labels_path = Path(labels_path)
    output_path = Path(output_path)

    if (not descriptors_path.is_file()):
        print(' - Error: invalid descriptors path \'{}\''.format(
            descriptors_path))

    if (not labels_path.is_file()):
        print(' - Error: invalid dataset path \'{}\''.format(
            labels_path))

    if (not output_path.is_dir()):
        os.mkdir(str(output_path))

    descriptors = pd.read_csv(str(descriptors_path))
    labels = pd.read_csv(str(labels_path))

    dataset = descriptors.merge(labels, left_on='path', right_on='path')

    # Remove unnamed columns
    dataset = dataset.loc[:, ~dataset.columns.str.contains('^Unnamed')]

    # Remove non numeric IC50 (exclamation mark!)
    f_dataset = dataset[pd.to_numeric(dataset.IC50,
                                      errors='coerce').notnull()]

    # Calculate BE / number of atoms
    f_dataset['be / atoms'] = f_dataset['be'] / f_dataset['atoms']

    # Drop unnecessary columns
    f_dataset.drop(['toten', 'weight', 'atoms', 'be', 'clusters'], axis=1, inplace=True)

    # Split by activity
    f_dataset.loc[:, 'IC50'] = f_dataset.loc[:, 'IC50'].apply(pd.to_numeric)
    f_dataset['pIC50'] = - np.log10(f_dataset.loc[:, 'IC50'] / 1000000)
    actives, inactives = [x for _, x in f_dataset.groupby(
        [f_dataset['IC50'] < 0])]

    # Convert activity in a boolean
    f_dataset['activity'] = f_dataset['IC50'] > 0
    f_dataset.drop(['IC50'], axis=1, inplace=True)

    """
    # Split data into training and test set
    X_train, X_test, y_train, y_test = train_test_split(
        f_dataset.loc[:, ['be / atoms', 'rotamers', 'internal']].to_numpy(),
        f_dataset['activity'].to_list(),
        train_size=0.75,
        random_state=123,
        stratify=f_dataset['activity'].to_list())

    activity_classifier_1 = generate_logistic_model(X_train, y_train,
                                                    X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_1.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_1, handle)

    plot_classification(activity_classifier_1, X_test, y_test,
                        "Activity logistic regression classification",
                        str(output_path.joinpath('activity_classifier_1.png')))

    activity_classifier_2 = generate_forest_model(X_train, y_train,
                                                  X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_2.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_2, handle)

    plot_classification(activity_classifier_2, X_test, y_test,
                        "Activity random forest classification",
                        str(output_path.joinpath('activity_classifier_2.png')))

    activity_classifier_3 = generate_neighbors_model(X_train, y_train,
                                                     X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_3.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_3, handle)

    plot_classification(activity_classifier_3, X_test, y_test,
                        "Activity k-nearest neighbor classification",
                        str(output_path.joinpath('activity_classifier_3.png')))

    activity_classifier_4 = generate_ensemble_classifier(
        [('Logistic regression', activity_classifier_1),
         ('Random forest', activity_classifier_2,),
         ('k-nearest neighbors', activity_classifier_3)],
        X_train, y_train, X_test, y_test)

    with open(str(output_path.joinpath('activity_classifier_4.pkl')), 'wb') \
            as handle:
        pickle.dump(activity_classifier_4, handle)

    plot_classification(activity_classifier_4, X_test, y_test,
                        "Activity ensemble classification",
                        str(output_path.joinpath('activity_classifier_4.png')))
    """

    # Split data into training and test set
    X_train, X_test, y_train, y_test = train_test_split(
        actives.loc[:, ['be / atoms', 'rotamers', 'internal']].to_numpy(),
        actives['pIC50'].to_numpy(),
        train_size=0.75,
        random_state=123)

    thresholds = domain_analysis(X_train)

    nn_results = check_applicability_domain(X_train, X_test, thresholds)

    min_nn = 0
    f_X_test, f_y_test = filter_entries_by_nn(X_test, y_test,
                                              nn_results,
                                              min_nn=min_nn)
    affinity_predictor_1 = generate_regression_model(X_train, y_train,
                                                     f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_1.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_1, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_1, X_test, y_test, X_train, y_train,
                   'Linear regression',
                   str(output_path.joinpath('affinity_predictor_1.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_2 = generate_lasso_model(X_train, y_train,
                                                f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_2.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_2, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_2, X_test, y_test, X_train, y_train,
                   'Lasso regression',
                   str(output_path.joinpath('affinity_predictor_2.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_3 = generate_ridge_model(X_train, y_train,
                                                f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_3.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_3, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_3, X_test, y_test, X_train, y_train,
                   'Ridge regression',
                   str(output_path.joinpath('affinity_predictor_3.png')),
                   nn=nn_results, min_nn=min_nn)

    affinity_predictor_4 = generate_ensemble_regressor(
        [('Linear regression', affinity_predictor_1),
         ('Ridge regression', affinity_predictor_3)],
        X_train, y_train, f_X_test, f_y_test)

    with open(str(output_path.joinpath('affinity_predictor_4.pkl')),
              'wb') as handle:
        pickle.dump(affinity_predictor_4, handle)
        pickle.dump(thresholds, handle)

    plot_regressor(affinity_predictor_4, X_test, y_test, X_train, y_train,
                   'Ensemble regression',
                   str(output_path.joinpath('affinity_predictor_4.png')),
                   nn=nn_results, min_nn=min_nn)

if __name__ == "__main__":
    main()


import urllib.request
from tqdm import tqdm

l = []

def conv_url(url, q, p):
    url = url.replace('%%%query%%%', q)
    url = url.replace('%%%page%%%', p)
    return url

url = 'https://pixabay.com/api/?key=4683416-8754135a461a502021a00143e&q=%%%query%%%&image_type=photo&per_page=200&page=%%%page%%%'

query = ['cat', 'dog', 'mouse', 'horse', 'panda', 'duckling', 'hedghog', 'kitten', 'lamb', 'owl', 'bunny', 'piglet', 'penguins', 'baby+hipo', 'fox', 'rainbow', 'seal', 'dolphin', 'deer', 'crocodile', 'pet', 'chick', 'bird', 'rat', 'puppy', 'retriever', 'buldog']
query2 = []
page = ['1', '2']
for q in query:
    print('Fetching {}'.format(q))
    for p in page:
        d = eval(urllib.request.urlopen(conv_url(url, q, p)).read().decode('utf-8'))
        for photo in d['hits']:
            l.append(photo['webformatURL'])

print('Download started')
num = 0
for i in tqdm(range(len(l))):
    num += 1
    f = open('{}.jpg'.format(num),'wb')
    f.write(urllib.request.urlopen(l[i]).read())
    f.close()


import pandas as pd
import numpy as np
from numpy.typing import ArrayLike
from typing import Tuple
from rdkit import Chem
from sklearn.model_selection import RepeatedKFold, train_test_split, ShuffleSplit
from sklearn.utils import shuffle

def load_dataset(csv_file: str, verbose: bool = False):
    '''Loads in a csv dataset of reactions.

    Dataset should be of the same format as defined by the
    supplied b97d3 dataset. This includes SMILES strings of
    reactants and products, the activation energy barriers and
    the enthalpies of the reactions.

    Arguments:
        csv_file: Path to csv dataset of reactions.
        verbose: Whether to print extra info.
    '''

    if verbose: print(f'Loading in dataset from {csv_file}')

    data = pd.read_csv(csv_file)
    ea = data.ea.tolist()              # Activation energies
    dh = data.dh.tolist()              # Enthalpy changes
    rs = data.rsmi.tolist()            # Reactant smiles
    ps = data.psmi.tolist()            # Product smiles

    if verbose: print('Dataset loaded.')

    return ea, dh, rs, ps

def extract_data(ea: list, dh: list, rs: list, ps: list, num_reacs: int, train_direction: str):
    '''Extract data from a dataset to work with a given train_direction.

    Takes the original data from the dataset and reorders it
    in the provided train direction, while also extracting
    RDKit MOL objects from the reactnat and product SMILES.

    'forward' train_direction simply extracts data in the
    order presented in the dataset, 'backward' extracts
    the reverse reactions of the ones in the dataset, and
    'both' extracts both the forward and backward reactions.

    Arguments:
        ea: Raw activation energies from dataset.
        dh: Raw enthalpy changes from dataset.
        rs: Reactant SMILES strings from dataset.
        ps: Product SMILES strings from dataset.
        num_reacs: Number of reactions in dataset (including train_direction).
        train_direction: Direction of reactions to train from.
    '''
    Eact = np.zeros(num_reacs)
    dH = np.zeros(num_reacs)
    rmol = []
    pmol = []

    # Rework arrays based on train_direction
    if train_direction == 'forward':
        for i in range(num_reacs):
            Eact[i] = ea[i]
            dH[i] = dh[i]
            rmol.append(Chem.MolFromSmiles(rs[i]))
            pmol.append(Chem.MolFromSmiles(ps[i]))
    elif train_direction == 'backward':
        for i in range(num_reacs):
            Eact[i] = ea[i] - dh[i]
            dH[i] = -dh[i]
            rmol.append(Chem.MolFromSmiles(ps[i]))
            pmol.append(Chem.MolFromSmiles(rs[i]))
    elif train_direction == 'both':
        half_reacs = int(num_reacs/2)
        for i in range(half_reacs):
            Eact[i] = ea[i]
            dH[i] = dh[i]
            rmol.append(Chem.MolFromSmiles(rs[i]))
            pmol.append(Chem.MolFromSmiles(ps[i]))
        for i in range(half_reacs):
            Eact[i+half_reacs] = ea[i] - dh[i]
            dH[i+half_reacs] = -dh[i]
            rmol.append(Chem.MolFromSmiles(ps[i]))
            pmol.append(Chem.MolFromSmiles(rs[i]))

    return Eact, dH, rmol, pmol

def split_data(split_method: str, X: ArrayLike, y: ArrayLike, split_ratio:Tuple[float, float], seed: int,
               n_splits: int = 5, index_path=None, verbose=False):
    '''Split data into train/test sets.

    Implements methods to split data through Scikit-learn's
    RepeatedKFold cross-validation, train_test_split,
    ShuffleSplit or manualSplit methods.

    Arguments:
        split_methods: One of ['cv','train_test_split','ShuffleSplit','manualSplit'].
        X: array of x data.
        y: Array of y data.
        split_ratio: Ratio to split data into testing/training by.
        seed: Random number generator seed.
        n_splits: (Optional; Default = 5) Number of shuffle splits or CV folds.
        index_path: (Optional) Path to save split indices to.
        verbose: (Optional; Default: False) Whether or not to pint extra information.
    '''
    if split_method == 'cv':
        kf = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=seed)
        for train_idx, test_idx in kf.split(X):
            X_train = X[train_idx]
            y_train = y[train_idx]
            X_test = X[test_idx]
            y_test = y[test_idx]
    elif split_method == 'train_test_split':
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=split_ratio[1], random_state=seed,
                                                            shuffle=True)
    elif split_method == 'ShuffleSplit':
        rs = ShuffleSplit(n_splits=n_splits, test_size=split_ratio[1], train_size=split_ratio[0],
                          random_state=seed)
        for train_idx, test_idx in rs.split(X):
            X_train = X[train_idx]
            y_train = y[train_idx]
            X_test = X[test_idx]
            y_test = y[test_idx]
    elif split_method == 'manualShuffle':
        raise NotImplementedError()

    if index_path is not None:
        if split_method == 'train_test_split':
            if verbose: print('Test/train indices were requested but these are not available for the chosen splitting method.')
        else:
            if verbose: print(f'Saving train/test split indices to {index_path}.')
            np.savez(index_path, train_idx=train_idx, test_idx=test_idx)

    return X_train, X_test, y_train, y_test

def normalise(x: ArrayLike, x_mean: float, x_range: float, norm_type: str):
    '''Normalise data.

    Data can be mean-normalised (Zscore), shifted, or
    returned without any normalisation.

    Arguments:
        x: Array of data to be normalised.
        x_mean: Mean of x array.
        x_range: Standard deviation of x array.
        norm_type: One of ['Zscore', 'shift', 'none'].
    '''
    if norm_type == 'Zscore':
        return (x - x_mean) / x_range
    elif norm_type == 'none':
        return x
    elif norm_type == 'shift':
        return x - x_mean

def un_normalise(x_norm: ArrayLike, x_mean: float, x_range: float, norm_type: str):
    '''Un-normalise data.

    Input data can be mean-normalised (Zscore), shifted, or
    without any normalisation. Output will reverse whichever
    normalisation process occurred.

    Arguments:
        x_norm: Array of data to be un-normalised.
        x_mean: Mean of original un-normalised x array.
        x_range: Standard deviation of original un-normalised x array.
        norm_type: One of ['Zscore', 'shift', 'none'].
    '''
    if norm_type == 'Zscore':
        return x_norm * x_range + x_mean
    elif norm_type == 'none':
        return x_norm
    elif norm_type == 'shift':
        return x_norm + x_mean


from keras.preprocessing.text import *
from keras.models import *
from keras.layers import *
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import EarlyStopping, ModelCheckpoint
from scipy.stats import pearsonr
from tensorflow.contrib.metrics import streaming_pearson_correlation
import numpy as np
import tensorflow as tf
import json
import pickle

GLOBAL_MAP = {}
GLOBAL_FILE_W2V_PATH = ""
GLOBAL_VEC_COLUMNS_SIZE = 0

def pearson(y_true, y_pred):
    result = streaming_pearson_correlation(y_true, y_pred)
    return result[0]

def pearson3(y_true, y_pred):
    fsp = y_pred - K.mean(y_pred)
    fst = y_true - K.mean(y_true)

    dev_p = K.std(y_pred)
    dev_t = K.std(y_true)

    return (fsp * fst) / (dev_p * dev_t)

def pearson2(y_true, y_pred):
    fsp = y_pred - K.mean(y_pred)
    fst = y_true - K.mean(y_true)

    dev_p = K.std(y_pred)
    dev_t = K.std(y_true)

    return K.mean(fsp * fst) / (dev_p * dev_t)

def clean_number(x):
    return x[3].replace("\n", "")

def load_word_to_vec(path, quant_columns):
    file = open(path, mode="r")

    global GLOBAL_MAP

    file.readline()

    for line in file:
        split_data = line.split()
        index = 1
        word = split_data[0]
        quant_columns_vec = len(split_data) - 1

        if quant_columns_vec > quant_columns:
            local_dif = quant_columns_vec - quant_columns
            index += local_dif
            for w in split_data[1:index]:
                word += " " + w

        GLOBAL_MAP[word] = np.asarray(split_data[index:], dtype='float32')

    file.close()

    return

def is_pair_file(sentences_results):
    line = next(iter(sentences_results or []), None)
    return len(line) == 6

def open_file_sentence_pair_similarity(file_path):
    file = open(file_path, mode="r")
    lines = file.readlines()
    sentences_result = [i.split("\t")[1:] for i in lines]
    index = 4 if is_pair_file(sentences_result) else 2

    for i in sentences_result:
        i[index] = float(i[index].replace("\n", ""))
        i[index] = i[index] / 5

    file.close()

    return sentences_result

def load_global_data():
    if len(GLOBAL_MAP) < 1:
        global GLOBAL_FILE_W2V_PATH
        global GLOBAL_VEC_COLUMNS_SIZE
        load_word_to_vec(GLOBAL_FILE_W2V_PATH, GLOBAL_VEC_COLUMNS_SIZE)
    return

def sentence_to_vec_default(line, index):
    sentence = line[index]

    load_global_data()

    list_words = text_to_word_sequence(sentence, lower=True)
    vec_result = []

    for word in list_words:
        vect_values = GLOBAL_MAP.get(word)
        if vect_values is not None:
            for i in range(len(vect_values)):
                if i > len(vec_result)-1:
                    vec_result.append(0)
                vec_result[i] += vect_values[i]

    return [v/len(list_words) for v in vec_result]

import numpy as np

def cos_sim(a, b):
    dot_product = np.dot(a, b)
    norma = np.linalg.norm(a)
    normb = np.linalg.norm(b)
    mult = norma*normb
    cos = dot_product/mult
    return cos

def create_pair_vector(pairs, sentence_to_vec_function, merge_vector_function):
    vector_result = []

    load_global_data()

    for pair in pairs:
        vec1 = GLOBAL_MAP.get(pair['c1'].lower())
        vec2 = GLOBAL_MAP.get(pair['c2'].lower())
        if vec1 is None:
            vec1 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        if vec2 is None:
            vec2 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        vector_result.append(merge_vector_function(vec1, vec2))

    return vector_result

def create_pair_sep_vector(pairs, sentence_to_vec_function, merge_vector_function):
    vector_result = np.zeros((GLOBAL_VEC_COLUMNS_SIZE,2,GLOBAL_VEC_COLUMNS_SIZE))

    load_global_data()

    for i,pair in enumerate(pairs):
        vec1 = GLOBAL_MAP.get(pair['c1'].lower())
        vec2 = GLOBAL_MAP.get(pair['c2'].lower())
        if vec1 is None:
            vec1 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        if vec2 is None:
            vec2 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        vector_result[i][0] = vec1
        vector_result[i][1] = vec2

    return vector_result

def create_pair_cosine(pairs):
    vector_result_cos = np.zeros((300))
    load_global_data()

    for i,pair in enumerate(pairs):
        vec1 = GLOBAL_MAP.get(pair['c1'].lower())
        vec2 = GLOBAL_MAP.get(pair['c2'].lower())

        if vec1 is None:
            vec1 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        if vec2 is None:
            vec2 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        vector_result = abs(cos_sim(vec1,vec2))
        print(vector_result)
        if not(vector_result is np.nan):
            #vector_result[i] = pair['r']
            if float(vector_result) >= 0.5:
                vector_result_cos[i] = 1.0
            else:
                vector_result_cos[i] = 0.0
        #vec2 = GLOBAL_MAP.get(pair['c2'].lower())
        #if vec is None:
        #    vec1 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        #vector_result.append(vec)

    return vector_result_cos

def create_pair_dice(pairs):
    vector_result = np.zeros((300))

    load_global_data()

    for i,pair in enumerate(pairs):
        if not(pair['r'] is np.nan):
            #vector_result[i] = pair['r']
            if float(pair['r']) >= 0.66:
                vector_result[i] = "1.0"
            else:
                vector_result[i] = "0.0"
        #vec2 = GLOBAL_MAP.get(pair['c2'].lower())
        #if vec is None:
        #    vec1 = np.zeros(GLOBAL_VEC_COLUMNS_SIZE)
        #vector_result.append(vec)

    return vector_result

def vectorize_sentence_pair(dados_param, sentence_to_vec_function, merge_vector_function):
    merge_result = []
    results = []

    for dado in dados_param:
        pairs = json.loads(dado[5])
        merge_result.append(create_pair_vector(pairs, sentence_to_vec_function, merge_vector_function))
        results.append(dado[4])

    return merge_result, results

def vectorize_sentence_pair_dice(dados_param, sentence_to_vec_function, merge_vector_function):
    result_dice = []
    merge_result = []
    results = []

    for dado in dados_param:
        pairs = json.loads(dado[5])
        #merge_result.append(create_pair_vector(pairs, sentence_to_vec_function, merge_vector_function))
        merge_result.append(create_pair_vector(pairs, sentence_to_vec_function, merge_vector_function))
        result_dice.append(create_pair_dice(pairs))
        results.append(dado[4])

    return merge_result,result_dice, results

def vectorize_sentence_pair_cos(dados_param, sentence_to_vec_function, merge_vector_function):
    result_cos = []
    merge_result = []
    results = []

    for dado in dados_param:
        pairs = json.loads(dado[5])
        #merge_result.append(create_pair_vector(pairs, sentence_to_vec_function, merge_vector_function))
        merge_result.append(create_pair_sep_vector(pairs, sentence_to_vec_function, merge_vector_function))
        result_cos.append(create_pair_cosine(pairs))
        results.append(dado[4])

    return merge_result,result_cos, results

def merge_vector(s1, s2):
    vec_dif = [s1[i]-s2[i] for i in range(len(s1))]
    vec_multi = np.multiply(s1, s2)
    return [vec_dif[i]-vec_multi[i] for i in range(len(vec_dif))]

def merge_vector_combine(s1, s2):
    return np.append(s1, s2)

def vectorize_sentence(dados_param, sentence_to_vec_function, merge_vector_function):
    merge_result = []
    results = []

    for dado in dados_param:
        s1 = sentence_to_vec_function(dado, 0)
        s2 = sentence_to_vec_function(dado, 1)
        merge = merge_vector_function(s1, s2)
        merge_result.append(merge)
        if len(dado) > 2:
            results.append(dado[2])

    return merge_result, results

def create_default_model(num_column_vector):
    model = Sequential()
    model.add(Dense(1, input_dim=num_column_vector, kernel_initializer='normal', activation='relu'))
    model.add(Dense(1, activation='softmax'))

    return model

def crete_maia_layer(num_column_vector, train_data,dice_layer):
    model1 = Sequential()
    model1.add(LSTM(output_dim=num_column_vector,
                   return_sequences=True,
                   input_shape=(train_data.shape[1], train_data.shape[2])))
    model1.add(LSTM(output_dim=num_column_vector,
                   return_sequences=False,
                   input_shape=(train_data.shape[1], train_data.shape[2])))
    model1.add(Dense(output_dim=num_column_vector,input_dim=train_data.shape[2],activation='relu'))
    #model1.add(LSTM(output_dim=num_column_vector,
    #               return_sequences=False,
    #               input_shape=(train_data.shape[1], train_data.shape[2])))
    #model1.add(TimeDistributed(LSTM(100, batch_input_shape=(None, train_data.shape[1], train_data.shape[2],train_data.shape[3]), return_sequences=False), batch_input_shape=(None, train_data.shape[1], train_data.shape[2], train_data.shape[3])))

            #model.add(LSTM(100, batch_input_shape=(None, clauses.shape[1],clauses.shape[2],clauses.shape[3])))

    #model1.add(LSTM(100, batch_input_shape=(None, train_data.shape[1],train_data.shape[2])))

    model2 = Sequential()
    model2.add(Dense(num_column_vector, input_dim=dice_layer.shape[1], init='normal', activation='relu'))
    #model2.add(Dense(num_column_vector, activation='relu'))

    final_model = Merge([model1,model2], mode='concat')
    model = Sequential()
    model.add(final_model)
    model.add(Dense(128, activation='sigmoid'))
    model.add(Dropout(0.25))
    model.add(Dense(1, init='normal',activation='sigmoid'))
    print(model.summary())

    return model

def traini_data(x_train,dice_train, y_train, x_val,dice_val, y_val,x_test,dice_test, y_test, num_column_vector, mode_creator, function_evaluation, is_pair):
    #train_data_size = round(len(x_train)*0.8)
    if is_pair is False:

        x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1]))
        x_val = np.reshape(x_val, (x_val.shape[0], 1, x_val.shape[1]))

    if mode_creator is not None:
        model = mode_creator(num_column_vector,x_train,dice_train)
    else:
        model = create_default_model(num_column_vector)

    model.compile(loss='mean_squared_error',
                  optimizer='adam'
                  , metrics=[
            'mse'
            #            pearson
        ])

    early_stopping = EarlyStopping(monitor='val_loss', patience=40)
    checkpointer = ModelCheckpoint(filepath='graph_ee_kfold.hdf5',
                    monitor='val_loss',
                    verbose=1,
                    save_best_only=True,
                    mode='min')

    model.fit([x_train,dice_train], y_train
              , validation_data=([x_val,dice_val], y_val)
              , nb_epoch=70, batch_size=400
              , callbacks=[early_stopping,checkpointer]
               )

    model.load_weights('graph_ee_kfold.hdf5')
    X_test = [x_test,dice_test]
    if function_evaluation is None:

        scores = model.evaluate([x_test,dice_test], y_test, verbose=1)
    else:
        scores = function_evaluation(model, dice_test, y_test)
    print("Person: " + str(scores[0]))

    return model

#def traini_data(x_train, y_train, x_val, y_val, num_column_vector, mode_creator, function_evaluation, is_pair):
#    if is_pair is False:
#        x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1]))
#        x_val = np.reshape(x_val, (x_val.shape[0], 1, x_val.shape[1]))

#    if mode_creator is not None:
#        model = mode_creator(num_column_vector, x_train)
#    else:
#        model = create_default_model(num_column_vector)

#     model.compile(loss='mean_squared_error',
#                  optimizer='adam'
#                  , metrics=[
#            'mse'
            #            pearson
#        ])
#    early_stopping = EarlyStopping(monitor='loss', patience=50)

#                checkpointer = ModelCheckpoint(filepath='graph_ee_kfold.hdf5',
#                    monitor='loss',
#                    verbose=1,
#                    save_best_only=True,
#                    mode='min')

#    model.fit(x_train, y_train
#              , validation_data=(x_val, y_val)
#              , epochs=100, batch_size=128,)

#    if function_evaluation is None:
#        scores = model.evaluate(x_val, y_val, verbose=1)
#    else:
#        scores = function_evaluation(model, x_val, y_val)
#    print("Person: " + str(scores[0]))

#    return model

def pearson_evaluation(model, x_val, y_val):
    values2 = []
    values = model.predict(x_val)
    values = values.tolist()

    [values2.append(float("{0:.2f}".format(x[0]))) for x in values]
    print(values2)
    print(y_val)
    return pearsonr(values2, y_val.tolist())

def execute_experiment(path_treino, path_validation,path_testing, path_word_embemding, num_column_vector, model_creation,
                       function_evaluation, slice_teste, vectoriz_function, sentence_to_vec_function, merge_vector_function, is_pair):
    training_data = open_file_sentence_pair_similarity(path_treino)
    validate_data = open_file_sentence_pair_similarity(path_validation)
    test_data = open_file_sentence_pair_similarity(path_validation)

    global GLOBAL_FILE_W2V_PATH
    global GLOBAL_VEC_COLUMNS_SIZE

    GLOBAL_FILE_W2V_PATH = path_word_embemding
    GLOBAL_VEC_COLUMNS_SIZE = num_column_vector

    vec_sentence_training, vector_dice_train, result_t = vectoriz_function(training_data, sentence_to_vec_function, merge_vector_function)

    vec_sentence_validate, vector_dice_validate, result_v = vectoriz_function(validate_data, sentence_to_vec_function, merge_vector_function)

    vec_sentence_test, vector_dice_test, result_te = vectoriz_function(test_data, sentence_to_vec_function, merge_vector_function)

    vec_sentence_training = pad_sequences(vec_sentence_training, maxlen=num_column_vector, dtype='float32')
    result_t = np.asarray(result_t)

    vec_sentence_validate = pad_sequences(vec_sentence_validate, maxlen=num_column_vector, dtype='float32')
    result_v = np.asarray(result_v)

    vec_sentence_test = pad_sequences(vec_sentence_test, maxlen=num_column_vector, dtype='float32')
    result_te = np.asarray(result_te)

    vector_dice_train = np.array(vector_dice_train)
    vector_dice_validate = np.array(vector_dice_validate)
    vector_dice_test = np.array(vector_dice_test)
    model = None
    print("training embedding vectors shape")
    print(vec_sentence_training.shape)
    print("training dice vectors shape")
    print(vector_dice_train.shape)
    print("training data output")
    print(result_t.shape)

    if slice_teste is not None and slice_teste is True:
        x_slice = np.split(vec_sentence_validate, 2)
        y_slice = np.split(result_v, 2)
        for i in range(len(x_slice)):
            vec_sentence_training = np.concatenate([vec_sentence_training, x_slice[i]])
            result_t = np.concatenate([result_t, y_slice[i]])
            index = 0 if i == 1 else 1
            vec_sentence_validate = np.concatenate([vec_sentence_validate, x_slice[index]])
            result_v = np.concatenate([result_v, y_slice[index]])
            traini_data(vec_sentence_training, result_t, vec_sentence_validate, result_v, num_column_vector,
                        model_creation, function_evaluation, is_pair)
    else:
    #    model = traini_data(vec_sentence_training, result_t, vec_sentence_validate, result_v, num_column_vector, model_creation,
    #                function_evaluation, is_pair)
         model = traini_data(vec_sentence_training,vector_dice_train, result_t, vec_sentence_validate,vector_dice_validate, result_v, vec_sentence_test,vector_dice_test, result_te, num_column_vector, model_creation,
                    function_evaluation, is_pair)
    return model

def main():
    tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
    path_corpus = "/home/sousa/janio_project/"
    model =  execute_experiment(path_corpus+"result_max(dice_nc)_ptbr_propor_train_2016.csv",
    #path_corpus+"assin-ptbr-test.csv",
    path_corpus+"result_max(dice_nc)_ptbr_propor_dev_2016.csv",
    path_corpus+"result_dice_ptbr_propor_test_2016.csv",
    path_corpus+"glove_s300.txt",
                       300,
                       crete_maia_layer,
                       pearson_evaluation,
                       False,
#                       vectorize_sentence,
                       #vectorize_sentence_pair_dice,
                       vectorize_sentence_pair_dice,
                       sentence_to_vec_default,
#                       merge_vector
                       merge_vector_combine,
                       True
                       )
    output = open(path_corpus+'modelo.pkl', 'wb')
    pickle.dump(model, output)

if __name__ == '__main__':
   main()


from __future__ import print_function
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
from collections import defaultdict
from collections import Counter
from numpy import linalg as LA
import base64
import json
import matplotlib.pyplot as plt
from scipy.sparse.linalg import svds
from sklearn.preprocessing import normalize
from nltk.tokenize import TreebankWordTokenizer

class NumpyEncoder(json.JSONEncoder):

    def default(self, obj):
        """If input object is an ndarray it will be converted into a dict
        holding dtype, shape and the data, base64 encoded.
        """
        if isinstance(obj, np.ndarray):
            if obj.flags['C_CONTIGUOUS']:
                obj_data = obj.data
            else:
                cont_obj = np.ascontiguousarray(obj)
                assert(cont_obj.flags['C_CONTIGUOUS'])
                obj_data = cont_obj.data
            data_b64 = base64.b64encode(obj_data)
            return dict(__ndarray__=data_b64,
                        dtype=str(obj.dtype),
                        shape=obj.shape)
        # Let the base class default method raise the TypeError
        return json.JSONEncoder(self, obj)

def build_review_lengths(reviews):
    """ Builds an length dictionary from the beer reviews.

    reviews: dictionary of reviews for beers
        Each beer in this list already contains the tokenized review.
    Returns
    =======
    review_lengths: dict
        For each term, the index contains a list of
        tuples (doc_id, count_of_term_in_doc):
        index[term] = [(d1, tf1), (d2, tf2), ...]"""

    review_lengths = defaultdict(int)

    for key in reviews.keys():
        review_lengths[key] = len(reviews[key])

    return review_lengths

def build_inverted_index(reviews):
    """ Builds an inverted index from the beer reviews.

    reviews: dictionary of reviews for beers
        Each beer in this list already contains the tokenized review.
    Returns
    =======
    index: dict
        For each term, the index contains a list of
        tuples (doc_id, count_of_term_in_doc):
        index[term] = [(d1, tf1), (d2, tf2), ...]
    Example
    =======

    >> test_idx = build_inverted_index([
    ...    {'toks': ['to', 'be', 'or', 'not', 'to', 'be']},
    ...    {'toks': ['do', 'be', 'do', 'be', 'do']}])

    >> test_idx['be']
    [(0, 2), (1, 2)]

    """
    inverted_index = defaultdict(list)
    counter = 0
    for key in reviews.keys():
        print("%d, %d" % (counter, len(reviews.keys())))
        beer_id = beer_name_to_index[key]
        counts = Counter(reviews[key])
        for term in counts:
            if term in vocab_to_index.keys():
                inverted_index[term].append((beer_id, counts[term]))
        counter+=1
    return inverted_index

file = open("beer_all.json")
transcripts = json.load(file)
new_dict = defaultdict(list)
all_keys = transcripts.keys()
keys_length = len(all_keys)
counter = 0
print("ascii conversion")
print("================")
for key in all_keys[0:90]:
    print("%d, %d" % (counter, keys_length))
    new_key = key.encode('ascii', 'ignore')
    for index_dict in transcripts[key]:
        inner_dict = {}
        for k in index_dict.keys():
            inner_dict[k.encode('ascii', 'ignore')] = index_dict[k].encode('ascii', 'ignore')
        new_dict[new_key].append(inner_dict)
    counter+=1

new_transcripts = new_dict
json.dump(new_transcripts, open('new_transcripts_2.json', 'w'))
data = defaultdict(list)
text_data = defaultdict(str)
print("constructing beer_all_json and text_data")
print("================")
counter = 0
for key in new_transcripts.keys():
    print("%d/%d" % (counter, len(new_transcripts.keys())))
    review = new_transcripts[key][0]
    inner_dict = {}
    inner_dict['ABV'] = review['ABV']
    inner_dict['appearance'] = review['appearance']
    inner_dict['aroma'] = review['aroma']
    inner_dict['brewerId'] = review['brewerId']
    inner_dict['beerId'] = review['beerId']
    inner_dict['overall'] = review['overall']
    inner_dict['palate'] = review['palate']
    inner_dict['style'] = review['style']
    inner_dict['taste'] = review['appearance']
    inner_dict['appearance'] = review['taste']
    inner_dict['time'] = review['time']
    data[key] = inner_dict
    for review_index in new_transcripts[key]:
        text_data[key] += review_index['text'] + ' '
    counter+=1

print("================")
json.dump(data, open('beer_data_all_2.json', 'w'))
print("================")
json.dump(text_data, open('new_transcripts_text_2.json', 'w'))
print("=========")
n_feats = 10000
doc_by_vocab = np.empty([len(text_data.keys()), n_feats])
tfidf_vec = TfidfVectorizer(input='context',stop_words='english', max_df=0.8,
                            min_df=0.0162074554, max_features=10000, norm='l2')
tfidf_vec_scripts = []
beer_name_to_index = {}
beer_index_to_name = []
counter = 0
print("contructing tfidf vectors, beer name and index")
print("=============")
for key,value in text_data.iteritems():
    print("%d/%d" % (counter, len(text_data.keys())))
    beer_name_to_index[key] = counter
    beer_index_to_name.append(key)
    tfidf_vec_scripts.append(value)
    counter+=1
json.dump(beer_name_to_index, open('beer_name_to_index_2.json', 'w'))
json.dump(beer_index_to_name, open('beer_index_to_name_2.json', 'w'))
print("computed beer_name and index")
doc_voc_matrix = tfidf_vec.fit_transform(tfidf_vec_scripts).toarray()
doc_by_vocab = doc_voc_matrix
print("transformed tfidf matrix")
json.dump(doc_voc_matrix, open('doc_voc_matrix_2.json', 'w'), cls=NumpyEncoder)
print("NOW HERE HERE HERE")
print("========================")
print("grabbing index_to_vocab and vocab_to_index")
index_to_vocab = {i:v for i, v in enumerate(tfidf_vec.get_feature_names())}
vocab_to_index = {v:i for i, v in enumerate(tfidf_vec.get_feature_names())}
json.dump(index_to_vocab, open('index_to_vocab_2.json', 'w'))
json.dump(vocab_to_index, open('vocab_to_index_2.json', 'w'))
print("computed vocab name and index")
print("========")
print("building review tokens")
tokenizer = TreebankWordTokenizer()
reviews_tokens = defaultdict(list)
for key in text_data.keys():
    toks = tokenizer.tokenize(text_data[key].replace('/', ' ')) #to separate cases like spicy/citrus/sweet
    toks_stripped = []
    for term in toks:
        toks_stripped.append(term.strip().strip('.').lower()) #to deal with cases like 'Light' and 'light.'
    reviews_tokens[key] = toks_stripped
print("building review_lengths")
review_lengths_data = build_review_lengths(reviews_tokens)
json.dump(review_lengths_data, open('review_lengths_2.json', 'w'))
print("computed review lengths")
print("building inverted index")
inv_index = build_inverted_index(reviews_tokens)
json.dump(inv_index, open('inv_index_2.json', 'w'))
print("computed inv index")
print("==================")
print("computing SVD")
beers_compressed, _, features_compressed = svds(doc_voc_matrix, k=80)
json.dump(beers_compressed, open('beers_compressed_2.json', 'w'), cls=NumpyEncoder)
json.dump(features_compressed, open('features_compressed_2.json', 'w'), cls=NumpyEncoder)
print("finished")


import unittest
import pycodestyle
from os.path import dirname, join

SCRIPT_PATH = dirname(dirname(__file__))

class TestCodeFormat(unittest.TestCase):

    def test_conformance(self):
        """Test that we conform to PEP-8."""
        path_to_config = join(SCRIPT_PATH, 'setup.cfg')
        style = pycodestyle.StyleGuide(quiet=True, config_file=path_to_config)
        result = style.check_files(['./'])
        self.assertEqual(result.total_errors, 0,
                         "Found code style errors (and warnings). {0}".format(result.messages))

if __name__ == '__main__':
    unittest.main()


from sentence_transformers import SentenceTransformer, util
import numpy as np

model_path = '../Checkpoint/KoSBERT/kosbert-klue-bert-base'

embedder = SentenceTransformer(model_path)

# Corpus with example sentences
corpus = ['한 남자가 음식을 먹는다.',
          '한 남자가 빵 한 조각을 먹는다.',
          '그 여자가 아이를 돌본다.',
          '한 남자가 말을 탄다.',
          '한 여자가 바이올린을 연주한다.',
          '두 남자가 수레를 숲 솦으로 밀었다.',
          '한 남자가 담으로 싸인 땅에서 백마를 타고 있다.',
          '원숭이 한 마리가 드럼을 연주한다.',
          '치타 한 마리가 먹이 뒤에서 달리고 있다.']

corpus_embeddings = embedder.encode(corpus, convert_to_tensor=True)

# Query sentences:
queries = ['한 남자가 파스타를 먹는다.',
           '고릴라 의상을 입은 누군가가 드럼을 연주하고 있다.',
           '치타가 들판을 가로 질러 먹이를 쫓는다.']

# Find the closest 5 sentences of the corpus for each query sentence based on cosine similarity
top_k = 5
for query in queries:
    query_embedding = embedder.encode(query, convert_to_tensor=True)
    cos_scores = util.pytorch_cos_sim(query_embedding, corpus_embeddings)[0]
    cos_scores = cos_scores.cpu()

    #We use np.argpartition, to only partially sort the top_k results
    top_results = np.argpartition(-cos_scores, range(top_k))[0:top_k]

    print("\n\n======================\n\n")
    print("Query:", query)
    print("\nTop 5 most similar sentences in corpus:")

    for idx in top_results[0:top_k]:
        print(corpus[idx].strip(), "(Score: %.4f)" % (cos_scores[idx]))