"""Log functions"""
import time
from datetime import datetime, timedelta
from pytz import timezone
import logging

def build_logger(log_file_path: str) -> logging.RootLogger:
    """
    Creates logger file.

    Parameters
    ----------
    log_file_path: str
        Path to save log file

    Returns
    ----------
    logger: logging.RootLogger
        Logger with file path.
    """
    # Reference
    # https://qiita.com/r1wtn/items/d615f19e338cbfbfd4d6
    # https://www.sejuku.net/blog/23149

    logger = logging.getLogger()
    logger.setLevel(logging.INFO)
    fh = logging.FileHandler(log_file_path)
    formatter = logging.Formatter(
        fmt="%(asctime)s: %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
    )

    def japanTime(*args):
        return datetime.now(timezone("Asia/Tokyo")).timetuple()

    formatter.converter = japanTime
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    return logger

def get_log_file_name(name: str = "") -> str:
    """
    Creates log file name based on executed time in Japan time.

    Parameters
    ----------
    name: str, optional
        Name of log file

    Returns
    ----------
    filename: str
        Generated log file name.
    """
    now = datetime.now(timezone("Asia/Tokyo"))
    date_time_str = now.strftime("%Y-%m-%d_%H:%M")
    filename = name + date_time_str + ".log"
    return filename

def log_execution_time(logger: logging.RootLogger) -> None:
    """
    Decorator to log function execution time to logger.

    Parameters
    ----------
    logger: logging.RootLogger
        The logger to write execution time for the function.

    Returns
    ----------
    None
    """

    def decorator(function):
        def wrapper(*args, **kwargs):
            start = time.time()
            function(*args, **kwargs)
            elapsed = time.time() - start
            elapsed_time = str(timedelta(seconds=elapsed))
            logger.info(f"Execution time: {elapsed_time}")

        return wrapper

    return decorator


from wsgiref.simple_server import make_server

# python fwsgi_3.py
# Заходим в браузер: http://127.0.0.1:8000/
#                    http://127.0.0.1:8000/abc/

def application(environ, start_response):
    """
    00:47
    simple PC: маршрутизатор
    :param environ: словарь данных от сервера
    :param start_response: функция для ответа серверу
    """
    # print(type(environ))  # <class 'dict'>
    # print(environ)
    path = environ['PATH_INFO']  # 'PATH_INFO': '/'
    if path == '/':
        start_response('200 OK', [('Content-Type', 'text/html')])
        return [b'Index']
    elif path == '/abc/':
        start_response('200 OK', [('Content-Type', 'text/html')])
        return [b'ABC']
    else:
        start_response('404 NOT FOUND', [('Content-Type', 'text/html')])
        return [b'404 Not Found']

with make_server('', 8000, application) as httpd:
    """Запуск непосредственно из PyCharm"""
    print("Serving on port 8000...")
    httpd.serve_forever()


import requests
import bs4
import os
import sys
import urllib.request
import time
import shutil

# Create the file of today's day in the current path
form="%Y-%m-%d"
a = time.strftime(form,time.localtime())
if os.path.exists(a):
    shutil.rmtree(a)
os.mkdir(a)
os.chdir(a)

# The online catalogue url
originurl='http://www.pxmart.com.tw/px/edm.px'

response = requests.get(originurl)
soup = bs4.BeautifulSoup(response.text,'html.parser')

w1 = soup.find('button')
w2 = soup.button['onclick']

# w3 is the monthly url info for the catalogue
w3 = w2.split("'")[1]

pxurl = 'http://www.pxmart.com.tw/px/www.pxmart.com.tw/px/edm/edm/'

# final online catalogue url
finurl = pxurl+w3

response = requests.get(finurl)
soup2 = bs4.BeautifulSoup(response.text,'html.parser')

# find the image of the online catalogue
p3 = soup2.select("div[data-double='false']")
final = ''
n = len(p3)
count = 1
pxmpic = 'http://www.pxmart.com.tw/'
for i in range(0,n-1):
    final = p3[i]
    final_string = str(final)
    if i == 0:
        finalword = final_string.split('"')[5]
    else:
        finalword = final_string.split('"')[3]
    finalpath = pxmpic+finalword
    f = open(str(count)+'.png','wb')

    with urllib.request.urlopen(finalpath) as response2:
        html = response2.read()
        f.write(html)
    count +=1


from collections import deque

t = int(input())

for _ in range(t):
    n = int(input())
    arr = list(map(int, input().split()))
    arr.sort()
    q = deque(arr)
    k = arr[-1] - arr[0]
    maxendinghere = float('-inf')
    minendinghere = float('inf')
    res = []
    for i in range(n):
        for pos, j in [(q[0], 0), (q[-1], 1)]:
            newmaxendinghere = max(pos, pos + maxendinghere)
            newminendinghere = min(pos, pos + minendinghere)
            if newmaxendinghere < k and newminendinghere > -k:
                res.append(pos)
                if j == 0:
                    q.popleft()
                else:
                    q.pop()
                maxendinghere = newmaxendinghere
                minendinghere = newminendinghere
                break
    if len(res) == n:
        print("Yes")
        print(*res)
    else:
        print("No")


from data import verbal_const

from utils.logic.functions import l_scale

from devices.factories.luxmeters.base_luxmeter import BaseLuxmeter

#region File Attributes

__author__ = "Orlin Dimitrov"
"""Author of the file."""

__copyright__ = "Copyright 2020, POLYGON Team Ltd."
"""Copyrighter
@see http://polygonteam.com/"""

__credits__ = ["Angel Boyarov"]
"""Credits"""

__license__ = "GPLv3"
"""License
@see http://www.gnu.org/licenses/"""

__version__ = "1.0.0"
"""Version of the file."""

__maintainer__ = "Orlin Dimitrov"
"""Name of the maintainer."""

__email__ = "or.dimitrov@polygonteam.com"
"""E-mail of the author.
@see or.dimitrov@polygonteam.com"""

__status__ = "Debug"
"""File status."""

#endregion

class LightSensor(BaseLuxmeter):
    """Light sensor."""

#region Attributes

    __analog_input = verbal_const.OFF
    """Sensor analog input.
    """

    __state = {}
    """Value from the input.
    """

    __min_measured = 0.0
    """Minimum value of the sensor.
    """

    __max_measured = 15000.0
    """Maximum of te sensor.
    """

#endregion

    def __init__(self, **config):

        super().__init__(config)

        self._vendor = "POLYGONTeam"

        self._model = "MI"

        self.__analog_input = self._config["analog_input"]

#region Public Methods

    def update(self):
        """Update sensor data.
        """

        self.__state = self._controller.analog_read(self.__analog_input)

    def get_value(self):
        """Get value.
        """

        raw = 0.0

        if bool(self.__state):
            raw = l_scale(self.__state["value"],
                          [self.__state["min"], self.__state["max"]],
                          [self.__min_measured, self.__max_measured])

        return raw

#endregion


import argparse
import os
from string import Template

import torch
from huggingface_hub import snapshot_download
from torch import nn
from transformers import AutoModelForSequenceClassification, AutoTokenizer

parser = argparse.ArgumentParser()

parser.add_argument("--device", type=int, default=0)
args = parser.parse_args()

model_name = "RM_model_onnx"
device = torch.device('cuda:7')

RM_model_path = "/search/ai/kaitongyang/RLHF_DEBUG/RM/summarization_reward_model/checkpoint-58/"
RM_model_path = '/search/ai/pretrain_models/roberta-base-finetuned-jd-binary-chinese'
RM_tokenizer = AutoTokenizer.from_pretrained(RM_model_path)
RM_model = AutoModelForSequenceClassification.from_pretrained(RM_model_path, torchscript=True)
RM_model.to(device)
query_text = '什么人不能喝三七粉'
response_text = '服用三七粉期间,孕妇和儿童不宜使用。 三七粉是处方药,不是药品。 过量服用会引起中毒。'
input = RM_tokenizer(query_text, response_text, truncation=True, max_length=512, padding="max_length", return_tensors="pt").to(device)
print(input)
print(RM_model(input['input_ids'], input['attention_mask'], input['token_type_ids']))
RM_model = RM_model.eval()  # 转换为eval模式
inputs = (input['input_ids'], input['attention_mask'], input['token_type_ids'])  # 模型测试输入数据
os.makedirs(f"model_store/{model_name}/1", exist_ok=True)
torch.onnx.export(
	RM_model,
	inputs,
	f"model_store/{model_name}/1/model.onnx",  # 输出模型文件名
	input_names=['input_ids', 'attention_mask', 'token_type_ids'],  # 输入节点名，每一个名称对应一个输入名
    output_names=['output'],  # 输出节点名，每一个名称对应一个输出名
	opset_version=11,
	dynamic_axes={'input_ids': {0: 'B', 1: 'C'}, 'attention_mask': {0: 'B', 1: 'C'}, 'token_type_ids': {0: 'B', 1: 'C'}}  # 声明动态维度，默认输入维度固定，可以声明为可变维度（用字符串占位符表示）
)

# traced_script_module.save(f"model_store/{model_name}/1/traced-model.pt")


# As for one-dimensional signals, images also can be filtered with various low-pass filters (LPF), high-pass filters (HPF), etc.
# A LPF helps in removing noise, or blurring the image.
# A HPF filters helps in finding edges in an image.

# OpenCV provides a function, cv2.filter2D(), to convolve a kernel with an image.
# As an example, we will try an averaging filter on an image. A 5x5 averaging filter kernel can be defined.
# Filtering with the above kernel results in the following being performed:
#   for each pixel, a 5x5 window is centered on this pixel,
#   all pixels falling within this window are summed up, and the result is then divided by 25.
#   This equates to computing the average of the pixel values inside that window.
#   This operation is performed for all the pixels in the image to produce the output filtered image.

import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.imread('Messi.jpg')

kernel = np.ones((5,5),np.float32)/25
dst = cv2.filter2D(img,-1,kernel)

plt.subplot(121),plt.imshow(img),plt.title('Original')
plt.xticks([]), plt.yticks([])
plt.subplot(122),plt.imshow(dst),plt.title('Averaging')
plt.xticks([]), plt.yticks([])
plt.show()


# Download the raw data of dropsondes from https://www.aoml.noaa.gov/ftp/hrd/data/dropsonde/
import libs.banner
import os
import requests
from bs4 import BeautifulSoup
import re
import libs.tools as tools
import sys
from datetime import datetime

url = "https://www.aoml.noaa.gov/ftp/hrd/data/dropsonde/"

if __name__ == "__main__":
    # Record log
    sys.stdout = tools.Logger(
        "logs/download_raw_" + datetime.now().strftime(r"%Y_%m_%d_%H_%M_%S") + ".log")

    # Get download urls
    response = requests.get(url)
    if response.status_code != 200:
        print(f"{response.status_code}: Failed to obtain {url}")
        exit(-1)
    soup = BeautifulSoup(response.text, "html.parser")
    response.close()
    ul_tag = soup.ul
    li_tags = ul_tag.find_all('li')
    li_tags.pop(0)
    li_urls = [li_tag.a["href"] for li_tag in li_tags]
    raw_urls = [url + li_url + "raw/" for li_url in li_urls]
    save_paths = ["./data/sondes/raw" + li_url for li_url in li_urls]

    for i in range(75, len(raw_urls)):
        print("========================>")
        print(f"Downloading folder {i+1}")
        response = requests.get(raw_urls[i])
        if response.status_code != 200:
            print(
                f"{response.status_code}: Failed to obtain urls from {raw_urls[i]}")
            print("<========================")
            continue
        soup = BeautifulSoup(response.text, "html.parser")
        response.close()
        ul_tag = soup.ul
        li_tags = ul_tag.find_all('li')
        li_tags.pop(0)
        li_urls = [li_tag.a["href"] for li_tag in li_tags]
        # Only get *avp.tar.gz or *.avp.tgz package
        pat1 = r'.*avp\.tar\.gz$'
        pat2 = r'.*AVP\.tar\.gz$'
        pat3 = r'.*avp\.tgz$'
        pat4 = r'.*AVP\.tgz$'
        li_urls = [li_url for li_url in li_urls if re.search(
            pat1, li_url) or re.search(pat2, li_url) or re.search(pat3, li_url) or re.search(pat4, li_url)]
        if len(li_urls) == 0:
            print(f"{response.status_code}: Got nothing from {raw_urls[i]}")
            print("<========================")
            continue
        package_urls = [raw_urls[i] + li_url for li_url in li_urls]
        if not os.path.isdir(save_paths[i]):
            os.mkdir(save_paths[i])
        for url in package_urls:
            tools.download_package(url, save_paths[i])
        print("<========================")


##punto fijo
import numpy as np
def f1(x):
    return - (np.sqrt(3.0*x)**(2.0/5.0))+x**3.0*np.cos(3.0*x)+4.0*x**2.0 - 7.0

#g(x) candidatas para f1
def gf11(x):
    return np.sqrt((1.0/4.0)*((np.sqrt(3*x))**(2.0/5.0)-x**3*np.cos(3*x)+7))
def gf12(x):
    return (((np.sqrt(3.0*x)**(2.0/5.0))-4.0*(x**2.0)-7.0)/np.cos(3.0*x))**(3.0)
def gf13(x):
    return - (np.sqrt(3*x)**(2/5))+x**3*np.cos(3*x)+4*x**2 - 7 + x

def f2(x):
    return -x**(0.25)+np.sin(3.5*x)+4*x**(0.5)+2*x - 5
#g(x) candidatas para f2
def gf21(x):
    return (x**0.25-np.sin(3.5*x)-4.0*x**0.5+5)/2
def gf22(x):
    return (-np.sin(3.5*x)-4*x**(0.5)+2*x + 5)**4.0

def puntofijo(f,tol,g):
    diverge = False
    x1=2.0
    tolx = tol
    tolf = tolx
    x2prev = x1
    iteraciones = 0
    while True:
        iteraciones +=1
        if iteraciones > 100:
            diverge = True
            print("la función diverge probando con {} iteraciones".format(iteraciones))
            break
        x2 = g(x1)
        if np.abs(x2-x2prev) <= tolx:
            break
        if np.abs(f(x2)) <= tolf:
            break
        x1=x2
        x2prev=x2

    if not diverge:
        print("Punto fijo: con {} iteraciones, {} de tolerancia, la raiz es: {}".format(iteraciones,tolx,x2))

print("f1\n")
print("g(x) candidata np.sqrt((1.0/4.0)*((np.sqrt(3*x))**(2.0/5.0)-x**3*np.cos(3*x)+7))")
puntofijo(f1,10**-10,gf11)
puntofijo(f1,10**-15,gf11)

print("g(x) candidata (((np.sqrt(3.0*x)**(2.0/5.0))-4.0*(x**2.0)-7.0)/np.cos(3.0*x))**(3.0)")
puntofijo(f1,10**-5,gf12)
print("g(x) candidata - (np.sqrt(3*x)**(2/5))+x**3*np.cos(3*x)+4*x**2 - 7 + x")
puntofijo(f1,10**-5,gf13)

print("\nf2")
print("g(x) candidata (x**0.25-np.sin(3.5*x)-4.0*x**0.5+5)/2")
puntofijo(f2,10**-5,gf21)
print("g(x) candidata (-np.sin(3.5*x)-4*x**(0.5)+2*x + 5)**4.0" )
puntofijo(f2,10**-5,gf22)

##syms newton
import numpy as np
import sympy as sym

def newton(f, tol, df):
    x1 = 2.0
    tolx = tol
    tolf = tolx
    x2prev = x1
    iteraciones = 0
    while True:
        iteraciones += 1
        x2 = x1 - (f.subs(xx,x1) / df.subs(xx,x1))
        if np.abs(x2 - x2prev) <= tolx:
            break
        if np.abs(f.subs(xx,x2)) <= tolf:
            break
        x1 = x2
        x2prev = x2
    print("Newton: con {} iteraciones, {} de tolerancia, la raiz es: {}".format(iteraciones, tolx, x2))

xx = sym.Symbol("xx")
f1 = -(sym.sqrt(3.0*xx)**(2.0/5.0))+xx**3.0*sym.cos(3.0*xx)+4.0*xx**2.0 - 7.0
df1 = f1.diff()

f2 = -xx**(0.25)+sym.sin(3.5*xx)+4*xx**(0.5)+2*xx - 5
df2 = f2.diff()

print('para F1')
newton(f1,10**-15,df1)

print('para F2')
newton(f2,10**-5,df2)
newton(f2,10**-15,df2)


import cv2
import numpy as np

def callback(x):
    pass

cv2.namedWindow('Image')

ref_point = [(0, 800)]
idx = 0

# Mouse Events
def shape_selection(event, x, y, flags, param):
    global ref_point, idx, drawing, point1, point2
    if event == cv2.EVENT_LBUTTONDOWN:
        ref_point = [(x, y)]
        print (ref_point)
        print (image[ref_point[0][1], ref_point[0][0]])

        hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

        dlow = 0
        dup = 179

        lower_color = np.array([dlow, 60, 120])
        upper_color = np.array([dup, 255, 255])

        mask = cv2.inRange(hsv, lower_color, upper_color)
        # cv2.imshow('Mask', mask)

        contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

        for contour in contours:
            if (cv2.pointPolygonTest(contour, ref_point[0], False)) == 1:
                print("point is inside contour")
                cv2.drawContours(image, contour, -1, (0, 255, 0), 3)

                # Centre Calculation
                M = cv2.moments(contour)
                cx = int(M['m10'] / M['m00'])
                cy = int(M['m01'] / M['m00'])
                centre = (cx, cy)
                print(centre)

                # Saving Image
                a, b, w, h = cv2.boundingRect(contour)
                print("abc", a,b, w, h)
                if w > 50 and h > 50:
                    idx += 1
                    new_img = image_1[b:b + h, a:a + w]
                    cv2.imwrite(str(idx) + '.png', new_img)

        cv2.imshow("Image", image)

cv2.setMouseCallback("Image", shape_selection)

# Image Reading
image = cv2.imread("Pics/leaves2.jpg")
#blurred_image = cv2.GaussianBlur(image, (5, 5), 1)
image_1= cv2.imread("Pics/leaves2.jpg")

hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

print(image.shape)

cv2.imshow("Image", image)

cv2.waitKey(0)
cv2.destroyAllWindows()


import time
from datetime import datetime as dt
temp_path='hosts'
hosts_path='\/etc\/hosts'
redirect='127.0.0.1'
websites=['www.reddit.com','www.facebook.com','www.pornhub.com','www.xhamster.com','www.myfreecams.com']
cy = dt.now().year
cm = dt.now().month
cd = dt.now().day

while True:
    if dt(cy,cm,cd,8) < dt.now() < dt(cy,cm,cd,12):
        print('test')
        with open(temp_path,'r+') as f:
            content = f.read()
            for website in websites:
                if website in content:
                    pass
                else:
                    f.write(redirect + ' ' + website + '\n')
    else:
        with open(temp_path, 'r+') as f:
            content = f.readlines()
            f.seek(0)
            for ln in content:
                if not any(website in ln for website in websites):
                    f.write(ln)
            f.truncate()

    time.sleep(5)


"""
URL configuration for investments project - portfolio app.
"""
from django.urls import path, include, re_path
from django.views.static import serve
from django.conf import settings
from portfolio import views

urlpatterns = [
    path('', views.customer_list),
    path('api/customers/', views.customer_list),
    path('api/mycustomers/', views.my_customer_list),
    path('api/customers/<int:pk>', views.getCustomer),
    path('api/investments/', views.investment_list),
    path('api/investments/<int:pk>', views.getInvestment),
    path('api/stocks/', views.stock_list),
    path('api/stocks/<int:pk>', views.getStock),
    re_path(r'^media/(?P<path>.*)$', serve, {'document_root':
settings.MEDIA_ROOT}), #serve media files when deployed
    re_path(r'^static/(?P<path>.*)$', serve, {'document_root':
settings.STATIC_ROOT}), #serve static files when deployed
]


import pygame
from pygame.locals import *

#debug = False
debug = True
numberOfFlowers = 4
flowersSeparation = 210
characterBaseX = 67
characterBaseY = 550
flowersBaseX = 167
flowersBaseY = 530

speedFactor = 1.0

imagedb = { }

def loadImages() :

    # Background
    bg      = pygame.image.load("assets/bg.png")
    imagedb["bg"] = bg

    # Indicators
    indicators = pygame.image.load("assets/indicators.png")
    imagedb["indicator"] = {}
    imagedb["indicator"]["life"] = []
    imagedb["indicator"]["combo"] = []
    for i in range(4) :
        imagedb["indicator"]["life"].append(getSprite(indicators, i, 0, 87, 87))
        imagedb["indicator"]["combo"].append(getSprite(indicators, i, 2, 87, 87))

    # Tuto stuff
    tutostuff = pygame.image.load("assets/tuto.png")
    imagedb["tuto"] = {}
    imagedb["tuto"]["leftkey"]  = []
    imagedb["tuto"]["rightkey"] = []
    imagedb["tuto"]["Dkey"]     = []
    imagedb["tuto"]["Fkey"]     = []
    imagedb["tuto"]["poof"]     = []
    for i in range(2) :
        imagedb["tuto"]["rightkey"].append(getSprite(tutostuff, i, 0, 175, 175))
        imagedb["tuto"]["leftkey"] .append(getSprite(tutostuff, i, 1, 175, 175))
        imagedb["tuto"]["Dkey"]    .append(getSprite(tutostuff, i, 2, 175, 175))
        imagedb["tuto"]["Fkey"]    .append(getSprite(tutostuff, i, 3, 175, 175))
        imagedb["tuto"]["poof"]    .append(getSprite(tutostuff, i, 4, 175, 175))

    # Title
    title = pygame.image.load("assets/title.png")
    imagedb["title"] = []
    imagedb["title"].append(getSprite(title, 0, 0, 610, 250))
    imagedb["title"].append(getSprite(title, 0, 1, 610, 250))

    # Game over stuff
    gameover = pygame.image.load("assets/gameover.png")
    imagedb["gameoverscorebg"] = getSprite(gameover, 0, 0, 400, 250)
    imagedb["again"]           = getSprite(gameover, 1, 0, 400, 250)

    # Stems
    stems =  pygame.image.load("assets/stems.png")
    imagedb["stems"] = [[],[]]
    for i in range(4) :
        imagedb["stems"][0].append(getSprite(stems, i, 0, 140, 54))
        imagedb["stems"][1].append(getSprite(stems, i, 1, 140, 54))

    # Flower sprites
    flowersHead = pygame.image.load("assets/flowersHead.png")
    imagedb["flowersHead"] = {}
    imagedb["flowersHead"]["neutral"] = []
    imagedb["flowersHead"]["good"] = []
    imagedb["flowersHead"]["evil"] = []
    for i in range(3) :
        imagedb["flowersHead"]["good"]   .append(getSprite(flowersHead, i, 0, 175, 175))
        imagedb["flowersHead"]["evil"]   .append(getSprite(flowersHead, i, 1, 175, 175))
        imagedb["flowersHead"]["neutral"].append(getSprite(flowersHead, i, 2, 175, 175))

    imagedb["flowersDead"] = {}
    imagedb["flowersDead"]["good"] = getSprite(flowersHead, 3, 0, 175, 175)
    imagedb["flowersDead"]["evil"] = getSprite(flowersHead, 3, 1, 175, 175)

    flowersPop = pygame.image.load("assets/flowersPop.png")
    imagedb["flowersPop"] = {}
    imagedb["flowersPop"]["good"] = []
    imagedb["flowersPop"]["evil"] = []
    for i in range(4) :
        imagedb["flowersPop"]["good"].append(getSprite(flowersPop, i, 0, 250, 250))
        imagedb["flowersPop"]["evil"].append(getSprite(flowersPop, i, 1, 250, 250))

    # Character sprites
    #char = pygame.image.load("assets/perso.png")
    #imagedb["char"] = {}
    #imagedb["char"]["standing"] = getSprite(char, 0, 0, 48, 64)
    #imagedb["char"]["left"]  = []
    #imagedb["char"]["right"] = []

    #for i in range(4) :
    #    imagedb["char"]["left"].append(getSprite(char, i, 1, 48, 64))
    #    imagedb["char"]["right"].append(getSprite(char, i, 2, 48, 64))

    # Cut animation
    persoActions = pygame.image.load("assets/persoActions.png")
    imagedb["actions"] = {}
    imagedb["actions"]["movingleft"]  = []
    imagedb["actions"]["movingright"] = []
    imagedb["actions"]["idle"]        = []
    imagedb["actions"]["cutting"]     = []
    imagedb["actions"]["watering"]    = []
    imagedb["actions"]["gethit"]      = []

    for i in range(4) :
        imagedb["actions"]["movingleft"] .append(getSprite(persoActions, i, 0, 225, 225))
        imagedb["actions"]["movingright"].append(getSprite(persoActions, i, 1, 225, 225))

    for i in range(2) :
        imagedb["actions"]["idle"]    .append(getSprite(persoActions, i, 2, 225, 225))
        imagedb["actions"]["cutting"] .append(getSprite(persoActions, i, 3, 225, 225))
        imagedb["actions"]["watering"].append(getSprite(persoActions, i, 4, 225, 225))
        imagedb["actions"]["gethit"]  .append(getSprite(persoActions, i, 5, 225, 225))

def getSprite(img, x, y, w=32, h=32) :

    return img.subsurface((x*w, y*h, w, h))


from .BaseNLP import NLPBase

import re
from nltk.stem import WordNetLemmatizer
import nltk
from nltk.corpus import stopwords
from sklearn.metrics import precision_recall_fscore_support
from sklearn.svm import  SVC
from nltk.classify import SklearnClassifier
from sklearn.metrics import classification_report, accuracy_score, precision_recall_fscore_support
from alive_progress import alive_bar
import random
import pickle
from joblib import dump, load

nltk.download('wordnet')
nltk.download('stopwords')
nltk.download('omw-1.4')

class LinearSVC(NLPBase):
    def pre_process(self,text):
        text = text.lower()
        lemma = WordNetLemmatizer()
        words_filter = [""]
        text = re.sub(r"(\w)([.,;:!?'\"”\)\[\]])", r"\1 \2", text)
        text = re.sub(r"([.,;:!?'\"“\(\)\[\]])(\w)", r"\1 \2", text)
        words = text.split(" ")
        final_tokenized_string = []
        for word in words:
            if word not in stopwords.words('english'):
                lemma_word = lemma.lemmatize(word)
                final_tokenized_string.append(lemma_word)

        # remove blank words from final_tokenized_string
        final_tokenized_string = list(filter(lambda val: val not in words_filter, final_tokenized_string))

        # print(final_tokenized_string)
        return final_tokenized_string

    def split_and_preprocess_data(self, data):
        """Split the data between train_data and test_data according to the percentage
        and performs the preprocessing."""
        train_data = []
        test_data = []

        num_samples = len(data)
        num_training_samples = int((self.percentage * num_samples))
        print("Generating training data")
        with alive_bar(len(data[:num_training_samples])) as bar:
            for (text, label) in data[:num_training_samples]:
                train_data.append((self.to_feature_vector(self.pre_process(text)),label))
                bar()

        print("Generating test data")
        with alive_bar(len(data[num_training_samples:])) as bar:
            for (text, label) in data[num_training_samples:]:
                test_data.append((self.to_feature_vector(self.pre_process(text)),label))
                bar()

        return train_data, test_data

    def to_feature_vector(self, tokens):
        # Should return a dictionary containing features as keys, and weights as values

        local_feature_dict = {}
        word_index = 0
        # Loop through every word in token
        for word in tokens:

            # Check to see if the word is not in the global_feature_dict
            if word in self.global_feature_dict:
                # get the index of the word
                word_index = self.global_feature_dict[word]
            else:
                # Add new word with corresponding index to global_feature_dict
                word_index = len(self.global_feature_dict) + 1
                self.global_feature_dict[word] = word_index

            # Check to see if word is in the local_feature_dict
            # Only applies to this set of tokens
            if word_index in local_feature_dict:
                # Increment value in local_feature_dict
                local_feature_dict[word_index] += 1/len(tokens)
            else:
                # Set the key of word to 1/length of tokens in local_feature_dict
                local_feature_dict[word_index] = 1/len(tokens)

        # Return local_feature_dict as global_feature_dict is set globally (as stated in the name)
        return local_feature_dict

    def cross_validate(self, dataset, folds, classifications):
        results = []
        fold_size = int(len(dataset)/folds) + 1
        for i in range(0,len(dataset),int(fold_size)):
            # insert code here that trains and tests on the 10 folds of data in the dataset
            if self.debug:
                print("Fold start on items %d - %d" % (i, i+fold_size))

            # FILL IN THE METHOD HERE
            statements = [x[0] for x in dataset[i:i+fold_size]]
            correct_outputs = [x[1] for x in dataset[i:i+fold_size]]
            classifier = self.train_classifier(dataset[i:i+fold_size])
            predicted_outputs = self.predict_labels(statements,classifier)
            if len(set(predicted_outputs)) != len(classifications):
                # raise Exception("Classifier did not learn all labels")
                print("Case of classifier not learning all classes in fold")
                continue

            if self.debug:
                print(classification_report(correct_outputs,predicted_outputs,target_names=classifications))

            (precision, recall, f1, _) = precision_recall_fscore_support(correct_outputs,predicted_outputs, average = "macro")
            accuracy = accuracy_score(correct_outputs,predicted_outputs)

            results.append(( precision, recall, f1, accuracy))

        cv_results = []
        for result_index in range(4):
            sum_of_values = 0
            for result in results:
                sum_of_values += result[result_index]
            cv_results.append(sum_of_values/len(results))
        if self.debug:
            print(cv_results)
        return cv_results, classifier

    def get_model_stats_against_test_set(self, classifier, test_set, classifications):
        statements = [x[0] for x in test_set]
        correct_outputs = [x[1] for x in test_set]
        predicted_outputs = self.predict_labels(statements,classifier)
        print(classification_report(correct_outputs,predicted_outputs,target_names=classifications))
        (precision, recall, f1, _) = precision_recall_fscore_support(correct_outputs,predicted_outputs, average = "macro")
        accuracy = accuracy_score(correct_outputs,predicted_outputs)
        print("============")
        print("Model Against Test Split")
        print("============")
        print(f"Precision: {precision}")
        print(f"Recall: {recall}")
        print(f"F1: {f1}")
        print(f"Accuracy: {accuracy}")

    def predict_labels(self, samples, classifier):
        """Assuming preprocessed samples, return their predicted labels from the classifier model."""
        return classifier.classify_many(samples)

    def predict_label_from_raw(self, sample, classifier):
        """Assuming raw text, return its predicted label from the classifier model."""
        return classifier.classify(self.to_feature_vector(self.pre_process(sample)))

    def predict_label_list_from_raw(self, sample, classifier):
        prob = classifier.prob_classify(self.to_feature_vector(self.pre_process(sample)))
        non_zero = prob.samples()
        prob_dict = {}
        for i in non_zero:
            prob_dict[i] = prob.prob(i)
        return prob_dict

    def train_classifier(self, data):
        # pipeline =  Pipeline([('svc', LinearSVC())])
        # return SklearnClassifier(pipeline).train(data)
        return SklearnClassifier(SVC(kernel='linear',probability=True)).train(data)

    def save_model(self, x_classifier, y_classifier):
        folder = 'NLPModel/Linear SVC Models/'
        filename = 'x_classifier.joblib'
        dump(x_classifier, folder+filename)

        filename = 'y_classifier.joblib'
        dump(y_classifier, folder+filename)

        filename = 'global_feature_vector.joblib'
        dump(self.global_feature_dict, folder+filename)

    def start(self):
        # load in data
        # raw_data defined here
        self.load_data()

        x_data, y_data = self.define_categories()

        # train_data and test_data defined here
        x_train, x_test = self.split_and_preprocess_data(x_data)
        y_train, y_test = self.split_and_preprocess_data(y_data)

        print("Training X Classifier...")
        x_results, x_classifier = self.cross_validate(x_train, 10, self.x_split_cats)
        print(x_results)
        print("Training Y Classifier...")
        y_results, y_classifier = self.cross_validate(y_train, 10, self.y_split_cats)
        print(y_results)
        # self.save_model(x_classifier, y_classifier)
        print("\n")
        print("X Axis against test set...")
        self.get_model_stats_against_test_set(x_classifier, x_test, self.x_split_cats)
        print("\n")
        print("Y Axis against test set...")
        self.get_model_stats_against_test_set(y_classifier, y_test, self.y_split_cats)

    def load_model(self):
        folder = 'NLPModel/Linear SVC Models/'
        filename = 'x_classifier.joblib'
        x_classifier = load(folder+filename)
        filename = 'y_classifier.joblib'
        y_classifier = load(folder+filename)
        filename = 'global_feature_vector.joblib'
        self.global_feature_dict = load(folder+filename)

        return x_classifier, y_classifier

    def get_manual_prediction(self, x_classifier, y_classifier, statement = None):
        print("Manual Classifier...")
        if statement is None:
            statement = input("Input to Classify:\n")
        print("X Classifier")
        x_confidence = self.predict_label_list_from_raw(statement, x_classifier)
        print(x_confidence)
        print("Y Classifier")
        y_confidence = self.predict_label_list_from_raw(statement, y_classifier)
        print(y_confidence)
        return x_confidence, y_confidence


# Page 272 hello.py

print("Hello World")

# Page 273 hello_from.py

from mpi4py import MPI
comm = MPI.COMM_WORLD
print("Hello World from process", comm.rank, "of", comm.size)

# Page 273 Example 12.2 hello_to.py

from mpi4py import MPI
comm = MPI.COMM_WORLD
if comm.rank == 0:
    comm.send("Hello World", dest=1)
if comm.rank == 1:
    msg = comm.recv(source=0)
    print("Message received:", msg)

# Page 274 Deadlock dl.py

from mpi4py import MPI
comm = MPI.COMM_WORLD
msg = comm.recv(source = comm.size - 1 - comm.rank)
comm.send(22, dest = comm.size - 1 - comm.rank)
print(msg)

# From now on always include:

from mpi4py import MPI

# and wherever needed:

size = MPI.COMM_WORLD.size
rank = MPI.COMM_WORLD.rank
send = MPI.COMM_WORLD.send
recv = MPI.COMM_WORLD.recv
reduce = MPI.COMM_WORLD.reduce
bcast = MPI.COMM_WORLD.bcast
Scatter = MPI.COMM_WORLD.Scatter
Gather = MPI.COMM_WORLD.Gather
Scatterv = MPI.COMM_WORLD.Scatterv

# Page 275 Example 12.3 Summation sum.py

n = 100
assert n % size == 0
k = rank * n // size + 1
l = (rank + 1) *  n // size + 1
sm_loc = sum(range(k, l))
if rank == 0:
    sm = sm_loc
    for i in  range(1, size):
        sm += recv(source=i)
else: send(sm_loc, dest=0)
if rank == 0: print(sm)

# Page 276 Example 12.4 "reduce"

n = 100
assert n % size == 0
k = rank * n // size + 1
l = (rank + 1) *  n // size + 1
sm_loc = sum(range(k, l))
sm = reduce(sm_loc)
if rank == 0: print(sm)

# Page 276 Example 12.4 "bcast"

q_loc = None
sm_loc = None
if rank == 0:
    n = 100
    assert n % size == 0
    q_loc = n//size
q = bcast(q_loc)
k =  rank*q + 1
l = (rank + 1)*q + 1
sm_loc = sum(range(k,l))
if rank == 0:
    sm = sm_loc
    for i in  range(1, size):
        sm += recv(source=i)
else: send(sm_loc, dest=0)
if rank == 0: print(sm)

# Page 277 Example 12.6 Load Balancing

tup = None
if rank == 0:
    n = 100
    q = n//size; u = q*size; r = n % size
    tup = q, u, r
q, u, r = bcast(tup)
k =  rank*q + 1
l = (rank + 1)*q + 1
sm = sum(range(k,l))
if rank < r: sm += u+1  + rank
sm = reduce(sm)
if rank == 0: print(sm)

# Page 278 Example 12.7 Midpoint Rule

def f(x): return 4/(1 + x*x)
tup = None
if rank == 0:
    n = 100_000
    a = 0.0; b = 1.0; h = (b - a)/n
    tup = n, a, h
n, a, h = bcast(tup)
add = 0.0
for i in range(rank, n, size):
    x = a + h*(i + 0.5)
    add += f(x)
int_loc = h*add
int = reduce(int_loc)
if rank == 0: print(int)

# Page 280 Trapezoidal Rule

def f(x): return 4/(1 + x*x)
tup = None
if rank == 0:
    a = 0; b = 1; n = 100_000
    h = (b - a)/n
    from numpy import linspace
    x = linspace(a, b, n+1)
    q = n//size
    tup = q, x, h
q, x, h = bcast(tup)
k = rank*q; l = k + q
int_loc = h/2*(f(x[k]) + 2*sum(f(x[k+1: l])) + f(x[l]))
int = reduce(int_loc)
if rank == 0: print(int)

# Page 280, 281 Dot Product Evenly Divided

from numpy import array, zeros
v = w = n_loc = None
if rank == 0:
    n = 12
    assert n % size == 0
    v = array([float(i) for i in range(1, n+1)])
    w = array([float(i) for i in range(n, 0, -1)])
    n_loc = n//size
n_loc = bcast(n_loc)
v_loc = zeros(n_loc); w_loc = zeros(n_loc)
Scatter(v, v_loc); Scatter(w, w_loc)  # note capital letter S, see below
dot_loc = v_loc @ w_loc  # recall: @ denotes matrix multiplication
dot = reduce(dot_loc)
if rank == 0: print(dot)

# Page 282 Dot Product with Load Balance

from numpy import array, zeros
v = w = sendcts = None
if rank == 0:
    n = 10
    v = array([float(i) for i in range(1, n+1)])
    w = array([float(i) for i in range(n, 0, -1)])
    n_loc = n//size  # even divisibility not required
    sendcts = [n_loc for i in range(size)]
    for i in range(n % size): sendcts[i] += 1
sendcts = bcast(sendcts)
v_loc = zeros(sendcts[rank]); w_loc = zeros(sendcts[rank])
Scatterv([v, sendcts, MPI.DOUBLE], v_loc)
Scatterv([w, sendcts, MPI.DOUBLE], w_loc)
dot_loc = v_loc @ w_loc
dot = reduce(dot_loc)
if rank == 0: print(dot)

# Page 284 Laplace Equation

if rank == 0:
    from numpy import linspace, zeros
    n = 10; m = n//2
    u = zeros((n,n))
    x = linspace(0, 1, n)
    u[0,:] = x*(1-x)
    ul = u[:, :m]
    ur = u[:, m:]
    send(ur, dest=1)
if rank == 1: ur = recv(source=0)
def stencil_it(u):
    u_it = u.copy()
    r, c = u.shape
    for i in range(1, r-1):
        for j in range(1, c-1):
             u_it[i,j] = (u[i+1,j] + u[i-1,j]
                        + u[i,j+1] + u[i,j-1]) / 4
    return u_it
from numpy import column_stack, delete
iter = 40
for i in range(iter):
    if rank == 0:
        send(ul[:, -1], dest=1)
        v = recv(source=1)
        ul_v = column_stack([ul, v])
        ul_v = stencil_it(ul_v)
        ul = delete(ul_v, -1, 1)
    if rank == 1:
       send(ur[:, 0], dest=0)
       v = recv(source=0)
       v_ur = column_stack([v, ur])
       v_ur = stencil_it(v_ur)
       ur = delete(v_ur, 0, 1)
if rank == 1: send(ur, dest=0)
if rank == 0:
    ur = recv(source=1)
    u = column_stack([ul, ur])
if rank == 0:
    from matplotlib.pyplot import figure, show
    fig = figure()
    ax = fig.add_subplot(projection='3d')
    from numpy import meshgrid
    x, y = meshgrid(x,x)  # linspace from line 5
    ax.plot_surface(x, y, u,
        rstride=1, cstride=1, cmap='jet', linewidth=0)
    show()

# Page 287 Conjugate Gradient

from numpy import array, zeros, sqrt
A = n = p = None
if rank == 0:
    A = array([[  9.,   3.,  -6.,  12.], [  3.,  26.,  -7., -11.],
               [ -6.,  -7.,   9.,   7.], [ 12., -11.,   7.,  65.]])
    b = array([ 18.,  11.,   3.,  73.])
    n = len(b)
    x = zeros(n)
    r = b.copy(); p = r.copy()
    rs_old = r @ r
n = bcast(n)
n_loc = n//size
A_loc = zeros((n_loc, n))
Scatter(A, A_loc)
for i in range(n):
    p = bcast(p)
    Ap_loc = A_loc @ p
    Ap = zeros(n)
    Gather(Ap_loc, Ap)
    if rank == 0:
        alpha = rs_old / (p @ Ap)
        x += alpha*p
        r -= alpha*Ap
        rs_new = r @ r
        if sqrt(rs_new) < 1e-10: break
        p = r + (rs_new / rs_old)*p
        rs_old = rs_new
if rank == 0: print(x)


from pyspark.sql import SparkSession, SQLContext
from pyecharts import options as opts
from pyecharts.charts import Pie
import os

os.environ['JAVA_HOME'] = "/usr/local/java/jdk1.8"

spark = SparkSession.builder.getOrCreate()
df = spark.read.format("jdbc") \
    .option("driver", "com.mysql.cj.jdbc.Driver") \
    .option("url", "jdbc:mysql://plane.mysql.database.azure.com:3306/weibo") \
    .option("user", "mykaneki") \
    .option("dbtable", "weibo_new") \
    .option("password", "azureLinux668CMY...").load()

load_dict = df.select('text', 'attitudes_count', 'comments_count').take(1000)

dic_attitudes = {}
dic_comments = {}
for dic in load_dict:
    title = dic['text']
    attitudes_count = dic['attitudes_count']
    comments_count = dic['comments_count']
    dic_attitudes[title] = attitudes_count
    dic_comments[title] = comments_count

d_attitudes = sorted(dic_attitudes.items(), key=lambda d: int(d[1]), reverse=True)
d_comments = sorted(dic_comments.items(), key=lambda d: int(d[1]), reverse=True)

columns_attitudes = [i[0] for i in d_attitudes[0:5]]
columns_comments = [i[0] for i in d_comments[0:5]]
data_attitudes = [i[1] for i in d_attitudes[0:5]]
data_comments = [i[1] for i in d_comments[0:5]]

c = (
    Pie(init_opts=opts.InitOpts(width="1440px", height="300px"))
    .add(
        "",
        [
            list(z)
            for z in zip(
            columns_attitudes,
            data_attitudes,
        )
        ],
        center=["40%", "50%"],

    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="top5点赞帖子饼图"),
        legend_opts=opts.LegendOpts(type_="scroll", pos_left="60%", orient="vertical"),
    )
    .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
    .render("top5_attitudes.html")
)

d = (
    Pie(init_opts=opts.InitOpts(width="1440px", height="300px"))
    .add(
        "",
        [
            list(z)
            for z in zip(
            columns_comments,
            data_comments,
        )
        ],
        center=["40%", "50%"],

    )
    .set_global_opts(
        title_opts=opts.TitleOpts(title="top5评论帖子饼图"),
        legend_opts=opts.LegendOpts(type_="scroll", pos_left="60%", orient="vertical"),
    )
    .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
    .render("top5_comments.html")
)


import torch
import numpy as np
import scipy.sparse as sp

from utils import encode_onehot, normalize, sparse_mx_to_torch_sparse_tensor, text_to_label_map, text_to_array

def load_cora(feature_mode = 0, feature_scale = 1):
  print('Loading cora dataset...')

  idx_features_labels = np.genfromtxt("cora.content",
                                      dtype=np.dtype(str))

  labels = encode_onehot(idx_features_labels[:, -1])

  idx = np.array(idx_features_labels[:,0], dtype=np.int32)
  idx_map = {j: i for i, j in enumerate(idx)}
  edges_unordered = np.genfromtxt("cora.cites", dtype=np.int32)
  edges = np.array(list(map(idx_map.get, edges_unordered.flatten())),
                   dtype=np.int32).reshape(edges_unordered.shape)
  adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])),
                      shape=(labels.shape[0], labels.shape[0]),
                      dtype=np.float32)

  adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
  adj = normalize (adj + sp.eye(adj.shape[0]))

  if (feature_mode == -1):
    features_given = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32)
    features_given = normalize(features_given)
    features_given = torch.FloatTensor(np.array(features_given.todense()))

    louvain_label_map = text_to_label_map('cora.label', ' ')
    tot_labels = len(set(louvain_label_map))
    features_louvain = np.zeros((len(idx_map),tot_labels * feature_scale))
    for i in range(len(idx_map)):
      for j in range(feature_scale):
        features_louvain[i, louvain_label_map[i] * feature_scale + j] = 1
    features_louvain = torch.FloatTensor(normalize(features_louvain))

    features = torch.cat((features_given, features_louvain), 1)
  elif (feature_mode == 0):
    features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32)
    features = normalize(features)
    features = torch.FloatTensor(np.array(features.todense()))
  elif (feature_mode == 1):
    louvain_label_map = text_to_label_map('cora.label', ' ')
    tot_labels = len(set(louvain_label_map))
    features = np.zeros((len(idx_map),tot_labels * feature_scale))
    for i in range(len(idx_map)):
      for j in range(feature_scale):
        features[i, louvain_label_map[i] * feature_scale + j] = 1
    features = torch.FloatTensor(normalize(features))
  elif (feature_mode == 2):
    features = torch.randn(len(idx_map), feature_scale)
    features = features / (features * features).sum(1).sqrt().reshape(len(idx_map), 1)
  elif (feature_mode == 3):
    features = torch.ones(len(idx_map), feature_scale) / feature_scale

  idx_train = range(140)
  idx_val = range(200, 500)
  idx_test = range(500, 1500)

  labels = torch.LongTensor(np.where(labels)[1])
  adj = sparse_mx_to_torch_sparse_tensor(adj)

  idx_train = torch.LongTensor(idx_train)
  idx_val = torch.LongTensor(idx_val)
  idx_test = torch.LongTensor(idx_test)

  return adj, features, labels, idx_train, idx_val, idx_test

def load_amazon(feature_scale = 1):
  print('Loading Amazon dataset')

  f_edge = open('com-amazon.ungraph.txt', 'r')
  lines = f_edge.readlines()
  edges = []
  cnt = 0
  max_idx = 0
  for line in lines:
    line = line.lstrip().rstrip().split('\t')
    line = list(map(lambda x : int(x), line))
    edges.append(line)
    max_idx = max(max_idx, line[0], line[1])
  f_edge.close()

  labels = []
  f_label = open('com-amazon.top297.cmty.txt', 'r')
  lines = f_label.readlines()
  for line in lines:
    line = line.strip().split(' ')
    labels.append(int(line[1]))
  f_label.close()
  labels = encode_onehot(labels)

  present = [0 for _ in range(max_idx + 1)]
  for edge in edges:
    present[edge[0]] += 1
    present[edge[1]] += 1

  nodes = []
  for i in range(len(present)):
    if (present[i] != 0):
      nodes.append(i)

  node_map = {j : i for i, j in enumerate(nodes)}
  nodes = [i for i in range(len(node_map))]
  num_node = len(nodes)

  edges = np.array(list(map(lambda x : [node_map[x[0]], node_map[x[1]]], edges)))

  adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])), shape = (num_node, num_node), dtype = np.float32)
  adj = adj + adj.T.multiply(adj.T) - adj.multiply(adj.T > adj)
  adj = normalize(adj + sp.eye(adj.shape[0]))
  adj = sparse_mx_to_torch_sparse_tensor(adj)

  train_size = num_node * 2 // 10
  train_val_size = num_node * 4 // 10

  np.random.shuffle(nodes)
  train_idx = nodes[:train_size]
  val_idx = nodes[train_size: train_val_size]
  test_idx = nodes[train_val_size:]

  louvain_label_map = text_to_label_map('amazon.label', ' ')
  tot_labels = len(set(louvain_label_map))
  features = np.zeros((num_node,tot_labels * feature_scale))
  for i in range(num_node):
    for j in range(feature_scale):
      features[i, louvain_label_map[i] * feature_scale + j] = 1
  features = torch.FloatTensor(normalize(features))

  labels = torch.LongTensor(np.where(labels)[1])

  train_idx = torch.LongTensor(train_idx)
  val_idx = torch.LongTensor(val_idx)
  test_idx = torch.LongTensor(test_idx)

  return adj, features, labels, train_idx, val_idx, test_idx

def load_citeseer(feature_mode, feature_scale):
  print('Loading cora dataset...')

  f_edge = open('citeseer.edges', 'r')
  lines = f_edge.readlines()
  edges = []
  for line in lines:
    line = line.strip().split(',')
    line = list(map(lambda x : int(x) - 1, line))
    assert line[2] == 0
    edges.append([line[0], line[1]])
  f_edge.close()

  edges = np.array(edges)

  f_node = open('citeseer.node_labels', 'r')
  lines = f_node.readlines()
  labels = []
  for line in lines:
    line = line.strip().split(',')
    labels.append(int(line[1]))

  nodes = [i for i in range(len(lines))]
  num_node = len(nodes)

  labels = encode_onehot(labels)

  adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])),
                      shape=(labels.shape[0], labels.shape[0]),
                      dtype=np.float32)

  adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
  adj = normalize (adj + sp.eye(adj.shape[0]))

  if (feature_mode == 0):
    features = torch.randn(num_node, feature_scale)
    features = features / (features * features).sum(1).sqrt().reshape(num_node, 1)
  elif (feature_mode == 1):
    features = torch.ones(num_node, feature_scale) / feature_scale
  elif (feature_mode == 2):
    louvain_label_map = text_to_label_map('citeseer.label', ' ')
    tot_labels = len(set(louvain_label_map))
    features = np.zeros((num_node ,tot_labels * feature_scale))
    for i in range(num_node):
      for j in range(feature_scale):
        features[i, louvain_label_map[i] * feature_scale + j] = 1
    features = torch.FloatTensor(normalize(features))

  train_size = num_node * 2 // 10
  train_val_size = num_node * 4 // 10
  np.random.shuffle(nodes)
  idx_train = nodes[:train_size]
  idx_val = nodes[train_size: train_val_size]
  idx_test = nodes[train_val_size:]

  labels = torch.LongTensor(np.where(labels)[1])
  adj = sparse_mx_to_torch_sparse_tensor(adj)

  idx_train = torch.LongTensor(idx_train)
  idx_val = torch.LongTensor(idx_val)
  idx_test = torch.LongTensor(idx_test)

  return adj, features, labels, idx_train, idx_val, idx_test


import nltk
import random
import pickle
from nltk.classify import ClassifierI
from statistics import mode
from nltk.tokenize import word_tokenize

documents_f = open("pickle/documents.pickle", "rb")
documents = pickle.load(documents_f)
documents_f.close()

word_features_f = open("pickle/word_features.pickle", "rb")
word_features = pickle.load(word_features_f)
word_features_f.close()

def find_features(document):
    words = word_tokenize(document)
    features = {}
    for w in word_features:
        features[w] = (w in words)

    return features

featuresets_f = open("pickle/featuresets.pickle", "rb")
featuresets = pickle.load(featuresets_f)
featuresets_f.close()

random.shuffle(featuresets)

testing_set = featuresets[10000:]
training_set = featuresets[:10000]

open_file = open("pickle/naivebayes.pickle", "rb")
classifier = pickle.load(open_file)
open_file.close()

def sentiment(text):
	feats=find_features(text)
	return classifier.classify(feats)


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
Test classes
"""

from time import sleep

from scapy.sendrecv import sendp
from scapy.packet import Raw
from scapy.volatile import RandShort
from scapy.layers.l2 import Ether, ARP
from scapy.layers.inet import IP, ICMP, TCP, UDP, fragment, defragment
from scapy.data import ETHER_BROADCAST

from utils import Sniffer

class TestSuite(object):

    """ Test class for all kind of packet pattern. """

    def __init__(self, name, log, conf, bpfilter, payload=None,
                 local_sniff=False, count=10):
        """
        :name: Test name
        :log: log object for output
        :conf: the network configuration
        :bpfilter: the bpfilter string
        :payload: custom payload to add
        :local_sniff: whether to sniff packet on the sending interface or not
        :count: the number of packet to send

        """
        self._name = name
        self._log = log
        self._conf = conf
        self._bpfilter = bpfilter
        self._payload = payload
        self._local_sniff = local_sniff
        self._count = count
        self._test_result = True

    def build_query(self, conf):
        """Method used to build the packet to send
        :conf: the namespace configuration
        :returns: packet built

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= self._payload

        # Force building the packet
        return pkt.__class__(str(pkt))

    def checksum_is_valid(self, pkt, proto):
        """ Method to validate the protocol checksum by computing the checksum
        with Scapy and compare it with the checksum reveived

        :pkt: the packet to validate
        :proto: The protocol, can be IP, TCP, or UDP.
        :returns: Bool depending on the result

        """
        received_csum = pkt[proto].chksum
        # force the checksum to None
        del pkt[proto].chksum
        # rebuild the packet to calculate the checksum
        pkt = pkt.__class__(str(pkt))
        if pkt[proto].chksum != received_csum:
            self._log.error('Bad %s checksum detected 0x%x, expecting 0x%x' %
                            (proto.name, received_csum, pkt[proto].chksum))
            return False
        return True

    def validate_l2l3_answer(self, pkts, conf, layer3=True, layer2=True,
                             count_exp=None, payload=True):
        """ This method validates the l2 and l3 informations on the answred
        packet this informations remain the same on all kind of tested traffic
        except ARP test.

        :pkts: packets list to validate
        :conf: the namespace configuration
        :layer2: test layer2
        :layer3: test layer3
        :expected: the expected number of packets

        :returns: True if the pkts are valid else False

        """
        log = self._log
        count = self._count * 2 if not count_exp else count_exp

        # Invert the MAC and IP @ when checking on the same iface that we send on
        if self._local_sniff:
            mac_src = conf['mac_local']
            mac_dst = conf['mac_nh']
            ip_src = conf['ip_priv']
            ip_dst = conf['ip_rmt']
        else:
            mac_src = conf['mac_nh']
            mac_dst = conf['mac_local']
            ip_src = conf['ip_pub']
            ip_dst = conf['ip_rmt_priv']

        if len(pkts) != count:
            log.error('Expecting %d captured packets found %d' %
                      (count, len(pkts)))
            return False

        _iter = iter(pkts)
        for pkt in _iter:
            try:
                req = pkt
                rsp = next(_iter)
            except StopIteration:
                rsp = None

            if layer2:
                if(req[Ether].src != mac_src or
                   req[Ether].dst != mac_dst):
                    log.error('Expecting packet with MAC@(src="%s", dst="%s"),'
                              ' got MAC@(src="%s" dst="%s")' %
                              (mac_src, mac_dst,
                               req[Ether].src, req[Ether].dst))
                    return False
                if rsp:
                    if(rsp[Ether].src != mac_dst or
                       rsp[Ether].dst != mac_src):
                        log.error('Expecting packet with MAC@(src="%s", dst="%s"),'
                                  ' got MAC@(src="%s" dst="%s")' %
                                  (mac_dst, mac_src,
                                   rsp[Ether].src, rsp[Ether].dst))
                        return False
            if layer3:
                if(req[IP].src != ip_src or
                   req[IP].dst != ip_dst):
                    log.error('Expecting packet with IP@(src="%s", dst="%s"),'
                              ' got IP@(src="%s" dst="%s")' %
                              (ip_src, ip_dst,
                               req[IP].src, req[IP].dst))
                    return False
                if rsp:
                    if(rsp[IP].src != ip_dst or
                       rsp[IP].dst != ip_src):
                        log.error('Expecting packet with IP@(src="%s", dst="%s"),'
                                  ' got IP@(src="%s" dst="%s")' %
                                  (ip_dst, ip_src,
                                   rsp[IP].src, rsp[IP].dst))
                        return False

                # Validate the l3 checksum on natted packet
                if  not self.checksum_is_valid(rsp if self._local_sniff else
                                               req, IP):
                    return False

            # Validate the payload
            if payload:
                pload = rsp[Raw].load if self._local_sniff else req[Raw].load
                if pload != self._payload:
                    log.error('Expecting packet with payload "%s", '
                              'got "%s"' % (self._payload, pload))
                    return False

        log.info('Received packets have a correct layer2, layer3 and payload'
                 ' informations')
        return True

    def validate_answer(self, req, pkts, conf):
        """ Wrapper for all validation methods

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return self.validate_l2l3_answer(pkts, conf)

    def run(self):
        """TODO: Docstring for run.

        :returns: Bool depending on test result

        """
        self._log.info("Starting %s test" % self._name)
        # send traffic flow in both sides
        for namespace, conf in self._conf.iteritems():
            iface = conf['dev'] if self._local_sniff else conf['peer_dev']
            pkt = self.build_query(conf)
            self._log.info("From namespace %s:" % namespace)
            self._log.debug("Packet built, ready to send.")
            if not isinstance(pkt, list):
                self._log.debug("%s" % pkt.summary())

            sniffer = Sniffer(log=self._log,
                              iface=iface,
                              bpfilter=self._bpfilter)
            sniffer.start()
            sleep(2) # let the thread start and start sniffing, very important
            self._log.debug('Sending %d packets...' % self._count)
            sendp(pkt, iface=conf['dev'],
                  count=self._count,
                  verbose=False)
            self._log.debug("Sending done.")
            sniffer.stop()

            self._log.debug("Start DPI...")

            # force IP reassembly if it exists
            pkts = defragment(sniffer.get_pcap())

            result = self.validate_answer(pkt, pkts, conf)
            if result:
                self._log.debug("DPI done, traffic is correct as expected")
            else:
                self._log.error("DPI done, traffic is incorrect")
            # Don't exit the test if it fails in one way, let's test the other
            # way too
            self._test_result &= result

        if self._test_result:
            self._log.info("%s Test done with success --> OK" % self._name)
        else:
            self._log.error("%s Test failed --> KO" % self._name)

        return self._test_result

class ICMPTest(TestSuite):

    """ICMP test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= ICMP(id=RandShort(), seq=1)
        pkt /= self._payload

        return pkt.__class__(str(pkt))

    def validate_icmp_answer(self, pkts, conf):
        """Validate the ICMP answers

        :pkts: received ICMP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log
        count = self._count
        echo_replays = 0

        for pkt in pkts:
            if pkt.haslayer(ICMP) and pkt[ICMP].type == 0:
                if (pkt[IP].src == conf['ip_rmt'] and
                        pkt[IP].dst == conf['ip_priv'] and
                        pkt[Ether].src == conf['mac_nh'] and
                        pkt[Ether].dst == conf['mac_local']):
                    echo_replays += 1
        if echo_replays != count:
            log.error('Expecting %d ICMP echo reply, got %d' % (count,
                                                                echo_replays))
            return False
        log.info('Expected ICMP echo replys correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate ICMP answers, expecting ICMP echo and echo_replays.

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_icmp_answer(pkts, conf))

class TCPTest(TestSuite):

    """TCP test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        TCP test need to validate any kind of TCP packet so let's use TCP SYN

        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= TCP(sport=RandShort(), dport=RandShort(), flags='S')
        pkt /= self._payload

        return pkt.__class__(str(pkt))

    def validate_l4_answer(self, pkts):
        """Validate the TCP answers
        TCP protol sends RST (reset connexion) when the port is not open
        let's filter and check only SYN packets

        :pkts: received TCP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log

        # test only received TCP packet with SYN flag
        for pkt in pkts:
            if pkt.haslayer(TCP) and pkt[TCP].flags == 'S':
                if  not self.checksum_is_valid(pkt, TCP):
                    return False
        log.info('Received TCP traffic is correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate answers, expecting TCP SYN and TCP RESET traffic

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_l4_answer(pkts))

class TCPFragTest(TCPTest):

    """TCP Fragmented test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        TCP test need to validate any kind of TCP packet so let's use TCP SYN

        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= TCP(sport=RandShort(), dport=RandShort(), flags='S')
        pkt /= self._payload

        return fragment(pkt.__class__(str(pkt)))

class UDPTest(TestSuite):

    """UDP test class"""

    def build_query(self, conf):
        """Method used to build the UDP packet to send.
        TCP test need to validate any kind of TCP packet so let's use TCP SYN

        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= UDP(sport=RandShort(), dport=RandShort())
        pkt /= self._payload

        return pkt.__class__(str(pkt))

    def validate_l4_answer(self, pkts):
        """Validate the UDP answers
        Linux Kernel sends an ICMP port unreachable when a UDP port is not open
        let's filter and check only UDP packets

        :pkts: received UDP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log

        for pkt in pkts:
            if pkt.haslayer(UDP):
                if not self.checksum_is_valid(pkt, UDP):
                    return False
        log.info('Received UDP traffic is correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate answers, we expect UDP packets and ICMP port unreachable
        packets as well

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_l4_answer(pkts))

class UDPDFTest(UDPTest):

    """UDPDF test class same as UDP test but with DF IP flag"""

    def build_query(self, conf):
        """Method used to build the UDP packet to send.

        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort(),
                  flags=2)
        pkt /= UDP(sport=RandShort(), dport=RandShort())
        pkt /= self._payload

        return pkt.__class__(str(pkt))

class UDPFragTest(TestSuite):

    """
    UDP Fragmented test class This test need to validate that the received
    reasemble packet has a good UDP checksum after being natted.

    """

    def build_query(self, conf):
        """Method used to build a serie of fragment of a huge UDP packet.

        :conf: the namespace configuration
        :returns: packet to send

        """
        payload = self._payload
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= UDP(sport=RandShort(), dport=RandShort())
        pkt /= payload

        return fragment(pkt.__class__(str(pkt)))

    def validate_l4_answer(self, pkts):
        """Validate the UDP answers
        Linux Kernel sends an ICMP port unreachable when a UDP port is not open
        let's filter and check only UDP packets

        :pkts: received UDP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log

        for pkt in pkts:
            if pkt.haslayer(UDP):
                if not self.checksum_is_valid(pkt, UDP):
                    return False
        log.info('Received UDP traffic is correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate answers, we expect UDP packets and ICMP port unreachable
        packets as well

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        # received packet are reassemble so we dont't expect the same number we
        # sent. let's recalculate it and taking in consideration reassembled
        # request

        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_l4_answer(pkts))

class UDPZeroCsumTest(UDPTest):

    """UDPDF test class same as UDP test but with checksum = 0"""

    def build_query(self, conf):
        """Method used to build the UDP packet to send.

        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort(),
                  flags=2)
        pkt /= UDP(sport=RandShort(), dport=RandShort(), chksum=0)
        pkt /= self._payload

        return pkt.__class__(str(pkt))

    def validate_l4_answer(self, pkts):
        """Validate the UDP answers
        Linux Kernel sends an ICMP port unreachable when a UDP port is not open
        let's filter and check only UDP packets

        :pkts: received UDP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log

        for pkt in pkts:
            if pkt.haslayer(UDP):
                # we exect the checksum to remain 0 after being natted
                if pkt[UDP].chksum != 0:
                    return False
        log.info('Received UDP traffic is correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate answers, we expect UDP packets and ICMP port unreachable
        packets as well

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_l4_answer(pkts))

class ICMPFragTest(ICMPTest):

    """ICMPFrag test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= ICMP(id=RandShort(), seq=1)
        pkt /= self._payload

        return fragment(pkt.__class__(str(pkt)))

    def validate_icmp_answer(self, pkts, conf):
        """Validate the ICMP answers

        :pkts: received ICMP packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log
        count = self._count
        echo_replays = 0

        for pkt in pkts:
            if pkt.haslayer(ICMP) and pkt[ICMP].type == 0:
                if (pkt[IP].src == conf['ip_rmt'] and
                        pkt[IP].dst == conf['ip_priv'] and
                        pkt[Ether].src == conf['mac_nh'] and
                        pkt[Ether].dst == conf['mac_local']):
                    echo_replays += 1
        if echo_replays != count:
            log.error('Expecting %d ICMP echo reply, got %d' % (count,
                                                                echo_replays))
            return False
        log.info('Expected ICMP echo replys correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate ICMP answers, expecting ICMP echo and echo_replays.

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf) and
                self.validate_icmp_answer(pkts, conf))

class TraceRouteTest(ICMPTest):

    """TraceRouteTest test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=conf['mac_nh'])
        # Outer packet
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= ICMP(id=RandShort(), seq=1, type=11, code=0) # ICMP TTL exceeded
        # Inner packet
        pkt /= IP(src=conf['ip_priv'], dst=conf['ip_rmt'], id=RandShort())
        pkt /= UDP(sport=RandShort(), dport=RandShort())
        pkt /= self._payload

        return pkt

    def validate_traceroute_answer(self, pkts, conf):
        """Validate the inner packet over ICMP answers

        :pkts: received ICMP ttl expired packets
        :conf: the namespace configuration
        :returns: bool

        """
        log = self._log

        for pkt in pkts:
            if pkt.haslayer(ICMP) and pkt[ICMP].type == 13: # ttl-exceeded
                inner_pkt = []
                inner_pkt += IP(str(pkt[Raw].load))
                if not self.validate_l2l3_answer(pkts, conf, count_exp=1):
                    return False
                if inner_pkt.haslayer(UDP):
                    if not self.checksum_is_valid(inner_pkt, UDP):
                        return False

        log.info('Received packets have a correct inner layer3, layer4 and '
                 'payload informations')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate ICMP answers, expecting ICMP echo and echo_replays.

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """
        return (self.validate_l2l3_answer(pkts, conf, payload=False,
                                          count_exp=1) and
                self.validate_traceroute_answer(pkts, conf))

class ARPTest(TestSuite):

    """ARP test class"""

    def build_query(self, conf):
        """Method used to build the packet to send
        :conf: the namespace configuration
        :returns: packet to send

        """
        pkt = Ether(src=conf['mac_local'], dst=ETHER_BROADCAST)
        pkt /= ARP(op=1, pdst=conf['ip_nh'])

        return pkt.__class__(str(pkt))

    def validate_arp_answer(self, pkts, conf):
        """Validate the ARP answers

        :pkts: received ARP packets
        :conf: the namespace configuration
        :returns: bool

        """

        log = self._log
        count = self._count
        arp_replies = 0
        mac_src = conf['mac_nh']
        mac_dst = conf['mac_local']
        ip_src = conf['ip_nh']
        ip_dst = conf['ip_priv']

        for pkt in pkts:
            if pkt.haslayer(ARP) and pkt[ARP].op == 2:
                if(pkt[Ether].src != mac_src or pkt[Ether].dst != mac_dst):
                    log.error('Expecting packet with MAC@(src="%s", dst="%s"),'
                              ' got MAC@(src="%s" dst="%s")' % (mac_dst,
                                                                mac_src,
                                                                pkt[Ether].src,
                                                                pkt[Ether].dst))
                    return False

                if(pkt[ARP].hwsrc != mac_src or pkt[ARP].hwdst != mac_dst):
                    log.error('Expecting packet with ARP '
                              'MAC@(src="%s", dst="%s"), got ARP '
                              'MAC@(src="%s" dst="%s")' % (mac_dst, mac_src,
                                                           pkt[ARP].hwsrc,
                                                           pkt[ARP].hwdst))
                    return False

                if(pkt[ARP].psrc != ip_src or pkt[ARP].pdst != ip_dst):
                    log.error('Expecting packet with ARP '
                              'IP@(src="%s", dst="%s"), got ARP '
                              'IP@(src="%s" dst="%s")' % (ip_src, ip_dst,
                                                          pkt[ARP].psrc,
                                                          pkt[ARP].pdst))
                    return False

                arp_replies += 1

        if arp_replies != count:
            log.error('Expecting %d ARP reply, got %d' % (count, arp_replies))
            return False

        log.info('Expected ARP reply correct')
        return True

    def validate_answer(self, req, pkts, conf):
        """Validate ARP.

        :req: the packet generated and sent
        :pkts: captured packts
        :conf: the namespace configuration
        :returns: Bool

        """

        return self.validate_arp_answer(pkts, conf)


import asyncio
import functools

import discord
from discord.ext import commands

class PaginationError(Exception):
    pass

class Paginator:
    __slots__ = (
        "ctx",
        "bot",
        "user",
        "channel",
        "msg",
        "execute",
        "embed",
        "max_pages",
        "entries",
        "paginating",
        "current",
        "reactions",
    )

    def __init__(self, ctx):
        self.ctx = ctx
        self.bot = ctx.bot
        self.user = ctx.author
        self.channel = ctx.channel
        self.msg = ctx.message

        self.execute = None
        self.entries = []
        self.embed = None
        self.max_pages = None
        self.paginating = True
        self.current = 0

        part = functools.partial(self.stop, delete=True)
        part.__doc__ = self.stop.__doc__

        self.reactions = [
            ("\N{BLACK LEFT-POINTING TRIANGLE}", self.backward),
            ("\N{BLACK RIGHT-POINTING TRIANGLE}", self.forward),
            ("\N{BLACK SQUARE FOR STOP}", part),
            ("\N{INFORMATION SOURCE}", self.info),
        ]

    def add_entry(self, entry):
        self.entries.append(entry)

    PAGINATION_PERMS = frozenset({"add_reaction", "embed_links", "read_message_history"})

    async def setup(self):
        perms = self.channel.permissions_for(self.ctx.me)
        missing = [perm for perm, value in perms
                   if perm in self.PAGINATION_PERMS and not value]
        if missing:
            raise commands.BotMissingPermissions(missing)

        if not self.entries:
            e = PaginationError("No pagination entries.")
            raise commands.CommandInvokeError(e) from e

        if not self.embed:
            self.entries = [entry + f"\n\nPage {page} of {len(self.entries)}"
                            for page, entry in enumerate(self.entries, 1)]
            try:
                self.msg = await self.channel.send(self.entries[0])
            except AttributeError:
                await self.channel.send(self.entries)
        else:
            for page, embed in enumerate(self.entries, 1):
                embed.set_author(name=f"Page {page} of {len(self.entries)}")
            try:
                self.msg = await self.channel.send(embed=self.entries[0])
            except (AttributeError, TypeError):
                await self.channel.send(embed=self.entries)

        if len(self.entries) == 1:
            return

        self.max_pages = len(self.entries) - 1

        for (r, _) in self.reactions:
            await self.msg.add_reaction(r)

    async def alter(self, page: int):
        try:
            await self.msg.edit(embed=self.entries[page])
        except (AttributeError, TypeError):
            await self.msg.edit(content=self.entries[page])

    async def backward(self):
        """takes you to the previous page or the last if used on the first one."""
        if self.current == 0:
            self.current = self.max_pages
            await self.alter(self.current)
        else:
            self.current -= 1
            await self.alter(self.current)

    async def forward(self):
        """takes you to the next page or the first if used on the last one."""
        if self.current == self.max_pages:
            self.current = 0
            await self.alter(self.current)
        else:
            self.current += 1
            await self.alter(self.current)

    async def stop(self, *, delete=False):
        """stops the paginator session."""
        try:
            if delete:
                await self.msg.delete()
            else:
                await self.msg.clear_reactions()
        except discord.HTTPException:
            pass
        finally:
            self.paginating = False

    async def info(self):
        """shows this page."""
        fmt = []

        for emoji, func in self.reactions:
            fmt.append(f"{emoji} {func.__doc__}")

        if not self.embed:
            fmt.insert(0, "**Instructions**\n"
                          "This is a reaction paginator, when you react to one of the buttons below the message gets"
                          " edited. Below you will find what each does.\n\n"
                          "**Reactions**")
            return await self.msg.edit(content="\n".join(fmt))

        embed = discord.Embed(color=discord.Color(0x008CFF))

        embed.set_author(name="Instructions")

        embed.description = (
            "This is a reaction paginator, when you react to one of the buttons below "
            "the message gets edited. Below you will find what each does."
        )

        embed.add_field(name="Reactions", value="\n".join(fmt))

        await self.msg.edit(embed=embed)

    def check(self, reaction, user):
        if not user.id == self.user.id:
            return False

        if not reaction.message.id == self.msg.id:
            return False

        for (emoji, func) in self.reactions:
            if reaction.emoji == emoji:
                self.execute = func
                return True
        return False

    async def paginate(self, *, embed: bool = True):
        self.embed = embed

        await self.setup()

        while self.paginating:
            done, pending = await asyncio.wait(
                [
                    self.bot.wait_for("reaction_add", check=self.check),
                    self.bot.wait_for("reaction_remove", check=self.check),
                ],
                return_when=asyncio.FIRST_COMPLETED, timeout=120
            )

            try:
                done.pop().result()
            except (discord.HTTPException, asyncio.TimeoutError, KeyError):
                return await self.stop()

            for future in pending:
                future.cancel()

            await self.execute()

class Tabulator:
    __slots__ = ("_widths", "_columns", "_rows")

    def __init__(self):
        self._widths = []
        self._columns = []
        self._rows = []

    def set_columns(self, columns):
        self._columns = columns
        self._widths = [len(c) + 2 for c in columns]

    def add_row(self, row):
        rows = [str(r) for r in row]
        self._rows.append(rows)
        for index, element in enumerate(rows):
            width = len(element) + 2
            if width > self._widths[index]:
                self._widths[index] = width

    def add_rows(self, rows):
        for row in rows:
            self.add_row(row)

    def render(self):
        sep = "╬".join("═" * w for w in self._widths)
        sep = f"╬{sep}╬"

        to_draw = [sep]

        def get_entry(d):
            elem = "║".join(f"{e:^{self._widths[i]}}" for i, e in enumerate(d))
            return f"║{elem}║"

        to_draw.append(get_entry(self._columns))
        to_draw.append(sep)

        for row in self._rows:
            to_draw.append(get_entry(row))

        to_draw.append(sep)
        return "\n".join(to_draw)

class Plural:
    def __init__(self, value):
        self.value = value

    def __format__(self, format_spec):
        v = self.value
        singular, _, pl = format_spec.partition('|')
        pl = pl or f"{singular}s"
        if not abs(v) == 1:
            return f"{v} {pl}"
        return f"{v} {singular}"


import psycopg2
import database_service

conn = database_service.connect()

def create_tables():
    """ create tables in the PostgreSQL database"""
    commands = (
        """
        CREATE TABLE IF NOT EXISTS athlete (
            athlete_id SERIAL PRIMARY KEY,
            name VARCHAR(255) NOT NULL,
            email VARCHAR(255) NOT NULL,
            phone VARCHAR(255) NOT NULL,
            gender VARCHAR(255) NOT NULL,
            birthday DATE NOT NULL
        )
        """,
        """ CREATE TABLE IF NOT EXISTS plan (
                   plan_id SERIAL PRIMARY KEY,
                   athlete_id INTEGER NOT NULL,
                   name VARCHAR(255) NOT NULL,
                   description TEXT NOT NULL,
                   difficulty VARCHAR(255) NOT NULL,
                        FOREIGN KEY (athlete_id)
                            REFERENCES athlete (athlete_id)
                            ON UPDATE CASCADE ON DELETE CASCADE
               )
        """,
        """
                CREATE TABLE IF NOT EXISTS exercise (
                    exercise_id SERIAL PRIMARY KEY,
                    name VARCHAR(255) NOT NULL,
                    description TEXT NOT NULL
                )
                """
    )

    try:
        cur = conn.cursor()
        for c in commands:
            cur.execute(c)
            print("Table was created successfully!")
        cur.close()
        conn.commit()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()

if __name__ == '__main__':
    create_tables()


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rcParams

config = {
    "font.family": 'serif',
    "font.size": 10,
    "mathtext.fontset": 'stix',
    "font.serif": ['Times New Roman'],
}
rcParams.update(config)

table = np.loadtxt('result2.csv', delimiter=',')

epoch = table[:, 0]
DCT = table[:, 1]
DCT_E = table[:, 2]
DCT_N = table[:, 3]

plt.figure(figsize=(12, 10))
plt.axis('off')
plt.subplot(2, 1, 1)
plt.title('Global', fontsize=15)
plt.xlabel('Epoch', fontsize=10)
plt.ylabel('Top1 Accuracy [%]', fontsize=20)
plt.plot(epoch, DCT, label='ResNet50', linewidth=2)
plt.plot(epoch, DCT_E, label='ResNet50-E', linewidth=2)
plt.plot(epoch, DCT_N, label='ResNet50-N', linewidth=2)
plt.legend(fontsize=20, loc='lower right')

plt.subplot(2, 1, 2)
plt.title('Partial', fontsize=15)
plt.xlabel('Epoch', fontsize=10)
plt.ylabel('Top1 Accuracy [%]', fontsize=20)
plt.plot(epoch[-3:], DCT[-3:], label='ResNet50', linewidth=2)
plt.plot(epoch[-3:], DCT_E[-3:], label='ResNet50-E', linewidth=2)
plt.plot(epoch[-3:], DCT_N[-3:], label='ResNet50-N', linewidth=2)
plt.legend(fontsize=20, loc='lower right')
plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.1)

plt.suptitle('ResNet Group', fontsize=20)
plt.savefig('out2.png', dpi=100)
plt.show()


from flask import Flask,request,render_template
import numpy as np
import pandas as pd

from sklearn.preprocessing import StandardScaler
from src.pipeline.predict_pipeline import CustomData,PredictPipeline

application=Flask(__name__)

app=application

## Route for a home page

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/predictdata',methods=['GET','POST'])
def predict_datapoint():
    if request.method=='GET':
        return render_template('home.html')
    else:
        data=CustomData(
            radius_mean=request.form.get('radius_mean'),
            concave_points_mean=request.form.get('concave_points_mean'),
            compactness_mean=request.form.get('compactness_mean'),
            concave_points_worst=request.form.get('concave_points_worst'),
            concavity_worst=request.form.get('concavity_worst'),
            compactness_worst=float(request.form.get('compactness_worst')),

        )
        pred_df=data.get_data_as_data_frame()
        print(pred_df)
        print("Before Prediction")

        predict_pipeline=PredictPipeline()
        print("Mid Prediction")
        results=predict_pipeline.predict(pred_df)
        print("after Prediction")
        if results[0] == 'B':
            result = 'Mammogram Results indicate Benign tumor'
        else:
            result = 'Mammogram Results indicate Malignant tumor'
        return render_template('home.html',results=result)

if __name__=="__main__":
    app.run(debug=True,host="0.0.0.0")


import numpy as np
from functools import partial
from menpo.feature import no_op
from menpofit.result import euclidean_bb_normalised_error

from .base import (BaseSupervisedDescentAlgorithm,
                   compute_parametric_delta_x, features_per_patch,
                   update_parametric_estimates, print_parametric_info,
                   build_appearance_model, fit_parametric_shape)
from menpo.model import PCAVectorModel
from menpofit.math import IIRLRegression, IRLRegression, OPPRegression
from menpofit.modelinstance import OrthoPDM
from menpofit.visualize import print_progress

class FullyParametricSDAlgorithm(BaseSupervisedDescentAlgorithm):
    r"""
    """

    def __init__(self, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel):
        super(FullyParametricSDAlgorithm, self).__init__()
        self.regressors = []
        self.shape_model_cls = shape_model_cls
        self.appearance_model_cls = appearance_model_cls
        self.appearance_model = None
        self.shape_model = None

    def _compute_delta_x(self, gt_shapes, current_shapes):
        # This is called first - so train shape model here
        if self.shape_model is None:
            self.shape_model = self.shape_model_cls(gt_shapes)

        return compute_parametric_delta_x(gt_shapes, current_shapes,
                                          self.shape_model)

    def _update_estimates(self, estimated_delta_x, delta_x, gt_x,
                          current_shapes):
        update_parametric_estimates(estimated_delta_x, delta_x, gt_x,
                                    current_shapes, self.shape_model)

    def _compute_training_features(self, images, gt_shapes, current_shapes,
                                   prefix='', verbose=False):
        if self.appearance_model is None:
            self.appearance_model = build_appearance_model(
                images, gt_shapes, self.patch_shape, self.patch_features,
                self.appearance_model_cls, verbose=verbose, prefix=prefix)

        wrap = partial(print_progress,
                       prefix='{}Extracting patches'.format(prefix),
                       end_with_newline=not prefix, verbose=verbose)

        features = []
        for im, shapes in wrap(zip(images, current_shapes)):
            for s in shapes:
                param_feature = self._compute_test_features(im, s)
                features.append(param_feature)

        return np.vstack(features)

    def _compute_parametric_features(self, patch):
        raise NotImplementedError()

    def _compute_test_features(self, image, current_shape):
        patch_feature = features_per_patch(
            image, current_shape, self.patch_shape, self.patch_features)
        return self._compute_parametric_features(patch_feature)

    def _print_regression_info(self, _, gt_shapes, n_perturbations,
                               delta_x, estimated_delta_x, level_index,
                               prefix=''):
        print_parametric_info(self.shape_model, gt_shapes, n_perturbations,
                              delta_x, estimated_delta_x, level_index,
                              self._compute_error, prefix=prefix)

    def run(self, image, initial_shape, gt_shape=None, **kwargs):
        return fit_parametric_shape(image, initial_shape, self,
                                    gt_shape=gt_shape)

class ParametricAppearanceProjectOut(FullyParametricSDAlgorithm):

    def _compute_parametric_features(self, patch):
        return self.appearance_model.project_out(patch.ravel())

class ParametricAppearanceWeights(FullyParametricSDAlgorithm):

    def _compute_parametric_features(self, patch):
        return self.appearance_model.project(patch.ravel())

class ParametricAppearanceMeanTemplate(FullyParametricSDAlgorithm):

    def _compute_parametric_features(self, patch):
        return patch.ravel() - self.appearance_model.mean().ravel()

class FullyParametricWeightsNewton(ParametricAppearanceWeights):
    r"""
    """

    def __init__(self, patch_features=no_op, patch_shape=(17, 17),
                 n_iterations=3, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel,
                 compute_error=euclidean_bb_normalised_error,
                 eps=10 ** -5, alpha=0, bias=True):
        super(FullyParametricWeightsNewton, self).__init__(
            shape_model_cls=shape_model_cls,
            appearance_model_cls=appearance_model_cls)

        self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias)
        self.patch_shape = patch_shape
        self.patch_features = patch_features
        self.n_iterations = n_iterations
        self._compute_error = compute_error
        self.eps = eps

class FullyParametricMeanTemplateNewton(ParametricAppearanceMeanTemplate):
    r"""
    """

    def __init__(self, patch_features=no_op, patch_shape=(17, 17),
                 n_iterations=3, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel,
                 compute_error=euclidean_bb_normalised_error,
                 eps=10 ** -5, alpha=0, bias=True):
        super(FullyParametricMeanTemplateNewton, self).__init__(
            shape_model_cls=shape_model_cls,
            appearance_model_cls=appearance_model_cls)

        self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias)
        self.patch_shape = patch_shape
        self.patch_features = patch_features
        self.n_iterations = n_iterations
        self._compute_error = compute_error
        self.eps = eps

class FullyParametricProjectOutNewton(ParametricAppearanceProjectOut):
    r"""
    """

    def __init__(self, patch_features=no_op, patch_shape=(17, 17),
                 n_iterations=3, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel,
                 compute_error=euclidean_bb_normalised_error,
                 eps=10 ** -5, alpha=0, bias=True):
        super(FullyParametricProjectOutNewton, self).__init__(
            shape_model_cls=shape_model_cls,
            appearance_model_cls=appearance_model_cls)

        self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias)
        self.patch_shape = patch_shape
        self.patch_features = patch_features
        self.n_iterations = n_iterations
        self._compute_error = compute_error
        self.eps = eps

# TODO: document me!
class FullyParametricProjectOutGaussNewton(ParametricAppearanceProjectOut):
    r"""
    """

    def __init__(self, patch_features=no_op, patch_shape=(17, 17),
                 n_iterations=3, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel,
                 compute_error=euclidean_bb_normalised_error,
                 eps=10 ** -5, alpha=0, bias=True, alpha2=0):
        super(FullyParametricProjectOutGaussNewton, self).__init__(
            shape_model_cls=shape_model_cls,
            appearance_model_cls=appearance_model_cls)

        self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias,
                                      alpha2=alpha2)
        self.patch_shape = patch_shape
        self.patch_features = patch_features
        self.n_iterations = n_iterations
        self._compute_error = compute_error
        self.eps = eps

class FullyParametricProjectOutOPP(ParametricAppearanceProjectOut):
    r"""
    """

    def __init__(self, patch_features=no_op, patch_shape=(17, 17),
                 n_iterations=3, shape_model_cls=OrthoPDM,
                 appearance_model_cls=PCAVectorModel,
                 compute_error=euclidean_bb_normalised_error,
                 eps=10 ** -5, bias=True):
        super(FullyParametricProjectOutOPP, self).__init__(
            shape_model_cls=shape_model_cls,
            appearance_model_cls=appearance_model_cls)

        self._regressor_cls = partial(OPPRegression, bias=bias)
        self.patch_shape = patch_shape
        self.patch_features = patch_features
        self.n_iterations = n_iterations
        self._compute_error = compute_error
        self.eps = eps


from flask import Flask, render_template, session, request
import secrets

app = Flask(__name__)
app.secret_key = secrets.token_bytes()

items = {5427: ("White Cat", 1024),
         5428: ("Orange Cat", 1024),
         5429: ("Ranibow Cat", 4096),
         5430: ("FLAG", 2147483648),
         5431: ("Evil Cat", 65536),
         5432: ("Null Cat", 0)}

@app.route("/")
def home():
    if 'money' not in session:
        session['money'] = 65536
    if 'stuff' not in session:
        session['stuff'] = []
    return render_template("index.html", items=items)

@app.route("/item/<int:item_id>")
def view_item(item_id):
    return render_template("item.html", item=items[item_id], item_id=item_id)

@app.route("/buy", methods=['POST'])
def buy_item():
    cost = int(request.form.get("cost"))
    item_id = int(request.form.get("item_id"))
    if cost > session['money']:
        return "<script>alert(`You don't have enough money Q_Q`); location.href='/';</script>"
    session['money'] -= cost
    session['stuff'].append(items[item_id])
    return "<script>alert(`Success!`); location='/'</script>"

if __name__ == "__main__":
    app.run(debug=True)