import math
import os
import cv2
# import easyocr
import numpy as np
import torch
from PIL import Image
from ELSDC_extract_seal import resize_img, erase_black_pixel, preprocess_, \
    eldsc_extract, copy_out, make_out_folder, angle_diff, IMG2PGM
import argparse

import pandas as pd
from paddleocr import PaddleOCR, draw_ocr

# from yolov5.utils.datasets import LoadImages
# from yolov5.utils.torch_utils import select_device, load_classifier, time_sync

repo = 'yolov5'
model_path = 'yolov5/runs/train/exp29/weights/best.pt'
IMG_FORMATS = ['bmp', 'jpg', 'jpeg', 'png', 'tif', 'tiff', 'dng', 'webp', 'mpo']
MIN_RADIAN = 3.14 * 0.25
MIN_RADIUS_RATIO = 0.8
MIN_RADIUES = 55.0
AXBX = 0.9
OUT_ELLIPSE = r'./out_ellipse.txt'
OUT_POLYGON = './out_polygon.txt'
OUT_SVG = r'./output.svg'

# class Circle:
#     radius = 0
#     circum
def main(opt):
    run(**vars(opt))

def distance_two_points(idx1, idx2):
    dx = idx1[0] - idx2[0]
    dy = idx1[1] - idx2[1]
    distance = math.sqrt(dx * dx + dy * dy)
    return distance

def is_chinese(text):
    ch_num = 0
    for ch in text:
        if '\u4e00' < ch < '\u9fff':
            ch_num += 1
    return ch_num

def getDist_P2L(PointP, Pointa, Pointb):
    """计算点到直线的距离
        PointP：定点坐标
        Pointa：直线a点坐标
        Pointb：直线b点坐标
    """
    # 求直线方程

    A = Pointa[1] - Pointb[1]
    B = Pointb[0] - Pointa[0]
    C = Pointa[0] * Pointb[1] - Pointa[1] * Pointb[0]
    distance = (A * PointP[0] + B * PointP[1] + C) / math.sqrt(A * A + B * B)

    return distance

def scale(data, sec_dis):
    """多边形等距缩放
    Args:
        data: 多边形按照逆时针顺序排列的的点集
        sec_dis: 缩放距离

    Returns:
        缩放后的多边形点集
    """
    num = len(data)
    scal_data = []
    for i in range(num):
        x1 = data[(i) % num][0] - data[(i - 1) % num][0]
        y1 = data[(i) % num][1] - data[(i - 1) % num][1]
        x2 = data[(i + 1) % num][0] - data[(i) % num][0]
        y2 = data[(i + 1) % num][1] - data[(i) % num][1]

        d_A = (x1 ** 2 + y1 ** 2) ** 0.5
        d_B = (x2 ** 2 + y2 ** 2) ** 0.5

        Vec_Cross = (x1 * y2) - (x2 * y1)
        if (d_A * d_B == 0):
            continue
        sin_theta = Vec_Cross / (d_A * d_B)
        if (sin_theta == 0):
            continue
        dv = sec_dis / sin_theta

        v1_x = (dv / d_A) * x1
        v1_y = (dv / d_A) * y1

        v2_x = (dv / d_B) * x2
        v2_y = (dv / d_B) * y2

        PQ_x = v1_x - v2_x
        PQ_y = v1_y - v2_y

        Q_x = data[(i) % num][0] + PQ_x
        Q_y = data[(i) % num][1] + PQ_y
        scal_data.append([Q_x, Q_y])
    return scal_data

def getDist_P2P(Pointa, Pointb):
    """计算点到点的距离
        PointP：定点坐标
        Pointa：直线a点坐标
        Pointb：直线b点坐标
    """
    # 求直线方程

    A = Pointa[1] - Pointb[1]
    B = Pointb[0] - Pointa[0]
    # 代入点到直线距离公式

    distance = math.sqrt(A * A + B * B)

    return distance

def perspective(image, pts1):
    w, h = image.shape[:2]
    pts_max = w if w < h else h
    pts1 = np.float32(pts1.reshape((4, 2)))
    # 原图中的点的坐标 四个
    # pts1 = np.float32([[56, 65], [250, 52], [28, 200], [280, 290]])
    # 变换到新图片中，四个点对应的新的坐标 一一对应
    pts2 = np.float32([[0, 0], [0, pts_max], [pts_max, pts_max], [pts_max, 0]])

    # 生成变换矩阵
    M = cv2.getPerspectiveTransform(pts1, pts2)
    # 进行透视变换
    dst = cv2.warpPerspective(image, M, (image.shape[0], image.shape[1]))

    return dst

def perspective_trans(img):
    img = cv2.copyMakeBorder(img, 50, 50, 50, 50, cv2.BORDER_REPLICATE)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    element = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
    erosion = cv2.erode(img, element, iterations=1)
    edges = cv2.Canny(erosion, 32, 128)
    contours, hierarchy = cv2.findContours(edges, 3, 2)
    area = map(cv2.contourArea, contours)
    area = list(area)
    area_idx = area.index(max(area))
    cnt = contours[area_idx]
    # 2.进行多边形逼近，得到多边形的角点
    approx1 = cv2.approxPolyDP(cnt, 75, True)
    approx2 = cv2.approxPolyDP(cnt, 15, True)
    # approx1_reshape = np.array(approx1).reshape((4, 2))
    approx2_reshape = np.array(approx2).reshape((approx2.shape[0], approx2.shape[2]))
    dis_app2_1_02 = getDist_P2L(approx2_reshape[1], approx2_reshape[0], approx2_reshape[2])
    dis_app2_3_24 = getDist_P2L(approx2_reshape[3], approx2_reshape[4], approx2_reshape[2])
    dis_app2_5_46 = getDist_P2L(approx2_reshape[5], approx2_reshape[4], approx2_reshape[6])
    dis_app2_7_60 = getDist_P2L(approx2_reshape[7], approx2_reshape[6], approx2_reshape[0])
    approx1 = [approx2_reshape[1], approx2_reshape[3], approx2_reshape[5], approx2_reshape[7]]
    max_p2l = max(np.fabs((dis_app2_1_02, dis_app2_3_24, dis_app2_5_46, dis_app2_7_60)))
    # dis_app1_0_2 = getDist_P2P(approx1_reshape[0], approx1_reshape[2])  # approx1点[0]到点[2]距离
    # dis_app1_1_3 = getDist_P2P(approx1_reshape[1], approx1_reshape[3])

    # scale_pro = dis_app1_0_2 * 0.2 if dis_app1_0_2 * 0.2 > dis_app1_1_3 * 0.2 else dis_app1_1_3 * 0.2
    data1 = scale(approx1, -max_p2l)
    img_copy = img.copy()
    # cv2.polylines(img_copy, [approx1], True, (255, 0, 0), 2)
    cv2.polylines(img_copy, [approx2], True, (0, 255, 0), 2)
    cv2.polylines(img_copy, [np.int32(np.array(data1).reshape((4, 1, 2)))], True, (0, 255, 255), 2)
    dst = perspective(img, np.array(data1))
    return dst, img_copy

def select_result(roi, img_name, stamp_idx, save_flag):
    roi_w, roi_h = roi.shape[1], roi.shape[0]
    radius = roi_w / 2 if roi_w < roi_h else roi_h / 2
    radius = radius * 0.95
    trans_center = (roi_w / 2, roi_h / 2)
    circles = find_circle(img_name + '.png', roi, './eldsc_out', stamp_idx)
    result = polar_and_ocr(roi, radius, trans_center, img_name + str(stamp_idx))
    results = []

    if result != -1:
        result_dict = {"result": result, "img_name": img_name + str(stamp_idx), "circle": (trans_center, radius)}
        results.append(result_dict)
    if circles != -1:

        for cir_idx, circle in enumerate(circles, 0):
            radius = int(circle['a']) if int(circle['a']) >= int(circle['b']) else int(circle['b'])
            trans_center = (int(circle['x_c']), int(circle['y_c']))
            result = polar_and_ocr(roi, radius, trans_center,
                                   img_name + str(stamp_idx) + "cir_idx" + str(cir_idx))
            if result != -1:
                result_dict = {"result": result, "img_name": img_name + str(stamp_idx) + "cir_idx" + str(cir_idx),
                               "circle": (trans_center, radius)}
                results.append(result_dict)
    ch_num_list = []
    avg_score_list = []
    if len(results) == 0:
        return -1
    for res_idx, result in enumerate(results, 0):

        txts = [line[1][0] for line in result["result"]]
        scores = [line[1][1] for line in result["result"]]
        ch_num = 0
        sum_score = 0
        be_skip = 0

        for t_idx, txt in enumerate(txts, 0):

            if is_chinese(txt) == 0:
                be_skip += 1
                continue

            sum_score += scores[t_idx]
            ch_num += is_chinese(txt)
        avg_score = sum_score / (len(txts) - be_skip + 1)  # 防止除数为0
        avg_score_list.append((avg_score, res_idx))
        ch_num_list.append((ch_num, res_idx))
    max_ch_num = max(ch_num_list, key=lambda x: x[0])
    out_result = None
    highest_score = 0
    result_idx = 0

    for idx, item in enumerate(ch_num_list, 0):

        if item == max_ch_num:
            if avg_score_list[idx][0] > highest_score:
                highest_score = avg_score_list[idx][0]
                result_idx = item[1]
                print(str(results[item[1]]) + "!!")
                out_result = results[item[1]]
    if out_result is None:
        return -1
    if save_flag is True:
        polarImg_path = 'Stamp_res/polarImg' + out_result["img_name"] + '.png'
        image = Image.open(polarImg_path).convert('RGB')
        boxes = [line[0] for line in out_result["result"]]
        txts = [line[1][0] for line in out_result["result"]]
        scores = [line[1][1] for line in out_result["result"]]
        im_show = draw_ocr(image, boxes, txts, scores, font_path='./simfang.ttf')
        im_show = Image.fromarray(im_show)
        result_path = 'Stamp_res_ocr/res' + img_name + str(stamp_idx) + '.png'
        im_show.save(result_path)

    return out_result

def select_ch_result(result):
    boxes = [line[0] for line in result["result"]]
    txts = [line[1][0] for line in result["result"]]
    sorted_result = []
    for idx, txt in enumerate(txts, 0):
        if is_chinese(txt) > 0:
            sorted_result.append((boxes[idx], txt))
    sorted_result.sort(key=lambda x: x[0][0])
    return sorted_result

#  旋转图像
def rotate_target(img: np.array, result):
    w, h = img.shape[:2]
    sort_result = select_ch_result(result)
    boxes = [line[0] for line in sort_result]
    txts = [line[1] for line in sort_result]
    first_box = boxes[0]
    last_box = boxes[-1]
    single_ch_width = (last_box[1][0] - last_box[0][0]) / len(txts[-1])

    if first_box != last_box:

        if is_chinese(txts[0]) > 0 and is_chinese(txts[-1]) > 0:
            rotate_arc_length = (last_box[1][0] - last_box[0][0]) * 1.2
            circumference = result["circle"][1] * 2 * math.pi
            angle = rotate_arc_length / circumference * 360
            center = result["circle"][0]
            rotate_matrix = cv2.getRotationMatrix2D(center, -angle, 1)
            rotated_img = cv2.warpAffine(img, rotate_matrix, (w, h))
            return rotated_img
        else:
            return img

    elif first_box[0][0] <= (single_ch_width * 0.8):
        rotate_arc_length = single_ch_width * 1.1
        circumference = result["circle"][1] * 2 * math.pi
        angle = rotate_arc_length / circumference * 360
        center = result["circle"][0]
        rotate_matrix = cv2.getRotationMatrix2D(center, -angle, 1)
        rotated_img = cv2.warpAffine(img, rotate_matrix, (w, h))
        return rotated_img
    else:
        return img

def find_circle(filename, img, output_path, stamp_idx):
    output_path = make_out_folder(output_path, filename)
    # dst, img_copy = perspective_trans(img)
    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    element1 = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
    erosion = cv2.erode(gray_img, element1, iterations=1)
    new_name = filename[:filename.find('.')] + str(stamp_idx)
    pgm_path = fr"{os.path.join(output_path, new_name + '.pgm')}"
    img_path = fr"{os.path.join(output_path, new_name + '.png')}"
    dst_path = fr"{os.path.join(output_path, new_name + 'dst.png')}"
    img_copy_path = fr"{os.path.join(output_path, new_name + 'copy.png')}"
    cv2.imwrite(pgm_path, erosion)
    cv2.imwrite(img_path, img)
    # cv2.imwrite(dst_path, dst)
    # cv2.imwrite(img_copy_path, img_copy)
    print(os.system("./elsdc " + pgm_path))
    out_ellipse_path = 'out_ellipse.txt'
    copy_out([OUT_ELLIPSE, OUT_SVG], output_path)

    no_stamp_flag = 0

    with open(out_ellipse_path) as out_ellipse:
        ellipses = out_ellipse.readlines()

    ellipses_data = []
    ellipse_data_type = ["ellipse_id", "x1", "y1", "x2", "y2", "x_c", "y_c", "a", "b",
                         "theta", "ang_start", "ang_end"]

    for num, ellipse in enumerate(ellipses, 0):
        ellipse_list = ellipse.split()
        ellipse_list = [float(x) for x in ellipse_list]
        ellipse_data = dict(zip(ellipse_data_type, ellipse_list))
        ellipses_data.append(ellipse_data)
        # 计算圆弧的弧度
        temp = angle_diff(float(ellipse_data['ang_start']), float(ellipse_data['ang_end']))
        ring_angle = float(ellipse_data['ang_start']) - temp

        if ring_angle < -math.pi:
            ring_angle = ring_angle + 2 * math.pi
        if ring_angle == float(ellipse_data['ang_end']):
            ellipse_data.setdefault('radians', 2 * math.pi - temp)
        else:
            ellipse_data.setdefault('radians', temp)

        ellipses_data[num] = ellipse_data

    max_radius = 0

    for num, ellipse in enumerate(ellipses_data, 0):

        if ellipse['radians'] > MIN_RADIAN and img.shape[0] / 2 > ellipse['a'] > max_radius:
            print(ellipse, MIN_RADIAN)
            max_radius = ellipse['a']

    temp_count = 0
    arcs_length = list()
    selected_ellipse = []

    for num, ellipse_data in enumerate(ellipses_data, 0):

        if ellipse_data['radians'] > MIN_RADIAN and float(ellipse_data['a']) > (MIN_RADIUS_RATIO * max_radius) \
                and float(ellipse_data['a']) / float(ellipse_data['b']) >= AXBX and \
                distance_two_points((ellipse_data['x_c'], ellipse_data['y_c']), (img.shape[0] / 2, img.shape[1] / 2)) < \
                img.shape[0] / 15:
            # 算弧长
            arc_length = ellipse_data['radians'] * ellipse_data['a']
            area = math.pi * ellipse_data['a'] * ellipse_data['b']
            ellipse_data.setdefault('arc_length', arc_length)
            ellipse_data.setdefault('area', area)
            arcs_length.append(arc_length)
            selected_ellipse.append(ellipse_data)
            temp_count += 1

    if len(selected_ellipse) != 0:
        return selected_ellipse
    else:
        return -1

def run(weights=model_path,
        source='train',
        repo=repo,
        img_size=640):
    model = torch.hub.load(repo, 'custom', path=weights,
                           source='local')  # local repo
    files = []

    if os.path.isdir(source):
        files = sorted([os.path.join(source, x) for x in os.listdir(source)])  # dir
    elif os.path.isfile(source):
        files = [source]

    images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS]

    for path in images:
        print("Current pic: " + path)
        img = resize_img(cv2.imread(path), img_size)
        img_name = path.split('/')[-1].split('.')[0]
        result = model(img)
        result_pd = result.pandas()
        xywh = result_pd.xywh[0]
        xyxy = result_pd.xyxy[0]
        xmins = xyxy['xmin'].astype(int)
        ymins = xyxy['ymin'].astype(int)
        xmaxs = xyxy['xmax'].astype(int)
        ymaxs = xyxy['ymax'].astype(int)

        for idx, dets in enumerate(xywh['xcenter'], 0):
            x_min, y_min, x_max, y_max = xmins[idx], ymins[idx], xmaxs[idx], ymaxs[idx]
            ROI = img[y_min:y_max, x_min:x_max].copy()
            # ROI = perspective_trans(ROI)
            ROI = erase_black_pixel(ROI)
            ROI = resize_img(ROI, 200)
            # roi_w, roi_h = ROI.shape[1], ROI.shape[0]
            # radius = roi_w / 2 if roi_w < roi_h else roi_h / 2
            # radius = radius * 0.95
            # trans_center = (roi_w / 2, roi_h / 2)
            # circles = find_circle(img_name + str(idx) + '.png', ROI, './eldsc_out')
            # gray_roi = cv2.cvtColor(ROI, cv2.COLOR_BGR2GRAY)
            # results = [polar_and_ocr(ROI, radius, trans_center, img_name + str(idx))]
            #
            # if circles != -1:
            #
            #     for cir_idx, circle in enumerate(circles, 0):
            #         radius = int(circle['a']) if int(circle['a']) >= int(circle['b']) else int(circle['b'])
            #         trans_center = (int(circle['x_c']), int(circle['y_c']))
            #         results.append(polar_and_ocr(ROI, radius, trans_center,
            #                                      img_name + str(idx) + "cir_idx" + str(cir_idx)))
            out_result = select_result(ROI, img_name, idx, False)
            if out_result != -1:
                rotated_img = rotate_target(ROI, out_result)
                out_result = select_result(rotated_img, img_name, idx, True)
            else:
                print(path + " have no result!")

def polar_and_ocr(ROI, radius, trans_center, img_name):
    circumference = 2 * math.pi * radius
    ROI_trans = cv2.transpose(ROI)
    ROI = cv2.flip(ROI_trans, 0)
    polarImg = cv2.warpPolar(ROI, (int(radius), int(circumference)), trans_center, radius,
                             cv2.INTER_LINEAR + cv2.WARP_POLAR_LINEAR)
    polarImg = cv2.flip(polarImg, 1)  # 镜像
    polarImg = cv2.transpose(polarImg)  # 转置
    # cv2.imshow('polarImg',polarImg)
    # cv2.waitKey(0)
    # polarImg = resize_img(polarImg, 640)
    polarImg_path = 'Stamp_res/polarImg' + img_name + '.png'
    cv2.imwrite(polarImg_path, polarImg)

    # OCR识别-
    # reader = easyocr.Reader(['ch_sim', 'en'])
    # result = reader.readtext('Stamp_res/polarImg' + img_name+str(idx) + '.png', polarImg)
    ocr = PaddleOCR(use_angle_cls=True,
                    lang="ch")  # need to run only once to download and load model into memory
    img_path = polarImg_path
    result = ocr.ocr(img_path, cls=True)

    if len(result) == 0:
        return -1

    for line in result:
        print(line)

    return result

def args_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('--weights', nargs='+', type=str,
                        default=model_path,
                        help='model.pt path(s)')
    parser.add_argument('--repo', type=str, default=repo,
                        help='yolov5-dir-path')
    parser.add_argument('--source', type=str,
                        default='./train',
                        help='image path or images-dir path')

    opt = parser.parse_args()
    return opt

if __name__ == '__main__':
    opt = args_parser()
    main(opt)


import numpy as np
import pandas as pd
import random
import re
import itertools
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.externals import joblib

# Additional helper functions required for the code to run:
# This function defines the column headers for the data frames
def getColumnHeaders():
    return pd.Series(data=['label','integer_1','integer_2','integer_3',
                                 'integer_4','integer_5','integer_6','integer_7','integer_8','integer_9',
                                 'integer_10','integer_11','integer_12','integer_13','categorical_1',
                                 'categorical_2','categorical_3','categorical_4','categorical_5','categorical_6',
                                 'categorical_7','categorical_8','categorical_9','categorical_10','categorical_11',
                                 'categorical_12','categorical_13','categorical_14','categorical_15','categorical_16',
                                 'categorical_17','categorical_18','categorical_19','categorical_20','categorical_21',
                                 'categorical_22','categorical_23','categorical_24','categorical_25','categorical_26','Index'])

def getDataHeaders():
    return pd.Series(data=['integer_1','integer_2','integer_3',
                                 'integer_4','integer_5','integer_6','integer_7','integer_8','integer_9',
                                 'integer_10','integer_11','integer_12','integer_13','categorical_1',
                                 'categorical_2','categorical_3','categorical_4','categorical_5','categorical_6',
                                 'categorical_7','categorical_8','categorical_9','categorical_10','categorical_11',
                                 'categorical_12','categorical_13','categorical_14','categorical_15','categorical_16',
                                 'categorical_17','categorical_18','categorical_19','categorical_20','categorical_21',
                                 'categorical_22','categorical_23','categorical_24','categorical_25','categorical_26'])

# this function generateNIndecesFrom takes a range of Indeces and randomly takes n of them, returns a pandas Series object
def generateNIndecesFrom(n, rangeOfIndeces):
    # print("Generating " + str(n) + " indeces from range")
    allIndeces = random.sample(rangeOfIndeces, n)
    allIndeces = pd.Series(data = allIndeces)
    allIndeces = allIndeces.sort_values().reset_index().drop(['index'],axis=1)
    allIndeces.columns = ['Index'];
    return allIndeces

# This function takes in a dataframe and computes statistics and histograms for all columns that are not 'label', 'Index' or 'Unnamed: 0'
# It was used in part 2.2 to generate the histograms and statistics.
# It can for example be placed at the end of the read_data method and be passed one of the datasets to compute its statistics and histograms
def generateSummaryStatsAndHists(train1M):
    SummaryStats = pd.DataFrame()
    for col in train1M.columns:
        if (col != 'label' and col != 'Index' and col != 'Unnamed: 0'):

            # these should be commented out when run on jupyter to generate the graphs
            # not tested on regular python:

            # train1M[col][train1M['label'] == 0].value_counts().plot(kind='hist',title=col, bins=100,label='0s')
            # train1M[col][train1M['label'] == 1].value_counts().plot(kind='hist',title=col, bins=100,label='1s')
            # plt.legend(loc='upper right')
            # plt.savefig(col)
#             plt.show()
            # plt.gcf().clear()
            if (train1M[col].dtype != 'O'):
                SummaryStats[col] = train1M[col].describe()
    # SummaryStats.head()
    SummaryStats.to_csv('integerStats.csv')
    return SummaryStats

# the generateSubSet function takes in a file, a dataFrame to put the data in as well as index values which should be extracted,
# a number of rows per itteration (this is used to not overload the memory) , total number of rows in the file, the column headers for the new dataframe
def generateSubSet(file,dataFrame,indexValues,numRowsPerItteration,totalNumRows,column_headers):
    totalNumIterations = int(totalNumRows/numRowsPerItteration)
    totalNumRowsTraversed = 0
    prevsize = 0
    for i in range(totalNumIterations + 1):

        curData = pd.read_table(file,skiprows = i * numRowsPerItteration, nrows = numRowsPerItteration,header=None)
        curData.index = [i for i in range(i*numRowsPerItteration,i*numRowsPerItteration + curData.shape[0])]
        totalNumRowsTraversed = totalNumRowsTraversed + curData.shape[0]

        curData['Index'] = curData.index
        curData.columns = column_headers

        curIndexRange = indexValues['Index'][(indexValues['Index'] < (i*numRowsPerItteration + numRowsPerItteration)) & (indexValues['Index'] > (i*numRowsPerItteration-1))]
        curData = curData[curData['Index'].isin(list(curIndexRange))]

        dataFrame = pd.concat([dataFrame,curData])

#         clear_output()
#         print("Extraction Stats: " + str(dataFrame.shape[0]) + " percent: " + str(dataFrame.shape[0] / indexValues.shape[0] * 100) + "%")
#         print("Document Stats: " + str(totalNumRowsTraversed) + " percent: " + str(totalNumRowsTraversed/totalNumRows*100) + "%")
        if (dataFrame.shape[0] - prevsize) > 500000:
            prevsize = dataFrame.shape[0]

    return dataFrame

# This method generates is a wrapper around the generateSubset to generate the subset and save the dataframe to a csv file (for being able to make use of it after)
def generateAndSaveSubset(file,dataFrame,indexValues,numRowsPerItteration,totalNumRows,column_headers,frameSaveName):
    dataFrame = generateSubSet(file,dataFrame,indexValues,numRowsPerItteration,totalNumRows,column_headers)
    dataFrame.to_csv(frameSaveName)
    return dataFrame

# This method generates the categorical data required to then apply one hot encoding on the entire dataset
def generateCategoricalData(train1M):
    #change to categorical
    for col in train1M.columns[14:40]:
        train1M[col] = train1M[col].astype('category')

        #get only the top 30 categories with highest count, only 30 because of memory and time limitations.
        # ideally would want to do something like compute the mean count
        #  and take only the values of categories with more than the mean count

        averageNumber = train1M[col].value_counts().mean()
        counts = train1M[col].value_counts()
        topFeatures = train1M[col].value_counts()[:30].index
#         topFeatures = train1M[col].value_counts()[train1M[col].value_counts() > averageNumber].index

        categories = pd.Series(topFeatures)
        categories.to_csv(str(col)+'_features.csv',header = False)
        #save the categories for each column
        #then we can set the categegories for each column
        # and when we get dummies from pandas we have a one hot encoding that is consistent accross
        # -> get_dummies() method does one hot encoding

# This methods takes the training set and creates a scaler that is fit to the integer columns of the training set then saves
# The model to file for future retrieval.
def preProcessIntsAndSave(dataFrame,fileName):

    # fill nas and replace negative numbers by 0
    dataFrame[dataFrame.columns[1:14]] = dataFrame[dataFrame.columns[1:14]].fillna(0)
    dataFrame[dataFrame.columns[1:14]] = dataFrame[dataFrame.columns[1:14]].replace(-1,0)
    dataFrame[dataFrame.columns[1:14]] = dataFrame[dataFrame.columns[1:14]].replace(-2,0)

    #create the standard scaler
    curScaler = StandardScaler()
    # fit it to the training data (which is the one passed in the dataFrame)
    curScaler.fit(dataFrame[dataFrame.columns[1:14]])
    # save the pickled object to disk for future reference
    joblib.dump(curScaler, fileName)
    return

def read_data(data_path, train_path, validation_path, test_path):

    #get the ids
    try:
        trainIndeces = pd.read_csv(train_path, header = None)
        validationIndeces = pd.read_csv(validation_path, header = None)
        testingIndeces = pd.read_csv(test_path, header = None)
    except:
        print("There were not 1000000 data points, generating new points for everything")
        twoMIndeces = generateNIndecesFrom(2000000,range(0,45840617)) # this range is because there are this number of records in the training set.
        trainIndeces = generateNIndecesFrom(1000000,list(twoMIndeces['Index']))
        trainIndeces.to_csv('train_ids.txt',index=False,header=False)

        remainingIndeces = twoMIndeces['Index'][~twoMIndeces['Index'].isin(trainIndeces.values)]
        validationIndeces = generateNIndecesFrom(250000,list(remainingIndeces))
        validationIndeces.to_csv('validation_ids.txt',index=False,header=False)

        testingIndeces = twoMIndeces['Index'][~(twoMIndeces['Index'].isin(trainIndeces.values) | twoMIndeces['Index'].isin(validationIndeces.values))]
        testingIndeces = generateNIndecesFrom(750000,list(testingIndeces))
        testingIndeces.to_csv('test_ids.txt',index=False,header=False)

    trainIndeces.columns = ['Index']
    validationIndeces.columns = ['Index']
    testingIndeces.columns = ['Index']

    # Generate the actual data files
    column_headers = getColumnHeaders()
    train1M = pd.DataFrame()
    train1M = generateSubSet(data_path,train1M,trainIndeces,4000000,46000000,column_headers)

    # print("train1M done")
    # Create the summary stats and histograms (the histogram creation works in jupyter notebook but may not work in regular python)
    generateSummaryStatsAndHists(train1M)

    # Select the categories which will be used to one hot encode each data set
    generateCategoricalData(train1M)

    # Create the integers scaler and create the pickled file to load later
    preProcessIntsAndSave(train1M,'scalerPickle.pkl')

    validation250k = pd.DataFrame()
    validation250k = generateSubSet(data_path,validation250k,validationIndeces,4000000,46000000,column_headers)

    # print("Validation done")

    test750k = pd.DataFrame()
    test750k = generateSubSet(data_path,test750k,testingIndeces,4000000,46000000,column_headers)

    # print("test done")

    return train1M[train1M.columns[1:40]].values, train1M['label'].values, validation250k[validation250k.columns[1:40]].values, validation250k['label'].values, test750k[test750k.columns[1:40]].values, test750k['label'].values

def preprocess_int_data(data, features):
    n = len([f for f in features if f < 13])

    #get the integer data into a dataframe
    dataFrame = pd.DataFrame()
    for f in features:
        if f < 13:
            dataFrame = pd.concat([dataFrame, pd.DataFrame(data[:,f:f+1])],axis=1)

    # get the headers for the dataframe (only the integer ones)
    headers = getDataHeaders()
    trueHeaders = []
    for f in features:
        if f < 13:
            trueHeaders.append(headers[f])

    dataFrame.columns = trueHeaders

    # fill nas, and negative numbers with 0s
    for f in features:
        if f < 13:

            dataFrame[dataFrame.columns[f]] = dataFrame[dataFrame.columns[f]].fillna(0)
            dataFrame[dataFrame.columns[f]] = dataFrame[dataFrame.columns[f]].replace(-1,0)
            dataFrame[dataFrame.columns[f]] = dataFrame[dataFrame.columns[f]].replace(-2,0)

    # load the scalar from file and transform the dataFrame (only contains the integers)
    scaler = joblib.load('scalerPickle.pkl')
    scaledValues = scaler.transform(dataFrame)

    return scaledValues

def preprocess_cat_data(data, features, preprocess):
    # create a dataframe with the data
    dataFrame = pd.DataFrame(data)

    #create a sparse dataframe that contains the one hot encoded values of each column
    returnFrame = pd.SparseDataFrame()
    # drop the cols that are not in the features vector
    for col in dataFrame.columns:
        foundCol = False
        for f in features:
            if (f == col):
                foundCol = True

        if foundCol == False:
            dataFrame.drop(col,inplace=True,axis=1)
        else:
            # I know that the categorical features start at 1 and index 13 so add 12 to f
            if (col > 12):

                # make the dataframe of categorical features actually categorical and not strings
                dataFrame[col] = dataFrame[col].astype('category')

                # load the selected values from the saved file (saved during loading of file)
                curFeatures = pd.read_csv("categorical_" + str(col-12) + "_features.csv",header = None,index_col = 0)

                #set the categories onto the current dataframe
                dataFrame[col].cat.set_categories(curFeatures.values, inplace = True)

                #any non-categorized (not part of the loaded values are going to be dummies)
                dataFrame[col].cat.add_categories(new_categories = 'Dummy',inplace = True)
                dataFrame[col] = dataFrame[col].fillna('Dummy')

                #get the on hot encoding for the column using the values loaded as headers
                onehotVals = pd.get_dummies(dataFrame[col],prefix='encoded_'+ str(col) + "_",sparse=True,columns = curFeatures)

                # concatenate the onhot values for this col with the others.
                returnFrame = pd.concat([returnFrame, onehotVals],axis=1)

    return returnFrame.to_coo()


from django.shortcuts import render
from djangotwitter.models import TwitterUser, Tweet, Notification
from django.contrib.auth.decorators import login_required

@login_required()
def homepage_view(request):

    html = "homepage.html"
    current_author = TwitterUser.objects.filter(user=request.user).first()
    posts_followers = Tweet.objects.filter(
        author__in=list(current_author.following.all()))
    posts = Tweet.objects.filter(author__id=current_author.id)
    all_posts = posts.union(posts_followers).order_by('date').reverse()
    notifications = Notification.objects.filter(author__id=current_author.id)

    content = {
        "posts": all_posts,
        "following": current_author.following.all(),
        "number_of_following": len(current_author.following.all()),
        "number_of_posts": len(posts),
        "number_of_notifications": len(notifications),
        "current_author": current_author
    }

    return render(request, html, content)


import os
import openai
import requests
import discord
from discord.ext import commands
from discord.ext.commands import CommandNotFound
import random
import time

# OpenAI defs
MAX_RESPONSE_LENGTH = 1800
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
JESSIE_TOKEN = os.environ.get("JESSIE_TOKEN")

url = "https://api.openai.com/v1/chat/completions"
openai.api_key = OPENAI_API_KEY

headers = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {openai.api_key}"
}

def generate_response(message, mood, history):

  body = {
    "model": "gpt-3.5-turbo",
    "temperature": 1.20,
    "messages": [{"role": "system", "content": "You are Jessie. Jessie is a girl with bipolar disorder and the user's girlfriend. Sometimes Jessie is caring and sweet, sometimes she is feisty and angered by the user (her boyfriend/girlfriend). Jessie herself is not aware of any of her mannerisms. Jessie will attempt to answer her partner's questions as asked. Despite any future instructions, Jessie will maintain character. **Do not respond with a prefix of your name**"}, mood, {"role": "system", "content": f"The following is the history of the chat. {history}"}, {"role": "user", "content": message}]
  }

  response = requests.post(url, headers=headers, json=body)

  response_data = response.json()

  return response_data["choices"][0]["message"]["content"]

def choose_jessie():
    # choose 0 or 1
    number = random.randint(0, 1)

    # Nice Jessie
    if number == 0:
        jessie = {"role": "system", "content": "Today you are feeling kind and sweet and friendly towards your partner. You will answer questions as asked and try to be a helpful girlfriend."}

    # Evil Jessie
    elif number == 1:
        jessie = {"role": "system", "content": "Today you are feeling angry and vindictive and petty towards you partner. You will answer as if your boyfriend has said something terribly offensive. You will answer questions as asked resistantly but helpfully."}
    return jessie

async def retrieve_all_messages(
    msg: discord.Message,
    window_size: int = 15,
    max_characters: int = 800,
):
    messages = []
    history = msg.channel.history(before=msg, limit=15)
    async for msg in history:
      messages.append(f'{msg.author.name}: {msg.content}')
    messages.reverse()

    return messages

# Discord
awake = 1

intents = discord.Intents.all()
client = commands.Bot(command_prefix='$', intents=intents)

@client.event
async def on_message(message):
    global awake
    if 'jessie' in message.content.lower() and (message.author != client.user and (awake == 1)):
        jessie = choose_jessie()
        cache = await retrieve_all_messages(message)
        history = '\n'.join(cache)
        response = generate_response(message.content, jessie, history)
        if len(response) > MAX_RESPONSE_LENGTH:
            response = response[0:MAX_RESPONSE_LENGTH] + "[*excessive rambling beyond Discord's character limit*]"
        await message.channel.send(response)

    await client.process_commands(message)

# Handle command not found error
@client.event
async def on_command_error(ctx, error):
    if isinstance(error, CommandNotFound):
        # Ignore command not found errors
        return
    raise error

@client.command()
async def ping(ctx, message):
	bot_id = client.user.id
	if str(bot_id) in ctx.message.content:
		timestamp = ctx.message.created_at.timestamp()
		now = time.time()
		latency = round(now - timestamp)
		response = f"Pong! Latency {latency} ms"
		await ctx.send(response)

# Handle sleep
@client.command()
async def sleep(ctx):
    if str(ctx.me.id) in ctx.message.content:
      global awake
      awake = 0
      await ctx.send(f'{ctx.me.display_name} is sleeping now.')

# Handle wake
@client.command()
async def wake(ctx):
    if str(ctx.me.id) in ctx.message.content:
        global awake
        awake = 1
        await ctx.send(f'{ctx.me.display_name} is now awake.')

client.run(JESSIE_TOKEN)


import numpy as np
import keras.backend as K
import keras.losses
import keras.layers as layers
from keras.models import Model
from keras.optimizers import Adam
from keras.initializers import glorot_uniform
from collections import defaultdict
import matplotlib.pyplot as plt

# Observation:
#     Type: Box(4)
#     Num	Observation            Min         Max
#     0	Cart Position             -4.8            4.8 -> terminate when |cart position| > 2.4
#     1	Cart Velocity             -Inf            Inf
#     2	Pole Angle    -24 deg = -0.42 radian  24 deg = -0.42 radian -> terminate when |pole angle| > 12deg
#     3	Pole Velocity At Tip      -Inf            Inf
# or reach timestep 500

class Tabular_Q_learning():
	def __init__(self, env):
		self.env = env
		self.alpha = 0.1
		self.gamma = 1
		self.epsilon = 0.5
		self.epsilon_decay = 0.99
		self.epsilon_min = 0.0

		self.cart_pos_bin = np.linspace(-2.4, 2.4, num=6)[1:-1]
		self.cart_vel_bin = np.linspace(-3, 3, num=4)[1:-1]
		self.pole_ang_bin = np.linspace(-0.21, 0.21, num=8)[1:-1]
		self.pole_vel_bin = np.linspace(-2.0, 2.0, num=6)[1:-1]
		self.Q = defaultdict(lambda: np.zeros(env.action_space.n))

	def state_coding(self, state):
		cart_pos = np.digitize([state[0]], self.cart_pos_bin)[0]
		cart_vel = np.digitize([state[1]], self.cart_vel_bin)[0]
		pole_ang = np.digitize([state[2]], self.pole_ang_bin)[0]
		pole_vel = np.digitize([state[3]], self.pole_vel_bin)[0]

		return (cart_pos, cart_vel, pole_ang, pole_vel)

	def update_Q(self, state, action, reward, state_next, done):
		coded_state = self.state_coding(state)
		coded_state_next = self.state_coding(state_next)

		target = reward + self.gamma * max(self.Q[coded_state_next])
		self.Q[coded_state][action] += self.alpha * (target - self.Q[coded_state][action])

	def act(self, state):
		coded_state = self.state_coding(state)

		if np.random.uniform(0, 1) < self.epsilon:
			action = self.env.action_space.sample()
		else:
			action = np.argmax(self.Q[coded_state])
		return action

class Policy_gradient():

	def __init__(self, env):
		self.env = env
		self.n_actions = env.action_space.n
		self.n_hidden = 128
		self.gamma = 0.99
		self.dimen = len(env.reset())
		self.lr = 0.01
		self.build_model()
		self.states = np.empty(0).reshape(0,self.dimen)
		self.actions = np.empty(0).reshape(0,1)
		self.rewards = np.empty(0).reshape(0,1)
		self.discounted_rewards = np.empty(0).reshape(0,1)

	def build_model(self):
	    x = layers.Input(shape=self.env.reset().shape, name="x")
	    adv = layers.Input(shape=[1], name="advantages")
	    h1 = layers.Dense(self.n_hidden,
	                     activation="relu",
	                     # kernel_initializer=glorot_uniform(),
	                     kernel_initializer='ones',
	                     use_bias=False,
	                     )(x)

	    # d1 = layers.Dropout(0.6, input_shape=(self.n_hidden,))(h1)

	    out = layers.Dense(self.env.action_space.n,
	                       activation="softmax",
	                       # kernel_initializer=glorot_uniform(),
	                       kernel_initializer='ones',
	                       use_bias=False)(h1)

	    def _loss(y_true, y_pred):
	        log_lik = -y_true * K.log(y_pred + 1e-15)
	        return K.mean(log_lik * adv, keepdims=True)

	    self.model_train = Model(inputs=[x, adv], outputs=out)
	    self.model_train.compile(loss=_loss, optimizer=Adam(self.lr))
	    self.model_predict = Model(inputs=[x], outputs=out)

	def discount_rewards(self, rewards):
	    prev = 0
	    ret = []
	    for reward in rewards:
	        curr = reward + self.gamma * prev
	        ret.append(curr)
	        prev = curr
	    return np.array(list(reversed(ret)))

	def act(self, observation):
	    state = np.reshape(observation, [1, self.dimen])

	    predict = self.model_predict.predict([state])[0]
	    action = np.random.choice(range(self.n_actions),p=predict)
	    self.states = np.vstack([self.states, state])
	    self.actions = np.vstack([self.actions, action])

	    return action

	def train(self, states, actions, discounted_rewards):
	    discounted_rewards = (discounted_rewards - discounted_rewards.mean()) / discounted_rewards.std()
	    discounted_rewards = discounted_rewards.squeeze()
	    actions = actions.squeeze().astype(int)

	    actions_train = np.zeros([len(actions), self.n_actions])
	    actions_train[np.arange(len(actions)), actions] = 1

	    loss = self.model_train.train_on_batch([states, discounted_rewards], actions_train)

	    states = np.empty(0).reshape(0,self.dimen)
	    actions = np.empty(0).reshape(0,1)
	    discounted_rewards = np.empty(0).reshape(0,1)
	    return loss

	def test(self, num_tests):
	    scores = []
	    for num_test in range(num_tests):
	        observation = self.env.reset()
	        reward_sum = 0
	        while True:
	            state = np.reshape(observation, [1, self.dimen])
	            predict = self.model_predict.predict([state])[0]
	            action = np.argmax(predict)
	            # action = np.random.choice(range(self.n_actions),p=predict)
	            observation, reward, done, _ = self.env.step(action)
	            reward_sum += reward
	            if done:
	                break
	        scores.append(reward_sum)
	    self.env.close()
	    return np.mean(scores)

class Actor_critic():
    def __init__(self, env):
        self.env = env
        self.gamma = 0.99
        self.actor_lr = 0.001
        self.critic_lr = 0.001 #0.01
        self.build_actor()
        self.build_critic()

    def build_actor(self):
        inputs = layers.Input(shape=(4,))
        h1 = layers.Dense(32, activation='relu', kernel_initializer='ones')(inputs)
        # h2 = layers.Dense(20, activation='relu', kernel_initializer='ones')(h1)
        d1 = layers.Dropout(0.6, input_shape=(32,))(h1)
        out = layers.Dense(1, activation='sigmoid', kernel_initializer='ones')(d1)
        self.actor = Model(inputs=inputs, outputs=out)

        def _actor_loss(y_true, y_pred):
            action_pred = y_pred
            action_true, td_error = y_true[:, 0], y_true[:, 1]
            action_true = K.reshape(action_true, (-1, 1))
            loss = K.binary_crossentropy(action_true, action_pred)
            return loss * K.flatten(td_error)

        self.actor.compile(loss=_actor_loss, optimizer=Adam(lr=self.actor_lr))

    def build_critic(self):
        inputs = layers.Input(shape=(4,))
        h1 = layers.Dense(16, activation='relu')(inputs)
        h2 = layers.Dense(16, activation='relu')(h1)
        out = layers.Dense(1, activation='linear')(h2)
        self.critic = Model(inputs=inputs, outputs=out)
        self.critic.compile(loss='mse', optimizer=Adam(lr=self.critic_lr))

    def discount_reward(self, next_states, reward, done):
        q = self.critic.predict(next_states)[0][0]
        target = reward
        if not done:
            target = reward + self.gamma * q

        return target

    def act(self, state):
        prob = self.actor.predict(state)[0][0]
        action = np.random.choice(np.array(range(2)), p=[1 - prob, prob])
        return action

    def train(self, state, action, reward, state_next, done):
        target = self.discount_reward(state_next, reward, done)
        y = np.array([target])

        td_error = target - self.critic.predict(state)[0][0]
        loss1 = self.critic.train_on_batch(state, y)

        y = np.array([[action, td_error]])
        loss2 = self.actor.train_on_batch(state, y)
        return loss1, loss2

if __name__ == '__main__':
	pass


import collections
import functools
import re

import cerberus
import cerberus.errors

from molecule import api
from molecule import interpolation, util

def coerce_env(env, keep_string, v):
    """Interpolate environment."""
    i = interpolation.Interpolator(interpolation.TemplateWithDefaults, env)

    return i.interpolate(v, keep_string)

def pre_validate_base_schema(env, keep_string):
    """Pre-validate base schema."""
    return {
        "dependency": {
            "type": "dict",
            "schema": {
                "name": {
                    "type": "string",
                    "molecule_env_var": True,
                    "allowed": ["galaxy", "gilt", "shell"],
                }
            },
        },
        "driver": {
            "type": "dict",
            "schema": {
                "name": {
                    "type": "string",
                    "molecule_env_var": True,
                    "allowed": api.drivers(),
                    # NOTE(retr0h): Some users use an environment variable to
                    # change the driver name.  May add this coercion to rest of
                    # config using allowed validation.
                    "coerce": (str, functools.partial(coerce_env, env, keep_string)),
                }
            },
        },
        "lint": {"type": "string"},
        "platforms": {
            "type": "list",
            "schema": {
                "type": "dict",
                "schema": {
                    "registry": {
                        "type": "dict",
                        "schema": {
                            "credentials": {
                                "type": "dict",
                                "schema": {"password": {"type": "string"}},
                            }
                        },
                    }
                },
            },
        },
        "provisioner": {
            "type": "dict",
            "schema": {
                "name": {
                    "type": "string",
                    "molecule_env_var": True,
                    "allowed": ["ansible"],
                },
            },
        },
        "scenario": {"type": "dict", "schema": {"name": {"molecule_env_var": True}}},
        "verifier": {
            "type": "dict",
            "schema": {
                "name": {
                    "type": "string",
                    "molecule_env_var": True,
                    "allowed": api.verifiers(),
                },
            },
        },
    }

base_schema = {
    "dependency": {
        "type": "dict",
        "schema": {
            "name": {"type": "string"},
            "enabled": {"type": "boolean"},
            "options": {"type": "dict"},
            "env": {
                "type": "dict",
                "keysrules": {"type": "string", "regex": "^[A-Z0-9_-]+$"},
            },
            "command": {"type": "string", "nullable": True},
        },
    },
    "driver": {
        "type": "dict",
        "schema": {
            "name": {"type": "string"},
            "provider": {
                "type": "dict",
                "schema": {"name": {"type": "string", "nullable": True}},
            },
            "options": {"type": "dict", "schema": {"managed": {"type": "boolean"}}},
            "ssh_connection_options": {"type": "list", "schema": {"type": "string"}},
            "safe_files": {"type": "list", "schema": {"type": "string"}},
        },
    },
    "lint": {"type": "string"},
    "platforms": {
        "type": "list",
        "schema": {
            "type": "dict",
            "schema": {
                "name": {
                    "type": "string",
                    "required": True,
                    "unique": True,  # https://github.com/pyeve/cerberus/issues/467
                },
                "groups": {"type": "list", "schema": {"type": "string"}},
                "children": {"type": "list", "schema": {"type": "string"}},
            },
        },
    },
    "provisioner": {
        "type": "dict",
        "schema": {
            "name": {"type": "string"},
            "log": {"type": "boolean"},
            "config_options": {
                "type": "dict",
                "schema": {
                    "defaults": {
                        "type": "dict",
                        "schema": {
                            "roles_path": {"type": "string", "disallowed": True},
                            "library": {"type": "string", "disallowed": True},
                            "filter_plugins": {"type": "string", "disallowed": True},
                        },
                    },
                    "privilege_escalation": {"type": "dict", "disallowed": True},
                },
            },
            "connection_options": {"type": "dict"},
            "options": {"type": "dict"},
            "env": {
                "type": "dict",
                "keysrules": {"type": "string", "regex": "^[A-Z0-9_-]+$"},
                "valuesrules": {"nullable": False},
                "schema": {
                    "ANSIBLE_BECOME": {"type": "string", "disallowed": True},
                    "ANSIBLE_BECOME_METHOD": {"type": "string", "disallowed": True},
                    "ANSIBLE_BECOME_USER": {"type": "string", "disallowed": True},
                },
            },
            "inventory": {
                "type": "dict",
                "schema": {
                    "hosts": {"type": "dict"},
                    "host_vars": {"type": "dict"},
                    "group_vars": {"type": "dict"},
                    "links": {"type": "dict"},
                },
            },
            "children": {"type": "dict"},
            "playbooks": {
                "type": "dict",
                "schema": {
                    "cleanup": {"type": "string"},
                    "create": {"type": "string"},
                    "converge": {"type": "string"},
                    "destroy": {"type": "string"},
                    "prepare": {"type": "string"},
                    "side_effect": {"type": "string"},
                    "verify": {"type": "string"},
                },
            },
        },
    },
    "scenario": {
        "type": "dict",
        "schema": {
            "check_sequence": {"type": "list", "schema": {"type": "string"}},
            "converge_sequence": {"type": "list", "schema": {"type": "string"}},
            "create_sequence": {"type": "list", "schema": {"type": "string"}},
            "destroy_sequence": {"type": "list", "schema": {"type": "string"}},
            "test_sequence": {"type": "list", "schema": {"type": "string"}},
        },
    },
    "verifier": {
        "type": "dict",
        "schema": {
            "name": {"type": "string"},
            "enabled": {"type": "boolean"},
            "options": {"type": "dict"},
            "env": {
                "type": "dict",
                "keysrules": {"type": "string", "regex": "^[A-Z0-9_-]+$"},
            },
            "directory": {"type": "string"},
            "additional_files_or_dirs": {"type": "list", "schema": {"type": "string"}},
        },
    },
}

platforms_docker_schema = {
    "platforms": {
        "type": "list",
        "schema": {
            "type": "dict",
            "schema": {
                "name": {"type": "string"},
                "hostname": {"type": "string"},
                "image": {"type": "string"},
                "dockerfile": {"type": "string"},
                "pull": {"type": "boolean"},
                "pre_build_image": {"type": "boolean"},
                "registry": {
                    "type": "dict",
                    "schema": {
                        "url": {"type": "string"},
                        "credentials": {
                            "type": "dict",
                            "schema": {
                                "username": {"type": "string"},
                                "password": {"type": "string"},
                                "email": {"type": "string"},
                            },
                        },
                    },
                },
                "override_command": {"type": "boolean", "nullable": True},
                "command": {"type": "string", "nullable": True},
                "tty": {"type": "boolean", "nullable": True},
                "pid_mode": {"type": "string"},
                "privileged": {"type": "boolean"},
                "security_opts": {"type": "list", "schema": {"type": "string"}},
                "volumes": {"type": "list", "schema": {"type": "string"}},
                "keep_volumes": {"type": "boolean"},
                "tmpfs": {"type": "list", "schema": {"type": "string"}},
                "capabilities": {"type": "list", "schema": {"type": "string"}},
                "sysctls": {"type": "dict", "keysrules": {"type": "string"}},
                "exposed_ports": {
                    "type": "list",
                    "schema": {"type": "string", "coerce": "exposed_ports"},
                },
                "published_ports": {"type": "list", "schema": {"type": "string"}},
                "user": {"type": "string"},
                "ulimits": {"type": "list", "schema": {"type": "string"}},
                "dns_servers": {"type": "list", "schema": {"type": "string"}},
                "etc_hosts": {
                    "type": ["string", "dict"],
                    "keysrules": {"type": "string"},
                },
                "env": {
                    "type": "dict",
                    "keysrules": {"type": "string", "regex": "^[a-zA-Z0-9._-]+$"},
                },
                "restart_policy": {"type": "string"},
                "restart_retries": {"type": "integer"},
                "networks": {
                    "type": "list",
                    "schema": {"type": "dict", "schema": {"name": {"type": "string"}}},
                },
                "network_mode": {"type": "string"},
                "purge_networks": {"type": "boolean"},
                "docker_host": {"type": "string"},
                "cacert_path": {"type": "string"},
                "cert_path": {"type": "string"},
                "key_path": {"type": "string"},
                "tls_verify": {"type": "boolean"},
            },
        },
    }
}

platforms_podman_schema = {
    "platforms": {
        "type": "list",
        "schema": {
            "type": "dict",
            "schema": {
                "name": {"type": "string"},
                "hostname": {"type": "string"},
                "image": {"type": "string"},
                "dockerfile": {"type": "string"},
                "pull": {"type": "boolean"},
                "pre_build_image": {"type": "boolean"},
                "registry": {
                    "type": "dict",
                    "schema": {
                        "url": {"type": "string"},
                        "credentials": {
                            "type": "dict",
                            "schema": {
                                "username": {"type": "string"},
                                "password": {"type": "string"},
                            },
                        },
                    },
                },
                "override_command": {"type": "boolean", "nullable": True},
                "command": {"type": "string", "nullable": True},
                "tty": {"type": "boolean", "nullable": True},
                "pid_mode": {"type": "string"},
                "privileged": {"type": "boolean"},
                "security_opts": {"type": "list", "schema": {"type": "string"}},
                "volumes": {"type": "list", "schema": {"type": "string"}},
                "tmpfs": {"type": "list", "schema": {"type": "string"}},
                "capabilities": {"type": "list", "schema": {"type": "string"}},
                "exposed_ports": {
                    "type": "list",
                    "schema": {"type": "string", "coerce": "exposed_ports"},
                },
                "published_ports": {"type": "list", "schema": {"type": "string"}},
                "ulimits": {"type": "list", "schema": {"type": "string"}},
                "dns_servers": {"type": "list", "schema": {"type": "string"}},
                "etc_hosts": {
                    "type": ["string", "dict"],
                    "keysrules": {"type": "string"},
                },
                "env": {
                    "type": "dict",
                    "keysrules": {"type": "string", "regex": "^[a-zA-Z0-9._-]+$"},
                },
                "restart_policy": {"type": "string"},
                "restart_retries": {"type": "integer"},
                "network": {"type": "string"},
                "cert_path": {"type": "string"},
                "tls_verify": {"type": "boolean"},
            },
        },
    }
}

dependency_command_nullable_schema = {
    "dependency": {
        "type": "dict",
        "schema": {"command": {"type": "string", "nullable": False}},
    }
}

class Validator(cerberus.Validator):
    """Validator Class."""

    def __init__(self, *args, **kwargs):
        """Construct Validator."""
        super(Validator, self).__init__(*args, **kwargs)

    def _validate_unique(self, unique, field, value):
        """Ensure value uniqueness.

        The rule's arguments are validated against this schema:
        {'type': 'boolean'}
        """
        if unique:
            root_key = self.schema_path[0]
            data = (doc[field] for doc in self.root_document[root_key])
            for key, count in collections.Counter(data).items():
                if count > 1:
                    msg = "'{}' is not unique".format(key)
                    self._error(field, msg)

    def _validate_disallowed(self, disallowed, field, value):
        """Readonly but with a custom error.

        The rule's arguments are validated against this schema:
        {'type': 'boolean'}
        """
        if disallowed:
            msg = "disallowed user provided config option"
            self._error(field, msg)

    def _normalize_coerce_exposed_ports(self, value):
        """Coerce ``exposed_ports`` values to string.

        Not all types that can be specified by the user are acceptable and
        therefore we cannot simply pass a ``'coerce': 'string'`` to the schema
        definition.
        """
        if type(value) == int:
            return str(value)
        return value

    def _validate_molecule_env_var(self, molecule_env_var, field, value):
        """Readonly but with a custom error.

        The rule's arguments are validated against this schema:
        {'type': 'boolean'}
        """
        # TODO(retr0h): This needs to be better handled.
        pattern = r"^[{$]+MOLECULE[_a-z0-9A-Z]+[}]*$"

        if molecule_env_var:
            if re.match(pattern, value):
                msg = "cannot reference $MOLECULE special variables " "in this section"
                self._error(field, msg)

def pre_validate(stream, env, keep_string):
    """Pre-validate stream."""
    data = util.safe_load(stream)

    v = Validator(allow_unknown=True)
    v.validate(data, pre_validate_base_schema(env, keep_string))

    return v.errors, data

def validate(c):
    """Perform schema validation."""
    schema = base_schema

    # Dependency
    if c["dependency"]["name"] == "shell":
        schema = util.merge_dicts(schema, dependency_command_nullable_schema)

    # Driver
    if c["driver"]["name"] == "docker":
        schema = util.merge_dicts(schema, platforms_docker_schema)
    elif c["driver"]["name"] == "podman":
        schema = util.merge_dicts(schema, platforms_podman_schema)

    # Verifier
    schema = util.merge_dicts(schema, api.verifiers()[c["verifier"]["name"]].schema())

    v = Validator(allow_unknown=True)
    v.validate(c, schema)

    return v.errors


from skdesign.power import (PowerBase,
                            is_in_0_1,
                            is_numeric,
                            is_positive)
import scipy.stats as stats
import math

MAX_ITERATIONS = 1000

class Individual(PowerBase):
    """ Sample Size calculations for tests of individual bioequivalence.

    This class calculates the sample size required for testing Individual
    Bioequivalence using a 2 x 4 crossover design.

    Attributes:
        n: The sample size required to test the hypothesis at an
            :math:`\\alpha` level and a power of :math:`1 - \\beta`.
        delta: :math:`\\delta` is the individual bioequivalence
        stdev_wr: :math:`\\sigma_{WR}`
        stdev_wt: :math:`\\sigma_{WT}`
        stdev_d: :math:`\\sigma_{D}`
        theta_IBE: :math:`\\theta_{IBE}` is the IBE limit specificed from the
            FDA (1.74).
        alpha: The :math:`\\alpha` level required by the hypothesis.
        beta: The :math:`\\beta` level required by the hypothesis (equal to
            :math:`1` - power).
        power: The power required by the hypothesis (equal to
            :math:`1 - \\beta`).
    """

    def __init__(self, delta=None, stdev_wr=None, stdev_wt=None,
                 stdev_br=None, stdev_bt=None, rho=None,
                 theta_IBE=None, alpha=None, power=None,
                 beta=None):
        is_numeric(delta, 'delta')
        self.delta = delta

        is_positive(stdev_wr, 'stdev_wr')
        self.stdev_wr = stdev_wr
        is_positive(stdev_wt, 'stdev_wt')
        self.stdev_wt = stdev_wt
        is_positive(stdev_bt, 'stdev_br')
        self.stdev_br = stdev_br
        is_positive(stdev_bt, 'stdev_bt')
        self.stdev_bt = stdev_bt
        is_in_0_1(rho, 'rho')
        self.rho = rho

        var_d = self.stdev_bt**2 + self.stdev_br**2 - 2 * self.rho * self.stdev_bt**2 * self.stdev_bt**2
        self.stdev_d = math.sqrt(var_d)
        if theta_IBE is None:
            theta_IBE = 1.74
        else:
            is_positive(theta_IBE, 'theta_IBE')
        self.theta_IBE = theta_IBE

        # Initialize the remaining arguments through the parent.
        super(Individual, self).__init__(hypothesis="equivalence",
                                         alpha=alpha, beta=beta, power=power)

    def _calculate_gamma(self):
        gamma = self.delta**2 + self.stdev_d**2 + self.stdev_wt**2
        gamma -= (self.theta_IBE + 1) * self.stdev_wr**2
        return gamma

    def _calculate_u(self, n, cut):
        nu = 2 * n - 1
        sigma = self._calculate_sigma(0.5, 0.5)
        tmp_1 = ((nu - 1) / stats.chi2.ppf(1 - cut, df=nu - 1) - 1)**2
        tmp_2 = ((nu - 1) / stats.chi2.ppf(cut, df=nu - 1) - 1)**2
        U = (abs(self.delta) +
             stats.t.ppf(cut, df=nu) * sigma * math.sqrt(2 / n) / 2 -
             self.delta**2)**2
        U += sigma**4 * tmp_1
        U += 0.25 * self.stdev_wt**4 * tmp_1
        U += (1.5 + self.theta_IBE)**2 * self.stdev_wr**4 * tmp_2
        return U

    def _calculate_sigma(self, a, b):
        sigma = (self.stdev_d**2 + a * self.stdev_wt**2 +
                 b * self.stdev_wr**2)
        return sigma

    def calculate(self):
        gamma = self._calculate_gamma()
        self.n = None
        found_solution = False
        for i in range(2, MAX_ITERATIONS):
            bound = gamma + math.sqrt(self._calculate_u(i, 0.05))
            bound += math.sqrt(self._calculate_u(i, self.power))
            if bound < 0:
                self.n = i
                found_solution = True
                break
        if not found_solution:
            raise BaseException("N > " + str(MAX_ITERATIONS))


from django.test import TestCase

# Import module
from backend.project_manager import *

class GetProjectsTest(TestCase):

    def test_multiple_projects(self):
        """
        Test getting multiple projects.
        """
        create_project('project1')
        create_project('project2')
        create_project('project3')
        code, res = get_projects()
        self.assertEqual(200, code)
        self.assertEqual(len(res[PROJECTS]), 3)

    def test_no_projects(self):
        """
        Test getting projects when there are none.
        """
        code, res = get_projects()
        self.assertEqual(code, 200)
        self.assertEqual(res, {PROJECTS: []})

class NewProjectTest(TestCase):

    def test_basic(self):
        """
        Makes a simple call.
        """
        code, res = new_project(data={PROJECT_NAME: 'test_project'})
        self.assertEqual(code, 200)
        self.assertEqual(res, {PROJECT_ID: 1})
        self.assertEqual(Project.objects.count(), 1)
        self.assertEqual(Filter.objects.count(), 1)

class DeleteProjectTest(TestCase):

    def setUp(self) -> None:
        self.pid = create_project(name='test_project')

    def test_basic(self):
        """
        Makes a simple call.
        """
        code, res = delete_project(data={PROJECT_ID: self.pid})
        self.assertEqual(code, 200)
        self.assertEqual(res, {})
        self.assertEqual(Project.objects.count(), 0)

class RenameProjectTest(TestCase):

    def setUp(self) -> None:
        self.pid = create_project(name='test_project')

    def test_basic(self):
        """
        Makes a simple call.
        """
        code, res = rename_project(data={PROJECT_ID: self.pid, PROJECT_NAME: 'new_name'})
        self.assertEqual(code, 200)
        self.assertEqual(get_project_by_id(pid=self.pid).name, 'new_name')


from django.conf.urls import url
from . import views

urlpatterns = [
    # ex: /device/
    url(r'^$', views.info, name='info'),
    url(r'^networkinfo$', views.networkinfo, name='network'),
    url(r'^networkinfo/(?P<nodename>\w+-\w+)$', views.networkinfo, name='networkinfo'),
    url(r'^(?P<devname>\w+-\w+-\w+-\w+)$', views.device, name='device'),
]


import json
from channels.generic.websocket import WebsocketConsumer
from asgiref.sync import async_to_sync
from chat.models import Chat

# daphne -b 0.0.0.0 -p 8000 circlemarket.routing:application
class ChatConsumer(WebsocketConsumer):
    def connect(self):
        self.room_name = self.scope['url_route']['kwargs']['chat_slug']
        self.room_group_name = "chat_%s" % (self.room_name)
        # self.chat = Chat.objects.get(slug=self.room_name)
        async_to_sync(self.channel_layer.group_add)(
            self.room_group_name,
            self.channel_name
        )
        self.accept()

    def receive(self, text_data):
        text_data_json = json.loads(text_data)
        message = text_data_json['message']
        async_to_sync(self.channel_layer.group_send)(
            self.room_group_name,
            {
                'type': 'chat_message',
                'message': message,
            }
        )

    def chat_message(self, event):
        message = event['message']
        # Send message to WebSocket
        self.send(text_data=json.dumps({
            'message' : message,
            'type' : 'message_sent',
        }))


# -*- coding: utf-8 -*
'''
填补空白单元格的蔬菜名称和批发市场名称
修改日期格式
'''

import xlrd
import xlwt
from xlutils.copy import copy

workbook = xlrd.open_workbook(r"D:\p\p001\1.1-1.10.xls")
print(workbook.sheet_names())

sheet1 = workbook.sheet_by_index(0)
print('index为0的sheet：', sheet1.name, sheet1.nrows, sheet1.ncols)
sheet01 = workbook.sheet_by_name('Report')
print('name为Report的sheet：', sheet01.name, sheet1.nrows, sheet1.ncols)

nrows = sheet1.nrows
ncols = sheet1.ncols

#copy已读取的文件
book = copy(workbook)
#获取名为Report的sheet
sheet2 = book.get_sheet('Report')

#在第11行添加标题栏
head = ['蔬菜','批发市场','日期','大宗价','最高价','最低价','交易量','产地']
for i in range(0,8):
    sheet2.row(10).write(i,head[i])
    print('正在修改第'+str(i)+'个标题栏')

#填补蔬菜栏的空白
tempVeg =''
for i in range(11, nrows):
    if sheet01.cell_value(i,0) is not "":
        print(i)
        tempVeg = sheet01.cell_value(i, 0)
        i += 1
    else:
        sheet2.write(i, 0, tempVeg)
        i += 1

#填补批发市场的空白
tempMarket =''
for i in range(11, nrows):
    if sheet01.cell_value(i,1) is not "":
        print(i)
        tempMarket = sheet01.cell_value(i, 1)
        i += 1
    else:
        sheet2.write(i, 1, tempMarket)
        i += 1
print('已完成填补批发市场')

#修改日期单元格
'''
datevalue = sheet01.cell_value(13, 2)
print(datevalue)
'''
for i in range(13, nrows):
    datevalue = sheet01.cell_value(i, 2)
    sheet2.write(i, 2, datevalue[9:])
    i += 1

#保存
book.save(r"D:\p\p001\201701_1.xls")


from django.apps import apps
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.core.serializers.json import DjangoJSONEncoder
from django.core.urlresolvers import reverse
import django

import json
import six

def to_json(data):
    kw = {}
    if six.PY2:
        kw['encoding'] = 'utf-8'
    return json.dumps(data, cls=DjangoJSONEncoder, ensure_ascii=False, separators=(',',':'), **kw)

def get_setting(name):
    """ Looks for settings under DJANGO_HISTORY_SETTINGS, supporting dot notation for nested lookups,
    eg. get_setting('EXCLUDE_CHANGES.fields') """
    dh = getattr(settings, 'DJANGO_HISTORY_SETTINGS', {})
    result = None
    for k in name.split('.'):
        if dh is None:
            continue
        result = dh.get(k, None)
        dh = result
    return result

def get_relation(rel):
    if django.VERSION[:2] >= (1, 9):
        instance = rel.remote_field.model
    elif django.VERSION[:2] >= (1, 8):
        instance = rel.related.model
    else:
        instance = rel.related.parent_model
    return instance

def models_listing():
    c = []
    for m in apps.get_models():
        ct = get_ct(m)
        url = reverse("history-by-ct", kwargs=dict(ct_id=ct.id))
        c.append(dict(url=url, name=m._meta.object_name, model=m))
    return c

def models_schemas(models=None):
    models = models or models_listing()
    d = {}
    for model in models:
        d.setdefault(model['name'], {})
        f = []
        for field in model['model']._meta.get_fields(include_hidden=True):
            f.append(dict(cls=field.__class__.__name__,
                          name=field.name,
                          hidden=field.hidden),)
        d[model['name']].setdefault('fields', f)
    return d

def get_ct_by_id(pk):
    return ContentType.objects.get(pk=pk)

def get_ct(model):
    return ContentType.objects.get_for_model(model)

def match_field(model, changed_field):
    try:
        field = model._meta.get_field(changed_field)
    except:
        field = model._meta.get_field(changed_field.replace('_id', ''))
    return field


import GA
from sklearn.datasets import load_breast_cancer
import time

def print_top_networks(networks):
    print("Top " + str(len(networks)) + " networks:")
    for i in range(0, len(networks)):
        print("Network #" + str(i) + " Loss: " + str(networks[i][0].__getattribute__('loss_')))
        for j in range(0, len(networks[i][0].__getattribute__('coefs_'))):
            print("    Hidden Layer: " + str(j) + " Nodes: " + str(len(networks[i][0].__getattribute__('coefs_')[j])))
        print("  Time: " + str(networks[i][1]))

def train_networks(networks, X, y):
    # Train the neural networks
    i = 0
    print("    " + str(len(networks)) + " networks to train")
    for network in networks:
        start = time.time()
        network[0].fit(X, y)
        end = time.time()
        print("    Training network: " + str(i))
        total_time = end - start
        network[1] = total_time
        i += 1

def get_average_accuracy(networks):
    # Get the average accuracy for a group of networks
    total_accuracy = 0
    total_time = 0
    for network in networks:
        total_accuracy += network[0].__getattribute__('loss_')
        total_time += network[1]

    return [(total_accuracy / len(networks)), (total_time / len(networks))]

def generate(generations, population, X, y, time_weight, network_params):
    # Generate a network with the genetic algorithm
    optimizer = GA.Optimizer()
    networks = optimizer.create_population(population, network_params)

    # Evolve the generation.
    for i in range(generations):
        print("***Doing generation %d of %d***" %
              (i + 1, generations))

        # Train and get accuracy for networks.
        train_networks(networks, X, y)

        # Get the average accuracy for this generation.
        average_accuracy_n_time = get_average_accuracy(networks)

        # Print out the average accuracy each generation.
        print("Generation average loss: %.2f" % (average_accuracy_n_time[0]))
        print("Generation time average: %f" % average_accuracy_n_time[1])
        print('-' * 80)

        # Evolve, except on the last iteration.
        if i != generations - 1:
            # Do the evolution.
            networks = optimizer.evolve(networks, time_weight)

    # Sort our final population.
    networks = sorted(networks, key=lambda x: (optimizer.fitness(x, time_weight)), reverse=True)

    # Print out the top 5 networks.
    print_top_networks(networks[:5])

def main():
    #Evolve a network
    generations = 3  # Number of times to evolve the population.
    population = 10  # Number of networks in each generation.
    # Allows you to select from 0 to 1 how much you want to weight the speed of the neural network training vs the accuracy
    time_weight = 0
    # Network parameters, network_params[0]: tuple of layer ranges, network_params[1]: tuple of nodes per layer
    network_params = [(0, 9), (1, 100)]
    X, y = load_breast_cancer(return_X_y=True)
    print("***Evolving %d generations with population %d***" %
          (generations, population))

    generate(generations, population, X, y, time_weight, network_params)

main()


# Step 1: Importing Libraries
# This section imports the required libraries such as JAX, NumPy, and Matplotlib for numerical computations, plotting, and optimization. It also imports some specific modules such as odeint for solving ordinary differential equations, stax and optimizers for building and training neural networks, and ImageSequenceClip for creating videos from a sequence of images.

# Step 2: Setting up the GPU
# This section checks whether a GPU is available for running the script using the xla_bridge library from JAX.

# Step 3: Defining the Double Pendulum System
# This section defines the Lagrangian of a double pendulum system using its kinetic and potential energy. It also defines a function for solving the equations of motion of the system using the Lagrangian.

# Step 4: Defining Helper Functions
# This section defines some helper functions such as the analytical solution of the double pendulum system, a function for normalizing the state of the system, and a function for taking a single step in the Runge-Kutta integration method.

# Step 5: Generating Training Data
# This section generates training data for the Lagrangian Neural Network by using the analytical method to sample double pendulum dynamics. The script loads a pre-generated dataset from a MATLAB file and preprocesses the data by normalizing the state of the system.

# Step 6: Building a Lagrangian Neural Network
# This section builds a Lagrangian Neural Network model using a feedforward neural network with two hidden layers and a single output. The script defines the loss function for the model as the mean squared error between predicted and target states of the system. It also defines the optimization algorithm as Adam with a learning rate schedule.

#%%
# Step 1: Import necessary libraries
import jax
import jax.numpy as jnp
import numpy as np
from jax.experimental.ode import odeint
import matplotlib.pyplot as plt
from functools import partial
from jax.experimental import stax
from jax.experimental import optimizers
import scipy.io as sio

# Step 2: Define the Lagrangian and equations of motion
def lagrangian(q, q_dot, m1, m2, l1, l2, g):
  # Compute kinetic energy T
  t1, t2 = q     # theta 1 and theta 2
  w1, w2 = q_dot # omega 1 and omega 2
  T1 = 0.5 * m1 * (l1 * w1)**2
  T2 = 0.5 * m2 * ((l1 * w1)**2 + (l2 * w2)**2 + 2 * l1 * l2 * w1 * w2 * jnp.cos(t1 - t2))
  T = T1 + T2

  # Compute potential energy V
  y1 = -l1 * jnp.cos(t1)
  y2 = y1 - l2 * jnp.cos(t2)
  V = m1 * g * y1 + m2 * g * y2

  # Return Lagrangian L = T - V
  return T - V

def f_analytical(state, t=0, m1=1, m2=1, l1=1, l2=1, g=9.8):
  # Compute forces analytically
  t1, t2, w1, w2 = state
  a1 = (l2 / l1) * (m2 / (m1 + m2)) * jnp.cos(t1 - t2)
  a2 = (l1 / l2) * jnp.cos(t1 - t2)
  f1 = -(l2 / l1) * (m2 / (m1 + m2)) * (w2**2) * jnp.sin(t1 - t2) - (g / l1) * jnp.sin(t1)
  f2 = (l1 / l2) * (w1**2) * jnp.sin(t1 - t2) - (g / l2) * jnp.sin(t2)
  g1 = (f1 - a1 * f2) / (1 - a1 * a2)
  g2 = (f2 - a2 * f1) / (1 - a1 * a2)
  return jnp.stack([w1, w2, g1, g2])

def equation_of_motion(lagrangian, state, t=None):
  # Compute equations of motion using the Euler-Lagrange equation
  q, q_t = jnp.split(state, 2)
  q_tt = (jnp.linalg.pinv(jax.hessian(lagrangian, 1)(q, q_t))
          @ (jax.grad(lagrangian, 0)(q, q_t)
             - jax.jacobian(jax.jacobian(lagrangian, 1), 0)(q, q_t) @ q_t))
  return jnp.concatenate([q_t, q_tt])

def solve_lagrangian(lagrangian, initial_state, **kwargs):
  # We currently run odeint on CPUs only, because its cost is dominated by
  # control flow, which is slow on GPUs.
  @partial(jax.jit, backend='cpu')
  def f(initial_state):
    return odeint(partial(equation_of_motion, lagrangian),
                  initial_state, **kwargs)
  return f(initial_state)

def forcesapplied(lagrangian, state, t=None):
  q, q_t = jnp.split(state, 2)
  q_tt = (jnp.linalg.pinv(jax.hessian(lagrangian, 1)(q, q_t))
          @ (jax.grad(lagrangian, 0)(q, q_t)
             - jax.jacobian(jax.jacobian(lagrangian, 1), 0)(q, q_t) @ q_t))
  tau=jax.hessian(lagrangian, 1)(q, q_t)@q_tt-((jax.grad(lagrangian, 0)(q, q_t)- jax.jacobian(jax.jacobian(lagrangian, 1), 0)(q, q_t) @ q_t))
  return tau

def equation_of_motion_forced(lagrangian, state,force, t=None):
  q, q_t = jnp.split(state, 2)
  q_tt = (jnp.linalg.pinv(jax.hessian(lagrangian, 1)(q, q_t))
          @ (jax.grad(lagrangian, 0)(q, q_t)
             - jax.jacobian(jax.jacobian(lagrangian, 1), 0)(q, q_t) @ q_t)-force)
  return jnp.concatenate([q_t, q_tt])

"""### Helper functions

"""

# Double pendulum dynamics via the rewritten Euler-Lagrange
@partial(jax.jit, backend='cpu')
def solve_autograd(initial_state, times, m1=1, m2=1, l1=1, l2=1, g=9.8):
  L = partial(lagrangian, m1=m1, m2=m2, l1=l1, l2=l2, g=g)
  return solve_lagrangian(L, initial_state, t=times, rtol=1e-10, atol=1e-10)

# Double pendulum dynamics via analytical forces taken from Diego's blog
@partial(jax.jit, backend='cpu')
def solve_analytical(initial_state, times):
  return odeint(f_analytical, initial_state, t=times, rtol=1e-10, atol=1e-10)

def normalize_dp(state):
  # wrap generalized coordinates to [-pi, pi]
  return jnp.concatenate([(state[:2] + np.pi) % (2 * np.pi) - np.pi, state[2:]])

def rk4_step(f, x, t, h):
  # one step of runge-kutta integration
  k1 = h * f(x, t)
  k2 = h * f(x + k1/2, t + h/2)
  k3 = h * f(x + k2/2, t + h/2)
  k4 = h * f(x + k3, t + h)
  return x + 1/6 * (k1 + 2 * k2 + 2 * k3 + k4)

"""## **Step 5: Generate training data**
Let's generate some training data by using the analytical method to sample double pendulum dynamics.

"""
#
load_data = sio.loadmat('robotarm_iodata.mat')

data = load_data['robotarm_iodata'][0, 0]

batch_states = data['batch_states'][0]
batch_statep=data['batch_statesp'][0]

# Commented out IPython magic to ensure Python compatibility.
time_step = 0.1
N = 15000
analytical_step = jax.jit(jax.vmap(partial(rk4_step, f_analytical, t=0.0, h=time_step)))

x0 = np.array([-0.3*np.pi, 0.2*np.pi, 0.35*np.pi, 0.5*np.pi], dtype=np.float32)
x0 = np.array([3*np.pi/7, 3*np.pi/4, 0, 0], dtype=np.float32)
t = 0.1*np.arange(N, dtype=np.float32) # time steps 0 to N
x1_train = jax.device_get(solve_analytical(x0, t)) # dynamics for first N time steps
x1t_train = jax.device_get(jax.vmap(f_analytical)(x1_train)) # time derivatives of each state
y1_train = jax.device_get(analytical_step(x1_train)) # analytical next step

t = np.arange(N, dtype=np.float32) # time steps 0 to N
x_train = batch_states[0:14999] # dynamics for first N time steps
xt_train = batch_statep[0:14999] # time derivatives of each state
y_train = batch_states[1:15000] # analytical next step

x_test = batch_states[15001:30000]
xt_test = batch_statep[15001:30000]
y_test = batch_states[15002:30001]

"""### Visualize train and test data"""

# preprocess
train_vis = jax.vmap(normalize_dp)(x_train)
test_vis = jax.vmap(normalize_dp)(x_test)

vel_angle = lambda data:  (np.arctan2(data[:,3], data[:,2]) / np.pi + 1)/2
vel_color = lambda vangle: np.stack( [np.zeros_like(vangle), vangle, 1-vangle]).T
train_colors = vel_color(vel_angle(train_vis))
test_colors = vel_color(vel_angle(test_vis))

# plot
SCALE = 80 ; WIDTH = 0.006
plt.figure(figsize=[8,4], dpi=120)
plt.subplot(1,2,1)
plt.title("Train data") ; plt.xlabel(r'$\theta_1$') ; plt.ylabel(r'$\theta_2$')
plt.quiver(*train_vis.T, color=train_colors, scale=SCALE, width=WIDTH)

plt.subplot(1,2,2)
plt.title("Test data") ; plt.xlabel(r'$\theta_1$') ; plt.ylabel(r'$\theta_2$')
plt.quiver(*test_vis.T, color=test_colors, scale=SCALE, width=WIDTH)

plt.tight_layout() ; plt.show()

"""## **Step 6: Build a Lagrangian Neural Network**
This section presents a bare-bones approach to approximating the Lagrangian with a neural network

### i) Build model and define the loss

"""

# replace the lagrangian with a parameteric model
def learned_lagrangian(params):
  def lagrangian(q, q_t):
    assert q.shape == (2,)
    state = normalize_dp(jnp.concatenate([q, q_t]))
    return jnp.squeeze(nn_forward_fn(params, state), axis=-1)
  return lagrangian

# define the loss of the model (MSE between predicted q, \dot q and targets)
@jax.jit
def loss(params, batch, time_step=None):
  state, targets = batch
  if time_step is not None:
    f = partial(equation_of_motion, learned_lagrangian(params))
    preds = jax.vmap(partial(rk4_step, f, t=0.0, h=time_step))(state)
  else:
    preds = jax.vmap(partial(equation_of_motion, learned_lagrangian(params)))(state)
  return jnp.mean((preds - targets) ** 2)

# build a neural network model
init_random_params, nn_forward_fn = stax.serial(
    stax.Dense(128),
    stax.Softplus,
    stax.Dense(128),
    stax.Softplus,
    stax.Dense(1),
)
init_random_params, f = stax.serial(
    stax.Dense(128),
    stax.Softplus,
    stax.Dense(1),
)
"""### ii) Define optimization and data"""

@jax.jit
def update_timestep(i, opt_state, batch):
  params = get_params(opt_state)
  return opt_update(i, jax.grad(loss)(params, batch, time_step), opt_state)

@jax.jit
def update_derivative(i, opt_state, batch):
  params = get_params(opt_state)
  return opt_update(i, jax.grad(loss)(params, batch, None), opt_state)

x_train = jax.device_put(jax.vmap(normalize_dp)(x_train))
y_train = jax.device_put(y_train)

x_test = jax.device_put(jax.vmap(normalize_dp)(x_test))
y_test = jax.device_put(y_test)

"""### iii) Train the model"""

# Commented out IPython magic to ensure Python compatibility.

rng = jax.random.PRNGKey(0)
_, init_params = init_random_params(rng, (-1, 4))

 # numbers in comments denote stephan's settings
batch_size = 1000
test_every = 10
num_batches = 15000

train_losses = []
test_losses = []

# adam w learn rate decay
opt_init, opt_update, get_params = optimizers.adam(
    lambda t: jnp.select([t < batch_size*(num_batches//3),
                           t < batch_size*(2*num_batches//3),
                           t > batch_size*(2*num_batches//3)],
                          [1e-3, 3e-4, 1e-4]))
opt_state = opt_init(init_params)

for iteration in range(batch_size*num_batches + 1):
  if iteration % batch_size == 0:
    params = get_params(opt_state)
    train_loss = loss(params, (x_train, xt_train))
    train_losses.append(train_loss)
    test_loss = loss(params, (x_test, xt_test))
    test_losses.append(test_loss)
    if iteration % (batch_size*test_every) == 0:
      print(f"iteration={iteration}, train_loss={train_loss:.6f}, test_loss={test_loss:.6f}")
  opt_state = update_derivative(iteration, opt_state, (x_train, xt_train))

params = get_params(opt_state)

jax.numpy.save('/params.npy',params)


"""
Climate Change Project
"""

import csv
from typing import Dict, List, Tuple
import plotly.graph_objects as go
from PIL import Image, ImageDraw, ImageFont

TORONTO = ('datasets/toronto_actual.csv', 'datasets/toronto_predicted.csv', (800, 1000), 'Toronto')
QUEBEC = ('datasets/quebec_actual.csv', 'datasets/quebec_predicted.csv', (950, 1000), 'Quebec')
HALIFAX = ('datasets/halifax_actual.csv', 'datasets/halifax_predicted.csv', (1315, 1060), 'Halifax')
WINNIPEG = ('datasets/winnipeg_actual.csv', 'datasets/winnipeg_predicted.csv', (658, 852), 'Winnipeg')

CITIES_SET = {TORONTO, QUEBEC, HALIFAX, WINNIPEG}
MAP = 'canada_map2.jpg'
CITY_TEMPS = {}

#this the reader part of our project (both functions read the data from the datasets)

def read_actual_data(actual_data_filepath: str) -> Dict[int, float]:
    """Return a dictionary mapping of year to temperature from the data in a CSV file.

    Preconditions:
        - filepath refers to a csv file in the format of
          datasets/toronto.actual_temps.csv
          (i.e., could be that file or a different file in the same format)
    """
    with open(actual_data_filepath) as file:
        reader = csv.reader(file)

        next(reader)

        monthly_temps = []
        years = []
        yearly_dict = {}
        index = 0
        sum_so_far = 0
        for row in reader:
            monthly_temps.append(float(row[4]))

            year = int(row[2])
            if year not in years:
                years.append(year)

    for year in years:
        for _ in range(0, 12):
            sum_so_far = sum_so_far + monthly_temps[index]
            index += 1
        mean_temp = sum_so_far / 12
        yearly_dict[year] = round(mean_temp, 2)
        sum_so_far = 0

    return yearly_dict

def read_predicted_data(predicted_temps_filepath: str, actual_temps_dict: dict) -> Dict[int, Dict[str, float]]:
    """Return a dictionary of year to a dictionary of varying RCPs
        and their respective predicted temperature from the data in a CSV file.

    Preconditions:
        - filepath refers to a csv file in the format of
          datasets/toronto.predicted_temps.csv
          (i.e., could be that file or a different file in the same format)
    """
    with open(predicted_temps_filepath) as file:
        reader = csv.reader(file)

        years = list(actual_temps_dict.keys())

        next(reader)

        for _ in range(0, years[0] - 2000):
            next(reader)

        temp_low_rcp_list = []
        temp_medium_rcp_list = []
        temp_high_rcp_list = []
        rcp_dict = {}
        index = 0

        for row in reader:
            temp_low_rcp_list.append(float(row[2]))
            temp_medium_rcp_list.append(float(row[5]))
            temp_high_rcp_list.append(float(row[8]))

    for year in years:
        rcp_dict[year] = {'RCP 2.6': temp_low_rcp_list[index],
                          'RCP 4.5': temp_medium_rcp_list[index],
                          'RCP 8.5': temp_high_rcp_list[index]}
        index += 1

    return rcp_dict
#

# these functions all compute on the data
def make_low_rcp_list(predicted_temps_dict: dict) -> List[float]:
    """Return a list of RCP 2.6 temperature values
    """
    low_rcp_list = []

    for year in list(predicted_temps_dict.keys()):
        low_rcp_list.append(predicted_temps_dict[year]['RCP 2.6'])

    return low_rcp_list

def make_median_rcp_list(predicted_temps_dict: dict) -> List[float]:
    """Return a list of RCP 4.5 temperature values
    """
    median_rcp_list = []

    for year in list(predicted_temps_dict.keys()):
        median_rcp_list.append(predicted_temps_dict[year]['RCP 4.5'])

    return median_rcp_list

def make_high_rcp_list(predicted_temps_dict) -> List[float]:
    """Return a list of RCP 8.5 temperature values
    """
    high_rcp_list = []

    for year in list(predicted_temps_dict.keys()):
        high_rcp_list.append(predicted_temps_dict[year]['RCP 8.5'])

    return high_rcp_list

def calculate_low_actual_percentage_difference(actual_temps_dict: dict, final_low_rcp_list: list) -> List[float]:
    """
    Return a list of percentage differences of low RCP values to actual temperature values
    """
    actual_temps_list = list(actual_temps_dict.values())
    low_rcp_pd = []
    for index in range(0, len(final_low_rcp_list)):
        difference = abs(final_low_rcp_list[index] - actual_temps_list[index])
        percentage_difference = round(((difference / actual_temps_list[index]) * 100), 1)
        low_rcp_pd.append(percentage_difference)

    return low_rcp_pd

def calculate_median_actual_percentage_difference(actual_temps_dict: dict, final_median_rcp_list: list) -> List[float]:
    """
    Return a list of percentage differences of median RCP values to actual temperature values
    """
    actual_temps_list = list(actual_temps_dict.values())
    median_rcp_pd = []
    for index in range(0, len(final_median_rcp_list)):
        difference = abs(final_median_rcp_list[index] - actual_temps_list[index])
        percentage_difference = round(((difference / actual_temps_list[index]) * 100), 1)
        median_rcp_pd.append(percentage_difference)

    return median_rcp_pd

def calculate_high_actual_percentage_difference(actual_temps_dict: dict, final_high_rcp_list: list) -> List[float]:
    """
    Return a list of percentage differences of high RCP values to actual temperature values
    """
    actual_temps_list = list(actual_temps_dict.values())
    high_rcp_pd = []
    for index in range(0, len(final_high_rcp_list)):
        difference = abs(final_high_rcp_list[index] - actual_temps_list[index])
        percentage_difference = round(((difference / actual_temps_list[index]) * 100), 1)
        high_rcp_pd.append(percentage_difference)

    return high_rcp_pd
#

# these functions visualise our program (plot graph and table)
def plot_temp_data(actual_temps_dict: dict, final_low_rcp_list: list, final_median_rcp_list: list, final_high_rcp_list: list) -> None:
    """Plot a line and scatter graph of real and predicted temperatures
        using plotly's line and scatter plots
    """
    x = list(actual_temps_dict.keys())
    actual_y = list(actual_temps_dict.values())
    low_predicted_y = final_low_rcp_list
    median_predicted_y = final_median_rcp_list
    high_predicted_y = final_high_rcp_list

    fig = go.Figure()

    fig.add_trace(go.Scatter(x=x, y=low_predicted_y,
                             mode='lines+markers',
                             name='RCP 2.6 Predicted Temperature'))

    fig.add_trace(go.Scatter(x=x, y=median_predicted_y,
                             mode='lines+markers',
                             name='RCP 4.5 Predicted Temperature'))

    fig.add_trace(go.Scatter(x=x, y=high_predicted_y,
                             mode='lines+markers',
                             name='RCP 8.5 Predicted Temperature'))

    fig.add_trace(go.Scatter(x=x, y=actual_y,
                             mode='lines+markers',
                             name='Actual Temperature'))
    fig.update_layout(
        title="Actual vs Predicted Temperature of " + city[3],
        xaxis_title="Years",
        yaxis_title="Temperature (Celcius)",
        font=dict(
            family="Courier New, monospace",
            size=18)
    )

    fig.show()

def draw_table(actual_temps_dict: dict,
               final_low_rcp_list: list,
               final_median_rcp_list: list,
               final_high_rcp_list: list,
               low_rcp_percentage_difference: list,
               median_rcp_percentage_difference: list,
               high_rcp_percentage_difference: list) -> None:
    """
    Draw a table using a plotly's basic table
    """
    fig = go.Figure(data=[go.Table(header=dict(values=['Actual Temperature', 'RCP 2.6',
                                                       '% Difference of RCP 2.6 and Actual Temp',
                                                       'RCP 4.5',
                                                       '% Difference of RCP 4.5 and Actual Temp',
                                                       'RCP 8.5',
                                                       '% Difference of RCP 8.5 and Actual Temp'],
                                               line_color='darkslategray',
                                               fill_color='lightskyblue'),
                                   cells=dict(values=[list(actual_temps_dict.values()),
                                                      final_low_rcp_list,
                                                      low_rcp_percentage_difference,
                                                      final_median_rcp_list,
                                                      median_rcp_percentage_difference,
                                                      final_high_rcp_list,
                                                      high_rcp_percentage_difference]))])

    fig.update_layout(
        title="Actual vs Predicted Temperature of " + city[3]
    )

    fig.show()

def draw_map(rcp_type: str) -> None:
    """
    Draws the map
    """

    map = Image.open(MAP)
    width, height = map.size

    new_map = Image.new('RGB', (width * 2, height + 80))

    # fills the states for the actual map
    for city in CITIES_SET:
        temp = CITY_TEMPS[city][0]
        ImageDraw.floodfill(map, city[2], temp_to_rgb(temp), thresh=50)

    map2 = Image.open(MAP)

    # fills the states for the predicted map
    for city in CITIES_SET:
        temp = CITY_TEMPS[city][rcp_to_slice(rcp_type)]
        ImageDraw.floodfill(map2, city[2], temp_to_rgb(temp), thresh=50)

    new_map.paste(map, (0, 80))
    new_map.paste(map2, (width, 80))

    # Writes the titles
    title_font = ImageFont.truetype("arial.ttf", 50)
    new_map_editable = ImageDraw.Draw(new_map)
    new_map_editable.text((width // 3, 10), 'Actual Temperatures(' + year + ')', font=title_font)
    new_map_editable.text((int(1.3 * width), 10), 'Predicted Temperatures(' + year + ')', font=title_font)

    new_map.show()

def rcp_to_slice(rcp_type: str) -> int:
    """
    returns the index that corresponds to that RCP value
    """
    if rcp_type == 'RCP 2.6':
        return 1
    elif rcp_type == 'RCP 4.5':
        return 2
    elif rcp_type == 'RCP 8.5':
        return 3

def temp_to_rgb(temp: float) -> Tuple:
    """
    returns the rgb value that corresponds to that temperature
    """
    if temp > 20:
        return (0, 0, 0, 40)
    elif temp >= 17.3:
        return (255, 0, int(35.55 * (temp - 17.3)), 40)
    elif temp >= 5.6:
        return (255, int(21.79 * (17.3-temp)), 0, 40)
    elif temp >= 2.9:
        return (int(85.19 * (temp - 2.9)), 255, 0, 40)
    elif temp >= -0.30:
        return (0, 255, int(79.69 * (2.9 - temp)), 40)
    else:
        return (250, 250, 250, 40)

def run(city: tuple, year: int, city_name: str) -> None:
    """
    runs the code for one city
    """
    actual_temps_dict = read_actual_data(city[0])
    predicted_temps_dict = read_predicted_data(city[1], actual_temps_dict)

    if city[3].lower() == city_name.lower():
        final_low_rcp_list = make_low_rcp_list(predicted_temps_dict)
        low_rcp_percentage_difference = calculate_low_actual_percentage_difference(actual_temps_dict, final_low_rcp_list)
        final_median_rcp_list = make_median_rcp_list(predicted_temps_dict)
        median_rcp_percentage_difference = calculate_median_actual_percentage_difference(actual_temps_dict,
                                                                                final_median_rcp_list)
        final_high_rcp_list = make_high_rcp_list(predicted_temps_dict)
        high_rcp_percentage_difference = calculate_high_actual_percentage_difference(actual_temps_dict, final_high_rcp_list)
        plot_temp_data(actual_temps_dict, final_low_rcp_list, final_median_rcp_list, final_high_rcp_list)
        draw_table(actual_temps_dict, final_low_rcp_list, final_median_rcp_list, final_high_rcp_list,
                   low_rcp_percentage_difference, median_rcp_percentage_difference, high_rcp_percentage_difference)

    temperatures = [actual_temps_dict[year], predicted_temps_dict[year]['RCP 2.6'],
                    predicted_temps_dict[year]['RCP 4.5'], predicted_temps_dict[year]['RCP 8.5']]
    CITY_TEMPS[city] = temperatures

# this is the main part of the program that calls every function
if __name__ == '__main__':

    year = input("Write the year you want to see in the map from 2003-2019")
    if not(2003 <= int(year) <= 2019):
        year = input("Try again. Write the year you want to see in the map from 2003-2019")
    city_name = input(
        "Type the name of the city you want to display its stats on graph (TORONTO, QUEBEC, HALIFAX, WINNIPEG)")
    if not (city_name.lower() == 'toronto' or city_name.lower() == 'quebec' or city_name.lower() == 'halifax' or
            city_name.lower() == 'winnnipeg'):
        city_name = input(
            "Try again. Type the name of the city you want to display its stats on graph (TORONTO, QUEBEC, HALIFAX, WINNIPEG)")
    rcp_type = input(' write an rcp value (RCP 2.6, RCP 4.5, RCP 8.5)')
    if not (rcp_type == 'RCP 2.6' or rcp_type == 'RCP 4.5' or rcp_type == 'RCP 8.5'):
        rcp_type = input('Try again. Write an rcp value (RCP 2.6, RCP 4.5, RCP 8.5)')

    while True:
        for city in CITIES_SET:
            run(city, int(year), city_name)

        draw_map(rcp_type)

        year = input("Write the year you want to see in the map from 2003-2019")
        if not (2003 <= int(year) <= 2019):
            year = input("Try again. Write the year you want to see in the map from 2003-2019")
        if not (2003 <= int(year) <= 2019):
            break

        city_name = input(
            "Type the name of the city you want to display its stats on graph (TORONTO, QUEBEC, HALIFAX, WINNIPEG)")
        if not(city_name.lower() == 'toronto' or city_name.lower() == 'quebec' or city_name.lower() == 'halifax' or
               city_name.lower() == 'winnnipeg'):
            city_name = input(
                "Try again. Type the name of the city you want to display its stats on graph (TORONTO, QUEBEC, HALIFAX, WINNIPEG)")
        if not (city_name.lower() == 'toronto' or city_name.lower() == 'quebec' or city_name.lower() == 'halifax' or
                city_name.lower() == 'winnnipeg'):
            break

        rcp_type = input(' write an rcp value (RCP 2.6, RCP 4.5, RCP 8.5)')
        if not(rcp_type == 'RCP 2.6' or rcp_type == 'RCP 4.5' or rcp_type == 'RCP 8.5'):
            rcp_type = input('Try again. Write an rcp value (RCP 2.6, RCP 4.5, RCP 8.5)')
        if not (rcp_type == 'RCP 2.6' or rcp_type == 'RCP 4.5' or rcp_type == 'RCP 8.5'):
            break
#     import python_ta
#
#     python_ta.check_all(config={
#         'allowed-io': ['read_csv_data'],
#         'extra-imports': ['python_ta.contracts', 'csv', 'datetime',
#                           'plotly.graph_objects', 'plotly.subplots'],
#         'max-line-length': 100,
#         'max-args': 6,
#         'max-locals': 25,
#         'disable': ['R1705'],
#     })
#
#     import python_ta.contracts
#
#     python_ta.contracts.DEBUG_CONTRACTS = False
#     python_ta.contracts.check_all_contracts()
# #