#!/usr/bin/env python3
#Author: Rohit Singh
#Date: 04/01/2019
#convert a .dat file to .csv, .vtk, .vtp and .pvd file for planar view of crack
#use this planar data in crackGen3D.py to calculate 3D view of crack based on crack opening value

import re
import os, sys
import vtk
import os
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap
from collections import  defaultdict
from collections import OrderedDict

#creates required folder structure to save the files
def createfolder():
    try:
        if os.path.exists("./Crack Data"):
            return
        os.makedirs("./Crack Data")
        os.makedirs("./Crack Data/Plane")
        os.makedirs("./Crack Data/Plane/VTK")
        os.makedirs("./Crack Data/Plane/VTP")

        os.makedirs("./Crack Data/3D_UP")
        os.makedirs("./Crack Data/3D_UP/VTK")
        os.makedirs("./Crack Data/3D_UP/VTP")

        os.makedirs("./Crack Data/3D_DOWN")
        os.makedirs("./Crack Data/3D_DOWN/VTK")
        os.makedirs("./Crack Data/3D_DOWN/VTP")
    except:
        print("Could not create directory for storing output files")

#reads the data from the .dat input file.
#read is dependent on the data formatting in the input file so refer to the sample input file
def readData(file_path):
    with open(file_path, "r") as file:
        data=file.read()
    data_row= data.split("\n")
    data_formatted=""
    for row in data_row:
        row=row.strip(" ")
        row=re.sub("        ", ",", row)
        row = re.sub("    ", ",", row)
        data_formatted+=row+"\n"
    return data_formatted

#saves the input data in .csv format
def convertToCSV(data_formatted, no_of_stages):
    with open('Crack Data\crack_data.csv', mode='w') as file:
        file.write(data_formatted)

#parses thorugh the input data and generates the .vtk format for the input data
def convertToVTK(data_formatted, colormap, stage_to_extract):
    data_list=data_formatted.split("\n")
    header="# vtk DataFile Version 2.0 \n" \
            "Crack Data \n" \
            "ASCII \n" \
            "DATASET POLYDATA \n" \
            "POINTS"

    dataVTK = ""
    dataVTK+=header
    count=0
    stage=0
    skip=0
    no_points=0
    no_polygons=0
    no_triangles=0

    adjacent_faces=defaultdict(set)

    #specifying vertex positions-(x,y,z)
    for row in data_list:
        if row[:4] == "ZONE":
            stage += 1
        if(stage <stage_to_extract):
            skip+=1
        if(stage==stage_to_extract):
            if row[:4]=="ZONE":
                i=row.find("N=")
                no_points=int(row[i+2:i+row[i:].find(",")])+1
                j=row.find("E=")
                b=row[j:].find("\n")
                no_polygons = int(row[j + 2:])
                dataVTK+=" "+str(no_points)+" float \n" + \
                    "0.0000,0.0000,0.0000 \n"
                count=1
            else:
                if(count<no_points):
                    dataVTK+=row[:len(row)-row[::-1] .find(",")-1]+" \n"
                    count+=1
                else:
                    if(count == no_points):
                        dataVTK+="POLYGONS"
                        no_vertices=row.count(",")
                        temp = row.split(",")
                        if no_vertices==2:
                            no_triangles+=1
                            #create a dictionary with all adjacent vertices for the vertex
                            adjacent_faces[int(temp[0].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[0].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[1].strip()))
                        else:
                            adjacent_faces[int(temp[0].strip())].add(int(temp[3].strip()))
                            adjacent_faces[int(temp[0].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[3].strip()))
                            adjacent_faces[int(temp[3].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[3].strip())].add(int(temp[0].strip()))
                        dataVTK+=str(no_vertices+1)+","+row+" \n"
                        count+=1
                    else:
                        no_vertices = row.count(",")
                        temp = row.split(",")
                        if no_vertices==2:
                            no_triangles+=1
                            adjacent_faces[int(temp[0].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[0].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[1].strip()))
                        else:
                            adjacent_faces[int(temp[0].strip())].add(int(temp[3].strip()))
                            adjacent_faces[int(temp[0].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[0].strip()))
                            adjacent_faces[int(temp[1].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[1].strip()))
                            adjacent_faces[int(temp[2].strip())].add(int(temp[3].strip()))
                            adjacent_faces[int(temp[3].strip())].add(int(temp[2].strip()))
                            adjacent_faces[int(temp[3].strip())].add(int(temp[0].strip()))
                        dataVTK += str(no_vertices+1)+","+ row+" \n"
                        count += 1
            if count==no_points+no_polygons:
                break
    count=0

    #specifying vertex ordering for polygons
    index = dataVTK.find("POLYGONS")
    dataVTK=dataVTK[:(index+8)]+" "+str(no_polygons)+" "+str(no_polygons*5-no_triangles)+"\n" + dataVTK[index+8:]
    dataAtt ="POINT_DATA "+str(no_points)+" \n" \
            "SCALARS point_scalars float 1 \n" \
             "LOOKUP_TABLE my_table"+"\n"
    dataVTK += dataAtt
    dataVTK = re.sub(",", " ", dataVTK)
    scalar_val=[0.0]
    max_val=0

    #specifying scalar values at vertices
    for row in data_list:
        if(count==0):
            dataVTK +="0"+"\n"
            count+=1
        elif(count>skip and count-skip<no_points):
            n=len(row)
            val=float(row[n-row[::-1].find(",")+1:])
            scalar_val.append(val)
            if(val>max_val):
                max_val=val
            dataVTK+=str(val)+ " \n"
            count += 1
        else:
            count+=1

    #perform linear interpolation to determine scalar value at a vertex
    interpolated_scalars=set()
    last_color = 0
    for i in range(0, len(scalar_val)):
        neighbors=adjacent_faces[i]
        sum_scalars=scalar_val[i]
        count=1
        if(len(neighbors)!=0):
            for j in neighbors:
                if(j!=i):
                    sum_scalars+=scalar_val[j]
                    count+=1
            last_color=sum_scalars/(count)
        else:
            pass
        interpolated_scalars.add(last_color)

    #specifying color map for vertices
    dataVTK+="LOOKUP_TABLE my_table"+" "+str(len(interpolated_scalars))+"\n"

    interpolated_scalars=sorted(list(interpolated_scalars))
    max_val=max(interpolated_scalars)
    min_val=min(interpolated_scalars)
    for i in interpolated_scalars:
        color_val=(i-min_val)/(max_val-min_val)
        color=colormap(color_val)
        rgb=str(color[0])+" "+str(color[1])+" "+str(color[2])+" 1.0 \n"
        dataVTK+=rgb

    #store the formatted data in .vtk and .vtp format
    fileno=""
    if(stage_to_extract<10):
        fileno="0"+str(stage_to_extract)
    else:
        fileno=str(stage_to_extract)
    with open('Crack Data\Plane\VTK\crack_data_vtk'+ \
              fileno+'.vtk', mode='w') as file:
        file.write(dataVTK)

    vtkf='Crack Data\Plane\VTK\crack_data_vtk'+fileno+'.vtk'
    vtpf='Crack Data\Plane\VTP\crack_data_vtp'+fileno+'.vtp'
    vtk2vtp(vtkf, vtpf, binary=False)

#source: https://gist.github.com/thomasballinger/1281457
def vtk2vtp(invtkfile, outvtpfile, binary=False):
    """What it says on the label"""
    reader = vtk.vtkPolyDataReader()
    reader.SetFileName(invtkfile)

    writer = vtk.vtkXMLPolyDataWriter()
    writer.SetFileName(outvtpfile)
    if binary:
        writer.SetFileTypeToBinary()
    writer.SetInputConnection(reader.GetOutputPort())
    writer.Update()

#link all.vtp files to produce a time varying data set
def createPVDFile(no_of_stages):
    header="<?xml version=\"1.0\"?> \n" \
                "<VTKFile type=\"Collection\" version=\"0.1\">\n" \
                    "  <Collection>\n"

    tail="  </Collection>\n" \
            "</VTKFile>\n"

    data=header

    for i in range(1,no_of_stages+1):
        if i<10:
            data += "    <DataSet timestep=\"" + str(i - 1) + "\" file=\"crack_data_vtp0" + str(i) + ".vtp\"/>\n"
        else:
            data += "    <DataSet timestep=\"" + str(i - 1) + "\" file=\"crack_data_vtp" + str(i) + ".vtp\"/>\n"

    data+=tail

    with open('D:\CS 6635 Vis or Data Sceince\Final Project\Crack Data\Plane\VTP\crack_data.pvd', mode='w') as file:
        file.write(data)

#create a cool to warm color map for the input data set based on max. crack opening
def createColorMap(data_formatted, no_of_stages):
    data_list=data_formatted.split("\n")
    scalar_val=[0]
    skip=0
    stage=0
    count=0
    for row in data_list:
        if row[:4] == "ZONE":
            stage += 1
        if(stage <no_of_stages):
            skip+=1
        if stage==no_of_stages:
            if row[:4] == "ZONE":
                i = row.find("N=")
                no_points = int(row[i + 2:i + row[i:].find(",")]) + 1
                count=1
            else:
                if (count < no_points):
                    n = len(row)
                    val = float(row[n - row[::-1].find(",") + 1:])
                    scalar_val.append(val)
                    count+=1
    max_val=max(scalar_val)
    min_val=min(scalar_val)
    range_val= len(set(scalar_val))
    for i in range(0, range_val):
        scalar_val[i]=(scalar_val[i]-min_val)/(max_val-min_val)
    color=cm.get_cmap('coolwarm', range_val)
    newcolors=color(np.linspace(0, 1, range_val))
    newcmp = ListedColormap(newcolors, name='CoolWarm')
    '''
    fg, ax = plt.subplots()
    psm = ax.pcolormesh(np.array([sorted(scalar_val)]), cmap=newcmp, rasterized=True, vmin=0, vmax=1)
    fg.colorbar(psm, ax=ax)
    plt.show()
    '''
    return newcmp

def main():
    no_of_stages = 98 #total number of stages of crack opening
    createfolder() #create folder structure
    data_formatted=readData("sp2_lc2_test_crack2.dat") #input file
    colormap=createColorMap(data_formatted,no_of_stages) #generate a colormap
    convertToCSV(data_formatted, no_of_stages) #convert to csv(optional)
    for i in range(1,no_of_stages+1):
        convertToVTK(data_formatted, colormap, i) #generate a .vtk and .vtp file for each stage
    createPVDFile(no_of_stages) #create a .pvd file linking all stages

if __name__ == '__main__':
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import time
import os
import pigpio
import syslog
import shutter as sh
import zangi_gui as gui
import pygame
from pygame.locals import *
import sys

# global
SleepStepSec = 0.1

preview_numb = 0

loop = True

power_button = 6
preview_button = 26
shutter_button = 5

# gpio
pi = pigpio.pi()
pi.set_mode(power_button, pigpio.INPUT)
pi.set_mode(preview_button, pigpio.INPUT)
pi.set_mode(shutter_button, pigpio.INPUT)
pi.set_pull_up_down(power_button, pigpio.PUD_UP)
pi.set_pull_up_down(preview_button, pigpio.PUD_UP)
pi.set_pull_up_down(shutter_button, pigpio.PUD_UP)

shutter_state = 0

def go_home():
    global preview_numb
    preview_numb = sh.shutter_numb
    gui.screen_home()

def cb_power(gpio, level, tick):
    global shutter_state
    # print (gpio, level, tick)
    if KeepWatchForSeconds(0.5, gpio):
        if KeepWatchForSeconds(5, gpio):
            CallShutdown()

        else:
            if gui.hmi_state == gui.PRVIEW_STATE:
                go_home()
            else:
                if shutter_state == 0:
                    sh.camera.start_preview()
                    shutter_state = 1
                else:
                    sh.camera.stop_preview()
                    gui.screen_shutter()
                    # sh.cameraLoad()
                    sh.shutter()
                    # sh.cameraSave()
                    time.sleep(2)
                    go_home()
                    shutter_state = 0

def cb_shutter(gpio, level, tick):
    # print (gpio, level, tick)
    if KeepWatchForSeconds(0.5, gpio):
        sh.camera.start_preview()
        time.sleep(3)
        sh.camera.stop_preview()
        gui.screen_shutter()
        sh.shutter()
        time.sleep(2)
        go_home()

def cb_preview(gpio, level, tick):
    # print (gpio, level, tick)
    if KeepWatchForSeconds(0.5, gpio):
        preview()

def KeepWatchForSeconds(seconds, pin_numb):
    GoFlag = True
    while seconds > 0:
        time.sleep(SleepStepSec)
        seconds -= SleepStepSec
        if (pi.read(pin_numb) == True):
            GoFlag = False
            break
    return GoFlag

def CallShutdown():
    # print("Going shutdown by GPIO.")
    sh.camera.close()
    syslog.syslog(syslog.LOG_NOTICE, "Going shutdown by GPIO.")
    os.system("/sbin/shutdown -h now 'Poweroff by GPIO'")

def preview():
    # print ("preview")
    global preview_numb
    if preview_numb == 0:
        gui.screen_nophoto()
    else:
        filename = os.path.join(sh.photo_dir, str("{0:06d}".format(preview_numb)) + '.jpg')
        gui.screen_preview(filename)

        preview_numb -= 1
        if preview_numb < 1:
            preview_numb = sh.shutter_numb

cb1 = pi.callback(power_button, pigpio.FALLING_EDGE, cb_power)
cb2 = pi.callback(preview_button, pigpio.FALLING_EDGE, cb_preview)
cb3 = pi.callback(shutter_button, pigpio.FALLING_EDGE, cb_shutter)

if __name__ == '__main__':
    if not pi.connected:
        exit()

    sh.setting()
    sh.loadFile()
    preview_numb = sh.shutter_numb

    gui.screen_opening()

    gui.screen_home()

    while loop == True:
        gui.fpsclock.tick(10)
        for event in pygame.event.get():
            if event.type == KEYDOWN:
                if event.key == K_ESCAPE:
                    sh.camera.close()
                    sys.exit()


import json
import socketserver
import threading
import time
from http.server import BaseHTTPRequestHandler, HTTPServer
from io import BytesIO
import depthai as dai
import cv2
from PIL import Image
from pathlib import Path
import numpy as np
import pickle
import select
import socket
import argparse
from helpers.server_classes import TCPServerRequest, VideoStreamHandler, ThreadedHTTPServer

"""
   Parsing arguments
"""

parser = argparse.ArgumentParser()
parser.add_argument("-d", "--device", help="Choose host: \n0 - delta simulation\n1 - real delta",
                    type=int, choices=[0, 1], default=0)
parser.add_argument("-i", "--ip", help="Set http and json servers ip-s. The default ip would be localhost",
                    type=str, default='localhost')
parser.add_argument("-p", "--preview", help="Choose preview: \n0 - preview off\n1 - preview on",
                    type=int, choices=[0, 1], default=1)
parser.add_argument("-D", "--depth", help="Choose depth: \n0 - depth off\n1 - depth on",
                    type=int, choices=[0, 1], default=1)

args = parser.parse_args()

if args.device == 0:
    delta_host, delta_port = "127.0.0.1", 2137
else:
    delta_host, delta_port = "192.168.0.155", 10

IPAddress = args.ip

if args.preview:
    previewBool = True
else:
    previewBool = False

if args.depth:
    depthBool = True
else:
    depthBool = False

# PORTS
HTTP_SERVER_PORT = 8090
HTTP_SERVER_PORT2 = 8080
if depthBool:
    HTTP_SERVER_PORT3 = 8070
JSON_PORT = 8060

"""
    Define pipeline & nodes
"""

pipeline = dai.Pipeline()

if depthBool:
    camRgb = pipeline.create(dai.node.ColorCamera)
    detectionNetwork = pipeline.create(dai.node.YoloSpatialDetectionNetwork)
    monoLeft = pipeline.create(dai.node.MonoCamera)
    monoRight = pipeline.create(dai.node.MonoCamera)
    stereo = pipeline.create(dai.node.StereoDepth)
else:
    camRgb = pipeline.create(dai.node.ColorCamera)
    detectionNetwork = pipeline.create(dai.node.YoloDetectionNetwork)

# outputs nodes
if depthBool:
    nnNetworkOut = pipeline.create(dai.node.XLinkOut)
    xoutNN = pipeline.create(dai.node.XLinkOut)
    xoutRgb = pipeline.create(dai.node.XLinkOut)
    xoutDepth = pipeline.create(dai.node.XLinkOut)

    xoutRgb.setStreamName("rgb")
    xoutNN.setStreamName("detections")
    nnNetworkOut.setStreamName("nnNetwork")
    xoutDepth.setStreamName("depth")
else:
    xoutNN = pipeline.create(dai.node.XLinkOut)
    xoutRgb = pipeline.create(dai.node.XLinkOut)

    xoutRgb.setStreamName("rgb")
    xoutNN.setStreamName("detections")

"""
    Define pipeline nodes properties
"""

# nodes properties
camRgb.setPreviewSize(416, 416)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
camRgb.setInterleaved(False)
camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.BGR)
camRgb.setFps(40)

if depthBool:
    monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
    monoLeft.setCamera("left")
    monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P)
    monoRight.setCamera("right")

    # setting node configs
    stereo.setDefaultProfilePreset(dai.node.StereoDepth.PresetMode.HIGH_DENSITY)

    # Align depth map to the perspective of RGB camera, on which inference is done
    stereo.setDepthAlign(dai.CameraBoardSocket.CAM_A)
    stereo.setOutputSize(monoLeft.getResolutionWidth(), monoLeft.getResolutionHeight())
    stereo.setSubpixel(True)

"""
    Configure Yolo NN model
"""

# blob model path
detectionNetwork.setBlobPath(Path("config/yoloModel.blob"))

# open NN config
configPath = Path("config/yoloConfig.json")

if not configPath.exists():
    raise ValueError(f"Path {configPath} does not exist!")

print("Loading Yolo config...")
with open(configPath) as file:
    config = json.load(file)
nnConfig = config["nn_config"]
if nnConfig:
    print("Successfully loaded config")

if depthBool:
    # spatial Yolo detection parameters
    detectionNetwork.input.setBlocking(False)
    detectionNetwork.setBoundingBoxScaleFactor(0.5)
    detectionNetwork.setDepthLowerThreshold(100) # Min 10 centimeters
    detectionNetwork.setDepthUpperThreshold(5000) # Max 5 meters

# configure Yolo
detectionNetwork.setNumClasses(nnConfig["NN_specific_metadata"]["classes"])
detectionNetwork.setCoordinateSize(nnConfig["NN_specific_metadata"]["coordinates"])
detectionNetwork.setAnchors(nnConfig["NN_specific_metadata"]["anchors"])
detectionNetwork.setAnchorMasks(nnConfig["NN_specific_metadata"]["anchor_masks"])
detectionNetwork.setIouThreshold(nnConfig["NN_specific_metadata"]["iou_threshold"])
detectionNetwork.setConfidenceThreshold(nnConfig["NN_specific_metadata"]["confidence_threshold"])

# get labels
labels = config["mappings"]["labels"]

"""
    Link pipeline nodes
"""
if depthBool:
    monoLeft.out.link(stereo.left)
    monoRight.out.link(stereo.right)

    stereo.depth.link(detectionNetwork.inputDepth)

    camRgb.preview.link(detectionNetwork.input)

    detectionNetwork.passthrough.link(xoutRgb.input)  # TODO should be sync with detection ?
    detectionNetwork.passthroughDepth.link(xoutDepth.input)
    detectionNetwork.outNetwork.link(nnNetworkOut.input)
    detectionNetwork.out.link(xoutNN.input)

else:
    camRgb.preview.link(detectionNetwork.input)
    detectionNetwork.passthrough.link(xoutRgb.input)
    detectionNetwork.out.link(xoutNN.input)

"""
    Start servers
"""

# start TCP data server (JSON)
try:
    server_TCP = socketserver.TCPServer(("127.0.0.1", JSON_PORT), TCPServerRequest)
    th = threading.Thread(target=server_TCP.serve_forever)
    th.daemon = True
    th.start()
except Exception as e:
    print(e)

# start MJPEG HTTP Servers
try:
    server_HTTP = ThreadedHTTPServer((args.ip, HTTP_SERVER_PORT), VideoStreamHandler)
    th2 = threading.Thread(target=server_HTTP.serve_forever)
    th2.daemon = True
    th2.start()
except Exception as e:
    print(e)

try:
    server_HTTP2 = ThreadedHTTPServer((args.ip, HTTP_SERVER_PORT2), VideoStreamHandler)
    th3 = threading.Thread(target=server_HTTP2.serve_forever)
    th3.daemon = True
    th3.start()
except Exception as e:
    print(e)

if depthBool:
    try:
        server_HTTP3 = ThreadedHTTPServer((args.ip, HTTP_SERVER_PORT3), VideoStreamHandler)
        th4 = threading.Thread(target=server_HTTP3.serve_forever)
        th4.daemon = True
        th4.start()
    except Exception as e:
        print(e)

# connect to device and start pipeline
with dai.Device(pipeline) as device:
    print("DepthAI running.")
    print(f"Navigate to '{str(IPAddress)}:{str(HTTP_SERVER_PORT)}' for normal video stream.")
    print(f"Navigate to '{str(IPAddress)}:{str(HTTP_SERVER_PORT2)}' for warped video stream.")
    if depthBool:
        print(f"Navigate to '{str(IPAddress)}:{str(HTTP_SERVER_PORT3)}' for depth heatmap video stream.")
    print(f"Navigate to '{str(IPAddress)}:{str(JSON_PORT)}' for detection data in json format.")

    # load transformation matrix
    with open("perspectiveCalibration/calibration_result", "rb") as ifile:
        transformation_matrix = pickle.load(ifile)

    if depthBool:
        # output queues will be used to get the rgb frames and nn data from the outputs defined above
        previewQueue = device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
        detectionNNQueue = device.getOutputQueue(name="detections", maxSize=4, blocking=False)
        depthQueue = device.getOutputQueue(name="depth", maxSize=4, blocking=False)
        networkQueue = device.getOutputQueue(name="nnNetwork", maxSize=4, blocking=False)
    else:
        previewQueue = device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
        detectionNNQueue = device.getOutputQueue(name="detections", maxSize=4, blocking=False)

    startTime = time.monotonic()
    counter = 0
    fps = 0
    color = (255, 255, 255)

    while True:
        inPreview = previewQueue.get()
        inDet = detectionNNQueue.get()
        if depthBool:
            depthBool = depthQueue.get()
            inNN = networkQueue.get()

        frame = inPreview.getCvFrame()
        frame_copy = frame
        if depthBool:
            depthFrame = depthBool.getFrame()  # depthFrame values are in millimeters
            depth_downscaled = depthFrame[::4]
            min_depth = np.percentile(depth_downscaled[depth_downscaled != 0], 1)
            max_depth = np.percentile(depth_downscaled, 99)
            depthFrameColor = np.interp(depthFrame, (min_depth, max_depth), (0, 255)).astype(np.uint8)
            depthFrameColor = cv2.applyColorMap(depthFrameColor, cv2.COLORMAP_HOT)

        counter += 1
        current_time = time.monotonic()
        if (current_time - startTime) > 1:
            fps = counter / (current_time - startTime)
            counter = 0
            startTime = current_time

        detections = inDet.detections

        # If the detections is available, draw bounding boxes on it and show the frame
        height = frame.shape[0]
        width = frame.shape[1]

        # prepare dictionary for json format send
        send = {el: [] for el in labels}

        for detection in detections:

            # TODO delete?
            # if depthBool:
            #     roiData = detection.boundingBoxMapping
            #     roi = roiData.roi
            #     roi = roi.denormalize(depthFrameColor.shape[1], depthFrameColor.shape[0])
            #     topLeft = roi.topLeft()
            #     bottomRight = roi.bottomRight()
            #     xmin = int(topLeft.x)
            #     ymin = int(topLeft.y)
            #     xmax = int(bottomRight.x)
            #     ymax = int(bottomRight.y)

            # Denormalize bounding box
            x1 = int(detection.xmin * width)
            x2 = int(detection.xmax * width)
            y1 = int(detection.ymin * height)
            y2 = int(detection.ymax * height)

            try:
                label = labels[detection.label]
            except:
                label = detection.label

            # bbox middle coordinates
            bbox_x, bbox_y = int((x1 + x2) // 2), int((y1 + y2) // 2)
            if transformation_matrix.any():
                # if perspective calibration was done calculate detection (x,y) on warped img
                t_bbox_x, t_bbox_y, scale = np.matmul(transformation_matrix, np.float32([bbox_x, bbox_y, 1]))
                t_bbox_x, t_bbox_y = int(t_bbox_x / scale), int(t_bbox_y / scale)
            else:
                t_bbox_x, t_bbox_y = None, None

            if depthBool:
                cv2.putText(frame, str(label), (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
                cv2.putText(frame, "{:.2f}".format(detection.confidence * 100), (x1 + 10, y1 + 35),
                            cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
                cv2.putText(frame, f"X: {int(detection.spatialCoordinates.x)} mm", (x1 + 10, y1 + 50),
                            cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
                cv2.putText(frame, f"Y: {int(detection.spatialCoordinates.y)} mm", (x1 + 10, y1 + 65),
                            cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
                cv2.putText(frame, f"Z: {int(detection.spatialCoordinates.z)} mm", (x1 + 10, y1 + 80),
                            cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)

            else:
                cv2.putText(frame, str(label), (x1 + 10, y1 + 20), cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)
                cv2.putText(frame, "{:.2f}".format(detection.confidence * 100), (x1 + 10, y1 + 35),
                            cv2.FONT_HERSHEY_TRIPLEX, 0.5, color)

            cv2.rectangle(frame, (x1, y1), (x2, y2), color, cv2.FONT_HERSHEY_SIMPLEX)

            # append "send" json file
            if depthBool:
                spatialXYZ = (detection.spatialCoordinates.x, detection.spatialCoordinates.y,
                                   detection.spatialCoordinates.z)
            else:
                spatialXYZ = (None, None, None)

            det = {"x_max": detection.xmax, "x_min": detection.xmin, "y_max": detection.ymax, "y_min": detection.ymin,
                   "middle": (bbox_x, bbox_y), "middle_transformed": (t_bbox_x, t_bbox_y), "conf": detection.confidence,
                   "spatial_xyz": spatialXYZ}
            send[label].append(det)

        # send birdview camera if perspective calibration was done
        if transformation_matrix.any():

            # transform frame
            transformed_frame = cv2.warpPerspective(frame_copy, transformation_matrix, (width, height))

            # draw circle for every bar recognized in new perspective
            for choclate_bar_name in send:
                for detected_bar in send[choclate_bar_name]:
                    coordinates = detected_bar["middle_transformed"]
                    cv2.circle(transformed_frame, coordinates, 5, (255, 255, 255), -1)
            server_HTTP2.frametosend = transformed_frame

        # encode json file and send it using TCP
        json_send = json.dumps(send)
        server_TCP.datatosend = json_send

        # send frames using http servers
        server_HTTP.frametosend = frame
        if depthBool:
            server_HTTP3.frametosend = depthFrameColor

        if previewBool:
            cv2.putText(frame, "NN fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.4, color)
            cv2.putText(transformed_frame, "NN fps: {:.2f}".format(fps), (2, frame.shape[0] - 4), cv2.FONT_HERSHEY_TRIPLEX, 0.4, color)

            cv2.imshow("rgb", frame)
            cv2.imshow("Transformed frame", transformed_frame)

            if depthBool:
                cv2.imshow("depth", depthFrameColor)

        if cv2.waitKey(1) == ord('q'):
            break


from dataclasses import dataclass, field
from typing import List, Optional
from ojp.natural_language_string_structure import NaturalLanguageStringStructure

__NAMESPACE__ = "http://www.siri.org.uk/siri"

@dataclass
class StopAssignmentStructure:
    """
    Type for assignment of a SCHEDULED STOP POINT to a specific QUAY or platform
    +SIRI v2.0.

    :ivar aimed_quay_ref: Physical QUAY to use according to the planned
        timetable. +SIRI v2.0
    :ivar aimed_quay_name: Scheduled Platform name. Can be used to
        indicate platfrom change. +SIRI v2.0
    :ivar expected_quay_ref: Physical QUAY to use accoring to the real-
        time prediction. +SIRI v2.0
    :ivar actual_quay_ref: Physical QUAY actually used. +SIRI v2.0
    """
    aimed_quay_ref: Optional[str] = field(
        default=None,
        metadata={
            "name": "AimedQuayRef",
            "type": "Element",
            "namespace": "http://www.siri.org.uk/siri",
        }
    )
    aimed_quay_name: List[NaturalLanguageStringStructure] = field(
        default_factory=list,
        metadata={
            "name": "AimedQuayName",
            "type": "Element",
            "namespace": "http://www.siri.org.uk/siri",
        }
    )
    expected_quay_ref: Optional[str] = field(
        default=None,
        metadata={
            "name": "ExpectedQuayRef",
            "type": "Element",
            "namespace": "http://www.siri.org.uk/siri",
        }
    )
    actual_quay_ref: Optional[str] = field(
        default=None,
        metadata={
            "name": "ActualQuayRef",
            "type": "Element",
            "namespace": "http://www.siri.org.uk/siri",
        }
    )


import pandas as pd
import chart_studio.plotly as py
from plotly.offline import init_notebook_mode,plot,download_plotlyjs
import plotly.graph_objects as go

import os

def excercise1():
    init_notebook_mode(connected=True)

    df = pd.read_csv(os.getcwd() + '/' + '2014_World_Power_Consumption' )

    print(df.head())

    data = dict( type='choropleth',
                locations=df['Country'],
                locationmode='country names',
                text=df['Text'],
                z=df['Power Consumption KWH'],
                colorbar= {'title' : '2014 World Power Consumption Data'})

    layout = dict(title= '2014 World Power Consumption Data',
                  geo= dict(showframe=False, projection= {'type' : 'robinson'}))

    choromap1 =go.Figure(data=[data], layout=layout)

    plot(choromap1)

def excercise2():
    df = pd.read_csv(os.getcwd() + '/' + '2012_Election_Data')

    print(df.columns)
    print(df.head())

    data=dict(type='choropleth',
              locations=df['State Abv'],
              locationmode='USA-states',
              z=df['Voting-Age Population (VAP)'],
              text=df['State'],
              colorbar={'title' : '2012 Election Data'})

    layout=dict(title='2012 Election Data',
                geo=dict(scope='usa', showlakes=True, lakecolor='rgb(85, 173, 240)'))

    choromap1 = go.Figure(data=[data], layout=layout)

    plot(choromap1)

if __name__ == "__main__":
    #excercise1()
    excercise2()


import requests

def converter(valor, moeda_destino):
    url = f'https://economia.awesomeapi.com.br/json/last/BRL-{moeda_destino}'
    response = requests.get(url)

    if response.status_code == 200:
        data = response.json()
        ask = data[f'BRL{moeda_destino}']['ask']
        return valor / float(ask)
    else:
        print(f"Erro: {response.status_code} ao acessar {url}")
        return None

valor_reais = float(input("Digite o valor em reais: "))
moeda_destino = input("Digite a moeda de destino (USD, EUR, GBP, ARS, BTC, CAD, AUD, JPY, CHF, CNY): ")

resultado = converter(valor_reais, moeda_destino.upper())

if resultado is not None:
    print(f"{valor_reais:.2f} BRL equivale a {resultado:.2f} {moeda_destino.upper()}")

Computational Thinking Usinf Python/exApiConversao.py


from django.db import models

class Profile(models.Model):
    FIRSTNAME_MAX_LENGTH = 20
    LASTNAME_MAX_LENGTH = 20

    first_name = models.CharField(
        max_length=FIRSTNAME_MAX_LENGTH,
        verbose_name='First Name',
    )

    last_name = models.CharField(
        max_length=LASTNAME_MAX_LENGTH,
        verbose_name='Last Name',
    )

    age = models.IntegerField(
        verbose_name='Age',
    )

    image = models.URLField(
        verbose_name='Link to Profile Image',
    )

    def __str__(self):
        return f'{self.first_name} {self.last_name}'

class Note(models.Model):
    TITLE_MAX_LENGTH = 30

    title = models.CharField(
        max_length=TITLE_MAX_LENGTH,
        verbose_name="Title",
    )

    image = models.URLField(
        verbose_name="Link To Image",
    )

    content = models.TextField(
        verbose_name="Content",
    )

    def __str__(self):
        return self.title


import os
from os import path
import pathlib
import shutil
import time

target_folder = "/home/bali/Downloads/"
is_image = ['.png', '.jpg', '.jpeg', '.tiff', '.bmp', '.gif', '.psd', '.webp', '.svg', '.raw', '.arw', '.cr2', '.nrw', '.k25', '.svgz', '.ai', '.eps', '.xcf']
is_video = video_file_extensions = [
    '.264', '.3g2', '.3gp', '.3gp2', '.3gpp', '.3gpp2', '.3mm', '.3p2', '.60d', '.787', '.89', '.aaf', '.aec', '.aep', '.aepx',
    '.aet', '.aetx', '.ajp', '.ale', '.am', '.amc', '.amv', '.amx', '.anim', '.aqt', '.arcut', '.arf', '.asf', '.asx', '.avb',
    '.avc', '.avd', '.avi', '.avp', '.avs', '.avs', '.avv', '.axm', '.bdm', '.bdmv', '.bdt2', '.bdt3', '.bik', '.bin', '.bix',
    '.bmk', '.bnp', '.box', '.bs4', '.bsf', '.bvr', '.byu', '.camproj', '.camrec', '.camv', '.ced', '.cel', '.cine', '.cip',
    '.clpi', '.cmmp', '.cmmtpl', '.cmproj', '.cmrec', '.cpi', '.cst', '.cvc', '.cx3', '.d2v', '.d3v', '.dat', '.dav', '.dce',
    '.dck', '.dcr', '.dcr', '.ddat', '.dif', '.dir', '.divx', '.dlx', '.dmb', '.dmsd', '.dmsd3d', '.dmsm', '.dmsm3d', '.dmss',
    '.dmx', '.dnc', '.dpa', '.dpg', '.dream', '.dsy', '.dv', '.dv-avi', '.dv4', '.dvdmedia', '.dvr', '.dvr-ms', '.dvx', '.dxr',
    '.dzm', '.dzp', '.dzt', '.edl', '.evo', '.eye', '.ezt', '.f4p', '.f4v', '.fbr', '.fbr', '.fbz', '.fcp', '.fcproject',
    '.ffd', '.flc', '.flh', '.fli', '.flv', '.flx', '.gfp', '.gl', '.gom', '.grasp', '.gts', '.gvi', '.gvp', '.h264', '.hdmov',
    '.hkm', '.ifo', '.imovieproj', '.imovieproject', '.ircp', '.irf', '.ism', '.ismc', '.ismv', '.iva', '.ivf', '.ivr', '.ivs',
    '.izz', '.izzy', '.jss', '.jts', '.jtv', '.k3g', '.kmv', '.ktn', '.lrec', '.lsf', '.lsx', '.m15', '.m1pg', '.m1v', '.m21',
    '.m21', '.m2a', '.m2p', '.m2t', '.m2ts', '.m2v', '.m4e', '.m4u', '.m4v', '.m75', '.mani', '.meta', '.mgv', '.mj2', '.mjp',
    '.mjpg', '.mk3d', '.mkv', '.mmv', '.mnv', '.mob', '.mod', '.modd', '.moff', '.moi', '.moov', '.mov', '.movie', '.mp21',
    '.mp21', '.mp2v', '.mp4', '.mp4v', '.mpe', '.mpeg', '.mpeg1', '.mpeg4', '.mpf', '.mpg', '.mpg2', '.mpgindex', '.mpl',
    '.mpl', '.mpls', '.mpsub', '.mpv', '.mpv2', '.mqv', '.msdvd', '.mse', '.msh', '.mswmm', '.mts', '.mtv', '.mvb', '.mvc',
    '.mvd', '.mve', '.mvex', '.mvp', '.mvp', '.mvy', '.mxf', '.mxv', '.mys', '.ncor', '.nsv', '.nut', '.nuv', '.nvc', '.ogm',
    '.ogv', '.ogx', '.osp', '.otrkey', '.pac', '.par', '.pds', '.pgi', '.photoshow', '.piv', '.pjs', '.playlist', '.plproj',
    '.pmf', '.pmv', '.pns', '.ppj', '.prel', '.pro', '.prproj', '.prtl', '.psb', '.psh', '.pssd', '.pva', '.pvr', '.pxv',
    '.qt', '.qtch', '.qtindex', '.qtl', '.qtm', '.qtz', '.r3d', '.rcd', '.rcproject', '.rdb', '.rec', '.rm', '.rmd', '.rmd',
    '.rmp', '.rms', '.rmv', '.rmvb', '.roq', '.rp', '.rsx', '.rts', '.rts', '.rum', '.rv', '.rvid', '.rvl', '.sbk', '.sbt',
    '.scc', '.scm', '.scm', '.scn', '.screenflow', '.sec', '.sedprj', '.seq', '.sfd', '.sfvidcap', '.siv', '.smi', '.smi',
    '.smil', '.smk', '.sml', '.smv', '.spl', '.sqz', '.srt', '.ssf', '.ssm', '.stl', '.str', '.stx', '.svi', '.swf', '.swi',
    '.swt', '.tda3mt', '.tdx', '.thp', '.tivo', '.tix', '.tod', '.tp', '.tp0', '.tpd', '.tpr', '.trp', '.ts', '.tsp', '.ttxt',
    '.tvs', '.usf', '.usm', '.vc1', '.vcpf', '.vcr', '.vcv', '.vdo', '.vdr', '.vdx', '.veg','.vem', '.vep', '.vf', '.vft',
    '.vfw', '.vfz', '.vgz', '.vid', '.video', '.viewlet', '.viv', '.vivo', '.vlab', '.vob', '.vp3', '.vp6', '.vp7', '.vpj',
    '.vro', '.vs4', '.vse', '.vsp', '.w32', '.wcp', '.webm', '.wlmp', '.wm', '.wmd', '.wmmp', '.wmv', '.wmx', '.wot', '.wp3',
    '.wpl', '.wtv', '.wve', '.wvx', '.xej', '.xel', '.xesc', '.xfl', '.xlmv', '.xmv', '.xvid', '.y4m', '.yog', '.yuv', '.zeg',
    '.zm1', '.zm2', '.zm3', '.zmv'
]
is_document = ['.pdf', '.doc', '.docx', '.odt', '.xls', '.xlsx', '.ods', '.ppt', '.pptx', '.txt']
is_program = ['.html', '.htm', '.php', '.py', '.sh', '.sql', '.rpm', '.deb', '.appimage', '.dart', '.css', '.js', '.jsx', '.env', '.json', '.md', '.htaccess', '.log', '.conf', '.xml', '.yml', '.key']
is_compressed = ['.tar', '.gz', '.zip', '.rar', '.xz', '.7zip']
is_font = ['.jfproj', '.ttf', '.pfa', '.woff', '.fnt', '.otf', '.woff2', '.odttf']

def move_file(destination_path, file_name, file_ext, file_name_plain):
    if not path.exists(destination_path):
        os.mkdir(destination_path)
    for files in os.listdir(destination_path):
        file_path = os.path.join(target_folder, files)
        if os.path.exists(destination_path+file_name):
            file_name_new = file_name_plain+"copy("+ str(round(time.time())) +")"+file_ext
    else:
        file_name_new = file_name

    if(os.path.exists(target_folder+file_name)):
        shutil.move(target_folder+file_name, destination_path+file_name_new)

for files in os.listdir(target_folder):
    file_path = os.path.join(target_folder, files)
    if os.path.isfile(file_path):
        file_extension = pathlib.Path(file_path).suffix
        file_name_plain = pathlib.Path(file_path).stem
        if file_extension.lower() in is_image:
            move_file(target_folder + "Images/", files, file_extension, file_name_plain)
        if file_extension.lower() in is_video:
            move_file(target_folder + "Videos/", files, file_extension, file_name_plain)
        if file_extension.lower() in is_document:
            move_file(target_folder + "Documents/", files, file_extension, file_name_plain)
        if file_extension.lower() in is_program:
            move_file(target_folder + "Programs/", files, file_extension, file_name_plain)
        if file_extension.lower() in is_compressed:
            move_file(target_folder + "Compressed/", files, file_extension, file_name_plain)
        if file_extension.lower() in is_font:
            move_file(target_folder + "Fonts/", files, file_extension, file_name_plain)
        else :
            move_file(target_folder + "Miscs/", files, file_extension, file_name_plain)


'''from functools import reduce
a=[1,2,3,4,5,6,7,8]
b=list(map(lambda x:x>3,a))
print(b)
c=list(filter(lambda x:x>3,a))
print(c)
d=reduce(lambda x,y:x+y,a)
print(d)
e=reduce(lambda x,y:x+y,map(lambda x:x+3,filter(lambda x:x>3,a)))
print(e)

quadratic_equation=lambda x,y:(x**2)+(y**2)
print(quadratic_equation(3,4))

def linear_equation(o,p):
    print(lambda o,p:(3*o)+(4*p))
linear_equation(2,2)

l=[5,6,4,3,7,8,1,9,12,43,21]
print(l.sort())'''
from functools import reduce
restart=('Y')
while restart not in ('n','N','no','NO'):
    print('1.Detect if the given number is octal number')
    print('2.pass and sum the list to the method and return value')
    print('3.check the number is less than 10')
    print('4.exit the menu')
    menu=int(input('enter your choice:'))
    if menu==1:
        try:
            num=int(input("enter octal number:"),8)
            print("num(decimal format):",num)
            print("num(octal format):",oct(num))
        except ValueError:
            print("Plese enter only Octal Value")
        restart='Y'
    elif menu==2:
        def my_function(numbers):
            for x in numbers:
                print(x)

        inputs=list(input('enter sequence of numbers').split())
        my_function(inputs)
        print('sum of numbers:',reduce(lambda x,y:x+y,inputs))
        restart = 'Y'
    elif menu==3:
        input=list(input('enter sequence of numbers').split())
        print(input)
        restart = 'Y'
    elif menu==4:
        print('Thank You')
        break


from django import forms
from .models import Schoolrool,Familymemberone,Familymembertwo

class Schoolroolform(forms.ModelForm):
    class Meta:
        model = Schoolrool
        fields = '__all__'
        exclude = ['province_ID','country_ID']
        labels = {
        'name':'姓名',
        'alread_name':'曾用名',
        'sex':'性别',
        'IDcard':'身份证号',
        'nation':'民族',
        'school':'学校',
        'grade':'年级',
        'class_bj':'班级',
        'by_type':'户口性质',
        'address':'现住地址',
        'is_child':'是否是独生子女',
        'is_leftover_child':'是否留守儿童',
        'is_healdy':'健康状况',
        'entrance_date':'入学年月',
        'entrance_go':'入学方式',
        'how_go_school':'就读方式',
        'is_disable':'是否残疾人',
        'country_ID':'国网学籍号',
        'province_ID':'省网学籍号',
        }
    def clean(self):
        name = self.cleaned_data['name']
        print(1111)
        return self.cleaned_data
#['alread_name','name','sex','IDcard','nation','school','grade','class_bj','by_type','address','is_child','is_leftover_child','is_healdy','entrance_date','entrance_go','how_go_school','is_disable','country_ID','province_ID']
#['member_name_one','member_ralation_one','member_address_one','member_phone_one','is_guardian_one','member_idcard_one','member_nation_one','member_job_one','member_duty_one']
#['member_name_two','member_nation_two','member_idcard_two','member_ralation_two','member_address_two','member_phone_two','is_guardian_two','member_job_two','member_duty_two']

class Familymemberoneform(forms.ModelForm):
    class Meta:
        model = Familymemberone
        fields = ['member_name_one','member_nation_one','member_idcard_one','member_ralation_one','member_address_one','member_phone_one','is_guardian_one','member_job_one','member_duty_one']
        labels ={
            'name':'姓名',
            'member_name_one':'成员1姓名',
            'member_ralation_one':'成员1关系',
            'member_address_one':'成员1地址',
            'member_phone_one':'成员1电话',
            'is_guardian_one':'成员1是否监护人',
            'member_idcard_one':'成员1身份证号',
            'member_nation_one':'成员1民族',
            'member_job_one':'成员1工作单位',
            'member_duty_one':'成员1职务',
        }

class Familymembertwoform(forms.ModelForm):
    class Meta:
        model = Familymembertwo
        fields = ['member_name_two','member_nation_two','member_idcard_two','member_ralation_two','member_address_two','member_phone_two','is_guardian_two','member_job_two','member_duty_two']
        labels ={
            'name':'姓名',
            'member_name_two':'成员2姓名',
            'member_ralation_two':'成员2关系',
            'member_address_two':'成员2地址',
            'member_phone_two':'成员2电话',
            'is_guardian_two':'成员2是否监护人',
            'member_idcard_two':'成员2身份证号',
            'member_nation_two':'成员2民族',
            'member_job_two':'成员2工作单位',
            'member_duty_two':'成员2职务',
        }

class Alterstudentinfo(forms.Form):
    IDcard = forms.CharField(label='身份证号',max_length=18)
'''
    def clean(self):
        IDcard = self.cleaned_data['IDcard']
        if Schoolrool.objects.filter(IDcard=IDcard).exists():
        else:
            raise forms.ValidationError("你查询的学生不存在，请先注册！")
        return IDcard

'''


import attr
from attr.validators import instance_of
from pathlib import Path
from typing import Union

@attr.s(frozen=True)
class BatFile:
    """File object path attribute and several metadata attributes.
    Instantiated by the ProcessFilelist class, passed to the ProcessData class
    Attributes: path, fname, experiment."""
    path = attr.ib(validator=instance_of(Path))
    fname = attr.ib(validator=instance_of(str))
    date = attr.ib(validator=instance_of(str))
    tent = attr.ib(validator=instance_of(str))
    # maybe add user input...
    env = attr.ib(validator=instance_of(str))
    # bat_name = attr.ib(validator=instance_of(str))
    # tag = attr.ib(validator=instance_of(str))
    # cond (right or left)

# @attr.s
class ProcessFile:
    """Pipeline to process a list of files.
    Reads attributes from filename and creates a list of EyeFile objects to pass.
    Attributes: filelist, invalid_files, eyedict.
    Methods: instantiate_eye_file, assert_csv, extract_file_attrs.
    """
    # filelist = attr.ib(validator=instance_of(list))
    # batitem = attr.ib() #  BatFile instances to pass forward
    # invalid_files = attr.ib(default=attr.Factory(list)) ### should add invalid files output ###
    def __init__(self, filelist):
        self.filelist = filelist
        self.invalid_files = []

    def get_file_attrs(self) -> None:
        """Analizes file attributes and instantiate BatFile objects"""
        for batfile in self.filelist:
            path = Path(batfile)
            fname = path.name
            if not self.assert_csv(path): # accepts only .csv files
                self.invalid_files.append(fname)
                continue

            fattrs = self.extract_file_attrs(fname)
            if not fattrs: # accepts files only if named in the appropriate pattern
                self.invalid_files.append(fname)
                continue
            date, tent = fattrs[0], fattrs[1]

            if 'slow' in fname:
                env = 'slow'
            elif 'fast' in fname:
                env = 'fast'
            elif 'test' in fname:
                env = 'test'
            else: # accepts only slow, fast or test files
                self.invalid_files.append(fname)
                continue
            self.instantiate_bat_file(path, fname, date, tent, env)

    def assert_csv(self, path: Path) -> bool:
        """Asserts that a file is a csv file"""
        return str.lower(path.suffix) == '.csv'

    def extract_file_attrs(self, fname: str) -> Union[list, bool]:
        """If the file named appropriately, extracts its attributes from filename"""
        fattrs = fname.split('_')
        # return fattrs if len(fattrs) == 10 else False
        return fattrs

    def instantiate_bat_file(self, path: Path, fname: str, date: str, tent: str, env: str) -> BatFile:
        """Instantiates BatFile objects"""
        self.batitem = BatFile(path=path, fname=fname, date=date, tent=tent, env=env) #to add user input too

if __name__ == "__main__":
    # filelist =    ['/Users/gonina/Dropbox/classes/machine_learning/project/1_clio/After feeding/food/Yovel_20190513_3_neg.txt','/Users/gonina/Dropbox/classes/machine_learning/project/1_clio/After feeding/food/Yovel_20190513_3_pos.txt']
    filelist =    ['/Users/gonina/Library/Mobile Documents/com~apple~CloudDocs/lab/python_codes/feeders/feeders analysis/2020-03-20-07_B_Shin_slow.csv']
    files = ProcessFile(filelist)
    files.get_file_attrs()

    print(files.batitem)


#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
#@File        :inverse_kinematics.py
#@Date        :2022/06/29 09:55:39
#@Author      :zerui chen
#@Contact     :zerui.chen@inria.fr

import torch
import numpy as np
from mano.manolayer import ManoLayer
from mano.rodrigues_layer import batch_rodrigues
from kornia.geometry.conversions import rotation_matrix_to_angle_axis

def ik_solver_mano(mano_shape, pred_joints):
    mano_layer = ManoLayer(flat_hand_mean=True, side="right", mano_root='../common/mano/assets', use_pca=False, center_idx=0)
    mano_layer = mano_layer.cuda()
    batch_size = pred_joints.shape[0]

    mano_pose = torch.eye(3).repeat(batch_size, 16, 1, 1).to(pred_joints.device)
    mano_axisang = torch.zeros((batch_size, 16, 3), dtype=torch.float32, device=pred_joints.device)
    target_joints = (pred_joints[:, :21] - pred_joints[:, [0]]).clone().detach()
    if mano_shape is None:
        target_shape = torch.zeros((batch_size, 10), dtype=torch.float32, device=pred_joints.device)
    else:
        target_shape = mano_shape.clone().detach()
    _, template_joints, _, _, _ = mano_layer(mano_axisang.reshape((batch_size, -1)), target_shape)

    P_0 = torch.cat([target_joints[:, [1]] - target_joints[:, [0]], target_joints[:, [5]] - target_joints[:, [0]], target_joints[:, [9]] - target_joints[:, [0]], target_joints[:, [13]] - target_joints[:, [0]], target_joints[:, [17]] - target_joints[:, [0]]], axis=1).transpose(1, 2)
    T_0 = torch.cat([template_joints[:, [1]] - template_joints[:, [0]], template_joints[:, [5]] - template_joints[:, [0]], template_joints[:, [9]] - template_joints[:, [0]], template_joints[:, [13]] - template_joints[:, [0]], template_joints[:, [17]] - template_joints[:, [0]]], axis=1).transpose(1, 2)
    H = torch.matmul(T_0, P_0.transpose(1, 2))
    U, S, V_T = torch.linalg.svd(H)
    V = V_T.transpose(1, 2)
    R = torch.matmul(V, U.transpose(1, 2)).to(pred_joints.device)

    det0 = torch.linalg.det(R)
    valid_idx = (abs(det0 + 1) > 1e-6).unsqueeze(-1).long()
    batch_id = torch.where(abs(det0 + 1) > 1e-6)[0]
    mano_axisang[batch_id, 0] = rotation_matrix_to_angle_axis(R)[batch_id]
    mano_pose[batch_id, 0] = R[batch_id]

    finger_list = [[0, 5, 6, 7, 8], [0, 9, 10, 11, 12], [0, 17, 18, 19, 20], [0, 13, 14, 15, 16], [0, 1, 2, 3, 4]]
    for group_idx, group in enumerate(finger_list):
        recon_joints = torch.zeros((batch_size, 5, 3), dtype=torch.float32, device=pred_joints.device)
        for joint_idx, joint in enumerate(group):
            if joint_idx < 2:
                continue

            vec_template = template_joints[:, group[joint_idx]] - template_joints[:, group[joint_idx - 1]]

            R_pa = R.clone()
            for i in range(joint_idx - 2):
                R_pa = torch.matmul(R_pa, mano_pose[:, group_idx * 3 + i + 1])

            recon_joints[:, joint_idx - 1] = torch.matmul(R_pa, (template_joints[:, group[joint_idx - 1]] - template_joints[:, group[joint_idx - 2]]).unsqueeze(-1)).squeeze(-1) + recon_joints[:, joint_idx - 2]

            vec_target = torch.matmul(R_pa.transpose(1, 2), (target_joints[:, group[joint_idx]] - recon_joints[:, joint_idx - 1]).unsqueeze(-1)).squeeze(-1)

            temp_axis = torch.cross(vec_template, vec_target)
            temp_axis = temp_axis / (torch.norm(temp_axis, dim=-1, keepdim=True) + 1e-7)
            overall_angle = torch.acos(torch.clamp(torch.einsum('bk, bk->b', vec_template, vec_target).unsqueeze(-1) / (torch.norm(vec_template, dim=-1, keepdim=True) + 1e-7) / (torch.norm(vec_target, dim=-1, keepdim=True) + 1e-7), -1 + 1e-7, 1 - 1e-7))
            mano_axisang[batch_id, group_idx * 3 + joint_idx - 2 + 1] = (overall_angle * temp_axis)[batch_id]
            local_R = batch_rodrigues(overall_angle * temp_axis).reshape(batch_size, 3, 3)
            mano_pose[batch_id, group_idx * 3 + joint_idx - 2 + 1] = local_R[batch_id]

    verts, joints, _, global_trans, rot_center = mano_layer(mano_axisang.reshape((batch_size, -1)), target_shape)
    verts += pred_joints[:, [0]]
    joints += pred_joints[:, [0]]

    mean_pose = torch.from_numpy(np.array(mano_layer.smpl_data['hands_mean'], dtype=np.float32)).to(pred_joints.device)
    mano_axisang = mano_axisang.reshape((batch_size, -1))

    results = {"verts": verts, "joints": joints, "shape": target_shape, "pose": mano_axisang, "global_trans":global_trans, "rot_center": rot_center, 'vis': valid_idx, 'mean_pose': mean_pose}

    return results


import pandas as pd
import numpy as np
import statsmodels.api as sm
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.stattools import grangercausalitytests
from statsmodels.tsa.api import VAR # vector autoregression
from statsmodels.stats.stattools import durbin_watson
from statsmodels.tsa.base.datetools import dates_from_str
import matplotlib.pyplot as plt
import seaborn as sns
import warnings

warnings.filterwarnings("ignore")

sns.set(rc={'figure.figsize':(20, 4)})

get_ipython().run_line_magic('matplotlib', 'inline')

data = pd.read_csv("ModifiedChange-ETH.csv")
data = data.set_index("Time")
data.head()

def grangers_causality_matrix(data, variables, test='ssr_chi2test', maxlag=2, verbose=True):

    dataset = pd.DataFrame(np.zeros((len(variables), len(variables))), columns=variables, index=variables)

    for c in dataset.columns:
        for r in dataset.index:
            test_result = grangercausalitytests(data[[r,c]], maxlag=maxlag, verbose=False)
            p_values = [round(test_result[i+1][0][test][1], 5) for i in range(maxlag)]
            if verbose:
                print(f'Y = {r}, X = {c}, P Values = {p_values}')

            min_p_value = np.min(p_values)
            dataset.loc[r,c] = min_p_value

    dataset.columns = [var + '_x' for var in variables]

    dataset.index = [var + '_y' for var in variables]

    return dataset

grangers_causality_matrix(data, variables=data.columns)

BTC = data["BTC_Growth"].values
ETH = data["ETH_Growth"].values

BTC_result = adfuller(BTC)
print('BTC - ADF Statistic: %f' % BTC_result[0])
print('BTC - p-value: %f' %  BTC_result[1])

ETH_result = adfuller(ETH)
print('\nETH - ADF Statistic: %f' % ETH_result[0])
print('ETH - p-value: %f' %  ETH_result[1])

model = VAR(data)
lag_orders = model.select_order(25)
lag_orders.summary()

lag_order = 1
results = model.fit(lag_order, ic="aic")
results.summary()

dw_r = durbin_watson(results.resid)

for col, val in zip(data.columns, dw_r):
    print(col, ':', round(val, 2))

fc = results.forecast(data.values[-lag_order:], steps=20
                     )
fc = pd.DataFrame(fc, columns=["BTC_forecast", "ETH_forecast"])
fc

results.plot_forecast(20);


"""
- This sheet will process and clean data in DataFrame follow standard
- After that, return df_clean

"""
from Extract_To_DataFrame import *
from pyspark.sql import SparkSession
from pyspark.sql.functions import split, col, lit
from queries_db import *
from Parameter import *
from datetime import date
from pyspark.sql.functions import udf
import datetime as dt

def Process_Mapping_Dim(df_port,df_state, df_mode, df_visa, df_country, Dim):
    """
    - process mapping dim
    - input dim data frame from extract module
    - output: clean dim data Frame after process
    """
    Dimmesion = Dim
    spark = create_sparksession()

    if Dimmesion == 'port_dim':
        # read dimmension df:

        # clean df_port: split port_cd -->  update port_cd and add new column name "sta_cd"
        split_value = df_port['port_nm'].str.split(", ", n = 1, expand = True)
        df_port['port_nm'] = split_value[0]
        df_port['sta_cd']  = split_value[1]

        # clean record have value not in US or not defined: join with df_state
        df_state_join = df_state['sta_cd'].str.replace("\t", "")
        df_port_clean = pd.merge(df_port, df_state_join, left_on= 'sta_cd', right_on='sta_cd', how='inner')

        # final --> select cloumn: 'port_cd', 'port_nm','sta_cd', 'lst_mdf_data_x'
        df_port_clean = df_port_clean[['port_cd', 'port_nm','sta_cd', 'lst_mdf_data']]
        df_port_clean = spark.createDataFrame(df_port_clean)

        return df_port_clean

    elif Dimmesion == 'state_dim':
        df_state['sta_cd'] = df_state['sta_cd'].str.replace("\t", "")
        df_state_clean = spark.createDataFrame(df_state)
        return df_state_clean

    elif Dimmesion == 'mode_dim':
        df_mode_clean = spark.createDataFrame(df_mode)
        return df_mode_clean
    # clean visa dim:
    elif Dimmesion == 'visa_dim':
        df_visa_clean = spark.createDataFrame(df_visa)
        return df_visa_clean

    # clean country: có nhiều mã quốc gia không hợp lệ, không tồn tại ==> chuyển về 'OTHER'
    elif Dimmesion == 'country_dim':
        df_country.loc[df_country['cntr_nm'].str.contains('No Country'), 'cntr_nm'] = 'OTHER'
        df_country.loc[df_country['cntr_nm'].str.contains('INVALID'), 'cntr_nm'] = 'OTHER'
        df_cntr_clean = spark.createDataFrame(df_country)

        return df_cntr_clean

def Process_Temp(df_temp):
    """
    - xử lý và làm sạch dữ liệu Temp theo phương án làm sạch đã đưa ra
    - sau đó, trả về df_temp_clean

    """
    # time in 2016:
    # only US:
    df_temp_clean = df_temp.filter(df_temp.Country =='US')\
                           .filter(df_temp.Year == '2016')\
                           .withColumn("lst_mdf_data", lit(date.today().strftime("%Y%m%d")))\
                           .withColumnRenamed("State", "sta_nm")\
                           .withColumnRenamed("City", "cit")\
                           .withColumnRenamed("Month", "mon")\
                           .withColumnRenamed("Year", "yr")\
                           .withColumnRenamed("AvgTemperature", "avg_temp")\
                           .drop("Region")\
                           .drop("Country")
    print("already cleaned temp data")
    return df_temp_clean

def Process_Airport(df_airport):
    """
    process airport data
    """
    # clean data:
    df_airport_clean = df_airport.filter(df_airport.iso_country == 'US')\
                             .withColumn("sta_cd", split(col("iso_region"), "-")[1])\
                             .withColumn("lst_mdf_data", lit(date.today().strftime("%Y%m%d")))\
                             .withColumnRenamed("ident", "arpt_cd")\
                             .withColumnRenamed("elevation_ft", "ele_ft")\
                             .withColumnRenamed("iso_country", "cntr")\
                             .withColumnRenamed("name", "arpt_nm")\
                             .withColumnRenamed("municipality", "mun")\
                             .withColumnRenamed("type", "arpt_tp")\
                             .withColumnRenamed("gps_code", "gps_cd")\
                             .withColumnRenamed("local_code", "local_cd")\
                             .drop("iso_region")\
                             .drop("coordinates")\
                             .drop("continent")\
                             .drop("iata_code")
    print("already cleaned airport data")
    return df_airport_clean

def Process_city_demo(df_city_demo):

    # process:delelte record has state field null. After that clean duplicate record at 3 field: state, city, race:
    df_city_demo_clean = df_city_demo.filter(df_city_demo.State.isNotNull())\
                                     .withColumn("LST_MDF_DATA", lit(date.today().strftime("%Y%m%d")))\
                                     .dropDuplicates(subset = ['State', 'City', 'Race'])\
                                     .withColumnRenamed("City", "CIT")\
                                     .withColumnRenamed("Median Age", "MD_AG")\
                                     .withColumnRenamed("Male Population", "M_PPLT")\
                                     .withColumnRenamed("Female Population", "FM_PPLT")\
                                     .withColumnRenamed("Total Population", "TT_PPLT")\
                                     .withColumnRenamed("Number of Veterans", "NUM_VTR")\
                                     .withColumnRenamed("Foreign-born", "FRN_BR")\
                                     .withColumnRenamed("Average Household Size", "AVR_HOSE_SZ")\
                                     .withColumnRenamed("State Code", "STA_CD")\
                                     .withColumnRenamed("Race", "RAC")\
                                     .withColumnRenamed("Count", "CNT")\
                                     .drop("State")
    print("already cleaned city_demo data")
    return df_city_demo_clean

def Process_i94_immigration(df_i94,df_state_clean, df_visa_clean, df_mode_clean):
    """
    - INPUT: dataframe df_94, df_state, df_visa, df_mode
    - OUTPUT: df_i94_clean

    """
    # call datafram extract
    spark = create_sparksession()
    # make temp table
    df_state_clean.createOrReplaceTempView("df_state_clean")
    df_visa_clean.createOrReplaceTempView("df_visa_clean")
    df_mode_clean.createOrReplaceTempView("df_mode_clean")
    df_i94.createOrReplaceTempView("df_i94")

    # clean data:
    df_i94_clean = spark.sql("""
                            select
                                   i.cicid as id
                                  ,i.i94yr as yr
                                  ,i.i94mon as mon
                                  ,i.i94cit as cit
                                  ,i.i94res as res
                                  ,i.i94port as port
                                  ,i.arrdate as arrdt
                                  ,coalesce(m.mode_cd, 'other') as mode
                                  ,coalesce(s.sta_cd, '99') as state
                                  ,i.depdate as depdt
                                  ,i.i94bir as bir
                                  ,coalesce(v.visa_cd, 'other') as visa_cd
                                  ,i.dtadfile as dtadfile
                                  ,i.occup as occup
                                  ,i.gender as gender
                                  ,i.airline as airline
                                  ,i.fltno as fltno
                                  ,i.visatype as visa_tp
                                  ,i.lst_mdf_data as lst_mdf_data

                            from df_i94 i left join df_mode_clean m on i.i94mode = m.mode_cd
                                      left join df_visa_clean v on i.i94visa = v.visa_cd
                                      left join df_state_clean s on i.i94addr = s.sta_cd

                            """)
    get_date = udf(lambda x: (dt.datetime(1960, 1, 1).date() + dt.timedelta(x)).isoformat() if x else None)
    # clean data: only date add data in file > 20160101 and drop duplicate value of some field
    df_i94_clean = df_i94_clean.filter(df_i94_clean.dtadfile > '20160101')\
                            .dropDuplicates(subset = ['id','yr', 'mon', 'cit', 'port', 'arrdt'])\
                            .withColumnRenamed("id", "i94_id")\
                            .withColumnRenamed("port", "port_cd")\
                            .withColumnRenamed("state", "sta_cd")\
                            .withColumnRenamed("mode", "mode_cd")\
                            .withColumn("arrdt", get_date(df_i94_clean.arrdt))
    print("already processed i94 data")

    return df_i94_clean


import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
#https://stackoverflow.com/questions/61867945/python-import-error-cannot-import-name-six-from-sklearn-externals
#Fixed six import issue using stackoverflow from above
import six
import sys
sys.modules['sklearn.externals.six'] = six
import mlrose
from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
import time

data = pd.read_csv("file")
X = data.iloc[:,2:-1]
Y = data.iloc[:,-1:]
Xtrain, Xtest, ytrain, ytest = train_test_split(X,Y,test_size=.2, random_state=0)
scaler = MinMaxScaler()
print()
Xtrain = scaler.fit_transform(Xtrain)
Xtest = scaler.transform(Xtest)
randhillacc_table = []
simacc_table = []
genacc_table = []

randhillacc_test = []
simacc_test = []
genacc_test = []

#Placing in loop allows for a accurate plot graph
for i in range(1,1000,200):

    model = mlrose.NeuralNetwork(hidden_nodes = [2], activation = 'relu', \
                                     algorithm = 'random_hill_climb', max_iters = i, \
                                     bias = True, is_classifier = True, learning_rate = 0.1, \
                                     early_stopping = True, clip_max = 2, max_attempts = 100, \
                                     random_state = 3)

    model.fit(Xtrain, ytrain)
    pred = model.predict(Xtrain)
    trainval_x = accuracy_score(ytrain, pred)
    randhillacc_table.append(trainval_x)

    testpred = model.predict(Xtest)
    testval_x = accuracy_score(ytest, testpred)
    randhillacc_test.append(testval_x)

    modely = mlrose.NeuralNetwork(hidden_nodes = [2], activation = 'relu', \
                                     algorithm = 'simulated_annealing', max_iters = i, \
                                     bias = True, is_classifier = True, learning_rate = 0.1, \
                                     early_stopping = True, clip_max = 2, max_attempts = 100, \
                                     random_state = 3)

    modely.fit(Xtrain, ytrain)
    pred = modely.predict(Xtrain)
    trainval_y = accuracy_score(ytrain, pred)
    simacc_table.append(trainval_y)

    testpred = modely.predict(Xtest)
    testval_y = accuracy_score(ytest, testpred)
    simacc_test.append(testval_y)

    modelz = mlrose.NeuralNetwork(hidden_nodes = [2], activation = 'relu', \
                                     algorithm = 'genetic_alg', max_iters = i, \
                                     bias = True, is_classifier = True, learning_rate = 0.1, \
                                     early_stopping = True, clip_max = 2, max_attempts = 100, \
                                     random_state = 3)

    modelz.fit(Xtrain, ytrain)
    pred = modelz.predict(Xtrain)
    trainval_z = accuracy_score(ytrain, pred)
    genacc_table.append(trainval_z)

    testpred = modelz.predict(Xtest)
    testval_z = accuracy_score(ytest, testpred)
    genacc_test.append(testval_z)

plt.plot(np.arange(1, 1000,200), np.array(randhillacc_table), label='Random Hill Climb')
plt.plot(np.arange(1, 1000,200), np.array(simacc_table), label='Simulated Annealing')
plt.plot(np.arange(1, 1000,200), np.array(genacc_table), label='Genetic Algorithm')
plt.xlabel('Iterations')
plt.ylabel('Training Acccuracy')
plt.title('Training Rates')
plt.legend()
plt.show()
print(np.array(model))
print(accuracy_score(ytrain, pred))

print("RHC {}, SIM {}, GEN {}.".format(trainval_x, trainval_y, trainval_z))

plt.figure()
plt.plot(np.arange(1, 1000,200), np.array(randhillacc_test), label='Random Hill Climb')
plt.plot(np.arange(1, 1000,200), np.array(simacc_test), label='Simulated Annealing')
plt.plot(np.arange(1, 1000,200), np.array(genacc_test), label='Genetic Algorithm')
plt.xlabel('Iterations')
plt.ylabel('Test Accuracy')
plt.title('Testing Rates')
plt.legend()
plt.show()

#From mlrose
def random_hill_climb(problem, max_attempts=10, max_iters=np.inf, restarts=0,
                      init_state=None, curve=False, random_state=None):

    if (not isinstance(max_attempts, int) and not max_attempts.is_integer()) \
       or (max_attempts < 0):
        raise Exception("""max_attempts must be a positive integer.""")

    if (not isinstance(max_iters, int) and max_iters != np.inf
            and not max_iters.is_integer()) or (max_iters < 0):
        raise Exception("""max_iters must be a positive integer.""")

    if (not isinstance(restarts, int) and not restarts.is_integer()) \
       or (restarts < 0):
        raise Exception("""restarts must be a positive integer.""")

    if init_state is not None and len(init_state) != problem.get_length():
        raise Exception("""init_state must have same length as problem.""")

    # Set random seed
    if isinstance(random_state, int) and random_state > 0:
        np.random.seed(random_state)

    best_fitness = -1*np.inf
    best_state = None

    if curve:
        fitness_curve = []

    for _ in range(restarts + 1):
        if init_state is None:
            problem.reset()
        else:
            problem.set_state(init_state)

        attempts = 0
        iters = 0

        while (attempts < max_attempts) and (iters < max_iters):
            iters += 1

            next_state = problem.random_neighbor()
            next_fitness = problem.eval_fitness(next_state)

            if next_fitness > problem.get_fitness():
                problem.set_state(next_state)
                attempts = 0

            else:
                attempts += 1

            if curve:
                fitness_curve.append(problem.get_fitness())

        # Update best state and best fitness
        if problem.get_fitness() > best_fitness:
            best_fitness = problem.get_fitness()
            best_state = problem.get_state()

    best_fitness = problem.get_maximize()*best_fitness

    if curve:
        return best_state, best_fitness, np.asarray(fitness_curve)

    return best_state, best_fitness
#From mlrose
def simulated_annealing(problem, schedule=mlrose.GeomDecay(), max_attempts=10,
                        max_iters=np.inf, init_state=None, curve=False,
                        random_state=None):

    if (not isinstance(max_attempts, int) and not max_attempts.is_integer()) \
       or (max_attempts < 0):
        raise Exception("""max_attempts must be a positive integer.""")

    if (not isinstance(max_iters, int) and max_iters != np.inf
            and not max_iters.is_integer()) or (max_iters < 0):
        raise Exception("""max_iters must be a positive integer.""")

    if init_state is not None and len(init_state) != problem.get_length():
        raise Exception("""init_state must have same length as problem.""")

    # Set random seed
    if isinstance(random_state, int) and random_state > 0:
        np.random.seed(random_state)

    # Initialize problem, time and attempts counter
    if init_state is None:
        problem.reset()
    else:
        problem.set_state(init_state)

    if curve:
        fitness_curve = []

    attempts = 0
    iters = 0

    while (attempts < max_attempts) and (iters < max_iters):
        temp = schedule.evaluate(iters)
        iters += 1

        if temp == 0:
            break

        else:
            # Find random neighbor and evaluate fitness
            next_state = problem.random_neighbor()
            next_fitness = problem.eval_fitness(next_state)

            # Calculate delta E and change prob
            delta_e = next_fitness - problem.get_fitness()
            prob = np.exp(delta_e/temp)

            # If best neighbor is an improvement or random value is less
            # than prob, move to that state and reset attempts counter
            if (delta_e > 0) or (np.random.uniform() < prob):
                problem.set_state(next_state)
                attempts = 0

            else:
                attempts += 1

        if curve:
            fitness_curve.append(problem.get_fitness())

    best_fitness = problem.get_maximize()*problem.get_fitness()
    best_state = problem.get_state()

    if curve:
        return best_state, best_fitness, np.asarray(fitness_curve)

    return best_state, best_fitness
#From mlrose
def genetic_alg(problem, pop_size=200, mutation_prob=0.1, max_attempts=10,
                max_iters=np.inf, curve=False, random_state=None):

    if pop_size < 0:
        raise Exception("""pop_size must be a positive integer.""")
    elif not isinstance(pop_size, int):
        if pop_size.is_integer():
            pop_size = int(pop_size)
        else:
            raise Exception("""pop_size must be a positive integer.""")

    if (mutation_prob < 0) or (mutation_prob > 1):
        raise Exception("""mutation_prob must be between 0 and 1.""")

    if (not isinstance(max_attempts, int) and not max_attempts.is_integer()) \
       or (max_attempts < 0):
        raise Exception("""max_attempts must be a positive integer.""")

    if (not isinstance(max_iters, int) and max_iters != np.inf
            and not max_iters.is_integer()) or (max_iters < 0):
        raise Exception("""max_iters must be a positive integer.""")

    # Set random seed
    if isinstance(random_state, int) and random_state > 0:
        np.random.seed(random_state)

    if curve:
        fitness_curve = []

    # Initialize problem, population and attempts counter
    problem.reset()
    problem.random_pop(pop_size)
    attempts = 0
    iters = 0

    while (attempts < max_attempts) and (iters < max_iters):
        iters += 1

        # Calculate breeding probabilities
        problem.eval_mate_probs()

        # Create next generation of population
        next_gen = []

        for _ in range(pop_size):
            # Select parents
            selected = np.random.choice(pop_size, size=2,
                                        p=problem.get_mate_probs())
            parent_1 = problem.get_population()[selected[0]]
            parent_2 = problem.get_population()[selected[1]]

            # Create offspring
            child = problem.reproduce(parent_1, parent_2, mutation_prob)
            next_gen.append(child)

        next_gen = np.array(next_gen)
        problem.set_population(next_gen)

        next_state = problem.best_child()
        next_fitness = problem.eval_fitness(next_state)

        # If best child is an improvement,
        # move to that state and reset attempts counter
        if next_fitness > problem.get_fitness():
            problem.set_state(next_state)
            attempts = 0

        else:
            attempts += 1

        if curve:
            fitness_curve.append(problem.get_fitness())

    best_fitness = problem.get_maximize()*problem.get_fitness()
    best_state = problem.get_state()

    if curve:
        return best_state, best_fitness, np.asarray(fitness_curve)

    return best_state, best_fitness


# 基于Tkinter 绘制界面版本的邀请函（海报）生成器
import tkinter
from PIL import Image
from tkinter import Tk
from PIL import ImageDraw
from PIL import ImageFont
from tkinter import messagebox
from tkinter import colorchooser
from tkinter.filedialog import askopenfilename, askdirectory

class GUIGenerator:
    def __init__(self):
        """内部变量初始化"""
        self.messagebox = messagebox
        self.image_file = None
        self.name_list = []
        self.font = None
        self.color = None
        self.position = {"x": 0, "y": 0}
        self.save_path = None
        """图形化界面变量初始化"""
        self.generator = Tk()
        self.generator.title("邀请函生成器V1.1")
        self.generator.geometry("500x400")
        self.source_image_path_label = tkinter.Label(self.generator, text="原始图片文件：")
        self.source_image_path_input = tkinter.Entry(self.generator, width=40)
        self.source_image_file_select_button = tkinter.Button(
            self.generator,
            height=1,
            width=3,
            text="选择",
            command=self.source_image_file_select_function
        )
        self.invitee_name_list_file_label = tkinter.Label(self.generator, text="被邀请人名称列表文件：")
        self.invitee_name_list_file_input = tkinter.Entry(self.generator, width=40)
        self.invitee_name_list_file_select_button = tkinter.Button(
            self.generator,
            height=1,
            width=3,
            text="选择",
            command=self.invitee_name_list_file_select_function
        )
        self.invitation_card_save_path_label = tkinter.Label(self.generator, text="邀请函图片保存路径：")
        self.invitation_card_save_path_input = tkinter.Entry(self.generator, width=40)
        self.invitation_card_save_path_select_button = tkinter.Button(
            self.generator,
            height=1,
            width=3,
            text="选择",
            command=self.invitation_card_save_path_select_function
        )
        self.font_config_label = tkinter.Label(self.generator, text="字体属性：")
        self.font_color_select_label = tkinter.Label(self.generator, text="字体颜色：")
        self.font_color_select_input = tkinter.Entry(self.generator, width=10, bg="white")
        self.font_color_select_button = tkinter.Button(
            self.generator,
            height=1,
            width=5,
            text="颜色选择",
            command=self.font_color_select_function
        )
        self.font_size_label = tkinter.Label(self.generator, text="字体大小：")
        self.font_size_input = tkinter.Entry(self.generator, width=8)
        self.font_type_file_path_label = tkinter.Label(self.generator, text="字体文件路径：")
        self.font_type_file_path_input = tkinter.Entry(self.generator, width=40)
        self.font_type_file_select_button = tkinter.Button(
            self.generator,
            height=1,
            width=3,
            text="选择",
            command=self.font_type_file_select_function
        )
        self.word_position_label = tkinter.Label(self.generator, text="文字位置：")
        self.word_position_x_label = tkinter.Label(self.generator, text="横坐标：")
        self.word_position_x_input = tkinter.Entry(self.generator, width=5)
        self.word_position_y_label = tkinter.Label(self.generator, text="纵坐标：")
        self.word_position_y_input = tkinter.Entry(self.generator, width=5)
        self.generate_button = tkinter.Button(self.generator, height=1, width=10, text="开始生成", command=self.generate)
        self.copyright_label = tkinter.Label(self.generator, text="作者：b0b@c 一个不误正业的安全从业人员 联系：crsecscu@gmail.com")

    def source_image_file_select_function(self):
        selected_file_path = askopenfilename(
            parent=self.generator,
            title="原始图片选择",
            initialdir="~/"
        )
        self.source_image_path_input.insert("end", selected_file_path)

    def invitee_name_list_file_select_function(self):
        selected_file_path = askopenfilename(
            parent=self.generator,
            title="受邀嘉宾姓名文件选择",
            initialdir="~/"
        )
        self.invitee_name_list_file_input.insert("end", selected_file_path)

    def invitation_card_save_path_select_function(self):
        selected_path = askdirectory(
            parent=self.generator,
            title="邀请函存储路径选择",
            initialdir="~/"
        )
        self.invitation_card_save_path_input.insert("end", selected_path)

    def font_type_file_select_function(self):
        selected_file_path = askopenfilename(
            parent=self.generator,
            title="字体类型文件选择",
            initialdir="~/"
        )
        self.font_type_file_path_input.insert("end", selected_file_path)

    def font_color_select_function(self):
        color = colorchooser.askcolor(
            parent=self.generator,
            title="字体颜色选择"
        )
        self.font_color_select_input.config(bg=str(color[1]))
        self.font_color_select_input.place(x=100, y=230)
        self.color = str(color[1])

    def graph(self):
        self.source_image_path_label.place(x=30, y=20)
        self.source_image_path_input.place(x=30, y=50)
        self.source_image_file_select_button.place(x=401, y=47)
        self.invitee_name_list_file_label.place(x=30, y=80)
        self.invitee_name_list_file_input.place(x=30, y=110)
        self.invitee_name_list_file_select_button.place(x=401, y=107)
        self.invitation_card_save_path_label.place(x=30, y=140)
        self.invitation_card_save_path_input.place(x=30, y=170)
        self.invitation_card_save_path_select_button.place(x=401, y=167)
        self.font_config_label.place(x=30, y=200)
        self.font_color_select_label.place(x=30, y=230)
        self.font_color_select_input.place(x=100, y=230)
        self.font_color_select_button.place(x=215, y=227)
        self.font_size_label.place(x=310, y=230)
        self.font_size_input.place(x=380, y=230)
        self.font_type_file_path_label.place(x=30, y=270)
        self.font_type_file_path_input.place(x=30, y=300)
        self.font_type_file_select_button.place(x=401, y=297)
        self.word_position_label.place(x=30, y=340)
        self.word_position_x_label.place(x=100, y=340)
        self.word_position_x_input.place(x=160, y=340)
        self.word_position_y_label.place(x=220, y=340)
        self.word_position_y_input.place(x=280, y=340)
        self.generate_button.place(x=340, y=337)
        self.copyright_label.place(x=30, y=375)
        self.generator.mainloop()

    def warning(self, title, message):
        self.messagebox.showinfo(title, message)

    def generate_init(self):
        source_image_file_path = self.source_image_path_input.get()
        if source_image_file_path in ["", " ", None] or not isinstance(source_image_file_path, str):
            self.warning("注意", "请正确输入源图片文件路径!")
            return False
        self.image_file = source_image_file_path
        invitee_name_list_file = self.invitee_name_list_file_input.get()
        if invitee_name_list_file in ["", " ", None] or not isinstance(invitee_name_list_file, str):
            self.warning("注意", "请正确输入姓名文件路径!")
            return False
        with open(invitee_name_list_file, 'r') as filereader:
            for name in filereader.readlines():
                name = name.split("\n")[0].split("\r")[0]
                self.name_list.append(name)
        self.name_list = list(set(self.name_list))
        if len(self.name_list) <= 0:
            self.warning("注意", "没有找到需要生成邀请函的姓名!")
            return False
        if self.color is None:
            self.warning("注意", "没有设置颜色")
            return False
        font_type_file = self.font_type_file_path_input.get()
        if font_type_file in ["", " ", None] or not isinstance(font_type_file, str):
            self.warning("注意", "请正确输入字体文件路径!")
            return False
        font_size = self.font_size_input.get()
        if font_size in ["", " ", None] or not isinstance(font_size, str):
            self.warning("注意", "请正确输入字体大小!")
            return False
        try:
            font_size = int(font_size)
        except Exception as reason:
            self.warning("异常", str(reason))
            return False
        self.font = ImageFont.truetype(font_type_file, font_size)
        try:
            x = int(self.word_position_x_input.get())
            y = int(self.word_position_y_input.get())
            self.position["x"] = x
            self.position["y"] = y
        except Exception as reason:
            self.warning("异常", str(reason))
            return False
        self.save_path = self.invitation_card_save_path_input.get()
        if self.save_path in ["", " ", None] or not isinstance(self.save_path, str):
            self.warning("注意", "请正确输入姓名文件路径!")
            return False
        return True

    def generate(self):
        if not self.generate_init():
            return
        else:
            for name in self.name_list:
                if name in ["", " ", None]:
                    continue
                text_width = self.font.getsize(name)
                try:
                    image_file = Image.open(self.image_file)
                except Exception as reason:
                    self.warning("异常", str(reason))
                    return
                drawer = ImageDraw.Draw(image_file)
                text_coordinate = (
                    int((image_file.size[0] - text_width[0]) / 2 - self.position.get("x")),
                    int(self.position.get("y"))
                )
                drawer.text(text_coordinate, name, self.color, font=self.font)
                image_file.save(self.save_path + "/" + str(name) + ".jpg", "jpeg", quality=95)

if __name__ == "__main__":
    generator = GUIGenerator()
    generator.graph()


import numpy as np
import cv2 as cv
import glob

chessboardSize = (10, 14)
criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((chessboardSize[0] * chessboardSize[1], 3), np.float32)
objp[:, :2] = np.mgrid[0:chessboardSize[0], 0:chessboardSize[1]]
prev_img_shape = None

objpoints = []
imgpoints = []
images = glob.glob('*.jpg')
for image in images:
    print(image)
    img = cv.imread(image)
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    ret, corners = cv.findChessboardCorners(gray, chessboardSize)
    if ret:
        objpoints.append(objp)
        corners2 = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners)
        cv.drawChessboardCorners(img, chessboardSize, corners2, ret)

    cv.imshow('img', img)
    cv.imwrite('caliResult.png', img)
    cv.waitKey(100)
cv.destroyAllWindows()


import logging
import os
from types import *
from clientagent.common.platform_id import PlatformID
from clientagent.common.config import Config

class ClientAgentState:
    '''
    Very basic state machine used by various client agent modules.
    '''
    INIT_SETUP = False
    CLIENTAGENT_ROOT = ""
    CONFIG = None
    # The following are largely used in the Win32 service APIs, but provided
    # in case we find a use for them elsewhere
    SRV_NAME = "EILClientAgent"
    SRV_DISPLAY_NAME = "EIL Client Agent"
    SRV_DESCRIPTION = "EIL Portal Unified Client Service (Python) - resides on client - interfaces with CCMS"
    # Version information
    VERSION = "Undefined!"

    # Various Linux-isms we need for compatibility with the dispatcher scripts
    COMDIR = None
    BINDIR = None

def updateLogger():
    debug_level = 2
    if(ClientAgentState.CONFIG.C.has_option('main', 'log_level')):
        _debug_level = ClientAgentState.CONFIG.C.get('main', 'log_level')
        try:
            debug_level = int(_debug_level)
        except:
            pass

    Logger = logging.getLogger()

    if debug_level < 1:
        Logger.setLevel(logging.CRITICAL)
    elif debug_level == 1:
        Logger.setLevel(logging.ERROR)
    elif debug_level == 2:
        Logger.setLevel(logging.WARNING)
    elif debug_level == 3:
        Logger.setLevel(logging.INFO)
    else:
        # Anything higher will be debug to full
        Logger.setLevel(logging.DEBUG)

    Logger.info('Log level set to %d' % debug_level)

    # FIXME - Temp debugging suds client items
    logging.getLogger('suds.client').setLevel(logging.DEBUG)

if not ClientAgentState.INIT_SETUP:
    platformID = PlatformID()

    # Set up the logging filename
    # FIXME - What do we want for a debug mode? Would be nice if we could run
    # from command-line and have the logging go to stdout
    if platformID.IS_WINDOWS:
        # FIXME - Will want this to be the same log we have used previously
        ClientAgentState.CLIENTAGENT_ROOT = 'C:\\eil'
        logging.basicConfig(filename='%s\\clienagent.log' % ClientAgentState.CLIENTAGENT_ROOT,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    else:
        # Our root will be defined by previously issued LANANA/LSB requirements
        ClientAgentState.CLIENTAGENT_ROOT = '/opt/intel/eil/clientagent'
        fn = '%s/home/client-agent-base.log' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --stdlog')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                fn = output[0]
        finally:
            stream.close()

        logging.basicConfig(filename=fn,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        comdir = '%s/home/commands' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --comdir')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                comdir = output[0]
        finally:
            stream.close()

        ClientAgentState.COMDIR = comdir

        bindir = '%s/bin' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --bin')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                bindir = output[0]
        finally:
            stream.close()

        ClientAgentState.BINDIR = bindir

    ClientAgentState.CONFIG = Config(ClientAgentState.CLIENTAGENT_ROOT)

    try:
        verFile = os.path.join(ClientAgentState.CLIENTAGENT_ROOT, 'lib', 'VERSION')
        version = open(verFile, 'r')
        for rawline in version:
            verInfo = rawline.strip()
            if len(verInfo) > 0:
                ClientAgentState.VERSION = verInfo
        version.close()
    except:
        ClientAgentState.VERSION = 'Undefined'

    updateLogger()
    ClientAgentState.CONFIG.setCallback(updateLogger)

    ClientAgentState.INIT_SETUP = True

def get_config():
    '''
    Returns the config instance
    '''
    return ClientAgentState.CONFIG

# vim:set ai et sts=4 sw=4 tw=80:


class heatmapfinal:
    registered = True  # Value to define db operator

    def __init__(self):
        self.n = 0
        #  self.mydata = []
        self.heatmap = {}

    def step(self, *args):

        if self.n == 0:
            # print args, len(args)

            self.nomydatacolumns = int(len(args) / 4.0)
            self.colnames = [None] * self.nomydatacolumns
            self.curvalues = [None] * self.nomydatacolumns
            self.minvalues = [0.0] * self.nomydatacolumns
            self.step = [0.0] * self.nomydatacolumns
            self.index = [0.0] * self.nomydatacolumns
            # print self.nomydatacolumns, self.colnames ,self.curvalues , self.minvalues ,self.step ,self.index

        try:
            for i in xrange(self.nomydatacolumns):
                self.curvalues[i] = float(args[i * 4 + 1])
                if self.n == 0:
                    self.minvalues[i] = float(args[i * 4 + 2])
                    self.colnames[i] = (args[i * 4])
                    self.step[i] = float(args[i * 4 + 3])
            self.n += 1

        except (ValueError, TypeError):
            raise

        # print  self.step #["A", self.curvalues, self.colnames, self.minvalues, self.step]

        for i in xrange(self.nomydatacolumns):
            # print self.step[i]
            self.index[i] = max(int(round((self.curvalues[i] - self.minvalues[i]) / self.step[i])), 0)

        if tuple(self.index) in self.heatmap.keys():
            self.heatmap[tuple(self.index)] += 1
        else:
            self.heatmap[tuple(self.index)] = 1

    def final(self):
        import itertools
        yield tuple(itertools.chain.from_iterable((tuple(itertools.chain.from_iterable(
            [("colname" + str(i), "id" + str(i), "minvalue" + str(i), "maxvalue" + str(i)) for i in
             xrange(self.nomydatacolumns)])), ['num'])))
        # yield ('colname0', 'minvalue0', 'maxvalue1', 'colname1', 'minvalue1', 'maxvalue1')

        if self.n == 0:
            result = []
            for i in xrange(self.nomydatacolumns):
                result.append("None")
                yield result
        else:
            for item in self.heatmap:
                result = []
                for i in xrange(self.nomydatacolumns):
                    result.append(self.colnames[i])
                    result.append(item[i])
                    result.append(self.minvalues[i] + item[i] * self.step[i])
                    result.append(self.minvalues[i] + (item[i] + 1) * self.step[i])
                result.append(self.heatmap[item])
                yield result

if not ('.' in __name__):
    """
    This is needed to be able to test the function, put it at the end of every
    new function you create
    """
    import sys
    from functions import *

    testfunction()
    if __name__ == "__main__":
        reload(sys)
        sys.setdefaultencoding('utf-8')
        import doctest

        doctest.testmod()


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from tqdm import tqdm
from load_data import DataGetter
import pickle

def similarities_matrix(tfidf_data: pd.DataFrame) -> None:
    """
        Process and builds cosine similarities dataframe between chuncks of apps ids for
        memory optimization.
            inputs:
                    tfidf_data: TF-IDF map of apps text description.
    """
    store = pd.HDFStore('../data/raw/similiarities.h5')
    last_idx = 0
    for idx in tqdm(range(1000, tfidf_data.shape[0], 1000)):
       temp_data = tfidf_data.dot(tfidf_data.iloc[last_idx:idx].transpose())
       store.append(f'val_{int(idx/1000)}', temp_data)
       last_idx = idx
       del temp_data

    #last apps in the range
    temp_data = tfidf_data.dot(tfidf_data.iloc[last_idx:tfidf_data.shape[0]].transpose())
    store.append('val_final', temp_data)
    store.close()

def tfidf_map(data: pd.DataFrame, column: str) -> pd.DataFrame:
    """TF-IDF of corpus."""

    # build TF-IDF sparse matrix.
    tfidf_vector = TfidfVectorizer(ngram_range=(1,1),
                                   min_df=0.005,
                                   use_idf=True)
    tf_idf_data = tfidf_vector.fit_transform(data[column])

    # build dataframe of TF-IDF Map
    data = pd.DataFrame(tf_idf_data.toarray(),
                        columns = tfidf_vector.get_feature_names_out(),
                        index=data.item_id)
    return(data)

def get_top10_similiarities_from_h5_store() -> None:
    """Gets top 10 most similars apps for each app."""

    store = pd.HDFStore('../data/raw/similiarities.h5')
    similiarities = {}
    for key in tqdm(store.keys()):
        df = store.get(key=key)
        for column in df.columns:
            similiarities[column] = df[column].nlargest(11).iloc[1:10]

    with open('../data/final/similiarities.p', 'wb') as fp:
        pickle.dump(similiarities, fp)

def main() -> None:
    """Builds the pipeline of Similarities Matrix."""

    app_metadata = DataGetter(['app_metadata'], data_form='process').app_metadata_df
    tfidf_data = tfidf_map(app_metadata, 'description')
    similarities_matrix(tfidf_data)
    get_top10_similiarities_from_h5_store()

if __name__ == "__main__":
    main()


import dns
import os
import time
from recursortests import RecursorTest

class testOOOTCP(RecursorTest):
    _confdir = 'OOOTCP'

    _config_template = """dnssec=validate
"""

    @classmethod
    def generateRecursorConfig(cls, confdir):
        super(testOOOTCP, cls).generateRecursorConfig(confdir)

    def testOOOVeryBasic(self):
        expected = {}
        queries = []
        for zone in ['5.delay1.example.', '0.delay2.example.']:
            expected[zone] = dns.rrset.from_text(zone, 0, dns.rdataclass.IN, 'TXT', 'a')
            query = dns.message.make_query(zone, 'TXT', want_dnssec=True)
            query.flags |= dns.flags.AD
            queries.append(query)

        ress = self.sendTCPQueries(queries)

        self.assertEqual(len(ress), len(expected))

        i = 0
        for exp in [expected['0.delay2.example.'], expected['5.delay1.example.']]:
            print('ress0')
            print(ress[i].answer[0].to_text())
            print('exp')
            print(exp.to_text())
            self.assertMessageIsAuthenticated(ress[i])
            self.assertRRsetInAnswer(ress[i], exp)
            self.assertMatchingRRSIGInAnswer(ress[i], exp)
            i = i + 1

    def testOOOTimeout(self):
        expected = {}
        queries = []
        for zone in ['25.delay1.example.', '1.delay2.example.']:
            query = dns.message.make_query(zone, 'TXT', want_dnssec=True)
            query.flags |= dns.flags.AD
            queries.append(query)

        ress = self.sendTCPQueries(queries)

        self.assertEqual(len(ress), 2)
        exp = dns.rrset.from_text('1.delay2.example.', 0, dns.rdataclass.IN, 'TXT', 'a')
        self.assertRRsetInAnswer(ress[0], exp)
        self.assertMatchingRRSIGInAnswer(ress[0], exp)
        self.assertRcodeEqual(ress[1], dns.rcode.SERVFAIL)

        # Let the auth timeout happen to not disturb other tests
        # this can happen if the auth is single-threaded
        time.sleep(1)


#! /usr/bin/env python3

'''--------------------------------------------------------------
Programmer: Nick
Program:
Date:
Description:
--------------------------------------------------------------'''

import os
import json
import boto3
from datetime import datetime, date, timedelta
import sys

def get_paths():

    scripts = os.path.dirname(__file__)
    secrets = "/".join(scripts.split('/')[:-1]) + '/secrets'

    return scripts, secrets

def get_s3_conn(secrets):

    with open(secrets + '/creds.json') as creds:
        creds = json.load(creds)

    s3 = boto3.client(
        's3',
        region_name='us-east-1',
        aws_access_key_id=creds["aws_access_key_id"],
        aws_secret_access_key=creds["aws_secret_access_key"]
    )

    return s3

def get_argv_date_minus_1(argv_1):

    date = datetime.strptime(argv_1, '%Y-%m-%d') - timedelta(days=1)
    date_str = date.strftime('%Y%m%d')
    print('Date = ' + date_str)

    return date_str