import argparse
import os

from datasets.data_split import divide_data

import numpy as np
import torch.nn.functional as F
from torchvision import transforms

from skimage import io
import os
import openslide as ops
from PIL import Image
from metrics.evaluation import *

from models.u_net import U_Net, AttU_Net, XXU_Net
from models.unet import UNet, UNet_V1, UNet_V2, UNet_V3, UNet_V4 # new version

import torch
import csv

def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

def wsi_prediction(s, model, foreground, level, n_class=2, patch_size = 384, step = 200, foreground_thres = 0.25, t=transforms.ToTensor()):
    step = patch_size//2

    whole_size_i = s.level_dimensions[level][1] # y
    whole_size_j = s.level_dimensions[level][0] # x

    patch_size_i = patch_size
    patch_size_j = patch_size

    result = np.zeros((n_class,whole_size_i,whole_size_j))
    for i in range(0, whole_size_i, step):
        for j in range(0, whole_size_j, step):
            if i+patch_size_i>whole_size_i:
                i = whole_size_i-patch_size_i
            if j+patch_size_j>whole_size_j:
                j = whole_size_j-patch_size_j

            sub_foreground=foreground[i:i+patch_size, j:j+patch_size]
            if sub_foreground.sum()/(patch_size*patch_size)<=foreground_thres:
                continue

            img = s.read_region((int(j*s.level_downsamples[level]),int(i*s.level_downsamples[level])),level,(patch_size,patch_size))
            img = t(img)[0:3,:,:].unsqueeze(0)
            SR, CR = model(img)
            SR = SR.squeeze().detach().numpy()

            if i==0 and j==0:
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            elif i==0 and j>0:
                overlay1 = result[:,i:i+patch_size_i,j:j+patch_size_j-step]
                overlay2 = SR[:,:,0:patch_size_j-step]
                SR[:,:,0:patch_size_j-step] = (overlay1+overlay2)/2
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            elif j==0 and i>0:
                overlay1 = result[:,i:i+patch_size_i-step,j:j+patch_size_j]
                overlay2 = SR[:,0:patch_size_j-step,:]
                SR[:,0:patch_size_j-step,:] = (overlay1+overlay2)/2
                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
            else:
                overlay1 = result[:,i:i+patch_size_i-step,j:j+patch_size_j]
                overlay2 = SR[:,0:patch_size_j-step,:]
                SR[:,0:patch_size_j-step,:] = (overlay1+overlay2)/2

                overlay3 = result[:,i+patch_size_i-step:i+patch_size_i,j:j+patch_size_j-step]
                overlay4 = SR[:,patch_size_i-step:patch_size_i,0:patch_size_j-step]
                SR[:,patch_size_i-step:patch_size_i,0:patch_size_j-step] = (overlay3+overlay4)/2

                result[:,i:i+patch_size_i,j:j+patch_size_j] = SR
    return result

def test_wsi(model_type, unet, svsf, p1, level, patch_size = 384, category = 1, t=transforms.ToTensor()):
    '''
    svsf: (list) wsi to validate
    '''
    Image.MAX_IMAGE_PIXELS = 933120000000

    score = 0
    jss = []
    for i, f in enumerate(svsf):
        print('%d/%d'%(i+1,len(svsf)))

        s=ops.open_slide(p1+f)
        foreground=io.imread(p1+f.split('.')[0]+'_level%s.tif'%level)
        result = wsi_prediction(s, unet, foreground, level, patch_size = patch_size, foreground_thres = 0.25)

        # save numpy as binary map
        heatmap = result[category,:,:]
        segmap = result.argmax(0).astype('uint8')

        heatmap_pil = Image.fromarray((heatmap*255).astype('uint8'), 'L')
        heatmapf = model_type + '_'+f.split('.')[0]+'_heatmap.png'
        heatmap_pil.save(os.path.join(p1, heatmapf))

        segmap_pil = Image.fromarray((segmap*255), 'L')
        segmapf = model_type + '_'+f.split('.')[0]+'_segmap.png'
        segmap_pil.save(os.path.join(p1, segmapf))

    return 0

def test_patch(model_type, unet, patches, root, category = 1, t=transforms.ToTensor()):
    '''

    '''
    Image.MAX_IMAGE_PIXELS = 933120000000

    for i, f in enumerate(patches):
        print('%d/%d'%(i+1,len(patches)))

        img = Image.open(os.path.join(root, f)).convert("RGB")
        img = t(img)[0:3,:,:].unsqueeze(0)
        SR, CR = unet(img)
        result = SR.squeeze().detach().numpy()

        # save numpy as binary map
        heatmap = result[category,:,:]
        segmap = result.argmax(0).astype('uint8')

        heatmap_pil = Image.fromarray((heatmap*255).astype('uint8'), 'L')
        heatmapf = model_type + '_'+f.split('.')[0]+'_heatmap.png'
        heatmap_pil.save(os.path.join(root, heatmapf))

        segmap_pil = Image.fromarray((segmap*255), 'L')
        segmapf = model_type + '_'+f.split('.')[0]+'_segmap.png'
        segmap_pil.save(os.path.join(root, segmapf))

    return 0

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    # model
    parser.add_argument('--depth', type=int, default=5)
    parser.add_argument('--width', type=int, default=32)
    parser.add_argument('--image_size', type=int, default=384)
    parser.add_argument('--n_classes', type=int, default=2)
    parser.add_argument('--category', type=int, default=1, help='category for evaluation label')
    parser.add_argument('--t', type=int, default=2, help='t for Recurrent step of R2U_Net or R2AttU_Net')
    parser.add_argument('--reduction_ratio', type=int, default=None, help='reduction ratio for attention layer')
    parser.add_argument('--n_skip', type=int, default=4, help='number of skip-connection layers, <= depth-1')
    parser.add_argument('--n_head', type=int, default=1, help='number of heads for prediction, 1 <= depth-1')
    parser.add_argument('--att_mode', type=str, default='cbam', help='cbam/bam/se')
    parser.add_argument('--conv_type', type=str, default='basic', help='basic/sk')
    parser.add_argument('--is_shortcut', type=str2bool, default=False)

    # training hyper-parameters
    parser.add_argument('--img_ch', type=int, default=3)
    parser.add_argument('--num_epochs', type=int, default=100)
    parser.add_argument('--num_epochs_decay', type=int, default=30)
    parser.add_argument('--batch_size', type=int, default=4)
    parser.add_argument('--num_workers', type=int, default=8)
    parser.add_argument('--lr', type=float, default=0.0001)
    parser.add_argument('--beta1', type=float, default=0.5)        # momentum1 in Adam
    parser.add_argument('--beta2', type=float, default=0.999)      # momentum2 in Adam
    parser.add_argument('--augmentation_prob', type=float, default=0.5)

    # log
    parser.add_argument('--log_step', type=int, default=1)
    parser.add_argument('--val_step', type=int, default=2)

    # loss
    parser.add_argument('--loss_type', type=str, default='nll', help='l1/nll/focal/iou/dice/multitask/multiloss')
    parser.add_argument('--alpha', type=float, default=1)        # alpha for l1 loss
    parser.add_argument('--gamma', type=float, default=1)        # gamma for l1/Focal loss
    parser.add_argument('--balance', type=list, default=None)   # balance factor

    # misc
    parser.add_argument('--mode', type=str, default='train', help='train/test')
    parser.add_argument('--model_type', type=str, default='XXU_Net', help='XXU_Net/U_Net/R2U_Net/AttU_Net/R2AttU_Net')
    parser.add_argument('--model_path', type=str, default='/mnt/DATA_OTHER/paip2020/results/checkpoint/')
    parser.add_argument('--root_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/')
    parser.add_argument('--train_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/train/')
    parser.add_argument('--train_anno_path', type=str, default=None)
    parser.add_argument('--valid_path', type=str, default='/mnt/DATA_OTHER/paip2019/patches/seg/level1_400/validation/')
    parser.add_argument('--wsi_path', type=str, default='/mnt/DATA_OTHER/paip2020/original/images/')
    parser.add_argument('--valid_anno_path', type=str, default=None)
    parser.add_argument('--test_path', type=str, default='./fundus_images/test/')
    parser.add_argument('--result_path', type=str, default='./results/')
    parser.add_argument('--fold', type=int, default=3, help='5-fold cross validation')
    parser.add_argument('--level', type=int, default=3, help='1/2/3')

    # other
    parser.add_argument('--cuda_idx', type=int, default=1)

    config = parser.parse_args()
    config.root_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/'%(config.level, config.image_size)
    config.train_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/train/'%(config.level, config.image_size)
    config.valid_path = '/mnt/DATA_OTHER/paip2020/patches/level%d_%d/validation/'%(config.level, config.image_size)
    if config.n_skip>config.depth-1:
        config.n_skip = config.depth-1

    # divide data for k-fold cross validation
#     dd = divide_data(config.root_path)
#     dd.reset() # move data to all/
#     dd.divide(config.fold)

    """Build model"""
    unet = None
    if config.model_type =='U_Net':
        unet = U_Net(img_ch=3, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type =='R2U_Net':
        unet = R2U_Net(img_ch=3,t=config.t, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type =='AttU_Net':
        unet = AttU_Net(img_ch=3, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'R2AttU_Net':
        unet = R2AttU_Net(img_ch=3,t=config.t, n_classes = config.n_classes, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['UNet', 'SEU_Net', 'CBAMU_Net', 'BAMU_Net']:
        unet = UNet(img_ch=3, n_classes=config.n_classes, init_features=config.width, network_depth=config.depth, reduction_ratio=config.reduction_ratio, att_mode = config.att_mode, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['SKU_Net', 'SK-SC-U_Net', 'SK-SE-U_Net']:
        unet = UNet_V1(reduction_ratio=config.reduction_ratio, att_mode = config.att_mode, is_shortcut = config.is_shortcut, conv_type = config.conv_type, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'MHU_Net':
        unet = UNet_V2(img_ch=3, n_classes=config.n_classes, n_head = config.n_head, is_head_selective = False, is_shortcut = False, activation = torch.nn.Softmax(dim=1))
    elif config.model_type == 'SCU_Net':
        unet = UNet_V3(img_ch=3, n_classes=config.n_classes, n_head = config.n_head, is_scale_selective = False, is_shortcut = True, activation = torch.nn.Softmax(dim=1))
    elif config.model_type in ['SSU_Net', 'SK-SSU_Net', 'SE-SSU_Net', 'SC-SSU_Net' ]:
        unet = UNet_V4(reduction_ratio=config.reduction_ratio, n_head = config.n_head, att_mode = config.att_mode, is_scale_selective = True, is_shortcut = config.is_shortcut, conv_type = config.conv_type, activation = torch.nn.Softmax(dim=1))
    else:
        raise NotImplementedError(config.model_type+" is not implemented")

    unet_path = os.path.join(config.model_path, '%s-%s-level%s-size%s-depth%s-width%s-n_classes%s-alpha%s-gamma%s-nhead%s-fold%s.pkl'%(config.model_type, config.loss_type, config.level, config.image_size, config.depth, config.width, config.n_classes, config.alpha, config.gamma, config.n_head, config.fold))
    print('try to load weights from: %s'%unet_path)
    unet.load_state_dict(torch.load(unet_path))
    unet.train(False)
    unet.eval()

    # validation slide, fold1
    valf=['13', '24', '28', '32', '38', '39', '4', '44', '47']
    svsf=[]
    for f in os.listdir(config.wsi_path):
        if f.endswith(('.svs','.SVS')) and f.split('.')[0] in valf:
            svsf.append(f)
    print(svsf)

    # wsi prediction
    s = test_wsi(config.model_type+config.loss_type, unet, svsf, p1 = config.wsi_path, level = config.level, patch_size = config.image_size, category = config.category, t=transforms.ToTensor())
    print(s)

#     patchesf = []
#     proot = config.test_path
#     for f in os.listdir(proot):
#         if f.endswith(('.png','.PNG')):
#             patchesf.append(f)
#     print(patchesf)

#     # patches prediction
#     stage = test_patch(config.model_type, unet, patchesf, proot, category = 1, t=transforms.ToTensor())
#     print(stage)


import numpy as np
import matplotlib.pyplot as plt
import h5py
import scipy
from PIL import Image
from scipy import ndimage
from lr_utils import load_dataset
import re
import sys
import time
train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, classes = load_dataset()
train_set_x_flatten = train_set_x_orig.reshape(train_set_x_orig.shape[0], -1).T
test_set_x_flatten = test_set_x_orig.reshape(test_set_x_orig.shape[0], -1).T
train_set_x = train_set_x_flatten/255.
test_set_x = test_set_x_flatten/255.
var_lr = float(sys.argv[1])
var_num_iteration = int(sys.argv[2])
print_every = int(sys.argv[3])
variable_learning_rate = sys.argv[4]
cat_identification_number = sys.argv[5]
cat_file_name = sys.argv[6]
def sigmoid(z):
    s = 1/(1+np.exp(-z))
    return s
def initialize_with_zeros(dim):
    w = np.zeros((dim,1))
    b = 0
    return w, b
def propagate(w, b, X, Y):
    m = X.shape[1]
    A = sigmoid(np.dot(w.T,X)+ b )
    cost = -(1/m)*np.sum((Y*np.log(A)+(1-Y)*np.log(1-A)))
    dw = 1/m*(np.dot(X,(A-Y).T))
    db = 1/m*np.sum((A-Y))
    cost = np.squeeze(cost)
    grads = {"dw": dw,
             "db": db}
    return grads, cost
def optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False, print_every = 100):
    costs = []
    for i in range(num_iterations):
        grads, cost = propagate(w, b, X, Y)
        dw = grads["dw"]
        db = grads["db"]
        w = w-learning_rate*dw
        b = b-learning_rate*db
        if i % print_every == 0:
            costs.append(cost)
        if print_cost and ((i % print_every) == 0):
            print ("Cost after iteration %i: %f" %(i, cost))
    params = {"w": w,
              "b": b}
    grads = {"dw": dw,
             "db": db}
    return params, grads, costs

def predict(w, b, X):
    m = X.shape[1]
    Y_prediction = np.zeros((1,m))
    w = w.reshape(X.shape[0], 1)
    A = sigmoid(np.dot(w.T,X)+ b )
    for i in range(A.shape[1]):
        if (A[0,i] >= 0.5):
            Y_prediction[0,i]= 1
        else:
            Y_prediction[0,i] = 0
        pass
    return Y_prediction
def model(X_train, Y_train, X_test, Y_test, num_iterations = 2000, learning_rate = 0.5, print_cost = False, print_every  = 100):
    w, b = initialize_with_zeros(X_train.shape[0])
    parameters, grads, costs = optimize(w, b, X_train, Y_train, num_iterations, learning_rate, print_cost, print_every)
    w = parameters["w"]
    b = parameters["b"]
    Y_prediction_test = predict(w, b, X_test)
    Y_prediction_train = predict(w, b, X_train)
    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test,
         "Y_prediction_train" : Y_prediction_train,
         "w" : w,
         "b" : b,
         "learning_rate" : learning_rate,
         "num_iterations": num_iterations}
    return d
d = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = var_num_iteration, learning_rate = var_lr, print_cost = True, print_every = print_every)
costs = np.squeeze(d['costs'])
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('iterations (per hundreds)')
plt.title("Learning rate =" + str(d["learning_rate"]))
plt.savefig('app/assets/images/'+ cat_identification_number+'.png')
# plt.show()
if(variable_learning_rate == "true"):
    learning_rates = [0.01, 0.001, 0.0001]
    models = {}
    for i in learning_rates:
        print ("learning rate is: " + str(i))
        models[str(i)] = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 1500, learning_rate = i, print_cost = False)
        print ('\n' + "-------------------------------------------------------" + '\n')
    for i in learning_rates:
        plt.plot(np.squeeze(models[str(i)]["costs"]), label= str(models[str(i)]["learning_rate"]))
    plt.ylabel('cost')
    plt.xlabel('iterations')
    legend = plt.legend(loc='upper center', shadow=True)
    frame = legend.get_frame()
    frame.set_facecolor('0.90')
    plt.show()
my_image = cat_file_name
fname = "public/" + my_image
image = np.array(ndimage.imread(fname, flatten=False))
num_px = train_set_x_orig.shape[1]
my_image = scipy.misc.imresize(image, size=(num_px,num_px)).reshape((1, num_px*num_px*3)).T
my_predicted_image = predict(d["w"], d["b"], my_image)
f = open(cat_identification_number+ ".txt","w+")
f.write(classes[int(np.squeeze(my_predicted_image)),].decode("utf-8"))
# plt.imshow(image)
print("y = " + str(np.squeeze(my_predicted_image)) + ", your algorithm predicts a \"" + classes[int(np.squeeze(my_predicted_image)),].decode("utf-8") +  "\" picture.")


#!/usr/bin/env python

import sys
import os
import re
import cv2
import numpy as np
import scipy.stats
import itertools
import glob

class Point:
    # 2D point (x,y).
    def __init__(self, _x=0, _y=0):
        self.x = _x
        self.y = _y

class Range:
    # 2D range p1 to p2.

    def __init__(self, _p1=Point(), _p2=Point()):
        self.p1 = _p1
        self.p2 = _p2

class CV2Window:
    # CV2 named window rapper.

    def __init__(self, _name):
        self.name = _name

        cv2.namedWindow(self.name)

    def imgshow(self, img, wait=False):
        cv2.imshow(self.name, img)
        if wait:
            cv2.waitKey(0)

    def close(self):
        cv2.destroyWindow(self.name)

def fit_size(raw_img, show=True, h=500, w=500):
    # Resize numpy image data fit to h, w. Contain aspect ratio.
    # IMG required to be a color numpy image data.
    # H is height, W is width as integer.

    size = raw_img.shape[:2]
    f = min(h / size[0], w / size[1])
    img = cv2.resize(raw_img, (int(size[1] * f), int(size[0] * f)), interpolation=cv2.INTER_AREA)
    if show:
        window = CV2Window('raw image')
        window.imgshow(img)

    return img

def get_edges(img, show=True):
    # Edge detection by 'canny edge detection'.
    # IMG required to be a color numpy image data.

    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    edges = cv2.Canny(gray, 50, 150)
    if show:
        window = CV2Window('edge detection')
        window.imgshow(edges)

    return edges

def get_lines(img, show=True, threshold=80, minLineLength=50, maxLineGap=5):
    # Line detection by 'stochastic hough transform'.
    # IMG required to be a color numpy image data.

    edges = get_edges(img, show)
    lines = cv2.HoughLinesP(edges, 1, np.pi/180, threshold, minLineLength, maxLineGap)
    if show:
        window = CV2Window('line detection')
        tmpimg = np.copy(img)
        for line in lines:
            for x1,y1,x2,y2 in line:
                cv2.line(tmpimg, (x1,y1), (x2,y2), (0,255,0), 2)

        window.imgshow(tmpimg)
    return lines

def get_contours(img, show=True):
    # Extract contours.
    # IMG required to be a color numpy image data.

    edges = get_edges(img, show)
    contours = cv2.findContours(edges, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)[1]

    # remove too small contours
    min_area = img.shape[0] * img.shape[1] * 0.2
    large_contours = [c for c in contours if cv2.contourArea(c) > min_area]

    if show:
        window = CV2Window('contour detection (raw)')
        tmpimg = np.copy(img)
        cv2.drawContours(tmpimg, contours, -1, (0,255,0), 3)
        window.imgshow(tmpimg)

    return large_contours

def get_convexes(img, show=True):
    # Get convex hull(凸包) as expansion small lak.
    # IMG required to be a color numpy image data.

    contours = get_contours(img, show)
    convexes = [cv2.convexHull(cont) for cont in contours]

    if show:
        window = CV2Window('contour detection (convex hull)')
        tmpimg = np.copy(img)
        cv2.drawContours(tmpimg, convexes, -1, (0,255,0), 2)
        window.imgshow(tmpimg)

    return convexes

def get_convex_poly(img, show=True):
    # Get convex polygon as linear approximation of contour.
    # IMG required to be a color numpy image data.

    contours = get_convexes(img, show)
    polies = [cv2.approxPolyDP(cont, 0.02*cv2.arcLength(cont,True), True) for cont in contours]

    if show:
        window = CV2Window('contour detection (linear approx)')
        tmpimg = np.copy(img)
        cv2.drawContours(tmpimg, polies, -1, (0,255,0), 2)
        window.imgshow(tmpimg)

    return polies

def get_best_poly(img, show=True):
    # Get best appropriate (closest to the square) polygon.
    # POLIES is 2D polygon array data obtained by get_converx_poly.

    # calc score
    def calc_score(poly):
        arclen = cv2.arcLength(poly, True)
        edge_lens = [np.linalg.norm(poly[i] - poly[(i+1) % 4]) for i in range(4)]
        score = 1 / (sum([abs(arclen/4 - edge_len) for edge_len in edge_lens]) + 1)    # avoid devide by zero
        return score

    # get all polygons
    polies = get_convex_poly(img, show)

    # exclude polygons other than 4 corners
    targetPolies = [poly for poly in polies if poly.shape[0] == 4]

    scores = [calc_score(poly) for poly in targetPolies]
    best_poly = targetPolies[scores.index(max(scores))]

    if show:
        window = CV2Window('contour detection (closest square)')
        tmpimg = np.copy(img)
        cv2.drawContours(tmpimg, [best_poly], 0, (0,255,0), 2)
        window.imgshow(tmpimg)

    return best_poly

def makeExactlyPoly(poly):
    def retPoint(xrank, yrank):
        if xrank == 1 or xrank == 2:
            if yrank == 1 or yrank == 2:
                result = [0, 0]
            else:
                result = [0, 300]
        else:
            if yrank == 1 or yrank == 2:
                result = [300, 0]
            else:
                result = [300, 300]
        return result

    tpoly = poly[:,0]
    xpoint = tpoly[:,0]
    ypoint = tpoly[:,1]
    xrank = scipy.stats.rankdata(xpoint, method='ordinal')
    yrank = scipy.stats.rankdata(ypoint, method='ordinal')

    result = [retPoint(xrank[i], yrank[i]) for i in range(4)]
    return result

def trans_square(img, poly, show=True):
    # transform to right square
    # IMG required to be a color numpy image data.
    # POLY is best fitted to board polygon

    srcPoly = np.float32(poly)
    transPoly = np.float32(np.array(makeExactlyPoly(srcPoly)))

    M = cv2.getPerspectiveTransform(srcPoly, transPoly)
    transImg = cv2.warpPerspective(img, M, (300, 300))

    if show:
        window = CV2Window('trans square')
        window.imgshow(transImg)

    return transImg

def cut_piecies(img, poly, filepath, show=True):
    # Cut each piecies.
    # IMG required to be a color numpy image data.
    # POLY is best fitted to board polygon

    # transform to right square
    # transimg = trans_square(img, poly, show)

    transimg = img

    diffx = transimg.shape[0]/9
    diffy = transimg.shape[1]/9
    for xindex in range(9):
        for yindex in range(9):
            currentx = diffx * xindex
            currenty = diffy * yindex
            dstimg = transimg[(int)(currentx):(int)(currentx+diffx), (int)(currenty):(int)(currenty+diffy)]

            dirpath, basename = os.path.split(filepath)
            _, exttype = os.path.splitext(filepath)

            dirpath = dirpath.replace('raw', 'piecies')
            retmp = re.match(r"([0-9]*).(.*)", basename)
            imgnum = retmp[1]

            graylevel = "%04d" % (int)(cv2.cvtColor(dstimg, cv2.COLOR_RGB2GRAY).sum()/(255*33*33)*10000)
            dstimgpath = "%s/%s-%s-(%s-%s)%s" % (dirpath, graylevel, imgnum, xindex+1, yindex+1, exttype)

            cv2.imwrite(dstimgpath, dstimg)

    dstimgpath = filepath.replace('raw', 'board')
    cv2.imwrite(dstimgpath, transimg)

def get_board_corners(raw_img, filepath, show=True):
    # Get range of shogi board as 2Dpoint (x1,y1), (x2,y2).
    # IMG required to be a color numpy image data.

    # resize
    img = fit_size(raw_img, show, 500, 500)

    # choose best fit polygon detection
    # poly = get_best_poly(img, show)

    # import pdb; pdb.set_trace()
    shape = img.shape[:2]
    xmax = shape[0]
    ymax = shape[1]
    poly = np.array([[[0, 0]], [[xmax, 0]], [[xmax, ymax]], [[0, ymax]]])

    # cut piecies
    cut_piecies(img, poly, filepath, show)

def main():
    if (len(sys.argv) == 0):
        frgdebug = False
    else:
        frgdebug = True

    imgpaths = sorted(glob.glob('../images/raw/*.png'))

    for imgpath in imgpaths:
        try:
            print("processing...: %s" % os.path.basename(imgpath))
            get_board_corners(cv2.imread(imgpath), imgpath, False)
            if frgdebug:
                cv2.waitKey(0)
        except ZeroDivisionError:
            print("%s" % imgpath)

if __name__ == '__main__':
    main()


# Create your views here.
from datetime import datetime

from django.http import JsonResponse
from django.shortcuts import get_object_or_404

from app.activity.function import create_activity
from app.comment.models import Comment
from app.like.function import return_total_like_post, return_total_like_comment
from app.like.models import LikePost, LikeComment
from app.post.models import Post
from app.user.models import Profile

def post_like(request):
    post_id = request.POST.get('post_id', False)
    reponse_data = {}

    if post_id and request.user:
        post = Post.objects.get(pk=post_id)
        like, create = LikePost.objects.get_or_create(profile=Profile.objects.get(user=request.user), post=post)
        like.save()

        # Update time when post have a activity
        post.date_activity = datetime.now()
        post.save()

        #  Create activity
        create_activity(request.user, 'like_post', like.post.pk, 'like status')
        reponse_data['result'] = True
        reponse_data['total_like_post'] = return_total_like_post(post_id)
        return JsonResponse(reponse_data)
    else:
        reponse_data['result'] = False
        return JsonResponse(reponse_data)

def post_unlike(request):
    post_id = request.POST.get('post_id', False)
    response_data = {}

    if post_id and request.user:
        like = get_object_or_404(LikePost, profile=Profile.objects.get(user=request.user),
                                 post=Post.objects.get(pk=post_id))
        like.delete()
        response_data['result'] = True
        response_data['total_like_post'] = return_total_like_post(post_id)
        return JsonResponse(response_data)
    else:
        response_data['result'] = False
        return JsonResponse(response_data)

def comment_like(request):
    comment_id = request.POST.get('comment_id', False)
    response_data = {}

    if comment_id and request.user:
        comment = Comment.objects.get(pk=comment_id)
        like, create = LikeComment.objects.get_or_create(profile=Profile.objects.get(user=request.user),
                                                         comment=comment)
        like.save()

        # Update time when post have a activity
        comment.date_activity = datetime.now()

        #  Create activity
        create_activity(request.user, 'like_comment', like.comment.pk, 'like comment')

        response_data['result'] = True
        response_data['total_like_comment'] = return_total_like_comment(comment_id)
        return JsonResponse(response_data)
    else:
        response_data['result'] = False
        return JsonResponse(response_data)

def comment_unlike(request):
    comment_id = request.POST.get('comment_id', False)
    reponse_data = {}

    if comment_id and request.user:
        comment = Comment.objects.get(pk=comment_id)
        like = get_object_or_404(LikeComment, profile=Profile.objects.get(user=request.user), comment=comment)
        like.delete()

        reponse_data['result'] = True
        reponse_data['total_like_post'] = return_total_like_post(comment_id)
        return JsonResponse(reponse_data)
    else:
        reponse_data['result'] = False
        return JsonResponse(reponse_data)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('listofplaces/',views.PlacesListView.as_view(),name="list-of-places"),
    path('place/<int:pk>/', views.PlaceDetailView.as_view(), name='place-detail'),
    path('all_cities/', views.all_cities, name='all-cities'),
    path('city/places/<str:inputcity>/',views.city_places,name="city-places"),
    path('place/new/', views.place_new, name='place_new'),
]


import sys
import os
import argparse
import torch
import torchtext
from itertools import product
from torch import nn
from torch.autograd import Variable
from nltk.translate.bleu_score import SmoothingFunction, sentence_bleu
from data import load_data
from generate import genmax
from categorical_loss import CategoricalLoss
import numpy as np

def train(M,DS,args,optimizer):
  data = DS.train_batches
  weights = torch.cuda.FloatTensor(args.vsz).fill_(1)
  weights[0] = 0
  criterion = CategoricalLoss(DS.stoi_targets,args)#nn.CrossEntropyLoss(weights)
  trainloss = []
  for sources,targets,_ in data:
    sources = Variable(sources.cuda())
    targets = Variable(targets.cuda())
    M.zero_grad()
    outs,attns,_ = M(sources,targets)
    logits = M.gen(outs)
    targets = targets[:,1:].contiguous()
    #targets = targets[:,1:].contiguous()
    #logits = logits[:,:targets.size(1),:].contiguous()
    #logits = logits.view(-1,logits.size(2))
    #if targets.size(1)>args.maxlen:
    #  targets = targets[:,:args.maxlen].contiguous()
    #targets = targets.view(-1)
    loss = criterion(logits, targets,attns)
    loss.backward()
    trainloss.append(loss.data.cpu()[0])
    optimizer.step()
  return sum(trainloss)/len(trainloss)

def drawEncAttns(fn,sources,attns,vocab,svocab):
  data = [",".join([' ']+vocab)]
  attns = attns.data.cpu()
  for i,s in enumerate(sources):
    tmp = [' '.join(svocab[w] for w in s)]
    tmp += [str(x) for x in attns[i,:]]
    data.append(",".join(tmp))
  with open(fn,'w') as f:
    f.write("\n".join(data))

def validate(M,DS,args,vocab):
  M.eval()
  data = DS.val_batches
  #weights = torch.cuda.FloatTensor(args.vsz).fill_(1)
  #weights[0] = 0
  #criterion = nn.CrossEntropyLoss(weights)
  criterion = CategoricalLoss(DS.stoi_targets,args)#nn.CrossEntropyLoss(weights)
  vloss = []
  vbleu = []
  for sources,targets,swords in data:
    sources = Variable(sources.cuda(),volatile=True)
    targets = Variable(targets.cuda(),volatile=True)
    M.zero_grad()
    outs,attns,eW = M(sources)
    logits = M.gen(outs)
    targets = targets[:,1:].contiguous()
    if args.gentargs:
      gentexts,gentargs = genmax(logits,DS.itos_targets,targets=targets)
      with open(args.savestr+"targets.txt",'w') as f:
        f.write("\n".join(gentargs))
      args.gentargs=False
    gentexts,_ = genmax(logits,DS.itos_targets)
    with open(args.savestr+args.epoch+".generated",'w') as f:
      f.write("\n".join(gentexts))
    drawEncAttns(args.savestr+args.epoch+".heatmap",swords,eW,vocab,DS.itos_sources)
    if args.debug:
      for x in gentexts:
        print(x)
    batchbleu = calcbleu(logits, targets, DS.stoi_targets["<end>"])
    vbleu.append(batchbleu)
    #logits = logits[:,:targets.size(1),:].contiguous()
    #logits = logits.view(-1,logits.size(2))
    if targets.size(1)>args.maxlen:
      targets = targets[:,:args.maxlen].contiguous()
    #targets = targets.view(-1)
    loss = criterion(logits, targets, attns)
    vloss.append(loss.data.cpu()[0])
  M.train()
  return sum(vloss)/len(vloss), sum(vbleu)/len(vbleu)

def calcbleu(generated, targets, end):
  #calcbleu(generated, targets, DS.stoi_targets["<end>"]):
  cc = SmoothingFunction()
  generated = torch.max(generated.data.cpu(),2)[1]
  targets = targets.data.cpu()
  bleu = 0
  for i in range(generated.size(0)):
    gen = list(generated[i])
    targ = list(targets[i])
    genend = gen.index(end) if end in gen else len(gen)
    targend = targ.index(end) if end in targ else len(targ)
    gen = gen[:genend]
    targ = targ[:targend]
    bleu += sentence_bleu([targ],gen,smoothing_function=cc.method3)
  return bleu/i

def init_vocab(M,DS,args):
  #get a vsz x dim matrix to give to the model
  with open(args.eventvocab) as f:
    vocab = f.read().strip().lower().split("\n")
  matrix = DS.matrix(vocab).cuda()
  M.mkevents(matrix)
  return vocab

def main():
  args = parseParams()
  if args.model=="s2s":
    from s2s import model
  elif args.model=="s2sattn":
    from s2sattn import model
  elif args.model=="eventattn":
    from eventattn import model
    args.hsz = 300
  else:
    print("Bad model option");exit()
  args.gentargs = True
  args.savestr = "saved_models/"+args.model+"/"
  try:
    os.stat(args.savestr)
  except:
    os.mkdir(args.savestr)
  if args.debug:
    #args.eventvocab = "two.txt"
    args.bsz = 2
    DS = load_data(args.bsz,debug=True)
  else:
    try:
      DS = torch.load(args.datafile)
    except:
      DS = load_data(args.bsz)
      torch.save(DS,args.datafile)
  args.vsz = len(DS.itos_targets)
  args.start = DS.stoi_targets["<start>"]
  if args.resume:
    M, optimizer = torch.load(args.resume)
    M.enc.flatten_parameters()
    M.dec.flatten_parameters()
    print("resuming from ",args.resume)
  else:
    M = model(args).cuda()
    print(M)
    if args.model == "eventattn":
      vocab = init_vocab(M,DS,args)
    optimizer = torch.optim.Adam(M.parameters(), lr=args.lr)
  for epoch in range(args.epochs):
    args.epoch = str(epoch)
    trainloss = train(M,DS,args,optimizer)
    print("train loss epoch",epoch,trainloss)
    vloss, vbleu = validate(M,DS,args,vocab)
    print("valid loss ",vloss)
    print("valid bleu ",vbleu)
    savestr = args.savestr+"epoch_"+args.epoch+"vbleu_"+str(vbleu)[:4]+"-vloss_"+str(vloss)[:4]+".pt"
    torch.save((M,optimizer),savestr)

def parseParams():
    parser = argparse.ArgumentParser(description='none')
    # learning
    parser.add_argument('-layers', type=int, default=2, help='min_freq for vocab [default: 1]') #
    parser.add_argument('-drop', type=float, default=0.3, help='min_freq for vocab [default: 1]') #
    parser.add_argument('-bsz', type=int, default=50, help='min_freq for vocab [default: 1]') #
    parser.add_argument('-hsz', type=int, default=500, help='min_freq for vocab [default: 1]') #
    parser.add_argument('-maxlen', type=int, default=75, help='min_freq for vocab [default: 1]') #
    parser.add_argument('-lr', type=float, default=0.001, help='initial learning rate [default: 0.001]') #
    parser.add_argument('-epochs', type=int, default=10, help='number of epochs for train [default: 100]') #
    #parser.add_argument('-savedir', type=str, default="saved_models/s2s/")
    parser.add_argument('-datafile', type=str, default="data/datafile.pt")
    parser.add_argument('-model', type=str, default="s2sattn")
    parser.add_argument('-eventvocab', type=str, default="data/categories.txt")
    parser.add_argument('-debug', action="store_true")
    parser.add_argument('-resume', type=str, default=None)

    args = parser.parse_args()
    return args

main()


#
# Jigna product code
#
# (C) Copyright 2013 Enthought, Inc., Austin, TX
# All right reserved.
#
# This file is confidential and NOT open source.  Do not distribute.
#

# Standard library imports.
import inspect
import logging

# System library imports.
from pyface.qt import QtCore

# Logger.
logger = logging.getLogger(__name__)

###############################################################################
# Functions.
###############################################################################

class PythonContainer(QtCore.QObject):
    """ Container class for python object to be exposed to js. """
    def __init__(self, parent=None, slot_map=None, wrapped_obj=None):
        super(PythonContainer, self).__init__(parent)

        # slot_name: callable
        self._slot_map = slot_map

        # the object being wrapped for property access
        self._wrapped_obj = wrapped_obj

def create_js_object_wrapper(obj, callbacks=[], properties=[], parent=None):
    """ Create an object wrapper for python objects.

    Usage:
    ------
    1. Auto wrap python objects:

       Call with a python object as `obj` and both `callbacks` and `properties`
       unspecified. Any attributes of `obj` wrappable in JS will be wrapped.

    2. Wrap specified attributes of object:

       Call with `callbacks` and `properties` as a list of attributes to be
       exposed to JS.

       Each element of the list must be either:
        - str, which is taken as the attribute name of `obj` to be
          exposed to JS with same name and whose value is the value of
          the attribute of `obj`
        - 2-tuple (str,obj/callable), where the second element object
          exposed to JS with first element str as the property name.
          In this case the argument `obj` is unused.

    Note: Arbitrary object cannot be wrapped and callables in the form of
    function objects cannot be exposed as functions in JS. If `obj` is a dict,
    the items of the dict are exposed instead of the attributes.

    Note: Set the parent (setParent()) of the returned QObject to make sure
    it is destroyed when the parent is destroyed, or manually destroy the
    object when it is no longer needed (such as when the `window` js object
    is cleared or the webview/webpage/webframe object is destroyed);
    else stray references to the object may cause memory leaks.

    """
    if isinstance(obj, dict):
        obj = type('CustomObject', (object,), obj)

    # Create the container class dict.
    class_dict = {}
    auto = not callbacks and not properties
    if auto:
        names = filter(lambda name: not name.startswith('_'), dir(obj))
        is_callable = lambda name: callable(getattr(obj, name))
        callbacks, properties = split_on_condition(names, is_callable)

    # Create the container callback slots, these need to be defined on the
    # class (associated with the QMetaObject).
    slot_map = {}
    for name_or_pair in callbacks:
        if isinstance(name_or_pair, basestring):
            name = name_or_pair
            callback = getattr(obj, name)
        else:
            name, callback = name_or_pair

        wrapped = wrap_func(callback, name)
        if wrapped:
            class_dict[name] = wrapped
            slot_map[wrapped.name] = callback
        elif not auto:
            logger.error('Callback %r is not translatable to JavaScript', name)

    # Create the container properties.
    for name_or_pair in properties:
        if isinstance(name_or_pair, basestring):
            jname, pname = name_or_pair, name_or_pair
        else:
            jname, pname = name_or_pair

        qproperty = wrap_property(obj, pname)
        if qproperty:
            class_dict[jname] = qproperty
        elif not auto:
            logger.error('Attribute %r is not translatable to JavaScript',
                         pname)

    # Create the container class.
    container_class = type('CustomPythonContainer', (PythonContainer, QtCore.QObject,), class_dict)
    qobj = container_class(parent=parent, slot_map=slot_map, wrapped_obj=obj)

    return qobj

def wrap_func(func, name=None):
    """ Wraps a Python callable as a Qt Slot.
    """
    try:
        args = inspect.getargspec(func).args
    except TypeError:
        return None

    if args and args[0] in ('self', 'cls'):
        args.pop(0)
    types = ['QVariant'] * len(args)

    slot_key = name if name is not None else 'func_' + hex(id(func))

    @QtCore.Slot(*types, name=name, result='QVariant')
    def wrapped(self, *args):
        return self._slot_map[slot_key](*args)
    wrapped.name = slot_key
    return wrapped

def wrap_property(obj, name, setter=True):
    """ Wraps a Python attribute as a Qt Property.
    """
    getter = lambda self: getattr(self._wrapped_obj, name)
    if setter:
        setter = lambda self, value: setattr(self._wrapped_obj, name, value)
    value = getattr(obj, name)
    signature = None
    if isinstance(value, dict):
        signature = 'QVariantMap'
    elif isinstance(value, list):
        signature = 'QVariantList'
    elif isinstance(value, (bool, int, long, float, basestring)):
        signature = 'QVariant'
    if signature:
        if setter:
            return QtCore.Property(signature, getter, setter)
        else:
            return QtCore.Property(signature, getter)
    return None

def split_on_condition(seq, condition):
    """ Split a list on a condition.
    """
    a, b = [], []
    for item in seq:
        (a if condition(item) else b).append(item)
    return a, b


import requests
from datetime import datetime
import os
import DataCheck
import crawlbotlogging

start = 0
BASE_DIR = os.path.dirname(os.path.abspath(__file__))

with open(os.path.join(BASE_DIR, "startBungae1.txt"), "r") as f:
    start = int(f.readline())

articles = str(start)
count = 0

end = 0

startTime = datetime.now()
crawlbotlogging.startLog("Bungae1.py", articles)

while 1:
    URL = 'https://core-api.bunjang.co.kr/api/1/product/'
    URL2 = '/detail_info.json?stat_uid=9056251&version=2'
    URL = URL + articles + URL2
    # print(URL)
    request = requests.get(URL)
    html = request.json()

    if html["result"] == "fail":
        count += 1
        articles = str(int(articles) + 2)
        if count >= 10:
            articles = str(int(articles) - 20)
            break
        continue

    if html['item_info'] == None:
        # print('None')
        count += 1
        articles = str(int(articles) + 2)
        if count >= 10:
            articles = str(int(articles) - 20)
            break
        continue

    elif html['item_info']['status'] == "3":
        # print('Trade')
        count += 1
        articles = str(int(articles) + 2)
        if count >= 10:
            articles = str(int(articles) - 20)
            break
        continue
    else:
        title = html['item_info']['name']
        price = html['item_info']['price']
        description = html['item_info']['description']
        day = datetime.today().replace(microsecond=0)

    URL = 'https://m.bunjang.co.kr/products/' + articles
    if int(price) < 10000:
        articles = str(int(articles) + 2)
        continue

    # print(title, price, day, URL, description)
    try:
        DataCheck.DataCheck(title, str(price), URL, day, description, 2)
    except Exception:
        with open(os.path.join(BASE_DIR, "Error.txt"), "a+") as f:
            f.write("Error URL : " + URL + "\n")

    articles = str(int(articles) + 2)
    count = 0

request.close()

with open(os.path.join(BASE_DIR, "startBungae1.txt"), "w") as f:
    f.write(articles)

crawlbotlogging.endLog("Bungae1.py", articles, startTime)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import re, datetime
import urllib
from flask import request, session, redirect, abort

class Utils(object):

    logger = logging.getLogger('app')

    @staticmethod
    def company_required(f):
        def decorator(**keywords):
            if 'token' not in session or session['identity'] != 'company':
                return redirect('/login?' + urllib.urlencode({'next': request.path}))
            return f(**keywords)
        decorator.__name__ = f.__name__
        return decorator

    @staticmethod
    def person_required(f):
        def decorator(**keywords):
            if 'token' not in session or session['identity'] != 'person':
                return redirect('/login?' + urllib.urlencode({'next': request.path}))
            return f(**keywords)
        decorator.__name__ = f.__name__
        return decorator

    @staticmethod
    def is_login():
        if 'token' in session:
            return True
        else:
            return False

    @staticmethod
    def is_company():
        if "identity" not in session:
            return False
        elif session["identity"] == "company":
            return True
        else:
            return False

    @staticmethod
    def is_person():
        if "identity" not in session:
            return False
        elif session['identity'] == 'person':
            return True
        else:
            return False

    @staticmethod
    def deal_form_dict(res, keys, values):
        key = keys[0]
        if len(keys) <= 1:
            if key.endswith('[]'):
                res[key[:-2]] = values if isinstance(values, list) else [values]
            else:
                res[key] = values[0] if isinstance(values, list) else values
        else:
            if key.endswith('[]'):
                if key[:-2] not in res:
                    res[key[:-2]] = [{} for i in values]
                for new_res, value in zip(res[key[:-2]], values):
                    utils.deal_form_dict(new_res, keys[1:], value)
            else:
                if key not in res:
                    res[key] = {}
                Utils.deal_form_dict(res[key], keys[1:], values)

    @staticmethod
    def to_dict(form):
        if form == None:
            return {}
        res = {}
        for k, v in dict(form).items():
            Utils.deal_form_dict(res, k.split('.'), v)
        return res

    @staticmethod
    def change_page_to_offset(form, limit):
        res = {}
        res["limit"] = str(limit)
        for k,v in dict(form).items():
            if k == "page":
                res["offset"] = (int(v) - 1) * 10
            else:
                res[k] = v.encode("utf-8")
        return res

    @staticmethod
    def gen_url(base_url, param_dict):
        param_url = urllib.urlencode({k: param_dict[k] for k in param_dict if param_dict[k]})
        return base_url + '?' + param_url if len(param_url) > 0 else base_url

    @staticmethod
    def format_time(iso_date):
        d = datetime.datetime(*map(int, re.split('[^\d]', str(iso_date))[:6]))

        cur = datetime.datetime.now()

        if d.date() == cur.date():
            return d.strftime('%H:%M')
        elif (cur.date() - d.date()).days <= 3:
            return u"%d天前"%(cur.date() - d.date()).days
        elif d.year == cur.year:
            return u"%d月%d日"%(d.month, d.day)
        else:
            return u"%d年%d月%d日"%(d.year, d.month, d.day)

    @staticmethod
    def format_time_full(iso_date):
        iso_date = str(iso_date)
        return '/'.join([iso_date[0:4], iso_date[5:7], iso_date[8:10]]) + '  ' + iso_date[11:16]

    @staticmethod
    def summarize(string, num):
        return string[0:num+1] + '...' if len(string) > num else string

    @staticmethod
    def is_mobile():
        return re.match(r'.*(Android|webOS|iPhone|iPod|BlackBerry|IEMobile|Opera Mini).*', request.headers.get('User-Agent'), re.M|re.I)

    @staticmethod
    def content_chopper(s, offset, limit):
    	return re.sub(r'<.*?>|&.*?;', '', s)[offset:offset+limit]


import mxnet as mx
import numpy as np
import logging
import random

def get_symbol(num_class):
    data = mx.sym.Variable('data')
    cn1 = mx.sym.Convolution(data=data, kernel=(10, ), num_filter=128, name='cn1')
    bn1 = mx.sym.BatchNorm(data=cn1, fix_gamma=False, eps=2e-5, momentum=0.9, name='bn1')
    ac1 = mx.sym.Activation(data=bn1, act_type='relu', name='ac1')
    pool1 = mx.symbol.Pooling(data=ac1, pool_type="max", kernel=(7, ), stride=(4, ), name="pool1")

    cn2 = mx.sym.Convolution(data=pool1, kernel=(5,), num_filter=256, name='cn2')
    bn2 = mx.sym.BatchNorm(data=cn2, fix_gamma=False, eps=2e-5, momentum=0.9, name='bn2')
    ac2 = mx.sym.Activation(data=bn2, act_type='relu', name='ac2')
    pool2 = mx.symbol.Pooling(data=ac2, pool_type="max", kernel=(5, ), stride=(3, ), name="pool2")

    cn3 = mx.sym.Convolution(data=pool2, kernel=(4,), num_filter=512, name='cn3')
    bn3 = mx.sym.BatchNorm(data=cn3, fix_gamma=False, eps=2e-5, momentum=0.9, name='bn3')
    ac3 = mx.sym.Activation(data=bn3, act_type='relu', name='ac2')
    pool3 = mx.symbol.Pooling(data=ac3, pool_type="max", kernel=(2,), name="pool3")

    fla = mx.sym.Flatten(data=pool3)

    fc4 = mx.sym.FullyConnected(data=fla, num_hidden=1024, name='fc4')
    bn4 = mx.sym.BatchNorm(data=fc4, fix_gamma=False, eps=2e-5, momentum=0.99, name='bn4')
    ac4 = mx.sym.Activation(data=bn4, act_type='relu', name='ac4')

    fc5 = mx.sym.FullyConnected(data=ac4, num_hidden=num_class, name='fc5')

    return mx.sym.SoftmaxOutput(data=fc5, name='softmax')

class Param(object):
    def __init__(self):
        self.batch_size = 16
        self.record_len = 725
        self.learn_rate = 0.01
        self.seq_len = 3360
        self.end_lr = 0.00001
        self.pow = 0.6
        self.epoch = 100
        self.wd = 0.00005
        self.moment = 0.9
        self.gpu = 2
        self.frequent = 20
        self.global_step = int(self.record_len / self.batch_size * self.epoch)
        self.save_model_name = './d/project/rnn_data/model/model_name'

if __name__ == '__main__':
    param = Param()
    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.DEBUG, format=head)
    idx_file = './d/project/rnn_data/train_idx.txt'
    rec_file = './d/project/rnn_data/train_rec_data.rec'
    batch_size = param.batch_size
    idx_range = param.record_len
    label_list = []
    feature_list = []
    total_lst = []
    seq_len = param.seq_len
    record = mx.recordio.MXIndexedRecordIO(idx_file, rec_file, 'r')
    for idx in range(idx_range):
        item = record.read_idx(idx)
        header, data = mx.recordio.unpack(item)
        label = np.array(header.label)
        label = np.hstack((label, np.zeros(3)))
        label = label.reshape(4, -1)
        data = np.frombuffer(data, np.float32)
        data = data.reshape(-1, param.seq_len)
        # data = data[-seq_len:]
        label_add_data = np.concatenate((label, data), axis=1)
        total_lst.append(label_add_data)
        # feature_list.append(data[:seq_len])
        # label_list.append(label)
    random.shuffle(total_lst)
    for i in total_lst:
        feature_list.append(i[:, 1:])
        label_list.append(i[0][0])
    print(feature_list[0].shape)
    feature = np.array(feature_list, dtype=np.float32)
    label = np.array(label_list, dtype=np.float32)
    print('read data is over.')
    train_iter = mx.io.NDArrayIter(data=feature, label=label, batch_size=batch_size,
                                   shuffle=True, data_name='data', label_name='softmax_label')
    net = get_symbol(2)
    model = mx.mod.Module(symbol=net, context=[mx.gpu(i) for i in range(2)])

    sgd_opt = mx.optimizer.SGD(learning_rate=param.learn_rate, momentum=param.moment, wd=param.wd,
                               rescale_grad=1 / param.batch_size)

    def lr_callback(cp):
        global_nbatch = cp.epoch * int(param.record_len / param.batch_size) + cp.nbatch
        base_lr = param.learn_rate
        if cp.epoch >= param.epoch / 4:
            base_lr = param.learn_rate / 10
        if cp.epoch >= param.epoch / 1.5:
            base_lr = param.learn_rate / 20
        if global_nbatch < param.global_step:
            sgd_opt.lr = (1 - global_nbatch / param.global_step) ** param.pow * (base_lr - param.end_lr) + param.end_lr
        else:
            sgd_opt.lr = param.end_lr
        if cp.nbatch % 20 == 0:
            logging.info('Epoch[%d] Batch [%d]      learning rate:%f' % (cp.epoch, cp.nbatch, sgd_opt.lr))

    model.fit(train_data=train_iter,
              optimizer=sgd_opt,
              eval_metric=['acc', 'ce'],
              epoch_end_callback=mx.callback.do_checkpoint(param.save_model_name, 5),
              batch_end_callback=[mx.callback.Speedometer(param.batch_size, param.frequent), lr_callback],
              num_epoch=param.epoch)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : get_sina_news_thread.py.py
# @time: 2019/6/7 10:24
# @author: Charon
# @contact: 2630966248@qq.com
# @desc  :
import os
import threading
import time

import bs4
import requests
import wx

# 换成其他网址需要修改部分代码
url = "https://news.sina.com.cn/"
start_time = time.time()
if not os.path.exists("get_sina_news"):
    os.mkdir("get_sina_news")
now_time = str(time.time())
old_time = 0
try:
    # 注，要先读再写，就要用r+ 要先写再读，就要用w+
    f = open("get_sina_news/time.txt", "r+", encoding="utf-8")
    old_time = f.read()
    f.seek(0)
    f.write(now_time)
    f.close()
except:
    open("get_sina_news/time.txt", "w").write(now_time)

# 判断是否需要更新新闻信息,需要则开始下载
# 2019/6/7 10:29 done 把下载函数写成多线程，提高下载速度，多线程主要针对不同的新闻栏目，分开下载
def download_news(j, all_news):
    """
    :param j: 链接
    :param all_news: 获得的所有新闻
    :return:
    """
    title = j.getText()
    single_new_contet = ""
    try:  # 有的新闻这儿要出问题
        single_new_contet = str(requests.get(j.attrs["href"], timeout=5).content, encoding="utf-8")
    except:
        print(str(j) + "\n处的新闻好像出了点问题!")
    all_p = bs4.BeautifulSoup(single_new_contet, "lxml").select("#article_content p")
    article = ""
    for k in all_p:
        article += (k.getText() + "\n")
    article = article[0:article.find("更多猛料")]
    all_news[title] = article
    time.sleep(0.2)  # 休眠一下，免得被抓

if (float(now_time) - float(old_time)) > 300:
    # 得到网页源码
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like "
                             "Gecko) Chrome/27.0.1453.94 Safari/537.36"}
    main_content = str(requests.get(url, timeout=5).content, encoding="utf-8")
    open("get_sina_news/主页源码.txt", "w", encoding="utf-8").write(main_content)
    # 转换成beautifulsoup格式
    soup_content = bs4.BeautifulSoup(main_content, "lxml")
    all_part = {"要闻": "blk_yw_01", "军事": "blk_08_cont01", "内地": "blk_gnxw_01", "广东新闻": "blk_new_bdxw",
                "国际新闻": "blk_gjxw_01"}
    all_news = {}
    all_thread = []
    for i_name, i in all_part.items():
        temp_part = soup_content.select("#" + i + " a")
        for j in temp_part:
            if "java" not in str(j):
                # 多线程下载
                thread_obj = threading.Thread(target=download_news, args=[j, all_news])
                thread_obj.start()
                all_thread.append(thread_obj)
        print("Download....." + i_name)
    for thread in all_thread:
        thread.join()  # 等待下载完成
    f = open("get_sina_news/all_news.txt", "w", encoding="utf-8")
    for key, value in all_news.items():
        f.write(key + "\n" + value + "\n\n")
    f.close()
    print("下载完成！！")
else:
    print("已有5分钟前的缓存")
end_time = time.time()
print("用时: " + str(end_time - start_time))

# 显示新闻内容
f = open("get_sina_news/all_news.txt", encoding="utf-8")
all_news = {}  # 将所有新闻内容保存在这里
while True:
    title = f.readline()
    if title == "":
        break
    article = ""
    while True:
        temp = f.readline()
        if temp == "":
            break
        if "责任编辑" not in temp:
            article += temp
        else:
            article += temp
            f.readline()
            f.readline()
            all_news[title.strip()] = article
            break
title_list = []
for key in all_news:
    title_list.append(key)

class ListBoxFrame(wx.Frame):
    """
    一个显示新闻的窗口界面
    """
    def __init__(self):
        wx.Frame.__init__(self, None, -1, '新浪新闻',
                          size=(1200, 650))
        panel = wx.Panel(self, -1)
        self.listBox = wx.ListBox(panel, -1, (0, 0), (250, 600), title_list,
                                  wx.LB_SINGLE, name="listbox")
        self.listBox.SetSelection(0)
        self.Bind(wx.EVT_LISTBOX, self.on_choice, self.listBox)

        # 多行文本框
        self.area_text = wx.TextCtrl(panel, -1, all_news[self.listBox.GetStringSelection()], pos=(260, 0),
                                     size=(900, 600),
                                     style=(wx.TE_MULTILINE | wx.TE_WORDWRAP))

    def on_choice(self, event):
        """
        用户点击列表框的事件反应
        :param event:
        :return:
        """
        self.area_text.Clear()
        self.area_text.AppendText(all_news[self.listBox.GetStringSelection()])
        self.area_text.SetInsertionPoint(0)

class App(wx.App):
    """
    自定义一个界面
    """
    def OnInit(self):
        # 创建窗口对象
        frame = ListBoxFrame()
        frame.Show()
        return True

    def OnExit(self):
        return 0

if __name__ == '__main__':
    App = App()
    # ListBoxFrame().Show()
    App.MainLoop()


from ray.rllib.models.modelv2 import ModelV2
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
from ray.rllib.utils import try_import_torch
from ray.rllib.utils.annotations import override

torch, nn = try_import_torch()

class SecondModel(TorchModelV2, nn.Module):
    def __init__(self, obs_space, action_space, num_outputs, model_config, name, in_channels):
        nn.Module.__init__(self)
        super().__init__(obs_space, action_space, num_outputs, model_config, name)

        self.shared_layers = nn.Sequential(
            nn.Conv2d(
                in_channels=in_channels,
                out_channels=32,
                kernel_size=3,
                stride=1),
            nn.ReLU(),
            nn.Conv2d(
                in_channels=32,
                out_channels=64,
                kernel_size=3,
                stride=1),
            nn.ReLU(),
            nn.Conv2d(
                in_channels=64,
                out_channels=128,
                kernel_size=3,
                stride=1),
            nn.ReLU(),
            nn.Conv2d(
                in_channels=128,
                out_channels=256,
                kernel_size=3,
                stride=1),
            nn.ReLU(),
            nn.Flatten(),  # 1 * 13 * 256 = 3328
            nn.Linear(3328, 1024),
            nn.ReLU(),
            nn.Linear(1024, 512),
            nn.ReLU(),
            nn.Linear(512, 256),
            nn.ReLU(),
        )

        self.actor_layers = nn.Sequential(
            nn.Linear(256 + 2, 6)
        )

        self.critic_layers = nn.Sequential(
            nn.Linear(256 + 2, 1)
        )

        self._shared_layer_out = None
        self._features = None

    @override(ModelV2)
    def forward(self, input_dict, state, seq_lens):
        x = input_dict["obs"]["conv_features"]
        x = self.shared_layers(x)

        self._shared_layer_out = torch.cat((x, input_dict["obs"]["fc_features"]), dim=1)
        logits = self.actor_layers(self._shared_layer_out)

        return logits, state

    def predict(self, input_dict):
        return torch.argmax(self.forward(input_dict, [], None)[0]).item()

    def value_function(self):
        return torch.reshape(self.critic_layers(self._shared_layer_out), [-1])


from sgp.lib.base import BaseController
from sgp.model import metadata, DBSession
from sgp.model.auth import Item,Usuario, Rol, Permiso, Fase, Recurso, Proyecto
from sgp import model
import transaction

class FaseManager():

    def getAll(self):
        user = DBSession.query(Fase).all()
        return user

    def getByLogin(self, loginName):
        user = DBSession.query(Fase).filter(Fase.id_fase.like(loginName)).one();
        return user

    def getById(self, idUser):
        user = DBSession.query(Fase).filter(Fase.id_fase==(idUser)).one();
        return user

    def add(self, user):
        DBSession.add(user)
        transaction.commit()

        recurso = Recurso()
        recurso.fase = user
        recurso.tipo = 2
        DBSession.add(recurso)
        transaction.commit()

    def update(self,user):
        DBSession.merge(user)
        transaction.commit()

    def delete(self,user):
        recurso = DBSession.query(Recurso).filter(user.id_fase == Recurso.id_fase)
        DBSession.delete(recurso)
        transaction.commit()

        DBSession.delete(user)
        transaction.commit()

    def deleteById(self, id):
        recurso = DBSession.query(Recurso).filter(id == Recurso.id_fase).one()
        DBSession.delete(recurso)
        transaction.commit()

        u = self.getById(id)
        DBSession.delete(u)
        transaction.commit()

    def deleteByLogin(self,name):
        u = self.getByLogin(name)
        DBSession.delete(u)
        transaction.commit()
    def buscar(self, buscado, id_proyecto):

        lista = DBSession.query(Fase).filter(Fase.nombre.op('~*')(buscado) & (Fase.id_proyecto ==id_proyecto )).all()
        return lista
    def getListaFases(self, lista_id):
        listaFases = []
        for i in lista_id:
            p = self.getById(i)
            listaFases.append(p)
        return listaFases

    def buscar_por_proyecto(self, buscado):
        lista = DBSession.query(Fase).filter(Fase.id_proyecto == buscado).all()
        return lista
    def generarCodigo(self, id_proyecto):
        transaction.begin()
        idp=int(id_proyecto)
        p = DBSession.query(Proyecto).filter(Proyecto.id_proyecto==idp).one()
        nro = p.nro_fase+1
        p.nro_fase = nro
        DBSession.merge(p)
        transaction.commit()
        return nro
    def ordenarFase(self, orden, fase):
        transaction.begin()
        f = self.getById(fase)
        f.orden = orden
        DBSession.merge(f)
        transaction.commit()


import initDB
import psycopg2
import time
from datetime import date

def getStockList(connection):
    """
    getStockList: returns the list of all stocks with their end date and usable values
    connection - sql database connection object
    return: list of stocks (stocks are list of their info values)
    """
    sqlString = "SELECT * FROM StockInfo WHERE Usable=TRUE ORDER BY Symbol"
    # open cursor
    DBcurr = connection.cursor()
    DBcurr.execute(sqlString)
    stocksList = DBcurr.fetchall()
    DBcurr.close()
    connection.commit()
    return stocksList

def getPoints(symbol, columns, connection):
    """
    getPoints: gets the points from sql database that has the columns listed in the array columns
    symbol - string symbol for stock
    columns - array with strings of column names
    connection - sql database connection object
    return: array of points (points are themselves an array)
    """
    queryString = ""
    onItem = 0
    # add columns to select
    for column in columns:
        if onItem > 0:
            queryString += ","
        onItem += 1
        queryString += column
    sqlString = "SELECT " + queryString + " FROM StockPoints WHERE Symbol=%s ORDER BY Date"
    sqlItem = [symbol]
    # open cursor
    DBcurr = connection.cursor()
    DBcurr.execute(sqlString, sqlItem)
    pointsList = DBcurr.fetchall()
    DBcurr.close()
    connection.commit()
    return pointsList

def priceQuery(symbol, date, connection):
    """
    priceQuery: gets the price of stock symbol at date date
    symbol - string symbol of stock to pricecheck
    connection - sql database connection object
    return: integer stock price
    """
    sqlString = "SELECT AdjClose FROM StockPoints WHERE Symbol=%s AND Date=%s;"
    sqlItem = [symbol, date]
    DBcurr = connection.cursor()
    DBcurr.execute(sqlString,sqlItem)
    try:
        price = DBcurr.fetchall()[0][0]
    except:
        price = False
    DBcurr.close()
    connection.commit()
    return price

def fetchBrokerInfo(name, connection):
    """
    fetchBrokerInfo: fetches info for broker name
    name - string name of broker
    connection - database connection object
    return: array of broker info
    """
    sqlString = "SELECT * FROM StockBroker WHERE Name=%s;"
    sqlItems = [name]
    brokerInfo = False
    # execute query
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        brokerInfo = DBcurr.fetchone()
        DBcurr.close()
        connection.commit()
    except:
        connection.rollback()
        pass
    return brokerInfo

def setBrokerInfo(name, bank, date, connection):
    """
    setBrokerInfo: sets broker info for broker name
    name - string name of broker
    bank - float bankroll of broker
    date - date that broker is on
    connection - database connection object
    return: boolean of success
    """
    sqlString = "UPDATE StockBroker SET Bank=%s,Date=%s WHERE Name=%s;"
    sqlItems = [bank,date,name]
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        connection.commit()
        DBcurr.close()
    except:
        connection.rollback()
        return False
    return True

def queryAllOwned(name, connection):
    """
    queryAllOwned: queries for all owned stocks for broker name
    name - string name of broker
    connection - database connection object
    return: list of symbol,shares for all owned stocks
    """
    sqlString = "SELECT Symbol,Shares FROM OwnedStocks WHERE Broker=%s;"
    sqlItems = [name]
    ownedStocks = []
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        ownedStocks = DBcurr.fetchall()
        connection.commit()
        DBcurr.close()
    except:
        pass
    return ownedStocks

def queryLog(name, connection):
    """
    queryLog: queries for all log entries for name
    name - string name of broker
    connection - database connection object
    return: list of symbol,shares for all owned stocks
    """
    sqlString = "SELECT * FROM TransLog WHERE Broker=%s;"
    sqlItems = [name]
    ownedStocks = []
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        ownedStocks = DBcurr.fetchall()
        connection.commit()
        DBcurr.close()
    except:
        pass
    return ownedStocks

def queryOwnedStock(name, symbol, connection):
    """
    queryOwnedStock: queries to see if stock is owned and how many
    name - string name of broker
    symbol - string symbol of stock to query for
    connection - database connection object
    return: total shares owned
    """
    sqlString = "SELECT Shares FROM OwnedStocks WHERE Broker=%s AND Symbol=%s;"
    sqlItems = [name, symbol]
    owned = 0
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        owned = DBcurr.fetchone()[0]
        connection.commit()
        DBcurr.close()
    except:
        pass
    return owned

def clearTransLog(name, connection):
    """
    clearTransLog: clear transaction log for broker name
    name - string name of broker
    connection - database connection object
    return: boolean of success
    """
    sqlString = "DELETE FROM TransLog WHERE Broker=%s;"
    sqlItems = [name]
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        connection.commit()
        DBcurr.close()
    except:
        connection.rollback()
        return False
    return True

def clearOwnedStocks(name, connection):
    """
    clearOwnedStocks: clear owned stocks for broker name
    name - string name of broker
    connection - database connection object
    return: boolean of success
    """
    sqlString = "DELETE FROM OwnedStocks WHERE Broker=%s;"
    sqlItems = [name]
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        connection.commit()
        DBcurr.close()
    except:
        connection.rollback()
        return False
    return True

def sqlTransaction(date, name, buysell, symbol, shares, price, connection):
    """
    sqlTransaction: processes transaction on sql database
    name - string name of broker
    buysell - string buy or sell which denotes type of transaction
    shares - int number of shares to buy or sell
    price - float price of stock at buy
    connection - database connection object
    return: boolean of success
    """
    # initialize transaction string and items
    tranString = ""
    tranItems = []
    # initialize log string and items
    logString = "INSERT INTO TransLog (Date, Broker, Symbol, Shares, Price, BuySell) VALUES (%s,%s,%s,%s,%s,%s);"
    logItems = [date, name, symbol, shares, price, buysell]
    # check how much of the stock is owned
    owned = queryOwnedStock(name, symbol, connection)
    # buy transaction
    if buysell == "BUY":
        # if none are owned
        if owned == 0:
            tranString = "INSERT INTO OwnedStocks (Broker, Symbol, Shares) VALUES (%s,%s,%s);"
            tranItems = [name, symbol, shares]
        # if some are owned
        else:
            ownedVal = owned * price
            if ownedVal > 10000:
                return False
            tranString = "UPDATE OwnedStocks SET Shares=%s WHERE Broker=%s AND Symbol=%s;"
            newShares = owned + shares
            tranItems = [newShares, name, symbol]
    # sell transaction
    elif buysell == "SELL":
        # invalid if not enough shares are owned
        if shares > owned:
            return False
        # delete row if all shares sold
        elif shares == owned:
            tranString = "DELETE FROM OwnedStocks WHERE Broker=%s AND Symbol=%s;"
            tranItems = [name, symbol]
        # update row with remaining shares
        elif shares < owned:
            newShares = owned - shares
            tranString = "UPDATE OwnedStocks SET Shares=%s WHERE Broker=%s AND Symbol=%s;"
            tranItems = [newShares, name, symbol]
    # invalid transaction
    else:
        return False
    #try:
        # sql executions
    DBcurr = connection.cursor()
    DBcurr.execute(tranString,tranItems)
    connection.commit()
    DBcurr.execute(logString,logItems)
    connection.commit()
    DBcurr.close()
    #except:
        #return False
    return True

def dateExists(date, connection):
    """
    dateExists: queries for points on date to see if it exists
    date - date object to query
    connection - database connection object
    return: boolean if date exists
    """
    sqlString = "Select MAX(Date) FROM StockPoints WHERE Date=%s;"
    sqlItems = [date]
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString,sqlItems)
        dateNow = DBcurr.fetchone()
        connection.commit()
        DBcurr.close()
    except:
        connection.rollback()
        return False
    # check if date exists
    if dateNow:
        return True
    else:
        return False

def stockDateExists(date, symbol, connection):
    """
    dateExists: queries for points on date to see if it exists
    date - date object to query
    connection - database connection object
    return: boolean if date exists
    """
    sqlString = "Select MAX(Date) FROM StockPoints WHERE Date=%s AND Symbol=%s;"
    sqlItems = [date, symbol]
    # execute command
    try:
        DBcurr = connection.cursor()
        DBcurr.execute(sqlString, sqlItems)
        dateNow = DBcurr.fetchone()
        connection.commit()
        DBcurr.close()
    except:
        connection.rollback()
        return False
    # check if date exists
    if dateNow[0]:
        return True
    else:
        return False

def getPointsDateRange(d1, d2, symbol, columns, connection):
    """
    getPoints: gets the points from sql database that has the columns listed in the array columns
    d1, d2 - date range to load points from
    symbol - string symbol for stock
    columns - array with strings of column names
    connection - sql database connection object
    return: array of points (points are themselves an array)
    """
    queryString = ""
    onItem = 0
    # add columns to select
    for column in columns:
        if onItem > 0:
            queryString += ","
        onItem += 1
        queryString += column
    sqlString = "SELECT " + queryString + " FROM StockPoints WHERE Symbol=%s AND DATE BETWEEN %s AND %s ORDER BY Date"
    sqlItem = [symbol,d1,d2]
    # open cursor
    DBcurr = connection.cursor()
    DBcurr.execute(sqlString, sqlItem)
    pointsList = DBcurr.fetchall()
    DBcurr.close()
    connection.commit()
    return pointsList


import asyncio
import datetime
import logging

from .codecs import get_decoder
from .exceptions import InvalidStateError
from .jitterbuffer import JitterBuffer
from .mediastreams import MediaStreamTrack
from .rtp import RTCP_RR, RTCP_SR
from .stats import (RTCRemoteInboundRtpStreamStats,
                    RTCRemoteOutboundRtpStreamStats)

logger = logging.getLogger('rtp')

class RemoteStreamTrack(MediaStreamTrack):
    def __init__(self, kind):
        self.kind = kind
        self._queue = asyncio.Queue()

    async def recv(self):
        return await self._queue.get()

class RTCRtpReceiver:
    """
    The :class:`RTCRtpReceiver` interface manages the reception and decoding
    of data for a :class:`MediaStreamTrack`.

    :param: kind: The kind of media (`'audio'` or `'video'`).
    :param: transport: An :class:`RTCDtlsTransport`.
    """
    def __init__(self, kind, transport):
        if transport.state == 'closed':
            raise InvalidStateError

        self._decoders = {}
        self._kind = kind
        self._jitter_buffer = JitterBuffer(capacity=32)
        self._track = None
        self._started = False
        self._stats = {}
        self._stopped = asyncio.Event()
        self._transport = transport

    @property
    def transport(self):
        """
        The :class:`RTCDtlsTransport` over which the media for the receiver's
        track is received.
        """
        return self._transport

    async def receive(self, parameters):
        """
        Attempt to set the parameters controlling the receiving of media.

        :param: parameters: The :class:`RTCRtpParameters` for the receiver.
        """
        if not self._started:
            for codec in parameters.codecs:
                self._decoders[codec.payloadType] = get_decoder(codec)
            self._transport._register_rtp_receiver(self, parameters)
            self._started = True

    def setTransport(self, transport):
        self._transport = transport

    async def stop(self):
        """
        Irreversibly stop the receiver.
        """
        self._stopped.set()

    async def _handle_rtcp_packet(self, packet):
        logger.debug('receiver(%s) < %s' % (self._kind, packet))

        if packet.packet_type == RTCP_SR:
            stats = RTCRemoteOutboundRtpStreamStats(
                # RTCStats
                timestamp=datetime.datetime.now(),
                type='remote-outbound-rtp',
                id=str(id(self)),
                # RTCStreamStats
                ssrc=packet.ssrc,
                kind=self._kind,
                transportId=str(id(self.transport)),
                # RTCSentRtpStreamStats
                packetsSent=packet.sender_info.packet_count,
                bytesSent=packet.sender_info.octet_count,
                # RTCRemoteOutboundRtpStreamStats
                localId='TODO',
                remoteTimestamp=packet.sender_info.ntp_timestamp  # FIXME convert to a datetime
            )
            self._stats[stats.type] = stats

        if packet.packet_type in [RTCP_SR, RTCP_RR]:
            for report in packet.reports:
                stats = RTCRemoteInboundRtpStreamStats(
                    # RTCStats
                    timestamp=datetime.datetime.now(),
                    type='remote-inbound-rtp',
                    id=str(id(self)),
                    # RTCStreamStats
                    ssrc=packet.ssrc,
                    kind=self._kind,
                    transportId=str(id(self.transport)),
                    # RTCReceivedRtpStreamStats
                    packetsReceived=0,  # FIXME: where do we get this?
                    packetsLost=report.packets_lost,
                    jitter=report.jitter,
                    # RTCRemoteInboundRtpStreamStats
                    localId='TODO',
                    roundTripTime=0,  # FIXME: where do we get this?
                    fractionLost=report.fraction_lost
                )
                self._stats[stats.type] = stats

    async def _handle_rtp_packet(self, packet):
        logger.debug('receiver(%s) < %s' % (self._kind, packet))
        if packet.payload_type in self._decoders:
            decoder = self._decoders[packet.payload_type]
            loop = asyncio.get_event_loop()
            self._jitter_buffer.add(packet.payload, packet.sequence_number, packet.timestamp)

            if self._kind == 'audio':
                # FIXME: audio should use the jitter buffer!
                audio_frame = await loop.run_in_executor(None, decoder.decode, packet.payload)
                await self._track._queue.put(audio_frame)
            else:
                # check if we have a complete video frame
                payloads = []
                got_frame = False
                last_timestamp = None
                for count in range(self._jitter_buffer.capacity):
                    frame = self._jitter_buffer.peek(count)
                    if frame is None:
                        break
                    if last_timestamp is None:
                        last_timestamp = frame.timestamp
                    elif frame.timestamp != last_timestamp:
                        got_frame = True
                        break
                    payloads.append(frame.payload)

                if got_frame:
                    self._jitter_buffer.remove(count)
                    video_frames = await loop.run_in_executor(None, decoder.decode, payloads)
                    for video_frame in video_frames:
                        await self._track._queue.put(video_frame)


from django.conf import settings
from django.contrib.auth.models import User
from django.shortcuts import render

import pdb

class Common():

    PRIVATE_IPS_PREFIX = ('10.', '172.', '192.', )
    def get_client_ip(self,request):
        """get the client ip from the request
        """
        remote_address = request.META.get('REMOTE_ADDR')
        # set the default value of the ip to be the REMOTE_ADDR if available
        # else None
        ip = remote_address
        # try to get the first non-proxy ip (not a private ip) from the
        # HTTP_X_FORWARDED_FOR
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for:
            proxies = x_forwarded_for.split(',')
            # remove the private ips from the beginning
            while (len(proxies) > 0 and
                    proxies[0].startswith(PRIVATE_IPS_PREFIX)):
                proxies.pop(0)
            # take the first ip which is not a private one (of a proxy)
            if len(proxies) > 0:
                ip = proxies[0]

        return ip

    def redirect_login(self, isMobile, request):
        next_url = request.GET.get('next')
        context  = {'next':next_url}
        if isMobile:
            return render(request, 'registration/m_login.html', context)
        else:
            return render(request, 'registration/login.html', context)

    #user is not login, and redirect by server side.
    #we need the next parameter to direct the URL after login
    def redirect_login_path(self, isMobile, request):
        next_url = request.path
        context  = {'next':next_url}
        if isMobile:
            return render(request, 'registration/m_login.html', context)
        else:
            return render(request, 'registration/login.html', context)


import matplotlib.pyplot as plt
import numpy as np
from numpy import linalg
import cvxopt
import cvxopt.solvers

def linear_kernel(x1, x2):
    return np.dot(x1, x2)

def polynomial_kernel(x, y, p=3):
    return (1 + np.dot(x, y)) ** p

def gussian_kernel(x, y, sigma=5.0):
    return np.exp(-linalg.norm(x - y) ** 2 / (2 * (sigma ** 2)))

class Support_Vector_Machine:

    def __init__(self, kernel=linear_kernel, C=None):
        self.kernel = kernel
        self.C = C
        if self.C is not None:
            self.C = float(self.C)

    def fit(self, features, labels):
        n_samples, n_features = features.shape
        K = np.zeros((n_samples, n_samples))
        for i in range(n_samples):
            for j in range(n_samples):
                K[i, j] = self.kernel(features[i], features[j])

        # set parameters to solve the quadratic problem.
        P = cvxopt.matrix(np.outer(labels, labels) * K)
        q = cvxopt.matrix(np.ones(n_samples) * -1)
        A = cvxopt.matrix(labels, (1, n_samples))
        b = cvxopt.matrix(0.0)
        if self.C is None:
            G = cvxopt.matrix(np.diag(np.ones(n_samples) * -1))
            h = cvxopt.matrix(np.zeros(n_samples))
        else:
            tmp1 = np.diag(np.ones(n_samples) * -1)
            tmp2 = np.identity(n_samples)
            G = cvxopt.matrix(np.vstack((tmp1, tmp2)))
            tmp1 = np.zeros(n_samples)
            tmp2 = np.ones(n_samples) * self.C
            h = cvxopt.matrix(np.hstack((tmp1, tmp2)))

        # Solve quadratic problem.
        solution = cvxopt.solvers.qp(P, q, G, h, A, b)

        # Deduce the lagrange multiplier alpha.
        alpha = np.ravel(solution['x'])

        # Support vectors have non zero lagrange multipliers.
        # Determine the support vectors
        support_vector_alpha = alpha > 1e-5
        ind = np.arange(len(alpha))[support_vector_alpha]
        self.alpha = alpha[support_vector_alpha]
        self.support_vector_features = features[support_vector_alpha]
        self.support_vector_labels = labels[support_vector_alpha]
        print(len(self.alpha), "support vectors out of", n_samples, "points.")

        # Deduce the b value.
        # b = y - X * w.
        self.b = 0
        for n in range(len(self.alpha)):
            # plus y
            self.b += self.support_vector_labels[n]
            # - X * w
            self.b -= np.sum(self.alpha * self.support_vector_labels * K[ind[n], support_vector_alpha])
        # calculate mean
        self.b /= len(self.alpha)

        # Deduce w if possible.
        if self.kernel == linear_kernel:
            self.w = np.zeros(n_features)
            for n in range(len(self.alpha)):
                # w = sigma(alphai * yi * xi).
                self.w += self.alpha[n] * self.support_vector_labels[n] * self.support_vector_features[n]
        else:
            self.w = None

    def predict(self, features):
        if self.w is not None:
            sign = np.sign(np.dot(features, self.w) + self.b)
            if sign == 1:
                return "spam"
            else:
                return "not_spam"
        else:
            wx = 0
            for i in range(len(features)):
                for alpha, sv_y, sv_x in zip(self.alpha, self.support_vector_labels, self.support_vector_features):
                    # s = w * x
                    # w = sigma(alphai * yi * xi).
                    wx += alpha * sv_y * np.sum(self.kernel(features[i], sv_x))
                wx /= len(features)
                y_predict = wx + self.b
                # class = y_predict + b.
                print(y_predict + self.b, self.b)
                sign = np.sign(y_predict)
                if sign == 1:
                    return "spam"
                else:
                    return "not_spam"

svm = Support_Vector_Machine(kernel=polynomial_kernel)
featrues = np.array([[0, 0, 0, 0, 0, 0, 1],
                [1, 0, 0, 0, 0, 0, 0],
                [0, 0, 1, 0, 0, 0, 1],
                [1, 1, 0, 0, 0, 0, 0],
                [0, 0, 0, 0, 0, 0, 0],
                [1, 0, 1, 1, 0, 0, 1]])
labels = np.array([1.0, 1.0, -1.0, -1.0, 1.0, -1.0])
svm.fit(featrues, labels)
print(svm.predict([1, 0, 1, 0, 1, 0, 1]))


from yaml import load

def loadConfigFromFile(filepath):
    """
    Load the configuration options from a filepath

    @param filepath: The filepath to a configuration file.
    @type filepath: C{twisted.python.filepath.FilePath}
    """
    return loadConfigFromString(filepath.getContent())

def loadConfigFromString(s):
    """
    Load the configuration options from a string.

    @param s: a C{str} that contains the yaml formatted
                    configuration
    @type s: yaml formatted C{str}

    @returns: a tuple of hostname and L{engine.ThrottlingDeferred}s
                and the default behavior for unknown urls.
    """
    from turtle import engine

    loaded = load(s)

    # Remove the filtering param
    rest = loaded.pop('filter-rest', True)
    port = loaded.pop('port', 8080)
    # Remove the defaults to use them as
    # a base to work on.
    defaults = loaded.pop('defaults', {})
    # If nothing is in the config file, this is the default
    defaults.update({'calls': 1, 'interval': 1, 'concurrency': 10})

    urlmapping = {}
    for host, kwargs in loaded.iteritems():
        # Fill an empty dictionary with the defaults
        # And then override the defaults with the one
        # in the current block so that we use them
        kw = {}
        kw.update(defaults)
        kw.update(kwargs)
        urlmapping[host] = engine.ThrottlingDeferred(**kw)

    return urlmapping, rest, port