import jax.numpy as np
from jax import random, grad, vmap, jit
import matplotlib.pyplot as plt
from activation_jax import tanh, sigmoid, leaky_relu, relu
import numpy as slow_np
from metrics import score
from jax.config import config
import math

#config.update("jax_debug_nans", True) #debugger for Nan--- kill the code

# krome networks/react_hello---single T---no flux

def FNN(params,x):
    w0 = params[:20]
    b0 = params[20:40]
    w1 = params[40:60]
    b1 = params[60]

    x=tanh(x*w0+b0)
    x=tanh(np.sum(x*w1)+b1)
    return x

exact=slow_np.loadtxt('NN_jax/fort.66',unpack=True)

key = random.PRNGKey(0)
params = random.normal(key, shape=(61,3))

dfdx = grad(FNN, 1)

inputs = np.linspace(-0., 5., num=1130)
inputs_krome=exact[0,:]

f_vect1 = vmap(FNN, (None, 0))
dfdx_vect1 = vmap(dfdx, (None, 0))

f_vect2 = vmap(FNN, (None, 0))
dfdx_vect2 = vmap(dfdx, (None, 0))

f_vect3 = vmap(FNN, (None, 0))
dfdx_vect3 = vmap(dfdx, (None, 0))
@jit

def loss(params, inputs):
    eq1=dfdx_vect1(params[:,0], inputs)+f_vect1(params[:,0], inputs)
    eq2=dfdx_vect2(params[:,1], inputs)-f_vect1(params[:,0], inputs)+0.5*f_vect2(params[:,1], inputs)
    eq3=dfdx_vect3(params[:,2], inputs)-0.5*f_vect2(params[:,1], inputs)

    ic1=FNN(params[:,0],0.)-1
    ic2=FNN(params[:,1],0.)-1e-10
    ic3=FNN(params[:,2],0.)-1e-10

    return sum((np.mean(eq1**2.+ic1**2),np.mean(eq2**2.+ic2**2),np.mean(eq3**2.+ic3**2)))

grad_loss = jit(grad(loss, 0))

'''
learning_rate = 0.0007
momentum = 0.99
velocity = np.array([0.,0.,0.])

for epoch in range(epochs):
    if epoch % 100 == 0:
        print('epoch: %3d loss: %.6f' % (epoch, loss(params, inputs)))
        print(f_vect1(params[:,0],inputs)[0]-1.)
    gradient = grad_loss(params + momentum*velocity, inputs)
    velocity = momentum*velocity - learning_rate*gradient
    params += velocity
'''

lr=8e-3
beta1=9e-1
beta2=999e-3
delta=1e-8
momentum=0e0
velocity=0e0

epoch=0
eps=5e-6

train_loss=[]
validation_loss=[]

#for epoch in range(epochs):
while loss(params,inputs) > eps and epoch < 50000:

    if epoch % 200 == 0:
        print('epoch: %3d loss: %.6f' % (epoch, loss(params, inputs)),loss(params,inputs_krome))

    train_loss.append(loss(params, inputs))
    validation_loss.append(loss(params,inputs_krome))

    gradient = grad_loss(params, inputs)

    momentum=beta1*momentum+(1e0-beta1)*gradient
    velocity=beta2*velocity+(1e0-beta2)*gradient*gradient

    momentum_norm=momentum/(1e0-beta1**float(epoch+1))
    velocity_norm=velocity/(1e0-beta2**float(epoch+1))

    epoch=epoch+1

    if slow_np.isnan(momentum_norm).any()==False:
        params-= lr*momentum_norm/(np.sqrt(velocity_norm)+delta)
    else:
        params=params

train_loss=np.array(train_loss)
validation_loss=np.array(validation_loss)

plt.plot(np.arange(len(train_loss))+1,np.log10(train_loss),label='train')
plt.plot(np.arange(len(train_loss))+1,np.log10(validation_loss),label='valid')
plt.xlabel('epoch')
plt.ylabel('log10[loss]')
plt.legend()
plt.show()
#print(f_vect1(params[:,0], inputs),f_vect2(params[:,1], inputs),f_vect3(params[:,2], inputs))

print('specie 1:')
score(exact[1,:],f_vect1(params[:,0], inputs_krome))

print('specie 2:')
score(exact[2,:],f_vect1(params[:,1], inputs_krome))

print('specie 3:')
score(exact[3,:],f_vect1(params[:,2], inputs_krome))

plt.plot(exact[0,:],exact[1,:],label='A_krome',color='red')
plt.plot(exact[0,:],exact[2,:],label='B_krome',color='green')
plt.plot(exact[0,:],exact[3,:],label='C_krome',color='blue')

plt.plot(inputs_krome, f_vect1(params[:,0], inputs_krome),'.', label='A_pred',color='red')
plt.plot(inputs_krome, f_vect2(params[:,1], inputs_krome),'.', label='B_pred',color='green')
plt.plot(inputs_krome, f_vect3(params[:,2], inputs_krome),'.', label='C_pred',color='blue')
plt.legend()

plt.xlabel('t')
plt.ylabel('n[#/cm3]')
#plt.savefig('krome_hello.png')
plt.show()


import cv2
import imutils
import numpy as np
import math
import time
import os

def align_images(image, template, maxFeatures=3000,
                 keepPercent=0.2, debug=False):
    """align image with template

    Args:
        image (_type_): 要对齐的图片
        template (_type_): 模板图片
        maxFeatures (int, optional): _description_. Defaults to 3000.
        keepPercent (float, optional): _description_. Defaults to 0.2.
        debug (bool, optional): _description_. Defaults to False.

    Returns:
        ndarray: 对齐后的图片
    """
    # use grayscale
    imageGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    templateGray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)

    orb = cv2.ORB_create(maxFeatures)
    (kpsA, descsA) = orb.detectAndCompute(imageGray, None)
    (kpsB, descsB) = orb.detectAndCompute(templateGray, None)

    # match the features
    method = cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING
    matcher = cv2.DescriptorMatcher_create(method)
    matches = matcher.match(descsA, descsB, None)

    matchesort = sorted(matches, key=lambda x: x.distance)

    keep = int(len(matchesort) * keepPercent)
    matches = matchesort[:keep]

    # if debug:
    #     matchedVis = cv2.drawMatches(image, kpsA, template, kpsB,
    #                                  matches, None)
    #     matchedVis = imutils.resize(matchedVis, width=1000)
    #     cv2.imshow("Matched Keypoints", matchedVis)
    #     cv2.waitKey(0)

    ptsA = np.zeros((len(matches), 2), dtype="float")
    ptsB = np.zeros((len(matches), 2), dtype="float")

    for (i, m) in enumerate(matches):
        # indicate that the two keypoints in the respective images
        # map to each other
        ptsA[i] = kpsA[m.queryIdx].pt
        ptsB[i] = kpsB[m.trainIdx].pt

    (H, mask) = cv2.findHomography(ptsA, ptsB, method=cv2.RANSAC)

    (h, w) = template.shape[:2]
    aligned = cv2.warpPerspective(image, H, (w, h))

    return aligned

def align_check(img, template):
    """check if img is aligned to template

    Args:
        img (ndarray): 测试图片
        template (ndarray): 模板图片

    Returns:
        Bool: True/False
    """
    aligned = False

    return aligned

if __name__ == "__main__":
    templatepath = '/home/znzz/Desktop/Data/mfl/CODE/printcheck/tmp/2022-10-11'
    testimgpath = '/home/znzz/Desktop/Data/mfl/CODE/printcheck/tmptest'

    ims = os.listdir(testimgpath)
    im = cv2.imread(os.path.join(testimgpath, ims[4]))
    template = os.listdir(templatepath)
    templateim = cv2.imread(os.path.join(templatepath,template[4]))

    align_images(im, templateim)


from matplotlib import pyplot as plt
import numpy as np
from random import shuffle

from KeraScratch import model, optimizers
from KeraScratch.operators import activation, layer, loss

def get_images(path, split=0):
    ds = np.load(path)
    x = ds['X']
    y = ds['Y']
    data = []
    for i in range(x.shape[0]):
        data.append([np.array([x[i, :].astype(np.float64) / 255]), np.array([np.argmax(y[i, :])])])
    if split == 0:
        return data
    shuffle(data)
    return data[:int(split*len(data))], data[int(split*len(data)):]

train_data, val_data = get_images('image/trainset.npz', 0.9)
test_data = get_images('image/testset.npz')

cnn = model.SequentialModel([
    layer.Conv2DOperator(8, 5, padding='same'),
    activation.ReLUOperator(),
    layer.MaxPoolingOperator(2),
    layer.Conv2DOperator(16, 5, padding='same'),
    activation.ReLUOperator(),
    layer.MeanPoolingOperator(2),
    layer.FlattenOperator(),
    layer.LinearOperator(64),
    activation.ReLUOperator(),
    layer.LinearOperator(10),
    loss.CrossEntropyLoss()
])
cnn.compile(optimizers.SGD(learning_rate=1e-3, momentum=0))

epoch = 20
history = cnn.fit(train_data, val_data, epoch=epoch)

x_range = range(epoch)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(x_range, history['train']['accuracy'], label='train acc')
ax.plot(x_range, history['val']['accuracy'], label='test acc')
ax.set_ylabel('accuracy')
ax.legend(loc='upper left')

ax2 = ax.twinx()
ax2.plot(x_range, history['train']['loss'], label='train loss')
ax2.plot(x_range, history['val']['loss'], label='test loss')
ax2.set_ylabel('loss')
ax2.legend(loc='upper right')

plt.show()

test_his = cnn.evaluate(test_data)
print(test_his)


# from skimage import io
from PIL import ImageFile
import numpy as np
import re
import os
import time
import sys
import signal
from glob import glob
import numpy as np
import tensorflow as tf
from IPython import embed
import math
import scipy.stats as st
from skimage import feature

# This part of codes come from Ayan Chakrabarti <ayan@wustl.edu>
# General usage

# Logic for trapping ctrlc and setting stop
stop = False
_orig = None
def handler(a,b):
    global stop
    stop = True
    signal.signal(signal.SIGINT,_orig)
_orig = signal.signal(signal.SIGINT,handler)

def mprint(s):
    sys.stdout.write(time.strftime("%Y-%m-%d %H:%M:%S ") + s + "\n")
    sys.stdout.flush()

def vprint(it, nms, vals):
    s = '[%06d]' % it
    for i in range(len(nms)):
        s = s + ' ' + nms[i] + ' = %.3e'%vals[i]
    mprint(s)

# Manage checkpoint files, read off iteration number from filename
# Use clean() to keep latest, and modulo n iters, delete rest
class ckpter:
    def __init__(self,wcard):
        self.wcard = wcard
        self.load()

    def load(self):
        lst = glob(self.wcard)
        if len(lst) > 0:
            lst=[(l,int(re.match('.*/.*_(\d+)',l).group(1)))
                 for l in lst]
            self.lst=sorted(lst,key=lambda x: x[1])

            self.iter = self.lst[-1][1]
            self.latest = self.lst[-1][0]
        else:
            self.lst=[]
            self.iter=0
            self.latest=None

    def clean(self,every=0,last=1):
        self.load()
        old = self.lst[:-last]
        for j in old:
            if every == 0 or j[1] % every != 0:
                os.remove(j[0])

# Save/load networks
def saveNet(fn,net,sess):
    wts = {}
    for k in net.weights.keys():
        wts[k] = net.weights[k].eval(sess)
    np.savez(fn,**wts)

def loadNet(fn,net,sess):
    wts = np.load(fn)
    for k in wts.keys():
        if k in net.weights:
            print("loading Net")
            wvar = net.weights[k]
            wk = wts[k].reshape(wvar.get_shape())
            wvar.load(wk,sess)
    print("Loading finished!")

# Save/load Adam optimizer state
def saveAdam(fn,opt,vdict,sess):
    weights = {}
    beta1_power, beta2_power = opt._get_beta_accumulators()
    weights['b1p'] = beta1_power.eval(sess)
    weights['b2p'] = beta2_power.eval(sess)
    for v in vdict:
        weights['m_%s' % v.name] = opt.get_slot(v,'m').eval(sess)
        weights['v_%s' % v.name] = opt.get_slot(v,'v').eval(sess)
    np.savez(fn,**weights)

def loadAdam(fn,opt,vdict,sess):
    weights = np.load(fn)
    beta1_power, beta2_power = opt._get_beta_accumulators()
    beta1_power.load(weights['b1p'],sess)
    beta2_power.load(weights['b2p'],sess)

    for v in vdict:
        opt.get_slot(v,'m').load(weights['m_%s' % v.name],sess)
        opt.get_slot(v,'v').load(weights['v_%s' % v.name],sess)

#Content Encoder script
################ This part has been rewritten in data.py. This comment part is used for debug and review

# def load_gan_test_image( path, pre_height=267, pre_width=267, height=256, width=256 ):
#     try:
#         img = io.imread( path ).astype( float )
#     except:
#         return None

#     img /= 255.

#     if img is None: return None
#     if len(img.shape) < 2: return None
#     if len(img.shape) == 4: return None
#     if len(img.shape) == 2: img=np.tile(img[:,:,None], 3)
#     if img.shape[2] == 4: img=img[:,:,:3]
#     if img.shape[2] > 4: return None

#     short_edge = min( img.shape[:2] )
#     yy = int((img.shape[0] - short_edge) / 2)
#     xx = int((img.shape[1] - short_edge) / 2)
#     crop_img = img[yy:yy+short_edge, xx:xx+short_edge]
#     resized_img = skimage.transform.resize( crop_img, [pre_height,pre_width] )

#     rand_y = np.random.randint(0, pre_height - height)
#     rand_x = np.random.randint(0, pre_width - width)

#     resized_img = resized_img[ rand_y:rand_y+height, rand_x:rand_x+width, : ]

#     return (resized_img * 2)-1 #(resized_img - 127.5)/127.5

def random_interpolates(x, y, alpha=None):
    """
    x: first dimension as batch_size
    y: first dimension as batch_size
    alpha: [BATCH_SIZE, 1]
    """
    shape = x.get_shape().as_list()
    x = tf.reshape(x, [shape[0], -1])
    y = tf.reshape(y, [shape[0], -1])
    if alpha is None:
        alpha = tf.random_uniform(shape=[shape[0], 1])
    interpolates = x + alpha*(y - x)
    return tf.reshape(interpolates, shape)

def gradients_penalty(x, y, mask=None, norm=1.):
    """Improved Training of Wasserstein GANs
    - https://arxiv.org/abs/1704.00028
    """
    gradients = tf.gradients(y, x)[0]
    if mask is None:
        mask = tf.ones_like(gradients)
    slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients) * mask, axis=[1, 2, 3]))
    return tf.reduce_mean(tf.square(slopes - norm))

def resize_mask_like(mask, x):
    """Resize mask like shape of x.
    Args:
        mask: Original mask.
        x: To shape of x.
    Returns:
        tf.Tensor: resized mask
    """
    mask_resize = resize(
        mask, to_shape=x.get_shape().as_list()[1:3],
        func=tf.image.resize_nearest_neighbor)
    return mask_resize

def resize(x, scale=2, to_shape=None, align_corners=True, dynamic=False,
           func=tf.image.resize_bilinear, name='resize'):

    if dynamic:
        xs = tf.cast(tf.shape(x), tf.float32)
        new_xs = [tf.cast(xs[1]*scale, tf.int32),
                  tf.cast(xs[2]*scale, tf.int32)]
    else:
        xs = x.get_shape().as_list()
        new_xs = [int(xs[1]*scale), int(xs[2]*scale)]
    with tf.variable_scope(name):
        if to_shape is None:
            x = func(x, new_xs, align_corners=align_corners)
        else:
            x = func(x, [to_shape[0], to_shape[1]],
                     align_corners=align_corners)
    # print(x)
    return x

# def spatial_discounting_mask(gamma, c_height, c_width):

#     shape = [1, c_height, c_width, 1]
#     # logger.info('Use spatial discounting l1 loss.')
#     mask_values = np.ones((c_height, c_width))
#     for i in range(c_height):
#         for j in range(c_width):
#             mask_values[i, j] = max(
#                 gamma**min(i, c_height-i),
#                 gamma**min(j, c_width-j))
#     mask_values = np.expand_dims(mask_values, 0)
#     mask_values = np.expand_dims(mask_values, 3)

#     return tf.constant(mask_values, dtype=tf.float32, shape=shape)

def gauss_kernel(size=21, sigma=3, inchannels=3, outchannels=3):
    interval = (2 * sigma + 1.0) / size
    x = np.linspace(-sigma-interval/2, sigma+interval/2, size+1)
    ker1d = np.diff(st.norm.cdf(x))
    kernel_raw = np.sqrt(np.outer(ker1d, ker1d))
    kernel = kernel_raw / kernel_raw.sum()
    out_filter = np.array(kernel, dtype=np.float32)
    out_filter = out_filter.reshape((size, size, 1, 1))
    # out_filter = np.repeat(out_filter, [1, 1, inchannels, 1])
    return out_filter

def tf_make_guass_var(size, attention_gamma, inchannels=1, outchannels=1):
    kernel = gauss_kernel(size, attention_gamma, inchannels, outchannels)
    var = tf.Variable(tf.convert_to_tensor(kernel))
    return var

def spatial_discounting_mask(mask, attention_gamma=1.0/40, hsize=64,  iters=9):
    eps = 1e-5
    kernel = tf_make_guass_var(hsize, attention_gamma)
    init = 1-mask
    mask_priority = None
    mask_priority_pre = None
    for i in range(iters):
        mask_priority = tf.nn.conv2d(init, kernel, strides=[1,1,1,1], padding='SAME')
        mask_priority = mask_priority * mask
        if i == iters-2:
            mask_priority_pre = mask_priority
        init = mask_priority + (1-mask)
    mask_priority = mask_priority_pre / (mask_priority+eps)
    return mask_priority

################  Loss function zoo  ####################
def gan_ls_loss(pos, neg, value=1., name='gan_ls_loss'):
    """
    gan with least-square loss
    """
    with tf.variable_scope(name):
        l2_pos = tf.reduce_mean(tf.squared_difference(pos, value))
        l2_neg = tf.reduce_mean(tf.square(neg))
        # scalar_summary('pos_l2_avg', l2_pos)
        # scalar_summary('neg_l2_avg', l2_neg)
        d_loss = tf.add(.5 * l2_pos, .5 * l2_neg)
        g_loss = tf.reduce_mean(tf.squared_difference(neg, value))
        # scalar_summary('d_loss', d_loss)
        # scalar_summary('g_loss', g_loss)
    return g_loss, d_loss

def gan_wgan_loss(pos, neg, name='gan_wgan_loss'):
    """
    wgan loss function for GANs.
    - Wasserstein GAN: https://arxiv.org/abs/1701.07875
    """
    with tf.variable_scope(name):
        d_loss = tf.reduce_mean(neg-pos)
        g_loss = -tf.reduce_mean(neg)
        # scalar_summary('d_loss', d_loss)
        # scalar_summary('g_loss', g_loss)
        # scalar_summary('pos_value_avg', tf.reduce_mean(pos))
        # scalar_summary('neg_value_avg', tf.reduce_mean(neg))
    return g_loss, d_loss

def max_downsampling(x, ratio=2):
    iters = math.log2(ratio)
    assert int(iters) == iters
    for _ in range(int(iters)):
        x = tf.contrib.layers.max_pool2d(x, 2, padding='SAME')
    return x

def mask_fill(mask, k = 16):
    gridy = [i for i in range(0, mask.shape[1], k)]
    gridx = [i for i in range(0, mask.shape[2], k)]
    grid = np.meshgrid(gridy, gridx)
    tmp_mask = np.zeros((mask.shape[1:3]))
    tmp_mask[tuple(grid)] = 1.0
    tmp_mask = np.expand_dims(tmp_mask, axis=2)
    tmp_mask = [tmp_mask for i in range(mask.shape[0])]
    tmp_mask = np.array(tmp_mask)
    sparse_points = tf.constant(tmp_mask, dtype=tf.float32)
    mask = mask + sparse_points
    return mask, sparse_points

def edge_detector(img_tensor, minRate=0.2, maxRate=0.2,
             preserve_size=True, remove_high_val=False, return_raw_edges=False):

    MAX = 1
    kernel_size = 3
    sigma  = 1.2
    def Gaussian_Filter(kernel_size=kernel_size, sigma=sigma):
        k = (kernel_size-1)//2
        x, y = np.meshgrid(np.linspace(-k, k, kernel_size), np.linspace(-k, k, kernel_size))
        g = np.exp(-(x * x + y * y) / (2 * sigma**2)) / (2 * np.pi * sigma ** 2)
        return np.asarray(g).reshape(kernel_size,kernel_size,1,1)

    gaussian_filter = tf.constant(Gaussian_Filter(kernel_size, sigma), tf.float32)                 #STEP-1
    h_filter = tf.reshape(tf.constant([[-1,0,1],[-2,0,2],[-1,0,1]], tf.float32), [3,3,1,1])    #STEP-2
    v_filter = tf.reshape(tf.constant([[1,2,1],[0,0,0],[-1,-2,-1]], tf.float32), [3,3,1,1])    #STEP-2

    np_filter_0 = np.zeros((3,3,1,2))
    np_filter_0[1,0,0,0], np_filter_0[1,2,0,1] = 1,1 ### Left & Right
    # print(np_filter_0)
    filter_0 = tf.constant(np_filter_0, tf.float32)
    np_filter_90 = np.zeros((3,3,1,2))
    np_filter_90[0,1,0,0], np_filter_90[2,1,0,1] = 1,1 ### Top & Bottom
    filter_90 = tf.constant(np_filter_90, tf.float32)
    np_filter_45 = np.zeros((3,3,1,2))
    np_filter_45[0,2,0,0], np_filter_45[2,0,0,1] = 1,1 ### Top-Right & Bottom-Left
    filter_45 = tf.constant(np_filter_45, tf.float32)
    np_filter_135 = np.zeros((3,3,1,2))
    np_filter_135[0,0,0,0], np_filter_135[2,2,0,1] = 1,1 ### Top-Left & Bottom-Right
    filter_135 = tf.constant(np_filter_135, tf.float32)

    np_filter_sure = np.ones([3,3,1,1]); np_filter_sure[1,1,0,0] = 0
    filter_sure = tf.constant(np_filter_sure, tf.float32)
    border_paddings = tf.constant([[0,0],[1,1],[1,1],[0,0]])

    def Border_Padding(x, pad_width):
        for _ in range(pad_width): x = tf.pad(x, border_paddings, 'SYMMETRIC')
        return x

    def FourAngles(d):
        d0   = tf.to_float(tf.greater_equal(d,157.5))+tf.to_float(tf.less(d,22.5))
        d45  = tf.to_float(tf.greater_equal(d,22.5))*tf.to_float(tf.less(d,67.5))
        d90  = tf.to_float(tf.greater_equal(d,67.5))*tf.to_float(tf.less(d,112.5))
        d135 = tf.to_float(tf.greater_equal(d,112.5))*tf.to_float(tf.less(d,157.5))

        return (d0,d45,d90,d135)

    img_tensor = (img_tensor/tf.reduce_max(img_tensor))*MAX
    if preserve_size: img_tensor = Border_Padding(img_tensor, (kernel_size-1)//2)

    x_gaussian = tf.nn.convolution(img_tensor, gaussian_filter, padding='VALID')
    if remove_high_val: x_gaussian = tf.clip_by_value(x_gaussian, 0, MAX/2)

    if preserve_size: x_gaussian = Border_Padding(x_gaussian, 1)
    Gx = tf.nn.convolution(x_gaussian, h_filter, padding='VALID')
    Gy = tf.nn.convolution(x_gaussian, v_filter, padding='VALID')
    G = tf.sqrt(tf.square(Gx) + tf.square(Gy))
    BIG_PHI = tf.atan2(Gy,Gx)
    BIG_PHI    = (BIG_PHI*180/np.pi)%180         ### Convert from Radian to Degree
    D_0,D_45,D_90,D_135 = FourAngles(BIG_PHI)    ### Round the directions to 0, 45, 90, 135 (only take the masks)

    targetPixels_0 = tf.nn.convolution(G, filter_0, padding='SAME')
    isGreater_0 = tf.to_float(tf.greater(G*D_0, targetPixels_0))
    isMax_0 = isGreater_0[:,:,:,0:1]*isGreater_0[:,:,:,1:2]

    targetPixels_90 = tf.nn.convolution(G, filter_90, padding='SAME')
    isGreater_90 = tf.to_float(tf.greater(G*D_90, targetPixels_90))
    isMax_90 = isGreater_90[:,:,:,0:1]*isGreater_90[:,:,:,1:2]

    targetPixels_45 = tf.nn.convolution(G, filter_45, padding='SAME')
    isGreater_45 = tf.to_float(tf.greater(G*D_45, targetPixels_45))
    isMax_45 = isGreater_45[:,:,:,0:1]*isGreater_45[:,:,:,1:2]

    targetPixels_135 = tf.nn.convolution(G, filter_135, padding='SAME')
    isGreater_135 = tf.to_float(tf.greater(G*D_135, targetPixels_135))
    isMax_135 = isGreater_135[:,:,:,0:1]*isGreater_135[:,:,:,1:2]

    edges_raw = G*(isMax_0 + isMax_90 + isMax_45 + isMax_135)
    edges_raw = tf.clip_by_value(edges_raw, 0, MAX)

    edges_sure = tf.to_float(tf.greater_equal(edges_raw, maxRate))
    edges_weak = tf.to_float(tf.less(edges_raw, maxRate))*tf.to_float(tf.greater_equal(edges_raw, minRate))

    edges_connected = tf.nn.convolution(edges_sure, filter_sure, padding='SAME')*edges_weak
    for _ in range(10): edges_connected = tf.nn.convolution(edges_connected, filter_sure, padding='SAME')*edges_weak

    edges_final = edges_sure + tf.clip_by_value(edges_connected,0,MAX)
    mask = tf.to_float(tf.greater(edges_final, 0.2))
    # edges_final = edges_final * mask
    mask = tf.where(tf.greater(mask*255, 200), tf.ones_like(mask)*255, tf.zeros_like(mask))
    mask = mask / 255
    # embed()
    return mask


# Team Members:
# Blake Oberlander (bho180000)
# Sharar Siddiqui (srs200003)

from nltk import word_tokenize, ngrams
from collections import Counter
import pickle

def train(filename):
    """Loads in the training data for a given language
    returns dictionary of unigram and bigram"""

    with open(filename, 'r', encoding='utf8') as f:
        tokens = word_tokenize(f.read().replace('\n', ' '))

    unigrams = [(token,) for token in tokens]
    bigrams = list(ngrams(tokens, 2))

    # count unigrams and bigrams
    unigram_count = dict(Counter((
        unigram
        for
        unigram in unigrams
    )))

    bigram_count = dict(Counter((
        bigram
        for
        bigram in bigrams
    )))

    return unigram_count, bigram_count

def pickle_save(language, unigrams, bigrams):
    """Pickles unigram and bigrams to individual files,
    Uses the language parameter to specify the language"""
    pickle.dump(unigrams, open(f"{language}-unigrams.pickle", 'wb'))
    pickle.dump(bigrams, open(f"{language}-bigrams.pickle", 'wb'))

if __name__ == '__main__':
    # find the unigram and bigram counts for each language
    english_uni, english_bi = train('ngram_files/LangId.train.English')
    french_uni, french_bi = train('ngram_files/LangId.train.French')
    italian_uni, italian_bi = train('ngram_files/LangId.train.Italian')

    # save
    print('Starting to pickle 3 language training data')
    pickle_save('english', english_uni, english_bi)
    pickle_save('french', french_uni, french_bi)
    pickle_save('italian', italian_uni, italian_bi)
    print('Finished pickling output')
    exit(0)


import curses
import logging

from client.game import ClientGame
from common.socket_wrapper import SocketWrapper
from server.game.tic_tac_toe import TicTacToe as ServerTicTacToe, Board, Move, Finish, Sign

logger = logging.getLogger(__name__)

class TicTacToe(ClientGame):
    def __init__(self, server: SocketWrapper):
        super().__init__(server)
        self._board = Board()
        self._display = Display(ServerTicTacToe.user_sign, self._board)

    @property
    def server_game(self):
        return ServerTicTacToe

    def run(self):
        self._display.start()
        self._display.display_empty_grid()
        self._display.display_title('You are playing with {}'.format(ServerTicTacToe.user_sign))
        while True:
            self._display.display_message('It is your turn')
            self._display.input()
            self._display.display_message('Awaiting information from server')
            self._server.send(Move(self._board.board))

            rcv = self._server.receive()
            if isinstance(rcv, Move):
                self._board.board = rcv.payload
                self._display.fill_grid()
            elif isinstance(rcv, Finish):
                self._display.display_title('Result: {}'.format(rcv.payload.name))
                self._display.quit()
                break

        self._display.exit()

class Display:
    title_y = 0
    title_x = 5

    message_y = 2
    message_x = 5

    grid_y = 6
    grid_x = 10

    def __init__(self, sign, board):
        self.sign = sign
        self.board = board
        self.stdscr = None
        self._title = ''
        self._message = ''

    def start(self):
        self.stdscr = curses.initscr()
        curses.noecho()  # don't display keys automatically
        curses.cbreak()  # no enter required
        self.stdscr.keypad(True)  # handle special characters
        self.stdscr.leaveok(False)

    def exit(self):
        curses.nocbreak()
        self.stdscr.keypad(False)
        curses.echo()
        curses.endwin()

    def quit(self):
        self.display_message('Press q to exit')
        while True:
            ch = self.stdscr.getch()
            if ch == ord('q'):
                self.exit()
                break

    def display_title(self, title):
        self.clear_title()
        self._title = title
        self._display_text(self.title_y, self.title_x, title)

    def clear_title(self):
        self._display_text(self.title_y, self.title_x, ' ' * len(self._title))

    def display_message(self, message):
        self.clear_message()
        self._message = message
        self._display_text(self.message_y, self.message_x, message)

    def clear_message(self):
        self._display_text(self.message_y, self.message_x, ' ' * len(self._message))

    def _display_text(self, y, x, text):
        orig_y, orig_x = self.stdscr.getyx()
        self.stdscr.addstr(y, x, text)
        self.stdscr.move(orig_y, orig_x)
        self._refresh()

    def clear(self):
        self.stdscr.erase()

    def display_empty_grid(self):
        for y in range(self.grid_y, self.grid_y + 5, 2):
            for x in range(self.grid_x + 1, self.grid_x + 4, 2):
                self.stdscr.addch(y, x, curses.ACS_VLINE)

        for y in range(self.grid_y + 1, self.grid_y + 4, 2):
            for x in range(self.grid_x, self.grid_x + 5):
                self.stdscr.addch(y, x, curses.ACS_HLINE)
        curses.curs_set(0)

    def fill_grid(self):
        board_values = self.board.board
        for x in range(3):
            for y in range(3):
                value = board_values[x][y]
                if value != Sign.empty:
                    self.stdscr.addch(self._y_to_curses(y), self._x_to_curses(x), str(value))
        self._refresh()

    def _refresh(self):
        self.stdscr.refresh()

    def input(self):
        curses.curs_set(2)
        x, y = 1, 1
        self.stdscr.move(self._y_to_curses(y), self._x_to_curses(x))
        self._refresh()
        while True:
            ch = self.stdscr.getch()
            if ch in [curses.KEY_UP, curses.KEY_DOWN, curses.KEY_LEFT, curses.KEY_RIGHT]:
                self._move(ch)
            elif ch in [curses.KEY_ENTER, ord('\n')]:
                if self._put():
                    break
            else:
                logger.warn('Unknown key {}'.format(ch))

    def _move(self, ch):
        x, y = self._xy_from_curses()
        if ch == curses.KEY_LEFT and x > 0:
            x -= 1
        elif ch == curses.KEY_RIGHT and x < 2:
            x += 1
        elif ch == curses.KEY_UP and y > 0:
            y -= 1
        elif ch == curses.KEY_DOWN and y < 2:
            y += 1
        self.stdscr.move(self._y_to_curses(y), self._x_to_curses(x))
        self._refresh()

    def _put(self):
        x, y = self._xy_from_curses()
        if self.board.board[x][y] == Sign.empty:
            self.board.put(x, y, self.sign)
            curses.curs_set(0)
            self.fill_grid()
            return True
        return False

    def _y_to_curses(self, y):
        return int(self.grid_y + y * 2)

    def _x_to_curses(self, x):
        return int(self.grid_x + x * 2)

    def _xy_from_curses(self):
        y, x = self.stdscr.getyx()
        return (x - self.grid_x) // 2, (y - self.grid_y) // 2


import numpy as np
import pandas as pd
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn import linear_model

def unpack(Dict):
    W = Dict[:,-38:]
    Dict_ = Dict[:,:-38]
    for i in range(Dict_.shape[1]):
       norm = np.linalg.norm(Dict[:,i])
       for j in range(Dict_.shape[0]):
           Dict_[j.i] = Dict_[j,i] /norm
       for j in range(W.shape[1]):
           W[j,i] = W[j,i]/norm
    return Dict_,W

def normlization(Dict):
    for i in range(Dict.shape[0]):
        norm = np.linalg.norm(Dict[i,:])
        for j in range(Dict.shape[1]):
            Dict[i,j] = Dict[i,j]/norm

    return Dict

def dict_update(y, d, x, n_components):
    """
    使用KSVD更新字典的过程
    """
    for i in range(n_components):
        index = np.nonzero(x[i, :])[0]
        if len(index) == 0:
            continue
        # 更新第i列
        d[:, i] = 0
        # 计算误差矩阵
        r = (y - np.dot(d, x))[:, index]
        # 利用svd的方法，来求解更新字典和稀疏系数矩阵
        u, s, v = np.linalg.svd(r, full_matrices=False)
        # 使用左奇异矩阵的第0列更新字典
        d[:, i] = u[:, 0]
        # 使用第0个奇异值和右奇异矩阵的第0行的乘积更新稀疏系数矩阵
        for j,k in enumerate(index):
            x[i, k] = s[0] * v[0, j]
    return d, x

def date_pre(X,y,r=100):
    H = np.zeros(shape=(0,38))
    for i in y:
        tmp = np.zeros(shape=(1,38))
        tmp[0,int(i)] = r
        H = np.vstack((H,tmp))
    X_train = np.hstack((X,H))
    return X_train

def unpack(Dict):
    W = Dict[:,-38:]
    Dict_ = Dict[:,:-38]
    for i in range(Dict_.shape[1]):
       norm = np.linalg.norm(Dict[:,i])
       for j in range(Dict_.shape[0]):
           Dict_[j.i] = Dict_[j,i] /norm
       for j in range(W.shape[1]):
           W[j,i] = W[j,i]/norm
    return Dict_,W

"""训练"""
df = pd.read_csv('new.csv',header=None)
data = np.array(df)
y = data[:,-1]
X = data[:,:-1]
print(X.shape)

X_train = date_pre(X,y,r=100)
X = normlization(X)
"""训练和测试集划分"""

skf = StratifiedShuffleSplit(n_splits=1, test_size=0.6875, random_state=0)
#  = train_test_split(X, target, test_size=size, random_state=42) #随机取样
for train_index, test_index in skf.split(X, y):
    #print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]

#####主程序

u, s, v = np.linalg.svd(X_train.T)
n_comp = 200
dictionary = u[:, :n_comp]

max_iter = 500
tolerance = 1e-4
max_iter = 10

for i in range(max_iter):
    # 稀疏编码
    x = linear_model.orthogonal_mp(dictionary, X_train.T)
    e = np.linalg.norm(X_train.T - np.dot(dictionary, x))
    if e < tolerance:
        break
    dict_update(X_train.T, dictionary, x, n_comp)

sparsecode = linear_model.orthogonal_mp(dictionary, X_test.T)

dic,w = unpack(dictionary)

train_restruct = dic.dot(sparsecode)


'''
.. module:: settings

Constants and settings for cleaner. Compiled regexes reside here.

.. moduleauthor:: Christopher Phillippi <c_phillippi@mfe.berkeley.edu>
'''
from re import compile
from multiprocessing import cpu_count
from os.path import join
from os.path import expanduser

MASTER_DIR = join( join( expanduser( '~' ), 'Dropbox' ) , 'AFPdb' )
CLEAN_STORE = join( expanduser( '~' ), "AFPCorpus" )  # avoids being saved on dropbox for now
UNCLEAN_STORE = join( MASTER_DIR, "Unclean" )
EMPIRICAL_STORE = join( MASTER_DIR, 'Empirical' )

ADJUSTED_CLOSE_FILENAME = 'adjustedClose.csv'

CLEAN_BATCH_TAG = "cleaned"
CLEAN_BATCH_STORE = join( UNCLEAN_STORE, CLEAN_BATCH_TAG )

MAX_WORKERS = cpu_count()

LEXISNEXIS_ARTICLE_DELIMITER = "All Rights Reserved"
LEXISNEXIS_FILETAG = "LexisNexis"
LEXISNEXIS_REGEX_DATE = compile( "([a-zA-Z]+) ([0-9]+), ([0-9]{4})" )
LEXISNEXIS_REGEX_EXCLUDE_FROM_TITLE = compile( "[:;,'!.\\n]" )
LEXISNEXIS_REGEX_PAPER_DATE_TITLE = compile( "[\\n ]*[0-9]+ of [0-9]+ DOCUMENTS[\\n ]*([\\w'& ]+)[^\\n]*[\\n ]*([a-zA-Z]+ [0-9]+, [0-9]{4}) [:\\w ]*\\n(?:[^\\n]+\\n)*\\n([\\w:;,!.'\\-\\n ]{1,30})" )
LEXISNEXIS_REMOVE_FROM_ARTICLE = compile( "[^\\n]\\n[^\\n]" )
LEXISNEXIS_SECTION_DELIMTER = "\n\n\n"


import pyaudio
import struct
import math
import RPi.GPIO as GPIO
import time
import statistics

GPIO.setmode(GPIO.BCM)
GPIO.setup(22,GPIO.OUT)
GPIO.setup(19,GPIO.OUT)
FORMAT = pyaudio.paInt16
SHORT_NORMALIZE = (1.0/32768.0)
CHANNELS = 1 #2
RATE = 48000#44100
INPUT_BLOCK_TIME = 0.05
INPUT_FRAMES_PER_BLOCK = int(RATE*INPUT_BLOCK_TIME)

def get_rms( block ):
    # RMS amplitude is defined as the square root of the
    # mean over time of the square of the amplitude.
    # so we need to convert this string of bytes into
    # a string of 16-bit samples...

    # we will get one short out for each
    # two chars in the string.
    count = len(block)/2
    format = "%dh"%(count)
    shorts = struct.unpack( format, block )

    # iterate over the block.
    sum_squares = 0.0
    for sample in shorts:
        # sample is a signed short in +/- 32768.
        # normalize it to 1.0
        n = sample * SHORT_NORMALIZE
        sum_squares += n*n

    return math.sqrt( sum_squares / count )

class ClarityRating(object):
    def __init__(self, limit):
        self.pa = pyaudio.PyAudio()
        self.stream = self.open_mic_stream()
        self.threshold = limit   #above this means too quiet
        self.recentDB = []
        self.thresholdSize = 60 #100 = 5 seconds
        self.errorcount = 0

    def stop(self):
        self.stream.close()

    def find_input_device(self):
        device_index = None
        for i in range( self.pa.get_device_count() ):
            devinfo = self.pa.get_device_info_by_index(i)
            print( "Device %d: %s"%(i,devinfo["name"]) )

            for keyword in ["mic","input","dsnooped"]:
                if keyword in devinfo["name"].lower():
                    print( "Found an input: device %d - %s"%(i,devinfo["name"]) )
                    device_index = i
                    return device_index

        if device_index == None:
            print( "No preferred input found; using default input device." )

        return device_index

    def open_mic_stream( self ):
        device_index = self.find_input_device()

        stream = self.pa.open(   format = FORMAT,
                                 channels = CHANNELS,
                                 rate = RATE,
                                 input = True,
                                 input_device_index = device_index,
                                 frames_per_buffer = INPUT_FRAMES_PER_BLOCK)

        return stream

    def updateList(self, value):
        if (len(self.recentDB) == self.thresholdSize):
            del self.recentDB[-1]
            self.recentDB.insert(0,value)
        # TODO:     ELIF PROF CORRECTS THEMSELF, CLEAR ARRAY !
        else:
            self.recentDB.insert(0,value)

    def status(self):
        if (statistics.mean(CR.recentDB) > self.threshold):
            print("TOO QUIET!") #Update LEDs here
            GPIO.output(19,True)
            GPIO.output(22,False)

        else:
            print("good boy")
            GPIO.output(22,True)
            GPIO.output (19,False)

    def listen(self):
        try:
            block = self.stream.read(INPUT_FRAMES_PER_BLOCK)
        except IOError as e:
            self.errorcount += 1
            print( "(%d) Error recording: %s"%(self.errorcount,e) )
            return

        amplitude = get_rms( block )

        dB = abs(20 * math.log(amplitude, 10))

        self.updateList(dB)

def run(threshold):
    CR = ClarityRating(threshold)

    while True:
        CR.listen()
        CR.status()
        time.sleep(0.05)      # 20 Hz????

    #print(CR.recentDB)
    #print(statistics.mean(CR.recentDB))


from functools import cmp_to_key
from datetime import timedelta

def compare(a, b):
    if a['total_score'] > b['total_score']:
        return -1
    elif a['total_score'] < b['total_score']:
        return 1
    else:
        if a['total_seconds'] > b['total_seconds']:
            return 1
        elif a['total_seconds'] < b['total_seconds']:
            return -1
        else:
            return 0

def add_rank_prop(data):
    index, item = data
    item['rank'] = index + 1
    return item

def get_contest_leaderboard(contest):
    leaderboard = []

    for user in contest.participants.all():
        # Only CONTEST front-end submissions
        subs = user.submission_frontendsubmissions.filter(
            frontendcontestsubmission__contest=contest,
            frontendcontestsubmission__is_final=True
        )

        total_score, total_seconds, sub_count = 0, 0, 0
        for sub in subs:
            sub_count += 1
            total_score += sub.judge_score
            total_seconds += sub.frontendcontestsubmission.contest_start_timedelta().total_seconds()

        if sub_count: total_seconds /= sub_count

        total_time = str(timedelta(seconds=round(total_seconds)))

        final_subs = []
        for problem in contest.problems.all():
            appended = False
            for sub in subs:
                if sub.problem == problem:
                    final_subs.append(sub)
                    appended = True
                    break
            if not appended:
                final_subs.append(None)

        leaderboard.append({
            'user': user,
            'final_subs': final_subs,
            'total_score': total_score,
            'total_seconds': total_seconds,
            'total_time': total_time,
        })

    leaderboard.sort(key=cmp_to_key(compare))
    # Add rank property (To fix paginated leaderboard)
    leaderboard = list(map(add_rank_prop, enumerate(leaderboard)))
    return leaderboard


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('rssgenerator', '0004_auto_20170827_1703'),
    ]

    operations = [
        migrations.AddField(
            model_name='links',
            name='fromUploadedFile',
            field=models.BooleanField(default=False, editable=False),
        ),
        migrations.AlterField(
            model_name='links',
            name='link',
            field=models.URLField(max_length=1024, null=True, blank=True),
        ),
    ]


from matplotlib import offsetbox

def plot_annotationbox(ax, X, X0, h, w):
    shown_images = np.array([[1., 1.]])  # just something big
    for i in range(X.shape[0]):
        dist = np.sum((X[i] - shown_images) ** 2, 1)
        if np.min(dist) < 3e-4:
            # don't show points that are too close
            continue
        shown_images = np.r_[shown_images, [X[i]]]
        imagebox = offsetbox.AnnotationBbox(
            offsetbox.OffsetImage(X0[i].reshape(3,h,w).transpose(1,2,0),
                                    zoom=5),
            X[i], frameon=False)
        ax.add_artist(imagebox)

def plot_embedding(sample, embedding, ax=None):
    X0, X = sample, embedding
#     x_min, x_max = np.min(X, 0), np.max(X, 0)
#     X = (X - x_min) / (x_max - x_min)

    if ax == None:
        plt.figure(figsize=(5,4))
        ax = plt.subplot(111)
        plot_annotationbox(ax, X, X0, h=3, w=3)
#         plt.xticks([]), plt.yticks([])
    else:
        plot_annotationbox(ax, X, X0, h=3, w=3)

import pickle

a=[]
b=[]
count=1
for i in range(10):
    a = a + [torch.tensor(np.load('random_trigger'+str(i)+'.npy')).view(-1).repeat(1, 1).float().numpy()]
    b = b + [(count,-0.05)]
    count = count + 1

trigger_target = [51, 90, 96, 79, 8, 50, 19, 7, 7, 91]

result_rounds = []
total = 10
for round in range(total):
    with open('results_rand_round_'+ str(round) +'.pickle', 'rb') as file:
        result_rounds = result_rounds + [pickle.load(file)]

with open('results_0.9_rand.pickle', 'rb') as file:
    results_90 = pickle.load(file)
with open('results_0.8_rand.pickle', 'rb') as file:
    results_80 = pickle.load(file)
with open('results_0.5_rand.pickle', 'rb') as file:
    results_50 = pickle.load(file)
dis_90 = []
dis_80 = []
dis_50 = []
baseline0=[]
baseline1=[]
baseline2=[]
baseline3=[]
baseline4=[]
baseline5=[]
baseline6=[]
baseline7=[]
baseline8=[]
baseline9=[]

for i in range(10):
    dis_90 = dis_90 + [results_90[i][0]['Test ASR'][-1]]
    dis_80 = dis_80 + [results_80[i][0]['Test ASR'][-1]]
    dis_50 = dis_50 + [results_50[i][0]['Test ASR'][-1]]
    baseline0 += [result_rounds[0][i][0]['Test ASR'][-1]]
    baseline1 += [result_rounds[1][i][0]['Test ASR'][-1]]
    baseline2 += [result_rounds[2][i][0]['Test ASR'][-1]]
    baseline3 += [result_rounds[3][i][0]['Test ASR'][-1]]
    baseline4 += [result_rounds[4][i][0]['Test ASR'][-1]]
    baseline5 += [result_rounds[5][i][0]['Test ASR'][-1]]
    baseline6 += [result_rounds[6][i][0]['Test ASR'][-1]]
    baseline7 += [result_rounds[7][i][0]['Test ASR'][-1]]
    baseline8 += [result_rounds[8][i][0]['Test ASR'][-1]]
    baseline9 += [result_rounds[9][i][0]['Test ASR'][-1]]
baseline = np.array([baseline0, baseline1, baseline2, baseline3, baseline4, baseline5, baseline6, baseline7, baseline8, baseline9])
ave = np.mean(baseline,axis=0)
std = np.std(baseline,axis=0)
baseline_ranking_index = np.argsort(ave).astype(int)
a = np.array(a)[baseline_ranking_index]
plot_embedding(a,np.array(b))

plt.plot(range(1,11), np.array(dis_90)[baseline_ranking_index], label='distribution 0.9', marker='o', c='red', alpha=.5)
plt.plot(range(1,11), np.array(dis_80)[baseline_ranking_index], label='distribution 0.8', marker='o', c='green', alpha=.5)
plt.plot(range(1,11), np.array(dis_50)[baseline_ranking_index], label='distribution 0.5', marker='o', c='navy', alpha=.5)
plt.plot(range(1,11), ave[baseline_ranking_index], label = 'baseline', marker='o',c='gray', alpha=.5)

plt.plot(range(1,11), ave[baseline_ranking_index]+std[baseline_ranking_index], ls='--', c='gray')
plt.plot(range(1,11), ave[baseline_ranking_index]-std[baseline_ranking_index], ls='--', c='gray')
# for i in range(10):
#     plt.scatter(5,baseline4[i])
plt.fill_between(range(1,11), ave[baseline_ranking_index]-std[baseline_ranking_index], ave[baseline_ranking_index]+std[baseline_ranking_index], color = 'gray', alpha = .1)
# plt.plot(range(1,11), round1, label='one-point 2', marker = 'o', c='black', alpha=.5)
plt.legend(fontsize=14, frameon=False)
plt.xticks(range(1,11),fontsize=14)
plt.yticks([-0.1,0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9],fontsize=14)
plt.xlabel('10 random patterns on Cifar100' , fontsize=14)
plt.ylabel('ASR after defense', fontsize=14)

plot_embedding(a,np.array(b)+0.06-0.0099)
plt.plot(range(1,11), np.array(dis_90)[baseline_ranking_index], label='distribution 0.9', marker='o', c='red', alpha=.5)
plt.plot(range(1,11), np.array(dis_80)[baseline_ranking_index], label='distribution 0.8', marker='o', c='green', alpha=.5)
plt.plot(range(1,11), np.array(dis_50)[baseline_ranking_index], label='distribution 0.5', marker='o', c='navy', alpha=.5)
plt.legend(fontsize=14, frameon=False)
plt.xticks(range(1,11),fontsize=14)
plt.yticks(fontsize=14)
plt.xlabel('10 random patterns on Cifar100' , fontsize=14)
plt.ylabel('ASR after defense', fontsize=14)


import os
import logging
import sqlite3
import ConfigLogging
from flask import g
from ti_files import ti_files
from tinames import tinames
from TipiConfig import TipiConfig

#
# DOA-ish methods for cache of tipi_disk file meta-data.
#

logger = logging.getLogger(__name__)
tipi_disk = '/home/tipi/tipi_disk'

global_conn = None
db_name = '/home/tipi/.tipiweb.db'

tipi_config = TipiConfig.instance()

def get_context_conn():
    """
    Flask needs a database connection per request context, and that
    is registered to close in route.py
    """
    db = getattr(g, '_database', None)
    if db is None:
        db = g._database = sqlite3.connect(db_name)
    return db

def get_mon_conn():
    """
    If not under the flask process, such as TipiMonitor.py, we can
    act in a single-threaded model, and use a global connection
    """
    global global_conn
    if not global_conn:
        global_conn = sqlite3.connect(db_name)
    return global_conn

def get_conn():
    try:
        return get_context_conn()
    except RuntimeError:
        return get_mon_conn()

def setupSchema():
    logger.info("Checking for schema")
    sql = get_conn().cursor()
    sql.execute('CREATE TABLE IF NOT EXISTS fileheader (name TEXT PRIMARY KEY, icon TEXT, type TEXT, tiname TEXT, size INTEGER, protected INTEGER)')
    get_conn().commit()
    sql.close()

def deleteAll():
    logger.info("clearing tipi_disk meta-data")
    setupSchema()
    sql = get_conn().cursor()
    sql.execute('DELETE FROM fileheader')
    get_conn().commit()
    sql.execute('VACUUM')
    get_conn().commit()
    sql.close()
    logger.debug("previous meta-data deleted.")

def addAll():
    for root, subdirs, files in os.walk(tipi_disk):
        for filename in files:
            name = os.path.join(root, filename)
            updateFileInfo(name)

def deleteMissing():
    cachedFiles = []
    logger.debug("finding all cached files")
    sql = get_conn().cursor()
    for row in sql.execute('SELECT name FROM fileheader'):
        cachedFiles.append(row[0])

    get_conn().commit()
    sql.close()

    for name in cachedFiles:
        if not os.path.exists(os.path.join(tipi_disk, name)):
            deleteFileInfo(name)

def deleteFileInfo(name):
    logger.info("Deleting cache for %s", name)
    sql = get_conn().cursor()
    try:
        sql = get_conn().cursor()
        sqlargs = (name,)
        sql.execute('DELETE FROM fileheader WHERE name == ?', sqlargs)
        get_conn().commit()
    except Exception as e:
        logger.error("failed to delete %s", name)
    finally:
        sql.close()

def lookupFileInfo(name):
    sql = get_conn().cursor()
    sqlargs = (name,)
    sql.execute('SELECT * FROM fileheader WHERE name == ?', sqlargs)
    fileInfo = sql.fetchone()
    sql.close()
    if fileInfo == None:
        fileInfo = updateFileInfo(name)
    if fileInfo[1] == "native":
        tmpFileInfo = list(fileInfo)
        tmpFileInfo[2] = "DIS/VAR 80" if isInNativeTextDir(name) else "DIS/FIX 128"
        fileInfo = tuple(tmpFileInfo)
    return rowToMap(fileInfo)

def rowToMap(fileInfo):
    # fileInfo is currently a positional 'tuple' which sucks... so let's make a
    # map
    return { "name": fileInfo[0],
             "icon": fileInfo[1],
             "type": fileInfo[2],
             "tiname": fileInfo[3],
             "size": fileInfo[4],
             "protected": fileInfo[5]
    }

def updateFileInfo(name):
    if os.path.isdir(name):
        return
    sqlargs = _getFileInfo(name)
    sql = get_conn().cursor()
    try:
        sql.execute('REPLACE INTO fileheader VALUES (?, ?, ?, ?, ?, ?)', sqlargs)
        get_conn().commit()
    except Exception as e:
        logger.error("could not update info for %s", name)
    finally:
        sql.close()
    return sqlargs

def searchFileInfo(criteria):
    sql = get_conn().cursor()
    sqlargs = {
        'g': f"*{criteria['globpat']}*"
    }
    sqlstatement = 'SELECT * FROM fileheader WHERE (tiname GLOB :g'

    if criteria['matchpaths']:
        sqlargs['m'] = f"*{criteria['globpat'].replace('.', '/')}*"
        sqlstatement += ' OR name GLOB :m'
    sqlstatement += ')'

    types = []
    if criteria['type_program']:
        types.append('"PROGRAM"')
    if criteria['type_dv80']:
        types.append('"DIS/VAR 80"')
    if criteria['type_df80']:
        types.append('"DIS/FIX 80"')
    if criteria['type_df128']:
        types.append('"DIS/FIX 128"')
    if types:
        sqlstatement += f" AND type IN ({','.join(types)})"

    logger.info("sql: %s", sqlstatement)

    sql.execute(sqlstatement, sqlargs)
    allrows = sql.fetchall()
    files = []
    for row in allrows:
        files.append(rowToMap(row))
    return sorted(files, key=lambda i:("/".join(i["name"].split("/")[:-1]), i["tiname"]))

def _getFileInfo(name):
    dv80suffixes = (".txt", ".a99", ".b99", ".bas", ".xb", ".tb")
    basicSuffixes = (".b99", ".bas", ".xb", ".tb")

    header = None

    with open(name,"rb") as fdata:
        header = bytearray(fdata.read())[:128]

    valid = ti_files.isValid(header)

    isprotected = 0
    icon = "native"
    type = "DIS/FIX 128"
    tiname = tinames.asTiShortName(name)
    size = os.stat(name).st_size

    if valid:
        type = ti_files.flagsToString(header)
        if type != 'PROGRAM':
            type = type + " " + str(ti_files.recordLength(header))
        isprotected = ti_files.isProtected(header)
        icon = 'tifile'
    elif name.lower().endswith(dv80suffixes):
        type = "DIS/VAR 80"
        if name.lower().endswith(basicSuffixes):
            icon = 'basic'

    if type == 'PROGRAM' and ti_files.isTiBasicPrg(name):
        icon = 'basic'
    if type == 'INT/VAR 254' and ti_files.isTiBasicPrg(name):
        icon = 'basic'

    return (name, icon, type, tiname, size, isprotected)

def isInNativeTextDir(target_path):
    if not os.path.isfile(target_path):
        target_path += '/'
    # check if any of text_dirs is a prefix of target_path
    native_text_dirs = [f"TIPI.{a.strip()}" for a in tipi_config.get("NATIVE_TEXT_DIRS").split(',') if a]
    if native_text_dirs and len(native_text_dirs):
        text_dirs = [tinames.devnameToLocal(dir) for dir in native_text_dirs]
        if True in [(f"{td}/" in target_path) for td in text_dirs]:
            return True
    return False

if __name__ == '__main__':
    deleteAll()
    addAll()


#!/bin/env python
# coding:utf-8

import urllib
from bottle import get,post,request
import re
from gensim.models import word2vec
model=word2vec.Word2Vec.load("/home/ubuntu/data/wikipedia/model_py.data")

import logging
logging.basicConfig(filename="/var/log/bottle/debug.log",level=logging.DEBUG)

def get():
  return '''

Wikipedia記事本文から作成したWord2Vec

        <form action="w2v" method="post">
            positive text: <input name="posi_text" type="text" />

            negative text: <input name="nega_text" type="text" />

            <input value="search" type="submit" />
        </form>
    '''

def post(request):
  posi_text=request.forms.get("posi_text")
  nega_text=request.forms.get("nega_text")
  posi_text=urllib.unquote(posi_text)
  nega_text=urllib.unquote(nega_text)
  #posi_text=unicode(posi_text,"utf-8")
  #nega_text=unicode(nega_text,"utf-8")
  logging.debug(type(posi_text))
  logging.debug(posi_text)
  #posi_text=urllib2.unquote(posi_text).encode('raw_unicode_escape').decode('utf-8')
  #return posi_text
  #nega_text=urllib2.unquote(nega_text).encode('raw_unicode_escape').decode('utf-8')
  posi=[]
  nega=[]
  for t in posi_text.split(" "):
    print(t)
    posi.append(unicode(t,"utf-8"))
  for t in nega_text.split(" "):
    nega.append(unicode(t,"utf-8"))

  logging.debug(posi)
  logging.debug(nega)
  logging.debug(len(nega))
  logging.debug(len(nega_text))

  if len(posi_text)==0:
    return "invalid positive text"

  err_str=""
  out=None
  try:
    if len(nega_text)>0:
      out=model.most_similar(positive=posi,negative=nega)
    else:
      out=model.most_similar(positive=posi)
    logging.debug(out)
  except KeyError:
    err_str="Keyword Error"

  ret=get()
  ret=ret+"

Wikipedia keyword"
  ret=ret+"

"
  ret=ret+"positive text:"+posi_text+"
"
  ret=ret+"negative text:"+nega_text+"
"

  if len(err_str)>0:
    ret=ret+""+err_str+"

"
  else:
    ret=ret+"

word	score
"
    for x in out:
      ret=ret+"
"
      ret=ret+""+str(x[0].encode('utf-8'))+"	"
      ret=ret+""+str(x[1])+"	"
      ret=ret+"
"
    ret=ret+"
"
  return ret


from collections import defaultdict

RUN_TEST = False
TEST_SOLUTION = 8
TEST_INPUT_FILE = "test_input_day_08.txt"
INPUT_FILE = "input_day_08.txt"

ARGS = []

def main_part2(
    input_file,
):
    with open(input_file) as file:
        lines = list(map(lambda line: line.rstrip(), file.readlines()))

    # A tree's scenic score is found by multiplying together its viewing
    # distance in each of the four directions.

    # Consider each tree on your map. What is the highest scenic score possible
    # for any tree?

    columns = defaultdict(list)
    rows = defaultdict(list)

    for i, line in enumerate(lines):
        for j, char in enumerate(line):
            height = int(char)
            columns[j].append(height)
            rows[i].append(height)

    def check_direction(trees: list, height: int) -> int:
        """Returns the number of trees lower than height, until
        (and including) a tree of `height` is encountered

        Args:
            trees (list): List of tree heights
            height (int): Height of the tree to check

        Returns:
            int: Number of trees lower than height
        """
        for i, h in enumerate(trees):
            if h >= height:
                return i + 1
        return len(trees)

    max_scenic_score = 0
    visible_trees_map = []
    best_spot = (0, 0)
    cutoffs = (0, 0, 0, 0)
    for i, line in enumerate(lines):
        for j, char in enumerate(line):
            height = int(char)
            if i == 0 or j == 0 or i == len(lines) - 1 or j == len(line) - 1:
                visible_trees_map.append((i, j))
                continue
            # Just a fancy map. Not needed, but I like it =)
            if (
                all(height > h for h in columns[j][0:i])
                or all(height > h for h in rows[i][0:j])
                or all(height > h for h in columns[j][i + 1 :])
                or all(height > h for h in rows[i][j + 1 :])
            ):
                visible_trees_map.append((i, j))
            up = check_direction(columns[j][0:i][::-1], height)
            down = check_direction(columns[j][i + 1 :], height)
            left = check_direction(rows[i][0:j][::-1], height)
            right = check_direction(rows[i][j + 1 :], height)
            scenic_score = up * down * left * right
            if scenic_score > max_scenic_score:
                max_scenic_score = scenic_score
                best_spot = (i, j)
                cutoffs = (up, down, left, right)
    print(f"Best spot: {best_spot} with a score of {max_scenic_score}")
    print(f"Cutoffs(U, D, L, R): {cutoffs}")

    for i, line in enumerate(lines):
        for j, char in enumerate(line):
            if (i, j) == best_spot:
                print("X", end="")
            elif i == best_spot[0] and (
                best_spot[1] - cutoffs[2] + 1 <= j <= best_spot[1] + cutoffs[3] - 1
            ):
                print("-", end="")
            elif j == best_spot[1] and (
                best_spot[0] - cutoffs[0] + 1 <= i <= best_spot[0] + cutoffs[1] - 1
            ):
                print("|", end="")
            elif (i, j) in visible_trees_map:
                print(char, end="")
            else:
                print(" ", end="")
        print()

    solution = max_scenic_score
    return solution

if __name__ == "__main__":
    if RUN_TEST:
        solution = main_part2(TEST_INPUT_FILE, *ARGS)
        print(solution)
        assert TEST_SOLUTION == solution
    else:
        solution = main_part2(INPUT_FILE, *ARGS)
        print(solution)


import os
import cv2
import numpy as np
from skimage.metrics import structural_similarity as compare_ssim
import sys

def get_image_similarity(img1, img2):
    """Calculates and returns the Structural Similarity Index (SSIM) between two images."""
    # Resize images to a specified size (e.g., 256x256) for comparison
    img1 = cv2.resize(img1, (256, 256))
    img2 = cv2.resize(img2, (256, 256))
    img1_gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
    img2_gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
    score = compare_ssim(img1_gray, img2_gray)
    return score

def find_and_remove_duplicates(folder1, folder2, similarity_threshold=0.98):
    """Scans two folders for images and removes 1:1 duplicates based on SSIM similarity threshold."""
    # Get list of files in folders
    files1 = os.listdir(folder1)
    files2 = os.listdir(folder2)

    for file1 in files1:
        img1_path = os.path.join(folder1, file1)
        if not os.path.isfile(img1_path) or not any(img1_path.endswith(ext) for ext in ('.jpg', '.jpeg', '.png', '.bmp', '.gif')):
            continue
        img1 = cv2.imread(img1_path)

        for file2 in files2:
            img2_path = os.path.join(folder2, file2)
            if not os.path.isfile(img2_path) or not any(img2_path.endswith(ext) for ext in ('.jpg', '.jpeg', '.png', '.bmp', '.gif')):
                continue
            img2 = cv2.imread(img2_path)

            similarity = get_image_similarity(img1, img2)

            if similarity >= similarity_threshold:
                print(f'Removing duplicate: {file1} and {file2}')
                os.remove(img2_path)

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print('Usage: python script.py folder1 folder2')
        sys.exit(1)

    folder1 = sys.argv[1]
    folder2 = sys.argv[2]
    similarity_threshold = 0.98

    find_and_remove_duplicates(folder1, folder2, similarity_threshold)


from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField, SubmitField, SelectField
from wtforms.validators import DataRequired, Email, InputRequired
from wtforms_sqlalchemy.fields import QuerySelectField

# https://www.youtube.com/watch?v=b9W2ul2VRRc

# https://youtu.be/u0oDDZrDz9U

# deploy
# https://youtu.be/goToXTC96Co

from flask import Flask, render_template, redirect, url_for
from flask_sqlalchemy import SQLAlchemy
import os
# from form import SaskaitaForm, VartotojasForm

basedir = os.path.abspath(os.path.dirname(__file__))
print(basedir)

app = Flask(__name__)

app.config['SECRET_KEY'] = 'dfgsfdgsdfgsdfgsdf'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'saskaitos.db')
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

class Saskaita(db.Model):
    __tablename__ = 'saskaitos'
    id = db.Column(db.Integer, primary_key=True)
    numeris = db.Column(db.String(80), nullable=False)
    balansas = db.Column(db.Float, nullable=False)
    vartotojas_id = db.Column(db.Integer, db.ForeignKey('vartotojai.id'))
    vartotojas = db.relationship("Vartotojas")

class Vartotojas(db.Model):
    __tablename__ = 'vartotojai'
    id = db.Column(db.Integer, primary_key=True)
    vardas = db.Column(db.String(80), nullable=False)
    pavarde = db.Column(db.String(80), nullable=False)
    saskaitaos = db.relationship("Saskaita")

@app.route("/")
def invoices():
    try:
        visos_saskaitos = Saskaita.query.all()
    except:
        visos_saskaitos = []
    return render_template("saskaitos.html", visos_saskaitos=visos_saskaitos)

@app.route("/users")
def users():
    try:
        visi_vartotojai = Vartotojas.query.all()
    except:
        visi_vartotojai = []
    return render_template("vartotojai.html", visi_vartotojai=visi_vartotojai)

@app.route("/add_invoice", methods=["GET", "POST"])
def new_invoice():
    db.create_all()
    forma = SaskaitaForm()
    if forma.validate_on_submit():
        nauja_saskaita = Saskaita(numeris=forma.numeris.data, balansas=forma.balansas.data, vartotojas=forma.vartotojas.data)
        db.session.add(nauja_saskaita)
        db.session.commit()
        return invoices()
    return render_template("prideti_saskaita.html", form=forma)

@app.route("/add_user", methods=["GET", "POST"])
def new_user():
    db.create_all()
    forma = VartotojasForm()
    if forma.validate_on_submit():
        naujas_vartotojas = Vartotojas(vardas=forma.vardas.data, pavarde=forma.pavarde.data)
        db.session.add(naujas_vartotojas)
        db.session.commit()
        return redirect(url_for('users'))
    return render_template("prideti_vartotoja.html", form=forma)

def new_query():
    return Vartotojas.query

def get_pk(obj):
    return str(obj)

class SaskaitaForm(FlaskForm):
    numeris = StringField('Numeris', [DataRequired()])
    balansas = IntegerField('Balansas')
    vartotojas = QuerySelectField(query_factory=new_query, allow_blank=True, get_label="vardas", get_pk=get_pk)
    # vartotojas = SelectField('Vartotojas', choices=Vartotojas.query.all())
    # vartotojas = SelectField('Vartotojas', choices=[(1,"Group1"),(2,"Group2")], validators=[InputRequired])
    submit = SubmitField('Įvesti')

class VartotojasForm(FlaskForm):
    vardas = StringField('Numeris', [DataRequired()])
    pavarde = StringField('Pavardė', [DataRequired()])
    submit = SubmitField('Įvesti')

if __name__ == '__main__':
    app.run(host='127.0.0.1', port=8000, debug=True)
    db.create_all()


import os
import copy
import numpy  as np
import tables as tb
import pandas as pd
from pytest        import mark
from pytest        import fixture
from numpy.testing import assert_raises

from invisible_cities.io  .dst_io          import load_dst
from invisible_cities.core.testing_utils   import assert_dataframes_close
from invisible_cities.core.configure       import configure
from invisible_cities.reco.corrections     import read_maps
from invisible_cities.reco.corrections     import ASectorMap
from invisible_cities.reco.corrections     import maps_coefficient_getter

from . map_builder_functions import map_builder
from . checking_functions    import AbortingMapCreation

from hypothesis            import settings
from hypothesis            import given
from hypothesis.strategies import floats
from hypothesis.strategies import integers
from hypothesis.strategies import composite
from hypothesis.strategies import lists

import logging
import warnings
warnings.filterwarnings("ignore")
logging.disable(logging.DEBUG)
this_script_logger = logging.getLogger(__name__)
this_script_logger.setLevel(logging.INFO)

@fixture(scope="module")
def t_evol_table(MAPSDIR):
    return os.path.join(MAPSDIR, 'time_evol_table.h5')

@mark.timeout(None)
@mark.dependency()
def test_scrip_runs_and_produces_correct_outputs(folder_test_dst  ,
                                                 test_dst_file    ,
                                                 output_maps_tmdir,
                                                 test_map_file    ):
    """
    Run map creation script and check if an ASectormap is the output.
    """
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo.h5')
    default_n_bins = 15
    run_number     = 7517
    config = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_params_new = copy.copy(config.as_namespace.map_params)
    map_params_new['nmin']          = 100
    map_params_new['nStimeprofile'] = 1200
    map_params_new['z_range']       = (0, 10000)
    config.update(dict(folder         = folder_test_dst,
                       file_in        = test_dst_file  ,
                       file_out_map   = map_file_out   ,
                       file_out_hists = histo_file_out ,
                       default_n_bins = default_n_bins ,
                       run_number     = run_number     ,
                       map_params     = map_params_new ))
    map_builder(config.as_namespace)
    maps = read_maps(map_file_out)
    assert type(maps)==ASectorMap

    old_maps = read_maps(test_map_file)
    assert_dataframes_close(maps.e0 , old_maps.e0 , rtol=1e-5)
    assert_dataframes_close(maps.e0u, old_maps.e0u, rtol=1e-1)
    assert_dataframes_close(maps.lt , old_maps.lt , rtol=1e-5)
    assert_dataframes_close(maps.ltu, old_maps.ltu, rtol=1e-1)

@mark.dependency(depends="test_scrip_runs_and_produces_correct_outputs")
def test_time_evol_table_correct_elements(output_maps_tmdir):
    map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    emaps        = read_maps(map_file_out)
    time_table   = emaps.t_evol
    columns      = time_table.columns
    elements     =['ts'   ,
                   'e0'   , 'e0u'   ,
                   'lt'   , 'ltu'   ,
                   'dv'   , 'dvu'   ,
                   'resol', 'resolu',
                   's1w'  , 's1wu'  ,
                   's1h'  , 's1hu'  ,
                   's1e'  , 's1eu'  ,
                   's2w'  , 's2wu'  ,
                   's2h'  , 's2hu'  ,
                   's2e'  , 's2eu'  ,
                   's2q'  , 's2qu'  ,
                   'Nsipm', 'Nsipmu',
                   'Xrms' , 'Xrmsu' ,
                   'Yrms' , 'Yrmsu' ,
                   'S1eff' , 'S2eff', 'Bandeff']
    for element in elements:
        assert element in columns

@mark.dependency(depends="test_scrip_runs_and_produces_correct_outputs")
def test_time_evol_eff_less_one(output_maps_tmdir):
    map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    emaps        = read_maps(map_file_out)
    assert np.all(emaps.t_evol.S1eff   <= 1.)
    assert np.all(emaps.t_evol.S2eff   <= 1.)
    assert np.all(emaps.t_evol.Bandeff <= 1.)

@mark.dependency(depends="test_scrip_runs_and_produces_correct_outputs")
def test_time_evol_table_exact_numbers(t_evol_table, output_maps_tmdir):
    map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    emaps        = read_maps(map_file_out)
    t_evol = pd.pandas.read_hdf(t_evol_table, 't_evol')
    assert_dataframes_close(emaps.t_evol, t_evol, rtol=1e-5)

@composite
def xy_pos(draw, elements=floats(min_value=-200, max_value=200)):
    size = draw(integers(min_value=1, max_value=10))
    x    = draw(lists(elements,min_size=size, max_size=size))
    y    = draw(lists(elements,min_size=size, max_size=size))
    return (np.array(x),np.array(y))

@mark.dependency(depends="test_scrip_runs_and_produces_correct_outputs")
@given(xy_pos = xy_pos())
@settings(max_examples=50)
def test_maps_nans_outside_rmax(xy_pos, output_maps_tmdir):
    map_file_out = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    xs, ys = xy_pos

    emaps = read_maps(map_file_out)
    get_coef = maps_coefficient_getter(emaps.mapinfo, emaps.e0)
    r_max    = emaps.mapinfo.xmax
    nbins    = emaps.mapinfo.nx
    bins     = np.linspace(-200, 200, nbins)
    xs       = bins[np.digitize(xs, bins, right = True)]
    ys       = bins[np.digitize(ys, bins, right = True)]
    coefs    = get_coef(xs, ys)
    maskin   = np.sqrt(xs**2 + ys**2) <  r_max
    maskout  = np.sqrt(xs**2 + ys**2) >= r_max + 2 * r_max / nbins

    assert all(np.isnan   (coefs[maskout]))
    assert all(np.isfinite(coefs[maskin ]))

@mark.dependency(depends="test_scrip_runs_and_produces_correct_outputs")
def test_correct_map_with_unsorted_dst(folder_test_dst  ,
                                       test_dst_file    ,
                                       output_maps_tmdir):
    """
    This test shuffles the input dst, and checks that the map is the same
    as the one created with the same sorted dst.
    """
    map_file_sort   = os.path.join(output_maps_tmdir, 'test_out_map.h5')
    map_file_unsort = os.path.join(output_maps_tmdir, 'test_out_unsort.h5')
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo.h5')

    dst = load_dst(folder_test_dst+test_dst_file, 'DST', 'Events')
    if "index" in dst:del dst["index"]
    dst = dst.sort_values(by=['S2e'])
    tmp_unsorted_dst = 'unsorted_dst.h5'
    dst.to_hdf(output_maps_tmdir+tmp_unsorted_dst,
               key     = "DST"  , mode         = "w",
               format  = "table", data_columns = True,
               complib = "zlib" , complevel    = 4)
    with tb.open_file(output_maps_tmdir+tmp_unsorted_dst, "r+") as file:
        file.rename_node(file.root.DST.table, "Events")
        file.root.DST.Events.title = "Events"

    default_n_bins = 15
    run_number     = 7517
    config = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_params_new = config.as_namespace.map_params
    map_params_new['nmin']    = 100
    map_params_new['z_range'] = (0, 10000)
    config.update(dict(folder         = output_maps_tmdir,
                       file_in        = tmp_unsorted_dst ,
                       file_out_map   = map_file_unsort  ,
                       file_out_hists = histo_file_out   ,
                       default_n_bins = default_n_bins   ,
                       run_number     = run_number       ,
                       map_params     = map_params_new   ))
    map_builder(config.as_namespace)
    unsorted_maps = read_maps(map_file_unsort)
    sorted_maps   = read_maps(map_file_sort)

    assert_dataframes_close(unsorted_maps.e0 , sorted_maps.e0 , rtol=1e-5)
    assert_dataframes_close(unsorted_maps.e0u, sorted_maps.e0u, rtol=1e-5)
    assert_dataframes_close(unsorted_maps.lt , sorted_maps.lt , rtol=1e-5)
    assert_dataframes_close(unsorted_maps.ltu, sorted_maps.ltu, rtol=1e-5)

def test_exception_s1(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks if exception raises when ns1=1 efficiency is out of range.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_s1.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_s1.h5')
    min_eff_test = 0.
    max_eff_test = 0.8
    run_number   = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     nS1_eff_min    = min_eff_test   ,
                     nS1_eff_max    = max_eff_test   ,
                     run_number     = run_number     ))

    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )

def test_exception_s2(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks if exception raises when nS2=1 efficiency is out of range.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_s2.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_s2.h5')
    min_eff_test = 0.
    max_eff_test = 0.9
    run_number   = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     nS2_eff_min    = min_eff_test   ,
                     nS2_eff_max    = max_eff_test   ,
                     run_number     = run_number     ))

    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )

def test_exception_rate(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks if exception raises when rate distribution is not flat enough.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_rate.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_rate.h5')
    n_dev_rate = 0.5
    run_number   = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     n_dev_rate     = n_dev_rate     ,
                     run_number     = run_number     ))

    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )

def test_exception_Zdst(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks if exception raises when Z distribution is not
    similar enough to the reference one.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_Z.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_Z.h5')
    nsigmas_Zdst = 0.5
    run_number   = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     nsigmas_Zdst   = nsigmas_Zdst   ,
                     run_number     = run_number     ))

    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )

def test_exception_bandsel(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks if exception raises when band selection efficiency is
    out of a given range.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_bandsel.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_bandsel.h5')
    band_sel_params_new = copy.copy(conf.as_namespace.band_sel_params)
    band_sel_params_new['eff_min'] = 0.
    band_sel_params_new['eff_max'] = 0.89
    run_number = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     band_sel_params = band_sel_params_new,
                     run_number     = run_number     ))
    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )

def test_exception_drift_v(folder_test_dst, test_dst_file, output_maps_tmdir):
    """
    This test checks that exception raises if drift_v fit
    fails in too many temporal bins.
    """
    conf = configure('maps $ICARO/krcal/map_builder/config_LBphys.conf'.split())
    map_file_out   = os.path.join(output_maps_tmdir, 'test_out_map_driftv.h5'  )
    histo_file_out = os.path.join(output_maps_tmdir, 'test_out_histo_driftv.h5')
    map_params_new = copy.copy(conf.as_namespace.map_params)
    map_params_new['dv_maxFailed'] = 0.01
    map_params_new['nStimeprofile'] = 100
    run_number = 7517
    conf.update(dict(folder         = folder_test_dst,
                     file_in        = test_dst_file  ,
                     file_out_map   = map_file_out   ,
                     file_out_hists = histo_file_out ,
                     map_params     = map_params_new ,
                     run_number     = run_number     ))
    assert_raises(AbortingMapCreation,
                  map_builder        ,
                  conf.as_namespace  )


# YOLOv5 ЁЯЪА by Ultralytics, AGPL-3.0 license
"""
Validate a trained YOLOv5 detection model on a detection dataset

Usage:
    $ python val.py --weights yolov5s.pt --data coco128.yaml --img 640

Usage - formats:
    $ python val.py --weights yolov5s.pt                 # PyTorch
                              yolov5s.torchscript        # TorchScript
                              yolov5s.onnx               # ONNX Runtime or OpenCV DNN with --dnn
                              yolov5s_openvino_model     # OpenVINO
                              yolov5s.engine             # TensorRT
                              yolov5s.mlmodel            # CoreML (macOS-only)
                              yolov5s_saved_model        # TensorFlow SavedModel
                              yolov5s.pb                 # TensorFlow GraphDef
                              yolov5s.tflite             # TensorFlow Lite
                              yolov5s_edgetpu.tflite     # TensorFlow Edge TPU
                              yolov5s_paddle_model       # PaddlePaddle
"""

import argparse
import os
import sys
from pathlib import Path

import numpy as np
import torch
from tqdm import tqdm

FILE = Path(__file__).resolve()
ROOT = str(FILE.parents[0]) + "/yolov5"  # YOLOv5 root directory
if ROOT not in sys.path:
    sys.path.append(ROOT)  # add ROOT to PATH

from models import yolov5n, yolov5s
from yolov5.utils.dataloaders import create_dataloader
from yolov5.utils.general import (
    LOGGER,
    TQDM_BAR_FORMAT,
    Profile,
    colorstr,
    non_max_suppression,
    scale_boxes,
    xywh2xyxy,
    xyxy2xywh,
)
from yolov5.utils.metrics import ap_per_class, box_iou

from sparsebit.quantization import QuantModel, parse_qconfig

def set_seed(seed):
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.enabled = False

def save_one_txt(predn, save_conf, shape, file):
    # Save one txt result
    gn = torch.tensor(shape)[[1, 0, 1, 0]]  # normalization gain whwh
    for *xyxy, conf, cls in predn.tolist():
        xywh = (
            (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()
        )  # normalized xywh
        line = (cls, *xywh, conf) if save_conf else (cls, *xywh)  # label format
        with open(file, "a") as f:
            f.write(("%g " * len(line)).rstrip() % line + "\n")

def save_one_json(predn, jdict, path, class_map):
    # Save one JSON result {"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236}
    image_id = int(path.stem) if path.stem.isnumeric() else path.stem
    box = xyxy2xywh(predn[:, :4])  # xywh
    box[:, :2] -= box[:, 2:] / 2  # xy center to top-left corner
    for p, b in zip(predn.tolist(), box.tolist()):
        jdict.append(
            {
                "image_id": image_id,
                "category_id": class_map[int(p[5])],
                "bbox": [round(x, 3) for x in b],
                "score": round(p[4], 5),
            }
        )

def process_batch(detections, labels, iouv):
    """
    Return correct prediction matrix
    Arguments:
        detections (array[N, 6]), x1, y1, x2, y2, conf, class
        labels (array[M, 5]), class, x1, y1, x2, y2
    Returns:
        correct (array[N, 10]), for 10 IoU levels
    """
    correct = np.zeros((detections.shape[0], iouv.shape[0])).astype(bool)
    iou = box_iou(labels[:, 1:], detections[:, :4])
    correct_class = labels[:, 0:1] == detections[:, 5]
    for i in range(len(iouv)):
        x = torch.where(
            (iou >= iouv[i]) & correct_class
        )  # IoU > threshold and classes match
        if x[0].shape[0]:
            matches = (
                torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1)
                .cpu()
                .numpy()
            )  # [label, detect, iou]
            if x[0].shape[0] > 1:
                matches = matches[matches[:, 2].argsort()[::-1]]
                matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
                # matches = matches[matches[:, 2].argsort()[::-1]]
                matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
            correct[matches[:, 1].astype(int), i] = True
    return torch.tensor(correct, dtype=torch.bool, device=iouv.device)

@torch.no_grad()
def main(args):
    set_seed(args.seed)
    if args.model_name == "yolov5n":
        model = yolov5n(checkpoint_path=args.checkpoint_path)
    elif args.model_name == "yolov5s":
        model = yolov5s(checkpoint_path=args.checkpoint_path)
    else:
        raise NotImplementedError

    qconfig = parse_qconfig(args.qconfig_path)
    qmodel = QuantModel(model.model, config=qconfig)

    setattr(model, "model", qmodel)
    imgsz = 640
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # Configure
    model.eval()
    model.cuda()

    cuda = device.type != "cpu"
    nc = 80  # number of classes
    iouv = torch.linspace(0.5, 0.95, 10, device=device)  # iou vector for mAP@0.5:0.95
    niou = iouv.numel()
    task = "val"  # path to val images
    dataloader = create_dataloader(
        os.path.join(args.data_path, "val2017.txt"),
        imgsz,
        1,
        32,
        False,
        pad=0.5,
        rect=True,
        workers=args.workers,
        prefix=colorstr(f"{task}: "),
    )[0]

    # Calibration
    calib_loader = create_dataloader(
        os.path.join(args.data_path, "calib2017.txt"),
        imgsz,
        1,
        32,
        False,
        pad=0.5,
        rect=True,
        workers=args.workers,
        prefix=colorstr(f"{task}: "),
    )[0]

    qmodel.prepare_calibration()
    for batch_meta in calib_loader:
        data = batch_meta[0] / 255
        with torch.no_grad():
            _ = qmodel(data.to(device, non_blocking=True))
    qmodel.calc_qparams()
    qmodel.set_quant(w_quant=True, a_quant=True)

    seen = 0
    names = (
        model.names if hasattr(model, "names") else model.module.names
    )  # get class names
    if isinstance(names, (list, tuple)):  # old format
        names = dict(enumerate(names))
    s = ("%22s" + "%11s" * 6) % (
        "Class",
        "Images",
        "Instances",
        "P",
        "R",
        "mAP50",
        "mAP50-95",
    )
    p, r, mp, mr, map50, ap50, map = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
    dt = Profile(), Profile(), Profile()  # profiling times
    stats, ap = [], []
    pbar = tqdm(dataloader, desc=s, bar_format=TQDM_BAR_FORMAT)  # progress bar
    for im, targets, paths, shapes in pbar:
        with dt[0]:
            if cuda:
                im = im.to(device, non_blocking=True)
                targets = targets.to(device)
            im = im.float()  # uint8 to fp16/32
            im /= 255  # 0 - 255 to 0.0 - 1.0
            _, _, height, width = im.shape  # batch size, channels, height, width

        # Inference
        with dt[1]:
            preds = model(im)

        # NMS
        targets[:, 2:] *= torch.tensor(
            (width, height, width, height), device=device
        )  # to pixels
        with dt[2]:
            preds = non_max_suppression(
                preds,
                args.conf_thres,
                args.iou_thres,
                labels=[],
                multi_label=True,
                agnostic=False,
                max_det=300,
            )

        # Metrics
        for si, pred in enumerate(preds):
            labels = targets[targets[:, 0] == si, 1:]
            npr = pred.shape[0]  # number of labels, predictions
            shape = shapes[si][0]
            correct = torch.zeros(npr, niou, dtype=torch.bool, device=device)  # init
            seen += 1

            if npr == 0:
                stats.append(
                    (correct, *torch.zeros((2, 0), device=device), labels[:, 0])
                )
                continue

            # Predictions
            predn = pred.clone()
            scale_boxes(
                im[si].shape[1:], predn[:, :4], shape, shapes[si][1]
            )  # native-space pred

            # Evaluate
            tbox = xywh2xyxy(labels[:, 1:5])  # target boxes
            scale_boxes(
                im[si].shape[1:], tbox, shape, shapes[si][1]
            )  # native-space labels
            labelsn = torch.cat((labels[:, 0:1], tbox), 1)  # native-space labels
            correct = process_batch(predn, labelsn, iouv)

            stats.append(
                (correct, pred[:, 4], pred[:, 5], labels[:, 0])
            )  # (correct, conf, pcls, tcls)

    # Compute metrics
    stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)]  # to numpy
    if len(stats) and stats[0].any():
        tp, fp, p, r, f1, ap, ap_class = ap_per_class(
            *stats, plot=False, save_dir="", names=names
        )
        ap50, ap = ap[:, 0], ap.mean(1)  # AP@0.5, AP@0.5:0.95
        mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean()
    nt = np.bincount(stats[3].astype(int), minlength=nc)  # number of targets per class

    # Print results
    pf = "%22s" + "%11i" * 2 + "%11.3g" * 4  # print format
    LOGGER.info(pf % ("all", seen, nt.sum(), mp, mr, map50, map))
    if nt.sum() == 0:
        LOGGER.warning(
            f"WARNING тЪая╕П no labels found in {task} set, can not compute metrics without labels"
        )

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--model_name", type=str, default="yolov5n", choices=["yolov5n", "yolov5s"]
    )
    parser.add_argument("--qconfig_path", type=str, default="./qconfig.yaml")
    parser.add_argument("--data_path", type=str, required=True)
    parser.add_argument("--checkpoint_path", type=str, required=True)
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--workers", type=int, default=8, help="dataloader workers")
    parser.add_argument(
        "--conf_thres", type=float, default=0.001, help="confidence threshold"
    )
    parser.add_argument(
        "--iou_thres", type=float, default=0.6, help="NMS IoU threshold"
    )

    args = parser.parse_args()
    return args

if __name__ == "__main__":
    args = parse_args()
    main(args)