'''
UNIVERSIDAD DEL VALLE DE GUATEMALA
Criptografía y Cifrado de la Información
Sección 10

Walter Danilo Saldaña Salguero 19897
Jose Abraham Gutierrez Corado 19111
Javier Alejandro Cotto Argueta 19324

LABORATORIO #3
'''

import numpy as np
import matplotlib.pyplot as plt
import re
from skimage import data, io   # estas librerías solo se usan para
from PIL import Image               # llamar al ejemplo de cameraman.png
import streamCipher as sc

def xor(a, b):
    m = len(a)
    n = len(b)
    maxx = max(m,n)
    if (m < n):
        a = a + (n-m)*'0'
    if (n < m):
        b = b + (m-n)*'0'

    c = ''
    for i in range(0, maxx):
        c = c + str(int(a[i]) ^ int(b[i]))
    return c

def img2bits(I):
    ''' Convierte una imagen en escala de grises a cadena de bits.
        Input:  I = imagen, como numpy array de shape (m,n).
        Output: s = string de bits, donde se concatenan cada pixel de I.
    '''
    m, n = I.shape
    s = ''
    for i in range(0, m):
        for j in range(0, n):
            s = s + '{0:08b}'.format(I[i,j])
    return s

def bits2img(x, shape):
    ''' Convierte una cadena de bits a una imagen en escala de grises.
        Input:  s = string de bits, donde se concatenan cada pixel de I.
                shape = dimensiones (m,n) de la imagen de salida I.
        Output: I = imagen, como numpy array de shape (m,n).
    '''

    m, n = shape
    I = np.zeros(m*n).astype(np.uint8)
    bts = re.findall('........', x)
    for i in range(0, len(bts)-1):
        I[i] = int(bts[i], 2)
    I = I.reshape(m,n)
    return I

I = io.imread('mono.png', as_gray = True)

J = Image.fromarray(I)
J = J.resize((J.size[0]//2, J.size[1]//2), Image.LANCZOS)
I = np.array(J) * 255
I = I.astype(float).astype(int)

plt.figure()
plt.imshow(I, cmap='gray')
plt.show()

s0 = img2bits(I)

#Stream 1
s1 = sc.LCG(3,5,254,8,int(len(s0)/8))
r1 = xor(s0, s1[0])

#Stream 2
s2 = sc.LFSR(r1, len(r1), [1, 2, 3, 4, 5, 6, 9, 11, 14, 15], 10)
r2 = xor(r1, s2[0])

#Stream 3
s3 = sc.wichmanHill(8,int(len(s0)/8))
r3 = xor(r2, s3[0])

rf = xor(s0, r3)

I1 = bits2img(r1, I.shape)
I2 = bits2img(r2, I.shape)
I3 = bits2img(r3, I.shape)
I4 = bits2img(rf, I.shape)

plt.figure(figsize=(15,8))

#Graficar primera transformación (Con algoritmo LCG)
plt.subplot(2,2,1)
plt.imshow(I1, cmap='gray')

#Graficar segunda transformación (Con algoritmo LFSR)
plt.subplot(2,2,2)
plt.imshow(I2, cmap='gray')

#Graficar tercera transformación (Con algoritmo wichmanHill)
plt.subplot(2,2,3)
plt.imshow(I3, cmap='gray')

#Graficar cuarta transformación (xor de la transformación resultante con la imagen original)
plt.subplot(2,2,4)
plt.imshow(I4, cmap='gray')

plt.show()


'''
Descripttion:
version:
Author: Catop
Date: 2021-02-10 07:47:09
LastEditTime: 2021-06-19 16:34:27
'''
#coding:utf-8

import os
import time
import random
import json
from flask import Flask,request,jsonify

from globalAPI import goapi as goapi
from globalAPI import dbconn as dbconn
from globalAPI.CB_logger import plog as CB_logger

from router import CB_router as CB_router

app = Flask(__name__)
@app.route('/', methods=['POST'])
def getEvent():
    data = request.json
    post_type = data.get('post_type')
    if(post_type == 'message'):
        message_type = data.get('message_type')
        message = data.get('message')
        message_id = data.get('message_id')
        user_id = str(data.get('user_id'))
        sender = data.get('sender')
        #sender为dict
        if(message_type=='private'):
            #处理私聊消息
            CB_logger('Flask',f'接收私聊消息@{user_id}:{message[:20]}')
            CB_router(user_id,message,'private',sender=sender,message_id=message_id)
        elif(message_type=='group'):
            #处理群聊消息
            group_id = data.get('group_id')

            CB_logger('Flask',f'接收群消息@{group_id}@{user_id}:{message[:20]}')
            CB_router(user_id,message,'group',group_id,sender=sender,message_id=message_id)

    elif(post_type == 'request'):
        request_type = data.get('request_type')
        if(request_type=='friend'):
            user_id = str(data.get('user_id'))
            comment = str(data.get('comment'))
            flag = str(data.get('flag'))
            CB_logger('Flask',f'\n[flask]接收加好友请求@{user_id}:{comment[:20]}')
            time.sleep(random.randint(5,10))
            goapi.add_request(flag)
            time.sleep(random.randint(5,10))
            goapi.sendMsg(user_id,"欢迎！\n请先注册，例如'注册@张三@信安20-2'\n 班级请严格按格式输入，否则可能统计不上哦")
    else:
        #暂不处理其他类型上报，为防止go-cq报错而设置
        pass

    return data

if __name__ == '__main__':
    conf_info = "conf/botConf.json"
    with open(conf_info,"r") as f:
        conf_dict = json.load(f)
    flask_host = conf_dict['Flask']['Host']
    flask_port = conf_dict['Flask']['Port']
    flask_debug = conf_dict['Flask']['Debug']
    if flask_debug=="False":
        flask_debug = False
    else:
        flask_debug = True

    app.run(host=flask_host,port=flask_port,debug=flask_debug)


import torch
import sys
sys.path.append('../MatSciBERT')
from normalize_text import normalize
from transformers import AutoModel, AutoTokenizer
from sklearn.metrics.pairwise import cosine_similarity
import pickle
import pandas as pd
import re
from tqdm import tqdm
import os
import numpy as np

def sort_composition(compositions):

	composition_name_new = []

	for name in compositions:
		composition = name
		composition_split = re.findall('[A-Z][^A-Z]*', composition)
		composition_split.sort()

		for i, element in enumerate(composition_split):
			if not element[-1].isnumeric():
				composition_split[i] = composition_split[i] + '1'
		composition_sorted = "".join(composition_split)
		composition_name_new.append(composition_sorted)

	return composition_name_new

def make_robodata(save_dir, robo_data = None):

	if robo_data is None:
		robo_dir = 'robo_descriptions'

		file1 = open(robo_dir, "r")
		lines = file1.readlines()
		file1.close()

		lines = [l[:-1] for l in lines]
		names = [l.split(' ')[0] for l in lines]

		robo_data = pd.DataFrame({'composition_name':names,'sentence':lines})

	mat_table = 'JMC_Data_Kappa_Unique_Averaged.csv'
	table = pd.read_csv(mat_table)
	names_new = sort_composition(robo_data['composition_name'].values)
	robo_data['composition_name'] = names_new
	composition_name_new = sort_composition(table['composition_name'].values)

	robo_data_use = robo_data[(np.in1d(names_new, composition_name_new))]

	robo_data_use.to_pickle(save_dir+'/robo_data_kappa.pkl')

	return robo_data_use

def run_matscibert(robo_data):

	save_dir = '../robo_matscibert_embedding'
	model = AutoModel.from_pretrained('m3rg-iitd/matscibert')
	tokenizer = AutoTokenizer.from_pretrained('m3rg-iitd/matscibert')
	#sentences = ['SiO2 is a network former.']
	embeddings = []
	for row in tqdm(robo_data.iterrows()):

		sentence = row[1].sentence
		composition = row[1].composition_name

		norm_sents = [normalize(s) for s in [sentence]]
		tokenized_sents = tokenizer(norm_sents)
		tokenized_sents = {k: torch.Tensor(v).long() for k, v in tokenized_sents.items()}
		token_size = int(tokenized_sents['input_ids'].shape[1])
		sentence_embedding = []
		for i in range(0,token_size,512):
			with torch.no_grad():
				last_hidden_state = model(input_ids=tokenized_sents['input_ids'][:,i:i+512],token_type_ids=tokenized_sents['token_type_ids'][:,i:i+512],
				attention_mask=tokenized_sents['attention_mask'][:,i:i+512])[0]

			sentence_embedding.append(last_hidden_state.detach().cpu().numpy()[0])
			#np.save(save_dir+'/{}.npy'.format(composition),last_hidden_state.detach().cpu().numpy()[0])

		embeddings.append(np.concatenate(sentence_embedding).mean(0).reshape(1,-1))

	robo_data['embeddings'] = embeddings
	robo_data.to_pickle('robo_descriptions_with_embed_noprop.pkl')

if __name__ == "__main__":

	save_dir = '../'
	data = pd.read_pickle('robo_descriptions.pkl')

	if not os.path.isfile(save_dir+'robo_data_kappa_with_prop.pkl'):
		robo_data = make_robodata(save_dir, data)
	else:
		robo_data = pd.read_pickle(save_dir+'robo_data_kappa_with_prop.pkl')

	run_matscibert(data)


import matplotlib
matplotlib.use('TKagg')
import matplotlib.pyplot as plt
import numpy as np
import cv2
from scipy import optimize
import os
plt.ion()

def get_disk(radius):

    r = get_radius_array(radius)
    r[r < radius] = 1
    r[r > radius] = 0
    disk = r.astype(np.bool)

    return disk

def get_radius_array(disk_center, nx, ny):

    x = np.arange(0, nx)
    y = np.arange(0, ny)
    xv, yv = np.meshgrid(x, y)
    radius_array = np.sqrt( (xv - disk_center[0])**2 + (yv - disk_center[1])**2 )

    return radius_array

def fit_bkg(bkg_avg, xmin, xmax, radius_grid, rmin):
    # Define function for calculating a power law
    powerlaw = lambda x, amp, index: amp * (x ** index)
    # define our (line) fitting function
    fitfunc = lambda p, x: p[0] + p[1] * x
    errfunc = lambda p, x, y, err: (y - fitfunc(p, x)) / err

    xdata = np.arange(xmin, xmax, 1)
    ydata = bkg_avg[xdata]
    yerr = 0.2 * ydata
    logx = np.log10(xdata)
    logy = np.log10(ydata)
    logyerr = 0.001 * np.ones(ydata.shape)

    pinit = [1.0, -1.0]
    out = optimize.leastsq(errfunc, pinit,
                           args=(logx, logy, logyerr), full_output=1)
    pfinal = out[0]
    covar = out[1]
    index = pfinal[1]
    amp = 10.0 ** pfinal[0]

    bkg_2d = powerlaw(radius_grid, amp, index)
    bkg_2d[rr <= rmin] = 0

    return bkg_2d, amp, index

files = ['/Users/rattie/Data/Eclipse/Stack_mean_aligned_series/aligned/stack_mean_C_exp250.tiff',
         '/Users/rattie/Data/Eclipse/Stack_mean_aligned_series/aligned/stack_mean_C_exp30.tiff',
         '/Users/rattie/Data/Eclipse/Stack_mean_aligned_series/aligned/stack_mean_C_exp1.tiff']

savedir = '/Users/rattie/Data/Eclipse/Background_removed/'

# Exposure times
nexp = 3
times_array = np.array([250, 30, 1])
sum_times = times_array.sum()
# Normalization factors
max16 = 2**16-1
max14 = 2**14-1
# Load a sample
img0 = cv2.imread(files[0], cv2.IMREAD_COLOR + cv2.IMREAD_ANYDEPTH)
nx = img0.shape[1]
ny = img0.shape[0]
# Load all images and convert from openCV BGR to RGB
images = np.zeros([nexp, ny, nx, 3])
for i in range(nexp):
    images[i, :, :, :] = cv2.imread(files[i], cv2.IMREAD_COLOR + cv2.IMREAD_ANYDEPTH)
    images[i, :, :, :] = np.fliplr(images[i,:,:,:].reshape(-1,3)).reshape(images[i,:,:,:].shape)

# Get a reference version that can be visualized as float, normalized from [0-1].
# Because of the flat fielding, saturation occurs above max14. So we choose to normalize to a meaningful value,
# maxval =  maximum near-saturating intensity in the prominence.
maxval = images[:, 1080:1120, 1900:1960, 0].max() # ~ 20000
imagesf = images/maxval
# Clip within 0, 1 - for visualization only. Processing is made on an unclipped version.
imagesf = np.clip(imagesf, 0, 1)
# Coordinate of the center
center = (2120, 1361)
radius = 323
# Plot one image and the circle to check the center and radius
#circle1 = plt.Circle((0, 0), 0.2, color='r')
plt.figure(0, figsize=(15,11))
ax1 = plt.gcf().add_subplot(111)
ax1.imshow(np.clip(imagesf[0,...]*5, 0, 1), origin='lower')
#ax1.imshow(np.clip(imagesf[0,...], 0, 1), origin='lower')
ax1.add_artist(plt.Circle(center, radius, color='green', fill=False, linewidth=1))
ax1.add_artist(plt.Circle(center, 330, color='yellow', fill=False))
ax1.axis([1000, 3000, 500, 2500])
plt.title('1/250s , brightness x 5')
plt.tight_layout()

# Normalize to the exposure time, so we get the same intensity per second + noise
tarray3 = np.reshape(times_array, [nexp, 1, 1, 1])
images_exp = images * tarray3 # * (2**16 -1) / (2**14 -1)
# Divide again to maxval to have a visualization equivalent to "imagesf" (consider that 1s should look the same).
images_expf = images_exp/maxval
# Clip saturated pixels in the exposure-normalized images
images_expf_clipped = np.clip(images_expf, 0, 1)
# Display the 3 images in RGB.
fig = plt.figure(1, figsize=(19,10))
plt.subplot(231)
plt.imshow(imagesf[0,...])
plt.title('1/250s')
plt.subplot(232)
plt.imshow(imagesf[1,...])
plt.title('1/30s')
plt.subplot(233)
plt.imshow(imagesf[2,...])
plt.title('1s')

plt.subplot(234)
plt.imshow(images_expf_clipped[0,...])
plt.title('x 250')
plt.subplot(235)
plt.imshow(images_expf_clipped[1,...])
plt.title('x 30')
plt.subplot(236)
plt.imshow(images_expf_clipped[2,...])
plt.title('x 1')
plt.tight_layout()
plt.savefig('/Users/rattie/Data/Eclipse/figures/imagesRGB_and_exp_normalized.png')

## Remap to polar coordinates
pimages_rgb = np.zeros([nexp, ny, nx, 3])
for exp in range(0,3):
    for ch in range(0,3):
        # Azimuthal average
        pimages_rgb[exp,: , :, ch] = cv2.linearPolar(images_exp[exp, :, :, ch], center, nx, cv2.INTER_LANCZOS4 + cv2.WARP_FILL_OUTLIERS)

pimages_rgbf = np.clip(pimages_rgb / maxval, 0, 1)

# Get a selection to take the median over the y-axis.
ymin = 990
ymax = 1100

fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(19, 8))
axes[0].imshow(np.clip(pimages_rgbf[0,...], 0, 1), origin='lower')
axes[0].set_title('1/250s')
axes[1].imshow(np.clip(pimages_rgbf[1,...], 0, 1), origin='lower')
axes[1].set_title('1/30s')
axes[2].imshow(np.clip(pimages_rgbf[2,...]*5, 0, 1), origin='lower')
axes[2].set_title('1s x 5')

axes[1].axhline(y=ymin, ls='--', linewidth=1, color='yellow')
axes[1].axhline(y=ymax, ls='--', linewidth=1, color='yellow')

plt.tight_layout()

# Extract and average background radial profile
bkg_az_avg1 = pimages_rgbf[1, ymin:ymax, :, :].mean(axis=0)
bkg_az_avg2 = pimages_rgbf[2, ymin:ymax, :, :].mean(axis=0)
rr = get_radius_array(center, nx, ny)

rmin = 1300
rmax = 1800
# Red
bkg_r, amp_r, index_r = fit_bkg(bkg_az_avg1[:,0], rmin, rmax, rr, radius)
# Green
bkg_g, amp_g, index_g = fit_bkg(bkg_az_avg1[:,1], rmin, rmax, rr, radius)
# Blue
bkg_b, amp_b, index_b = fit_bkg(bkg_az_avg1[:,2], rmin, rmax, rr, radius)

# 1s exposure background fit
rmin = 1300
rmax = 1800
# Red
bkg_r2, amp_r2, index_r2 = fit_bkg(bkg_az_avg2[:,0], rmin, rmax, rr, radius)
# Green
bkg_g2, amp_g2, index_g2 = fit_bkg(bkg_az_avg2[:,1], rmin, rmax, rr, radius)
# Blue
bkg_b2, amp_b2, index_b2 = fit_bkg(bkg_az_avg2[:,2], rmin, rmax, rr, radius)

# for plotting, get a longer array of radius values for extrapolating the profile to the center.
xdata2 = np.arange(radius, nx, 1)
powerlaw = lambda x, amp, index: amp * (x ** index)

fig = plt.figure(figsize=(15, 10))
plt.subplot(2,1,1)
plt.plot(bkg_az_avg1[:,0], 'r-')
plt.plot(bkg_az_avg1[:,1], 'g-')
plt.plot(bkg_az_avg1[:,2], 'b-')
plt.plot(xdata2, powerlaw(xdata2, amp_r, index_r), 'k--')
plt.plot(xdata2, powerlaw(xdata2, amp_g, index_g), 'k-.')
plt.plot(xdata2, powerlaw(xdata2, amp_b, index_b), 'k:')
plt.title('Background 1/30 x 30 azimuthal average over y=[{:d} ; {:d}]'.format(ymin, ymax))
plt.subplot(2,1,2)
plt.plot(bkg_az_avg2[:,0], 'r-')
plt.plot(bkg_az_avg2[:,1], 'g-')
plt.plot(bkg_az_avg2[:,2], 'b-')
plt.plot(xdata2, powerlaw(xdata2, amp_r2, index_r2), 'k--')
plt.plot(xdata2, powerlaw(xdata2, amp_g2, index_g2), 'k-.')
plt.plot(xdata2, powerlaw(xdata2, amp_b2, index_b2), 'k:')
plt.title('Background 1s azimuthal average over y=[{:d} ; {:d}]'.format(ymin, ymax))
plt.tight_layout()

vmin = 0
vmax = 0.5
fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(18, 6))
# Show fitted background image
axs[0].imshow(bkg_r, vmin=vmin, vmax=vmax, cmap='gray', origin='lower')
axs[1].imshow(bkg_g, vmin=vmin, vmax=vmax, cmap='gray', origin='lower')
axs[2].imshow(bkg_b, vmin=vmin, vmax=vmax, cmap='gray', origin='lower')
axs[0].set_title('Fitted red background')
axs[1].set_title('Fitted green background')
axs[2].set_title('Fitted blue background')
plt.tight_layout()
plt.savefig('/Users/rattie/Data/Eclipse/figures/fitted_backgrounds.png')

# Remove background from the images
bkg_stack = np.stack([bkg_r, bkg_g, bkg_b], axis=-1)
bkg_stack2 = np.stack([bkg_r2, bkg_g2, bkg_b2], axis=-1)
images_bkg_removed = np.clip(images_expf - bkg_stack, 0, 1)
images_bkg_removed2 = images_expf - bkg_stack2
images_bkg_removed2[images_bkg_removed2 < 0 ] = 0

images_bkg_removed_v = np.clip((images_expf - bkg_stack)*5, 0, 1)

# Display the 3 images in RGB with background removed
fig, axs = plt.subplots(nrows=3, ncols=3, figsize=(19,11))
axs[0,0].imshow(images_expf_clipped[0,...])
axs[0,0].set_title('x 250')
axs[0,1].imshow(images_expf_clipped[1,...])
axs[0,1].set_title('x 30')
axs[0,2].imshow(images_expf_clipped[2,...])
axs[0,2].set_title('x 1')

axs[1,0].imshow(images_bkg_removed[0,...])
axs[1,0].set_title('bkg removed')
axs[1,1].imshow(images_bkg_removed[1,...])
axs[1,1].set_title('bkg removed')
axs[1,2].imshow(images_bkg_removed[2,...])
axs[1,2].set_title('bkg removed')

axs[2,0].imshow(images_bkg_removed2[0,...])
axs[2,0].set_title('bkg removed (2)')
axs[2,1].imshow(images_bkg_removed2[1,...])
axs[2,1].set_title('bkg removed (2)')
axs[2,2].imshow(images_bkg_removed2[2,...])
axs[2,2].set_title('bkg removed (2)')

plt.tight_layout()

plt.savefig('/Users/rattie/Data/Eclipse/figures/Images_background_removed.png')

# Compare the far end of the corona once the backgroun is removed => multiply brightness by some factor
images_bkg_removed2_v = np.clip((images_expf - bkg_stack2)*5, 0, 1)

fig, axs = plt.subplots(nrows=2, ncols=3, figsize=(19,11))
axs[0,0].imshow(images_bkg_removed_v[0,...])
axs[0,0].set_title('bkg removed')
axs[0,1].imshow(images_bkg_removed_v[1,...])
axs[0,1].set_title('bkg removed')
axs[0,2].imshow(images_bkg_removed_v[2,...])
axs[0,2].set_title('bkg removed')

axs[1,0].imshow(images_bkg_removed2_v[0,...])
axs[1,0].set_title('bkg removed (2)')
axs[1,1].imshow(images_bkg_removed2_v[1,...])
axs[1,1].set_title('bkg removed (2)')
axs[1,2].imshow(images_bkg_removed2_v[2,...])
axs[1,2].set_title('bkg removed (2)')
plt.tight_layout()

plt.savefig('/Users/rattie/Data/Eclipse/figures/Images_background_removed_compare.png')

# Radial filter. Try with 1/r

# Exporting to Tiff files: must restore the equivalent data range by multiplying by the exposure and maxval

suffix = ['250th', '30th', '1']
exposures = [250, 30, 1]

# Save the images as numpy arrays
fname = os.path.join(savedir, 'img_bkg_removed.npz')
np.savez(fname, images_bkg_removed2)

#images = np.zeros([nexp, ny, nx, 3])
# "De-normalize" the data to restore the equivalent data range falling within 14-bit depth.
images = images_bkg_removed2 * maxval / tarray3

for exp, s in enumerate(suffix):
    # convert from openCV RGBB to BGR
    image_bgr = cv2.cvtColor(images[exp, ...].astype(np.uint16), cv2.COLOR_RGB2BGR)
    fname = os.path.join(savedir, 'bkg_removed_{:s}.tiff'.format(s))
    cv2.imwrite(fname, image_bgr)

#TODO: remove background independent from 30th and 1s exposure. The 30th is done already. Only the 1s needs to be done.
#TODO: No need to remove the background from the 250s
#TODO: The corona will have to be radially attenuated at the begnining for any overlay with prominence, and increased with radius.


"""
Find top result from spotify with search term
and reply to posts with the keyword
SpotifyIt!
"""

import os
import sys
import time
import json
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import spotipy
import praw
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def get_spotify_result(comment_text):
    """
    Query Spotify to return top result
    """
    print('getting spotify result...')
    sys.stdout.flush()
    try:
        query = comment_text.split('!', 1)

        spotify_response = SPOTIFY.search(q=query[1], limit=1)
        spotify_response = json.dumps(spotify_response)
        spotify_response = json.loads(spotify_response)

        spotify_response = spotify_response["tracks"]["items"]
        artist = [item["album"]["artists"][0]["name"] for item in spotify_response]
        track = [item["name"] for item in spotify_response]
        album = [item["album"]["name"] for item in spotify_response]
        url = [item["external_urls"]["spotify"] for item in spotify_response]

        top_result = 'Artist: {0} \n\nTrack: {1} \n\nAlbum: \
        {2} \n\nURL: {3}'.format(artist[0], track[0], album[0], url[0])

    except Exception as error:
        top_result = 'No results returned. \n\nPlease try again with different keywords.'
        print('[spotify] Error: {0}'.format(error))
        sys.stdout.flush()

    return top_result

def search_for_comments():
    """
    Request comments with KEYWORD
    """
    print('getting comment list...')
    sys.stdout.flush()
    try:
        request = requests.get(\
        'https://api.pushshift.io/reddit/search?q=%22SpotifyIt!%22&limit=100',\
        headers={'User-Agent':'SpotifyItBot-Agent'}, verify=False)
        pushshift_response = request.json()
        pushshift_response = pushshift_response["data"]
    except Exception as error:
        print('[pushshift] Error: {0}'.format(error))
        sys.stdout.flush()

    return pushshift_response

def main():
    """
    Check for duplicates and comment on new searches
    """
    while True:
        comments = search_for_comments()
        try:
            for com in comments:
                comment_text = com["body"]
                comment_id = com["id"]
                submission = REDDIT.comment(comment_id)
                if comment_id not in SPOTIFIED_POSTS:
                    reply = get_spotify_result(comment_text)
                    SPOTIFIED_POSTS.append(comment_id)
                    disclaimer = "I'm a bot bleep bloop. \
                    \n\nPM me for more information or to report any issues."
                    submission.reply('{0}\n\n\n{1}'.format(reply, disclaimer))
                    print('Replied to {0} \n{1}'.format(comment_id, reply))
                    sys.stdout.flush()
                    print('writing to file...')
                    sys.stdout.flush()
                    # write to file
                    with open("SPOTIFIED_POSTS.txt", "w") as text_file:
                        for comment_id in SPOTIFIED_POSTS:
                            text_file.write(comment_id + "\n")
        except Exception as error:
            print('[main] Error: {0}'.format(error))
            sys.stdout.flush()
        time.sleep(60)

# runner
print("start...")
sys.stdout.flush()

REDDIT = praw.Reddit('spotify_it_bot')
SPOTIFY = spotipy.Spotify()
SUB = REDDIT.subreddit('all')
KEYWORD = 'SpotifyIt!'

# create file to record replies
if not os.path.isfile("SPOTIFIED_POSTS.txt"):
    SPOTIFIED_POSTS = []
else:
    with open("SPOTIFIED_POSTS.txt", "r") as f:
        SPOTIFIED_POSTS = f.read()
        SPOTIFIED_POSTS = SPOTIFIED_POSTS.split("\n")
        SPOTIFIED_POSTS = list(filter(None, SPOTIFIED_POSTS))

main()


# -*- coding: utf-8 -*-
import whisper
import zhconv
import time
from collections import deque
import logging
from logging.handlers import RotatingFileHandler
import os, json
from math import ceil
import queue
from recorder import read_audio_stream
from threading import Thread
import wave
import numpy as np

error_logger = logging.getLogger(__name__)

# 日志文件
ai_log_path = '/data/ai/ai.log'
ai_audio_path = '/data/ai/audio'
# 字典文件 json字典文件中char_map为替换的单字,words为替换的词语
with open('speech_command_dict.json', 'r') as f:
    speech_command_dict = json.load(f)

#  正式：rtsp://127.0.0.1:554/stream0/asound_rockchiphdmi   测试用：export RTSP_URL=rtmp://192.168.1.8:1935/live/1
# 设置错误信息日志的保存位置和文件名
file_handler = RotatingFileHandler(ai_log_path, maxBytes=10*1024*1024, backupCount=5)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
error_logger.addHandler(file_handler)
error_logger.setLevel(logging.ERROR)

if not os.path.exists(ai_audio_path):
    os.makedirs(ai_audio_path)
else:
    # 清理掉ai_audio_path下的所有文件
    for file_name in os.listdir(ai_audio_path):
        file_path = os.path.join(ai_audio_path, file_name)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
        except Exception as e:
            error_logger.error("Unable to delete file: " + str(e))

def speech_rec(wait_rec_audio_queue, message_queue,):

    """
    语音识别函数
    :param wait_rec_audio_queue: 存放音频文件地址和能量阈值的队列
    :param message_queue: 存放识别结果的队列
    """

    speech_config = '/data/ai/speech_config.json'
    rtsp_url_env = os.environ.get('RTSP_URL')
    try:
        with open(speech_config, 'r') as f:
            config = json.load(f)
            rtsp_url = config['RTSP_URL'] if not rtsp_url_env else rtsp_url_env
            model_type = str(config['MODEL_TYPE'])
    except Exception as e:
        rtsp_url = '......'
        model_type = '1'

    if model_type == '1':
        model = whisper.load_model("/usr/lib/wpm/base.pt")
    elif model_type == '2':
        model = whisper.load_model("/usr/lib/wpm/small.pt")
    elif model_type == '3':
        model = whisper.load_model("/usr/lib/wpm/medium.pt")
    else:
        model = whisper.load_model("/usr/lib/wpm/tiny.pt")
    # 第一次启动程序识别
    # result = model.transcribe('start_voice.mp3', language="Chinese")
    # print(result['text'])
    result = model.transcribe('start_test.wav', language="Chinese")
    print(result['text'])

    while True:
        wav_path, ENERGY_THRESHOLD = wait_rec_audio_queue.get()
        start_time = time.time()
        # with wave.open(wav_path, 'rb') as wav_file:
        #     audio_data = wav_file.readframes(-1)
        #     audio_data = np.frombuffer(audio_data, dtype=np.int16).flatten().astype(np.float32) / 32768.0
        if not os.path.exists(wav_path):
            continue

        for i in range(5):
            try:
                print("%s speech rec start..." % wav_path)
                result = model.transcribe(wav_path, language="Chinese")
                break
            except Exception as e:
                print(e)
                time.sleep(0.1)

        audio_rec_time = time.time() - start_time
        audio_start_time = int(wav_path.split('/')[-1].split('.')[0]) - 1

        segments = result['segments']
        if len(segments) == 0:
            print("No speech content recgnation")
            continue

        duration = ceil(max([i['end'] for i in segments])) + 1
        text_result = zhconv.convert(result['text'], 'zh-cn')
        text_result = text_translate(text_result)
        # 清理音频文件
        if os.path.exists(wav_path):
            os.remove(wav_path)

        print("********************************************************* 推理时间: %d 等待时间：%f 音频时长：%f " % (audio_rec_time, time.time() - audio_start_time, duration))
        print("********************************************************* 识别内容: " + result['text'])
        print("********************************************************* 翻译内容: " + text_result)

        # 组织返回的数据
        data = {
            "rtsp_id": rtsp_url,
            "start_time": audio_start_time,
            "duration": duration,
            "text": text_result,
            "energy_threashold": ENERGY_THRESHOLD
        }
        # 发送消息
        message_queue.put((1, data))

def main(message_queue):
    # 注意 这个队列中存的是两个值，一个音频文件地址 一个energy_threashold
    wait_rec_audio_queue = queue.Queue()
    get_rec_audio = Thread(target=read_audio_stream, args=(wait_rec_audio_queue,))
    rec_audio = Thread(target=speech_rec, args=(wait_rec_audio_queue, message_queue,))
    get_rec_audio.start()
    rec_audio.start()
    wait_rec_audio_queue.join()
    get_rec_audio.join()
    rec_audio.join()

def text_translate(text):
    """
    翻译文本
    :param text: 待翻译文本
    :return: 翻译后文本
    """
    global speech_command_dict
    char_map = speech_command_dict['char_map']
    words = speech_command_dict['words']

    translate_text = ""
    for o_char in text:
        if o_char in char_map:
            translate_text += char_map[o_char]
        else:
            translate_text += o_char

    for i in words:
        translate_text = translate_text.replace(i[0], i[1]).replace("音响", "影响").replace("Go", "9")

    return translate_text

if __name__ == "__main__":
    message_queue = queue.Queue()
    main(message_queue)


"""project URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from django.conf.urls import url
from app import views

urlpatterns = [
    path('admin/', admin.site.urls),
    url(r'^$', views.home, name="home"),
    url(r'^signup/$', views.signup, name="signup"),
    url(r'^user_login/', views.user_login, name="user_login"),
    url(r'^menu$', views.menu, name="menu"),
    url(r'^logout/$', views.user_logout, name="logout"),
    url(r'^mess/$', views.mess, name="mess"),
    url(r'^breakfast/$', views.breakfast, name="breakfast"),
    url(r'^lunch/$', views.lunch, name="lunch"),
    url(r'^dinner/$', views.dinner, name="dinner"),
    url(r'^restaurant/$', views.restaurant, name="restaurant"),
    url(r'^rbreakfast/$', views.rbreakfast, name="rbreakfast"),
    url(r'^rlunch/$', views.rlunch, name="rlunch"),
    url(r'^rdinner/$', views.rdinner, name="rdinner"),
]


import pytest
import responses

from norris import get_joke, joke_length

"""
Response will look something like this
{
  "category": "['science']",
  "icon_url": "https://assets.chucknorris.host/img/avatar/chuck-norris.png",
  "id": "7SIYDCQ6QiiTLtd-B3sBEQ",
  "url": "http://api.chucknorris.io/jokes/7SIYDCQ6QiiTLtd-B3sBEQ",
  "value": "Chuck Norris broke the speed of sound. With his elbow."
}
"""

@pytest.fixture
def my_responses():

    test_response = {
        "category": ['science'],
        "icon_url": "https://assets.chucknorris.host/img/avatar/chuck-norris.png",
        "id": "7SIYDCQ6QiiTLtd-B3sBEQ",
        "url": "http://api.chucknorris.io/jokes/7SIYDCQ6QiiTLtd-B3sBEQ",
        "value": "Chuck Norris broke the speed of sound. With his elbow."
    }

    with responses.RequestsMock() as rsps:
        rsps.add(responses.GET, 'https://api.chucknorris.io/jokes/random?category=science',
                 json=test_response, status=200)
        yield rsps

def test_pytest_norris_basic(my_responses):
    verify_response = {
        "category": ['science'],
        "icon_url": "https://assets.chucknorris.host/img/avatar/chuck-norris.png",
        "id": "7SIYDCQ6QiiTLtd-B3sBEQ",
        "url": "http://api.chucknorris.io/jokes/7SIYDCQ6QiiTLtd-B3sBEQ",
        "value": "Chuck Norris broke the speed of sound. With his elbow."
    }
    result = get_joke()
    assert result == verify_response

def test_norris_value(my_responses):

    test_value = len("Chuck Norris broke the speed of sound. With his elbow.")
    # test_value = 5

    result = joke_length()
    assert result == test_value

# This test is to demonstrate when you might want to use curl or postman
# to troubleshoot an API call
"""
Imagine the response used to look like:
{
  "data": {
      "category": "['science']",
      "icon_url": "https://assets.chucknorris.host/img/avatar/chuck-norris.png",
      "id": "7SIYDCQ6QiiTLtd-B3sBEQ",
      "url": "http://api.chucknorris.io/jokes/7SIYDCQ6QiiTLtd-B3sBEQ",
      "value": "Chuck Norris broke the speed of sound. With his elbow."
  }
}
"""
def test_norris_badkey():
    result = get_joke()
    # This will fail because there's no such key as data anymore
    assert result["data"]["value"] == "This is a bad key"


import os
import math
import sklearn.metrics
import pickle
import numpy as np
import sklearn.linear_model
import argparse
import scipy
import collections
import datetime
import csv

parser = argparse.ArgumentParser(
    description="Get model performance and simplified pesi score performance",
)
parser.add_argument("--path_to_data", type=str, required=True)
parser.add_argument("--path_to_output", type=str, required=True)
parser.add_argument("--compare_vs_pesi", action=argparse.BooleanOptionalAction)
parser.add_argument("--require_pe_status", type=str, required=False, default=None)
parser.add_argument("--compute_confidence", action=argparse.BooleanOptionalAction)
parser.add_argument("--relative_confidence", action=argparse.BooleanOptionalAction)
parser.add_argument("--metric", type=str, default="auroc")

args = parser.parse_args()

if args.require_pe_status is not None:
    args.require_pe_status = args.require_pe_status.lower() == "true"

tasks = (
    "PE",
    "1_month_mortality",
    "6_month_mortality",
    "12_month_mortality",
    "1_month_readmission",
    "6_month_readmission",
    "12_month_readmission",
    "12_month_PH",
)
ct_names = (
    "pe",
    "mort_1m",
    "mort_6m",
    "mort_12m",
    "read_1m",
    "read_6m",
    "read_12m",
    "ph_12m",
)

ct_name_map = {k: v for k, v in zip(tasks, ct_names)}

if args.require_pe_status != None:
    tasks = tasks[1:]

performances = {}

has_pe = {}
pid_split_assignment = {}
pesi_map = {}

with open(os.path.join(args.path_to_data, "cohort_0.2.0_master_file.csv")) as f:
    reader = csv.DictReader(f)
    for row in reader:
        pid = int(row["patient_id"])
        pid_split_assignment[pid] = row["split"]
        time = datetime.datetime.fromisoformat(row["procedure_time"])
        has_pe[(pid, time)] = row["pe_positive_nlp"] == "True"
        pesi_map[(pid, time)] = float("nan")

with open(f"{args.path_to_output}/simplified_pesi/pesi_scores.pkl", "rb") as f:
    pesi_predictions, pesi_patient_ids, pesi_times = pickle.load(f)

for prediction, patient_id, time in zip(pesi_predictions, pesi_patient_ids, pesi_times):
    pesi_map[(patient_id, time)] = prediction

print("Have pesi scores for ", sum(v == v for k, v in pesi_map.items()), " labels")

from itertools import chain, combinations

def powerset(iterable):
    "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"
    s = list(iterable)
    return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1))

def load_data(name, task, split):
    if name == "GBM":
        with open(
            f"{args.path_to_output}/gbm_model_results/{task}/predictions.pkl", "rb"
        ) as f:
            result = pickle.load(f)
    elif name == "MOTOR":
        with open(
            f"{args.path_to_output}/motor_results/{task}/predictions.pkl", "rb"
        ) as f:
            result = pickle.load(f)
    elif name == "sPESI":
        baseline = load_data("GBM", task, split)
        for i, (pid, time) in enumerate(zip(baseline[1], baseline[3])):
            if task == "PE":
                time = time + datetime.timedelta(days=1)
            baseline[0][i] = scipy.special.expit(pesi_map[(pid, time)])
        return baseline
    elif name == "CT":
        ct_name = ct_name_map[task]
        pids = []
        prediction_dates = []
        labels = []
        probs = []

        with open(f"{args.path_to_output}/ct_results/{ct_name}_pred_proba.csv") as f:
            reader = csv.DictReader(f)
            for line in reader:
                pid = int(line["patient_id"])
                prediction_date = datetime.datetime.fromisoformat(
                    line["procedure_time"]
                )

                if task == "PE":
                    prediction_date = prediction_date - datetime.timedelta(days=1)

                label = float(line["label"]) == 1
                prob = float(line["prob"])
                pids.append(pid)
                prediction_dates.append(prediction_date)
                labels.append(label)
                probs.append(prob)

        result = [probs, pids, labels, prediction_dates]

    if not isinstance(result[3][0], datetime.datetime):
        result[3] = result[3].astype(datetime.datetime)

    o = np.lexsort((result[1], result[3]))
    result = [np.array(v)[o] for v in result]
    valid_indices = []

    for i, (pid, time) in enumerate(zip(result[1], result[3])):
        if task == "PE":
            time = time + datetime.timedelta(days=1)
        good = pid_split_assignment[pid] == split
        if args.compare_vs_pesi:
            if pesi_map[(pid, time)] != pesi_map[(pid, time)]:
                good = False

        if args.require_pe_status is not None:
            if has_pe[(pid, time)] != args.require_pe_status:
                good = False

        if good:
            valid_indices.append(i)
    result = [a[valid_indices] for a in result]

    return result

baseline_model = "MOTOR"

baseline_performance = {}

def get_performance(labels, probabilities):
    if args.metric == "auroc":
        return sklearn.metrics.roc_auc_score(labels, probabilities)
    elif args.metric == "auprc":
        return sklearn.metrics.average_precision_score(labels, probabilities)
    elif args.metric == "ece":
        bins = np.linspace(0, 1, num=10)
        start_bin = bins[:-1]
        end_bin = bins[1:]

        total = 0

        for start, end in zip(start_bin, end_bin):
            total_true = 0
            total_prob = 0
            num_in_bin = 0
            for label, prob in zip(labels, probabilities):
                if start <= prob < end:
                    total_true += label
                    total_prob += prob
                    num_in_bin += 1
            if num_in_bin != 0:
                total += num_in_bin * abs(
                    total_true / num_in_bin - total_prob / num_in_bin
                )

        return total / len(labels)
    else:
        assert False, f"{args.metric} is not a valid metric"

if args.compute_confidence:
    for task in tasks:
        predictions, pids, labels, times = load_data(baseline_model, task, "test")

        samples = []
        for _ in range(1000):
            indices = sklearn.utils.resample(list(range(len(predictions))))
            l_s, p_s = labels[indices], predictions[indices]
            performance = get_performance(l_s, p_s)
            samples.append((indices, performance))

        baseline_performance[task] = samples

def compute_score_and_bootstrap(task, labels, predictions):
    total_score = get_performance(labels, predictions)

    if args.compute_confidence:
        samples = []
        for indices, baseline_perf in baseline_performance[task]:
            l_s, p_s = labels[indices], predictions[indices]
            perf = get_performance(l_s, p_s)
            if args.relative_confidence:
                perf -= baseline_perf
            samples.append(perf)
        confidence = np.quantile(samples, (0.025, 0.975))
    else:
        confidence = None

    return total_score, confidence

def get_model_performance(task, name):
    predictions, pids, labels, times = load_data(name, task, "test")

    return compute_score_and_bootstrap(task, labels, predictions)

def get_combined_model_performance(task, names):
    pids = {}
    times = {}
    labels = {}
    probs = {}

    for split in ["valid", "test"]:
        for name in names:
            k = (name, split)
            probs[k], pids[k], labels[k], times[k] = load_data(name, task, split)

            if name != names[0]:
                assert len(pids[(names[0], split)]) == len(
                    pids[(name, split)]
                ), f"{len(pids[(names[0], split)])} {len(pids[(name, split)])}"

                assert np.all(pids[(names[0], split)] == pids[(name, split)])
                assert np.all(times[(names[0], split)] == times[(name, split)])
                assert np.all(labels[(names[0], split)] == labels[(name, split)])

    all_probs = np.stack([probs[(name, "valid")] for name in names], axis=-1)
    features = scipy.special.logit(all_probs)
    features = np.clip(features, -1e10, 1e10)

    model = sklearn.linear_model.LogisticRegression(penalty=None, solver="newton-cg")

    model.fit(features, labels[(names[0], "valid")])

    if np.any(np.isnan(model.coef_)):
        # Could not converge. Fallback to average
        model.coef_ = np.ones_like(model.coef_)
        model.intercept_ = 0

    test_probs = np.stack([probs[(name, "test")] for name in names], axis=-1)
    test_features = scipy.special.logit(test_probs)
    test_features = np.clip(test_features, -1e10, 1e10)

    computed_probs = model.predict_proba(test_features)[:, 1]

    return compute_score_and_bootstrap(task, labels[(names[0], "test")], computed_probs)

# models = ["CT", "MOTOR", "GBM"]
models = ["MOTOR", "GBM"]
if args.compare_vs_pesi:
    models.append("sPESI")

aurocs = collections.defaultdict(dict)
fusion_aurocs = collections.defaultdict(dict)

best_perfs = {}
best_m_perfs = {}

def compare(a, b):
    if args.metric == "ece":
        return a > b
    else:
        return a < b

for task in tasks:
    for model in models:
        aurocs[model][task] = get_model_performance(task, model)
        if not args.compare_vs_pesi:
            if task not in best_perfs or compare(
                best_perfs[task][0][0], aurocs[model][task][0]
            ):
                best_perfs[task] = aurocs[model][task], model
        if task not in best_m_perfs or compare(
            best_m_perfs[task][0][0], aurocs[model][task][0]
        ):
            best_m_perfs[task] = aurocs[model][task], model

    if not args.compare_vs_pesi:
        for names in powerset(models):
            if len(names) > 1:
                fusion_aurocs[names][task] = get_combined_model_performance(task, names)

                if task not in best_m_perfs or compare(
                    best_m_perfs[task][0][0], fusion_aurocs[names][task][0]
                ):
                    best_m_perfs[task] = fusion_aurocs[names][task], names

print(aurocs)

print("MAIN TABLE")

for model, performances in aurocs.items():
    row = " "
    for i, model_column in enumerate(models):
        if i != 0:
            row += " &"
        if model == model_column:
            row += " $\\checkmark$"
    for task in tasks:
        row += "& "
        value = f"{performances[task][0]:0.3f} "
        if task in best_perfs and model == best_perfs[task][1]:
            value = f"\\underline{{" + value + f"}}"
        if task in best_m_perfs and model == best_m_perfs[task][1]:
            value = f"\\textbf{{" + value + f"}}"

        row += value
    row += "\\\\"
    print(row)

if not args.compare_vs_pesi:
    print("\\midrule")

    for eval_models, performances in fusion_aurocs.items():
        row = " "
        for i, model_column in enumerate(models):
            if i != 0:
                row += " &"
            if model_column in eval_models:
                row += "$\\checkmark$"
        for task in tasks:
            row += "& "
            if eval_models == best_m_perfs[task][1]:
                row += f"\\textbf{{" + f"{performances[task][0]:0.3f}" + f"}}"
            else:
                row += f"{performances[task][0]:0.3f} "
        row += "\\\\"
        print(row)

if args.compute_confidence:
    print("CONFIDENCE INTERVAL TABLE")

    for model, performances in aurocs.items():
        row = " "
        for i, model_column in enumerate(models):
            if i != 0:
                row += " &"
            if model == model_column:
                row += " $\\checkmark$"
        for task in tasks:
            row += "& "
            text = f"({performances[task][1][0]:0.2f}, {performances[task][1][1]:0.2f})"
            if args.relative_confidence and (
                all(v < 0 for v in performances[task][1])
                or all(v > 0 for v in performances[task][1])
            ):
                text = f"\\textbf{{" + text + f"}}"
            row += text
        row += "\\\\"
        print(row)

    print("\\midrule")

    for eval_models, performances in fusion_aurocs.items():
        row = " "
        for i, model_column in enumerate(models):
            if i != 0:
                row += " &"
            if model_column in eval_models:
                row += "$\\checkmark$"
        for task in tasks:
            row += "& "
            text = f"({performances[task][1][0]:0.2f}, {performances[task][1][1]:0.2f})"
            if args.relative_confidence and (
                all(v < 0 for v in performances[task][1])
                or all(v > 0 for v in performances[task][1])
            ):
                text = f"\\textbf{{" + text + f"}}"
            row += text
        row += "\\\\"
        print(row)


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 26 20:49:44 2020

@author: Home
"""

from collections import Counter
import numpy as np
import pandas as pd
import nltk
import re
from nltk.corpus import stopwords
import fuzzywuzzy
from fuzzywuzzy import fuzz
from  nltk import word_tokenize
import torch.optim as optim

def isEnglish(s):
    try:
        s.encode(encoding='utf-8').decode('ascii')
    except UnicodeDecodeError:
        return False
    else:
        return True
def remove_emoji(string):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', string)

f = "english/agr_en_train.csv"

# preprocessing english tweets.
#ingesting english csv file
df = pd.read_csv(f,names = ['source','comment','annotation'],encoding='UTF-8')
df['comment'] = df.comment.str.strip()   # removing spaces

comments = np.asarray(df['comment'])    # dividing the dataframe into comments and tags and converting to array
tags = np.asarray(df['annotation'])
print((len(comments)))
print(len(tags))

stop_words = set(stopwords.words('english'))  #english stop words list
processed_tokens = []
for comment in comments:
#    comment = "Also see ....hw ur RSS activist caught in Burkha .... throwing beef in d holy temples...https://www.google.co.in/amp/www.india.com/news/india/burkha-clad-rss-activist-caught-throwing-beef-at-temple-pictures-go-viral-on-facebook-593154/amp/,NAGfacebook_corpus_msr_403402,On the death of 2 jawans in LOC CROSS FIRING"
    comment = comment.lower()   #lower casing each tweets
    Digit_REMOVAL = re.sub(r'[0-9]+', '',comment) #removal of numbers
    URL_REMOVAL = re.sub(r"http\S+", "", Digit_REMOVAL) # removal of URLS
    tokenizer = nltk.RegexpTokenizer(r"\w+")   # removal of punctuation and tokenizing
    new_words = tokenizer.tokenize(URL_REMOVAL)
    sentence = []
    for word in new_words:
        if word not in stop_words:           #checking for stop words on each sentence
            sentence.append(word)
    processed_tokens.append(sentence)

#-----------------For hinglish dataset

Hindi_text  = "hindi/agr_hi_dev.csv"
df1 = pd.read_csv(Hindi_text,names = ['source','comment','annotation'],encoding='UTF-8')
df1['comment'] = df1.comment.str.strip()   # removing spaces
hindi_comments = np.asarray(df1['comment'])    # dividing the dataframe into comments and tags and converting to array
hindi_tags = np.asarray(df1['annotation'])
print((hindi_comments[1]))
processed_Hindi_tokens = []
for comment in hindi_comments:
#    comment = "Also see ....hw ur RSS activist caught in Burkha .... throwing beef in d holy temples...https://www.google.co.in/amp/www.india.com/news/india/burkha-clad-rss-activist-caught-throwing-beef-at-temple-pictures-go-viral-on-facebook-593154/amp/,NAGfacebook_corpus_msr_403402,On the death of 2 jawans in LOC CROSS FIRING"
    comment = comment.lower()   #lower casing each tweets
    Digit_REMOVAL = re.sub(r'[0-9]+', '',comment) #removal of numbers
    URL_REMOVAL = re.sub(r"http\S+", "", Digit_REMOVAL) # removal of URLS
    Emoji_removal = remove_emoji(URL_REMOVAL)
    if (isEnglish(Emoji_removal) == True):
        Emoji_removal = re.sub(r'[^\w\s]','',Emoji_removal)# removal of punctuation and tokenizing
    processed_Hindi_tokens.append(word_tokenize(Emoji_removal))
processed_Hindi_tokens[0]
processed_Hindi_tokens[11]
processed_Hindi_tokens[6]
processed_Hindi_tokens[11]

#-----------Transliteration and translation
transliteration_dict = "B:\CS 695\Assignment3\Classification-of-Offensive-Tweets-in-Hinglish-Language/transliterations.hi-en.csv"
t_dict = pd.read_csv(transliteration_dict,names = ['Hinglish','Hindi'],encoding='UTF-8',sep='\t')
t_dict['Hinglish'] = t_dict['Hinglish'].str.strip()
t_dict['Hindi'] = t_dict['Hindi'].str.strip()
t_dict = np.asarray(t_dict)

#--------------profanity dictionary
profanity_dict = "ProfanityText.txt"
P_dict = pd.read_csv(profanity_dict,names = ['Hinglish','English'],encoding='UTF-8',sep='\t')
P_dict['Hinglish'] = P_dict['Hinglish'].str.strip()
P_dict['English'] = P_dict['English'].str.strip()
P_dict = np.asarray(P_dict)

print(t_dict)
processed_Hindi_tokens[4]
for i in range(0,len(processed_Hindi_tokens)):
    print(i)
    for j in range (0,len(processed_Hindi_tokens[i])):
        flag = 0
        Str1 = (processed_Hindi_tokens[i][j])
        max_ratio = 60
        max_ratio_P = 75   #needs to be adjusted
        if (Str1 in EH_dict): # check whether the values exists in english dictionary or not.
            continue;
        for l in range(0,len(P_dict)):
            Str2 = P_dict[l][0]
            Ratiostr1 = fuzz.ratio(Str1,Str2)
            if (Ratiostr1 >= max_ratio_P):
                print(Ratiostr1)
                max_ratio_P = Ratiostr1
                processed_Hindi_tokens[i][j] = P_dict[l][1]
                flag = 1
                print(flag)
                break;
        for p in EH_dict_F:
            RAtionstr1 = fuzz.ratio(Str1,str(p))
            if(Ratiostr1 >= 98):
                flag = 1
                break;
        if (flag == 1):
            continue;

        else:
            for k in range(0,len(t_dict)):
                Str2 = t_dict[k][0]
                Ratiostr1 = fuzz.ratio(Str1,Str2)
                if (Ratiostr1 > max_ratio):
                    max_ratio = Ratiostr1
                    processed_Hindi_tokens[i][j] = t_dict[k][1]
processed_Hindi_tokens[0]
processed_Hindi_tokens[1]
(processed_tokens[12])

#------------------Translation of hindi text back to english-------

Hindi_dict = "Hindi_English_Dict.csv"
H_dict = pd.read_csv(Hindi_dict,names = ['Hindi','English'],encoding='UTF-8')

HE_dict_F = "HE_dictionary_functions.csv"
H_dict_F = pd.read_csv(HE_dict_F,names = ['Hindi','English'],encoding='UTF-8')
H_dict_F['Hindi'] = H_dict_F['Hindi'].str.strip()
H_dict_F['English'] = H_dict_F['English'].str.strip()
H_hindi_F = np.asarray(H_dict_F['Hindi'])
H_english_F = np.asarray(H_dict_F['English'])

H_dict['Hindi'] = H_dict['Hindi'].str.strip()
H_dict['English'] = H_dict['English'].str.strip()
H_hindi = np.asarray(H_dict['Hindi'])
H_english = np.asarray(H_dict['English'])

HE_dict = dict(zip(H_hindi,H_english))
H_dict_F = dict(zip(H_hindi_F,H_english_F))

EH_dict = {v:k for k, v in HE_dict.items()}
EH_dict_F = {v:k for k, v in H_dict_F.items()}

for i in range(0,len(processed_Hindi_tokens)):
    print(i)
    for j in range (0,len(processed_Hindi_tokens[i])):
        Str = processed_Hindi_tokens[i][j]
        if(Str in HE_dict):
            processed_Hindi_tokens[i][j] = HE_dict[Str]
        elif(Str in H_dict_F):
            processed_Hindi_tokens[i][j] = H_dict_F[Str]

#------------------------------MODEL

import torch
from torch import nn
from torch.autograd import Variable
import torch.nn.functional as F

class MIMCT(nn.Module):
    def __init__(self,input_channel,output_channel,embedding_dim,hidden_dim,kernel_size,feature_linear):
        super(MIMCT, self).__init__()
        self.CNN_Layers = nn.Sequential(
            nn.Conv1d(input_channel, output_channel,kernel_size[0], stride=1),
            nn.Conv1d(input_channel, output_channel, kernel_size[1], stride=1),
            nn.Conv1d(input_channel, output_channel, kernel_size[2], stride=1),
            nn.Flatten(),nn.Dropout(p=0.25),
            nn.Linear(feature_linear, 3),
            nn.Softmax()
            )
        #create a sequential for LSTM.
        self.LSTM_Layers = nn.Sequential(
            nn.LSTM(embedding_dim, hidden_dim, output_channel,dropout),
            nn.Linear(hidden_dim, 3),
            nn.Dropout(p=0.20),
            nn.Softmax()
        )
    def forward(self,x):
        x = self.CNN_Layers(x)
      #  y = self.LSTM_Layers(x)
        #concat the outputs the compile layer with categorical cross-entropy the loss function,
       # print(y)
        return x

relu = nn.ReLU()

#try with output channel 1 as well.
batch_size = 1
input_channel = 16
embedding_dim = 200
output_channel = 16
kernel_size = [20,15,10]
Feature_layer1 = embedding_dim - kernel_size[0] + 1
Feature_layer2 = Feature_layer1 - kernel_size[1] + 1
Feature_layer3 = Feature_layer2 - kernel_size[2] + 1
feature_linear = Feature_layer3 * input_channel

#Parameters for LSTM
hidden_dim = 128
dropout = 0.25,
#recurrent_dropout = 0.3

model = MIMCT(input_channel,output_channel,embedding_dim,hidden_dim,kernel_size,feature_linear)

#Adam Optimizer
optimizer = optim.Adam(model.parameters(), lr=0.01)
input1 = torch.randn(batch_size,input_channel,embedding_dim)
output = model(input1)
#create the loss cretirion and training loops

'''
m = nn.Conv1d(1, 2,1,stride=2)
input1 = torch.randn(10)
cnn1d_2 = nn.Conv1d(in_channels=1, out_channels=1, kernel_size=3, stride=2)
input1 = input1.unsqueeze(0).unsqueeze(0)
input1.shape
cnn1d_2(input1)

#------------CNN_Class----------------#
kernel_size = [4,3,2]
embedding_dim = 10
Feature_layer1 = embedding_dim - kernel_size[0] + 1
Feature_layer2 = Feature_layer1 - kernel_size[1] + 1
Feature_layer3 = Feature_layer2 - kernel_size[2] + 1

Layer1 = nn.Conv1d(16, 16, 4, stride=1)
Layer2 = nn.Conv1d(16, 16, 3, stride=1)
Layer3 = nn.Conv1d(16, 16, 2, stride=1)
Dropout_layer = nn.Dropout(p=0.25)
dense_layer = nn.Linear(Feature_layer3, 3)
nn.Flatten

#------------Forward------------------#
batch_size = 1
input_channel = 16
embedding_dim = 10
input1 = torch.randn(batch_size,input_channel,embedding_dim)
output = Layer1(input1)
output1 = Layer2(output)
output2 = Layer3(output1)
output3 = Dropout_layer(output2)
output4 = dense_layer(output3)
nn.flatten

x = torch.tensor([[1.0, -1.0],[0.0,  1.0],[0.0,  0.0]])

in_features = x.shape[1]  # = 2
out_features = 2

m = nn.Linear(in_features, out_features)
m.weight

class MIMCT(nn.Module):
    def __init__(self,input_channel,output_channel,embedding_dim,kernel_size,feature_linear):
        super(MIMCT, self).__init__()
        self.CNN_Layers = nn.Sequential( nn.Conv1d(input_channel, output_channel,kernel_size[0], stride=1),nn.Conv1d(input_channel, output_channel, kernel_size[1], stride=1),nn.Conv1d(input_channel, output_channel, kernel_size[2], stride=1),nn.Flatten(),nn.Dropout(p=0.25),nn.Linear(feature_linear, 3),nn.Softmax())

    def forward(self,x):
        x = self.CNN_Layers(x)
        return x

#try with output channel 1 as well.
batch_size = 1
input_channel = 16
embedding_dim = 200
output_channel = 16
kernel_size = [20,15,10]
embedding_dim = 200
Feature_layer1 = embedding_dim - kernel_size[0] + 1
Feature_layer2 = Feature_layer1 - kernel_size[1] + 1
Feature_layer3 = Feature_layer2 - kernel_size[2] + 1
feature_linear = Feature_layer3 * input_channel
model = MIMCT(input_channel,output_channel,embedding_dim,kernel_size,feature_linear)

output = model(input1)
'''

''''
kernel_size = [20,15,10]
embedding_dim = 200
Feature_layer1 = embedding_dim - kernel_size[0] + 1
Feature_layer2 = Feature_layer1 - kernel_size[1] + 1
Feature_layer3 = Feature_layer2 - kernel_size[2] + 1
feature_linear = Feature_layer3 * input_channel
m = nn.Sequential( nn.Conv1d(input_channel,16,kernel_size[0], stride=1),nn.Conv1d(input_channel, output_channel, kernel_size[1], stride=1),nn.Conv1d(input_channel, output_channel, kernel_size[2], stride=1),nn.Flatten(),nn.Dropout(p=0.25),nn.Linear(feature_linear, 3),nn.Softmax())

#------------Forward------------------#
batch_size = 1
input_channel = 16
embedding_dim = 200
input1 = torch.randn(batch_size,input_channel,embedding_dim)
test_input = input1
output = m(test_input)
nn.flatten
'''


from comet_ml import Experiment

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from sklearn.datasets import load_iris
from tqdm.auto import tqdm

import enchanter.tasks as tasks
import enchanter.engine.modules as modules
import enchanter.addons as addons
import enchanter.addons.layers as layers

experiment = Experiment()
model = layers.MLP([4, 512, 128, 3], addons.mish)
optimizer = optim.Adam(model.parameters())

x, y = load_iris(return_X_y=True)
x = x.astype("float32")
y = y.astype("int64")

train_ds = modules.get_dataset(x, y)
val_ds = modules.get_dataset(x, y)
test_ds = modules.get_dataset(x, y)

train_loader = DataLoader(train_ds, batch_size=32)
val_loader = DataLoader(val_ds, batch_size=32)
test_loader = DataLoader(test_ds, batch_size=32)

with tasks.ClassificationRunner(model, optimizer, nn.CrossEntropyLoss(), experiment) as runner:
    for epoch in tqdm(range(10)):
        with runner.experiment.train():
            for train_batch in train_loader:
                runner.optimizer.zero_grad()
                train_out = runner.train_step(train_batch)
                runner.backward(train_out["loss"])
                runner.update_optimizer()

                with runner.experiment.validate(), torch.no_grad():
                    for val_batch in val_loader:
                        val_out = runner.val_step(val_batch)["loss"]
                        runner.experiment.log_metric("val_loss", val_out)

        with runner.experiment.test(), torch.no_grad():
            for test_batch in test_loader:
                test_out = runner.test_step(test_batch)["loss"]
                runner.experiment.log_metric("test_loss", test_out)


import pygame
import json
from Weapon.Weapon import Weapon

with open("Config/ScreenSettingsData.json", "r") as f:
    SCREEN_SETTINGS = json.load(f)
WIDTH = SCREEN_SETTINGS["width"]

class ActionEntityTemplate(pygame.sprite.Sprite):
    class_health = None

    def __init__(self,
                 start_x_position,
                 start_y_position,
                 image_filename):
        super(ActionEntityTemplate, self).__init__()
        self.image = pygame.image.load(image_filename)
        self.rect = self.image.get_rect()
        self.rect.centerx = start_x_position
        self.rect.centery = start_y_position
        self.health = self.class_health

    def get_health(self):
        return self.health

    def kill_action(self):
        pass

    def reduce_health(self, reduce):
        self.health -= reduce
        if self.health <= 0:
            self.kill_action()

class ShootingAbilityEntity(ActionEntityTemplate):
    def __init__(self,
                 start_x_position,
                 start_y_position,
                 default_bullet_preset,
                 group_of_bullets,
                 reload_change_index,
                 image_filename):
        super(ShootingAbilityEntity, self).__init__(start_x_position,
                                                    start_y_position,
                                                    image_filename)
        self.weapon = Weapon(self,
                             default_bullet_preset,
                             reload_change_index)
        self.group_of_bullets = group_of_bullets

    def get_conflict_side(self):
        pass

    def shoot(self):
        self.weapon.shoot()

    def change_bullet_preset(self, preset):
        self.weapon.change_bullet_preset(preset)

class EnemyEntityTemplate(ShootingAbilityEntity):
    class_price = None
    calls_to_be_killed = 2

    def __init__(self,
                 start_x_position,
                 start_y_position,
                 default_bullet_preset,
                 group_of_bullets,
                 reload_change_index,
                 image_filename):
        super(EnemyEntityTemplate, self).__init__(start_x_position,
                                                  start_y_position,
                                                  default_bullet_preset,
                                                  group_of_bullets,
                                                  reload_change_index,
                                                  image_filename)
        self.radius = 15
        self.price = self.class_price
        self.calls_after_killing = 0
        self.killed = False

    def get_conflict_side(self):
        return False

    def get_price(self):
        return self.price

    def get_position_status(self):
        if self.rect.right >= WIDTH or self.rect.left <= 0:
            return True
        return False

    def kill_action(self):
        self.killed = True
        self.image = pygame.image.load("ActionEntities/ActionEntitiesAssets/Enemy/KIllState.png")

    def update(self, x_shift, y_shift):
        self.rect.centerx += x_shift
        self.rect.centery += y_shift
        if self.killed:
            self.calls_after_killing += 1
        if self.calls_after_killing == self.calls_to_be_killed:
            self.kill()


import torch
from torch.autograd import Variable
import torch.optim as optim

class CW(object):
    def __init__(self,model):
        self.model = model

    def get_loss(self,xi,label_onehot_v, c, modifier, TARGETED):
        #print(c.size(),modifier.size())
        loss1 = c*torch.sum(modifier*modifier)
        #output = net(torch.clamp(xi+modifier,0,1))
        output = self.model.predict(xi+modifier)
        real = torch.sum(torch.mul(output, label_onehot_v), 1)
        other = torch.max(torch.mul(output, (1-label_onehot_v))-label_onehot_v*10000,1)[0]
        #print(real,other)
        if TARGETED:
            loss2 = torch.sum(torch.clamp(other - real, min=0))
        else:
            loss2 = torch.sum(torch.clamp(real - other, min=0))
        error = loss2 + loss1.sum()
        #print(loss1,loss2)
        return error,loss1,loss2

    def cw(self, input_xi, label_or_target, c, TARGETED=False):

        modifier = Variable(torch.zeros(input_xi.size()).cuda(), requires_grad=True)
        yi = label_or_target
        label_onehot = torch.FloatTensor(yi.size()[0],self.model.num_classes).cuda()
        label_onehot.zero_()
        label_onehot.scatter_(1,yi.view(-1,1),1)
        label_onehot_v = Variable(label_onehot, requires_grad=False).cuda()
        xi = Variable(input_xi.cuda())
        optimizer = optim.Adam([modifier], lr = 0.1)
        best_loss1 = 1000
        best_adv = None
        for it in range(1000):
            optimizer.zero_grad()
            error,loss1,loss2 = self.get_loss(xi,label_onehot_v,c,modifier, TARGETED)
            #print(error)
            self.model.get_gradient(error)
            #error.backward()
            #if (it)%500==0:
            #    print(error.data,loss1.data,loss2.data)
            if loss2.data ==0:
                if best_loss1 >= loss1.data:
                    best_loss1 = loss1.data
                    best_adv = modifier.clone()
            optimizer.step()            # if (it)%10==0:
            #    print(error.data.item())
        if best_adv is None:
            #print(str(c)+'\t'+'None')
            return None
        else:
            return best_adv

    def __call__(self, input_xi, label_or_target, epsilon=None, TARGETED=False):
        dis_a = []
        c_hi = 1000*torch.ones(input_xi.size()[0],1).cuda()
        c_lo = 0.01*torch.ones(c_hi.size()).cuda()
        while torch.max(c_hi-c_lo) > 1e-1:
            c_mid = (c_hi + c_lo)/2.0
            adv = self.cw(input_xi, label_or_target, c_mid, TARGETED)
            if adv is None:
                c_hi = c_mid
                #print(c_mid)
            else:
                dis = torch.norm(adv).data
                dis_a.append(dis)
                print(dis)
                c_lo = c_mid
        return adv


from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers import pipeline
import torch
import gc
import time

from urllib import response
import requests
import json

start = time.time()

# device = "cuda" if torch.cuda.is_available() else "cpu"

USE_CUDA = torch.cuda.is_available()
device = torch.device('cuda:0' if USE_CUDA else 'cpu')
# gc.collect()
# torch.cuda.empty_cache()

# print('CUDA 사용 가능 여부 :', USE_CUDA)
# print('현재 사용 device :', device)
# print('CUDA Index :', torch.cuda.current_device())
# print('GPU 이름 :', torch.cuda.get_device_name())
# print('GPU 개수 :', torch.cuda.device_count())

# tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-5.8b")
# model = AutoModelForCausalLM.from_pretrained("EleutherAI/polyglot-ko-5.8b")
# tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-3.8b")
# model = AutoModelForCausalLM.from_pretrained("EleutherAI/polyglot-ko-3.8b")

# tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-1.3b",
#     bos_token='[BOS]', eos_token='[EOS]', unk_token='[UNK]', pad_token='[PAD]', mask_token='[MASK]')
# model = AutoModelForCausalLM.from_pretrained(
#     "EleutherAI/polyglot-ko-1.3b", torch_dtype='auto', low_cpu_mem_usage=True)

tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-3.8b",
                                          bos_token='[BOS]', eos_token='[EOS]', unk_token='[UNK]', pad_token='[PAD]',
                                          mask_token='[MASK]'
                                          )
model = AutoModelForCausalLM.from_pretrained(
    "EleutherAI/polyglot-ko-3.8b", torch_dtype='auto', low_cpu_mem_usage=True)

# tokenizer = AutoTokenizer.from_pretrained("EleutherAI/polyglot-ko-5.8b",
#                                           bos_token='[BOS]', eos_token='[EOS]', unk_token='[UNK]', pad_token='[PAD]',
#                                           mask_token='[MASK]'
#                                           )
# model = AutoModelForCausalLM.from_pretrained(
#     "EleutherAI/polyglot-ko-5.8b", torch_dtype='auto', low_cpu_mem_usage=True)

# prompt = "GPTNeoX20B is a 20B-parameter autoregressive Transformer model developed by EleutherAI."

# input_ids = tokenizer(prompt, return_tensors="pt").input_ids
# input_ids = tokenizer(prompt, return_tensors="pt").to(device).input_ids
# .input_ids는 리턴되는 객체(=튜플)의 요소중 하나.
# inputs_ids = tokenizer(prompt, return_tensors="pt").to(device)
# model = model.to(device)
# outputs = model(inputs_ids)
# prediction_logits = outputs.logits

# gen_tokens = model.generate(
#     input_ids,
#     do_sample=True,
#     temperature=0.9,
#     max_length=150,
# )
# gen_text = tokenizer.batch_decode(gen_tokens)[0]

# 바로 위의 코드들을 pipeline()메소드로 축약시킨 코드가 가능
pipe = pipeline(
    'text-generation',
    model=model,
    tokenizer=tokenizer,
    device=0,
    framework='pt'
)
prompt1 = """\
The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and very friendly.

Human: 오늘 날씨 어때?
AI: 좋습니다. 당신의 하루는 오늘 어땠나요?
Human: 난 오늘 꽤 괜찮았지~
AI: 그렇군요! 모르는 것 있으면 물어보세요. 친절히 알려드리겠습니다."""

# text = pipe(prompt, do_sample=True,  pad_token_id=50256,
#             num_return_sequences=3, top_k=50, top_p=0.95, max_new_tokens=50
#             )
# num_beams=4, no_repeat_ngram_size=2, temperature=0.75, early_stopping=True,
# max_length=50,
# print(text)
# time.sleep(0.9)
# end = time.time()
# print("Time consumed in working: ", end - start)

# text = pipe("아인슈타인은", do_sample=True, max_length=50, pad_token_id=50255,
#             num_return_sequences=2, top_k=50, top_p=0.95
#             )
# print(text)
# time.sleep(0.9)
# end = time.time()
# print("Time consumed in working: ", end - start)

# 텍스트 생성 기능위주인듯한데... 대화 기능을 찾아보자..

# print(f"gen_text: {pipe}")
# print(f"gen_text: {inputs_ids['input_ids']}")
# print(f"gen_text: {inputs_ids}")
# print(f"gen_text: {prediction_logits}")
# print(f"gen_text: {outputs}")

# Only recent response text will return
def processing_response(response_text: str ) -> str:
    ret_text = ""
    for i in range(0, len(response_text)):
        if response_text[i] == "\n" or response_text[i:i + 7] == "Human: " or response_text[i:i + 4] == "AI: ":
            break
        ret_text += response_text[i]
    return ret_text.strip()

url = "https://eleuther-ai-gpt-j-6b-float16-text-generation-api-ainize-team.endpoint.ainize.ai/predictions/text-generation"
# base_prompt = get_base_prompt()
base_prompt = prompt1
headers = {"Content-Type": "application/json"}
# End chat when user types "exit"
while True:
    chat = input("Human: ")
    if chat == "exit":
        break
    prompt = f"{base_prompt}\nHuman: {chat}\nAI:"

    # data == string type dumps()는 json객체를 json문자열로 변환시켜줌.
    data = json.dumps({
        "text_inputs":prompt,
        "temperature":0.9,
        "top_p":0.95,
        "repetition_penalty":0.8,
        "do_sample":True,
        "top_k":50,
        "length":50,
    })
    # loads()는 json문자열을 json객체로 변환시켜줌.
    jo = json.loads(data)
    # text = pipe(
    #     jo['text_inputs'],
    #     do_sample=jo['do_sample'],
    #     temperature=jo['temperature'],
    #     top_p=jo['top_p'],
    #     top_k=jo['top_k'],
    #     max_new_tokens=jo['length'],
    #     repetition_penalty=jo['repetition_penalty'],
    #     pad_token_id=50256
    # )
    text = pipe(
        prompt,
        do_sample=True,
        temperature=0.9,
        top_p=0.95,
        top_k=50,
        max_new_tokens=40,
        repetition_penalty=0.8,
        pad_token_id=50256
    )

    # res = requests.post(url, headers=headers, data=data)
    # if res.status_code == 200:
    # response_text = res.json()[0] # response에서 json응답일경우 바로 딕셔너리로 변환하고 0이라는 키의 값을 뽑아냄
    print(text[0])
    response_text = text[0]['generated_text']
    print("AI:", processing_response(response_text[len(prompt):]))


import datetime as dt
from dataclasses import dataclass
from typing import List, Tuple, Callable, Optional

from sova.asset import AbstractAsset, Deposit, Stock

@dataclass
class Trade:
    trade_time: dt.datetime
    stock_price: float
    asset: AbstractAsset

class Portfolio:
    def __init__(self, trades: List[Trade], delta_threshold: float = .005):
        self.trades = trades
        self.delta_threshold = delta_threshold

    def trade(self, trade_time: dt.datetime, stock_price: float, asset: AbstractAsset):
        self.trades.append(Trade(trade_time, stock_price, asset))

    def _apply_function(self, market_time: dt.datetime, f: Callable[[Trade], float]) -> float:
        return sum(map(f, filter(lambda t: t.trade_time <= market_time, self.trades)))

    def current_price(self,
                      stock_price: float,
                      market_time: dt.datetime,
                      risk_free_rate: float = .01) -> float:
        return self._apply_function(market_time, lambda t: t.asset.current_price(stock_price=stock_price,
                                                                                market_time=market_time,
                                                                                risk_free_rate=risk_free_rate))

    def current_delta(self,
                      stock_price: float,
                      market_time: dt.datetime,
                      risk_free_rate: float = .01) -> float:
        return self._apply_function(market_time, lambda t: t.asset.current_delta(stock_price=stock_price,
                                                                                market_time=market_time,
                                                                                risk_free_rate=risk_free_rate))

    def hedge_delta(self,
                    stock_price: float,
                    market_time: dt.datetime,
                    risk_free_rate: float = .01) -> Optional[Tuple[AbstractAsset, AbstractAsset]]:
        delta = self.current_delta(stock_price, market_time, risk_free_rate)
        if abs(delta) > self.delta_threshold:
            hedge_deposit = Trade(trade_time=market_time,
                                  stock_price=stock_price,
                                  asset=Deposit(delta * stock_price))
            hedge_asset = Trade(trade_time=market_time,
                                stock_price=stock_price,
                                asset=Stock(-1 * delta))
            self.trades.append(hedge_deposit)
            self.trades.append(hedge_asset)
            return hedge_deposit.asset, hedge_asset.asset

        return Deposit(0), Stock(0)


"""
Task
    Given an integer array nums sorted in non-decreasing order,
    return an array of the squares of each number sorted in non-decreasing order.

Constraints
    1 <= nums.length <= 104
    -104 <= nums[i] <= 104
    nums is sorted in non-decreasing order.
"""

from typing import List

class Solution:

    # O(n) when mirrored sorted list.
    def stage_sort(self, nums: List[int]) -> List[int]:
        lst = [] * len(nums)
        # index to start at left of list
        left_index: int = 0
        #  index tp start at right of list
        right_index: int = len(nums) -1
        # until the left and right index are eq or pass eachother
        while left_index <= right_index:
            # if the left value is greater than the right
            if nums[left_index] > nums[right_index]:
                # add to return list
                lst.append(nums[left_index])
                # increment left
                left_index += 1
            # if the right value is greater than the left
            else:
                # add to return list
                lst.append(nums[right_index])
                # decrement right
                right_index -= 1
        # return a reverse splice of our list
        return lst[::-1]

    def sortedSquares(self, nums: List[int]) -> List[int]:
        self.stage_sort(nums=[num ** 2 for num in nums])

print(Solution().sortedSquares([x * x for x in range(-10,11)]))


# %%
import pandas as pd
confirmedData = pd.read_csv("DXYvisualize/confirmedData.csv", index_col=0)
curedData = pd.read_csv("DXYvisualize/curedData.csv", index_col=0)
deadData = pd.read_csv("DXYvisualize/deadData.csv", index_col=0)

def getIncr(data: pd.DataFrame)->pd.DataFrame:
    incr = data.diff()
    incr.iloc[0] = data.iloc[0]
    return incr

confirmedIncr = getIncr(confirmedData)
curedIncr = getIncr(curedData)
deadIncr = getIncr(deadData)

# %%
def getRolling(data: pd.DataFrame)->pd.DataFrame:
    return data.rolling(7, center=True, min_periods=0).mean()

confirmedIncrRolling = getRolling(confirmedIncr)
curedIncrRolling = getRolling(curedIncr)
deadIncrRolling = getRolling(deadIncr)
# %%
totalData = pd.DataFrame(columns=["updateTime","confirmedIncr","curedIncr","deadIncr","provinceName"])
for province in confirmedData.columns.values:
    # if province=="湖北":continue
    provinceData = pd.DataFrame(data = confirmedData.index,columns=["updateTime"])
    # print( confirmedIncrRolling[province])
    provinceData.insert(1, 'provinceName', [province]*len(confirmedData.index))
    provinceData.insert(2, 'confirmedIncr', confirmedIncrRolling[province].values)
    provinceData.insert(3, 'curedIncr', curedIncrRolling[province].values)
    provinceData.insert(4, 'deadIncr', deadIncrRolling[province].values)
    totalData = pd.concat([totalData, provinceData])
totalData = totalData.reset_index(drop=True)
# %%
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

plt.figure(figsize=(20,20))
ax = plt.subplot(projection="3d")
ax.set_title("")
ax.scatter(totalData["confirmedIncr"],totalData["curedIncr"],totalData["deadIncr"],c='r')
ax.set_xlabel("感染")
ax.set_ylabel("治愈")
ax.set_zlabel("死亡")
plt.savefig("3D.png")
plt.show()
# %%


#!/usr/bin/env python

import os
import rospy
import rospkg
import lewansoul_xarm

def shutdown():
    print('\nshutting down xArm...')
    global controller
    controller.shutdown()

def ros_main():
    # connect to xArm
    global controller # so we can shutdown
    controller = lewansoul_xarm.controller()

    rospack = rospkg.RosPack()
    pkg_path = rospack.get_path('lewansoul_xarm')

    urdf_file = pkg_path + '/urdf/xArm.urdf'
    if not os.path.isfile(urdf_file):
        print('couldn\'t find file: ' + urdf_file + '. cartesian moves will be disabled')
        urdf_file = ''
    arm = controller.add_arm("arm", "xArm-49770F673737-arm.json", urdf_file)
    arm.enable()
    arm.home()

    gripper = controller.add_arm("gripper", "xArm-49770F673737-gripper.json")
    gripper.enable()
    gripper.home()

    rospy.init_node('xArm_ros', anonymous = True)
    rospy.on_shutdown(shutdown)

    controller_ros = lewansoul_xarm.controller_ros(controller)
    controller_ros.loop(30) # 30 Hz seems to be the maximum the controller can perform

if __name__ == '__main__':
    try:
        ros_main()
    except rospy.ROSInterruptException:
        pass


#!/usr/bin/python
import alsaaudio
import argparse
import rospy
import signal
import struct
import time
from audio_common_msgs.msg import AudioData #, AudioInfo
from Configuration import ConfigurationSection
from Serialization import Serialization
from UdpServer import UdpServer

class AudioCapture:
    DEFAULT_UDP_IDENTIFIER  = 0x19AB0A7D
    DEFAULT_UDP_LISTEN_PORT = 9562
    DEFAULT_UDP_REPLY_PORT  = 9562

    def __init__( self, params = None ):
        self._udpCommunicator = None
        self._udpIdentifier = AudioCapture.DEFAULT_UDP_IDENTIFIER
        self._udpListenPort = AudioCapture.DEFAULT_UDP_LISTEN_PORT
        self._udpReplyPort = AudioCapture.DEFAULT_UDP_REPLY_PORT
        self._audioData = None
        self._audioInfo = None
        try:
            enable = 0
            rosAudioCaptureName = "/audio"
            rosAudioCapInfoName = "/audio_info"
            if params is not None:
                enable = int( params.get( 'enable', enable ) )
                rosAudioCaptureName = str( params.get( 'rosAudioCaptureName', rosAudioCaptureName ) )
                rosAudioCapInfoName = str( params.get( 'rosAudioInfoName', rosAudioCapInfoName ) )
                self._udpIdentifier = int( params.get( 'udpIdentifier', self._udpIdentifier ) )
                self._udpListenPort = int( params.get( 'udpListenPort', self._udpListenPort ) )
                self._udpReplyPort = int( params.get( 'udpReplyPort', self._udpReplyPort ) )
            if enable != 0:
                rospy.init_node( "SS_AudioCapture", anonymous = True )
                self._audioData = rospy.Subscriber( rosAudioCaptureName, AudioData, self.onAudioDataReceived )
                #self._audioInfo = rospy.Subscriber( rosAudioCapInfoName, AudioInfo, self.onAudioInfoReceived )
                self._udpCommunicator = UdpServer( self._udpListenPort, self._udpReplyPort, "AudioCapture" )
                self._udpCommunicator.setCommunicationWaitTime( 11 )
        except Exception as ex:
            print( "[AudioCapture] Error: %s" % ex )
    #END __init__( )

    def __del__( self ):
        if self._audioData is not None:
            self._audioData.unregister( )
            self._audioData = None
        if self._audioInfo is not None:
            self._audioInfo.unregister( )
            self._audioInfo = None
        self._udpCommunicator = None
    #END __del__( )

    def threadUpdateInterval( self ):
        return 100 # 0.1 miliseconds
    #END threadUpdateInterval( )

    def onAudioDataReceived( self, data ):
        packet = struct.pack( "<I", self._udpIdentifier ) + Serialization.packAudioData( data )
        self._udpCommunicator.broadcast( packet )
    #END onAudioDataReceived( )

    def onAudioInfoReceived( self, data ):
        packet = struct.pack( "<I", self._udpIdentifier ) + Serialization.packAudioInfo( data )
        self._udpCommunicator.broadcast( packet )
    #END onAudioInfoReceived( )

    def update( self ):
        if self._udpCommunicator is None:
            return

        self._udpCommunicator.update( )
        packet, client = self._udpCommunicator.receive( )
        while packet is not None:
            if len( packet ) != 4 or struct.unpack( "<I", packet )[0] != self._udpIdentifier:
                # unexpected packet data -> disconnect
                client.disconnect( )
            packet, client = self._udpCommunicator.receive( )
    #END update( )
#END AudioCapture

def workerSignalHandler( signal, frame ):
    global threadRunning
    threadRunning = False
#END workerSignalHandler( )

def workerMain( ):
    global threadRunning
    threadRunning = True

    signal.signal( signal.SIGABRT, workerSignalHandler )
    signal.signal( signal.SIGHUP, workerSignalHandler )
    signal.signal( signal.SIGINT, workerSignalHandler )
    signal.signal( signal.SIGTERM, workerSignalHandler )

    arg_fmt = argparse.ArgumentDefaultsHelpFormatter
    parser = argparse.ArgumentParser( formatter_class=arg_fmt )
    parser.add_argument( "-a", "--audio-capture", dest="rosAudioCaptureName", default="/audio",      type=str, help="ROS audio capture topic name" )
    parser.add_argument( "-i", "--audio-info",    dest="rosAudioCapInfoName", default="/audio_info", type=str, help="ROS audio information topic name" )
    parser.add_argument( "-ui", "--udp-identifier",  dest="udpIdentifier", default=AudioCapture.DEFAULT_UDP_IDENTIFIER,  type=int, help="Connection identifier for the UDP communication" )
    parser.add_argument( "-up", "--udp-listen-port", dest="udpListenPort", default=AudioCapture.DEFAULT_UDP_LISTEN_PORT, type=int, help="Listening port for the UDP communication" )
    parser.add_argument( "-ur", "--udp-reply-port",  dest="udpReplyPort",  default=AudioCapture.DEFAULT_UDP_REPLY_PORT,  type=int, help="Reply port for the UDP communication" )
    args = parser.parse_args( rospy.myargv( )[1:] )

    params = ConfigurationSection( )
    params.add( 'enable', 1 )
    params.add( 'rosAudioCaptureName', args.rosAudioCaptureName )
    params.add( 'rosAudioInfoName', args.rosAudioCapInfoName )
    params.add( 'udpIdentifier', args.udpIdentifier )
    params.add( 'udpListenPort', args.udpListenPort )
    params.add( 'udpReplyPort', args.udpReplyPort )

    handler = AudioCapture( params )
    lastUpdated = time.time( ) * 1000000.0
    while threadRunning:
        now = time.time( ) * 1000000.0
        elapsed = now - lastUpdated
        diff = handler.threadUpdateInterval( ) - elapsed
        if diff <= 0.0:
            lastUpdated = now
            handler.update( )
            time.sleep( 0 )
        else:
            time.sleep( diff / 4000000 )
#END workerMain( )

if __name__ == '__main__':
    workerMain( )
#END if-else


#!/usr/bin/pythonRoot3.5
#----------------------------------------------------------------
# Dateiname:  inter_webcam.py
# CGI-Skript fuer interaktive Webcam
#
# Michael Weigend
# Raspberry Pi programmieren mit Python, 4. Auflage, mitp 2018
# Kap. 11
# Michael Weigend 20. September 2018
#--------------------------------------------------------------

import cgi, cgitb, os, subprocess, io
from PIL import Image

PATTERN='''Content-Type: text/html

        <title>Interaktive Webcam</title>

 Interaktive Webcam

   Wählen Sie den Bildausschnitt!
   <form action="http://localhost/cgi-bin/inter_webcam.py"
     method="GET">
      <input type="Radio" name="zoom" value="weit" {}/>
           Weit entfernt

      <input type="Radio" name="zoom" value="naeher" {}/>
           Näher

      <input type="Radio" name="zoom" value="nah" {}/>
          Nah

      <input type="Submit" value="Bild anfordern"/>
    </form>

'''                                            #1

PATH = 'image.jpg'
COMMAND = 'raspistill -t 500 -w {}  -h {} -o {} -n '                                #2

def takePhoto (width, height, bbox):
    imageData = io.BytesIO()
    command = COMMAND.format(width, height,'-')               #3
    try:
        imageData.write(subprocess.check_output(command, shell=True))                                          #4
        imageData.seek(0)
        image = Image.open(imageData).crop(bbox)          #5
        image.save(PATH)
    except: pass

form = cgi.FieldStorage()
zoom = form.getvalue('zoom', 'nah')
if zoom == 'nah':                                     #6
    takePhoto(1600, 1200, (600, 450, 1000, 750))
    response = PATTERN.format('', '', 'checked="checked"')
elif zoom == 'naeher':
    takePhoto(800, 600, (200, 150, 600, 450))         #7
    response = PATTERN.format('', 'checked="checked"', '')
else:
    takePhoto(400, 300, (0, 0, 400, 300))             #8
    response = PATTERN.fortmat('checked="checked"', '', '')

print(response)


import csv
import json
import re

# It tries to get the links for the coverage badges in the repos README.md

services = [
    'coveralls',
    'codecov',
    'codeclimate',
    'cov'
]

def get_badge_urls(text):
    result = set()
    matched = re.findall(r'(?:http\:|https\:)\/\/.*', text)

    for match in matched:
        for service in services:
            if service in match:
                result.add(match)
                break

    return list(result)

def get_badge_images(text):
    result = set()
    matched = re.findall(r'(?:http\:|https\:)\/\/.*\.(?:png|jpg|svg)', text)

    for match in matched:
        for service in services:
            if service in match:
                result.add(match)
                break

    return list(result)

f = open('data/cov_traces.json')
traces = json.load(f)

badges = []

for repo in traces:
    links = []
    images = []

    for line in traces[repo]:
        links.extend(get_badge_urls(line))
        images.extend(get_badge_images(line))

    if not (links == [] and images == []):
        badges.append({
            'name': repo,
            'links': links,
            'images': images
        })

with open('data/badges.csv', 'w', encoding='UTF8', newline='') as f:
    writer = csv.DictWriter(f, fieldnames=['name', 'links', 'images'])
    writer.writeheader()
    writer.writerows(badges)

f.close()