import novelpy
import tqdm

for focal_year in tqdm.tqdm(range(2000,2016), desc = "Computing indicator for window of time"):
    Lee = novelpy.indicators.Lee2015(collection_name = "references",
                                           id_variable = 'PMID',
                                           year_variable = 'year',
                                           variable = "c04_referencelist",
                                           sub_variable = "item",
                                           focal_year = focal_year)
    Lee.get_indicator()


from flask import Flask, json
from flask_restful import Resource, Api, reqparse

companies = []

api = Flask(__name__)

@api.route('/companies', methods=['GET'])
def get_companies():
    return json.dumps(companies)

@api.route('/companies', methods=['POST'])
def post_companies():
    parser = reqparse.RequestParser()
    parser.add_argument('id', required=True)
    parser.add_argument('name', required=False)

    args = parser.parse_args()

    new_company = {"id":args['id'], "name":args['name']}

    companies.append(new_company)
    return json.dumps({"succuessfully added company to list":True}), 201

@api.route('/companies', methods=['DELETE'])
def delete_companies():

    parser = reqparse.RequestParser()
    parser.add_argument('id', required=True)
    args = parser.parse_args()

    for i in range(len(companies)):
        if companies[i]['id'] == args['id']:
            del companies[i]
            break

    return json.dumps({"succuessfully deleted the company the list":True}), 201

api.run()


from bs4 import BeautifulSoup

#通过解析本地文件 来将bs4的基础语法进行讲解

#加载本地文件  默认打开文件的编码格式是gbk  所以在打开的时候要指定编码
# soup = BeautifulSoup(open('bs4的基本使用.html', encoding='utf-8'),'lxml')
# print(soup)

#根据标签的名字来查找节点
#例如  查找标签为a的节点
#  此处查找到的是第一个标签为a的节点
# soup = BeautifulSoup(open('bs4的基本使用.html', encoding='utf-8'),'lxml')
#print(soup.a)

#查找到的是第一个标签为a的属性和属性值
# soup = BeautifulSoup(open('bs4的基本使用.html', encoding='utf-8'),'lxml')
# print(soup.a.attrs)

#bs4 的一些函数
# 1）   find
# soup = BeautifulSoup(open('bs4的基本使用.html', encoding='utf-8'),'lxml')
#返回的是第一个符合条件的数据
#print(soup.find('a'))

#根据属性的值来找到对应的标签对象
# print(soup.find('a', id='222'))

#如果想查找属性为class的值 不可以直接查找class  因为class已经被定义  如果想查找class，可以将class改成class_
# print(soup.find('a', class_="a1"))

# 2)   find_all
soup = BeautifulSoup(open('bs4的基本使用.html', encoding='utf-8'),'lxml')

#find_all可查看到所有标签 并返回一个列表  并返回所有为a的标签
# print(soup.find_all('a'))

#查找所有a标签和span标签，注意 需要在find_all的参数中添加的是列表的数据
# print(soup.find_all(['a', 'span']))

#查找前两个标签为li的数据
# print(soup.find_all('li', limit=2))

# 3)    select

# select 方法返回的数一个列表  并且会返回多个数据
# print(soup.select('a'))

# 可以用.代表class  我们把这种操作叫做类选择器
# print(soup.select('.a1'))

# id  #可以代表id  例如  #l1可以查找id为l1的标签
# print(soup.select('#l1'))

#属性选择器   通过属性来寻找对应的标签
# 查找到li标签中有id的标签
# print(soup.select('li[id]'))

#层级选择器      空格  大于号  逗号

# 找div下面的li   后代选择器  可以利用 空格
# print(soup.select('div li'))

#子代选择器  查找某标签的第一级子标签
#注意  很多计算机编程语言中，如果不加空格不会输出内容，在bs4中不会报错，会显示内容
# print(soup.select('div > ul > li'))

#找到a标签和li标签的所有对象
# print(soup.select('a,li'))

#节点信息
#1 获取节点内容
# 如果标签对象中，除了内容还有标签  那么string就获取不到数据  而get_text()可以获取数据  所以推荐使用后者
# obj = soup.select('#d1')[0]  # 返回的是一个列表的值
# print(obj)
# print(obj.get_text())

#2节点的属性
# 通过id来获取p标签
obj = soup.select('#p1')[0]

#name 返回的是标签的名字
# print(obj.name)

# attrs  将属性值作为一个字典返回
# print(obj.attrs)

# 获取节点的属性
obj = soup.select('#p1')[0]
#方式1  print(obj.attrs.get('class'))
#方式2  print(obj.get('class'))
#方式3  print(obj['class'])


from mycroft import MycroftSkill, intent_handler
from adapt.intent import IntentBuilder
import requests
import json

class RedditNews(MycroftSkill):
    def __init__(self):
        MycroftSkill.__init__(self)

    @intent_handler(IntentBuilder("").optionally("Pre").require("News").require("Reddit"))
    def handle_news_reddit(self, message):
        r = requests.get("https://reddit.com/r/worldnews/hot/.json", headers={"User-Agent": "mycroft:reddit-news-skill:v0.1 (by /u/makeworld)"})
        data = r.json()
        headlines = ""
        for post in data["data"]["children"][:5]:  # First five posts
            headline = post["data"]["title"]
            if headline[-1:] != ".":  # Add a period at the end to make mycroft pause
                headline += "."
            # Add a space at the end, because all headlines will be in one string
            headline += " "
            # Add it to the string of headlines
            headlines += headline
        self.speak_dialog('here.news', data={"headlines": headlines})

def create_skill():
    return RedditNews()


from collections import deque

t = int(input())

for i in range(t):
    H, W = map(int, input().split())

    graph = []
    for j in range(H):
        graph.append(list(map(str, input())))

    visited = [[False] * W for _ in range(H)]

    dx = [-1, 0, 1, 0]
    dy = [0, -1, 0, 1]

    q = deque()

    def bfs(h, w):
        q.append([h, w])
        visited[h][w] = True
        while q:
            y, x = q.popleft()
            for i in range(4):
                nx = x + dx[i]
                ny = y + dy[i]
                if nx >= 0 and nx < W and ny >= 0 and ny < H:
                    if not visited[ny][nx] and graph[ny][nx] == '#':
                        visited[ny][nx] = True
                        q.append([ny, nx])
    result = 0

    for i in range(H):
        for j in range(W):
            if not visited[i][j] and graph[i][j] == '#':
                bfs(i, j)
                result += 1

    print(result)


import numpy as np
import scipy.optimize
import matplotlib.pyplot as plt

def basin_entropy(w):
    # use masked array (ma) to avoid problems with zero
    return -np.dot(w,np.ma.log2(w).filled(0.))

def min_basin_entropy(r,n):
    return (r-1)*n/2**n -(1-(r-1)/2**n)*np.log2((1-(r-1)/2**n))

def max_basin_entropy(r):
    """
    Note: This does not include a restriction due to having an
    integer number of states in each basin.  This could produce
    a bug in sampling near to the maximum entropy.
    """
    return np.log2(r)

def entropy_to_relative_basin_sizes(h_tilde,r,n,tol=1e-3):

    if h_tilde > max_basin_entropy(r):
        raise ValueError("Basin entropy cannot be larger than log2(r)")

    if h_tilde < min_basin_entropy(r,n):
        raise ValueError("Basin entropy cannot be below h_tilde_min = "+str(min_basin_entropy(r,n)))

    func = lambda w: (basin_entropy(w) - h_tilde)**2
    w0 = [ (i+1)/(r*(r+1)/2) for i in range(r) ]
    normConstraint = scipy.optimize.LinearConstraint(np.ones(r),lb=1.,ub=1.)
    bound = scipy.optimize.Bounds(lb=1./2**n,ub=1.)
    solution = scipy.optimize.minimize(func,w0,constraints=normConstraint,bounds=bound)
    if solution.success and np.sqrt(solution.fun) < tol:
        return solution.x
    else:
        print("entropy_to_relative_basin_sizes: ERROR: Solution not found within given tolerance.  Returning empty array.")
        return np.array([])

def entropy_to_basin_sizes(h_tilde,r,n,tol=1e-3):
    """
    To do: Add checks to this!!!
    """
    w = entropy_to_relative_basin_sizes(h_tilde,r,n,tol=tol)
    if len(w) > 0:
        w_int = [round(w[_]*2**n) for _ in range(len(w))]
        w_int[-1] = w_int[-1] +  2**n - np.sum(w_int)
        return np.array(w_int)
    else:
        return np.array([])


# Provide path to Dataset3.csv
path = 'C:/Users/karme/Documents/MS_IITM/Courses/CS5691/PRML_assignment1_2020/PRML_assignment1_2020/Dataset3.csv'
# Provide path to save plots
plot_save_path = 'C:/Users/karme/Desktop/'

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns; sns.set();

# Kernelization happens here
def generate_kernels(X_input, ker_name='poly', degree=2, sigma=0.1):
  if ker_name == 'poly':
    K = np.dot(X_input.T, X_input)
    K += 1
    K **= degree
  elif ker_name == 'Gaussian':
    X_sum = np.sum(X_input ** 2, axis=0)
    K = X_sum[:, None] + X_sum[None, :] - 2 * np.dot(X_input.T, X_input)

    K *= -0.5/(sigma*sigma)
    K = np.exp(K)
  return K

def centering_kernel(K):
  num_samples = K.shape[0]
  num_samples_const = 1/num_samples

  K_col_avg = np.sum(K, axis=0) / num_samples
  K_row_avg = (np.sum(K, axis=1) / num_samples)[:, np.newaxis]
  K_total_avg = np.sum(K_col_avg)/ num_samples
  Gram_K = K + K_total_avg - K_col_avg - K_row_avg
  return Gram_K

def kernel_PCA(X_input, num_components, kernel_name='poly', degree=2, sigma=0.1):
  # Generating and centering kernel matrix K
  K = generate_kernels(X_input, ker_name=kernel_name, degree=degree, sigma=sigma)
  K = centering_kernel(K)

  # Calculating eigen values and eigen vectors of kernel matrix K
  eigvals, eigvecs = np.linalg.eigh(K)

  #Sorting and filtering eigenvectors based on eigenvalues
  idx = eigvals.argsort()[::-1]
  eigvals = eigvals[idx][:num_components]
  eigvecs = np.atleast_1d(eigvecs[:, idx])[:, :num_components]

  # Normalizing eigenvalues and eigen vectors
  eigvals = 1/np.sqrt(1000*eigvals)
  eigvals = eigvals.reshape((len(eigvals), 1))
  normalized_eigvals = np.diagflat(eigvals)
  normalized_eigenvecs = np.dot(eigvecs, normalized_eigvals)

  # Projecting datapoints into feature space
  X_projected = np.dot(K, normalized_eigenvecs)
  return X_projected, normalized_eigenvecs, normalized_eigvals

def fetch_data_from_excel(file_path):
  my_data = np.genfromtxt(file_path, delimiter=',')
  my_data = np.transpose(my_data)
  return my_data

def plot_graphs(X_projected, j, k, sigma, poly_degree):
  sns.set();

  if sigma > 0 and poly_degree == 0:
    ax2[j, k].scatter(X_projected[:, 0], X_projected[:, 1], c='red', s=15)
    ax2[j, k].set_title(r"Projection by KPCA in feature space for sigma=%0.1f" %sigma)
    ax2[j, k].set_xlabel("1st PC")
    ax2[j, k].set_ylabel("2nd PC")
  elif sigma == 0 and poly_degree > 0:
    ax1[j].scatter(X_projected[:, 0], X_projected[:, 1])
    ax1[j].set_title(r"Projection by KPCA in feature space for d=%i" %poly_degree)
    ax1[j].set_xlabel("1st PC")
    ax1[j].set_ylabel("2nd PC")

# Fetch data from excel file
X = fetch_data_from_excel(path)

# Computing and plotting Polynomial kernel
max_components = 2
fig1, ax1 = plt.subplots(1, 2, figsize=(20, 5))
degrees = [2, 3]
j = 0
k = 0
for i in range(len(degrees)):
  X_projected, principal_components, variances = kernel_PCA(X, max_components, kernel_name='poly', degree=degrees[i], sigma=0.1)
  plot_graphs(X_projected, j, k, 0, degrees[i])
  j += 1

sns.set();
plt.savefig(plot_save_path +'Q3a_polynomial_kernel_plots_CS20S020_Assignment1_CS5691.png')

# Computing and plotting Gaussian kernel
max_components = 2
fig2, ax2 = plt.subplots(5, 2, figsize=(25, 30))
sigmas = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
j = -1
k = 0
for i in range(len(sigmas)):
  X_projected, principal_components, variances = kernel_PCA(X, max_components, kernel_name='Gaussian', degree=3, sigma=sigmas[i])
  j += 1
  plot_graphs(X_projected, j, k, sigmas[i], 0)
  if i == 4:
    j = -1
    k = 1

sns.set();
plt.savefig(plot_save_path +'Q3b_gaussian_kernel_plots_CS20S020_Assignment1_CS5691.png')


# -*- coding: utf-8 -*-
# Advanced Error Recovery during Bottom-Up Parsing
# File: views.py
# Author: Dominika Regeciova, xregec00@stud.fit.vutbr.cz

from django.shortcuts import render
from django.template import RequestContext
from django.http import HttpResponse
from .models import Nonterminal, Terminal, Rule
from .scanner import Scanner
from .parser import Parser
from .panic_mode import PanicModeParser
from .panic_mode_first import PanicModeParserFirst
from .ad_hoc import ParserAdHoc
from .alan_method import AlanMethodParser
from .lrtable import LRTable
from .forms import UploadFileForm
import os
import zipfile
import time

def get_grammar():
    """ Returns list of grammar rules order by id (number)  """
    grammar = Rule.objects.order_by('id')
    grammar_list = []
    grammar_list.append({'id': 0, 'left': '', 'right': ''})
    for g in grammar:
        grammar_list.append({'id': g.id, 'left': g.left_hand_side,
                             'right': g.right_hand_side})
    return grammar_list

def about(request):
    """ Render of about page """
    return render(request, 'alan/about.html', {})

def man(request):
    """ Render of man page """
    return render(request, 'alan/man.html', {})

def changelog(request):
    """ Render of changelog page """
    return render(request, 'alan/changelog.html', {})

def report(request):
    with open('./report.pdf', 'rb') as pdf:
        response = HttpResponse(pdf.read(), content_type='application/pdf')
        response['Content-Disposition'] = 'inline;filename=some_file.pdf'
        return response
    pdf.closed

def index(request):
    """ The page to obtain input for the scanner """
    # Initialization of input
    if 'source_code' in request.session:
        code = request.session['source_code']
    else:
        code = ''
    # Upload of file
    if request.method == 'POST':
        form = UploadFileForm(request.POST, request.FILES)
        if form.is_valid():
            code = ''
            path = (request.FILES['docfile'])
            for chunk in path.chunks():
                code = code + str(chunk.decode('utf-8', 'ignore'))
    else:
        form = UploadFileForm()
    return render(request, 'alan/index.html', {'code': code, 'form': form},
                  context_instance=RequestContext(request))

def run_scanner(request):
    """ This view is called by button 'Spustit lexikalni analyzu'
        in index.html and return results of Lexical analysis """
    # Initialization
    scanner = Scanner()
    tokens = []
    source_code = ''
    exit_code = rows = 0
    # Run of Lexical analysis
    if request.method == 'POST':
        if 'fun_code_area' in request.POST and request.POST['fun_code_area']:
            source_code = request.POST['fun_code_area']
            request.session['source_code'] = source_code
            tokens, exit_code = scanner.scanner_analysis(source_code)
            request.session['tokens'] = tokens
            rows = len(tokens)
    return render(request, 'alan/scanner.html', {'source_code': source_code,
                  'tokens': tokens, 'exit_code': exit_code, 'rows':rows})

def run_parser(request):
    """ This view is called by button 'Spustit syntaktickou analyzu'
        in scanner.html and return results of basic syntax analysis """
    parser = Parser()
    source_code = request.session.get('source_code', '')
    tokens = request.session.get('tokens', '')
    parser_result = stack = state = []
    exit_code = rows = 0
    grammar_list = get_grammar()
    if request.method == 'POST':
        parser_result, stack, state, lex_input, exit_code = parser.parser_analysis(tokens, grammar_list)
        results = zip(stack, state, parser_result, lex_input)
        rows = len(parser_result)
    return render(request, 'alan/parser.html', {'source_code': source_code,
                  'tokens': tokens, 'exit_code': exit_code, 'results': results,
                  'rows':rows})

def run_panic_mode_parser(request):
    """ This view is called by button 'Spustit Panic mode'
        in parser.html and return results of the first advanced
        syntax analysis """
    parser = PanicModeParser()
    source_code = request.session.get('source_code', '')
    tokens = request.session.get('tokens', '')
    parser_result = stack = state = panic_mode = []
    exit_code = rows = 0
    grammar_list = get_grammar()
    if request.method == 'POST':
        parser_result, stack, state, exit_code, panic_mode, lex_input = parser.parser_analysis(tokens, grammar_list)
        results = zip(stack, state, parser_result, lex_input)
        rows = len(parser_result)
    return render(request, 'alan/panic_mode_parser.html', {
                  'source_code': source_code, 'tokens': tokens,
                  'exit_code': exit_code, 'panic_mode': panic_mode, 'results': results,
                  'rows':rows})

def run_panic_mode_parser_first(request):
    """ This view is called by button 'Spustit Panic mode'
        in parser.html and return results of the first advanced
        syntax analysis """
    parser = PanicModeParserFirst()
    source_code = request.session.get('source_code', '')
    tokens = request.session.get('tokens', '')
    parser_result = stack = state = panic_mode = []
    exit_code = rows = 0
    grammar_list = get_grammar()
    if request.method == 'POST':
        parser_result, stack, state, exit_code, panic_mode, lex_input = parser.parser_analysis(tokens, grammar_list)
        results = zip(stack, state, parser_result, lex_input)
        rows = len(parser_result)
    return render(request, 'alan/panic_mode_parser_first.html', {
                  'source_code': source_code, 'tokens': tokens,
                  'exit_code': exit_code, 'panic_mode': panic_mode, 'results': results,
                  'rows':rows})

def run_parser_ad_hoc(request):
    """ This view is called by button 'Spustit Ad-hoc'
        in parser.html and return results of the first advanced
        syntax analysis """
    parser = ParserAdHoc()
    source_code = request.session.get('source_code', '')
    tokens = request.session.get('tokens', '')
    parser_result = stack = state = []
    exit_code = rows = 0
    grammar_list = get_grammar()
    lrtable = LRTable()
    table_action, table_goto = lrtable.generate_ad_hoc_table_print()
    if request.method == 'POST':
        parser_result, stack, state, lex_input, exit_code = parser.parser_analysis(tokens, grammar_list)
        results = zip(stack, state, parser_result, lex_input)
        rows = len(parser_result)
    return render(request, 'alan/ad_hoc_parser.html', {
                  'source_code': source_code, 'tokens': tokens,
                  'results': results, 'exit_code': exit_code, 'rows':rows,
                  'table_action': table_action, 'table_goto': table_goto})

def run_alan_method_parser(request):
    """ This view is called by button 'Spustit Alan method'
        in parser.html and return results of the first advanced
        syntax analysis """
    parser = AlanMethodParser()
    source_code = request.session.get('source_code', '')
    tokens = request.session.get('tokens', '')
    parser_result = stack = state = alan_method = []
    exit_code = rows = 0
    grammar_list = get_grammar()
    if request.method == 'POST':
        parser_result, stack, state, exit_code, alan_method, lex_input = parser.parser_analysis(tokens, grammar_list)
        results = zip(stack, state, parser_result, lex_input)
        rows = len(parser_result)
    return render(request, 'alan/alan_method_parser.html', {
                  'source_code': source_code, 'tokens': tokens,
                  'exit_code': exit_code, 'alan_method': alan_method, 'results': results,
                  'rows':rows})

def comparison(request):
    return render(request, 'alan/comparison.html', {})

def grammar(request):
    grammar = Rule.objects.order_by('id')
    terminals = Terminal.objects.order_by('id')
    t_list = ', '.join([t.char for t in terminals])
    nonterminals = Nonterminal.objects.order_by('id')
    n_list = ', '.join([n.char for n in nonterminals])
    return render(request, 'alan/grammar.html', {'grammar': grammar,
                  'terminals': t_list, 'nonterminals': n_list})

def lrtable(request):
    lrtable = LRTable()
    table_action, table_goto = lrtable.generate_table_print()
    return render(request, 'alan/lrtable.html', {'table_action': table_action,
                  'table_goto': table_goto})

def ad_hoc_lrtable(request):
    lrtable = LRTable()
    table_action, table_goto = lrtable.generate_ad_hoc_table_print()
    return render(request, 'alan/ahlrtable.html', {'table_action': table_action,
                  'table_goto': table_goto})

def download(request):
    zipf = zipfile.ZipFile('alanfiles.zip', 'w')
    fpath = 'files/'
    for root, dirs, files in os.walk(fpath):
        for file in files:
            zipf.write(os.path.join(root, file))
    zipf.close()
    response = HttpResponse(open('alanfiles.zip', 'rb').read())
    response['Content-Type'] = 'application/x-zip-compressed'
    response['Content-Disposition'] = 'attachment; filename=alanfiles.zip'
    return response


# !/usr/bin/env python
# -- coding: utf-8 --
# @Author zengxiaohui
# Datatime:7/26/2021 10:10 AM
# @File:infer_net3_tensorrt3_tta.py
import os
os.environ["CUDA_HOME"]="/usr/local/cuda-10.2"
os.environ["TENSORRT_HOME"]="/TensorRT-7.2.2.3"
os.environ["LD_LIBRARY_PATH"]="/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/TensorRT-7.2.2.3/lib:/usr/local/cuda-10.2/lib:/usr/local/cuda-10.2/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
os.environ["PATH"]='/opt/conda/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3:/usr/local/cuda-10.2/bin:/TensorRT-7.2.2.3'
from ai_hub import inferServer
import json
from io import BytesIO
import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit
from pycuda.tools import make_default_context
cuda.init()  # Initialize CUDA
ctx = make_default_context()  # Create CUDA context
import numpy as np
import cv2
from PIL import Image
import base64
import albumentations as A
from albumentations.pytorch import ToTensorV2
from torch.nn import functional as F
import torch

class myInfer(inferServer):
    def allocate_buffers(self, engine):
        inputs = []
        outputs = []
        bindings = []
        stream = cuda.Stream()  # pycuda 操作缓冲区
        for binding in engine:
            size = trt.volume(engine.get_binding_shape(binding)) * engine.max_batch_size
            dtype = trt.nptype(engine.get_binding_dtype(binding))

            host_mem = cuda.pagelocked_empty(size, dtype)
            device_mem = cuda.mem_alloc(host_mem.nbytes)  # 分配内存
            bindings.append(int(device_mem))

            if engine.binding_is_input(binding):
                inputs.append(HostDeviceMem(host_mem, device_mem))
            else:
                outputs.append(HostDeviceMem(host_mem, device_mem))
        return inputs, outputs, bindings, stream

    def do_inference(self, context, bindings, inputs, outputs, stream, batch_size=1):
        self.cfx.push()

        [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in inputs]  # 将输入放入device
        context.execute_async(batch_size=batch_size, bindings=bindings, stream_handle=stream.handle)  # 执行模型推理
        [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in outputs]  # 将预测结果从缓冲区取出
        stream.synchronize()  # 线程同步
        return [out.host for out in outputs]

    def get_engine(self, engine_file_path=""):
        print("Reading engine from file {}".format(engine_file_path))
        with open(engine_file_path, "rb") as f, trt.Runtime(self.trt_logger) as runtime:
            return runtime.deserialize_cuda_engine(f.read())

    def _create_context(self):
        return self.engine.create_execution_context()

    def __init__(self, model=None):
        super().__init__(model)
        self.cfx = cuda.Device(0).make_context()
        self.trt_logger = trt.Logger()

        engine_file_path = '/user_data/model_data/checkpoint-best.engine'
        self.engine = self.get_engine(engine_file_path)
        self.context = self._create_context()

        self.inputs, self.outputs, self.bindings, self.stream = \
            self.allocate_buffers(self.engine)

    # 数据前处理
    def pre_process(self, data):
        json_data = json.loads(data.get_data().decode("utf-8"))
        img = json_data.get("img")
        bast64_data = img.encode(encoding="utf-8")
        img = base64.b64decode(bast64_data)
        img = cv2.imdecode(np.fromstring(img, dtype=np.uint8), -1)
        # img = np.expand_dims(img, 0).astype(np.float16)
        return img

    # 数据后处理
    def post_process(self, data):
        # data = data.squeeze().data.numpy()
        # data = data[0].reshape(10,256, 256)
        # data = np.argmax(data, axis=0)
        # data = data.squeeze()
        # data = data.squeeze().cpu().data.numpy()
        # data = data+1
        data[data == 0]=1
        img_encode = np.array(cv2.imencode('.png', data)[1]).tobytes()
        bast64_data = base64.b64encode(img_encode)
        bast64_str = str(bast64_data, 'utf-8')
        return bast64_str

    # 模型预测：默认执行self.model(preprocess_data)，一般不用重写
    # 如需自定义，可覆盖重写
    def predict(self, data):
        np.copyto(self.inputs[0].host, data.ravel())
        trt_outputs = self.do_inference(
            context=self.context,
            bindings=self.bindings,
            inputs=self.inputs,
            outputs=self.outputs,
            stream=self.stream)
        self.cfx.pop()
        data = trt_outputs[0].reshape(256, 256)
        return data

class HostDeviceMem(object):
    def __init__(self, host_mem, device_mem):
        super(HostDeviceMem, self).__init__()
        self.host = host_mem
        self.device = device_mem

    def __str__(self):
        return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)

    def __repr__(self):
        return self.__str__()

if __name__ == "__main__":
    # net1
    # model_name = 'resnet50'#'efficientnet-b6'
    # n_class=10
    # net1=seg_qyl(model_name,n_class)
    # # net1 = torch.nn.DataParallel(net1) # 如果使用了多卡训练那么需要加这句话
    # checkpoints=torch.load('/user_data/model_data/checkpoint-best.pth')
    # net1.load_state_dict(checkpoints['state_dict'])
    # net1.cuda()
    # net1.eval()

    my_infer = myInfer()
    my_infer.run(ip="0.0.0.0",debuge=False)  # 默认为("127.0.0.1", 8080)，可自定义端口，如用于天池大赛请默认即可，指定debuge=True可获得更多报错信息


from django.urls import path
from .import views

urlpatterns = [

    path(r'',views.Index,name = 'Index'),
    path(r'login/',views.login_User, name='logUser'),
    path(r'logout/',views.logout_view, name='logout_view'),

    path(r'register/',views.register_User, name='logUser'),
    path(r'chat/', views.chat, name='chat'),

    path(r'post/', views.Post, name='post'),
    path(r'messages/', views.Messages, name='messages'),

    path(r'get_help/',views.get_help, name='get_help'),
    path(r'about_depression/',views.about_depression, name='about_depression'),

    path(r'stories/',views.stories, name='stories'),

    path(r'about_depression/know_depression/',views.know_depression, name='know_depression'),
    path(r'about_depression/signs_of_depression/', views.signs_of_depression, name='learn_the_signs'),
    path(r'about_depression/how_to_help/',views.how_to_help, name='how_to_help'),
    path(r'contributors',views.contributors, name='contributors'),
    path(r'videoprofile',views.videoprofile, name='videoprofile'),
    path(r'volunteer',views.volunteer, name='volunteer'),

]


import os
import json
import glob
import pandas
from bs4 import BeautifulSoup
from url_normalize import url_normalize

# Constants
PAGES_PATH = "pages"
files = glob.glob(PAGES_PATH + '/**/*.json', recursive=True)
savepath = "works_urls.txt"

# Parse files
works_urls = set()
for i, file in enumerate(files):
    print(f"Parsing page {i+1}/{len(files)}")
    with open(file, "r") as f:
        try:
            # Load JSON and extract HTML
            r = json.load(f)

            # Check if there is something to parse
            html = r.get('Value')
            if html:
                # Parse HTML
                soup = BeautifulSoup(html, 'html.parser')

                # Get links to works
                tmp_works_urls = soup.select("div.mostrable.miniaturas > figure > figcaption.presentacion-listado > dl > dt > a")
                tmp_works_urls = [url_normalize(elem.attrs['href']) for elem in tmp_works_urls]
                works_urls.update(tmp_works_urls)

        except Exception as e:
            print("[EXCEPTION]: " + str(e))

    # if i==10:  # Debugging
    #     break

# Save URLs
with open(savepath, "w") as f:
    for line in works_urls:
        f.write(line + "\n")

print("")
print(f"{len(works_urls)} URLs were saved!")
print("done!")


from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
#To use By
from selenium.webdriver.common.by import By
#For pause
import time

#Keep open the window
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
#Eliminate 'Bluetooth: bluetooth_adapter' error messeage
options.add_experimental_option("excludeSwitches", ["enable-logging"])
#Install driver and open page with it
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
driver.get('https://www.lg.com/global')
#Maximize window
driver.maximize_window()
#-------------------------------------------------------------------------

#Get total height of the page
last_height = driver.execute_script("return document.body.scrollHeight")
#Initial height value
new_height = 300
#Incremental value
increase_value = 700
#Scroll down by incremental value and click go to top button
for i in range (new_height, last_height-1000, increase_value):
    #scroll to new_height value
    driver.execute_script("window.scrollTo(0, '"+str(new_height)+"')")
    time.sleep(1)
    #The current window's y-value before click the top button
    height_before_topBtn = driver.execute_script("return window.pageYOffset")
    print("Height before top BTN click:" + str(height_before_topBtn))
    #Click Top BTN
    driver.find_element(By.CLASS_NAME, "back-to-top").click()
    new_height = new_height + increase_value
    time.sleep(1)
    #The current window's y-value after click the top button
    height_after_topBtn = driver.execute_script("return window.pageYOffset")
    print("Height after top BTN click:" + str(height_after_topBtn))
    #Print "pass" when the top button work properly, if not print "fail"
    result = ""
    if height_after_topBtn == 0:
        result = "pass"
    else:
        result = "fail"
driver.close()


# python 3.10

from functools import cmp_to_key

def solution(numbers):
    """
    Args:
        numbers : 0 또는 양의 정수
    Returns:
        정수를 이어 붙여 만들 수 있는 가장 큰 수
    """
    answer = ''

    def comp(a, b):
        t1 = int(a+b)
        t2 = int(b+a)
        return (t1 > t2) - (t1 < t2) # 양수면 두번째 인자가 앞에 정렬, 음수면 첫번째 인자가 앞에 정렬, 0이면 변화 없음

    numbers = sorted(list(map(str, numbers)), key=cmp_to_key(comp), reverse=True) # 큰 수를 만들어야하기 때문에 reverse

    answer = str(int(''.join(numbers)))

    return answer


'''
Toteuta Flask-taustapalvelu, joka ilmoittaa, onko parametrina saatu luku alkuluku vai ei.
Hyödynnä toteutuksessa aiempaa tehtävää, jossa alkuluvun testaus tehtiin.
Esimerkiksi lukua 31 vastaava GET-pyyntö annetaan muodossa: http://127.0.0.1:3000/alkuluku/31. Vastauksen on oltava muodossa: {"Number":31, "isPrime":true}.
'''

from flask import Flask, Response
import json

app = Flask(__name__)
@app.route('/alkuluku/<num>')
def alkuluku(num):
    try:
        num = int(num)
        isPrime = True
        if num == 1:
            isPrime = False
        elif num > 1:
            for i in range(2, num):
                if num % i == 0:
                    isPrime = False
                    break
        else:
            isPrime = True

        tilakoodi = 200
        vastaus = {
            "status" : tilakoodi,
            "Number" : num,
            "isPrime" : isPrime
        }

    except ValueError:
        tilakoodi = 400
        vastaus = {
            "status" : tilakoodi,
            "teksti" : "Virheellinen yhteenlaskettava"
        }
    jsonvastaus = json.dumps(vastaus)
    return Response(response=jsonvastaus, status=tilakoodi, mimetype="application/json")

@app.errorhandler(404)
def page_not_found(virhekoodi):
    vastaus = {
        "status" : "404",
        "teksti" : "Virheellinen päätepiste"
    }
    jsonvastaus = json.dumps(vastaus)
    return Response(response=jsonvastaus,status=404, mimetype="application/json")

if __name__ == '__main__':
    app.run(use_reloader=True, host='127.0.0.1', port=3000)


# Import packages
import os
import csv
import numpy as np
import sklearn
import argparse
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split

import cv2
from sklearn.utils import shuffle
from keras.models import Sequential
from keras.models import Model
from keras.layers import Flatten, Dense, Lambda, Dropout, Input, GlobalAveragePooling2D
from keras.layers import Convolution2D, AveragePooling2D, BatchNormalization, ELU, Conv2D
from keras.layers.pooling import MaxPooling2D
from keras.layers import Cropping2D
from keras.callbacks import ModelCheckpoint
from keras.regularizers import l2
from keras.applications.vgg16 import preprocess_input
from keras.optimizers import Adam
from keras.applications.vgg16 import VGG16

# Resized shape and input shape of the model
resized_shape = (80, 80)
INPUT_SHAPE = (80, 80, 3)

def read_csv(args):
    """
    # Reading the csv file and store in the list
    """
    samples = []
    # Using the supplied data of udacity
    if args.selected_dataset == 'udacity':
        with open('./data/data/driving_log.csv') as csvfile:
            reader = csv.reader(csvfile)
            for line in reader:
                samples.append(line)
        samples = samples[1:]

    # Using my data from the simulator
    elif args.selected_dataset == 'my_data':
        with open('./data1/driving_log.csv') as csvfile:
            reader = csv.reader(csvfile)
            for line in reader:
                samples.append(line)

    return samples

def normalize_data(image_data):
    """
    # Normalize the dataset
    """
    data = image_data / 255.0 - 0.5
    return data

def generator(args, samples, batch_size=32):
    """
    # The generator of the samples
    """
    num_samples = len(samples)
    while 1: # Loop forever so the generator never terminates
        shuffle(samples)
        for offset in range(0, num_samples, batch_size):
            batch_samples = samples[offset:offset+batch_size]

            images = []
            angles = []
            for batch_sample in batch_samples:
                for i in range(3):
                    # Selected the name path for each dataset
                    if args.selected_dataset == 'udacity':
                        name = './data/data/IMG/'+batch_sample[i].split('/')[-1]
                    elif args.selected_dataset == 'my_data':
                        name = './data1/IMG/'+batch_sample[i].split('\\')[-1]
                    # Read the image and preprocess the image: resize, stree angle, augment, filp

                    # Nvidia model: read the image only and when it run the 'model.h5',
                    #it shoud be used the 'python drive-original.py model.h5'
                    if args.selected_model == 'Nvidia':
                        image = cv2.imread(name)
                        images.append(image)

                    # Vgg16 transfer learning model: read and resize the image.
                    # When it run the 'model.h5',
                    #it should be used the 'python drive.py model.h5'
                    elif args.selected_model == 'Vgg16':
                        image = cv2.imread(name)
                        image = cv2.resize(image, resized_shape)
                        images.append(image)

                    center_angle = float(batch_sample[3])
                    correction = 0.2
                    if i == 0:
                        measurment = center_angle # center
                    elif i == 1:
                        measurment = center_angle + correction #left
                    elif i == 2:
                        measurment = center_angle - correction # right
                    angles.append(measurment)
            # Augment and flip the image
            augmented_images, augmented_angles = [], []
            for image, angle in zip(images, angles):
                augmented_images.append(image)
                augmented_angles.append(angle)
                augmented_images.append(cv2.flip(image, 1))
                augmented_angles.append(angle*-1.0)

            # Form the X train and y train and shuffle the generator
            X_train = np.array(augmented_images)
            X_train = normalize_data(X_train)
            y_train = np.array(augmented_angles)
            yield sklearn.utils.shuffle(X_train, y_train)

def Nvidia_model():
    """
    # Nvidia model:referenced https://arxiv.org/pdf/1604.07316v1.pdf
    # this model follwo the udacity lecture, and not modified the image size before traiing the model,
    # and so that when you run the simulator, you should run the 'drive-original.py' which don't change the
    # tesing image size
    """
    print("Nvidia-------------")
    model = Sequential()
    model.add(Lambda(lambda x : x / 255.0 - 0.5, input_shape=(160,320,3)))
    model.add(Cropping2D(cropping=((70,25),(0,0))))
    model.add(Convolution2D(24,5,5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(36,5,5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(48,5,5,subsample=(2,2),activation='relu'))
    model.add(Convolution2D(64,3,3,activation='relu'))
    model.add(Convolution2D(64,3,3,activation='relu'))
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(100))
    model.add(Dense(50))
    model.add(Dense(10))
    model.add(Dense(1))

    return model

def Vgg16_transfer_model():
    """
    # Vgg16 model
    # this model's input shape is resized, so when you run the simulator,
    # you should run the 'drive.py' which modified the testing image size
    """
    print("Vgg16---------------")
    input_shape = INPUT_SHAPE
    input_tensor = Input(shape=input_shape)

    base_model = VGG16(input_tensor=input_tensor, weights='imagenet', include_top=False)
    x = base_model.output
    x = GlobalAveragePooling2D()(x)
    x = Dense(512, activation='elu')(x)
    x = Dropout(0.5)(x)
    x = Dense(256, activation='elu')(x)
    x = Dropout(0.3)(x)
    x = Dense(64, activation='elu')(x)
    x = Dropout(0.1)(x)
    predictions = Dense(1, init='zero')(x)

    # the model we will train
    model = Model(input=base_model.input, output=predictions)

    return model

def main():
    """
    Load train/validation data set and train the model
    """
    parser = argparse.ArgumentParser(description='Behavioral Cloning Project')
    parser.add_argument('-d', help='input udacity or my_data', dest='selected_dataset', type=str, default='my_data')
    parser.add_argument('-n', help='number of epochs', dest='nb_epoch', type=int, default=5)
    parser.add_argument('-m', help='input Nvidia or Vgg16', dest='selected_model', type=str, default="Vgg16")
    args = parser.parse_args()

    print('-' * 40)
    print('Parameters')
    print('-' * 40)
    for key, value in vars(args).items():
        print('{:<20} := {}'.format(key, value))
    print('-' * 40)

    # Load dataset
    samples = read_csv(args)
    train_samples, validation_samples = train_test_split(samples, test_size=0.2)
    train_generator = generator(args, train_samples, batch_size=32)
    validation_generator = generator(args, validation_samples, batch_size=32)

    # Selected model and train the model
    # Nvidia model
    if args.selected_model == 'Nvidia':
        model = Nvidia_model()
        # Default parameters follow those provided in the original paper.
        model.compile(loss='mse', optimizer='adam')
        checkpoint = ModelCheckpoint('model.h5',
                                 monitor='val_loss',
                                 verbose=0,
                                 save_best_only=True,
                                 mode='auto')
        history_object = model.fit_generator(train_generator, samples_per_epoch=
                        len(train_samples)*6, validation_data=validation_generator,
                        nb_val_samples=len(validation_samples)*6, callbacks=[checkpoint], nb_epoch=args.nb_epoch)
        model.save('model.h5')

    # Vgg16 transfer learning model
    elif args.selected_model == 'Vgg16':
        model = Vgg16_transfer_model()

        # Choose top2 blocks to train
        for layer in model.layers[:11]:
            layer.trainable = False
        for layer in model.layers[11:]:
            layer.trainable = True

        # Default parameters follow those provided in the original paper.
        opt = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.6)
        model.compile(optimizer=opt, loss='mse')
        checkpoint = ModelCheckpoint('model.h5',
                                 monitor='val_loss',
                                 verbose=0,
                                 save_best_only=True,
                                 mode='auto')
        history_object = model.fit_generator(train_generator, samples_per_epoch=
                    len(train_samples)*6, validation_data=validation_generator,
                    nb_val_samples=len(validation_samples)*6, callbacks=[checkpoint], nb_epoch=args.nb_epoch)

    print(history_object.history.keys())

    ### plot the training and validation loss for each epoch
    plt.plot(history_object.history['loss'])
    plt.plot(history_object.history['val_loss'])
    plt.title('model mean squared error loss')
    plt.ylabel('mean squared error loss')
    plt.xlabel('epoch')
    plt.legend(['training set', 'validation set'], loc='upper right')
    plt.show()

if __name__ == '__main__':
    main()


from __future__ import annotations

import argparse
import os.path

import pytest

import support

INPUT_TXT = os.path.join(os.path.dirname(__file__), 'input.txt')

def compare_ints(left, right):
    if left < right:
        return True
    elif left > right:
        return False
    else:
        # No decision
        return None

def compare_lists(left, right):
    idx = 0
    while True:
        if idx >= len(left) and idx >= len(right):
            # Both ran out of items.
            # No decision
            return None
        elif idx >= len(left) and idx < len(right):
            # Left ran out first
            return True
        elif idx < len(left) and idx >= len(right):
            # Right ran out first
            return False
        else:
            # Neither ran out yet.
            ret_val = compare(left[idx], right[idx])
            if ret_val is not None:
                return ret_val
        idx += 1

def compare(left, right):
    if type(left) == int and type(right) == int:
        ret_val = compare_ints(left, right)
        if ret_val is not None:
            return ret_val
    elif type(left) == list and type(right) == list:
        ret_val = compare_lists(left, right)
        if ret_val is not None:
            return ret_val
    elif type(left) == int and type(right) == list:
        ret_val = compare_lists(list([left]), right)
        if ret_val is not None:
            return ret_val
    elif type(left) == list and type(right) == int:
        ret_val = compare_lists(left, list([right]))
        if ret_val is not None:
            return ret_val

    return None

def compute(s: str) -> int:
    lines = s.splitlines()
    sm = 0
    for idx in range(0, len(lines), 3):
        flag = compare(eval(lines[idx]), eval(lines[idx + 1]))
        if flag is True:
            sm += (idx // 3) + 1
        elif flag is False:
            pass
        else:
            raise Exception(eval(lines[idx]), eval(lines[idx + 1]))
    return sm

INPUT_S = '''\
[1,1,3,1,1]
[1,1,5,1,1]

[[1],[2,3,4]]
[[1],4]

[9]
[[8,7,6]]

[[4,4],4,4]
[[4,4],4,4,4]

[7,7,7,7]
[7,7,7]

[]
[3]

[[[]]]
[[]]

[1,[2,[3,[4,[5,6,7]]]],8,9]
[1,[2,[3,[4,[5,6,0]]]],8,9]
'''
EXPECTED = 13

@pytest.mark.parametrize(
    ('input_s', 'expected'),
    (
        (INPUT_S, EXPECTED),
    ),
)
def test(input_s: str, expected: int) -> None:
    assert compute(input_s) == expected

def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument('data_file', nargs='?', default=INPUT_TXT)
    args = parser.parse_args()

    with open(args.data_file) as f, support.timing():
        print(compute(f.read()))

    return 0

if __name__ == '__main__':
    raise SystemExit(main())


from sklearn.feature_extraction.text import CountVectorizer
from scipy.cluster.vq import kmeans,vq
from sklearn import mixture
import tensorflow as tf
import tensorflow_text
import tensorflow_hub as hub
import spacy
import time
import numpy as np
from gensim.models.doc2vec import Doc2Vec, TaggedDocument
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('punkt')
from tqdm.autonotebook import tqdm
from scipy.spatial.distance import cosine, euclidean
from transformers import *
from summarizer import Summarizer
from keybert import KeyBERT
from collections import Counter
import random

ROOT_ID = 0
DOC_ID = 1
YEAR_ID = 2
MONTH_ID = 3
SENTENCE_ID = 4
AUTHOR_ID = 5

class nlpTool:
    DB = None
    model = None

    def __init__(self, DB):
        self.DB = DB

    def createExSumModel(self, modelName):
        custom_config = AutoConfig.from_pretrained(modelName)
        custom_config.output_hidden_states=True
        custom_tokenizer = AutoTokenizer.from_pretrained(modelName)
        custom_model = AutoModel.from_pretrained(modelName, config=custom_config)
        model = Summarizer(custom_model=custom_model, custom_tokenizer=custom_tokenizer)
        return model

    def generateUSEembeddings(self, baseID):
        module_url = "https://tfhub.dev/google/universal-sentence-encoder-multilingual/3"
        USEmodel = hub.load(module_url)
        print ("module %s loaded" % module_url)

        print("Reading Texts...")
        cur = self.DB.conn.cursor()
        cur.execute("""SELECT * FROM public.spans
            inner join public.subsumptions
            on public.spans.term = public.subsumptions.term
            where public.subsumptions.parent = {}""".format(baseID))
        result = cur.fetchall()
        summarizer = self.createExSumModel('bert-base-german-cased')
        print("Creating Summaries...")
        textList = [(i[4], i[5]) for i in tqdm(result)]
        #textList = [(i[4], summarizer(i[5], num_sentences = 1)) for i in tqdm(result)]
        print("Generating Embeddings...")

        embeddings = [(i[0], USEmodel(i[1]).numpy()[0]) for i in tqdm(textList)]
        for i in tqdm(embeddings):
            string = ""
            for j in i[1]:
                string += str(j) + ","
            string = string[:-1]
            cur.execute("""INSERT INTO public.embeddings VALUES({}, ARRAY [{}])""".format(i[0], string))
        self.DB.conn.commit()
        cur.close()

        return

    def createD2VModel(self, filename):
        start = time.time()
        stop_words = stopwords.words('german')
        textList = []
        print("Reading Texts...")
        cur = self.DB.conn.cursor()
        cur.execute("""SELECT * FROM public.spans
            inner join public.subsumptions
            on public.spans.term = public.subsumptions.term
            where public.subsumptions.parent = {}""".format(DOC_ID))

        result = cur.fetchall()
        textList = [" ".join(w.lower() for w in i[5].split() if w.lower() not in stop_words) for i in result]

        #for root, dirs, files in os.walk(IF.path):
        #    for name in files:
        #        with open(os.path.join(root, name), encoding="utf-8") as f:
        #            data = f.read()
        #        textList.append(data)

        print("Creating Embedding Model...")
        taggedData = [TaggedDocument(words=word_tokenize(doc), tags=[i]) for i, doc in enumerate(textList)]
        modelD2V = Doc2Vec(taggedData,vector_size=512, window=15, min_count=1, workers=4)

        modelD2V.save("models/"+filename)
        self.model = modelD2V
        end = time.time()

        print("model built successfully. Stored as models/{}. Duration: {}.".format(filename, str(end-start)))
        return

    def loadModel(self, path):
        self.model = Doc2Vec.load(path)
        return

    def generateD2VEmbeddings(self, baseID):
        stop_words = stopwords.words('german')

        print("Reading Texts...")
        cur = self.DB.conn.cursor()
        cur.execute("""SELECT * FROM public.spans
            inner join public.subsumptions
            on public.spans.term = public.subsumptions.term
            where public.subsumptions.parent = {}""".format(baseID))
        result = cur.fetchall()
        textList = [(i[4],  " ".join(w.lower() for w in i[5].split() if w.lower() not in stop_words)) for i in result]

        print("Generating Embeddings...")

        embeddings = [(i[0], self.model.infer_vector(word_tokenize(i[1]))) for i in tqdm(textList)]
        for i in tqdm(embeddings):
            string = ""
            for j in i[1]:
                string += str(j) + ","
            string = string[:-1]
            cur.execute("""INSERT INTO public.embeddings VALUES({}, ARRAY [{}])""".format(i[0], string))
        self.DB.conn.commit()
        cur.close()
        return

    def similaritySearch(self, spanID, baseID, amount):
        cur = self.DB.conn.cursor()
        cur.execute("""SELECT * FROM public.embeddings
            inner join public.spans
            on public.embeddings.span = public.spans.id
            inner join public.subsumptions
            on public.spans.term = public.subsumptions.term
            WHERE public.subsumptions.parent = {}""".format(baseID))
        res = cur.fetchall()

        vectors = np.array([i[1] for i in res])
        search = [x for x, y in enumerate(res) if y[0] == spanID]
        searchArray = vectors[search]
        similarities = [cosine(searchArray, i) for i in vectors]
        #similarities = [euclidean(searchArray, i) for i in vectors]

        joined = []
        for i in range(len(res)):
            joined.append((res[i][0], similarities[i]))

        ordered = sorted(joined, key=lambda x: x[1])

        output = []
        for i in ordered[:amount]:
            cur.execute("""SELECT * FROM public.spans WHERE "id" = {} """.format(i[0]))
            span = cur.fetchone()
            output.append(span[5])

        return output

    def createClusters(self, numClusters, gmm):

        cur = self.DB.conn.cursor()

        print("Fetching embeddings...")
        cur.execute("""SELECT * FROM public.embeddings""")
        res = cur.fetchall()
        vectors = np.array([i[1] for i in res])
        ids = np.array([i[0] for i in res])

        if not gmm:
            print("Calculating centroids...")
            centroids, _ = kmeans(vectors, numClusters)

            print("Computing cluster id for each document...")
            # computes cluster Id for document vectors
            doc_ids,_ = vq(vectors,centroids)
            # zips cluster Ids back to document labels
            doc_labels = list(zip(ids, doc_ids))

        else:
            print("Performing GMM...")
            gmm = mixture.GaussianMixture(n_components=numClusters, covariance_type='full').fit(vectors)
            labels = gmm.predict(vectors)
            print(labels)
            doc_labels = list(zip(ids, labels))

        print("Adding to DB...")
        for i in tqdm(doc_labels):
            cur.execute("""INSERT INTO public.cluster VALUES({}, {})""".format(i[0], i[1]))
        self.DB.conn.commit()
        cur.close()
        return

    def getTopWords(self, corpus, n=None):
        vec = CountVectorizer().fit(corpus)
        bag_of_words = vec.transform(corpus)

        sum_words = bag_of_words.sum(axis=0)

        words_freq = [(word, sum_words[0, idx]) for word, idx in     vec.vocabulary_.items()]
        words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
        return words_freq[:n]

    def findKeywordsForClusters(self, keywordNum, ratio):

        cur = self.DB.conn.cursor()
        cur.execute("""SELECT MAX("cluster") from public.cluster""")
        clusterNum = cur.fetchone()[0]+1

        nlpSpacy = spacy.load('de_core_news_sm')
        stop_words = stopwords.words('german')

        kW_model = KeyBERT(model='distiluse-base-multilingual-cased-v1')

        keywordList = []
        for i in tqdm(range(clusterNum)):
            cur.execute("""SELECT text FROM public.spans
            join public.cluster
            on public.spans.id = public.cluster.span
            where public.cluster.cluster = {}""".format(i))

            #METHOD: FIND KEYWORDS IN ALL TEXTS, USE MOST COMMON KEYWORDS
            texts = [i[0] for i in cur.fetchall()]
            keywords = []
            for text in random.sample(texts, int(len(texts)*ratio)):
                for j in kW_model.extract_keywords(text, keyphrase_ngram_range=(1, 1), top_n=keywordNum):
                    keywords.append(j[0])
            ctr = Counter(keywords)
            keywords = ctr.most_common()[:keywordNum]
            keywordList.append(keywords)

        allKeys = []
        for i in range(len(keywordList)):
            wordList = [j[0] for j in keywordList[i]]
            cur.execute("""INSERT INTO public.cluster_keywords VALUES({}, ARRAY {})""".format(i, wordList))
            self.DB.conn.commit()

        print("Importing of Keywords successful.")
        return

    def keywordsToStrings(self):
        cur = self.DB.conn.cursor()
        cur.execute("""SELECT * FROM public.cluster_keywords""")
        keywords = cur.fetchall()
        allKeys = []
        for i in keywords:
            tmpStr = ""
            for j in i[1]:
                tmpStr += j+", "
            allKeys.append(tmpStr)
        return allKeys

    def resetClusters(self):
        cur = self.DB.conn.cursor()
        cur.execute("""DELETE FROM public.cluster""")
        cur.execute("""DELETE FROM public.cluster_keywords""")
        self.DB.conn.commit()
        return


import argparse
import asyncio
import random
from datetime import datetime
import numpy as np
from aiohttp import ClientSession
import concurrent.futures
import requests
import logging
import functools

MAX_IOV = 2147483647

def timer(func):
    @functools.wraps(func)
    def timed_func(*args, **kwargs):
        t_0 = datetime.now()
        result = func(*args, **kwargs)
        logging.info(f'{func.__name__} took {datetime.now() - t_0}')
        return result
    return timed_func

class RequestMaker:

    def __init__(self, host_name, end_point, global_tag, pattern):
        self.pattern = pattern
        self.base_url = f'http://{host_name}/api/cdb_rest/{end_point}/?gtName={global_tag}'
        logging.info(f'initialized {self.__class__.__name__} with\n'
                     f' base url = {self.base_url}\n'
                     f' access pattern = <{self.pattern}>')

    def get_piov_url(self):
        major = {'first': 0, 'last': MAX_IOV, 'random': random.randint(0, MAX_IOV)}[self.pattern]
        minor = {'first': 0, 'last': MAX_IOV, 'random': random.randint(0, MAX_IOV)}[self.pattern]
        return f'{self.base_url}&majorIOV={major}&minorIOV={minor}'

    def make_requests(self, n):
        t_0 = datetime.now()
        results = self._make_requests(n)
        logging.info(f'made {n} calls in {datetime.now() - t_0}')
        return results

    def _make_requests(self, n):
        raise NotImplemented

class AsyncRequestMaker(RequestMaker):

    def _make_requests(self, n):
        return asyncio.run(self.make_async_requests(n))

    async def make_async_request(self, session: ClientSession, queue: asyncio.Queue):
        url = self.get_piov_url()
        logging.debug(f'attempting to request following url:\n{url}')
        beg_ts = datetime.now().timestamp()
        async with session.get(url) as response:
            await queue.put((beg_ts, datetime.now().timestamp(), response))

    async def make_async_requests(self, n):
        results = []
        queue = asyncio.Queue()

        async with ClientSession() as session:
            async with asyncio.TaskGroup() as group:
                for _ in range(n):
                    group.create_task(self.make_async_request(session, queue))

        while not queue.empty():
            results.append(await queue.get())

        return results

class ThreadingRequestMaker(RequestMaker):

    def _make_requests(self, n):
        urls = [self.get_piov_url() for _ in range(n)]
        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
            results = executor.map(self.make_request, urls)
        return results

    def make_request(self, url):
        logging.debug(f'attempting to request following url:\n{url}')
        beg_ts = datetime.now().timestamp()
        res = requests.get(url)
        return beg_ts, datetime.now().timestamp(), res

def process_results(results):
    pass

def main(args):
    logging.getLogger().setLevel(args.log_level)

    cls_dict = {'threading': ThreadingRequestMaker, 'async': AsyncRequestMaker}
    MakerClass = cls_dict[args.style]
    req_maker = MakerClass(args.hostname, args.endpoint, args.gt, args.pattern)

    wall_beg = datetime.now().timestamp()
    results = req_maker.make_requests(args.ncalls)
    wall_duration = datetime.now().timestamp() - wall_beg

    durations, beg_ts, end_ts = [], [], []
    for p in results:
        beg_ts.append(p[0])
        end_ts.append(p[1])
        durations.append(p[1] - p[0])

    print(f'processed {args.ncalls} calls {wall_duration}')
    logging.info(f'made {args.ncalls} calls within {np.max(beg_ts) - np.min(beg_ts)}')
    logging.info(f'mean response time [ms]: {round(np.mean(durations) * 1000)}')
    logging.info(f'mean response frequency [Hz]: {round(args.ncalls/wall_duration, 2)}')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    date_str = datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')[:-3]
    parser.add_argument('--output', type=str, default=f'output/{date_str}', help='output folder')
    parser.add_argument('--hostname', type=str, default='test111.apps.usatlas.bnl.gov')
    parser.add_argument('--ncalls', type=int, default=10, help='number of total calls to service')
    parser.add_argument('--pattern', type=str, default='random', choices=['random', 'first', 'last'])
    parser.add_argument('--style', type=str, default='async', choices=['async', 'threading', 'htc'])
    parser.add_argument('--endpoint', type=str, default='payloadiovs')
    parser.add_argument('--gt', type=str, default='my_gt')
    parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING'],
                        default='WARNING')
    args = parser.parse_args()
    main(args)


#coding: utf-8
import utils
from scipy import sparse
import numpy as np

class Edge(object):

    def __init__(self, v1, v2, ids):
        """
        Construct an Edge from two vertex index

        :param v1:  First vertex index
        :param v2:  Second vertex index
        :param ids:  Edge ID
        """
        self._v1 = v1
        self._v2 = v2
        self.ids = ids
        self._f1 = None
        self._f2 = None

    def add_face(self, f_idx):
        if self._f1 is None:
            self._f1 = f_idx
        elif self._f2 is None:
            self._f2 = f_idx
        else:
            raise ValueError('Error, edge can not be linked to more than two faces')

    def num_face(self):
        """
        Indicate how many faces are attached to this edge

        :return: Number of face assigned
        """
        nf = 0
        nf += 1 if self._f1 else 0
        nf += 1 if self._f2 else 0
        return nf

    @property
    def f1(self):
        """
        Return first face associated to this edge

        :return: Face 1
        """
        return self._f1

    @property
    def f2(self):
        """
        Return second face associated to this edge

        :return: Face 2
        """
        return self._f2

class Mesh(object):

    def __init__(self, vertices=None, trilist=None):
        """
        Create a Mesh object from an array of vertices [Nx3] and an array of triangle [Kx3]

        :param vertices: Array of vertices
        :param trilist: Array of triangles
        """
        # List of vertices
        self.vertex = vertices
        # List of triangles
        self.tri = trilist
        # List of vertices connect to each vertices (neighbours)
        self.neighbour = None
        # List of edges connected to a given vertex
        self._eneighbour = dict()
        # List of edges
        self.edge = None
        # Adjacency matrix
        self._adj = None
        # Voronoi area
        self._voronoi_area = self._compute_voronoi_area()

    def save(self, filename):
        """
        Dump current mesh into an *.obj file

        :param filename:    Filename
        """
        utils.save_obj(self.vertex, self.tri, filename)

    def load(self, filename):
        """
        Load mesh from an *.obj file

        :param filename:    Path to obj file
        """
        self.vertex, self.tri = utils.load_obj(filename)
        self.neighbour = None
        self._eneighbour = dict()
        self.edge = None
        self._voronoi_area = self._compute_voronoi_area()

    def compute_laplacian(self, type):
        """
        Compute Laplacian operator for this mesh

        :param type:    Type of Laplacian (Combinatorial, Normalized, Cotan, ...)
        :return:    Degree, Adjacency matrix, Laplacian operator
        """
        if type == 'combinatorial':
            return self._combinatorial_laplacian()
        elif type == 'normalized':
            return self._normalized_laplacian()
        elif type == 'cotan':
            return self._cotan_laplacian()
        elif type == 'cotan_normalized':
            return self._cotan_laplacian(True)
        else:
            return None

    def _combinatorial_laplacian(self):
        """
        Compute combinatorial Laplacian, L = D - A

        :return:    Degree, Adjacency matrix, Laplacian operator
        """
        deg = self._compute_degree(self.tri)
        N = len(deg)
        adj = self._compute_adjacency_matrix()
        lap = sparse.spdiags(deg.T, [0], N, N, format='coo')
        lap -= adj
        return deg, adj, lap

    def _normalized_laplacian(self):
        """
        Compute combinatorial Laplacian, L = I - D ** -0.5 @ A @ D ** -0.5

        :return:    Degree, Adjacency matrix, Laplacian operator
        """
        deg = self._compute_degree(self.tri)
        N = len(deg)
        adj = self._compute_adjacency_matrix()
        lap = sparse.identity(N, dtype=np.float32, format='coo')
        D = sparse.spdiags(deg.T, [0], N, N, format='coo')
        D = D.power(-0.5)
        lap -= D.dot(adj.dot(D)) #@ (adj @ np.diagflat(deg ** -0.5))
        return deg, adj, lap

    def _cotan_laplacian(self, normalized=False):
        """
        Compute cotan laplacian operator

        :param normalized:  Indicate if the operator is normalized by the voronoi area
        :return:            Degree, Adjacency matrix, Laplacian
        """
        # Find edge's index between two vertex neighbour
        def _edge_common(v1, n1):
            for ve, idx in self._eneighbour[v1]:
                if ve == n1:
                    return self.edge[idx]
        # Compute degree
        deg = self._compute_degree(self.tri)
        # Adjacency
        adj = self._compute_adjacency_matrix()
        # Laplacian
        data = []
        ridx = []
        cidx = []
        for key, value in self._eneighbour.items():
            vi = key
            weights = []
            for neighbour, idx in value:
                edge = _edge_common(vi, neighbour)
                faces = [edge.f1, edge.f2]
                cotan = []
                ridx.append(vi)
                cidx.append(neighbour)
                for face in faces:
                    if face is not None:
                        # Compute cotan
                        v_idx = [v for v in self.tri[face, :] if v != vi and v != neighbour][0]
                        vp = self.vertex[v_idx, :]
                        u = self.vertex[vi, :] - vp
                        v = self.vertex[neighbour, :] - vp
                        cot = np.dot(u, v) / np.linalg.norm(np.cross(u, v))
                        cotan.append(cot)
                num = (self._voronoi_area[vi] * self._voronoi_area[neighbour])** 0.5 if normalized else len(cotan)
                w = -1.0 / num * np.sum(cotan)
                data.append(w)
                weights.append(w)
            # Add diag element
            ridx.append(vi)
            cidx.append(vi)
            data.append(-1.0 * np.sum(weights))
        # Create laplacian
        lap = sparse.coo_matrix((data, (ridx, cidx)), shape=adj.shape, dtype=adj.dtype)
        return deg, adj, lap

    def _compute_degree(self, trilist):
        """
        Compute node's degree

        :param trilist: Array of triangle
        :return:        Vertex degrees
        """
        return np.asarray(list(map(len, self.neighbour)),
                          dtype=np.float32).reshape((len(self.neighbour), 1))

    def _compute_adjacency_matrix(self):
        """
        Compute adjacency matrix

        :return:        Adjacency matrix
        """
        if self._adj is None:
            N = len(self.neighbour)

            ridx = []
            cidx = []
            data = []
            visited = []
            for idx, n_list in enumerate(self.neighbour):
                for n in n_list:
                    if (idx, n) and (n, idx) not in visited:
                        ridx.append(idx)
                        cidx.append(n)
                        data.append(1.0)
                        cidx.append(idx)
                        ridx.append(n)
                        data.append(1.0)
                        visited.append((idx, n))
                        visited.append((n, idx))
                    else:
                        a=0
            self._adj = sparse.coo_matrix((data, (ridx, cidx)), shape=(N,N), dtype=np.float32)
        return self._adj

    def _compute_edges(self, trilist):
        """
        Compute for each vertex the edges that are connected to them

        :param trilist: List of triangles
        :return:        Edge list
        """
        def _in_list(collections, value):
            f = [v for v in collections if v[0] == value]
            return f

        def _processed(p, v1, v2, ids):
            v1p = v1 in p
            v2p = v2 in p
            if v1p or v2p:
                f1 = []
                f2 = []
                if v1p:
                    f1 = _in_list(p[v1], v2)
                if v2p:
                    f2 = _in_list(p[v2], v1)
                if f1 and f2:
                    assert f1[0][1] == f2[0][1]
                    return f1[0][1]
                elif not f1 and not f2:
                    if v1p:
                        p[v1].append((v2, ids))
                    else:
                        p[v1] = [(v2, ids)]
                    if v2p:
                        p[v2].append((v1, ids))
                    else:
                        p[v2] = [(v1, ids)]
                    return -1
                else:
                    a = 0
                    raise ValueError('Should not reach this points')
            else:
                # New entry
                p[v1] = [(v2, ids)]
                p[v2] = [(v1, ids)]
                return -1

        edges = []
        self._eneighbour = dict()
        spoly = trilist.shape[1]
        # Iterate over all triangle
        for k, tri in enumerate(trilist):
            for i in range(spoly):
                v0 = tri[i]
                v1 = tri[(i + 1) % spoly]
                # Edge already in added ?
                idx = _processed(self._eneighbour, v0, v1, len(edges))
                if idx == -1:
                    # Create edge
                    idx = len(edges)
                    e = Edge(v0, v1, idx)
                    # Update list
                    edges.append(e)
                # Add face
                edges[idx].add_face(k)
        return edges

    def _compute_voronoi_area(self):
        """
        Compute for each vertex the voronoi cell's area
        :return:    Area
        """
        areas = []
        if self.vertex is not None:
            N = self.vertex.shape[0]
            for vidx in range(N):
                # Get all triangles connected to this vertex
                faces = set()
                for neighbour, eidx in self._eneighbour[vidx]:
                    edge = self.edge[eidx]
                    if edge.f1 is not None:
                        faces.add(edge.f1)
                    if edge.f2 is not None:
                        faces.add(edge.f2)
                # Compute area over all tri
                area = 0.0
                for f in faces:
                    # Query face
                    tri = self.tri[f,:]
                    v0 = tri[tri == vidx][0]
                    verts = np.where(tri != vidx)[0].tolist()
                    assert len(verts) == 2
                    v1 = tri[verts[0]]
                    v2 = tri[verts[1]]
                    # Compute Centroid + mid edges
                    centroid = np.mean(self.vertex[[v0, v1, v2], :], axis=0) - self.vertex[v0, :]
                    v1m = (self.vertex[v1, :] - self.vertex[v0, :]) / 2.0
                    v2m = (self.vertex[v2, :] - self.vertex[v0, :]) / 2.0
                    # Compute area
                    a = np.linalg.norm(np.cross(centroid, v1m)) / 2.0
                    a += np.linalg.norm(np.cross(centroid, v2m)) / 2.0
                    area += a
                # add to the list
                areas.append(area)
        return areas

    @property
    def vertex(self):
        """
        Access vertex storage

        :return: Array of vertices
        """
        return self.__vertex

    @property
    def tri(self):
        """
        Access triangle storage

        :return: Array of triangle
        """
        return self.__tri

    @property
    def neighbour(self):
        """
        Return list of vertices index connected to each node

        :return:    Adjacent vertex for each vertices
        """
        return self.__neighbour

    @property
    def edge(self):
        """
        Return list of edges

        :return:    Edges
        """
        return self.__edge

    @vertex.setter
    def vertex(self, vertices):
        """
        Set a new vertices

        :param vertices: Array of vertices to overwrite with
        """
        self.__vertex = vertices

    @tri.setter
    def tri(self, trilist):
        """
        Set a new list of triangles

        :param trilist: Array of triangles to overwrite with
        """
        self.__tri = trilist

    @neighbour.setter
    def neighbour(self, trilist):
        """
        Initialize neighbour structure

        :param trilist: Triangle list
        """
        self.__neighbour = utils.gather_neighbour(self.tri) if self.tri is not None else None

    @edge.setter
    def edge(self, edge):
        """
        Initialize edge list

        :param edge: edge
        """
        self.__edge = self._compute_edges(self.tri) if self.tri is not None else None


# decompyle3 version 3.8.0
# Python bytecode 3.7.0 (3394)
# Decompiled from: Python 3.8.9 (default, Mar 30 2022, 13:51:17)
# [Clang 13.1.6 (clang-1316.0.21.2.3)]
# Embedded file name: output/Live/mac_64_static/Release/python-bundle/MIDI Remote Scripts/ableton/v3/control_surface/skin.py
# Compiled at: 2022-01-27 16:28:17
# Size of source mod 2**32: 1375 bytes
from __future__ import absolute_import, print_function, unicode_literals
from itertools import chain
from . import BasicColors
ON_SUFFIXES = ('enabled', 'on', 'pressed', 'selected')

class Skin:

    def __init__(self, colors=None, *a, **k):
        (super().__init__)(*a, **k)
        self._colors = {}
        if colors is not None:
            self._fill_colors(colors)

    def _fill_colors(self, colors, pathname=''):
        if getattr(colors, '__bases__', None):
            for base in colors.__bases__:
                self._fill_colors(base, pathname=pathname)

        for k, v in vars(colors).items():
            if k[:1] != '_':
                if callable(v):
                    self._fill_colors(v, '{}{}.'.format(pathname, k))
                else:
                    self._colors['{}{}'.format(pathname, k)] = v

    def __getitem__(self, key):
        if key not in self._colors:
            if key.lower().endswith(ON_SUFFIXES):
                return BasicColors.ON
            return BasicColors.OFF
        return self._colors[key]

def merge_skins(*skins):
    skin = Skin()
    skin._colors = dict(chain(*map(lambda s: s._colors.items(), skins)))
    return skin


from typing import List

def daily_temperatures(t: List[int]) -> List[int]:
    result = [0] * len(t)
    stack: List[int] = []
    for i in range(len(t)):
        while stack and t[stack[-1]] < t[i]:
            last = stack.pop()
            result[last] = i - last
        stack.append(i)
    return result


import json
from flask import Blueprint, request, redirect, render_template
from bs4 import BeautifulSoup
import requests
from news.models import WorldNews
from news import db
from sqlalchemy import text, engine

cbsnews = Blueprint('cbsnews', __name__)

def get_cbs_news():

    world_url = "https://www.cbsnews.com/latest/world/"

    news_page = requests.get(world_url).text

    item = BeautifulSoup(news_page, 'lxml')

    world_page = item.find('div', {'class':'component__item-wrapper'})
    article_page = world_page.find_all('article', {'class':'item lazyload item--type-article item--topic-world'})

    cbs_news = []

    for item in article_page:
        title = item.find('div', {'class':'item__title-wrapper'}).h4.text.strip()
        link = item.a.get('href')
        image = item.span.img.get('src')
        description = item.find('div', {'class':'item__title-wrapper'}).p.text.strip()

        news_items = {'Title':title, 'Link':link, 'Image':image, 'Description':description}
        news_items.update(news_items)

        cbs_news.append({
            'Title': title,
            'Link': link,
            'Image': image,
            'Description':description
        })

    return cbs_news

@cbsnews.route('/cbs-news', methods=['GET', 'POST'])
def cbs_news():
    if request.method == 'POST':
        return redirect('/')

    # new data from world news
    data = get_cbs_news()

    # existing data from database
    wnews =WorldNews.get_all_news()

    # loop through data
    for news in data:
        # check if news already exists in database
        # news['title'] is the title of the news
        if news.get('title').lower() not in [wn.title.lower() for wn in wnews]:
            wnews = WorldNews(title=news['title'], link=news['link'])
            wnews.save()
        else:
            continue

    return render_template('world_news.html', data=wnews)

@cbsnews.route('/world-news/search', methods=['GET', 'POST'])
def search_cbs_news():
    # /world-news/search?q=python
    query = request.args['q']
    # search in database for anythign that matches the query
    # SELECT * FROM world_news WHERE title LIKE '%python%'
    # data = db.session.query(worldNews).filter(worldNews.title.like(f'%{query}%')).all()
    # sql = text('SELECT * FROM world_news WHERE title LIKE :query')
    # data = db.engine.execute(sql, query=f'%{query}%').fetchall()
    data = WorldNews.query.filter(WorldNews.title.like(f'%{query}%')).all()
    # print(data)
    # [<HackNews 1>, <HackNews 2>]
    return {'data': [news.serialize() for news in data]}

@cbsnews.route('/search')
def search():
    return render_template('search.html')