import sys
import asyncio
import websockets
import json
from pytube import Playlist
from pytube import YouTube
import speedtest

WSS_HOST = "127.0.0.1"
WSS_PORT = 5678

# list of all topics
CONNECTION_SUCCEEDED = "CONNECTION_SUCCEEDED"
PLAYLIST_CONTENTS = "PLAYLIST_CONTENTS"
SEED_VIDEO_SIZE = 'SEED_VIDEO_SIZE'

# download progression status
DOWNLOAD_FAILED = "DOWNLOAD_FAILED"
DOWNLOAD_PROCESSING = "DOWNLOAD_PROCESSING"
DOWNLOAD_FINISHED = "DOWNLOAD_FINISHED"

args = sys.argv

speed_test = speedtest.Speedtest()

async def connection():
    async with websockets.connect("ws://%s:%d" % (WSS_HOST, WSS_PORT)) as websocket:
        await websocket.send(json.dumps({
            "topic": CONNECTION_SUCCEEDED,
            "value": "Connected successfully",
            "success": True
        }))

async def get_playlist_contents():
    async with websockets.connect("ws://%s:%d" % (WSS_HOST, WSS_PORT)) as websocket:
        try:
            playlist_link = args[2]
            playlist = Playlist(playlist_link)
            output = {
                "playlist_id": playlist.playlist_id,
                "playlist_url": playlist.playlist_url,
                "title": playlist.title,
                "videos": []
            }

            try:
                output["owner"] = playlist.owner
            except:
                output["owner"] = ""

            for video in playlist.videos:
                video_details = video.vid_info['videoDetails']
                # description contains not escaped characters
                del video_details["shortDescription"]
                output["videos"].append(video_details)
            await websocket.send(json.dumps({
                "topic": PLAYLIST_CONTENTS,
                "value": output,
                "success": True
            }, default=str))
        except:
            await websocket.send(json.dumps({
                "topic": PLAYLIST_CONTENTS,
                "value": "Failed to fetch playlist",
                "success": False
            }, default=str))

async def videos_seeding():
    async with websockets.connect("ws://%s:%d" % (WSS_HOST, WSS_PORT)) as websocket:
        try:
            videos_ids = json.loads(args[2])
            output = []
            for video_id in videos_ids:
                video = YouTube(f"https://youtube.com/watch?v={video_id}")
                # video_stream = video.streams.get_by_resolution('480p')
                # if video_stream == None:
                video_stream = video.streams.get_by_resolution('360p')
                if video_stream == None:
                    video_stream = video.streams.get_by_resolution('240p')
                video_size = video_stream.filesize
                download_url = video_stream.url
                expires_at = video_stream.expiration
                output.append({
                    "video_id": video_id,
                    "video_size": video_size,
                    "download_url": download_url,
                    "expires_at": expires_at
                })
            await websocket.send(json.dumps({
                "topic": SEED_VIDEO_SIZE,
                "value": output,
                "success": True
            }, default=str))
        except:
            await websocket.send(json.dumps({
                "topic": SEED_VIDEO_SIZE,
                "value": "Failed to seed the playlist videos",
                "success": False
            }, default=str))

def main():
    if len(args) > 1:
        if args[1] == PLAYLIST_CONTENTS:
            asyncio.run(get_playlist_contents())
        elif args[1] == SEED_VIDEO_SIZE:
            asyncio.run(videos_seeding())
    else:
        asyncio.run(connection())

main()


from trading.get_account_info import get_all_stat, get_my_order, get_all_currency, get_all_securities, get_my_operations
from main import dp, bot
from aiogram.types import Message
from aiogram.dispatcher.filters import Text
from trading.get_securities import security_name_by_figi
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from trading.place_order import cancel_order
from config.personal_data import get_account_access

"""

    Здесь собраны все хэндлеры, которые отвечают за вывод информации о счёте

"""

"""

    Баланс

"""

@dp.message_handler(Text(contains="баланс", ignore_case=True))
async def get_balance(message: Message):
    currency_df = get_all_currency(message.from_user.id)

    await message.answer(f"💸Доступная валюта💸")

    text = ""

    for i in currency_df.index:

        if currency_df['exchange_rate'][i] != 0.0 and currency_df['exchange_rate'][i] != 1.0:
            text += (
                f"{currency_df['name'][i]}\n"
                f"{round(currency_df['sum'][i], 2)}{currency_df['sign'][i]} ({round(currency_df['sum_in_ruble'][i], 2)}₽)\n\n")
        else:
            text += (
                f"{currency_df['name'][i]}\n"
                f"{round(currency_df['sum'][i], 2)}{currency_df['sign'][i]}\n\n")

    if text:
        await message.answer(text=text)
    else:
        await message.answer(text="У Вас нет средств!")

"""

    Бумаги

"""

@dp.message_handler(Text(contains="бумаги", ignore_case=True))
async def get_share(message: Message):
    security_df = get_all_securities(message.from_user.id)

    empty_portfolio = True
    await message.answer(f"💼Ценные бумаги в портфеле💼")

    for i in security_df.index:

        inst = ""

        if security_df['instrument'][i] == "share":
            inst = "Акции"
            empty_portfolio = False

        elif security_df['instrument'][i] == "bond":
            inst = "Бонды"
            empty_portfolio = False

        elif security_df['instrument'][i] == "etf":
            inst = "ETF"
            empty_portfolio = False

        elif security_df['instrument'][i] == "currency":
            continue

        elif security_df['instrument'][i] == "future":
            inst = "Фьючерсы"
            empty_portfolio = False

        if security_df['exp_yield'][i] >= 0:
            exp_yield = f"Ожидаемый доход: {round(security_df['exp_yield'][i], 2)}₽"
        else:
            exp_yield = f"Ожидаемая убыль: {round(security_df['exp_yield'][i], 2)}₽"

        await message.answer(
            f"🧾{inst} {security_df['security_name'][i]}\n"
            f"FIGI: {security_df['figi'][i]}\n\n"
            f"Лотов: {int(security_df['lots'][i])}\n"
            f"Всего: {round(security_df['quantity'][i], 2)}\n\n"
            f"Средняя цена: {security_df['average_price'][i]}{security_df['currency_sign'][i]}\n"
            f"Средняя цена FIFO: {security_df['average_price_fifo'][i]}{security_df['currency_sign'][i]}\n"
            f"Текущая цена: {round(security_df['current_price'][i], 6)}{security_df['currency_sign'][i]}\n\n"
            f"НКД: {security_df['nkd'][i]}{security_df['currency_sign'][i]}\n"
            f"{exp_yield}{security_df['currency_sign'][i]}\n"
        )

    if empty_portfolio:
        await message.answer(f"У Вас нет ценных бумаг в портфеле!")

"""

    Статистика по счёту

"""

@dp.message_handler(Text(contains="статистика", ignore_case=True))
async def get_stat(message: Message):
    await message.answer(f"📈Статистика по счёту📉 ")

    stat = get_all_stat(message.from_user.id)

    # Посчитаем сумму всех бумаг
    sum = stat['sum_total'][0]

    # Переведём доход/убыток из процентов в рубли
    dif = round(sum - (sum / (100 + stat['exp_yield'][0])) * 100, 2)

    if dif >= 0:
        dif_text = f"Прибыль: {dif}₽ ({stat['exp_yield'][0]}%)"
    else:
        dif_text = f"Убыль: {dif}₽ ({stat['exp_yield'][0]}%)"

    await message.answer(
        f"Акции на сумму: {stat['sum_shares'][0]}₽\n"
        f"Бонды на сумму: {stat['sum_bonds'][0]}₽\n"
        f"ETF на сумму: {stat['sum_etf'][0]}₽\n"
        f"Валюта на сумму: {stat['sum_curr'][0]}₽\n"
        f"Фьючерсы на сумму: {stat['sum_fut'][0]}₽\n\n"
        f"Итого: {round(sum, 2)}₽\n"
        f"{dif_text}\n"

    )

'''
    Открытые ордера
'''

@dp.message_handler(Text(contains="ордер", ignore_case=True))
async def get_orders(message: Message):
    order_df = get_my_order(message.from_user.id)

    await message.answer(f"📋Открытые ордера📋")

    no_orders = True

    for i in order_df.index:

        no_orders = False

        cancel_order_keyboard = InlineKeyboardMarkup()
        if get_account_access(user_id=message.from_user.id) == 1:
            cancel_order_keyboard.add(InlineKeyboardButton(text=f"Отменить ордер", callback_data=f"cancel_order:{order_df['order_id'][i]}"))

        if order_df['direction'][i] == 2:
            direction = "Продажа"
        else:
            direction = "Покупка"

        await message.answer(
            f"✅{direction} бумаг {security_name_by_figi(order_df['figi'][i], message.from_user.id)}\n"
            f"Открыт: {order_df['order_date'][i]}\n\n"
            f"ID: {order_df['order_id'][i]}\n\n"
            f"FIGI: {order_df['figi'][i]}\n\n"
            f"Лотов запрошено: {order_df['lots_req'][i]}\n"
            f"Лотов исполнено: {order_df['lots_ex'][i]}\n\n"
            f"Сумма запрошена: {order_df['sum_req'][i]}{order_df['currency_sign'][i]}\n"
            f"Сумма исполнено: {order_df['sum_ex'][i]}{order_df['currency_sign'][i]}\n\n"
            f"Цена одной акции: {round(order_df['price_one'][i], 6)}{order_df['currency_sign'][i]}\n\n"
            f"Комиссия: {round(order_df['commission'][i], 3)}{order_df['currency_sign'][i]}\n"
            f"Комиссия сервиса: {round(order_df['serv_commission'][i], 3)}{order_df['currency_sign'][i]}\n\n"
            f"Итого: {order_df['sum_total'][i]}{order_df['currency_sign'][i]}\n",
            reply_markup=cancel_order_keyboard
        )
    if no_orders:
        await message.answer(text="У Вас нет открытых ордеров!")

'''
    Закрытие ордера по id
'''

@dp.callback_query_handler(lambda c: c.data and c.data.startswith('cancel_order'))
async def close_order(callback_query):
    data = callback_query.data.split(":")
    order_id = data[1]

    await cancel_order(order_id=order_id, user_id=callback_query.from_user.id)

    await bot.edit_message_text(chat_id=callback_query.from_user.id, message_id=callback_query.message.message_id,
                                text="❌Ордер отменён❌")

'''
    Операции
'''

@dp.message_handler(Text(contains="операции", ignore_case=True))
async def get_operations(message: Message):

    try:
        operations = get_my_operations(user_id=message.from_user.id)
    except:
        await message.answer("Ошибка!")

    else:
        if operations:
            await message.answer(f"Ваши операции:")

            with open(f"img/operations/all_operations_{message.from_user.id}.png", "rb") as operations_img:
                await bot.send_photo(chat_id=message.from_user.id, photo=operations_img)

        else:
            await message.answer(f"У Вас пока не было операций!")


from jinja2 import Environment
import nltk
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
from flask import Flask, request, render_template
import csv
import os

app = Flask(__name__, static_url_path='/static')

# Import the zip filter
env = Environment()
env.filters['zip'] = zip
def jinja2_enumerate(iterable):
    return enumerate(iterable)

def stemmer(documents):
    punc = ['.', ',', '"', "'", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}', "%", "'", "``"]
    stpwrd = nltk.corpus.stopwords.words('english')
    stpwrd.extend(punc)
    porter = PorterStemmer()
    filtered_docs = []
    for doc in documents:
        tokens = word_tokenize(doc)
        tmp = ""
        for word in tokens:
            if word.lower() not in stpwrd:
                tmp += porter.stem(word.lower()) + " "
        filtered_docs.append(tmp)

    return (filtered_docs)

def stemmer_query(query):
    punc = ['.', ',', '"', "'", '?', '!', ':', ';', '(', ')', '[', ']', '{', '}', "%", "'", "``"]
    stpwrd = nltk.corpus.stopwords.words('english')
    stpwrd.extend(punc)
    stemmer = PorterStemmer()
    title_tokens = [stemmer.stem(token) for token in query.split() if token.lower() not in stpwrd]
    return(title_tokens)

def build_inverted_index(documents):
    inverted_index = {}
    filtered_docs = stemmer(documents)
    # Tokenize each document and count the term frequency
    for idx, doc in enumerate(filtered_docs):
        tokens = doc.split()
        term_freq = {}
        for token in tokens:
            if token in term_freq:
                term_freq[token] += 1
            else:
                term_freq[token] = 1
        for token, tf in term_freq.items():
            if token in inverted_index:
                inverted_index[token][idx] = tf
            else:
                inverted_index[token] = {idx: tf}
    return (inverted_index)

def process_query(query, inverted_index):
    qdocs = set()
    filtered_query = stemmer_query(query)
    df = {}
    tf_idf = {}
    for j in range(len(filtered_query)):
        if filtered_query[j] in inverted_index:
            qdocs = qdocs.union(inverted_index[filtered_query[j]].keys())
            print(qdocs)
            df[filtered_query[j]] = len(inverted_index[filtered_query[j]])
            N = len(qdocs)
            for i, tf in inverted_index[filtered_query[j]].items():
                if i in tf_idf:
                    tf_idf[i] += tf * (N / df[filtered_query[j]])
                else:
                    tf_idf[i] = tf * (N / df[filtered_query[j]])
    sorted_tf_idf = sorted(tf_idf, key=lambda k: tf_idf[k] * -1)
    return (sorted_tf_idf)

filename = "result_papers.csv"
script_dir = os.path.dirname(os.path.abspath(__file__))
csv_path = os.path.join(script_dir, filename)

with open(csv_path, mode="r") as csvfile:
    papers = []
    reader = csv.DictReader(csvfile)
    for row in reader:
        papers.append(row)

corpus = []
for paper in papers:
    f = ""
    f = paper['title'] + " " + paper['cov_authors'] +  " " + paper['non_cov_authors'] +  " " + paper['abstract'] + \
        paper['date']
    f = ((f.replace('[','')).replace(']','')).replace("'",'')
    corpus.append(f)

inverted_index = build_inverted_index(corpus)

@app.route("/")
def index():
    return render_template("home.html")

@app.route("/search", methods=['GET', 'POST'])

def search():
    # Get the query
    query = request.form.get("query")
    sorted_tf_idf = process_query(query, inverted_index)
    print(sorted_tf_idf)
    results = []
    for i in sorted_tf_idf:
        paper = {}
        paper = papers[i]
        results.append({
            "title": paper["title"],
            "title_link": paper["title_link"],
            "date": paper["date"],
            "authors_cov": paper["cov_authors"].replace('[','').replace(']','').split("'"),
            "author_links": paper["author_links"].replace('[','').replace(']','').split("'"),
            "authors_noncov": paper["non_cov_authors"].replace('[','').replace(']','').split("'"),
            "abstract": paper["abstract"]
        })

    return render_template("search.html", query=query, papers=results, zip = zip, enumerate = enumerate)

if __name__ == "__main__":
    app.run(debug=True)


# encoding: utf8

import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.utils.data as Data
import os
from models.RF import getRF
import const_rf as const

EPOCH = 30
BATCH_SIZE = 200
LR = 0.0005
if_use_gpu = 1
num_classes = const.num_classes

def load_data(fpath):
    train = np.load(fpath, allow_pickle=True).item()
    train_X, train_y = train['dataset'], train['label']

    return train_X, train_y

def adjust_learning_rate(optimizer, echo):
    lr = LR * (0.2 ** (echo / EPOCH))
    for para_group in optimizer.param_groups:
        para_group['lr'] = lr

def val(cnn, test_x, test_y, result_file, test_file):
    cnn.eval()
    test_result = open(test_file, 'w+')
    test_data = Data.TensorDataset(test_x, test_y)
    test_loader = Data.DataLoader(dataset=test_data, batch_size=BATCH_SIZE, shuffle=True)
    for step, (tr_x, tr_y) in enumerate(test_loader):
        test_output = cnn(tr_x)
        if if_use_gpu:
            test_output = test_output.cpu()
        pred_y, accuracy = get_result(test_output, tr_y)
        resultfile = open(result_file, 'w+')
        for i in range(len(tr_y)):
            resultfile.write(str(tr_y[i].numpy()) + ',' + str(pred_y[i]) + '\n')
        resultfile.close()
        test_result.write(str(accuracy) + '\n')
        print(accuracy)
    test_result.close()

def get_result(output, true_y):
    pred_y = torch.max(output, 1)[1].data.numpy().squeeze()
    accuracy = (pred_y == true_y.numpy()).sum().item() * 1.0 / float(true_y.size(0))
    return pred_y, accuracy

def control(feature_file):
    x, y = load_data(feature_file)
    cnn = getRF(num_classes)

    if if_use_gpu:
        cnn = cnn.cuda()

    optimizer = torch.optim.Adam(cnn.parameters(), lr=LR, weight_decay=0.001)
    loss_func = nn.CrossEntropyLoss()
    train_x = torch.unsqueeze(torch.from_numpy(x), dim=1).type(torch.FloatTensor)
    train_x = train_x.view(train_x.size(0), 1, 2, -1)
    train_y = torch.from_numpy(y).type(torch.LongTensor)

    train_data = Data.TensorDataset(train_x, train_y)
    train_loader = Data.DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)

    cnn.train()

    for epoch in range(EPOCH):

        adjust_learning_rate(optimizer, epoch)
        for step, (tr_x, tr_y) in enumerate(train_loader):
            batch_x = Variable(tr_x.cuda())
            batch_y = Variable(tr_y.cuda())
            output = cnn(batch_x)
            _, accuracy = get_result(output.cpu(), tr_y.cpu())

            del batch_x
            loss = loss_func(output, batch_y)
            del batch_y
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            del output

            if step % 100 == 0:
                print(epoch, step, accuracy, loss.item())

    torch.save(cnn.state_dict(), os.path.join(const.model_path, method + '.pth'))

if __name__ == '__main__':
    # TODO: change the data file path
    defense = 'Undefended'
    feature_file = '../countermeasure/dataset/' + defense + '.npy'
    method = defense
    control(feature_file)


from django.urls import path
from django.urls import re_path
from . import views

app_name = 'treadweb'
urlpatterns = [
    re_path(r'^$', views.home_page, name='home'),
    re_path(r'^search/$', views.search_page, name='search'),
    path('search/<str:keyword>', views.search_keyword, name='search_keyword'),
    path('channel/', views.channel_page, name='channel'),
    path('video/', views.video_page, name='video'),
    path('video/<str:tag>', views.video_tag, name='video_tag'),
]


import pytest

from viposhu.exceptions import ThreadOver1000, ThreadStopped
from viposhu.hoshu5ch import Hoshu5ch

def test_over1000():
    h = Hoshu5ch("https://mi.5ch.net/test/read.cgi/news4vip/1667619945/l1")
    assert h
    with pytest.raises(ThreadOver1000) as e:
        b = h.is_in_time_threshold()
    assert e

def test_KakoLog():
    h = Hoshu5ch("https://mi.5ch.net/test/read.cgi/news4vip/1667714909/")
    assert h
    with pytest.raises(ThreadStopped) as e:
        b = h.is_in_time_threshold()
    assert e


from django.shortcuts import render
from django.template import loader
from django.http import HttpResponse, HttpResponseRedirect
from django.core import serializers
from .models import Cameras, Emails, Users, Properties, Agents
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.decorators import login_required
import re
from django.utils import timezone

from app.forms import EmailForm

def index(request):
    # context = {}
    # template = loader.get_template('app/index.html')
    # return HttpResponse(template.render(context, request))
    if request.user.is_superuser:
        return HttpResponseRedirect("/ipcameras")
    else:
        return HttpResponseRedirect("/monitor")

def gentella_html(request):
    context = {}
    # The template to be loaded as per gentelella.
    # All resource paths for gentelella end in .html.

    # Pick out the html file name from the url. And load that template.
    load_template = request.path.split('/')[-1]
    template = loader.get_template('app/' + load_template)
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def ipcamera_html(request):
    context = {}
    template = loader.get_template('app/ipcameras.html')
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def monitor_html(request):
    context = {}
    template = loader.get_template('app/monitor.html')
    return HttpResponse(template.render(context, request))

def login_html(request):
    next = request.GET.get('next')
    context = {"next" : next}
    template = loader.get_template('app/login.html')
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def users_html(request):
    context = {}
    template = loader.get_template('app/users.html')
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def properties_html(request):
    context = {}
    template = loader.get_template('app/properties.html')
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def agents_html(request):
    context = {}
    template = loader.get_template('app/agents.html')
    return HttpResponse(template.render(context, request))

@login_required(login_url='/login')
def alerts_html(request):
    context = {}
    template = loader.get_template('app/alerts.html')
    return HttpResponse(template.render(context, request))

def cameras_asJson(request):
    object_list = Cameras.objects.all()
    for item in object_list:
        if item.online_status == False:
            continue
        last_con = item.lastConnection
        now = timezone.now()
        delta = now - last_con
        if delta.total_seconds() > 1860:
            item.online_status = False
            item.save()
    json = serializers.serialize('json', object_list)
    return HttpResponse(json, content_type='application/json')

def users_asJson(request):
    object_list = Users.objects.all()
    json = serializers.serialize('json', object_list)
    return HttpResponse(json, content_type='application/json')

def properties_asJson(request):
    object_list = Properties.objects.all()
    json = serializers.serialize('json', object_list)
    return HttpResponse(json, content_type='application/json')

def agents_asJson(request):
    object_list = Agents.objects.all()
    for item in object_list:
        if item.status == "offline":
            continue
        last_con = item.lastConnection
        now = timezone.now()
        delta = now - last_con
        if delta.total_seconds() > 3:
            item.status = "offline"
            item.save()
    object_list = Agents.objects.all()
    json = serializers.serialize('json', object_list)
    return HttpResponse(json, content_type='application/json')

def get_serial(text):
    try:
        found = re.search('Alarm Device Name: (.+?) ', text).group(1)
    except AttributeError:
        # AAA, ZZZ not found in the original string
        found = '' # apply your error handling
    return found

@csrf_exempt
def email_receiver(request):
    if request.POST.get('subject') == "Health_Test":
        serial = get_serial(re.sub(r"\s+", " ", request.POST.get('stripped-text')))
        cam = Cameras.objects.get(serial_number=serial)
        cam.online_status = True
        cam.lastConnection = timezone.now()
        cam.save()
        return HttpResponse('')
    else:
        #form = EmailForm(request.POST)
        print(request.POST)
        print(request.POST.get('stripped-text'))
        new_email = Emails()
        new_email.message_id = request.POST.get('Message-Id')
        new_email.subject = request.POST.get('Subject')
        new_email.content = request.POST.get('stripped-text')
        serial = get_serial(re.sub(r"\s+", " ", new_email.content))
        new_email.serial = serial
        cam = Cameras.objects.filter(serial_number=serial)
        if (cam[0].auth_user != None):
            new_email.notify = True
        new_email.save()
        return HttpResponse("ok")

def get_notify(request):
    alert = request.GET.get('alert')
    if alert == "1":
        agent = Agents.objects.get(username = request.user.username)
        if agent.status != "offline":
            agent.status = "offline"
            agent.save()
    else :
        agent = Agents.objects.get(username = request.user.username)
        agent.lastConnection = timezone.now()
        if agent.status != "online":
            agent.status = "online"
        agent.save()
    if request.GET.get('notify') == '0':
        object_list = Emails.objects.filter(notify=False)
        if (len(object_list) > 0):
            email = object_list[0]
            emails = Emails.objects.filter(message_id=email.message_id)
            emails.update(notify=True)
            serial = email.serial
            cam = Cameras.objects.filter(serial_number=serial)
            cam.update(auth_user = request.user.username)
            json = serializers.serialize('json', cam)
            return HttpResponse(json, content_type='application/json')
        else:
            return HttpResponse('')
    else:
        object_list = Emails.objects.filter(notify=False)
        if (len(object_list) > 0):
            email = object_list[0]
            serial = email.serial
            cam = Cameras.objects.filter(serial_number=serial)
            json = serializers.serialize('json', cam)
            return HttpResponse(json, content_type='application/json')
        else:
            return HttpResponse('')

def remove_notis(request):
    Emails.objects.filter(notify=False).update(notify=True)
    return HttpResponse('')


import discord
from discord.ext import commands

from Models.Context import Context

class SameTimeAttackStartCog(commands.Cog):
    """
    同時凸開始処理
    """
    def __init__(self, bot: commands.Bot):
        self.bot = bot

    @commands.command("dstart")
    async def same_time_attack_start(self, ctx: commands.context.Context, *args):
        remain: int = int(args[0])

        # 同時凸データ追加
        context: Context = await Context.get_instance(self.bot)
        context.same_time_attack.start(remain)
        print(context.same_time_attack.format_status())
        await ctx.message.channel.send(context.same_time_attack.format_status())


"""AoC [[day]], [[year]][% if puzzle_name %]: [[puzzle_name]][% endif %]"""

# Standard library imports
import pathlib
import sys
import re
from itertools import repeat
import math

def parse(puzzle_input):
    """Parse input"""
    return puzzle_input.splitlines()

def part1(data):
    """Solve part 1"""
    highest = find_highest(data)
    the_array = create_2d_array(highest)

    for line in data:
        regex = r"^(\d+),(\d+) -> (\d+),(\d+)"
        match = re.search(regex, line)

        x0 = int(match.group(1))
        y0 = int(match.group(2))
        x1 = int(match.group(3))
        y1 = int(match.group(4))

        if x0 == x1:
            if y0 < y1 + 1:
                for y in range(y0, y1 + 1):
                    the_array[x0][y] += 1
            else:
                for y in range(y1, y0 + 1):
                    the_array[x0][y] += 1
        elif y0 == y1:
            if x0 < x1 + 1:
                for x in range(x0, x1 + 1):
                    the_array[x][y0] += 1
            else:
                for x in range(x1, x0 + 1):
                    the_array[x][y0] += 1

    return calc_result(the_array)

def find_highest(data):
    max_width = 0
    for line in data:
        regex = r"^(\d+),(\d+) -> (\d+),(\d+)"
        match = re.search(regex, line)

        for i in range(1, 5):
            if int(match.group(i)) > max_width:
                max_width = int(match.group(i))
    return max_width

def create_2d_array(max_width):
    the_array = []

    for i in range(0, max_width + 1):
        the_array.append(list(repeat(0, max_width + 1)))

    return the_array

def part2(data):
    """Solve part 2"""
    highest = find_highest(data)
    the_array = create_2d_array(highest)

    for line in data:
        regex = r"^(\d+),(\d+) -> (\d+),(\d+)"
        match = re.search(regex, line)

        x0 = int(match.group(1))
        y0 = int(match.group(2))
        x1 = int(match.group(3))
        y1 = int(match.group(4))

        if x0 == x1:
            if y0 < y1 + 1:
                for y in range(y0, y1 + 1):
                    the_array[x0][y] += 1
            else:
                for y in range(y1, y0 + 1):
                    the_array[x0][y] += 1
        elif y0 == y1:
            if x0 < x1 + 1:
                for x in range(x0, x1 + 1):
                    the_array[x][y0] += 1
            else:
                for x in range(x1, x0 + 1):
                    the_array[x][y0] += 1
        elif abs(x1 - x0) - abs(y1 - y0) == 0:
            x_sign = int(math.copysign(1, x1 - x0))
            y_sign = int(math.copysign(1, y1 - y0))
            for i in range(0, abs(x1 - x0) + 1):
                the_array[x0 + x_sign * i][y0 + y_sign * i] += 1

    return calc_result(the_array)

def calc_result(the_array):
    result = 0
    for x in the_array:
        for xy in x:
            if xy > 1:
                result += 1

    return result

def solve(puzzle_input):
    """Solve the puzzle for the given input"""
    data = parse(puzzle_input)
    solution1 = part1(data)
    solution2 = part2(data)

    # 8111
    # 22088

    return solution1, solution2

if __name__ == "__main__":
    for path in sys.argv[1:]:
        print(f"{path}:")
        solutions = solve(pathlib.Path(path).read_text().strip())
        print("\n".join(str(solution) for solution in solutions))


import sympy as sp
A = sp.Matrix([
    [1],
    [2],
    [3]])
B = sp.Matrix([
    [4],
    [5],
    [6]])
print("A的模为：", A.norm())
print("A的模的浮点数为：", A.norm().evalf())
print("A的转置矩阵为：", A.T)
print("A和B的点乘为：", A.dot(B))
print("A和B的叉乘为：", A.cross(B))

3_python在高等数学和线性代数中的应用/3_6线性代数问题的符号解和数值解/3_6_1线性代数问题的符号解及解线性方程组/1_矩阵的运算.py


# Scrapy's settings for scrapers project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import logging
import os
from datetime import timedelta

BOT_NAME = 'airline_scraper'

SPIDER_MODULES = ['scrapers.spiders']
NEWSPIDER_MODULE = 'scrapers.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) ' \
             'Chrome/113.0.0.0 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 10
# The download delay setting will honor only one of:
CONCURRENT_REQUESTS_PER_DOMAIN = 2
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
# COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
# }

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'scrapers.middlewares.AirlineScraperSpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'scrapers.middlewares.AirlineScraperDownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'scrapers.pipelines.AirlineScraperPipeline': 0,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
AUTOTHROTTLE_ENABLED = True

# The initial download delay.
# Default 5
AUTOTHROTTLE_START_DELAY = 5

# The maximum download delay to be set in case of high latencies
# Default 60
AUTOTHROTTLE_MAX_DELAY = 100

# The average number of requests Scrapy should be sending in parallel to each remote server
AUTOTHROTTLE_TARGET_CONCURRENCY = 0.25
# Enable showing throttling stats for every response received:
AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

# Default True
RANDOMIZE_DOWNLOAD_DELAY = True

# Default 2
RETRY_TIMES = 2

DOWNLOADER_MIDDLEWARES = {
}

LOG_LEVEL = os.environ.get('LOG_LEVEL', logging.INFO)
LOG_FORMAT = os.environ.get('LOG_FORMAT', '[%(name)s] %(asctime)s %(levelname)s: %(message)s')

ROUTES_FILE = os.path.join(
    os.path.dirname(os.path.abspath(__file__)), os.environ.get('ROUTES_FILE', 'airline_routes.json')
)

# Want to scrape 6 months in advance
DAYS_TO_SCRAPE = timedelta(days=os.environ.get('DAYS_TO_SCRAPE', 180))


from django.urls import path
from .views import*
from django.contrib.auth import views as auth_views
urlpatterns = [path('',home,name='home'),
               path('notes',notes,name='notes'),
               path('delete_note/<int:pk>',delete_note,name='delete_note'),
               path('notes_detail/<int:pk>',notedetails,name='notes_detail'),
               path('homework',homework,name='homework'),
               path('update_homework/<int:pk>',update_homework,name='updatehomework'),
               path('delete_homework/<int:pk>',delete_homework,name='deletehomework'),
               path('youtube',youtube,name='youtube'),
               path('todo',todo,name='todo'),
               path('updatetodo/<int:pk>',updatetodo,name='updatetodo'),
               path('deletetodo/<int:pk>',deletetodo,name='deletetodo'),
               path('books',books,name='books'),
               path('dictionary',dictionary,name='dictionary'),
               path('WikiPedia',WikiPedia,name='WikiPedia'),

               path('register',register,name='register'),
               path('login',auth_views.LoginView.as_view(template_name='login.html'),name='login'),
               path('profile',profile,name='profile'),
               path('logout',logout,name='logout'),

]


from PyQt5.QtWidgets import (
    QAction,
    QCheckBox,
    QComboBox,
    QHBoxLayout,
    QLabel,
    QSpinBox,
    QToolButton,
    QVBoxLayout,
    QWidget,
)

class PlayBackUi:
    def setupUi(self, parent: QWidget) -> None:
        self.followChk = QCheckBox("Follow")
        self.followChk.setToolTip("Follow playhead")
        self.followChk.setChecked(True)

        self.fpsLabel = QLabel("Fps")
        self.fpsSpin = QSpinBox()
        self.fpsSpin.setToolTip("Playback speed")
        self.fpsSpin.setRange(1, 60)
        self.fpsSpin.setValue(15)

        self.lengthLabel = QLabel("Length")
        self.lengthSpin = QSpinBox()
        self.lengthSpin.setRange(60, 2000)
        self.lengthSpin.setToolTip("Max animation length in frames")

        self.currentViewLabel = QLabel("View")
        self.currentView = QComboBox()
        self.currentView.setToolTip("Current edit view")
        self.currentView.addItem("Dope sheet")
        self.currentView.addItem("Easing curve")

        # self.editor = QToolButton()
        # self.editor.setPopupMode(QToolButton.MenuButtonPopup)
        # self.editor.triggered.connect(self.editor.setDefaultAction)
        # self.editor.addAction(QAction("Dopesheet", parent))
        # self.editor.addAction(QAction("Curve editor", parent))

        hbox = QHBoxLayout(parent)
        hbox.setContentsMargins(0, 0, 0, 5)
        hbox.addWidget(self.currentViewLabel)
        hbox.addWidget(self.currentView)
        hbox.addStretch()
        hbox.addWidget(self.followChk)
        hbox.addWidget(self.fpsLabel)
        hbox.addWidget(self.fpsSpin)
        hbox.addWidget(self.lengthSpin)
        hbox.addWidget(self.lengthLabel)


"""
Boilerplate functions used in defining binary operations.
"""
from collections import UserDict
from functools import wraps
from typing import Callable

import numpy as np

from pandas._libs.lib import is_list_like, item_from_zerodim
from pandas._typing import F

from pandas.core.dtypes.generic import ABCDataFrame, ABCIndexClass, ABCSeries

def unpack_zerodim_and_defer(name: str) -> Callable[[F], F]:
    """
    Boilerplate for pandas conventions in arithmetic and comparison methods.

    Parameters
    ----------
    name : str

    Returns
    -------
    decorator
    """

    def wrapper(method: F) -> F:
        return _unpack_zerodim_and_defer(method, name)

    return wrapper

def _unpack_zerodim_and_defer(method, name: str):
    """
    Boilerplate for pandas conventions in arithmetic and comparison methods.

    Ensure method returns NotImplemented when operating against "senior"
    classes.  Ensure zero-dimensional ndarrays are always unpacked.

    Parameters
    ----------
    method : binary method
    name : str

    Returns
    -------
    method
    """
    is_cmp = name.strip("__") in {"eq", "ne", "lt", "le", "gt", "ge"}

    @wraps(method)
    def new_method(self, other):

        if is_cmp and isinstance(self, ABCIndexClass) and isinstance(other, ABCSeries):
            # For comparison ops, Index does *not* defer to Series
            pass
        else:
            for cls in [ABCDataFrame, ABCSeries, ABCIndexClass]:
                if isinstance(self, cls):
                    break
                if isinstance(other, cls):
                    return NotImplemented

        other = item_from_zerodim(other)

        if isinstance(self, (ABCSeries, ABCDataFrame)) and isinstance(
            other, (ABCSeries, ABCDataFrame)
        ):
            # we dont require length matches
            pass
        elif is_list_like(other, allow_sets=False) and not isinstance(
            other, (dict, UserDict)
        ):
            if len(other) != len(self):
                if len(other) == 1 and not hasattr(other, "dtype"):
                    # i.e. unpack scalar list, but leave e.g. Categorical,
                    #  for which the scalar behavior doesnt match the
                    #  array behavior
                    other = other[0]
                else:
                    raise ValueError(
                        "Lengths must match", self.shape, np.shape(other), type(other)
                    )

        return method(self, other)

    return new_method


import numpy as np
import multiflap as mf
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from scipy.integrate import odeint
x = [.1, 5., 3.6]

time_array = np.linspace(0, 180, 90000)
mymodel = mf.Rossler(a=0.2, b=0.2, c=5.7)

ms_obj =  mf.MultipleShootingPeriod(x, M=30, period_guess= 5.,
                                    t_steps=5000, model=mymodel, integrator='odeint')

mysol = mf.SolverPeriod(ms_obj = ms_obj).lma()

jac = mysol[4]

eigenvalues, eigenvectors = np.linalg.eig(jac)

# plot the trajectory related to the initial condition
[_, initial_traj] = ms_obj.get_mappedpoint(x, 0, 100)
initial_points = ms_obj.get_initial_guess()

sol_array = mysol[3].space
sol_time = mysol[3].time
period = sol_time[-1]

initial_traj = initial_traj.x

fig1 = plt.figure(1)
ax = fig1.gca(projection='3d')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')
ax.set_zlabel('$z$')
ax.plot(sol_array[:, 0],
        sol_array[:, 1],
        sol_array[:, 2],color = 'b')
ax.scatter(initial_points[:, 0],
        initial_points[:, 1],
        initial_points[:, 2],color = 'red', alpha=0.5)
plt.legend()
plt.show()
fig2 = plt.figure(2)
ax = fig2.gca(projection='3d')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')
ax.set_zlabel('$z$')
ax.plot(sol_array[:, 0],
        sol_array[:, 1],
        sol_array[:, 2],color = 'b', label='limit cycle')
ax.plot(initial_traj[:, 0],
        initial_traj[:, 1],
        initial_traj[:, 2],color = 'red', alpha=0.5, label='initial point trajectory')
plt.legend()
plt.show()


"""
698. Partition to K Equal Sum Subsets
Given an integer array nums and an integer k, return true if it is possible to divide this array into k non-empty subsets whose sums are all equal.

Example1:
Input: nums = [4,3,2,3,5,2,1], k = 4
Output: true
Explanation: It is possible to divide it into 4 subsets (5), (1, 4), (2,3), (2,3) with equal sums.

Example2:
Input: nums = [1,2,3,4], k = 3
Output: false

Constraints:
1 <= k <= <= nums.length <= 16
1 <= nums[i] <= 10^4
The frequency of each element is in the range [1, 4].
"""

"""
Note:
1. DFS (Top Down) + Backtracking: O(n * n!) time | O(n) space
2. DFS (Bottom Up) + Backtracking + memo: O(klogk * k**n) | O(kn) space
"""

from typing import List
class Solution:
    def canPartitionKSubsets(self, nums: List[int], k: int) -> bool:
        total = sum(nums)
        if total % k != 0:
            return False

        target = total // k

        n = len(nums)
        visited = [False] * n

        nums.sort(reverse=True)
        def dfs(i, currentSum, k, visited) -> bool:
            if k == 1:
                return True
            for j in range(i, n):
                if visited[j]:
                    continue

                if currentSum + nums[j] > target:
                    break

                visited[j] = True
                if currentSum + nums[j] == target:
                    if dfs(0, 0, k-1, visited):
                        return True
                else:
                    if dfs(0, currentSum+nums[j], k, visited):
                        return True
                visited[j] = False

            return False
        return dfs(0, 0, k, visited)

    def canPartitionKSubsets2(self, nums: List[int], k: int) -> bool:
        total = sum(nums)
        if total % k != 0:
            return False

        buckets = [total // k] * k
        nums.sort(reverse=True)

        memo = {}
        def dfs(i, buckets, totalSum):
            key = (str(sorted(buckets)), i)
            if key in memo:
                return memo[key]
            if totalSum == 0:
                memo[key] = True
                return memo[key]

            num = nums[i]
            for j in range(k):
                if buckets[j] - num < 0:
                    continue
                buckets[j] -= num
                if dfs(i+1, buckets, totalSum - num):
                    memo[key] = True
                    return memo[key]
                buckets[j] += num

            memo[key] = False
            return memo[key]
        return dfs(0, buckets, total)

# Unit Tests
import unittest
funcs = [Solution().canPartitionKSubsets, Solution().canPartitionKSubsets2]

class TestCanPartitionKSubsets(unittest.TestCase):
    def testCanPartitionKSubsets1(self):
        for func in funcs:
            nums = [4,3,2,3,5,2,1]
            k = 4
            self.assertEqual(func(nums=nums, k=k), True)

    def testCanPartitionKSubsets2(self):
        for func in funcs:
            nums = [1,2,3,4]
            k = 3
            self.assertEqual(func(nums=nums, k=k), False)

    def testCanPartitionKSubsets3(self):
        for func in funcs:
            nums = [1,1,1,1,2,2,2,2]
            k = 4
            self.assertEqual(func(nums=nums, k=k), True)

if __name__ == "__main__":
    unittest.main()


from __future__ import annotations

import json
from typing import Union

from PySide2 import QtGui, QtWidgets

from usc_lr_timer import google_sheets, talk_to_google
from usc_lr_timer.constants import RESOURCES

JOURNALS = RESOURCES / 'journal_mapping.json'

class Model(object):
    def __init__(self):
        self._names = []
        self._name_index = 0
        self._name_mapping = {}
        self._journals = []
        self._journal_index = 0
        self._journal_mapping = {}
        self._pin = ''

    @property
    def names(self) -> list[str]:
        return self._names[:]

    @property
    def journals(self) -> list[str]:
        return self._journals[:]

    @property
    def name(self) -> Union[str, None]:
        if self._name_index == 0:
            return None
        else:
            return self._names[self._name_index]

    @property
    def journal(self) -> Union[str, None]:
        if self._journal_index == 0:
            return None
        else:
            return self._journals[self._journal_index]

    @property
    def sheet_id(self) -> Union[str, None]:
        if self.journal is None:
            return None
        else:
            return self._journal_mapping[self.journal]

    @property
    def pin(self) -> str:
        return self._pin

    def set_names(self):
        if self.sheet_id is not None:
            status, result = talk_to_google.talk_to_google(
                google_sheets.get_names, self.sheet_id,
            )
            if status == talk_to_google.Results.success:
                self._name_mapping = dict(result)
                self._names = [''] + list(self._name_mapping.keys())
                return

        self._name_mapping = {}
        self._names = []

    def set_journals(self):
        with open(JOURNALS) as stream:
            self._journal_mapping = json.load(stream)
        self._journals = [''] + list(self._journal_mapping.keys())

    def set_name_index(self, index: int):
        self._name_index = index

    def set_journal_index(self, index: int):
        self._journal_index = index

    def set_pin(self, pin: str):
        self._pin = pin

    def login(self) -> bool:
        if self.name is None:
            return False
        return self.pin == self._name_mapping[self.name]

class View(object):
    def __init__(self, model: Model, widget: LoginDialog):
        self.model = model
        self.widget = widget

    def set_names(self):
        self.model.set_names()
        self.widget.names_cb.clear()
        if self.model.journal:
            self.widget.names_cb.addItems(self.model.names)

    def set_journals(self):
        self.model.set_journals()
        self.widget.journals_cb.addItems(self.model.journals)

    def set_name_index(self, index: int):
        self.model.set_name_index(index)
        self.widget.names_cb.setCurrentIndex(index)

    def set_journal_index(self, index: int):
        self.model.set_journal_index(index)
        self.widget.journals_cb.setCurrentIndex(index)
        self.set_names()

    def set_pin(self, pin: str):
        self.model.set_pin(pin)

    def login(self) -> bool:
        if self.model.login():
            return True
        else:
            self.model.set_pin('')
            self.widget.pin_line.setText(self.model.pin)
            return False

class LoginDialog(QtWidgets.QDialog):
    def __init__(self, model: Model):
        super().__init__(None)
        self._layout = QtWidgets.QVBoxLayout()
        self.setLayout(self._layout)
        self.view = View(model, self)
        self.init_ui()
        self.init_controller()
        self.view.set_journals()
        self.view.set_names()

    def init_ui(self):
        self.setWindowIcon(QtGui.QIcon(str(RESOURCES / 'icon.ico')))
        self.setWindowTitle('Login')
        self.setWhatsThis('Login to track your time')
        form_layout = QtWidgets.QFormLayout()
        self.journals_cb = QtWidgets.QComboBox()
        form_layout.addRow('Journal: ', self.journals_cb)

        # Add names dropdown
        self.names_cb = QtWidgets.QComboBox()
        form_layout.addRow('Name: ', self.names_cb)

        # Add pin
        self.pin_line = QtWidgets.QLineEdit()
        self.pin_line.setEchoMode(QtWidgets.QLineEdit.Password)
        form_layout.addRow('Pin: ', self.pin_line)

        self._layout.addLayout(form_layout)

        self.button_box = QtWidgets.QDialogButtonBox(
            QtWidgets.QDialogButtonBox.Ok
        )
        ok_btn = self.button_box.button(QtWidgets.QDialogButtonBox.Ok)
        ok_btn.setText('Login')
        self._layout.addWidget(self.button_box)

    def init_controller(self):
        self.names_cb.currentIndexChanged[int].connect(self.view.set_name_index)
        self.journals_cb.currentIndexChanged[int].connect(
            self.set_journal_index
        )
        self.pin_line.textEdited.connect(self.view.set_pin)
        self.button_box.accepted.connect(self.login)

    def set_journal_index(self, index: int):
        self.view.set_journal_index(index)

    def login(self):
        if self.view.login():
            self.accept()
        else:
            QtWidgets.QMessageBox.critical(self, 'Failed', 'Incorrect Pin')

def login() -> tuple(bool, str, str, str):
    model = Model()
    dialog = LoginDialog(model)
    dialog.exec_()
    success = dialog.result() == dialog.Accepted
    return success, model.journal, model.sheet_id, model.name


"""
Created on Wed Jul 22 15:39:20 2020
Parse drugs info for disease name and MeSH ID

@author: Tianqi
"""

xml_file = '../data/full database.xml'
import collections
import xml.etree.ElementTree as etree
import pandas as pd
import os

import json
import csv
# import spacy
# from scispacy.abbreviation import AbbreviationDetector
# # nlp = spacy.load("en_core_sci_sm")
# nlp = spacy.load("en_ner_bc5cdr_md")
import requests

def query_raw(text, url="https://bern.korea.ac.kr/plain"):
    return requests.post(url, data={'sample_text': text}).json()
#%%
cnt = 0
rows = list()
ns = '{http://www.drugbank.ca}'
for event, elem in etree.iterparse(xml_file, events=('start','end')): ### iterate all elements
    ##### find </drug type='biotech'> element, event='end' is corresponding to event ='start', but elemnt size is
    #### mostly dependent on 'end'
    if event == 'end' and '{http://www.drugbank.ca}drug' == elem.tag and 'type' in elem.attrib:

        cnt += 1
        assert elem.tag == ns + 'drug'
        drugbank_id = elem.findtext(ns + "drugbank-id[@primary='true']")
        discription_text = elem.findtext(ns + 'indication')

        result = (query_raw(discription_text))
        if 'logits' in result:

            disease = result['logits']['disease']
            text = result['text']

            for idx in range(len(disease)):
                di_dict = disease[idx][0]
                if 'id' in di_dict:
                    meshId = di_dict['id'] # disease_id including MeSH ID and BERN-ID
                else:
                    meshId = None

                disease_name = text[di_dict['start']:di_dict['end']] # disease_name
                disease_name = disease_name.casefold()

                row = collections.OrderedDict()
                row['disease'] = disease_name
                row['drugbank_id'] = drugbank_id
                row['MeSHID'] = meshId
                row['cas_num'] = elem.findtext(ns + "cas-number")
                row['type'] = elem.get('type')
                row['name'] = elem.findtext(ns + "name")
                rows.append(row)

        else:
            continue

        elem.clear()

#%%
columns = ['drugbank_id', 'name', 'cas_num', 'type', 'disease', 'MeSHID']
disease_df = pd.DataFrame.from_dict(rows)[columns]
#protein_df.to_csv(path, sep='\t', index=False)
disease_df.to_csv('drugbank_compound_disease_BERN.tsv', sep='\t', index=False, columns=columns)

#%%
meshid_df = pd.read_csv('./disease_mesh_ALL_updated.tsv',  sep="\t")
#
drugbank_disease_mesh_df = disease_df.merge(meshid_df, how = 'left', on = 'disease').drop_duplicates()
drugbank_disease_mesh_df.to_csv('./drugbank_CtD_vocabulary.tsv', sep = '\t', index = False)
drugbank_disease_mesh_df1 = drugbank_disease_mesh_df[['drugbank_id','name','disease' ,'mesh_id']].dropna()
drugbank_disease_mesh_df1 = drugbank_disease_mesh_df1.drop_duplicates()
# drugbank_disease_mesh_df1.to_csv('drugbank_CtD_vocabulary_slim.tsv', sep = '\t', index = False)
drugbank_disease_mesh_df2 = drugbank_disease_mesh_df1[['drugbank_id','mesh_id']]
dups = drugbank_disease_mesh_df2[drugbank_disease_mesh_df2.duplicated(keep=False)]
drugbank_disease_mesh_df2 = drugbank_disease_mesh_df2.drop_duplicates()
drugbank_disease_mesh_df2.to_csv('./drugbank_CtD.tsv', sep = '\t', index = False)

#%%
df = pd.read_csv('./drugbank_CtD.tsv', sep = '\t')
df =df.drop_duplicates()
# a1 = df[df.duplicated()]

ddi_df_new = df.assign(mesh_id=df['mesh_id'].str.split(',')).explode('mesh_id')
ddi_df_new = ddi_df_new.reset_index(drop=True)
ddi_df_new = ddi_df_new.drop_duplicates()

ddi_df_new['source'] = 'Compound::' + ddi_df_new[['drugbank_id']]
ddi_df_new['target'] = 'Disease::MESH:' + ddi_df_new[['mesh_id']]
ddi_df_new['edges'] = 'DRUGBANK::treats::Compound:Disease'
ddi_df_new.to_csv('../data/drugbank_triplets_CtreatD_mesh.tsv', sep = '\t', index = False)
# ddi_df_new = pd.concat([pd.Series(row['drugbank_id'], row['mesh_id'].split(','))
#                         for _, row in ddi_df.iterrows()]).reset_index()


import pandas as pd

df = pd.read_csv("/Users/mona/Downloads/finalized_gv_ih_master_updated.csv")

frame9_text = ''

for i in range(df.shape[0]):
    #if df.iloc[i]['Q3 Theme'] == 9:
    frame9_text += df.iloc[i]['news_title']

print(frame9_text)

import collections
import pandas as pd
from nltk.stem import PorterStemmer
ps = PorterStemmer()
import nltk
lemma = nltk.wordnet.WordNetLemmatizer()

import matplotlib.pyplot as plt
# Read input file, note the encoding is specified here
# It may be different in your text file

# Stopwords
stopwords = set(line.strip() for line in open('stopwords.txt'))
stopwords = stopwords.union(set(['mr','mrs','one','two','said']))
# Instantiate a dictionary, and for every word in the file,
# Add to the dictionary if it doesn't exist. If it does, increase the count.
wordcount = {}
# To eliminate duplicates, remember to split by punctuation, and use case demiliters.
for word in frame9_text.lower().split():
    word = word.replace(".","")
    word = word.replace(",","")
    word = word.replace(":","")
    word = word.replace("\"","")
    word = word.replace("!","")
    word = word.replace("â€œ","")
    word = word.replace("â€˜","")
    word = word.replace("*","")
    if lemma.lemmatize(word) not in stopwords:
        if lemma.lemmatize(word) not in wordcount:
            #wordcount[ps.stem(word)] = 1
            wordcount[lemma.lemmatize(word)] = 1
        else:
            wordcount[lemma.lemmatize(word)] += 1
# Print most common word
n_print = int(input("How many most common words to print: "))
print("\nOK. The {} most common words are as follows\n".format(n_print))
word_counter = collections.Counter(wordcount)
for word, count in word_counter.most_common(n_print):
    print(word, ": ", count)
# Close the file

# Create a data frame of the most common words
# Draw a bar chart
lst = word_counter.most_common(n_print)
df = pd.DataFrame(lst, columns = ['Word', 'Count'])
df.plot.bar(x='Word',y='Count')


from __future__ import print_function
import os
import os.path as osp
from scipy import io
import datetime
import time

def time_str(fmt=None):
    if fmt is None:
        fmt = '%Y-%m-%d_%H:%M:%S'
    return datetime.datetime.today().strftime(fmt)

def save_mat(ndarray, path):
    """Save a numpy ndarray as .mat file."""
    io.savemat(path, dict(ndarray=ndarray))

def may_make_dir(path):
    """
    Args:
      path: a dir, or result of `osp.dirname(osp.abspath(file_path))`
    Note:
      `osp.exists('')` returns `False`, while `osp.exists('.')` returns `True`!
    """
    # This clause has mistakes:
    # if path is None or '':

    if path in [None, '']:
        return
    if not osp.exists(path):
        os.makedirs(path)

class AverageMeter(object):
    """Modified from Tong Xiao's open-reid.
    Computes and stores the average and current value"""

    def __init__(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = float(self.sum) / (self.count + 1e-20)

class RunningAverageMeter(object):
    """Computes and stores the running average and current value"""

    def __init__(self, hist=0.99):
        self.val = None
        self.avg = None
        self.hist = hist

    def reset(self):
        self.val = None
        self.avg = None

    def update(self, val):
        if self.avg is None:
            self.avg = val
        else:
            self.avg = self.avg * self.hist + val * (1 - self.hist)
        self.val = val

class RecentAverageMeter(object):
    """Stores and computes the average of recent values."""

    def __init__(self, hist_size=100):
        self.hist_size = hist_size
        self.fifo = []
        self.val = 0

    def reset(self):
        self.fifo = []
        self.val = 0

    def update(self, val):
        self.val = val
        self.fifo.append(val)
        if len(self.fifo) > self.hist_size:
            del self.fifo[0]

    @property
    def avg(self):
        assert len(self.fifo) > 0
        return float(sum(self.fifo)) / len(self.fifo)

class ReDirectSTD(object):
    """Modified from Tong Xiao's `Logger` in open-reid.
    This class overwrites sys.stdout or sys.stderr, so that console logs can
    also be written to file.
    Args:
      fpath: file path
      console: one of ['stdout', 'stderr']
      immediately_visible: If `False`, the file is opened only once and closed
        after exiting. In this case, the message written to file may not be
        immediately visible (Because the file handle is occupied by the
        program?). If `True`, each writing operation of the console will
        open, write to, and close the file. If your program has tons of writing
        operations, the cost of opening and closing file may be obvious. (?)
    Usage example:
      `ReDirectSTD('stdout.txt', 'stdout', False)`
      `ReDirectSTD('stderr.txt', 'stderr', False)`
    NOTE: File will be deleted if already existing. Log dir and file is created
      lazily -- if no message is written, the dir and file will not be created.
    """

    def __init__(self, fpath=None, console='stdout', immediately_visible=False):
        import sys
        import os
        import os.path as osp

        assert console in ['stdout', 'stderr']
        self.console = sys.stdout if console == 'stdout' else sys.stderr
        self.file = fpath
        self.f = None
        self.immediately_visible = immediately_visible
        if fpath is not None:
            # Remove existing log file.
            if osp.exists(fpath):
                os.remove(fpath)

        # Overwrite
        if console == 'stdout':
            sys.stdout = self
        else:
            sys.stderr = self

    def __del__(self):
        self.close()

    def __enter__(self):
        pass

    def __exit__(self, *args):
        self.close()

    def write(self, msg):
        self.console.write(msg)
        if self.file is not None:
            may_make_dir(os.path.dirname(osp.abspath(self.file)))
            if self.immediately_visible:
                with open(self.file, 'a') as f:
                    f.write(msg)
            else:
                if self.f is None:
                    self.f = open(self.file, 'w')
                self.f.write(msg)

    def flush(self):
        self.console.flush()
        if self.f is not None:
            self.f.flush()
            import os
            os.fsync(self.f.fileno())

    def close(self):
        self.console.close()
        if self.f is not None:
            self.f.close()


import os
import shutil
import imageio
import matplotlib
from matplotlib.figure import Figure
from matplotlib.axes import Axes
import matplotlib.pyplot as plt
from awpy.visualization import plot
from awpy.data import NAV
import networkx as nx
from tqdm import tqdm
import models
import mathing

CT_COLOR: str = "xkcd:azure"
T_COLOR: str = "xkcd:orange"
BOMB_COLOR: str = "yellow"
CT_TILE_COLOR: str = "xkcd:lightblue"
T_TILE_COLOR: str = "darkorange"
SMOKE_COLOR: str = "grey"
FIRE_COLOR: str = "red"
DEFAULT_COLOR: str = "black"

def plot_navigation_mesh(map_name: str, map_type: str, dark: bool) -> tuple[Figure, Axes]:
    """
    Plots the map's navigation mesh
    """

    f: Figure
    ax: Axes

    f, ax = plot.plot_map(map_name = map_name, map_type = map_type, dark = dark)

    for a in NAV[map_name]:
        tile = NAV[map_name][a]
        x_se = plot.position_transform(map_name, tile["southEastX"], "x")
        x_nw = plot.position_transform(map_name, tile["northWestX"], "x")
        y_se = plot.position_transform(map_name, tile["southEastY"], "y")
        y_nw = plot.position_transform(map_name, tile["northWestY"], "y")
        width = (x_se - x_nw)
        height = (y_nw - y_se)
        southwest_x = x_nw
        southwest_y = y_se
        rect = matplotlib.patches.Rectangle((southwest_x,southwest_y), width, height, linewidth=0.2, edgecolor="yellow", facecolor="None")
        ax.add_patch(rect)

    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

    return f, ax

def plot_rounds(demo_file_data, output_file_directory: str) -> None:
    """
    Given a parsed demo file, plot all rounds of the demo and save it to a file
    Output file name shouldn't contain extensions
    """
    total_rounds = len(demo_file_data["gameRounds"])
    for index, round in enumerate(demo_file_data["gameRounds"]):
        # TODO: Make sure that the output is sent to the right place
        print(f"Creating .gif for round {index}/{total_rounds}:")
        plot_round(f"{output_file_directory}/round{index}.gif", round["frames"], map_name=demo_file_data["mapName"])

def plot_round(
    image_directory: str,
    round_number: int,
    round: models.Round,
    vision_graphs: list[nx.Graph],
    vision_traces: list[dict[int, mathing.VisionTraceResults]],
    map_name: str,
    map_type: str = "simpleradar",
    dark: bool = False,
    show_tiles: bool = True,
    gif_frame_rate: int = 2
    ) -> bool:
    """Plots a round and saves as a .gif. CTs are blue, Ts are orange, and the bomb is an octagon. Only use untransformed coordinates.

    Args:
        folder_name (string): folder name to save the frame images and the round gif
        round (list): the round to plot
        map_name (string): Map to search
        map_type (string): "original" or "simpleradar"
        dark (boolean): Only for use with map_type="simpleradar". Indicates if you want to use the SimpleRadar dark map type
        show_tiles: (boolean): The round images will have the outlines of each map tile displayed
        gif_frame_rate (int): The frames per second at which the GIF will play

    """
    image_files: list[str] = []
    frames = round.frames
    grenade_throwers: dict[tuple[float, float, float], str] = {
        (g.grenade_x, g.grenade_y, g.grenade_z): g.thrower_side for g in round.grenades
    }
    with tqdm(total=len(frames), desc = "Drawing frames: ") as progress_bar:
        for index, frame in enumerate(frames):
            vision_graph = vision_graphs[index]
            vision_trace_results = vision_traces[index]
            f, a = plot_frame(
                frame=frame,
                grenade_throwers=grenade_throwers,
                map_graph=vision_graph,
                vision_trace_results=vision_trace_results,
                map_name=map_name,
                map_type=map_type,
                dark=dark,
                show_tiles=show_tiles
            )
            image_files.append(f"{image_directory}/frame_{index+1}.png")
            f.tight_layout(pad=0)
            f.savefig(image_files[-1], dpi=300, bbox_inches="tight", pad_inches=0)
            plt.close()
            progress_bar.update()
    images = []
    for file in image_files:
        images.append(imageio.imread(file))
    imageio.mimsave(f"{image_directory}/round_{round_number}.gif", images, fps=gif_frame_rate)
    return True

def plot_frame(
    frame: models.Frame,
    grenade_throwers: dict[int, str],
    map_graph: nx.Graph,
    vision_trace_results: dict[int, mathing.VisionTraceResults],
    map_name: str,
    map_type: str,
    dark: bool,
    show_tiles: bool = True
    ) -> tuple[Figure, Axes]:
    """
    Plots a frame and returns the figure, axes, and the map graph (containing area controlled information).
    CTs are blue, Ts are orange, and the bomb is an octagon.
    """
    if show_tiles is True:
        f, a = plot_navigation_mesh(map_name, map_type, dark)
    else:
        f, a = plot.plot_map(map_name, map_type, dark)

    x_transform = lambda x: plot.position_transform(map_name, x, "x")
    y_transform = lambda y: plot.position_transform(map_name, y, "y")

    if frame.bomb is not None:
        bomb_x = x_transform(frame.bomb.x)
        bomb_y = y_transform(frame.bomb.y)
        a.scatter(x=bomb_x, y=bomb_y, c=BOMB_COLOR, marker="8")

    team: models.TeamFrameState
    color: str
    for team, color in [(frame.ct, CT_COLOR), (frame.t, T_COLOR)]:
        player_group: list[models.PlayerFrameState] = team.players
        for index, player in enumerate(player_group):

            player_x: float = x_transform(player.x)
            player_y: float = y_transform(player.y)

            if player.hp <= 0:
                a.scatter(x=player_x, y=player_y, c=color, marker="x")
                continue

            a.scatter(x=player_x, y=player_y, c=color, marker=".")

            # Draw lines between this player and other alive players
            for player2 in [p for p in player_group[index:] if p.hp > 0]:
                player2_x = x_transform(player2.x)
                player2_y = y_transform(player2.y)
                x_values = [player_x, player2_x]
                y_values = [player_y, player2_y]
                a.plot(x_values, y_values, c=color, linestyle="dotted", linewidth="0.7")

            aim_end_point = vision_trace_results[player.steam_id].end_points[0]
            # Draw a line to where the player is aiming
            a.scatter(x=x_transform(aim_end_point[0]), y=y_transform(aim_end_point[1]), c=color, marker="2")
            a.plot([player_x, x_transform(aim_end_point[0])], [player_y, y_transform(aim_end_point[1])], c=color, linestyle="solid", linewidth=0.85)

            # Draw lines for every view vector cast
            for end_point in vision_trace_results[player.steam_id].end_points[1:]:
                view_vector_end_point_x = x_transform(end_point[0])
                view_vector_end_point_y = y_transform(end_point[1])
                x_values = [player_x, view_vector_end_point_x]
                y_values = [player_y, view_vector_end_point_y]
                # a.scatter(x=end_point[0], y=end_point[1], c=color, marker="2")
                a.plot(x_values, y_values, c=color, linestyle="dotted", linewidth="0.4", alpha=0.2)

    # Color tiles!
    for node in map_graph.nodes(data=True):
        node_data: dict = node[1]
        controlling_side = node_data.get("controlling_side", None)
        if controlling_side is None:
            continue
        elif controlling_side == "CT":
            color = CT_TILE_COLOR
        elif controlling_side == "T":
            color = T_TILE_COLOR

        area_id: int = node_data["areaID"]
        tile = NAV[map_name][area_id]

        x_se = x_transform(tile["southEastX"])
        x_nw = x_transform(tile["northWestX"])
        y_se = y_transform(tile["southEastY"])
        y_nw = y_transform(tile["northWestY"])
        width = (x_se - x_nw)
        height = (y_nw - y_se)
        southwest_x = x_nw
        southwest_y = y_se

        rect = matplotlib.patches.Rectangle((southwest_x,southwest_y), width, height, linewidth=0.4, edgecolor=color, facecolor=color, alpha=0.2)
        a.add_patch(rect)

    # TODO: Resize markers to match how big they are in game if necessary

    marker_size: int = 400

    def get_thrower_of_closest_grenade(x: float, y: float, z: float) -> str:
        """
        Given a grenade's x, y, z coordinates, find the thrower of the closest grenade
        """
        closest_grenade_id = min(grenade_throwers, key=lambda g: (g[0] - x)**2 + (g[1] - y)**2 + (g[2] - z)**2)
        return grenade_throwers[closest_grenade_id]

    for smoke in frame.smokes:
        smoke_x: float = plot.position_transform(map_name, smoke.x, "x")
        smoke_y: float = plot.position_transform(map_name, smoke.y, "y")
        owner = get_thrower_of_closest_grenade(smoke.x, smoke.y, smoke.z)
        outline_color = CT_COLOR if owner == "CT" else T_COLOR if owner == "T" else SMOKE_COLOR
        a.scatter(x=smoke_x, y=smoke_y, c=SMOKE_COLOR, marker=".", edgecolors=outline_color, s=marker_size,)

    for fire in frame.fires:
        fire_x: float = plot.position_transform(map_name, fire.x, "x")
        fire_y: float = plot.position_transform(map_name, fire.y, "y")
        owner = get_thrower_of_closest_grenade(fire.x, fire.y, fire.z)
        outline_color = CT_COLOR if owner == "CT" else T_COLOR if owner == "T" else FIRE_COLOR
        a.scatter(x=fire_x, y=fire_y, c=FIRE_COLOR, marker=".", edgecolors=outline_color, s=marker_size,)

    return f, a


import copy
import torch
import torch.nn as nn
import torch.optim as optim
from config import config

class Alg():
    def __init__(self, model):
        self.model = model
        self.target_model = copy.deepcopy(self.model)
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.model.to(self.device)
        self.target_model.to(self.device)
        self.gamma = config['gamma']
        self.lr = config['learning_rate']

        self.mse_loss = torch.nn.MSELoss()
        self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)

    def predict(self, state):
        with torch.no_grad():
            pred_q = self.model(state).cpu()
        action = pred_q.argmax(dim=-1).numpy()
        return action

    def learn(self, states, actions, rewards, next_states, done):
        pred_value = self.model(states).gather(1, actions)
        greedy_action = self.model(next_states).max(dim=-1, keepdim=True)[1]
        with torch.no_grad():
            max_v = self.target_model(next_states).gather(1, greedy_action)
            target = rewards + (1 - done) * self.gamma * max_v
        self.optimizer.zero_grad()
        loss = self.mse_loss(pred_value, target)
        loss.backward()
        self.optimizer.step()
        return loss.item()

    def update(self):
        self.target_model.load_state_dict(self.model.state_dict())

    def drop_reset(self, p=0.01):  # 将p部分的网络重置
        for name, h in self.model.named_children():
            if type(h) == nn.Linear:
                m = copy.deepcopy(h)
                m.requires_grad_(False)
                reset_matrix = copy.deepcopy(m)
                mask = (torch.rand(h.weight.shape) > p).float()
                anti_mask = 1 - mask
                mask = mask.clone().detach().to(self.device)
                anti_mask = anti_mask.clone().detach().to(self.device)
                torch.nn.init.normal_(reset_matrix.weight, std=0.01)
                m.weight = torch.nn.Parameter(m.weight * mask, requires_grad=False)
                reset_matrix.weight = torch.nn.Parameter(reset_matrix.weight * anti_mask, requires_grad=False)
                m.weight += reset_matrix.weight
                eval('self.model.' + name + '.load_state_dict(m.state_dict())')


import contextlib
import glob
import logging
import os
import re
import string
import threading

from ngamsLib.ngamsCore import NGAMS_NOT_RUN_STATE,\
    NGAMS_ONLINE_STATE, NGAMS_SUBSCRIBE_CMD,\
    NGAMS_SUCCESS, TRACE, genLog, NGAMS_DISK_INFO, checkCreatePath,\
    NGAMS_SUBSCRIBER_THR, NGAMS_UNSUBSCRIBE_CMD, NGAMS_HTTP_INT_AUTH_USER,\
    loadPlugInEntryPoint, toiso8601, fromiso8601
from ngamsLib import ngamsStatus, ngamsLib, ngamsHttpUtils
from ngamsLib import ngamsPhysDiskInfo
from ngamsLib import ngamsSubscriber
from ngamsLib import ngamsHighLevelLib, ngamsDiskUtils
from ngamsLib import ngamsNotification
import ngamsArchiveUtils
import ngamsSubscriptionThread

logger = logging.getLogger(__name__)

def ngamsBaseExitHandler(srvObj):
    """
    NG/AMS Exit Handler Function. Is invoked when the NG/AMS Server
    is killed terminated.

    srvObj:      Reference to NG/AMS server class object (ngamsServer).

    signalNo:    Number of signal received (integer).

    killServer:  If set to 1 the server will be killed by the
                 function (integer/0|1).

    exitCode:    The exit code with which NG/AMS should exit (integer).

    delPidFile:  Flag indicating if NG/AMS PID file should be deleted or
                 not (integer/0|1).

    Returns:     Void.
    """
    logger.info("NG/AMS Exit Handler - cleaning up ...")

    if (srvObj.getState() == NGAMS_ONLINE_STATE):
        handleOffline(srvObj)

    # Update the ngas_hosts table.
    srvObj.updateHostInfo("", None, 0, 0, 0, 0, 0, NGAMS_NOT_RUN_STATE)

def _create_remote_subscriptions(srvObj, stop_evt):
    """
    Creates subscriptions in remote servers so they push their data into
    ours.
    """

    subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic()
    subscriptions = [v for _,v in subscriptions_in_cfg.items()]
    subscriptions_created = 0

    while True:

        # Done with all of them
        if not subscriptions:
            break

        # Iterate over copy, since we modify the original inside the loop
        for subscrObj in list(subscriptions):

            if (not srvObj.getThreadRunPermission()):
                logger.info("Terminating Subscriber thread ...")
                return

            our_host, our_port = srvObj.get_self_endpoint()
            subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()

            # Not subscribing to ourselves
            if srvObj.is_it_us(subs_host, subs_port):
                logger.warning("Skipping subscription to %s:%d because that's us", subs_host, subs_port)
                return

            # Create the URL that needs to be set on the remote end so we
            # get subscribed to it.
            # TODO: include reverse proxy information when we add support
            # TODO: hardcoded http will need to be changed when we add support
            #       for https
            url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.getUrl() or 'QARCHIVE')
            logger.info("Creating subscription to %s:%d with url=%s", subs_host, subs_port, url)

            pars = [["subscr_id", url],
                    ["priority", subscrObj.getPriority()],
                    ["url",      url],
                    ["start_date", toiso8601(local=True)]]
            if subscrObj.getFilterPi():
                pars.append(["filter_plug_in", subscrObj.getFilterPi()])
            if subscrObj.getFilterPiPars():
                pars.append(["plug_in_pars", subscrObj.getFilterPiPars()])

            try:
                # Issue SUBSCRIBE command
                resp = ngamsHttpUtils.httpGet(subs_host, subs_port, cmd=NGAMS_SUBSCRIBE_CMD, pars=pars)
                with contextlib.closing(resp):
                    stat = ngamsStatus.to_status(resp, subscrObj.getHostId(), NGAMS_SUBSCRIBE_CMD)

                if stat.getStatus() != NGAMS_SUCCESS:
                    msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later"
                    logger.warning(msg, stat.getStatus(), stat.getMessage())
                    continue

                subscriptions_created += 1
                logger.info("Successfully subscribed to %s:%d with url=%s", subs_host, subs_port, subscrObj.getUrl())

                # Remember to unsubscribe when going Offline.
                srvObj.getSubscrStatusList().append(subscrObj)
                subscriptions.remove(subscrObj)

            except:
                logger.exception("Error while adding subscription, will try later")

        if stop_evt.wait(10):
            return

    if subscriptions_created:
        logger.info("Successfully established %d Subscription(s)", subscriptions_created)
    else:
        logger.info("No Subscriptions established")

def getDiskInfo(srvObj,
                reqPropsObj = None):
    """
    Invokes the plug-in listed in the configuration to extract the information
    about the disks mounted. The information is returned in a dictionary which
    contains the Slot IDs as keys, and elements which each is an object of type
    ngamsPhysDiskInfo.

    srvObj:        Reference to NG/AMS Server class instance (ngamsServer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    Returns:       Dictionary containing information about the
                   disks (dictionary).
    """
    T = TRACE()

    diskInfoDic = {}

    plugIn = srvObj.getCfg().getOnlinePlugIn()
    if (not srvObj.getCfg().getSimulation()):
        logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn)
        plugInMethod = loadPlugInEntryPoint(plugIn)
        diskInfoDic = plugInMethod(srvObj, reqPropsObj)
        if (len(diskInfoDic) == 0):
            if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())):
                errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL",
                                [srvObj.getHostId(),
                                 srvObj.getHostId()])
                logger.warning(errMsg)
    else:
        if (srvObj.getCfg().getAllowArchiveReq()):
            logger.debug("Running as a simulated archiving system - generating " +\
                 "simulated disk info ...")
            portNo = 0
            slotIdLst = srvObj.getCfg().getSlotIds()
            for slotId in slotIdLst:
                if (slotId != ""):
                    storageSet = srvObj.getCfg().\
                                 getStorageSetFromSlotId(slotId)
                    if (storageSet.getMainDiskSlotId() == slotId):
                        diskType = "Main"
                    else:
                        diskType = "Rep"
                    mtRootDir = srvObj.getCfg().getRootDirectory()
                    mtPt = os.path.normpath(mtRootDir +\
                                            "/"+storageSet.getStorageSetId() +\
                                            "-" + diskType + "-" + str(slotId))
                    diskId = re.sub("/", "-", mtPt)[1:]
                    serialNo = "SERIAL-NUMBER-" + str(portNo)
                    manufact = "TEST-MANUFACTURER"
                    diskInfoDic[slotId] = ngamsPhysDiskInfo.ngamsPhysDiskInfo()
                    diskInfoDic[slotId].\
                                          setPortNo(portNo).\
                                          setSlotId(slotId).\
                                          setMountPoint(mtPt).\
                                          setStatus("OK").\
                                          setCapacityGb(100).\
                                          setModel("TEST-MODEL").\
                                          setSerialNo(serialNo).\
                                          setType("TEST-TYPE").\
                                          setManufacturer(manufact).\
                                          setDiskId(diskId).\
                                          setDeviceName("/dev/dummy" + slotId)
                    # We create the mount directory if it does not exist
                    checkCreatePath(mtPt)

                    portNo = portNo + 1
        else:
            # It is not an Archiving System. We check which of the directories
            # in the Ngas.RootDirectory contain an NgasDiskInfo file.
            # These are considered as available disks in simulation mode.
            baseDir = srvObj.getCfg().getRootDirectory()
            dirList = glob.glob(os.path.normpath(baseDir + "/*"))
            for mtPt in dirList:
                checkPath = os.path.normpath(mtPt + "/" + NGAMS_DISK_INFO)
                logger.debug("Checking if path exists: %s", checkPath)
                if (os.path.exists(checkPath)):
                    slotId = string.split(mtPt, "-")[-1]
                    portNo = (int(slotId) - 1)
                    diskId = re.sub("/", "-", mtPt)
                    serialNo = "SERIAL-NUMBER-" + str(portNo)
                    diskInfoDic[slotId]=ngamsPhysDiskInfo.ngamsPhysDiskInfo().\
                                         setPortNo(portNo).\
                                         setSlotId(slotId).\
                                         setMountPoint(mtPt).\
                                         setStatus("OK").\
                                         setCapacityGb(100).\
                                         setModel("TEST-MODEL").\
                                         setSerialNo(serialNo).\
                                         setType("TEST-TYPE").\
                                         setManufacturer("TEST-MANUFACTURER").\
                                         setDiskId(diskId).\
                                         setDeviceName("/dev/dummy" + slotId)

    logger.debug("Disk Dictionary: %s", str(diskInfoDic))
    return diskInfoDic

def handleOnline(srvObj,
                 reqPropsObj = None):
    """
    Initialize the system. This implies loading the NG/AMS
    Configuration making the internal initialization from this,
    obtaining information about the disk configuration.

    srvObj:         Reference to NG/AMS server class object (ngamsServer).

    reqPropsObj:    NG/AMS request properties object (ngamsReqProps).

    Returns:        Void.
    """
    logger.debug("Prepare NGAS and NG/AMS for Online State ...")

    # Re-load Configuration + check disk configuration.
    srvObj.loadCfg()

    hostId = srvObj.getHostId()
    for stream in srvObj.getCfg().getStreamList():
        srvObj.getMimeTypeDic()[stream.getMimeType()] = stream.getPlugIn()

    # Flush/send out possible retained Email Notification Messages.
    flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\
               "forced at Online"
    ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1, flushMsg)

    # Get possible Subscribers from the DB.
    subscrList = srvObj.getDb().getSubscriberInfo("", hostId,
                                                  srvObj.getCfg().getPortNo())
    num_bl =  srvObj.getDb().getSubscrBackLogCount(hostId, srvObj.getCfg().getPortNo())
    #debug_chen
    if (num_bl > 0):
        logger.debug('Preset the backlog count to %d', num_bl)
        srvObj.presetSubcrBackLogCount(num_bl)

    # TODO: unify this "db-to-object" reading (there is something similar elsewhere,
    #       I'm pretty sure, and hide these low-level details from here
    logger.info("Creating %d subscrption objects from subscription DB info", len(subscrList))
    for subscrInfo in subscrList:
        start_date = fromiso8601(subscrInfo[5], local=True) if subscrInfo[5] else None
        last_ingested_date = fromiso8601(subscrInfo[8], local=True) if subscrInfo[8] else None
        tmpSubscrObj = ngamsSubscriber.ngamsSubscriber(subscrInfo[0],
                                                       subscrInfo[1],
                                                       subscrInfo[2],
                                                       subscrInfo[4],
                                                       start_date,
                                                       subscrInfo[6],
                                                       subscrInfo[7],
                                                       last_ingested_date,
                                                       subscrInfo[3])
        tmpSubscrObj.setConcurrentThreads(subscrInfo[9])
        # Take only subscribers for this NG/AMS Server.
        if ((tmpSubscrObj.getHostId() == hostId) and
            (tmpSubscrObj.getPortNo() == srvObj.getCfg().getPortNo())):
            #srvObj.getSubscriberDic()[tmpSubscrObj.getId()] = tmpSubscrObj
            #if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0):
            #   break #only load one subscriber under the data mover mode
            srvObj.registerSubscriber(tmpSubscrObj)

    try:
        # Get information about the disks of this system.
        srvObj.setDiskDic(getDiskInfo(srvObj, reqPropsObj))
        ngamsArchiveUtils.resetDiskSpaceWarnings()

        # Update disk info in DB.
        ngamsDiskUtils.checkDisks(hostId, srvObj.getDb(), srvObj.getCfg(),
                                  srvObj.getDiskDic())

        # Write status file on disk.
        ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(),
                                            srvObj.getDb(), srvObj.getCfg())
    except Exception:
        errMsg = "Error occurred while bringing the system Online"
        logger.exception(errMsg)
        handleOffline(srvObj)
        raise

    # Check that there is enough disk space in the various system directories.
    srvObj.checkDiskSpaceSat()

    # Check if files are located in the Staging Areas of the Disks. In case
    # yes, move them to the Back-Log Area to have them treated later.
    checkStagingAreas(srvObj)

    # Start threads + inform threads that they are allowed to execute.
    srvObj.setThreadRunPermission(1)
    srvObj.startJanitorThread()
    srvObj.startDataCheckThread()
    ngamsSubscriptionThread.startSubscriptionThread(srvObj)
    srvObj.startUserServiceThread()
    srvObj.startMirControlThread()
    srvObj.startCacheControlThread()

    # Change state to Online.
    srvObj.setState(NGAMS_ONLINE_STATE)

    # If specified in the configuration file that this server should act as a
    # Subscriber, then subscribe to the Data Providers specified. We do this
    # in a thread so that the server can continuously try to subscribe,
    # even after errors occur.
    if (srvObj.getCfg().getSubscrEnable()):
        srvObj.remote_subscription_creation_evt = threading.Event()
        thrObj = threading.Thread(target=_create_remote_subscriptions,
                                  name=NGAMS_SUBSCRIBER_THR,
                                  args=(srvObj, srvObj.remote_subscription_creation_evt))
        thrObj.setDaemon(0)
        thrObj.start()

    logger.info("NG/AMS prepared for Online State")

def handleOffline(srvObj,
                  reqPropsObj = None):
    """
    Carry out the actions to put the system in Offline state (Standby).

    srvObj:          Reference to NG/AMS server class object (ngamsServer).

    reqPropsObj:     NG/AMS request properties object (ngamsReqProps).

    stopJanitorThr:  Setting this to 0, the function will not try to
                     stop the Janitor Thread. The reason for this is that
                     the Janitor Thread may bring the server to Offline
                     (integer/0|1).

    Returns:         Void.
    """
    # Stop/delete Janitor Thread + Data Check Thread + inform other
    # possible threads to stop execution (if running).
    srvObj.stopJanitorThread()
    srvObj.stopDataCheckThread()
    ngamsSubscriptionThread.stopSubscriptionThread(srvObj)
    srvObj.stopUserServiceThread()
    srvObj.stopMirControlThread()
    srvObj.stopCacheControlThread()
    srvObj.setThreadRunPermission(0)
    if srvObj.getCfg().getSubscrEnable():
        srvObj.remote_subscription_creation_evt.set()

    logger.debug("Prepare NG/AMS for Offline State ...")

    # Unsubscribe possible subscriptions. This is tried only once.
    if (srvObj.getCfg().getAutoUnsubscribe()):
        for subscrObj in srvObj.getSubscrStatusList():
            subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()
            try:
                resp = ngamsHttpUtils.httpGet(subs_host, subs_port, NGAMS_UNSUBSCRIBE_CMD, [["url", subscrObj.getId()]])
                with contextlib.closing(resp):
                    stat = ngamsStatus.to_status(resp, "%s:%d" % (subs_host, subs_port), NGAMS_UNSUBSCRIBE_CMD)
                if stat.getStatus() != NGAMS_SUCCESS:
                    logger.error("Error when unsubscribing from %s:%d: %s", subs_host, subs_port, stat.getMessage())
            except Exception:
                msg = "Problem occurred while cancelling subscription " +\
                      "(host/port): %s/%d. Subscriber ID: %s"
                logger.exception(msg, subscrObj.getHostId(),
                                 subscrObj.getPortNo(), subscrObj.getId())
        srvObj.resetSubscrStatusList()

    # Check if there are files located in the Staging Areas of the
    # Disks. In case yes, move these to the Back-Log Area to have them
    # treated at a later stage.
    checkStagingAreas(srvObj)

    # Dump disk info on all disks, invoke the Offline Plug-In to prepare the
    # disks for offline, and mark the disks as unmounted in the DB.
    ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(),
                                        srvObj.getDb(), srvObj.getCfg())
    plugIn = srvObj.getCfg().getOfflinePlugIn()
    if (srvObj.getCfg().getSimulation() == 0):
        logger.info("Invoking System Offline Plug-In: %s(srvObj, reqPropsObj)", plugIn)
        plugInMethod = loadPlugInEntryPoint(plugIn)
        plugInRes = plugInMethod(srvObj, reqPropsObj)
    else:
        pass
    ngamsDiskUtils.markDisksAsUnmountedInDb(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg())

    # Send out possible Retained Email Notification Messages.
    flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\
               "forced at Offline"
    ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1, flushMsg)

    logger.info("NG/AMS prepared for Offline State")

def wakeUpHost(srvObj,
               suspHost):
    """
    Wake up a host which has been suspended. After invoking the appropriate
    Wake-Up Plug-In it is checked within the time-out defined in the
    NG/AMS Configuration File if the server, which was woken up is up
    and running. If this is not the case within the specified time-out,
    an exception is thrown.

    srvObj:         Reference to instance of the NG/AMS server (ngamsServer).

    suspHost:       Host name of the suspended host (string).

    Returns:        Void.
    """
    T = TRACE()

    wakeUpPi = srvObj.getCfg().getWakeUpPlugIn()
    portNo = srvObj.getDb().getPortNoFromHostId(suspHost)
    try:
        plugInMethod = loadPlugInEntryPoint(wakeUpPi)
        plugInMethod(srvObj, suspHost)

        ipAddress = srvObj.getDb().getIpFromHostId(suspHost)
        ngamsHighLevelLib.pingServer(ipAddress, portNo,
                                     srvObj.getCfg().getWakeUpCallTimeOut())
    except Exception:
        logger.exception("Error waking up host %s", suspHost)
        raise

def checkStagingAreas(srvObj):
    """
    Check the Staging Areas of the disks registered, and in case files
    are found on these move htese to the Back-Log Area.

    srvObj:    Reference to NG/AMS server class object (ngamsServer).

    Returns:   Void.
    """
    T = TRACE()

    diskList = ngamsDiskUtils.\
               getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
                                          srvObj.getCfg().\
                                          getRootDirectory())
    # Generate first a dictionary with all files in the staging areas.
    stagingFileDic = {}
    for disk in diskList:
        stagingArea = disk.getStagingArea()
        if (stagingArea != ""):
            fileList = glob.glob(stagingArea + "/*")
            fileList += glob.glob(stagingArea + "/.NGAMS*")
            for filename in fileList: stagingFileDic[filename] = 1
    # Go through all files in the staging file dictionary and move them to
    # the Bad Files Area.
    for filename in stagingFileDic.keys():
        ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)

def genIntAuthHdr(srvObj):
    """
    Generate an internal HTTP Authentication Header if possible.

    srvObj:    Reference to NG/AMS server class object (ngamsServer).

    Returns:   Authentication code or '' if no internal NGAS user defined
               (string).
    """
    # If the NGAS Internal Authorization User is defined generate
    # an internal Authorization Code.
    if (srvObj.getCfg().hasAuthUser(NGAMS_HTTP_INT_AUTH_USER)):
        authHttpHdrVal = srvObj.getCfg().\
                         getAuthHttpHdrVal(NGAMS_HTTP_INT_AUTH_USER)
    else:
        authHttpHdrVal = ""
    return authHttpHdrVal

# EOF


# -*- coding: utf-8 -*-
"""
Created on Tue Nov  5 09:12:30 2019

@author: George Barker and Andre Zeromski
@author: watso
"""

import csv
import numpy as np

from nltk.tokenize import word_tokenize
from gensim.models import Word2Vec
from collections import Counter
from nltk.corpus import stopwords

class RNN:

    def __init__(self, word_dim, hidden_dim, bptt_truncate=4):
        # Assign instance variables
        self.word_dim = word_dim
        self.hidden_dim = hidden_dim
        self.bptt_truncate = bptt_truncate
        # Randomly initialize the network parameters
        self.U = np.random.uniform(-np.sqrt(1./word_dim), np.sqrt(1./word_dim), (hidden_dim, word_dim))
        self.V = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (word_dim, hidden_dim))
        self.W = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (hidden_dim, hidden_dim))

def wordToVec(sentences):
    """ Given an array of tokenized words returns a Word2Vec model """
    model = Word2Vec(sentences, size = 10, min_count=1, iter = 5)
    return model

def tokenizeLines(arrayOfLines):
    """ Takes in a array of lines of text and returns tokenized lines """
    i = 0
    for line in arrayOfLines:
        arrayOfLines[i] = tokenizeText(line[0])
        i += 1
    flattened  = [val for sublist in arrayOfLines for val in sublist]
    vocabulary = findVocab(flattened)
    l = 0
    for line in arrayOfLines:
        w = 0
        for word in line:
            if word not in vocabulary:
                arrayOfLines[l][w] = "UNK"
            w += 1
        arrayOfLines[l].insert(0,"START")
        arrayOfLines[l].append("END")
        l += 1
    return arrayOfLines

def tokenizeText(text):
    """ Takes in a string of text and returns an array of tokenized words """
    # tokenize words based on seperating whitespace
    tokens = word_tokenize(text)
    # lowercase all words
    tokens = [token.lower() for token in tokens]
    # remove stop words
    stopWords = set(stopwords.words('english'))
    tokens = [word for word in tokens if not word in stopWords]
    return tokens

def findVocab(listOfWords):
    """ Take in a tokenized array of words and returns a list of top 8000 words """
    counterWords = Counter(listOfWords)
    most_occur = counterWords.most_common(8000)
    vocab = []
    for position in most_occur:
        vocab.append(position[0])
    return vocab

def importData(filename):
    """ Return the text from the file where each line is an element in an array """
    f = open(filename)
    csv_f = csv.reader(f)
    text = []
    for row in csv_f:
        text.append(row)
    return text

def main():
#    vocabulary_size = 8000
#    unknown_token = "UNK"
#    sentence_start_token = "START"
#    sentence_end_token = "END"
    text = importData("rnnDataset.csv")
    tokenizedTextLines = tokenizeLines(text)
    model = wordToVec(tokenizedTextLines)
    # Print first 10 tokenized words and vectorized 10 tokenized words
    for line in tokenizedTextLines[:10]:
        print()
        print(line)
        for word in line:
            print(word)
            print(model.wv.__getitem__(word))
    print()
    print(model)

if __name__ == "__main__":
    main()


# Import von Flask.
from flask import Flask, jsonify, render_template, request
from collections import defaultdict
# Import CSV-Datei mit Daten.
import csv

# Benennung der CSV-Datei.
filename = 'data/Avocado_v6.csv'

# Daten der Avocados werden eingelesen.
with open(filename, 'r') as csvfile:
    # Daten aus 'data/Avocado_v6.csv' werden in ein Dictionary konvertiert und einzelne Datenfelder werden durch ';' getrennt.
    datareader = csv.DictReader(csvfile, delimiter=';')
    # Daten werden als Liste abgespeichert.
    data = list(datareader)

app = Flask(__name__)

# Funktionen der Startseite.
@app. route('/', methods=['GET'])
def index():
    return render_template('index_avotastic.html')

# Funktion jsonify wandelt Daten in JSON-Daten um. Route besteht um korrektes Datenauslesen zu überprüfen.
@app. route('/data', methods=['GET'])
def dataview():
    return jsonify(data)

# Funktionen für das Volumen der Daten für ein Jahr.
@app. route('/dataperyear', methods=['GET'])
def dataperyear():
    # Hier wird das Jahr aus dem Request geholt.
    year = request.args.get("year")

    # Alle Zeilen für das Jahr werden geholt.
    filtereddata = [row for row in data if row["year"] == year]

    # Hier wird ein Dictionary erzeugt und das Volumen für jeden Bundesstaat berechnet. Der Wert ist 0.
    d = defaultdict(lambda: 0)
    for row in filtereddata:
        # Kuerzel steht für Abkürzung des Bundesstaates. Diesen wird jeweils das Volumen der verkaufen Avocados zugeordnet.
        # Volumen wird durch 1000 geteilt, damit die Kommastellen der Zahl sich verkleinern für mehr Übersichtlichkeit.
        d[row["Kuerzel"]] += (float(row["Total Volume"])/1000)

    # Hier wird die Liste erstellt, die im main.js in den Graphen geladen werden kann.
    liste = []
    for code, value in d.items():
        # Der Wert und Code wird der Liste angehängt.
        liste.append({"value":value, "code":code})

    # Funktion jsonify wandelt Daten in JSON-Daten um und gibt sie wieder als Liste zurück.
    return jsonify(liste)

if __name__ == '__main__':
    app.run(debug=True, port=5000)


#!/usr/bin/python3

import pathlib
import pandas as pd

# define the location of the enron dataset
enron_postgres = pathlib.Path('/opt/enron_processed/enron_postgresql/')

# read users table
users_table = enron_postgres / 'users.csv'
users_df = pd.read_csv(users_table, encoding="utf-8")

# count number of unique emails vs number of all emails
print(users_df['user_email'].nunique() / len(users_df['user_email']))

# drop duplicates emails
users_df = users_df.drop_duplicates(subset=['user_email'])

# export to csv
output_path = enron_postgres / 'unique_users.csv'
users_df.to_csv(output_path, index=False)