import os
import socket
import sys
import time

from PyQt5 import uic
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *

def resource_path(relative_path):
    """ Get absolute path to resource, works for dev and for PyInstaller """
    base_path = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(base_path, relative_path)

form = resource_path('QuizClient.ui')
form_class = uic.loadUiType(form)[0]

class connectQuizServer(QThread):
    sig_msg = pyqtSignal(str)
    sig_err = pyqtSignal(str)

    def __init__(self, parent):
        super().__init__(parent)

        self.HOST = ''
        self.PORT = 0
        self.msg = ''
        self.tmpSocket = ''

    def setInfo(self, host, port):
        self.HOST = host
        self.PORT = port

    def sendMsg(self, sock, msg):
        try:
            sock.send(msg.encode())
        except Exception as e:
            print('sendMsg: ' + e)
            pass

    def rcvMsg(self, sock):
        while True:
            try:
                data = sock.recv(1024)
                if not data:
                    break

                self.sig_msg.emit(data.decode())
                print(data.decode())
            except:
                pass

    def run(self):
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
            self.tmpSocket = sock
            try:
                sock.connect((self.HOST, self.PORT))
                if self.msg != '':
                    msg = self.msg
                    self.sendMsg(sock, msg)
                    self.msg = ''

                while True:
                    try:
                        data = sock.recv(1024)
                        if not data:
                            break

                        self.sig_msg.emit(data.decode())
                        print(data.decode())
                    except Exception as e:
                        pass
            except Exception as e:
                self.sig_err.emit(str(e))
                print('connect exception')

    def stop(self):
        if self.isRunning():
            self.sendMsg(self.tmpSocket, '/quit')
            self.tmpSocket = ''
            self.terminate()

class clientWindow(QMainWindow, form_class):
    isMe = False

    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.setFixedSize(490, 570)
        self.username = ''
        self.quizSvr = connectQuizServer(self)
        self.quizSvr.sig_msg.connect(self.incomingChat)
        self.quizSvr.sig_err.connect(self.errorChat)

        self.pb_connect.clicked.connect(self.runQuizSvr)
        self.pb_tbClear.clicked.connect(self.clearText)
        self.le_inputText.returnPressed.connect(self.sendMsg)
        self.lbl_question.setStyleSheet('color: red')

    def sendMsg(self):
        if self.quizSvr.isRunning():
            self.quizSvr.sendMsg(self.quizSvr.tmpSocket, self.le_inputText.text())
            self.le_inputText.clear()

            if self.lbl_question.text() != '' and self.isMe:
                self.isMe = False
                self.le_inputText.setEnabled(False)

    def runQuizSvr(self):
        host = self.le_hostip.text()
        port = int(self.le_hostport.text())
        self.username = self.le_username.text()
        if host and port:
            if self.username:
                if self.quizSvr.isRunning():
                    self.quizSvr.msg = '/quit'
                    try:
                        self.quizSvr.stop()
                    except Exception as e:
                        self.quizSvr.terminate()
                        pass
                    print('연결이 종료되었습니다.')
                    self.incomingChat('연결이 종료되었습니다.\n')
                    self.lbl_question.setText('')
                    self.le_inputText.setEnabled(True)
                    self.le_hostip.setEnabled(True)
                    self.le_hostport.setEnabled(True)
                    self.le_username.setEnabled(True)
                    self.pb_connect.setText('연결')
                else:
                    self.quizSvr.setInfo(host, port)
                    self.quizSvr.msg = self.username
                    self.quizSvr.start()
                    self.le_hostip.setEnabled(False)
                    self.le_hostport.setEnabled(False)
                    self.le_username.setEnabled(False)
                    self.pb_connect.setText('연결 끊기')

            else:
                self.tb_content.append('이름을 입력해주세요.')
                print('이름을 입력해주세요.')
        else:
            self.tb_content.append('IP와 포트를 입력해주세요.')

    def clearText(self):
        self.tb_content.clear()

    @pyqtSlot(str)
    def incomingChat(self, msg):
        text = ''
        if msg.find('*Q:') != -1:
            self.lbl_question.setText(msg.replace('*Q:', ''))
            if msg.replace('*Q:', '') != '':
                self.le_inputText.setEnabled(False)
        elif msg.find('*출제자:') != -1:
            presenter = msg.replace('*출제자:', '')
            text = '출제자는 초성(또는 단어)을 입력하여 퀴즈를 시작해주세요. (출제자: ' + presenter + ')'
            if presenter != self.username:
                self.le_inputText.setEnabled(False)
            else:
                self.le_inputText.setEnabled(True)
        elif msg.find('*/next:') != -1:
            if msg.replace('*/next:', '') != self.username:
                self.isMe = False
                self.le_inputText.setEnabled(False)
            else:
                self.isMe = True
                self.le_inputText.setEnabled(True)
        elif msg.find('*/reset:') != -1:
            self.le_inputText.setEnabled(True)
        elif msg.find('*/') != -1:
            text = msg.replace('*/', '')
            # 틀렸으면 다시 오픈
            if text.find('몽총이') != -1 or text.find('틀렸습니다') != -1 or text.find('초성 불가') != -1:
                self.le_inputText.setEnabled(True)
        else:
            text = msg
        # elif msg.find('######## 초성 퀴즈 ########') != -1:
        #     text = msg
        # else:
        #     text = time.strftime('%H:%M:%S', time.localtime(time.time())) + '  ' + msg

        if text:
            self.tb_content.append(text)

    @pyqtSlot(str)
    def errorChat(self, errmsg):
        self.tb_content.append(errmsg)
        self.le_hostip.setEnabled(True)
        self.le_hostport.setEnabled(True)
        self.le_username.setEnabled(True)
        self.pb_connect.setText('연결')

if __name__ == "__main__":
    # QApplication : 프로그램을 실행시켜주는 클래스
    app = QApplication(sys.argv)
    # WindowClass의 인스턴스 생성
    myWindow = clientWindow()
    # 프로그램 화면을 보여주는 코드
    myWindow.show()
    # 프로그램을 이벤트루프로 진입시키는(프로그램을 작동시키는) 코드
    app.exec_()


from collections import defaultdict

import pandas as pd
import os
import sys
import numpy as np
from numpy import sort
import copy
import math
import cmath
import requests
import io
import json
import jsonpath

import logging
from ponall import config,logconfig

logconfig.setup_logging()
log = logging.getLogger("ponall.extract_feature")
# constant
a_list = ('A', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'K', 'L', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'V', 'W', 'Y')
aa_list = [i + j for i in a_list for j in a_list]
log.debug("aaindex path: %s", os.path.abspath(config.aa_index_path))
aaindex = pd.read_csv(config.aa_index_path, sep="\t", header=None, names=['name'] + aa_list, index_col='name')
aaindex = aaindex.T

def check_aa(seq, aa):
    seq = "".join(seq.split())
    aaf = aa[0]  # from
    aai = int(aa[1:-1])  # index
    aat = aa[-1]  # to
    return seq, aaf, aat, aai

#record 错误信息记录：
def msg_find(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    msg = ""
    if aaf not in a_list:
        msg = "aa error, origin of aa is invalid."
    if aat not in a_list:
        msg = "aa error, nutation of aa is invalid."
    if aai < 1 or aai > len(seq):
        msg = "aa error, index of aa is invalid."
    if seq[aai - 1] != aaf:
        msg = "aa error, seq[{}] = {}, but origin of aa = {}".format(aai, seq[aai - 1], aaf)
    return msg

def get_aaindex(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    res = aaindex.loc["{}{}".format(aaf, aat), :]
    return res.to_dict()

def get_residue(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    res = {"mut_residue": aai}
    return res

def get_length(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    res = {"len": len(seq)}
    return res

def get_nutationAll(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    nutationAll = pd.DataFrame(columns=[i[0] + '.' + i[1] for i in aa_list], index=aa_list).fillna(0)
    np.fill_diagonal(nutationAll.values, 1)
    return nutationAll.loc[[aaf+aat]].to_dict(orient='records')[0]

def get_groupAll(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    group = {
        'g1': ['V', 'I', 'L', 'F', 'M', 'W', 'Y', 'C'],
        'g2': ['D', 'E'],
        'g3': ['R', 'K', 'H'],
        'g4': ['G', 'P'],
        'g5': ['N', 'Q', 'S'],
        'g6': ['A', 'T']
    }

    # 颠倒键值对用于映射
    group_r = {}
    for k, v in group.items():
        for i in v:
            group_r[i] = k
    # 全对角矩阵
    groupAll = pd.DataFrame(
        index=[i + '.' + j for i in group.keys() for j in group.keys()],
        columns=[i + '.' + j for i in group.keys() for j in group.keys()],
    ).fillna(0)
    np.fill_diagonal(groupAll.values, 1)
    return groupAll.loc[[group_r[aaf]+'.'+group_r[aat]]].to_dict(orient='records')[0]

def get_neighborhood_features(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)

    def find_win(seq, aai):
        # 确定边界
        index = aai - 1
        front = 0 if index - 11 < 0 else index - 11
        after = len(seq) if index + 12 > len(seq) - 1 else index + 12
        return seq[front: after]

    def get_count_a(win):
        """ count number of aa in windows"""
        a_dict = defaultdict(int)
        for i in win:
            a_dict[i] += 1  # 递增
        return {'AA20D.' + i: a_dict[i] for i in a_list}

    win = find_win(seq, aai)
    count_a = get_count_a(win)
    nei_feature = count_a
    # 1.NonPolarAA:Number of nonpolar neighborhood residues
    nei_feature['NonPolarAA'] = nei_feature['AA20D.' + a_list[0]] + nei_feature['AA20D.' + a_list[4]] + nei_feature[
        'AA20D.' + a_list[5]] + nei_feature['AA20D.' + a_list[7]] + nei_feature['AA20D.' + a_list[9]] + nei_feature[
                                    'AA20D.' + a_list[10]] + nei_feature['AA20D.' + a_list[12]] + nei_feature[
                                    'AA20D.' + a_list[17]] + nei_feature['AA20D.' + a_list[18]] + nei_feature[
                                    'AA20D.' + a_list[19]]
    # 2.PolarAA:Number of polar neighborhood residues
    nei_feature['PolarAA'] = nei_feature['AA20D.' + a_list[1]] + nei_feature['AA20D.' + a_list[11]] + nei_feature[
        'AA20D.' + a_list[13]] + nei_feature['AA20D.' + a_list[15]] + nei_feature['AA20D.' + a_list[16]]
    # 3.ChargedAA:Number of charged neighborhood residues
    nei_feature['ChargedAA'] = nei_feature['AA20D.' + a_list[2]] + nei_feature['AA20D.' + a_list[3]] + nei_feature[
        'AA20D.' + a_list[6]] + nei_feature['AA20D.' + a_list[8]] + nei_feature['AA20D.' + a_list[14]]
    # 4.PosAA:Number of Positive charged neighborhood residues
    nei_feature['PosAA'] = nei_feature['AA20D.' + a_list[2]] + nei_feature['AA20D.' + a_list[3]]
    # 5.NegAA:Number of Negative charged neighborhood residues
    nei_feature['NegAA'] = nei_feature['AA20D.' + a_list[6]] + nei_feature['AA20D.' + a_list[8]] + nei_feature[
        'AA20D.' + a_list[14]]
    return nei_feature

#Sift4G
# def get_sift4g(id, seq, aa):
    # seq, aaf, aat, aai = check_aa(seq, aa)
    # with open('seq.fa', 'w') as file_object:
        # file_object.write('>'+id+'\n')
        # file_object.write(seq)
    # if not os.path.exists('subst'):
        # os.makedirs('subst')
    # if not os.path.exists('sift_out'):
        # os.makedirs('sift_out')
    # with open('./subst/{}.subst'.format(id), 'w') as file_object:
        # file_object.write(aa)
    # os.system('sift4g -q ./seq.fa --subst ./subst/ -d ../sift4g/uniprot_sprot.fasta --out ./sift_out/')
    # files = ['./sift_out/{}.SIFTprediction'.format(id)]
    # files
    # sift = []
    # for file in files:
        # with open(file) as f:
            # 读取文件
            # tmp = f.read()
            # tmp = tmp.strip().split('\n')  # 每行分割
            # gi序号
        # (filepath, tempfilename) = os.path.split(file)
        # (filename, extension) = os.path.splitext(tempfilename)
            # 组成json
        # for i in tmp:
            # tmp_1 = []
            # if '\t' in i:
                # tmp_1 = i.split('\t')
                # sift.append({
                    # 'hits': int(tmp_1[5]),
                    # 'score': float(tmp_1[2])
                # }
                # )
    # sift = [{"hits": 29,'score':0.48}]
    # return sift[0]

def get_sift4g_file(id, seq):
    with open('seq.fa', 'w') as file_object:
        file_object.write('>'+id+'\n')
        file_object.write(seq+'\n\n')
    if not os.path.exists('sift_out_file'):
        os.makedirs('sift_out_file')
    os.system('sift4g -q ./seq.fa -d ../sift4g/uniprot_sprot.fasta --out ./sift_out_file/')
    filename = './sift_out_file/{}.SIFTprediction'.format(id)
    filename1 = './sift_out_file/{}.SIFTprediction1'.format(id)
    fin = open(filename, 'r')
    a = fin.readlines()
    for i in a:
        i = i.strip()
    fout = open(filename1, 'w')
    l = a[5:]
    l.insert(0, 'A  B  C  D  E  F  G  H  I  K  L  M  N  P  Q  R  S  T  V  W  X  Y  Z  *  -\n')
    b = ''.join(l)
    fout.write(b)

    sift = pd.read_csv(filename1,sep='  ')
    print(sift)
    # sift[aat][aai+1]
    return sift

def get_sift4g_hits(id, seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    with open('seq.fa', 'w') as file_object:
        file_object.write('>'+id+'\n')
        file_object.write(seq+'\n\n')
    if not os.path.exists('subst'):
        os.makedirs('subst')
    if not os.path.exists('sift_out'):
        os.makedirs('sift_out')
    with open('./subst/{}.subst'.format(id), 'w') as file_object:
        file_object.write(aa)
    os.system('sift4g -q ./seq.fa --subst ./subst/ -d ../sift4g/uniprot_sprot.fasta --out ./sift_out/')
    files = ['./sift_out/{}.SIFTprediction'.format(id)]
    files
    sift = []
    for file in files:
        with open(file) as f:
            # 读取文件
            tmp = f.read()
            tmp = tmp.strip().split('\n')  # 每行分割
            # gi序号
        (filepath, tempfilename) = os.path.split(file)
        (filename, extension) = os.path.splitext(tempfilename)
        #     组成json
        for i in tmp:
            tmp_1 = []
            if '\t' in i:
                tmp_1 = i.split('\t')
                sift.append({
                    'hits': int(tmp_1[5]),
                    'score': float(tmp_1[2])
                }
                )
    return sift[0]['hits']

def get_GO(id):
    requestURL = "https://www.ebi.ac.uk/QuickGO/services/annotation/search?geneProductId={}".format(id)

    r = requests.get(requestURL, headers={"Accept": "application/json"})

    if not r.ok:
        r.raise_for_status()
        sys.exit()

    responseBody = r.text
    responseBody = json.loads(responseBody)
    gos = jsonpath.jsonpath(responseBody, '$..goId')
    if gos!=False:
        #查找祖先
        ancestor = pd.read_csv(config.ancestor_path, sep=',', header=None, low_memory=False).set_index(0)

        l = []
        for i in gos:
            if i in ancestor.index:
                l = l + ancestor.loc[i, :].tolist()
            l.append(i)
        l = list(set(l))
        if np.nan in l:
            l.remove(np.nan)
        if 'all' in l:
            l.remove('all')
        gos = ','.join(l)

    res = {"ancestor": gos}
    return res

def get_pos_1(seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    pos_1=0
    if aai==1:
        pos_1=1
    res = {"pos_1": pos_1}
    return res

def get_Site_1(id):
    site=''
    if not os.path.exists('Site'):
        os.makedirs('Site')
    url = "https://www.uniprot.org/uniprot/{}.gff".format(id)
    urlData = requests.get(url).content
    rawData = pd.read_csv(io.StringIO('\n'.join(urlData.decode('utf-8').split('\n')[2:])), sep="\t", header=None,
                          names=['ID', 'database', 'site', 'from', 'to', '1', '2', '3', '4', '5'])
    return rawData

def get_Site_2(rawData, seq, aa):
    seq, aaf, aat, aai = check_aa(seq, aa)
    l = []
    for indexs2 in rawData.index:
        if rawData.loc[indexs2]['from'] <= aai and rawData.loc[indexs2]['to'] >= aai:
            l.append(rawData.loc[indexs2, 'site'])
    l = list(set(l))
    site = ','.join(l)
    res = {"site": site}
    return res

def get_all_features(n, seq, aa, kind):
    print('收集特征')
    list = []
    #err_list = []
    if (kind in ['uniprot id','ensembl id','vcf']):
        print('有GO')
        flag='*'
        GOs=''
        Sites=''
        sift_hits=0
        sift=None
        for i in range(len(n)):
            id_ = n[i]
            seq_ = seq[i]
            aa_ = aa[i]
            features = {}
            features.update({"id" : i})
            features.update({"msg": msg_find(seq_, aa_)})
            if msg_find(seq_, aa_)=="":
                if id_ != flag:
                    sift_hits = get_sift4g_hits(id_, seq_, aa_)
                    sift = get_sift4g_file(id_, seq_)
                    GOs = get_GO(id_)
                    Sites = get_Site_1(id_)
                    flag = id_
                seq1, aaf1, aat1, aai1 = check_aa(seq_, aa_)
                print("hits:{}".format(sift_hits),'score:{}'.format(sift[aat1][aai1-1]))
                features.update(get_residue(seq_, aa_))
                features.update(get_length(seq_, aa_))
                features.update(get_aaindex(seq_, aa_))
                features.update(get_nutationAll(seq_, aa_))
                features.update(get_groupAll(seq_, aa_))
                features.update(get_neighborhood_features(seq_, aa_))
                features.update({"hits": sift_hits,'score':sift[aat1][aai1-1]})
                features.update(GOs)
                features.update(get_pos_1(seq_, aa_))
                features.update(get_Site_2(Sites, seq_, aa_))
                df_features = pd.DataFrame([features])
                list.append(df_features)
            else:
                df_features = pd.DataFrame([features])
                # err_list.append(df_features)
                list.append(df_features)
    else:
        print('无GO')
        flag='*'
        sift_hits=0
        sift=None
        for i in range(len(seq)):
            id_ = n[i]
            seq_ = seq[i]
            aa_ = aa[i]
            features = {}
            features.update({"id" : i})
            features.update({"msg": msg_find(seq_, aa_)})
            if msg_find(seq_, aa_) == "":
                if id_ != flag:
                    sift_hits = get_sift4g_hits(id_, seq_, aa_)
                    sift = get_sift4g_file(id_, seq_)
                    flag = id_
                seq1, aaf1, aat1, aai1 = check_aa(seq_, aa_)
                print("hits:{}".format(sift_hits),'score:{}'.format(sift[aat1][aai1-1]))
                features.update(get_residue(seq_, aa_))
                features.update(get_length(seq_, aa_))
                features.update(get_aaindex(seq_, aa_))
                features.update(get_nutationAll(seq_, aa_))
                features.update(get_groupAll(seq_, aa_))
                features.update(get_neighborhood_features(seq_, aa_))
                features.update({"hits": sift_hits,'score':sift[aat1][aai1-1]})
                features.update(get_pos_1(seq_, aa_))
                df_features = pd.DataFrame([features])
                list.append(df_features)
            else:
                df_features = pd.DataFrame([features])
                list.append(df_features)
                # err_list.append(df_features)
    df2 = pd.concat(list)
    df2 = df2.reset_index()
    del df2['index']
    print('特征')
    print(df2)
    return df2
    # df1 = pd.concat(err_list)
    # df1 = df1.reset_index()
    # del df1['index']
    # print('错误')
    # print(df1)
    # return df1, df2


from fastapi import FastAPI
from connexion import DB

app = FastAPI()

@app.get("/")
def read_root():
    return {"Hello": "World"}

# aux infos d'un restaurant à partir de son id,
@app.get("/api/info/{id}")
async def get_info_by_id(id=None):
    return DB.get_info_by_id(id)

# à la liste des noms de restaurants à partir du type de cuisine,
@app.get("/api/name/{type_cuisine}")
async def get_resto_by_type(type_cuisine: str):
    return DB.get_name_by_type(type_cuisine)

# au nombre d'inspection d'un restaurant à partir de son id restaurant,
@app.get("/api/inspection/{idrestaurant}")
async def get_inspection_by_idrestaurant(idrestaurant: int):
    return DB.get_nb_inspec_by_id(idrestaurant)

# les noms des 10 premiers restaurants d'un grade donné.
@app.get("/api/classement/{grade}")
async def get_classement_by_grade(grade: str):
    return DB.get_top_10(grade)

# http://localhost:88/api/info/50041578
# http://localhost:88/api/name/Asian
# http://localhost:88/api/inspection/40859175
# http://localhost:88/api/classement/B


from re import sub
from django.db import connection
from .models import AlbumDetails, SongDescription

def get_music_menu_album_list(language):
    return AlbumDetails.objects.filter(language=language) \
        .select_related('album') \
        .order_by('-album__release_date') \
        .only(
            'album__id', 'album__release_date',
            'title', 'slug', 'album_id'
        )

def get_all_songs(language, album_id=None):
    songs = SongDescription.objects.select_related('song').filter(language=language)
    songs = songs.order_by('-song__album_id', 'song__album_order', 'song__release_date')
    songs = songs.only(
        'song__id',
        'song__album_id',
        'song__original_title',
        'song__length',
        'song__slug',
        'song__is_hidden',
        'title'
    )

    if album_id:
        songs.filter(song__album_id=album_id)

    return songs

def get_album_language_urls(album_id):
    albums = AlbumDetails.objects.filter(album_id=album_id).only('album_id', 'language', 'slug')

    slugs = {}
    for album in albums:
        slugs[album.language] = '%d-%s/' % (album.album_id, album.slug)
    return slugs

def get_album_durations():
    with connection.cursor() as cursor:
        cursor.execute(
            'SELECT \
                TIME(SUM(STRFTIME("%s", length)), "unixepoch") AS album_length, \
                album_id \
            FROM music_song \
            WHERE is_hidden = 0 \
            GROUP BY album_id'
        )

        return [
            {'album_id': row[1], 'album_duration': sub('00:0?', '', row[0])} for row in cursor.fetchall()
        ]


import sys
import os
import numpy as np
from pyproj import Proj, Transformer, CRS
import multiprocessing as mp
import pandas as pd
import argparse
from tqdm import tqdm

AUSTIN_PROJ4_DESC = "+proj=lcc \
    +lat_1=31.8833333333333 +lat_2=30.1166666666667 +lat_0=29.6666666666667 \
    +lon_0=-100.333333333333 +x_0=2296583.333333 +y_0=9842500 \
    +datum=NAD83 +units=m +no_defs"

ERR_UB = 2000

parser = argparse.ArgumentParser(description="Preprocess the GPS-data")
parser.add_argument('inputDir', type=str, metavar="DirIn", help='Raw dataset Dir')
parser.add_argument('outputDir', type=str, metavar="DirOut", help='Output Dir')
parser.add_argument("--size", type=int, metavar='Size', help='GeoHash Block Size', default=ERR_UB)
parser.add_argument("--thread", type=int, help='Multiprocessing Thread Pool Size', default=4)
parser.add_argument("--mapLbX", type=float, default=30.18)
parser.add_argument("--mapLbY", type=float, default=-97.92)

crsWGS84 = CRS.from_epsg(4326)
crsAustin = CRS.from_proj4(AUSTIN_PROJ4_DESC)
transformer = Transformer.from_crs(crsWGS84, crsAustin)

def applyTransform(wgs):
    return transformer.transform(wgs[0], wgs[1])

def preproc(filePath, fileName, outputPath, tqdmHandle, blockSize):
    stat = pd.DataFrame()   # Collect the statistic data

    # Load the dataset
    tqdmHandle.set_description("Loading Dataset")
    dataset = pd.read_csv(os.path.join(filePath, fileName))
    stat.loc[0, 'RawPings'] = dataset.shape[0]

    # Clean the accuracy - Drop pings with large acc error
    acc = dataset.loc[:, ('horizontal_accuracy', 'vertical_accuracy')].fillna(np.inf)
    err = np.linalg.norm(acc, ord=2, axis=1)
    errPings = np.logical_or(err == 0, err > ERR_UB)
    dataset.drop(dataset[errPings].index, inplace=True, axis=0)

    # Sort the data according to the time
    dataset = dataset.sort_values(by='location_at')

    # Get the map boundary
    wgsMapLb = np.zeros(2)
    wgsMapLb[0] = args.mapLbX
    wgsMapLb[1] = args.mapLbY
    mapLb = np.array(transformer.transform(wgsMapLb[0], wgsMapLb[1]))

    # Convert to local cartesian coordinate
    tqdmHandle.set_description("Convert the CRS")
    pos = dataset.loc[:, ("latitude", "longitude")]
    with mp.Pool(args.thread) as pool:
        localPos = pool.map(applyTransform, pos.values)
    localPos = np.array(localPos)

    # Get the geohash coordinate
    geohashBlk = np.around((localPos - mapLb) / blockSize).astype(np.int)

    # Remove blocks out of boundary
    validIdx = np.all(geohashBlk >= 0, axis=1)
    oobCount = np.sum(~validIdx)
    dataset = dataset[validIdx]
    tqdmHandle.set_description(f"{oobCount} pings out of boundary!")
    localPos = localPos[validIdx, :]
    geohashBlk = geohashBlk[validIdx, :]

    # Insert new data
    dataset.insert(7, 'x', localPos[:, 0])
    dataset.insert(8, 'y', localPos[:, 1])
    dataset.insert(9, 'bx', geohashBlk[:, 0])
    dataset.insert(10, 'by', geohashBlk[:, 1])
    outputPath = os.path.join(outputPath, fileName)
    tqdmHandle.set_description(f"Writing CSV to {outputPath}")
    # Export the CSV and write the stat
    dataset.to_csv(outputPath)
    stat.loc[0, 'Filename'] = fileName
    stat.loc[0, 'FilteredPings'] = dataset.shape[0]
    stat.loc[0, 'OOBPings'] = oobCount
    stat.loc[0, 'AccErrPings'] = np.sum(errPings)
    stat.loc[0, 'latitudeLb'] = wgsMapLb[0]
    stat.loc[0, 'longitudeLb'] = wgsMapLb[1]
    stat.loc[0, 'xLb'] = mapLb[0]
    stat.loc[0, 'yLb'] = mapLb[1]
    return stat

if __name__ == '__main__':
    args = parser.parse_args()
    blockSize = np.array((args.size, args.size))
    os.makedirs(args.outputDir, exist_ok=True)

    # Find all the CSV files
    csvFiles = []
    for path, _, files in os.walk(args.inputDir):
        for file in files:
            if file[-4:] == '.csv':
                csvFiles.append((path, file))
    print(f"Found {len(csvFiles)} files under {args.inputDir}")

    # Run!
    fileLoader = tqdm(csvFiles)
    stat = None
    for p, f in fileLoader:
        ret = preproc(p, f, args.outputDir, fileLoader, blockSize)
        if stat is None:
            stat = ret
        else:
            stat = pd.concat((stat, ret), ignore_index=True)

    # Export the stat data
    statPath = os.path.join(args.outputDir, "stat.csv")
    print(f"Export the stat to {statPath}")
    stat.to_csv(statPath)
    print("Done")


import argparse
import os
import time
import traceback

import matplotlib.pyplot as plt
import torch
from pytorch_lightning.utilities.seed import seed_everything

from scr.data import get_dataset
from scr.generative_models import get_model
from scr.notify import post_message

if __name__ == "__main__":
    seed_everything(42)

    n_ensemble = 5
    train, val, _, _ = get_dataset("mnist")

    def process_data(dataset):
        data, labels = dataset.dataset.tensors
        data = data[labels==3]
        labels = labels[labels==3]
        data = data.round()
        return torch.utils.data.TensorDataset(data, labels)

    train = process_data(train)
    val = process_data(val)

    train_dataloader = torch.utils.data.DataLoader(train, batch_size=64)
    val_dataloader = torch.utils.data.DataLoader(val, batch_size=64)

    model_class = get_model("vae")
    model = model_class.fit(n_ensemble, 1, train_dataloader, val_dataloader)
    model.eval()

    classifier = torch.nn.Sequential(
        torch.nn.Flatten(),
        torch.nn.Linear(784, 100),
        torch.nn.ReLU(),
        torch.nn.Dropout(p=0.1),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 1)
    )

    train_img = train.tensors[0]
    reg_score = train_img.sum(dim=[1,2,3])
    classifier_data = torch.utils.data.DataLoader(
        torch.utils.data.TensorDataset(train_img, reg_score),
        batch_size=64
    )
    data_loader = iter(classifier_data)

    opt = torch.optim.Adam(classifier.parameters(), lr=1e-3)
    count = 0
    while count < 10000:
        opt.zero_grad()

        try:
            batch = next(data_loader)
        except StopIteration:
            data_iter = iter(classifier_data)
            batch = next(data_iter)

        img, score = batch
        pred = classifier(img).squeeze()
        loss = (pred - score).pow(2.0).sum()
        if count % 100 == 0:
            print(loss)
        loss.backward()
        opt.step()
        count += 1

    classifier.eval()

    latent_coords = torch.nn.Parameter(torch.randn(1, 2))
    opt = torch.optim.SGD([latent_coords], lr=1e-2)

    count = 0
    while count < 10000:
        opt.zero_grad()
        recon = model.decoder(latent_coords)
        pred = classifier(recon)
        loss = -pred.sum()
        loss.backward()
        if count % 100 == 0:
            print(loss, latent_coords)
        opt.step()
        count += 1

    from bayes_opt import BayesianOptimization

    def g(x, y):
        return model.decoder(torch.tensor([[x, y]], dtype=torch.float)).detach()

    def f(x, y):
        recon = g(x, y)
        pred = classifier(recon)
        return pred.item()

    pbounds = {'x': (-5, 5), 'y': (-5, 5)}

    optimizer = BayesianOptimization(
        f=f,
        pbounds=pbounds,
        verbose=2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
        random_state=1,
    )
    optimizer.maximize()

    grid = torch.stack(
        torch.meshgrid(
            [torch.linspace(-10, 10, 50) for _ in range(2)], indexing='ij'
        )
    ).reshape(2, -1).T
    recons = torch.stack([g(z[0], z[1]) for z in grid], 0).squeeze().reshape(50, 50, 28, 28)
    bigimg = torch.zeros(28*50, 28*50)
    for i in range(50):
        for j in range(50):
            bigimg[28*i:28*(i+1), 28*j:28*(j+1)] = recons[i, j]

    #             train_end = time.time()

    #             os.makedirs("models/generative_models/", exist_ok=True)
    #             model_class.save_checkpoint(
    #                 model, f"models/generative_models/{model_name}_{dataset_name}.pt"
    #             )
    #         except Exception as e:
    #             print(f"Exception happened:")
    #             traceback.print_exc()
    #             post_message(
    #                 "the following exception happended: \n" f"{traceback.format_exc()}"
    #             )


from django.shortcuts import render

# Create your views here.

from .models import LbBiOrderFlow as LbOf
from django.http import HttpResponse, JsonResponse
from django.db.models import Q
import json
from django.core.paginator import Paginator, InvalidPage, EmptyPage
import datetime

# Create your views here.
def orderflow(request):
    """课程"""
    course_datetime1 = request.GET.get("course_datetime1")
    course_datetime2 = request.GET.get("course_datetime2")
    # course_state = request.GET.get('course_state')
    each_page = 30
    if course_datetime1 or course_datetime2:
        aQ1 = Q()
        aQ2 = Q()
        list1 = []
        if course_datetime1 or course_datetime2 is not None:

            start_date = datetime.datetime.strptime(course_datetime1, "%Y-%m-%d")
            end_data = datetime.datetime.strptime(course_datetime2, "%Y-%m-%d") + datetime.timedelta(days=1)

            print(start_date)
            aQ1.add(Q(course_datetime__range=(start_date, end_data)), Q.OR)
            aQ2.add(Q(course_datetime__range=(start_date, end_data)), Q.AND)
            list1.append(course_datetime1)
        print(aQ1)
        print(list1)
        page = int(request.GET.get('page', '1'))
        if len(list1) == 1:
            all_orders = LbOf.objects.filter(aQ1)
            if page == -1:
                p_num = LbOf.objects.filter(aQ1).count()
                print(p_num)
                contacts = all_orders
            else:
                paginator = Paginator(all_orders, each_page)
                p_num = paginator.count
                print(p_num)
                try:
                    contacts = paginator.page(page)
                    print(contacts)
                except(EmptyPage, InvalidPage):
                    contacts = paginator.page(paginator.num_pages)
        else:
            all_orders = LbOf.objects.filter(aQ2)
            if page == -1:
                p_num = LbOf.objects.filter(aQ2).count()
                print(p_num)
                contacts = all_orders
            else:
                paginator = Paginator(all_orders, each_page)
                p_num = paginator.count
                print(p_num)
                try:
                    contacts = paginator.page(page)
                    print(contacts)
                except(EmptyPage, InvalidPage):
                    contacts = paginator.page(paginator.num_pages)
    else:
        try:
            all_orders = LbOf.objects.all()
        except LbOf.DoesNotExist:
            return HttpResponse(status=404)
        page = int(request.GET.get('page', '1'))
        if page == -1:
            return JsonResponse({'code': 400, 'msg': '数据量太多，请输入查询时间'})
        else:
            paginator = Paginator(all_orders, each_page)
            p_num = paginator.count
            print(p_num)
            try:
                contacts = paginator.page(page)
                print(contacts)
            except(EmptyPage, InvalidPage):
                contacts = paginator.page(paginator.num_pages)
    resultList = []
    print(contacts)
    for index in contacts:
        resultList += [{
            "combo_name": index.combo_name,
            "class_number": index.class_number,
            "partial_delivery": index.partial_delivery,
            "pending_delivery": index.pending_delivery,
            "waiting_network_test": index.waiting_network_test,
            "waiting_garden_confirm": index.waiting_garden_confirm,
            "all_delivery": index.all_delivery,
            "order_complete": index.order_complete,
            "no_plan": index.no_plan,
            "is_plan": index.is_plan,
            "class_corrected_number": index.class_corrected_number,
            "arranged_formal_courses": index.arranged_formal_courses,
            "confirm_time": index.confirm_time,
        }]

    # 返回值
    # response = JsonResponse(resultList, safe=False)
    # response.status_code = 500  自定义响应码
    return JsonResponse({"code": 200, "msg": "操作成功", "total": p_num, "data": resultList})


import matplotlib.pyplot as plt
import numpy as np
import serial
import time

Fs = 300

t = np.arange(0,10,1/10)

x = np.zeros(100)
y = np.zeros(100)
z = np.zeros(100)

serdev = '/dev/ttyACM0'

s = serial.Serial(serdev,115200)

for i in range(300):
    line=s.readline()
    #print(line)
    if(i%3 == 0): x[int(i/3)] = float(line)
    elif(i%3 == 1): y[int(i/3)] = float(line)
    elif(i%3 == 2): z[int(i/3)] = float(line)

disa = np.zeros(100)
disb = np.zeros(100)
dis = np.zeros(100)
great =  np.zeros(100)
for i in range(100):
    tu = 0.05
    if (i==0):
        disa[i]=0
        disb[i]=0
    else:
        disa[i]=disa[i-1]+x[i-1]*9.8/2*0.1*0.1
        disb[i]=disb[i-1]+y[i-1]*9.8/2*0.1*0.1
    dis[i] = np.sqrt(disa[i]*disa[i]+disb[i]*disb[i])
    if dis[i] > tu:   great[i] = 1
    else:   great[i] = 0

fig, ax = plt.subplots(2, 1)

ax[0].plot(t,x, label = 'x')
ax[0].plot(t,y, label = 'y')
ax[0].plot(t,z, label = 'z')
ax[0].set_xlabel('Time')
ax[0].set_ylabel('Acc Vector')
ax[0].legend()

ax[1].stem(t,great)
ax[1].set_xlabel('Time')
ax[1].set_ylabel('displacement larger than 5')

plt.show()

s.close()


from tensorhive.core import ssh
from tensorhive.core.ssh import HostsConfig, ProxyConfig, Hostname, Username
from pssh.clients.native import ParallelSSHClient
from typing import List, Optional, Dict, Iterator, Tuple
import time
from datetime import datetime
import logging
log = logging.getLogger(__name__)

__author__ = '@micmarty'
__all__ = ['ExitCodeError', 'SpawnError', 'spawn', 'terminate', 'running', 'fetch_log']
"""
This module provides functionality for spawning commands on host machines via ssh.
It's divided into 3 parts:
    1) command builder(s):
        Classes that hold a set of commands (as strings) which can be launched by task executor
        in order to achieve some result, e.g. `spawn` and `terminate`
        You can implement your own command builder that uses different backend, currently it's `screen` program.

    2) task executor:
        Launches commands provided by command builder.
        It contains core logic for e.g. spawning, terminating processes on remote hosts via ssh.

    3) stateless API functions:
        Provides high-level interface for operations on remote host, like:
        * spawning, terminating processes
        * getting info about running processes
        * log fetching

Author's note:
    1) The ONLY recommended way to use this module from outside is by using
    exposed API functions (see __all__)
    # TODO Write tests
"""

# Custom exception names
class ExitCodeError(AssertionError):
    pass

class SpawnError(Exception):
    pass

class ScreenCommandBuilder:
    """Set of configurable commands built on top of `screen` program."""

    @staticmethod
    def spawn(command: str,
              session_name: str,
              capture_output: bool = True,
              custom_log_name: Optional[str] = None,
              keep_alive: bool = False) -> str:
        """Command that runs inside daemonized screen session.

        Command is put into background manually (-D + & instead of simply using -d) -> we want to know the PID
        Ways to capture output:
            * standard way: keep session alive, user resumes and inspects it by himself.
                May need to use some workarounds if we want to know when command stops.
            * alternative way: screen should run ``script some_output.log -c "sleep 20; echo TensorHive"`
            * tee way (implemented here): redirect command's stdout + stderr from within `screen` with `tee`
                When used with -i, --ignore-interrupts options it won't accept SIGINT so that
                the main command will catch it instead of `tee`. Use case for this is when your command
                prints something important after SIGINT and you want to see it + put that into log file.
            * the screen way: use `screen -L -Logfile name.log`, unfortunately
                              not all `screen` versions supports that feature.

        Side effects (incompatible with capture_output=True):
            * keep_alive will prevent saving command's output to log file.
            It is advised to keep this turned off, unless you really want to keep session
            alive even when the command has finished execution - sometimes useful!

        Note: `custom_log_name` argument will be ignored when `capture_output=False`

        """
        if capture_output:
            if keep_alive:
                log.debug('Forcing keep_alive=False - incompatible with capture_output=True')
                keep_alive = False

            if custom_log_name:
                create_logfile_command = ScreenCommandBuilder.custom_log_file(custom_log_name)
            else:
                create_logfile_command = ScreenCommandBuilder.tmp_log_file()
            # | -> stdout only, |& -> stdout + stderr (Bash 4), 2>&1 (old Bash)
            capturing_command = '|& tee --ignore-interrupts $({})'.format(create_logfile_command)

        return 'screen -Dm -S {sess_name} bash -c "{cmd}{keep_alive} {log}" {to_bg}'.format(
            sess_name=session_name,  # will help distinguishing between TensorHive and user's sessions
            cmd=command,
            log=capturing_command if capture_output else '',  # see docstring
            keep_alive='; exec sh' if keep_alive else '',  # see docstring
            to_bg='& echo $!'  # force printing process pid
        )

    @staticmethod
    def mkdir(target_dir: str) -> str:
        """Command that creates any inexisting directory given in path"""
        return 'mkdir --parents ' + target_dir

    @staticmethod
    def tmp_log_file() -> str:
        """Command that creates tmp file for storing log content.

        When command is executed it will print path to newly created file.
        Target directory will be created automatically when executing.
        Current implementation uses week number for easier differentiation.
        """
        target_dir = '~/TensorHiveLogs'
        week_no = datetime.today().strftime('%U')  # type: str
        name_template = 'week_' + week_no + '_XXXX'  # at least 3 'X' are required by mktemp

        mkdir_cmd = ScreenCommandBuilder.mkdir(target_dir)
        mktemp_cmd = 'mktemp {dir}/{templ} --suffix .log'.format(dir=target_dir, templ=name_template)
        return mkdir_cmd + ' && ' + mktemp_cmd

    @staticmethod
    def custom_log_file(filename: str, target_dir: str = '~/TensorHiveLogs') -> str:
        """Command that prints path to custom log file (`filename`)

        Note: if log file already exists `tee` will overwrite it!
        Target directory will be created automatically when executing, but
        log file won't be created (`touch`ed), assuming that `tee` will create it when executing.
        """
        mkdir_cmd = ScreenCommandBuilder.mkdir(target_dir)
        echopath_cmd = 'echo {dir}/{name}.log'.format(dir=target_dir, name=filename)
        return mkdir_cmd + ' && ' + echopath_cmd

    @staticmethod
    def interrupt(pid: int) -> str:
        """Command that sends SIGINT to screen session. Should be used to gracefully terminate running command."""
        return 'screen -S {} -X stuff "^C"'.format(pid)

    @staticmethod
    def terminate(pid: int) -> str:
        """Command that terminates screen session using only pid (we don't need the full name: 1234.tensorhive)."""
        return 'screen -X -S {} quit'.format(pid)

    @staticmethod
    def kill(pid: int) -> str:
        """Command that kills screen session by pid.

        It should also wipe dead sessions. Note that `kill` exit code is returned!
        """
        return 'kill -9 {}; KILL_EXIT=$?; screen -wipe; (exit $KILL_EXIT)'.format(pid)

    @staticmethod
    def get_active_sessions(grep_pattern: str) -> str:
        """Fetches the full names of screen sessions matching given grep pattern."""
        return 'screen -ls | cut -f 2 | sed -e "1d;$d" | grep -e "{}"'.format(grep_pattern)

class Task:
    """Represents task executed on one machine."""

    __slots__ = ['hostname', 'command', 'pid', '_command_builder']

    def __init__(self, hostname: str, command: str = None, pid: int = None):
        self.hostname = hostname
        self.command = command
        self.pid = pid
        # Here you can replace with your own backend/command builder
        self._command_builder = ScreenCommandBuilder

    def spawn(self, client: ParallelSSHClient, name_appendix: Optional[str] = None) -> int:
        """Spawns command via ssh client.
        Returns:
            pid of the process
        """
        sess_name = 'tensorhive_task'
        log_name = None
        if name_appendix:
            sess_name += '_' + name_appendix
            log_name = 'task_' + name_appendix

        command = self._command_builder.spawn(
            self.command, session_name=sess_name, capture_output=True, custom_log_name=log_name, keep_alive=False)
        output = ssh.run_command(client, command)
        stdout = ssh.get_stdout(host=self.hostname, output=output)

        if not stdout:
            reason = output[self.hostname].exception
            raise ValueError('Unable to acquire pid from empty stdout, reason: {}'.format(reason))
        # FIXME May want to decouple it somehow
        # FIXME pop() may theoretically fail (never stumbled upon this issue)
        pid = stdout.split().pop()
        self.pid = int(pid)
        return self.pid

    """
    Author's note: These three methods below are nearly identical, but I wanted
    to keep them separate in order to easily extend their logic in the future.
    """

    def terminate(self, client: ParallelSSHClient) -> int:
        """Terminates the task using it's pid.

        Returns exit code of the operation
        """
        assert self.pid, 'You must first spawn the task or provide pid manually.'
        command = self._command_builder.terminate(self.pid)
        output = ssh.run_command(client, command)
        exit_code = output[self.hostname].exit_code
        return exit_code

    def interrupt(self, client: ParallelSSHClient) -> int:
        """Interrupts the task gracefully by sending SIGINT signal

        Returns exit code of the operation
        """
        assert self.pid, 'You must first spawn the task or provide pid manually.'
        command = self._command_builder.interrupt(self.pid)
        output = ssh.run_command(client, command)
        exit_code = output[self.hostname].exit_code
        return exit_code

    def kill(self, client: ParallelSSHClient) -> int:
        """Kills the task using it's pid.

        Returns exit code of the operation
        """
        assert self.pid, 'You must first spawn the task or provide pid manually.'
        command = self._command_builder.kill(self.pid)
        output = ssh.run_command(client, command)
        exit_code = output[self.hostname].exit_code
        return exit_code

def spawn(command: str, host: Hostname, user: Username, name_appendix: Optional[str] = None) -> int:
    """Stateless, high-level interface for spawning process on remote host.

    name_appendix: string that will be attached to session name and log file name
        Example: appendix='99' will produce session='tensorhive_task_99', log='task_99.log'
    Returns pid of new process
    """
    config, pconfig = ssh.build_dedicated_config_for(host, user)
    client = ssh.get_client(config, pconfig)
    task = Task(host, command)
    try:
        pid = task.spawn(client, name_appendix)
    except ValueError as e:
        raise SpawnError('{} on {}@{} failed: {}'.format(command, user, host, e))
    else:
        log.debug('Command spawned, pid: {}'.format(pid))
        return pid

def terminate(pid: int, host: Hostname, user: Username, gracefully: Optional[bool] = True) -> int:
    """Stateless, high-level interface for terminating process on remote host.

    gracefully: determines how task will be stopped
        False:  SIGKILL - almost guaranteed termination, aggressive
        None:   SIGTERM - works in most cases, but rather aggresive
        True:   SIGINT - often does not work, but only this method allows for capturing logs when program is closing
    Returns exit code of termination operation, not running process
    """
    config, pconfig = ssh.build_dedicated_config_for(host, user)
    client = ssh.get_client(config, pconfig)
    task = Task(host, pid=pid)

    if gracefully is None:
        exit_code = task.terminate(client)
    elif gracefully is False:
        exit_code = task.kill(client)
    else:
        exit_code = task.interrupt(client)
    return exit_code

def running(host: Hostname, user: Username) -> List[int]:
    """Stateless, high-level interface for getting a list of running processes on remote host.

    Ignores sessions with names other than `pattern`
    Returns a list of pids
    """
    config, pconfig = ssh.build_dedicated_config_for(host, user)
    client = ssh.get_client(config, pconfig)
    pattern = '.*tensorhive_task.*'
    command = ScreenCommandBuilder.get_active_sessions(pattern)
    output = ssh.run_command(client, command)
    stdout = ssh.get_stdout(host, output)
    if not stdout:
        return []

    # '4321.foobar_session' -> 4321
    pid_from_session_name = lambda name: int(name.split('.')[0])
    pids = [pid_from_session_name(line) for line in stdout.split('\n')]
    log.debug('Running pids: {}'.format(pids))
    return pids

def fetch_log(host: Hostname, user: Username, task_id: int, tail: bool = False) -> Tuple[Iterator[str], str]:
    """Stateless, high-level interface for fetching log files from remote host.

    Seeks and reads files located under specific folder on remote host.
    Re-raises pssh exceptions.
    tail: whether to include full content or only last 10 lines
    """
    # TODO Path should be configurable from config files (see config.py)
    path = '~/TensorHiveLogs/task_{}.log'.format(task_id)
    program = 'tail' if tail else 'cat'
    command = '{} {}'.format(program, path)

    config, pconfig = ssh.build_dedicated_config_for(host, user)
    client = ssh.get_client(config, pconfig)
    output = ssh.run_command(client, command)

    if output[host].exception:
        # Propagage ssh exception
        raise output[host].exception
    if output[host].exit_code != 0:
        raise ExitCodeError(path)
    return output[host].stdout, path


# 1.2 report 3
#ex 11.1.1

from matplotlib.pyplot import figure, show, legend, plot, xlabel
import numpy as np
from scipy.io import loadmat
from __init__ import clusterplot
from sklearn.mixture import GaussianMixture
from clean_data import *

# LOAD AND TRANSFORM DATA

data1 = clean_data('Datasets/**videos.csv')
data = transform_data(data1,['likes','dislikes','views','comment_count','trending_time'])
index = [data1.category_id[id] in [10,15,20,22] for id in range(len(data1.category_id))]
data['class'] = data1['category_id']
data = data[index]
data =data.head(5000) #first n rows
#print(len(data))
#data = data.sample(100000)
#print(data1)
X = np.array(data[['likes','dislikes','views','comment_count','trending_time']])
y = np.array(data['class'])
attributeNames = ['likes','dislikes','views','comment_count','trending_time']
classNames = ["Mu","Ga","PA","PB"]
#classNames = [name[0][0] for name in mat_data['classNames']]
N, M = X.shape
C = len(classNames)

# Number of clusters
K = 4
cov_type = 'full' # e.g. 'full' or 'diag'

# define the initialization procedure (initial value of means)
initialization_method = 'random'#  'random' or 'kmeans'
# random signifies random initiation, kmeans means we run a K-means and use the
# result as the starting point. K-means might converge faster/better than
# random, but might also cause the algorithm to be stuck in a poor local minimum

reps = 1
# number of fits with different initalizations, best result will be kept

# Fit Gaussian mixture model
gmm = GaussianMixture(n_components=K, covariance_type=cov_type, n_init=reps,
                      tol=1e-6, reg_covar=1e-6, init_params=initialization_method).fit(X)
cls = gmm.predict(X)
# extract cluster labels
cds = gmm.means_
# extract cluster centroids (means of gaussians)
covs = gmm.covariances_

# extract cluster shapes (covariances of gaussians)
if cov_type.lower() == 'diag':
    new_covs = np.zeros([K,M,M])

    count = 0
    for elem in covs:
        temp_m = np.zeros([M,M])
        new_covs[count] = np.diag(elem)
        count += 1

    covs = new_covs

# Plot results:
from categories import find_cat_labels

cat_labels = find_cat_labels('Datasets/CA_category_id.json')
classLabels = [cat_labels[i] for i in ["10","15","20","22"]]

#figure(figsize=(14,9))
legend(classLabels)
#clusterplot(X, clusterid=cls, centroids=cds, y=y, covars=covs)
#show()

## In case the number of features != 2, then a subset of features most be plotted instead.
figure(figsize=(14,9))
idx = [0,1] # feature index, choose two features to use as x and y axis in the plot
clusterplot(X[:,idx], clusterid=cls, centroids=cds[:,idx], y=y, covars=covs[:,idx,:][:,:,idx])
show()

## In case the number of features != 2, then a subset of features most be plotted instead.
#figure(figsize=(14,9))
#idx = [0,1] # feature index, choose two features to use as x and y axis in the plot
#clusterplot(X[:,idx], clusterid=cls, centroids=cds[:,idx], y=y, covars=covs[:,idx,:][:,:,idx])
#show()

print('Ran Exercise 11.1.1')

#from ex 11.1.5:

from sklearn import model_selection

# Range of K's to try
KRange = range(1,15)
T = len(KRange)

covar_type = 'full'       # you can try out 'diag' as well
reps = 3                  # number of fits with different initalizations, best result will be kept
init_procedure = 'kmeans' # 'kmeans' or 'random'

# Allocate variables
#BIC = np.zeros((T,))
#AIC = np.zeros((T,))
CVE = np.zeros((T,))

# K-fold crossvalidation
CV = model_selection.KFold(n_splits=10,shuffle=True)

for t,K in enumerate(KRange):
        print('Fitting model for K={0}'.format(K))

        # Fit Gaussian mixture model
        gmm = GaussianMixture(n_components=K, covariance_type=covar_type,
                              n_init=reps, init_params=init_procedure,
                              tol=1e-6, reg_covar=1e-6).fit(X)

        # Get BIC and AIC
        #BIC[t,] = gmm.bic(X)
        #AIC[t,] = gmm.aic(X)

        # For each crossvalidation fold
        for train_index, test_index in CV.split(X):

            # extract training and test set for current CV fold
            X_train = X[train_index]
            X_test = X[test_index]

            # Fit Gaussian mixture model to X_train
            gmm = GaussianMixture(n_components=K, covariance_type=covar_type, n_init=reps).fit(X_train)

            # compute negative log likelihood of X_test
            CVE[t] += -gmm.score_samples(X_test).sum()

# Plot results

#Print K multivGaussians to model the data
#To evaluate the model fit on the test set in terms of negative log likelihood
minCVE = min(CVE)
KMIN = np.nonzero(CVE == minCVE)[0][0] +1
print("Best K", KMIN)

figure(1);
plot(KRange, 2*CVE,'-ok')
legend('Crossvalidation')
xlabel('K')
show()

print('Ran Exercise 11.1.5')

#Look at plot and determine for which number of K is best


import os
from datetime import datetime
import subprocess
import json

# submit the job to the cluster
def eval_edges(save_dir, exp_name, dataset='PascalContext', num_req_files=None):

    if dataset == 'PascalContext':
        database = 'PASCALContext'
        if num_req_files is None:
            num_req_files = 5105
    elif dataset == 'NYUD':
        database = 'NYUD'
        if num_req_files is None:
            num_req_files = 654
    else:
        raise ValueError('unknown database')

    chk_dir = os.path.join(save_dir, 'edges')
    if not os.path.exists(chk_dir):
        print('Experiment {} is not ready yet. Evaluation aborted.')
        return

    # check for filenames
    fnames = sorted(os.listdir(chk_dir))
    #if len(fnames) != num_req_files:
    if False:
        print('Number of files is incorrect')
    else:
        # seism path
        seism_cluster_dir = '/home/seism'

        # rsync to seism
        rsync_str = 'rsync -aP {}/ '.format(chk_dir)
        rsync_str += '{} '.format(os.path.join(seism_cluster_dir, 'datasets', database, exp_name))
        rsync_str += '--exclude=models --exclude=*.txt'
        print(rsync_str)
        os.system(rsync_str)

        # submit the job

        # subm_job_str = 'export SLURM_CONF=/home/sladmcvl/slurm/slurm.conf; '
        subm_job_str = 'cp {}/parameters/HED.txt {}/parameters/{}.txt && ' \
                       .format(seism_cluster_dir, seism_cluster_dir, exp_name)
        # subm_job_str += 'python {}/src/scripts/eval_in_cluster.py {} {} read_one_cont_png fb 1 102 val' \
        subm_job_str += 'sbatch -J evalFb --array 1-102 {}/src/scripts/eval_in_cluster.py {} {} read_one_cont_png fb 1 102 val' \
            .format(seism_cluster_dir, exp_name, database)
        print(subm_job_str)
        os.system(subm_job_str)

# after cluster job is done, parse the output to generate .json file
def generate_pr_curves(save_dir, exp_name, dataset='PascalContext'):
    if dataset == 'PascalContext':
        database = 'PASCALContext'
    elif dataset == 'NYUD':
        database = 'NYUD'
    else:
        raise ValueError('unknown database')
    seism_cluster_dir = '/home/seism'

    timestamp = datetime.now().strftime(r'%m%d_%H%M%S')
    chk_file = os.path.join(save_dir, 'edge_{}.json'.format(exp_name))

    os.chdir(seism_cluster_dir)
    command_to_run = ['matlab', '-nodesktop', '-nodisplay', '-nosplash', '-r', "install;pr_curves_to_file('"+database+"','val','"+exp_name+"');exit"]
    subprocess.run(command_to_run, check=True)
    eval_results = {}
    for measure in {'ods_f', 'ois_f', 'ap'}:
        tmp_fname = os.path.join(seism_cluster_dir, 'results', 'pr_curves', database,
                                 database + '_val_fb_' + exp_name + '_' + measure + '.txt')
        with open(tmp_fname, 'r') as f:
            eval_results[measure] = float(f.read().strip())
    try:
        with open(chk_file, 'r') as f:
            data = json.load(f)
        data.update(eval_results)
        eval_results = data
    except FileNotFoundError:
        pass
    with open(chk_file, 'w') as f:
        json.dump(eval_results, f)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--script', type=str, help='Run eval_edges or generate_pr_curves')
    parser.add_argument('--exp_names', type=str, help='Experiment names separated by comma. e.g. exp1 or exp1,exp2 ')
    parser.add_argument('--dataset', type=str, help='Name of dataset')
    opts = parser.parse_args()

    assert opts.dataset in ['PascalContext', 'NYUD']

    exp_names = opts.exp_names.split(',')
    for exp_name in exp_names:
        save_dir = '/home/{}'.format(exp_name)
        if opts.script == 'eval_edges':
            eval_edges(save_dir, exp_name, opts.dataset)
        elif opts.script == 'generate_pr_curves':
            generate_pr_curves(save_dir, exp_name, opts.dataset)
        else:
            raise ValueError('Select either eval_edges or generate_pr_curves')


import requests
from bs4 import BeautifulSoup
import pyautogui

input_query = pyautogui.prompt('검색어를 입력해주세요: ')
print(f'input_query: {input_query}')

response = requests.get(f"https://search.naver.com/search.naver?where=news&sm=tab_jum&query={input_query}")
html = response.text
soup = BeautifulSoup(html, 'html.parser')
links = soup.select(".news_tit") # 결과는 리스트
for link in links:
    title = link.text
    url = link.attrs['href']
    print(title, url)

05.뉴스제목과링크가져오기.py


import csv, requests
from .models import Location, EntryDate, create_friendly_name, create_hash
from .models import County, ProvinceState, CountryRegion, CaseType
from .models import CountEntry, CountIncreaseEntry, CountPercentIncreaseEntry
from .coronaVars import county_key, province_key, country_key, lat_key
from .coronaVars import long_key, us_keys, global_keys, csv_global_urls, csv_us_urls
from .coronaVars import get_file, case_status_type_names
from celery import shared_task
from celery.signals import worker_ready
from datetime import datetime as dt
from datetime import timedelta as td
from hashlib import sha256

def get_state_location(locs, state, case_status_type_id):
    state_hash = create_hash(state.province_state)
    if state_hash not in locs:
        state = Location(
            province_state = state,
            friendly_name = state.province_state,
            friendly_hash = state_hash
        )
        state.save()
        state.case_types.add(case_status_type_id)
        return state
    locs[state_hash].case_types.add(case_status_type_id)
    return locs[state_hash]

def get_region_location(locs, region, case_status_type_id):
    region_hash = create_hash(region.region_country)
    if region_hash not in locs:
        region = Location(
            region_country = region,
            friendly_name = region.region_country,
            friendly_hash = region_hash

        )
        region.save()
        region.case_types.add(case_status_type_id)
        return region
    locs[region_hash].case_types.add(case_status_type_id)
    return locs[region_hash]

# csv_file: csv.DictReader
# case_status_type: string, one of ['confirmed', 'deaths', 'recovered']
def update_database(csv_file, case_status_type_id, column_keys, row_start):

    # hashtable the relevant objects for O(1) lookups
    csts = {cst.case_type : cst for cst in  CaseType.objects.all() }

    # if this case status type doesn't exist create it
    if case_status_type_id not in csts:
        case_status_type_id = CaseType(case_type=case_status_type_id)
        case_status_type_id.save()
    # otherwise grab the object from the hash table
    else:
        case_status_type_id = csts[case_status_type_id]

    # for each row in the csv file
    for row in csv_file:
        locs = { loc.friendly_hash: loc for loc in Location.objects.all() }
        province_state = { ps.province_state : ps for ps in ProvinceState.objects.all() }
        country_region = { cr.region_country : cr for cr in CountryRegion.objects.all() }
        counties = { cnty.county : cnty for cnty in County.objects.all() }

        # not all rows have a county entry
        county = None

        # if there is a county key present in the keys sent to this function
        if county_key in column_keys:
            # assign the couny accordingly
            county = row[column_keys[county_key]]

        # otherwise get values from row with keys sent to this function
        province = row[column_keys[province_key]]
        region = row[column_keys[country_key]]
        lat = row[column_keys[lat_key]]
        lon = row[column_keys[long_key]]

        # if there was a county
        if county:
            # include it in the friendly name
            friendly_name = create_friendly_name(province, region, county=county)
        else:
            # don't include it
            friendly_name = create_friendly_name(province, region)

        # create the friendly hash for this location for lookup in hash table
        friendly_hash = create_hash(friendly_name)

        # if the region isn't in the region hash table
        if region not in country_region:
            # create it
            region = CountryRegion(region_country=region)
        else:
            # grab it
            region = country_region[region]

        # try to add this case type to the region
        region.save()
        region.case_types.add(case_status_type_id)

        # get or create a location object that is only associated with this region
        region_location = get_region_location(locs, region, case_status_type_id)

        if province != '':
            #if the state isn't in the state hash table
            if province not in province_state:
                # create it
                province = ProvinceState(
                    province_state = province,
                    region_country = region
                )
            else:
                # grab it
                province = province_state[province]

            # try to add this case type to the state
            province.save()
            province.case_types.add(case_status_type_id)

            # get or create a location object that is only assicated with this state
            state_location = get_state_location(locs, province, case_status_type_id)

        else:
            province = None
            state_location = None

        # if there was a county
        if county:
            # if the county isn't in the county hash table
            if county not in counties:
                # create it
                county = County(
                    county = county,
                    province_state = province,
                    region_country = region
                )
            else:
                # grab it
                county = counties[county]

            # try to add this case type to this county
            county.save()
            county.case_types.add(case_status_type_id)

        # if this location not in the hash table of locations
        if friendly_hash not in locs:
            # if there was a county
            if county:
                # create a location with a county
                location = Location(
                    county = county,
                    province_state = province,
                    region_country = region,
                    latitude = lat,
                    longitude = lon,
                    friendly_name = friendly_name,
                    friendly_hash = friendly_hash
                )
            else:
                # create a location with a null county
                location = Location(
                    province_state = province,
                    region_country = region,
                    latitude = lat,
                    longitude = lon,
                    friendly_name = friendly_name,
                    friendly_hash = friendly_hash
                )
        else:
            # grab it
            location = locs[friendly_hash]

        # try to add this case type to this location
        location.save()
        location.case_types.add(case_status_type_id)

        # get number entries of this type for this location
        locations_entries = CountEntry.objects.filter(date__location=location, date__case_status_type_id=case_status_type_id)
        num_historic_db_entries = len(locations_entries)

        # get all actual entries on this row
        list_row = [ item for item in row.items() ][row_start:]

        # if the number of entries for this row is equal to the number in the database
        if num_historic_db_entries == len(list_row):
            # no new entries, nothing to do
            print('no updates for', case_status_type_id)
            continue

        one_day = td(days=1)

        # for each new entry
        for entry in list_row[num_historic_db_entries:]:

            the_date = dt.strptime(entry[0], '%m/%d/%y').date()
            previous_date = (dt.strptime(entry[0], '%m/%d/%y') - one_day).date()

            # try to get the previous date of this location for calculations if it exists
            try:
                previous_entry_date = EntryDate.objects.get(
                    date=previous_date,
                    location=location,
                    case_status_type_id=case_status_type_id
                )
                previous_date_count = previous_entry_date.countentry
                divisor = previous_date_count
                if int(divisor) == 0:
                    divisor = 100
            # it doesn't exist, fallback to default values
            except Exception as e:
                print(e)
                previous_date_count = 0
                divisor = 100

            # create a new EntryDate
            date = EntryDate(
                date = the_date,
                location = location,
                case_status_type_id = case_status_type_id
            )
            date.save()

            # crete a new CountEntry
            count_entry = CountEntry(
                date = date,
                value = int(entry[1])

            )
            count_entry.save()

            # create a new CountIncreaseEntry
            count_inc_entry = CountIncreaseEntry(
                date = date,
                value =  int(entry[1]) - int(previous_date_count)
            )
            count_inc_entry.save()

            # create a new CountPercentIncreaseEntry
            count_perc_inc_entry = CountPercentIncreaseEntry(
                date = date,
                value = 100*(int(count_inc_entry)/int(divisor))
            )
            count_perc_inc_entry.save()

            # if processing a US file

            # UPDATE STATE
            # get state location associated previous EntryDate object count
            if state_location:
                try:
                    previous_state_date = EntryDate.objects.get(
                        date=previous_date,
                        location=state_location,
                        case_status_type_id=case_status_type_id
                    )
                    previous_state_date_count = previous_state_date.countentry
                    state_divisor = previous_state_date_count
                    if int(state_divisor) == 0:
                        state_divisor = 100
                except Exception as e:
                    print(e)
                    previous_state_date_count = 0
                    state_divisor = 100

                # get or create the current state location EntryDate object
                try:
                    state_date = EntryDate.objects.get(
                        location=state_location,
                        date=date.date,
                        case_status_type_id = case_status_type_id
                        )
                except Exception as e:
                    print(e)
                    state_date = EntryDate(
                        date = the_date,
                        location = state_location,
                        case_status_type_id = case_status_type_id
                    )
                    state_date.save()

                # get and update or create the current state location CountEntry
                try:
                    state_date.countentry.value += count_entry.value
                    state_date.countentry.save()
                except Exception as e:
                    print(e)
                    state_count_entry = CountEntry(
                        date = state_date,
                        value = int(entry[1])

                    )
                    state_count_entry.save()

                # get and update or create the current state location CountIncreaseEntry
                try:
                    state_date.countincreaseentry.value += count_inc_entry.value
                    state_date.countincreaseentry.save()
                    state_inc_entry = state_date.countincreaseentry
                except Exception as e:
                    print(e)
                    state_inc_entry = CountIncreaseEntry(
                        date = state_date,
                        value = count_inc_entry.value
                    )
                    state_inc_entry.save()

                # get and update or create the current state location CountPercentIncreaseEntry
                try:
                    state_date.countpercentincreaseentry.value = 100*(int(state_inc_entry)/int(state_divisor))
                    state_date.countpercentincreaseentry.save()
                except Exception as e:
                    print(e)
                    state_perc_inc_entry = CountPercentIncreaseEntry(
                        date = state_date,
                        value = count_perc_inc_entry.value
                    )
                    state_perc_inc_entry.save()

                # UPDATE REGION
                if not county:
                    try:
                        previous_region_date = EntryDate.objects.get(
                            date=previous_date,
                            location=region_location,
                            case_status_type_id = case_status_type_id
                        )
                        previous_region_date_count = previous_region_date.countentry
                        region_divisor = previous_region_date_count
                        if int(region_divisor) == 0:
                            region_divisor = 100
                    except Exception as e:
                        print(e)
                        previous_region_date_count = 0
                        region_divisor = 100

                    try:
                        region_date = EntryDate.objects.get(
                            location=region_location,
                            date=date.date,
                            case_status_type_id=case_status_type_id
                            )
                    except Exception as e:
                        print(e)
                        region_date = EntryDate(
                            date = the_date,
                            location = region_location,
                            case_status_type_id = case_status_type_id
                        )
                        region_date.save()

                    try:
                        region_date.countentry.value += count_entry.value
                        region_date.countentry.save()
                    except Exception as e:
                        print(e)
                        region_count_entry = CountEntry(
                            date = region_date,
                            value = int(entry[1])
                        )
                        region_count_entry.save()

                    try:
                        region_date.countincreaseentry.value += count_inc_entry.value
                        region_date.countincreaseentry.save()
                        region_inc_entry = region_date.countincreaseentry
                    except Exception as e:
                        print(e)
                        region_inc_entry = CountIncreaseEntry(
                            date = region_date,
                            value = count_inc_entry.value
                        )
                        region_inc_entry.save()

                    try:
                        region_date.countpercentincreaseentry.value = 100*(int(region_inc_entry)/int(region_divisor))
                        region_date.countpercentincreaseentry.save()
                    except Exception as e:
                        print(e)
                        region_perc_inc_entry = CountPercentIncreaseEntry(
                            date = region_date,
                            value = count_perc_inc_entry.value
                        )
                        region_perc_inc_entry.save()

            print('new {status_type} historical entry {num} for {date} {location}'.format(
                status_type=case_status_type_id,
                num=entry[1],
                date=entry[0],
                location=location.friendly_name))

@shared_task
def do_data_update():
    csv_global_files = [get_file(csv_url) for csv_url in csv_global_urls]
    csv_us_files = [get_file(csv_url) for csv_url in csv_us_urls]

    for case_status_type_name, csv_file in zip(case_status_type_names, csv_global_files):
        update_database(csv_file, case_status_type_name, global_keys, 4)

    for case_status_type_name, csv_file in zip(case_status_type_names[:2], csv_us_files):
        if case_status_type_name == 'recovered':
            update_database(csv_file, case_status_type_name, us_keys, 11)
        else:
            update_database(csv_file, case_status_type_name, us_keys, 12)

# @worker_ready.connect
# def update_data(sender=None, conf=None, **kwargs):
#     do_data_update()


import typer
import json

from pathlib import Path
from typing import Optional

from .converter import convert as convert_json

app = typer.Typer()

def check_input(input: Path):
    """
    Check if the input file is a valid input
    """
    if not input.exists():
        print('File does not exist')
        raise typer.Abort()

    if not input.is_file():
        print('It is not a file')
        raise typer.Abort()

    if not input.suffix == '.json':
        print('It is not a json file')
        raise typer.Abort()

def check_output(path: Path):
    """
    Check is the output path is a valid dir
    """
    if not path.is_dir():
        print('Output path is not a dir')
        raise typer.Abort()

@app.command()
def convert(
    input_file: Path,
    output_path: Optional[Path] = typer.Option(Path('.')),
):
    """
    Convert the JSON file into a CSV
    """

    check_input(input_file)
    check_output(output_path)

    filename = input_file.stem
    filename = f'{filename}-converted.csv'

    output_file = output_path.joinpath(filename)
    output_file.touch(exist_ok=True)

    try:
        convert_json(input_file, output_file)
    except Exception as e:
        raise typer.Abort()

    print('Successfuly converted!')


import os
import datetime
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
from matplotlib.ticker import MultipleLocator, FixedLocator
# ---------------------------
def plot_data(ax, label, c, filename, n_trial=30, time_shift=0,  ls="-", alpha=0.2, linewidth = None):
    for trial in range(n_trial):
        print(filename.replace("##", str(trial)))
        data = pd.read_csv(filename.replace("##", str(trial)), sep="\t")
        data = data.loc[~np.isnan(data['s'])]
        if linewidth is None:
            ax.plot(data['window_end']+time_shift, data['s'], alpha=alpha, label=label, c=c, ls=ls)
        else:
            ax.plot(data['window_end']+time_shift, data['s'], alpha=alpha, label=label, c=c, ls=ls, linewidth=linewidth)
        label = None
    return

def plot_statevar(ax, label, c, filename, time_shift=0,  ls="-", alpha=0.7, names = ["t", "S", "E", "I", "R"]):
    statevar = pd.read_csv(filename, sep="\t", comment="#", names=names)
    if "I" in names:
        ax.plot(statevar['t']+time_shift, statevar['I'], alpha=alpha, label=label, c=c, ls=ls)
    else:
        print(statevar)
        ax.plot(statevar['t'] + time_shift, statevar['I_l'] + statevar['I_h'], alpha=alpha, label=label, c=c, ls=ls)

def run(baseline_statevar,baseline_dense,baseline_sparse,baseline_full,
        highR0, highR0_dense, transhet, transhet_dense,
        reduced1p1, reduced1p1_dense, reduced0p75, reduced0p75_dense, figname):

    plt.rcParams.update({'font.size': 10})
    fig, axes = plt.subplots(2, 4, figsize=[13.5, 5], dpi=300)

    # ---------------------------
    ## Panel A, B: dense vs. sparse vs. full
    plot_statevar(axes[0, 0], 'dense',   "k",       baseline_statevar)
    plot_data    (axes[1, 0], 'dense',   "k",       baseline_dense)
    plot_data    (axes[1, 0], 'sparse',  "#9e9e9e", baseline_sparse)
    plot_data    (axes[1, 0], 'full',    "k"      , baseline_full, ls = "--", n_trial = 1, alpha = 1)

    ## inset - same data but zoomed out
    left, bottom, width, height = [0.07, 0.33, 0.075, 0.12]
    inset = fig.add_axes([left, bottom, width, height])

    plot_data    (inset, 'dense',   "k",       baseline_dense)
    plot_data    (inset, 'sparse',  "#9e9e9e", baseline_sparse)
    plot_data    (inset, 'full',    "k"      , baseline_full, ls = "--", n_trial = 1, alpha = 1)

    inset.patch.set_alpha(0.8)
    inset.set_xlim([-25, 205])
    inset.xaxis.set_ticklabels([])

    inset.yaxis.set_major_locator(FixedLocator([]))
    inset.set_ylim([-10, 2750])
    inset.yaxis.set_major_locator(FixedLocator([200, 1500, 2500]))

    inset.tick_params(axis = 'y', direction = 'in', pad = -67, labelsize = 6, labelright = True, labelleft = False, right = False, left = True)

    # ---------------------------
    ## Panel C, D: high R0 vs. low R0 - todo
    plot_statevar(axes[0, 1], r'R$_{0}$ = 1.6', "k",       baseline_statevar)
    plot_data    (axes[1, 1], r'R$_{0}$ = 1.6', "k",       baseline_dense)

    plot_statevar(axes[0, 1], r'R$_{0}$ = 2.0', "#318fdb", highR0)
    plot_data    (axes[1, 1], r'R$_{0}$ = 2.0', "#318fdb", highR0_dense)

    # ---------------------------
    # Panel E, F: transmission heterogeneity
    statevar_1 = pd.read_csv(baseline_statevar, sep="\t", comment="#", names=["t", "S", "E",  "I", "R"])
    statevar_2 = pd.read_csv(transhet, sep="\t", comment="#",names=["t", "S", "E", "I_l", "I_h", "R"])
    time_shift =  statevar_1['t'][np.argmax(statevar_1['I'])] - \
                  statevar_2['t'][np.argmax(statevar_2['I_l'] + statevar_2['I_h'])]

    plot_statevar(axes[0, 2], 'No TH',       "k",       baseline_statevar)
    plot_data    (axes[1, 2], 'No TH',       "k",       baseline_dense)

    plot_statevar(axes[0, 2], 'TH'          , "#80b564", transhet,       ls = "-.", names = ["t", "S", "E", "I_l", "I_h", "R"])
    plot_statevar(axes[0, 2], 'TH (shifted)', "#80b564", transhet,       time_shift= time_shift, names = ["t", "S", "E", "I_l", "I_h", "R"])
    plot_data    (axes[1, 2], 'TH (shifted)', "#80b564", transhet_dense, time_shift= time_shift)

    # ---------------------------
    # ## Panel G, H: reduced R0
    plot_statevar(axes[0, 3], r'R$_{0}$ unchanged',       "k",       baseline_statevar)
    plot_data    (axes[1, 3], r'R$_{0}$ unchanged',       "k",       baseline_dense)

    plot_statevar(axes[0, 3], r'R$_{0}$ reduced to 1.1' , "#ff5f43", reduced1p1)
    plot_data    (axes[1, 3], r'R$_{0}$ reduced to 1.1' , "#ff5f43", reduced1p1_dense)
    plot_statevar(axes[0, 3], r'R$_{0}$ reduced to 0.75', "#fcb708", reduced0p75)
    plot_data    (axes[1, 3], r'R$_{0}$ reduced to 0.75', "#fcb708", reduced0p75_dense)
    #

    # ---------------------------
    # set axis limit
    for i in range(4):
        axes[0, i].set_xlim([0, 200]); axes[0, i].set_ylim([0, 10000]);
        axes[1, i].set_xlim([0, 200]); axes[1, i].set_ylim([0, 225]);
        axes[0, i].ticklabel_format(axis='y', style="sci", scilimits=(3, 4))

        if i != 0:
            axes[0, i].legend(loc="upper right")
        axes[1, i].set_xlabel('Days post index case')

    axes[0, 0].set_ylabel("Number of\ninfected individuals")
    axes[1, 0].set_ylabel("Number of\nsegregating sites")

    ## add subplot label
    labels = ['ACEG'.lower(), 'BDFH'.lower()]
    for i, label_row in enumerate(labels):
        for j, l in enumerate(label_row):
            if j == 0:
                x_loc = -0.4
                axes[i][j].text(-0.27, 1.1, l, transform=axes[i][j].transAxes, fontsize=16, fontweight='bold', va='top', ha='left')
            else:
                x_loc = -0.3
                axes[i][j].text(-0.23, 1.1, l, transform=axes[i][j].transAxes, fontsize=16, fontweight='bold', va='top', ha='left')
            continue

    ## save figure
    fig.tight_layout(pad=0.5)
    fig.savefig(f'manuscript figures/pdf/{figname}.pdf')
    fig.savefig(f'manuscript figures/png/{figname}.png')

    fig.clf()
    plt.cla()

    return

if __name__ == "__main__":
    import os
    import argparse

    if os.getcwd().find("manuscript_figure_script") == -1:
        parser = argparse.ArgumentParser()
        parser.add_argument('--figname', type=str, default="figure3")

        parser.add_argument('--baseline_statevar',  type=str, required=True)
        parser.add_argument('--baseline_dense',     type=str, required=True)
        parser.add_argument('--baseline_sparse',    type=str, required=True)
        parser.add_argument('--baseline_full',      type=str, required=True)

        parser.add_argument('--highR0',         type=str, required=True)
        parser.add_argument('--highR0_dense',   type=str, required=True)

        parser.add_argument('--transhet',       type=str, required=True)
        parser.add_argument('--transhet_dense', type=str, required=True)

        parser.add_argument('--reduced1p1',         type=str, required=True)
        parser.add_argument('--reduced1p1_dense',   type=str, required=True)
        parser.add_argument('--reduced0p75',        type=str, required=True)
        parser.add_argument('--reduced0p75_dense',  type=str, required=True)

        args = parser.parse_args()

    else:
        os.chdir(os.getcwd().replace("manuscript_figure_script", ""))
        parser = argparse.ArgumentParser()
        args = parser.parse_args()
        args.figname = "figure1"

        args.baseline_statevar = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/seed20230201_statevar.tsv"
        args.baseline_dense    = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/seed20230201_unif40_win4/seed230207_segsites.tsv"
        args.baseline_sparse   = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/seed20230201_unif20_win4/seed230207_segsites.tsv"
        args.baseline_full     = "simulated_data/simpleSEIR_R0=16e-1_mu2e-1/seed20230201_full_win4/seed230201_segsites.tsv"

        args.highR0            = "simulated_data"
        args.highR0_dense      = "simulated_data"

        args.transhet          = "simul_dat"
        args.transhet_dense    = "simul_dat"

        args.reduced1p1        = "simul_dat"
        args.reduced1p1_dense  = "simul_dat"
        args.reduced0p75       = "simul_dat"
        args.reduced0p75_dense = "simul_dat"

    print(args)
    run(args.baseline_statevar,
        args.baseline_dense,
        args.baseline_sparse,
        args.baseline_full,

        args.highR0,
        args.highR0_dense,
        args.transhet,
        args.transhet_dense,

        args.reduced1p1,
        args.reduced1p1_dense,
        args.reduced0p75,
        args.reduced0p75_dense,

        args.figname)


from turtle import forward
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.init as init

class GraphSage(nn.Module):
    def __init__(self,input_dim:int,hidden_dim:list,num_neighbors_list:list):
        """实现GraphSage层

        Args:
            input_dim (int): 输入维度
            hidden_dim (list): 隐藏层节点数
            num_neighbors_list (list): 各阶邻居采样数
        """
        super().__init__()
        self.input_dim=input_dim
        self.hidden_dim=hidden_dim
        self.num_neighbors_list=num_neighbors_list
        self.num_layers=len(num_neighbors_list)
        self.gcn=nn.ModuleList()
        self.gcn.append(SageGcn(input_dim,hidden_dim[0]))
        for idx in range(0,len(hidden_dim)-2):
            self.gcn.append(SageGcn(hidden_dim[idx],hidden_dim[idx+1]))
        self.gcn.append(SageGcn(hidden_dim[-2],hidden_dim[-1],act=None))

    def forward(self,node_features_list):
        """
        Args:
            node_features_list (List[List[tensor]]): 模型层数x采样数x特征数
        """
        hidden=node_features_list #init feature list
        for l in range(self.num_layers): #forward
            next_hidden=[]
            gcn=self.gcn[l]
            for hop in range(self.num_layers-l):  #依次聚合source节点及其各阶子节点的特征
                src_node_features=hidden[hop]
                src_node_num=len(src_node_features)
                neighbor_node_features=hidden[hop+1].view((src_node_num,self.num_neighbors_list[hop],-1))
                h=gcn(src_node_features,neighbor_node_features)
                next_hidden.append(h)
            hidden=next_hidden
        return hidden[0] #返回src 节点聚合后的节点特征

class SageGcn(nn.Module):
    def __init__(self,input_dim,hidden_dim,act=F.leaky_relu,
                 aggr_neighbor_method='mean',
                 aggr_hidden_method='sum',
                 ):
        """执行聚合操作

        Args:
            input_dim (int): 输入维度
            hidden_dim (int): 隐藏维度
            act (torch.nn.functional, optional): 激活函数. Defaults to F.leaky_relu.
            aggr_neighbor_method (str, optional): neighbor feature 聚合方式. Defaults to 'mean'.
            aggr_hidden_method (str, optional): 隐藏层特征聚合方式. Defaults to 'sum'.
        """
        super().__init__()
        assert aggr_neighbor_method in ['mean','sum','max']
        assert aggr_hidden_method in ['sum','concat']
        self.input_dim=input_dim
        self.hidden_dim=hidden_dim
        self.aggr_neighbor_method=aggr_neighbor_method
        self.aggr_hidden_method=aggr_hidden_method
        self.act=act
        self.aggregator=NeighborAggregator(input_dim, hidden_dim,
                                             aggr_method=aggr_neighbor_method)
        self.weight=nn.Parameter(torch.Tensor(input_dim,hidden_dim))
        self.reset_parameter()

    def reset_parameter(self):
        init.kaiming_normal_(self.weight)

    def forward(self,src_node_features,neighbor_node_features):
        neighbor_hidden=self.aggregator(neighbor_node_features)
        self_hidden=torch.matmul(src_node_features,self.weight)

        if self.aggr_hidden_method=='sum':
            hidden=self_hidden+neighbor_hidden
        elif self.aggr_hidden_method=='concat':
            hidden=torch.cat([self_hidden,neighbor_hidden],dim=1)

        if self.act:
            return self.act(hidden)
        else:
            return hidden

class NeighborAggregator(nn.Module):
    def __init__(self,input_dim,output_dim,use_bias=False,aggr_method='mean'):
        """执行邻居聚合操作

        Args:
            input_dim (int): 输入维度
            output_dim (int): 输出维度
            use_bias (bool, optional): 是否使用bias. Defaults to False.
            aggr_method (str, optional): 聚合方法, 目前只支持 ['mean','concat']. Defaults to 'mean'.
        """
        super().__init__()
        assert aggr_method in ['mean','sum','max']
        self.input_dim=input_dim
        self.output_dim=output_dim
        self.use_bias=use_bias
        self.aggr_aggr_method=aggr_method
        self.weight=nn.Parameter(torch.tensor(input_dim,output_dim))
        if self.use_bias:
            self.bias=nn.Parameter(torch.tensor(self.output_dim))
        self.reset_parameter()

    def reset_parameter(self):
        init.kaiming_normal_(self.weight)
        if self.use_bias:
            init.zeros_(self.bias)

    def forward(self,neighbor_feature):
        if self.aggr_aggr_method=='mean':
            aggr_neighbor=neighbor_feature.mean(dim=1)
        elif self.aggr_aggr_method=='sum':
            aggr_neighbor=neighbor_feature.sum(dim=1)
        elif self.aggr_aggr_method=='max':
            aggr_neighbor=neighbor_feature.max(dim=1)

        neighbor_hidden=torch.matmul(aggr_neighbor,self.weight)
        if self.use_bias:
            neighbor_hidden+=self.bias

        return neighbor_hidden


import setuptools
import setuptools.extension

try:
    import Cython.Distutils
except ImportError:
    cmdclass = {}
    suffix = ".cpp"
else:
    cmdclass = {"build_ext": Cython.Distutils.build_ext}
    suffix = ".pyx"

ext_module_sources = [
    ("bdaq.wrapper", ["bdaq/wrapper"]),
    ("bdaq.wrapper_enums", ["bdaq/wrapper_enums"])]
ext_modules = []

for (module_name, module_paths) in ext_module_sources:
    ext_modules.append(
        setuptools.extension.Extension(
            module_name,
            map(
                lambda path: path + suffix,
                module_paths),
            include_dirs=[],
            undef_macros=["NDEBUG"],
            define_macros=[],
            libraries=["biodaq"],
            extra_compile_args=[
                "-std=gnu++0x",
                "-Werror",
                "-Wno-uninitialized",
                "-Wno-write-strings"],
            language="c++"))

requires = []

setuptools.setup(
    name="pybdaq",
    version="0.0.7",
    packages=setuptools.find_packages(),
    cmdclass=cmdclass,
    ext_modules=ext_modules,
    install_requires=requires,
    author="Angaza Design",
    author_email="bcs@angazadesign.com",
    description="data acquisition hardware (DAQ) SDK wrapper",
    license="MIT",
    keywords="daq wrapper library bionic advantech usb",
    url="http://github.com/angaza/pybdaq",
    include_package_data=True,
    classifiers=[
        "License :: OSI Approved :: MIT License",
        "Development Status :: 3 - Alpha",
        "Programming Language :: Cython",
        "Programming Language :: Python :: 2.7",
        "Operating System :: POSIX :: Linux",
        "Topic :: Utilities",
        "Topic :: System :: Hardware",
        "Topic :: Software Development :: Libraries"])


import sqlite3
con = sqlite3.connect("d:/第一个属于自己的数据库.db")
cur = con.cursor()
x = input("输入id：")
cur.execute("select name,zcj FROM  book where id = '{}'".format(x))
print(cur.fetchall())
cur.execute("select * FROM  book")
for i in cur.fetchall():
    print(i)
cur.close()
con.close()

第三方库的学习/数据库/查.py