'''
@version: 0.0.1
@Author: Huang
@dev: python3 vscode
@Date: 2019-11-10 11:39:30
@LastEditTime: 2019-11-10 18:11:16
@FilePath: \\机器学习实战\\10-K均值聚类算法\\kMeans.py
@Descripttion: 聚类是一种无监督的学习，它将相似的对象归到同一个簇中
'''

import numpy as np
import matplotlib.pyplot as plt

def loadDataSet(filename):
    """
    [summary]:加载数据

    Arguments:
        filename  -- 文件名

    Returns:
        [List] -- 数据集
    """
    dataMat = []
    with open(filename) as fr:
        for line in fr.readlines():
            curline = line.strip().split()
            fltline = list(map(float, curline))
            dataMat.append(fltline)
    return dataMat

def distEclud(vecA, vecB):
    """
    [summary]:计算两个向量的欧氏距离

    Arguments:
        vecA  -- A坐标
        vecB  -- B坐标

    Returns:
        两点之间的欧氏距离
    """
    # not sum(mat) but mat.sum()
    return np.sqrt(np.power(vecA - vecB, 2).sum())

def randCent(dataSet, k):
    """
    [summary]:为数据集构建k个随机质心的集合

    Arguments:
        dataSet {[mat]} -- 数据集
        k {[int} -- 聚类数

    Returns:
        [mat] -- k个中心点组成的矩阵
    """
    n = np.shape(dataSet)[1]  # 获取列数
    centroids = np.mat(np.zeros((k, n)))
    # 遍历所有列
    for j in range(n):
        minj = min(dataSet[:, j])
        rangej = float(max(dataSet[:, j]) - minj)
        # 最小值+区间×随机系数,确保生成的中心点在数据集边界之内
        centroids[:, j] = minj + rangej * np.random.rand(k, 1)
    return centroids

def my_KMeans(dataSet, k, distMeas=distEclud, createCent=randCent):
    """
    [summary]:
        创建k个点作为起始质心（经常是随机选择）
        当任意一个点的簇分配结果发生改变时
            对数据集中的每个数据点
                对每个质心
                    计算质心与数据点之间的距离
                将数据点分配到距其最近的簇
            对每一个簇，计算簇中所有点的均值并将均值作为质心

    Arguments:
        dataSet {[mat]} -- 数据集
        k {[int]} -- 聚类数

    Keyword Arguments:
        distMeas  -- 距离算法 (default: {distEclud})
        createCent -- 创建初始质心 (default: {randCent})

    Returns:
        centroids -- (k,n) 类质心
        clusterAssment -- (m,2) 点分配
    """
    m = np.shape(dataSet)[0]  # 行数
    # 簇分配结果：第一列记录索引值;第二列存储误差,当前点到簇质心的距离
    clusterAssment = np.mat(np.zeros((m, 2)))
    # 随机生成中心点完成初始化
    centroids = randCent(dataSet, k)  # (k,n)
    clusterchanged = True
    while clusterchanged:
        # 假定所有点分配都不发生改变，标记为False
        clusterchanged = False
        for i in range(m):
            cluster_i = clusterAssment[i, 0]  # 取出簇索引值
            dismax = np.inf
            for j in range(k):
                curdis = distEclud(centroids[j, :], dataSet[i, :])
                if curdis < dismax:
                    dismax = curdis
                    # 更新簇分配结果
                    clusterAssment[i, :] = j, dismax
            if cluster_i != clusterAssment[i, 0]:
                clusterchanged = True
        print(centroids)
        for cent in range(k):
            ptsInClust = dataSet[np.nonzero(clusterAssment[:, 0].A == cent)[0]]
            # 沿矩阵的列方向进行均值计算
            centroids[cent, :] = np.mean(ptsInClust, axis=0)
    # 返回所有的类质心与点分配结果
    return centroids, clusterAssment

def plt_my_KMeans():
    data = np.mat(loadDataSet(r'./10-K均值聚类算法/testSet.txt'))
    # 对数据进行聚类
    centroidsOfData, clusterAssmentOfData = my_KMeans(data, 4)
    # 数据集的数量
    m = np.shape(data)[0]
    # 画出数据的散点图
    plt.scatter(data[:, 0].A.reshape(m),
                data[:, 1].A.reshape(m),
                c=clusterAssmentOfData.A[:, 0].reshape(m))
    # 用红色的三角形符号画出聚类中心
    plt.scatter(centroidsOfData.A[:, 0],
                centroidsOfData.A[:, 1],
                c='red',
                marker='^')
    # 显示图片
    plt.show()

def biKmeans(dataSet, k, distMeas=distEclud):
    """
    [summary]:二分K-均值算法
        将所有点看成一个簇
        当簇数目小于k时
            对于每一个簇
                计算总误差
                在给定的簇上面进行K-均值聚类(k=2)
                计算将该簇一分为二之后的总误差
            选择使得误差最小的那个簇进行划分操作

    Arguments:
        dataSet {[mat]} -- 数据集
        k {[int]} -- 聚类数

    Keyword Arguments:
        distMeas  -- 距离算法 (default: {distEclud})

    Returns:
        centroids -- (k,n) 类质心
        clusterAssment -- (m,2) 点分配
    """
    m, n = np.shape(dataSet)
    clusterAssment = np.mat(np.zeros((m, 2)))
    # 创建一个初始簇
    centroid0 = np.mean(dataSet, axis=0).tolist()[0]
    cenList = [centroid0]  # 保留质心
    # 计算误差
    for j in range(m):
        clusterAssment[j, 1] = distMeas(np.mat(centroid0), dataSet[j, :])**2
    # 对簇进行划分
    while len(cenList) < k:
        lowestSSE = np.inf
        # 尝试划分每一簇
        for i in range(len(cenList)):
            #找出正在计算的簇
            ptscurrCluster = dataSet[np.nonzero(
                clusterAssment[:, 0].A == i)[0], :]
            # 对给定簇进行K-均值聚类
            centroidMat, splitClustAss = my_KMeans(ptscurrCluster, 2, distMeas)
            # 计算划分后的SSE(误差平方和)
            ssesplit = np.sum(splitClustAss[:, 1])
            # 计算剩余数据集的SSE(误差平方和)
            ssenotsplit = np.sum(
                clusterAssment[np.nonzero(clusterAssment[:, 0].A != i)[0], 1])
            # print(ssesplit, ssenotsplit)

            if ssesplit + ssenotsplit < lowestSSE:
                bestCentToSplit = i
                bestnewCent = centroidMat
                # numpy中赋值都是将索引赋值，把数据真正赋值要用copy()
                bestClustAss = splitClustAss.copy()
                lowestSSE = ssenotsplit + ssesplit

        # 更新簇的分配结果
        bestClustAss[np.nonzero(
            bestClustAss[:, 0].A == 1)[0], 0] = len(cenList)
        bestClustAss[np.nonzero(
            bestClustAss[:, 0].A == 0)[0], 0] = bestCentToSplit
        # print('the bestcenttosplit: ', bestCentToSplit)
        # print('len bestclustass: ', len(bestClustAss))
        # 要划分的簇的簇心坐标更新为其中一个簇心坐标
        cenList[bestCentToSplit] = bestnewCent[0, :].A.reshape(n)
        # 另一个簇心坐标要通过append添加进簇心坐标集合里
        cenList.append(bestnewCent[1, :].A.reshape(n))
        # reassign new clusters, and SSE
        clusterAssment[np.nonzero(
            clusterAssment[:, 0].A == bestCentToSplit)[0], :] = bestClustAss

    return np.mat(cenList), clusterAssment

def plt_biKmeans():
    data = np.mat(loadDataSet(r'./10-K均值聚类算法/testSet2.txt'))
    centroids, clusterAssment = biKmeans(data, 3)
    # 数据集的数量
    m = np.shape(data)[0]
    # 画出数据的散点图
    plt.scatter(data[:, 0].A.reshape(m),
                data[:, 1].A.reshape(m),
                c=clusterAssment.A[:, 0].reshape(m))
    # 用红色的三角形符号画出聚类中心
    plt.scatter(centroids.A[:, 0], centroids.A[:, 1], c='red', marker='+')
    # 显示图片
    plt.show()

def distSLC(vecA, vecB):
    # 使用球面余弦定理计算两点的距离
    a = np.sin(vecA[0, 1] * np.pi / 180) * np.sin(vecB[0, 1] * np.pi / 180)
    b = np.cos(vecA[0, 1] * np.pi / 180) * np.cos(
        vecB[0, 1] * np.pi / 180) * np.cos(np.pi *
                                           (vecB[0, 0] - vecA[0, 0]) / 180)
    return np.arccos(a + b) * 6371.0  # pi is imported with numpy

def clusterClubs(numClust=5):
    datList = []
    for line in open(r'./10-K均值聚类算法/places.txt').readlines():
        lineArr = line.split('\t')
        datList.append([float(lineArr[4]), float(lineArr[3])])
    datMat = np.mat(datList)
    myCentroids, clustAssing = biKmeans(datMat, numClust, distMeas=distSLC)
    fig = plt.figure()
    rect = [0.1, 0.1, 0.8, 0.8]
    scatterMarkers = ['s', 'o', '^', '8', 'p', 'd', 'v', 'h', '>', '<']
    axprops = dict(xticks=[], yticks=[])
    ax0 = fig.add_axes(rect, label='ax0', **axprops)
    # 基于图像创建矩阵
    imgP = plt.imread(r'./10-K均值聚类算法/Portland.png')
    ax0.imshow(imgP)
    ax1 = fig.add_axes(rect, label='ax1', frameon=False)
    for i in range(numClust):
        ptsInCurrCluster = datMat[np.nonzero(clustAssing[:, 0].A == i)[0], :]
        markerStyle = scatterMarkers[i % len(scatterMarkers)]
        ax1.scatter(ptsInCurrCluster[:, 0].flatten().A[0],
                    ptsInCurrCluster[:, 1].flatten().A[0],
                    marker=markerStyle,
                    s=90)
    ax1.scatter(myCentroids[:, 0].flatten().A[0],
                myCentroids[:, 1].flatten().A[0],
                marker='+',
                s=300)
    plt.show()

if __name__ == '__main__':
    clusterClubs(5)
    # plt_my_KMeans()
    # plt_biKmeans()


"""D4C1 scoring function

Based on an original matlab code from
Gustavo A. Stolovitzky, and Robert Prill.

"""
try:
    from StringIO import StringIO
except:
    from io import StringIO
from dreamtools.core.challenge import Challenge

import scipy.io
import numpy as np
import pandas as pd

class D4C1(Challenge):
    """A class dedicated to D4C1 challenge

    ::

        from dreamtools import D4C1
        s = D4C1()
        filename = s.download_template()
        s.score(filename)

    Data and templates are downloaded from Synapse. You must have a login.

    """
    def __init__(self, verbose=True, download=True, **kargs):
        """.. rubric:: constructor

        """
        super(D4C1, self).__init__('D4C1', verbose, download ,**kargs)

        self._indices_sh3 = [0, 1, 2, 4]  # gold standard for 3 was not ready
        self._indices_kinases = [5, 6, 7]
        self._indices_pdz = [8, 9, 10, 11, 12]

        self._init()

        self._load_golddata()
        self.results = {'kinase': {}, 'pdz': {}, 'sh3': {}}

    def score(self, filename):
        self._load_prediction(filename)
        self.score_kinases()
        self.score_pdz()
        self.score_sh3()
        return self.results

    def _load_golddata(self):
        filename = self._pj([self.classpath, 'goldstandard',
            'D4C1_goldstandard.txt'])
        data = open(filename).read()
        # in principle, the file contains 13 matrices with an empty
        # line in between (hence the \n\n and there is a line before
        # each matrix that is not a header hence header=None
        self.golddata = []
        for i in range(0, 13):
            df = pd.read_csv(StringIO(data.split("\n\n")[i]),
                    header=None, sep='\t', skiprows=1, index_col=0)
            self.golddata.append(df)

    def _load_prediction(self, filename):
        data = open(filename).read()
        # in principle, the file contains 13 matrices with an empty
        # line in between (hence the \n\n and there is a line before
        # each matrix that is not a header hence header=None
        self.prediction = []
        for i in range(0, 13):
            df = pd.read_csv(StringIO(data.split("\n\n")[i]),
                    header=None, sep='\t', skiprows=1, index_col=0)

            if all(df.apply(lambda x: abs(x.sum()-1) < 1e-12)) is False:
                print(all(df.apply(lambda x: abs(x.sum()-1) < 1e-12)))
                print('WARNING In Matrix %s, the sum over a column is not equal to 1!!' % (i+1))
                print(df.sum())
            self.prediction.append(df)

    def download_template(self):
        filename = self._pj([self.classpath, 'templates',
            'D4C1_templates.txt'])
        return filename

    def download_goldstandard(self):
        return  self._pj([self.classpath, 'goldstandard',
            'D4C1_goldstandard.txt'])

    def _probability(self, X, Y, x):
        dx = X[1]-X[0]
        P = sum(Y[X <= x])*dx
        return P

    def _load_probability(self, tag):
        filename = self._pj([self.directory, 'pdf_%s.mat' % str(tag)])
        pdffile = scipy.io.loadmat(filename)
        return pdffile

    def score_kinases(self):

        pvals = {}
        mydists = {}
        for i in self._indices_kinases:
            # i+1 since the original matlab file starts at 1 (not 0)
            pdffile = self._load_probability(i+1)

            X = pdffile['X'][0]
            Y = pdffile['Y'][0]
            mydist = self._frobenius_norm()[i]
            pval = self._probability(X, Y, mydist)
            mydists[i] = mydist
            pvals[i] = pval

        # cast to list required in py3
        self.results['kinase']['overall_score'] = -np.mean(
                np.log10(list(pvals.values())))
        self.results['kinase']['pvals'] = pvals
        self.results['kinase']['distances'] = mydists

    def score_pdz(self):

        pvals = {}
        mydists = {}
        for i in self._indices_pdz:
            # i+1 since the original matlab file starts at 1 (not 0)
            pdffile = self._load_probability(i+1)

            X = pdffile['X'][0]
            Y = pdffile['Y'][0]
            mydist = self._frobenius_norm()[i]
            pval = self._probability(X, Y, mydist)
            if pval < 1e-100:
                pval = 1e-100
            mydists[i] = mydist
            pvals[i] = pval

        # cast to list required in py3
        self.results['pdz']['overall_score'] = -np.mean(
                np.log10(list(pvals.values())))
        self.results['pdz']['pvals'] = pvals
        self.results['pdz']['distances'] = mydists

    def score_sh3(self):
        all_pvals = []
        all_offsets = []
        for i in self._indices_sh3:
            # i+1 since the original matlab file starts at 1 (not 0)
            pdffile = self._load_probability(i+1)

            XX = pdffile['XX']
            YY = pdffile['YY']
            G = self.golddata[i]
            T = self.prediction[i]
            scores, offsets = self._slide(G, T)

            pvals = []
            for k, score in enumerate(scores):
                X = XX[:, k]
                Y = YY[:, k]

                pval = self._probability(X, Y, score)
                if pval < 1e-100:
                    pval = 1e-100
                pvals.append(pval)

            index = np.argmin(pvals)
            all_offsets.append(offsets[index])
            all_pvals.append(pvals[index])

        self.results['sh3']['overall_score'] = -np.mean(np.log10(all_pvals))
        self.results['sh3']['pvals'] = all_pvals
        self.results['sh3']['offset'] = all_offsets

    def _frobenius_norm(self):
        return [np.linalg.norm(self.golddata[i]-self.prediction[i])
                for i in range(0, 13)]

    def _init(self):
        if self._standalone is True:
            return
        self._download_data('pdf_1.mat', 'syn4552308')
        self._download_data('pdf_2.mat', 'syn4552309')
        self._download_data('pdf_3.mat', 'syn4552310')
        self._download_data('pdf_4.mat', 'syn4552311')
        self._download_data('pdf_5.mat', 'syn4552312')
        self._download_data('pdf_6.mat', 'syn4552313')
        self._download_data('pdf_7.mat', 'syn4552314')
        self._download_data('pdf_8.mat', 'syn4552315')
        self._download_data('pdf_9.mat', 'syn4552316')
        self._download_data('pdf_10.mat', 'syn4552317')
        self._download_data('pdf_11.mat', 'syn4552318')
        self._download_data('pdf_12.mat', 'syn4552319')
        self._download_data('pdf_13.mat', 'syn4552320')

    def _slide(self, G, T):
        # FIXME: this code could be simplified (two loops into 1 ?)
        # slide T relative to G
        #Here we manipulate the dataframe. The columns are labelled 1 to 10
        score = []
        offset = []
        t_stop = 10
        g_start = 1
        for t_start in [6, 5, 4, 3, 2, 1]:
            g_stop = 10 - t_start + 1

            idx_t = list(range(t_start, t_stop+1))
            idx_g = list(range(g_start, g_stop+1))

            # matrix of element distances
            d = G[idx_g].values - T[idx_t].values

            # frob norm
            f = np.linalg.norm(d)

            # %% number of overlapping rows
            overlap = len(idx_t)

            # remember realtive position
            offset.append(g_start - t_start)

            # remember score
            score.append(f)

        # last position is complete overlap

        # continue sliding T relative to G
        t_start = 1
        g_stop = 10
        for t_stop in range(9, 5-1, -1):
            g_start = 10 - t_stop + 1

            idx_t = list(range(t_start, t_stop+1))
            idx_g = list(range(g_start, g_stop+1))

            # matrix of element distances
            d = G[idx_g].values - T[idx_t].values

            # frob norm
            f = np.linalg.norm(d)

            # number of overlapping rows
            overlap = len(idx_t)

            # remember realtive position
            offset.append(g_start - t_start)

            # remember score
            score.append(f)

        return score, offset


from sqlite3 import Time
from time import time
from django.shortcuts import render
from .models import *
from .utils import *
from .utils_nba import *
from Sports.newsApi import getNews
import requests
from .models import City
from .forms import CityForm, BaseballDateForm, BasketballDateForm
import datetime
from Sports.API.weatherAPI import fetchWeather
from django.http import HttpResponseRedirect

def WeatherHome(request):
    dat = {}
    ctx = {'weather': dat, 'form': CityForm(), 'alert':{}, 'teams':{}}
    if request.method == 'POST':
        ctx['form'] = CityForm(request.POST)
        if ctx['form'].is_valid():
            city = ctx['form'].cleaned_data['name']
            weather = fetchWeather(city)
            ctx['weather'] = weather['weather']
            ctx['alerts'] = weather['alerts']
            teams = cityToTeams(city.title())
            if teams:
                ctx['teams'] = teams.items()

    return render(request, 'Sports/weather-home.html', ctx)

def BaseballPage(request):
    context = {}
    context['form'] = BaseballDateForm()

    if request.GET:
        temp = request.GET['date_field']
        date_string = populateBoxScores(temp)
    else:
        date_string = populateBoxScoresDefault()

    mlbGames = MlbBoxScoreData.objects.filter(date_played=date_string)
    context['mlbGames'] = mlbGames
    context['datePlayed'] = date_string
    if mlbGames:
        context['GamesPlayed'] = True
    else:
        context['GamesPlayed'] = False
    return render(request, 'Sports/baseball.html', context)

def BasketballPage(request):
    context = {}
    context['form'] = BasketballDateForm()

    if request.GET:
        temp = request.GET['date_field']
        date_string = populateNBABoxScores(temp)
    else:
        date_string = populateNBABoxScores("2022-06-16")

    nbaGames = NbaBoxScoreData.objects.filter(date_played=date_string)
    context['nbaGames'] = nbaGames
    context['datePlayed'] = date_string
    if nbaGames:
        context['GamesPlayed'] = True
    else:
        context['GamesPlayed'] = False
    return render(request, 'Sports/basketball.html', context)

def Home(request):
    return render(request, 'Sports/home.html')

def NewsHome(request):
    input_value = 'null'
    context = getNews(input_value)
    return render(request, 'Sports/news-home.html', context)

def NewsHome_search(request):
    if request.method == 'GET':
        search_query = request.GET.get('search_box', None)

    context = getNews(search_query)
    return render(request, 'Sports/news-home.html', context)

def NewsHome_sort(request):
    if request.method == 'GET':
        sort_option = request.GET.get('option', None)

    context = getNews(sort_option)
    return render(request, 'Sports/news-home.html', context)

def SportsHome(request):
    return HttpResponseRedirect("/Sports/Baseball")


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random as rnd
import seaborn as sns

from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier

train_df = pd.read_csv('train.csv')
test_df = pd.read_csv('test.csv')
combine = [train_df,test_df]
train_df.shape
test_df.shape
combine[0].shape
combine[1].shape

train_df['Age'].plot()
train_df['Age'].plot(kind='line')
train_df['Age'].plot(kind='scatter')
train_df['Age'].plot(kind='box')

train_df.head()
train_df.tail()
train_df.describe()
train_df.describe(include=['O'])

train_df[['Pclass','Survived']].groupby(['Pclass'],
        as_index=False).mean().sort_values(by='Survived',ascending=True)
train_df[['Sex','Survived']].groupby(['Sex'],
        as_index=False).mean().sort_values(by='Sex',ascending=False)
train_df[['SibSp','Survived']].groupby(['SibSp'],
        as_index=False).mean().sort_values(by='SibSp',ascending=False)
train_df[['Parch','Survived']].groupby(['Parch'],
        as_index=False).mean().sort_values(by='Parch',ascending=False)

g = sns.FacetGrid(train_df,col='Survived')
g.map(plt.hist,'Age',bins=20)
plt.show()
g = sns.FacetGrid(train_df,col='Survived',row='Pclass',size=2.2,aspect=1.6)
g.map(plt.hist,'Age',bins=20)
plt.show()
g = sns.FacetGrid(train_df,row='Embarked',size=2.2,aspect=1.6)
g.map(sns.pointplot,'Pclass','Survived','Sex',palette='deep')
g.add_legend()
g = sns.FacetGrid(train_df,row='Embarked',size=2.2,aspect=1.6)
g.map(sns.barplot,'Sex','Fare',alpha=0.5,ci=None)
g.add_legend()

print('before',train_df.shape,test_df.shape,combine[0].shape,combine[1].shape)
train_df = train_df.drop(['Ticket','Cabin'],axis=1)
test_df = test_df.drop(['Ticket','Cabin'],axis=1)
combine = [train_df,test_df]
print('after',train_df.shape,test_df.shape,combine[0].shape,combine[1].shape)

for dataset in combine:
    dataset['Title'] = dataset.Name.str.extract(' ([A-Za-z]+)\.',expand=False)
print(train_df.shape,test_df.shape,combine[0].shape,combine[1].shape)
pd.crosstab(train_df['Title'],train_df['Sex'])

for dataset in combine:
    dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
           'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'],'Rare')
    dataset['Title'] = dataset['Title'].replace('Mlle','Miss')
    dataset['Title'] = dataset['Title'].replace('Ms','Miss')
    dataset['Title'] = dataset['Title'].replace('Mme','Mrs')
train_df[['Title','Survived']].groupby(['Title'],
        as_index=False).mean().sort_values(by='Survived',ascending=False)

title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
    dataset['Title'] = dataset['Title'].map(title_mapping)
    dataset['Title'] = dataset['Title'].fillna(0)
train_df.head()

print('before',train_df.shape,test_df.shape,combine[0].shape,combine[1].shape)
train_df = train_df.drop(['Name','PassengerId'],axis=1)
test_df = test_df.drop(['Name'],axis=1)
combine = [train_df,test_df]
print('after',train_df.shape,test_df.shape,combine[0].shape,combine[1].shape)

for dataset in combine:
    dataset['Sex'] = dataset['Sex'].map({'female':1,'male':0}).astype(int)
train_df.head()

g = sns.FacetGrid(train_df,row='Pclass',col='Sex',size=2.2,aspect=1.6)
g.map(plt.hist,'Age',alpha=0.5,bins=20)
g.add_legend()

guess_ages = np.zeros((2,3))
guess_ages

for dataset in combine:
    for i in range(0,2):
        for j in range(0,3):
            guess_df = dataset[(dataset['Sex']==i) &
                               (dataset['Pclass']==j+1)]['Age'].dropna()
            age_guess = guess_df.median()
            guess_ages[i,j] = int( age_guess/0.5 + 0.5) * 0.5
guess_ages
for dataset in combine:
   for i in range(0,2):
       for j in range(0,3):
           dataset.loc[(dataset.Age.isnull()) & (dataset.Sex ==i) & (dataset.Pclass ==j+1),\
                       'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
train_df.head(10)

train_df['AgeBand'] = pd.cut(train_df['Age'],5)
train_df[['AgeBand','Survived']].groupby(['AgeBand'],
        as_index=False).mean().sort_values(by='Survived',ascending=False)

for dataset in combine:
    dataset.loc[dataset['Age'] <= 16 , 'Age'] = 0
    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32),'Age'] = 1
    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48),'Age'] = 2
    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64),'Age'] = 3
    dataset.loc[dataset['Age'] > 64,'Age']
train_df.head(10)

train_df = train_df.drop(['AgeBand'],axis=1)
combine = [train_df,test_df]

for dataset in combine:
    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize','Survived']].groupby(['FamilySize'],
        as_index=False).mean().sort_values(by='Survived',ascending=False)

for dataset in combine:
    dataset['IsAlone'] = 0
    dataset.loc[dataset['FamilySize']==1,'IsAlone'] = 1
train_df[['IsAlone','Survived']].groupby(['IsAlone'],
        as_index=False).mean().sort_values(by='Survived',ascending=False)

train_df = train_df.drop(['Parch','SibSp','FamilySize'],axis=1)
test_df = test_df.drop(['Parch','SibSp','FamilySize'],axis=1)
combine = [train_df,test_df]

for dataset in combine:
    dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:,['Age*Class','Age','Pclass']].head(10)

freq_port = train_df.Embarked.dropna().mode()[0]
freq_port

for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
train_df[['Embarked','Survived']].groupby(['Embarked'],
        as_index=False).mean().sort_values(by='Survived',ascending=False)

for dataset in combine:
    dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2}).astype(int)
train_df.head()

test_df['Fare'].fillna(test_df['Fare'].dropna().median(),inplace=True)
test_df.head()

train_df['FareBand'] = pd.cut(train_df['Fare'],4)
train_df[['FareBand','Survived']].groupby(['FareBand'],
        as_index=False).mean().sort_values(by='FareBand',ascending=False)

for dataset in combine:
    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
    dataset.loc[(dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454),'Fare'] = 1
    dataset.loc[(dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31),'Fare'] = 2
    dataset.loc[(dataset['Fare'] > 31) ,'Fare'] = 3
    dataset['Fare'] = dataset['Fare'].astype(int)
train_df = train_df.drop(['FareBand'],axis=1)
combine = [train_df,test_df]
train_df.shape,test_df.shape

X_train = train_df.drop('Survived',axis=1)
Y_train = train_df['Survived']
X_test = test_df.drop('PassengerId',axis=1).copy()
X_train.shape,Y_train.shape,X_test.shape

from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train,Y_train)
Y_pred = logreg.predict(X_test)
acc_log = round(logreg.score(X_train,Y_train)*100,2)
acc_log

coeff_df = pd.DataFrame(train_df.columns.delete(0))
coeff_df.columns = ['Feature']
coeff_df['correlations'] = pd.Series(logreg.coef_[0])
coeff_df.sort_values(by='correlations',ascending=False)

from sklearn.svm import SVC
svc = SVC()
svc.fit(X_train,Y_train)
Y_pred = svc.predict(X_test)
acc_svc = round(svc.score(X_train,Y_train)*100,2)
acc_svc

from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors = 3)
knn.fit(X_train,Y_train)
Y_pred = knn.predict(X_test)
acc_knn = round(knn.score(X_train,Y_train)*100,2)
acc_knn

from sklearn.naive_bayes import GaussianNB
gaussian = GaussianNB()
gaussian.fit(X_train,Y_train)
Y_pred = gaussian.predict(X_test)
acc_gaussian = round(gaussian.score(X_train,Y_train)*100,2)
acc_gaussian

from sklearn.linear_model import Perceptron
perceptron = Perceptron()
perceptron.fit(X_train,Y_train)
Y_pred = perceptron.predict(X_test)
acc_perceptron = round(perceptron.score(X_train,Y_train)*100,2)
acc_perceptron

from sklearn.svm import LinearSVC
linear_svc = LinearSVC()
linear_svc.fit(X_train,Y_train)
Y_pred = linear_svc.predict(X_test)
acc_linear_svc = round(linear_svc.score(X_train,Y_train)*100,2)
acc_linear_svc

from sklearn.linear_model import SGDClassifier
sgd = SGDClassifier()
sgd.fit(X_train,Y_train)
Y_pred = sgd.predict(X_test)
acc_sgd = round(sgd.score(X_train,Y_train)*100,2)
acc_sgd

from sklearn.tree import DecisionTreeClassifier
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train,Y_train)
Y_pred = decision_tree.predict(X_test)
acc_decision_tree = round(decision_tree.score(X_train,Y_train)*100,2)
acc_decision_tree

from sklearn.ensemble import RandomForestClassifier
random_forest = RandomForestClassifier(n_estimators=100)
random_forest.fit(X_train,Y_train)
Y_pred = random_forest.predict(X_test)
acc_random_forest = round(random_forest.score(X_train,Y_train)*100,2)
acc_random_forest

models = pd.DataFrame({
        'Model':['Support Vector Machines', 'KNN', 'Logistic Regression',
                 'Random Forest', 'Naive Bayes', 'Perceptron',
                 'Stochastic Gradient Decent', 'Linear SVC',
                 'Decision Tree'],
         'Score':[acc_svc, acc_knn, acc_log,
                  acc_random_forest, acc_gaussian, acc_perceptron,
                  acc_sgd, acc_linear_svc, acc_decision_tree]})
models.sort_values(by='Score',ascending=False)

from sklearn.model_selection import cross_val_score
accuracies = cross_val_score(estimator = decision_tree, X = X_train, y = Y_train, cv = 10)
meanda = accuracies.mean()
deviationda = accuracies.std()

from sklearn.metrics import make_scorer
from sklearn.model_selection import GridSearchCV
decision_tree = DecisionTreeClassifier()
from sklearn.metrics import f1_score, make_scorer

parameters = {'max_depth':[2,4,6,8,10],
              'min_samples_leaf':[2,4,6,8,10],
              'min_samples_split':[2,4,6,8,10]}

scorer = make_scorer(f1_score)

grid_obj = GridSearchCV(decision_tree,parameters,scoring=scorer)

grid_fit = grid_obj.fit(X_train,Y_train)

best_fit = grid_fit.best_estimator_

best_fit.fit(X_train,Y_train)

best_train_predictions = best_fit.predict(X_train)
best_test_predictions = best_fit.predict(X_test)

from sklearn.metrics import confusion_matrix
best_cn_train = confusion_matrix(Y_train,best_train_predictions)

best_f1score_train = f1_score(Y_train,best_train_predictions)

plot_model(X, y, best_clf)


from bs4 import BeautifulSoup as bs
import pandas as pd
from splinter import Browser
import time

def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {"executable_path": "chromedriver"}
    return Browser("chrome", **executable_path, headless=False)

def scrape_info():
    browser = init_browser()

    # Visit NASA Mars News
    url = "http://mars.nasa.gov/news"
    browser.visit(url)

    time.sleep(1)

    # Scrape page into Soup
    html = browser.html
    soup = bs(html, "html.parser")

    # Get Mars Latest News and Description
    title = soup.find_all('div', class_='content_title')[0].text
    paragraph = soup.find_all('div', class_='article_teaser_body')[0].text

    # Featured Image
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    image_url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(image_url)

    # HTML Object
    featured_html = browser.html
    featured_soup = bs(featured_html, 'html.parser')
    images = featured_soup.find('footer')
    link = images.find('a')['data-fancybox-href']
    featured_image = image_url + link

    # Mars Facts
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    mars_facts_url = 'https://space-facts.com/mars/'
    mars_facts = pd.read_html(mars_facts_url)
    mars_df = mars_facts[0]
    mars_df.columns = ["Mars", "Mars Facts"]
    mars_facts_html = mars_df.to_html()
    mars_facts_html.replace('\n', '')

    # HEMISPHERE IMAGE LINKS
    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    cerberus_link = 'https://astrogeology.usgs.gov/search/map/Mars/Viking/cerberus_enhanced'
    browser.visit(cerberus_link)
    cerberus_html = browser.html
    cerberus_soup = bs(cerberus_html, 'html.parser')
    cerberus = cerberus_soup.find('li').a['href']
    #  print(cerberus)

    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    schiaparelli_link = 'https://astrogeology.usgs.gov/search/map/Mars/Viking/schiaparelli_enhanced'
    browser.visit(schiaparelli_link)
    schiaparelli_html = browser.html
    schiaparelli_soup = bs(schiaparelli_html, 'html.parser')
    schiaparelli = schiaparelli_soup.find('li').a['href']
    #print(schiaparelli)

    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    syrtis_major_link = 'https://astrogeology.usgs.gov/search/map/Mars/Viking/syrtis_major_enhanced'
    browser.visit(syrtis_major_link)
    syrtis_major_html = browser.html
    syrtis_major_soup = bs(syrtis_major_html, 'html.parser')
    syrtis_major = syrtis_major_soup.find('li').a['href']
    #print(syrtis_major)

    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    valles_marineris_link = 'https://astrogeology.usgs.gov/search/map/Mars/Viking/valles_marineris_enhanced'
    browser.visit(valles_marineris_link)
    valles_marineris_html = browser.html
    valles_marineris_soup = bs(valles_marineris_html, 'html.parser')
    valles_marineris = valles_marineris_soup.find('li').a['href']
    #print(valles_marineris)

    # Store data in a dictionary
    mars_data = {
        "title": title,
        "paragraph": paragraph,
        "featured_image": featured_image,
        "facts": str(mars_facts_html),
        "cerberus": cerberus,
        "schiaparelli": schiaparelli,
        "syrtis_major": syrtis_major,
        "valles_marineris": valles_marineris
     }

    print(mars_data)
    # Close the browser after scraping
    browser.quit()

    # Return results
    return mars_data


import tornado.ioloop
import tornado.web
import zip_push
import bot_telegram

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        args = {}
        for x in self.request.arguments:
            args[x] = self.request.arguments[x][0].decode('utf-8')

        files_p = args.get('filespath')
        yad_p = args.get('yandpath')
        tok = args.get('token')
        f = zip_push
        b = bot_telegram

        if f.main(files_p, yad_p, tok) == 0:
            self.write('Files uploaded')
            b.reminder_message('Archive with files is uploaded')
        else:
            b.reminder_message('Today files are missing!')
            self.write('Today files are not received')
            self.set_status(500)
            return

def make_app():
    return tornado.web.Application([
        (r"/", MainHandler),
    ])

if __name__ == "__main__":
    app = make_app()
    app.listen(8888)
    tornado.ioloop.IOLoop.current().start()


from typing import List, Dict
from mysql import connector
from parking_lot.db_conn.connection import DBConnection
from parking_lot.helper.debug_helper import DebugHelper

class ParkingLotRepository:
    def __init__(self):
        self.db = DBConnection.instance().connection

    def get_all(self) -> List[Dict]:
        m_cursor = self.db.cursor()
        lots = []
        try:
            m_cursor.execute("SELECT * FROM parking_lot WHERE is_available = '1'")
            rs = m_cursor.fetchall()
            for x in rs:
                lot = dict()
                lot["id"] = x[0]
                lot["name"] = x[1]
                lot["location"] = x[2]
                lot["pin"] = x[3]
                lot["size"] = x[6]
                lots.append(lot)
        except connector.Error as e:
            print(e)
            DebugHelper.print_exception()
        finally:
            if self.db.is_connected():
                m_cursor.close()
        return lots

    def get_lot(self, slot_id: int) -> Dict:
        m_cursor = self.db.cursor()
        lot = {}
        try:
            m_cursor.execute("SELECT * FROM parking_lot WHERE id = {}".format(slot_id))
            for row in m_cursor:
                lot["id"] = row[0]
                lot["name"] = row[1]
                lot["location"] = row[2]
                lot["pin"] = row[3]
                lot["size"] = row[6]
        except connector.Error as e:
            print(e)
            DebugHelper.print_exception()
        finally:
            if self.db.is_connected():
                m_cursor.close()
        return lot


# 18352
import sys
from collections import deque

input = sys.stdin.readline

# 도시의 개수 N, 도로의 개수 M, 거리 정보 K, 출발 도시의 번호 X
n, m, k, x = map(int,input().split())

INF = int(1e9)
graph = [ [] for _ in range(n+1)]

for _ in range(m):
    a,b = map(int, input().split())
    graph[a].append(b)

distance = [INF] * (n+1)
distance[x] = 0

#BFS
visited = [False] * (n+1)
visited[x] = True
q = deque([x])
while q:
    now = q.popleft()
    for i in graph[now]:
        if distance[i] > distance[now] + 1 :
            q.append(i)
            distance[i] = distance[now] + 1

answer = []
for idx, d in enumerate(distance):
    if d == k:
        answer.append(idx)

if len(answer) == 0 :
    print(-1)
else:
    for a in answer:
        print(a)


import cv2

import numpy as np
import tensorflow as tf
import tensorflow.keras as kr

import Configuration as cfg

def load_image(image_path):
    image = cv2.imread(image_path)
    height, width, channel = image.shape
    reshape_image = cv2.resize(image, dsize=(cfg.image_size_width, cfg.image_size_height), interpolation=cv2.INTER_CUBIC)
    np_image = np.array(reshape_image, dtype=float)
    expand_np_image = np.expand_dims(np_image, axis=0)
    return image, np_image, expand_np_image, width, height

class DataGenerator(kr.utils.Sequence):
    def __init__(self, data_path, batch_size, transform_param=None):
        self.data_image_info = []
        self.data_label_info = {}

        data_file = open(data_path, 'r')
        while True:
            line = data_file.readline()
            if not line:
                break

            image_name_box_cnt = line.split(' ')
            self.data_image_info.append(image_name_box_cnt[0])

            class_box_info_list = []
            for idx in range(int(image_name_box_cnt[1])):
                line = data_file.readline()

                class_box_info = line.split(' ')
                class_box_info_list.append((int(class_box_info[0]), int(class_box_info[1]), int(class_box_info[2]), int(class_box_info[3]), int(class_box_info[4])))

            self.data_label_info[image_name_box_cnt[0]] = class_box_info_list
        data_file.close()

        self.batch_size = batch_size
        self.image_data_generator = kr.preprocessing.image.ImageDataGenerator()
        self.transform_param = transform_param
        self.shuffle = True

        self.on_epoch_end()

    def __len__(self):
        return int(np.floor(len(self.data_image_info) / self.batch_size))

    def __getitem__(self, index):
        batch_data_image_info = self.data_image_info[index * self.batch_size:(index + 1) * self.batch_size]
        image, label = self.data_generation(batch_data_image_info)

        return image, label

    def on_epoch_end(self):
        if self.shuffle == True:
            np.random.shuffle(self.data_image_info)

    def data_generation(self, batch_data_image_info):
        batch_image = np.empty((self.batch_size, cfg.image_size_width, cfg.image_size_height, 3))
        batch_label = np.empty((self.batch_size, cfg.grid_size, cfg.grid_size, ((4 + 1) + cfg.object_class_num)))

        for batch_idx, data_image_info in enumerate(batch_data_image_info):
            _, np_image, _, width, height = load_image(data_image_info)

            np_image -= cfg.image_mean_value
            np_image /= 255.0

            if self.transform_param is not None:
                batch_image[batch_idx] = self.image_data_generator.apply_transform(np_image, self.transform_param)
            else:
                batch_image[batch_idx] = np_image

            data_label_info = np.array(self.data_label_info[data_image_info], dtype=float)

            class_info = np.array(data_label_info[:, 0], dtype=int)

            data_label_info[:, 1:5:2] /= width
            data_label_info[:, 2:5:2] /= height

            box_info = np.stack([
                (data_label_info[:, 1] + data_label_info[:, 3]) / 2,
                (data_label_info[:, 2] + data_label_info[:, 4]) / 2,
                (data_label_info[:, 3] - data_label_info[:, 1]),
                (data_label_info[:, 4] - data_label_info[:, 2])
            ], axis=1)

            x_grid_idx = np.array(box_info[:, 0] * cfg.grid_size, dtype=int)
            y_grid_idx = np.array(box_info[:, 1] * cfg.grid_size, dtype=int)

            label = np.zeros((cfg.grid_size, cfg.grid_size, ((4 + 1) + cfg.object_class_num)))
            label[y_grid_idx[:], x_grid_idx[:], 0] = 1
            label[y_grid_idx[:], x_grid_idx[:], 1:5] = box_info[:, ...]
            label[y_grid_idx[:], x_grid_idx[:], 5 + class_info[:]] = 1

            batch_label[batch_idx] = label

        return batch_image, batch_label


from flask import Flask, render_template, request
# from markupsafe import escape
from os import getcwd

directory = getcwd()
app = Flask(__name__, template_folder='template', static_folder='static')

@app.route('/<username>')
def user_page(username):
    html_page = f' <h1> Hello {username} </h1>'
    return html_page

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        message = request.form.get('UserChat')
        if message:
            print(message)
        else:
            print('message is empty')
        return render_template('index.html')
    elif request.method == 'GET':
        return render_template('index.html')

if __name__ == '__main__':
    app.run(debug=True)


from modules.IPlugin  import IPlugin

class AddTextFromTargetToDestination(IPlugin):

    def __init__(self):
        super().__init__()
        self.pluginName = "AddTextFromTargetToDestination"
        self.pluginAutor = "Sebastian Rottgardt"
        self.pluginVersion = 0.1
        self.pluginDescription = "Add Text to Destination Column"
        self.pluginCommand = "addText"

    def preExecute(self):
        self.execute()

    def execute(self):
        message = []

        for rowsData in self.commandData.getItems():

            # check targetColumn exists
            if self.commandData.getTargetColumn() in rowsData:
                rowsData[self.commandData.getDestinationColumn()] = rowsData[self.commandData.getTargetColumn()] + str(self.commandData.getOptionsValue())
            else:
                rowsData[self.commandData.getTargetColumn()] = "targetColumn not exists"

            message.append(rowsData)

        if message is not None:
            self.commandData.setItems(message)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Device',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('temperature', models.DecimalField(max_digits=5, decimal_places=2)),
                ('humidity', models.DecimalField(max_digits=5, decimal_places=2)),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Room',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(default=b'', max_length=50)),
                ('description', models.TextField(default=b'')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AddField(
            model_name='device',
            name='room',
            field=models.ForeignKey(to='api.Room'),
            preserve_default=True,
        ),
    ]


import copy
from collections import defaultdict
import inspect
from typing import Any, Callable, List, Optional, Union

__all__ = ['get', 'register', 'all']
__MODS__ = defaultdict(lambda: {})

def get(group_name: str, name: Optional[str]=None) -> Any:
    """
        :param group_name(str): group name of the class or function belong to.
        :param name(str): module name or alias name for the class or function
        >>> f = get('test_group', 'f_function')
        >>> f()
        >>> #============
        >>> A = get('test_group', 'a_function')
        >>> a = A()
    """
    if group_name not in __MODS__.keys():
        raise RuntimeError(
            f'No such mod group named ({group_name}).')
    group = __MODS__[group_name]

    if name is None:
        return group

    if name not in group.keys():
        raise RuntimeError(
            f'No such mod named ({name}) in group ({group_name}).')
    return group[name]

def add(group_name: str,
        mod: Any, *,
        alias: Optional[Union[str, List[str]]]=None
) -> None:
    mod_name = mod.__name__
    __MODS__[group_name][mod_name] = mod
    if alias is not None:
        if not isinstance(alias, (list, tuple)):
            alias = [alias]
        for alias_item in alias:
            __MODS__[group_name][alias_item] = mod

def all() -> dict:
    return dict(copy.copy(__MODS__))

def register(group_name: str, **kwargs) -> Callable:
    """
        :param group_name(str): group name of the class or function belong to.
        :param alias(str): bind name for the class or function,
            default: the class name or the function name
        >>> @register('test_group', alias='f_function')
        >>> def f(*args, **kwargs):
        >>>     pass
        >>> #=============
        >>> @register('test_group', alias='a_class')
        >>> class A:
        >>>     def __init__(self):
        >>>         pass
    """
    def _exec(func):
        if inspect.isfunction(func) or inspect.isclass(func):
            alias_name = kwargs.get('alias', None)
            mod_name = func.__name__
            __MODS__[group_name][mod_name] = func
            if alias_name is not None:
                if not isinstance(alias_name, (list, tuple)):
                    alias_name = [alias_name]

                for alias_name_item in alias_name:
                    __MODS__[group_name][alias_name_item] = func
        def __exec(*args, **kwargs):
            return func(*args, **kwargs)
        return __exec
    return _exec

if __name__ == '__main__':
    # test
    @register('test_func')
    def f(a):
        print(a)

    ff = get('test_func', 'f')
    ff(2)

    print('-' * 20)
    try:
        ff = get('undefined_group', 'f')
        print('ERROR')
    except RuntimeError:
        print('OK')

    try:
        ff = get('test_func', 'ff')
        print('ERROR')
    except RuntimeError:
        print('OK')

    print('-' * 20)

    @register('test_cls')
    class A(object):
        def __init__(self, a) -> None:
            super().__init__()
            print('init A', a)

    AA = get('test_cls', 'A')
    aa = AA(1)

    print('-' * 20)
    A(2)

    print('-' * 20)
    @register('test_func', alias='f2')
    def f22(a):
        print(a)

    ff = get('test_func', 'f2')
    ff(8)


import os
import sys
import sysconfig
import logging
import distutils
import subprocess
import textwrap

from setuptools import Extension
from distutils.spawn import find_executable
from setuptools.command.build_ext import build_ext as _build_ext
from distutils.file_util import copy_file
from typing import List

logger = logging.getLogger(__name__)
cocotb_share_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "cocotb", "share"))

def create_sxs_assembly_manifest(name: str, filename: str, libraries: List[str]) -> str:
    """
    Create side-by-side (sxs) assembly manifest

    It contains dependencies to other assemblies (in our case the assemblies are equal to the other libraries).
    For more details see:
     - https://docs.microsoft.com/en-us/windows/win32/sbscs/assembly-manifests
     - https://docs.microsoft.com/en-us/windows/win32/sbscs/using-side-by-side-assemblies

    Args:
        name: The name of the assembly for which the manifest is generated, e.g. ``libcocotbutils``.
        filename: The filename of the library, e.g. ``libcocotbutils.dll``.
        libraries: A list of names of dependent manifests, e.g. ``["libgpilog"]``.
    """

    architecture = "amd64" if sys.maxsize > 2**32 else "x86"
    dependencies = []

    for lib in libraries:
        dependencies.append(textwrap.dedent('''\
            <dependency>
                <dependentAssembly>
                    <assemblyIdentity name="%s" version="1.0.0.0" type="win32" processorArchitecture="%s" />
                </dependentAssembly>
            </dependency>
            ''') % (lib, architecture))

    manifest_body = textwrap.dedent('''\
        <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
        <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
            <assemblyIdentity name="%s" version="1.0.0.0" type="win32" processorArchitecture="%s" />
            <file name="%s" />
            %s
        </assembly>
        ''') % (name, architecture, filename, textwrap.indent("".join(dependencies), '    ').strip())

    return manifest_body

def create_sxs_appconfig(filename):
    """
    Create side-by-side (sxs) application configuration file.

    The application configuration specifies additional search paths for manifests.
    For more details see: https://docs.microsoft.com/en-us/windows/win32/sbscs/application-configuration-files
    """

    config_body = textwrap.dedent('''\
        <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
        <configuration>
            <windows>
                <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
                    <probing privatePath="libs" />
                </assemblyBinding>
            </windows>
        </configuration>
        ''')

    dirpath = os.path.dirname(filename)
    os.makedirs(dirpath, exist_ok=True)

    with open(filename + ".2.config", "w", encoding='utf-8') as f:
        f.write(config_body)

def create_rc_file(name, filename, libraries):
    """
    Creates windows resource definition script to embed the side-by-side assembly manifest into the libraries.

    For more details see: https://docs.microsoft.com/en-us/windows/win32/menurc/about-resource-files
    """

    manifest = create_sxs_assembly_manifest(name, filename, libraries)

    # Escape double quotes and put every line between double quotes for embedding into rc file
    manifest = manifest.replace('"', '""')
    manifest = '\n'.join(['"%s\\r\\n"' % x for x in manifest.splitlines()])

    rc_body = textwrap.dedent('''\
        #pragma code_page(65001) // UTF-8
        #include <WinUser.h>

        LANGUAGE 0x00, 0x00

        ISOLATIONAWARE_MANIFEST_RESOURCE_ID RT_MANIFEST
        BEGIN
        %s
        END
        ''') % manifest

    with open(name + ".rc", "w", encoding='utf-8') as f:
        f.write(rc_body)

def _get_lib_ext_name():
    """ Get name of default library file extension on given OS. """

    if os.name == "nt":
        ext_name = "dll"
    else:
        ext_name = "so"

    return ext_name

class build_ext(_build_ext):
    def run(self):

        def_dir = os.path.join(cocotb_share_dir, "def")
        self._gen_import_libs(def_dir)

        if os.name == "nt":
            create_sxs_appconfig(self.get_ext_fullpath(os.path.join("cocotb", "simulator")))

            ext_names = {os.path.split(ext.name)[-1] for ext in self.extensions}
            for ext in self.extensions:
                fullname = self.get_ext_fullname(ext.name)
                filename = self.get_ext_filename(fullname)
                name = os.path.split(fullname)[-1]
                filename = os.path.split(filename)[-1]
                libraries = {"lib" + lib for lib in ext.libraries}.intersection(ext_names)
                create_rc_file(name, filename, libraries)

        super().run()

    # Needed for Windows to not assume python module (generate interface in def file)
    def get_export_symbols(self, ext):
        return None

    # For proper cocotb library naming, based on https://github.com/cython/cython/issues/1740
    def get_ext_filename(self, ext_name):
        """
        Like the base class method, but for libraries that are not python extension:
         - removes the ``.cpython-36m-x86_64-linux-gnu.`` or ``-cpython-36m.`` part before the extension
         - replaces ``.pyd`` with ``.dll`` on windows.
        """

        filename = _build_ext.get_ext_filename(self, ext_name)

        # for the simulator python extension library, leaving suffix in place
        if "simulator" == os.path.split(ext_name)[-1]:
            return filename

        head, tail = os.path.split(filename)
        tail_split = tail.split(".")

        # mingw on msys2 uses `-` as seperator
        tail_split = tail_split[0].split("-")

        filename_short = os.path.join(head, tail_split[0] + "." + _get_lib_ext_name())

        # icarus requires vpl extension
        filename_short = filename_short.replace("libcocotbvpi_icarus.so", "libcocotbvpi_icarus.vpl")
        filename_short = filename_short.replace("libcocotbvpi_icarus.dll", "libcocotbvpi_icarus.vpl")

        return filename_short

    def finalize_options(self):
        """ Like the base class method,but add extra library_dirs path. """

        super().finalize_options()

        for ext in self.extensions:
            ext.library_dirs.append(os.path.join(self.build_lib, "cocotb", "libs"))

    def copy_extensions_to_source(self):
        """ Like the base class method, but copy libs into proper directory in develop. """

        build_py = self.get_finalized_command("build_py")
        for ext in self.extensions:
            fullname = self.get_ext_fullname(ext.name)
            filename = self.get_ext_filename(fullname)
            modpath = fullname.split(".")
            package = ".".join(modpath[:-1])
            package_dir = build_py.get_package_dir(package)
            # unlike the method from `setuptools`, we do not call `os.path.basename` here
            dest_filename = os.path.join(package_dir, filename)
            src_filename = os.path.join(self.build_lib, filename)

            os.makedirs(os.path.dirname(dest_filename), exist_ok=True)

            copy_file(
                src_filename, dest_filename, verbose=self.verbose, dry_run=self.dry_run
            )
            if ext._needs_stub:
                self.write_stub(package_dir or os.curdir, ext, True)

    def _gen_import_libs(self, def_dir):
        """
        On Windows generate import libraries that contains the code required to
        load the DLL (.a) based on module definition files (.def)
        """

        if os.name == "nt":
            for sim in ["icarus", "modelsim", "aldec", "ghdl"]:
                subprocess.run(
                    [
                        "dlltool",
                        "-d",
                        os.path.join(def_dir, sim + ".def"),
                        "-l",
                        os.path.join(def_dir, "lib" + sim + ".a"),
                    ]
                )

def _extra_link_args(lib_name=None, rpaths=[]):
    """
    Add linker argument to load dependencies from the directory where vpi/vhpi/fli library is located
    On osx use `install_name`.
    Use `rpath` on all platforms
    """

    args = []
    if sys.platform == "darwin" and lib_name is not None:
        args += ["-Wl,-install_name,@rpath/%s.so" % lib_name]
    for rpath in rpaths:
        if sys.platform == "darwin":
            rpath = rpath.replace("$ORIGIN", "@loader_path")
        args += ["-Wl,-rpath,%s" % rpath]
    if os.name == "nt":
        # Align behavior of gcc with msvc and export only symbols marked with __declspec(dllexport)
        args += ["-Wl,--exclude-all-symbols"]
    else:
        args += ["-flto"]
    return args

def _get_python_lib_link():
    """ Get name of python library used for linking """

    if sys.platform == "darwin":
        ld_library = sysconfig.get_config_var("LIBRARY")
    else:
        ld_library = sysconfig.get_config_var("LDLIBRARY")

    if ld_library is not None:
        python_lib_link = os.path.splitext(ld_library)[0][3:]
    else:
        python_version = sysconfig.get_python_version().replace(".", "")
        python_lib_link = "python" + python_version

    return python_lib_link

def _get_python_lib():
    """ Get the library for embedded the python interpreter """

    if os.name == "nt":
        python_lib = _get_python_lib_link() + "." + _get_lib_ext_name()
    else:
        python_lib = "lib" + _get_python_lib_link() + "." + _get_lib_ext_name()

    return python_lib

# TODO [gh-1372]: make this work for MSVC which has a different flag syntax
_base_warns = ["-Wall", "-Wextra", "-Wcast-qual", "-Wwrite-strings", "-Wconversion"]
_ccx_warns = _base_warns + ["-Wnon-virtual-dtor", "-Woverloaded-virtual"]
_extra_cxx_compile_args = ["-std=c++11", "-fvisibility=hidden", "-fvisibility-inlines-hidden"] + _ccx_warns
if os.name != "nt":
    _extra_cxx_compile_args += ["-flto"]

# Make PRI* format macros available with C++11 compiler but older libc, e.g. on RHEL6.
_extra_defines = [("__STDC_FORMAT_MACROS", "")]

def _get_common_lib_ext(include_dir, share_lib_dir):
    """
    Defines common libraries.

    All libraries go into the same directory to enable loading without modifying the library path (e.g. LD_LIBRARY_PATH).
    In Makefile `LIB_DIR` (s) is used to point to this directory.
    """

    #
    #  libcocotbutils
    #
    libcocotbutils_sources = [
        os.path.join(share_lib_dir, "utils", "cocotb_utils.cpp")
    ]
    if os.name == "nt":
        libcocotbutils_sources += ["libcocotbutils.rc"]
    libcocotbutils = Extension(
        os.path.join("cocotb", "libs", "libcocotbutils"),
        define_macros=[("COCOTBUTILS_EXPORTS", "")] + _extra_defines,
        include_dirs=[include_dir],
        libraries=["gpilog"],
        sources=libcocotbutils_sources,
        extra_link_args=_extra_link_args(lib_name="libcocotbutils", rpaths=["$ORIGIN"]),
        extra_compile_args=_extra_cxx_compile_args,
    )

    #
    #  libgpilog
    #
    python_lib_dirs = []
    if sys.platform == "darwin":
        python_lib_dirs = [sysconfig.get_config_var("LIBDIR")]

    libgpilog_sources = [
        os.path.join(share_lib_dir, "gpi_log", "gpi_logging.cpp")
    ]
    if os.name == "nt":
        libgpilog_sources += ["libgpilog.rc"]
    libgpilog = Extension(
        os.path.join("cocotb", "libs", "libgpilog"),
        define_macros=[("GPILOG_EXPORTS", "")]  + _extra_defines,
        include_dirs=[include_dir],
        library_dirs=python_lib_dirs,
        sources=libgpilog_sources,
        extra_link_args=_extra_link_args(lib_name="libgpilog", rpaths=["$ORIGIN"]),
        extra_compile_args=_extra_cxx_compile_args,
    )

    #
    #  libcocotb
    #
    libcocotb_sources = [
        os.path.join(share_lib_dir, "embed", "gpi_embed.cpp")
    ]
    if os.name == "nt":
        libcocotb_sources += ["libcocotb.rc"]
    libcocotb = Extension(
        os.path.join("cocotb", "libs", "libcocotb"),
        define_macros=[("COCOTB_EMBED_EXPORTS", ""), ("PYTHON_SO_LIB", _get_python_lib())] + _extra_defines,
        include_dirs=[include_dir],
        libraries=[_get_python_lib_link(), "gpilog", "cocotbutils"],
        library_dirs=python_lib_dirs,
        sources=libcocotb_sources,
        extra_link_args=_extra_link_args(lib_name="libcocotb", rpaths=["$ORIGIN"] + python_lib_dirs),
        extra_compile_args=_extra_cxx_compile_args,
    )

    #
    #  libgpi
    #
    libgpi_sources=[
        os.path.join(share_lib_dir, "gpi", "GpiCbHdl.cpp"),
        os.path.join(share_lib_dir, "gpi", "GpiCommon.cpp"),
    ]
    if os.name == "nt":
        libgpi_sources += ["libgpi.rc"]
    libgpi = Extension(
        os.path.join("cocotb", "libs", "libgpi"),
        define_macros=[("GPI_EXPORTS", ""), ("LIB_EXT", _get_lib_ext_name()), ("SINGLETON_HANDLES", "")] + _extra_defines,
        include_dirs=[include_dir],
        libraries=["cocotbutils", "gpilog", "cocotb"],
        sources=libgpi_sources,
        extra_link_args=_extra_link_args(lib_name="libgpi", rpaths=["$ORIGIN"]),
        extra_compile_args=_extra_cxx_compile_args,
    )

    #
    #  simulator
    #
    simulator_sources=[
        os.path.join(share_lib_dir, "simulator", "simulatormodule.cpp"),
    ]
    if os.name == "nt":
        simulator_sources += ["simulator.rc"]
    libsim = Extension(
        os.path.join("cocotb", "simulator"),
        define_macros=_extra_defines,
        include_dirs=[include_dir],
        libraries=["cocotbutils", "gpilog", "gpi"],
        library_dirs=python_lib_dirs,
        sources=simulator_sources,
        extra_compile_args=_extra_cxx_compile_args,
        extra_link_args=_extra_link_args(rpaths=["$ORIGIN/libs"]),
    )

    # The libraries in this list are compiled in order of their appearance.
    # If there is a linking dependency on one library to another,
    # the linked library must be built first.
    return [libgpilog, libcocotbutils, libcocotb, libgpi, libsim]

def _get_vpi_lib_ext(
    include_dir, share_lib_dir, sim_define, extra_lib=[], extra_lib_dir=[]
):
    lib_name = "libcocotbvpi_" + sim_define.lower()
    libcocotbvpi_sources = [
        os.path.join(share_lib_dir, "vpi", "VpiImpl.cpp"),
        os.path.join(share_lib_dir, "vpi", "VpiCbHdl.cpp"),
    ]
    if os.name == "nt":
        libcocotbvpi_sources += [lib_name + ".rc"]
    libcocotbvpi = Extension(
        os.path.join("cocotb", "libs", lib_name),
        define_macros=[("COCOTBVPI_EXPORTS", ""), ("VPI_CHECKING", "1")] + [(sim_define, "")] + _extra_defines,
        include_dirs=[include_dir],
        libraries=["gpi", "gpilog"] + extra_lib,
        library_dirs=extra_lib_dir,
        sources=libcocotbvpi_sources,
        extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
        extra_compile_args=_extra_cxx_compile_args,
    )

    return libcocotbvpi

def _get_vhpi_lib_ext(
    include_dir, share_lib_dir, sim_define, extra_lib=[], extra_lib_dir=[]
):
    lib_name = "libcocotbvhpi_" + sim_define.lower()
    libcocotbvhpi_sources = [
        os.path.join(share_lib_dir, "vhpi", "VhpiImpl.cpp"),
        os.path.join(share_lib_dir, "vhpi", "VhpiCbHdl.cpp"),
    ]
    if os.name == "nt":
        libcocotbvhpi_sources += [lib_name + ".rc"]
    libcocotbvhpi = Extension(
        os.path.join("cocotb", "libs", lib_name),
        include_dirs=[include_dir],
        define_macros=[("COCOTBVHPI_EXPORTS", ""), ("VHPI_CHECKING", 1)] + [(sim_define, "")] + _extra_defines,
        libraries=["gpi", "gpilog"] + extra_lib,
        library_dirs=extra_lib_dir,
        sources=libcocotbvhpi_sources,
        extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
        extra_compile_args=_extra_cxx_compile_args,
    )

    return libcocotbvhpi

def get_ext():

    cfg_vars = distutils.sysconfig.get_config_vars()

    if sys.platform == "darwin":
        cfg_vars["LDSHARED"] = cfg_vars["LDSHARED"].replace("-bundle", "-dynamiclib")

    share_lib_dir = os.path.relpath(os.path.join(cocotb_share_dir, "lib"))
    include_dir = os.path.relpath(os.path.join(cocotb_share_dir, "include"))
    share_def_dir = os.path.relpath(os.path.join(cocotb_share_dir, "def"))

    ext = []

    logger.info("Compiling interface libraries for cocotb ...")

    ext += _get_common_lib_ext(include_dir, share_lib_dir)

    #
    #  Icarus Verilog
    #
    icarus_extra_lib = []
    icarus_extra_lib_path = []
    logger.info("Compiling libraries for Icarus Verilog")
    if os.name == "nt":
        icarus_extra_lib = ["icarus"]
        icarus_extra_lib_path = [share_def_dir]

    icarus_vpi_ext = _get_vpi_lib_ext(
        include_dir=include_dir,
        share_lib_dir=share_lib_dir,
        sim_define="ICARUS",
        extra_lib=icarus_extra_lib,
        extra_lib_dir=icarus_extra_lib_path,
    )
    ext.append(icarus_vpi_ext)

    #
    #  Modelsim/Questa
    #
    modelsim_extra_lib = []
    modelsim_extra_lib_path = []
    logger.info("Compiling libraries for Modelsim/Questa")
    if os.name == "nt":
        modelsim_extra_lib = ["modelsim"]
        modelsim_extra_lib_path = [share_def_dir]

    modelsim_vpi_ext = _get_vpi_lib_ext(
        include_dir=include_dir,
        share_lib_dir=share_lib_dir,
        sim_define="MODELSIM",
        extra_lib=modelsim_extra_lib,
        extra_lib_dir=modelsim_extra_lib_path,
    )
    ext.append(modelsim_vpi_ext)

    vsim_path = find_executable("vdbg")
    if vsim_path is None:
        logger.warning(
            "Modelsim/Questa executable (vdbg) executable not found. No FLI interface will be available."
        )
    else:
        modelsim_dir = os.path.dirname(os.path.dirname(vsim_path))
        modelsim_include_dir = os.path.join(modelsim_dir, "include")
        mti_path = os.path.join(modelsim_include_dir, "mti.h")
        if os.path.isfile(mti_path):
            lib_name = "libcocotbfli_modelsim"
            fli_sources = [
                os.path.join(share_lib_dir, "fli", "FliImpl.cpp"),
                os.path.join(share_lib_dir, "fli", "FliCbHdl.cpp"),
                os.path.join(share_lib_dir, "fli", "FliObjHdl.cpp"),
            ]
            if os.name == "nt":
                fli_sources += [lib_name + ".rc"]
            fli_ext = Extension(
                os.path.join("cocotb", "libs", lib_name),
                define_macros=[("COCOTBFLI_EXPORTS", "")] + _extra_defines,
                include_dirs=[include_dir, modelsim_include_dir],
                libraries=["gpi", "gpilog"] + modelsim_extra_lib,
                library_dirs=modelsim_extra_lib_path,
                sources=fli_sources,
                extra_link_args=_extra_link_args(lib_name=lib_name, rpaths=["$ORIGIN"]),
                extra_compile_args=_extra_cxx_compile_args,
            )

            ext.append(fli_ext)

        else:
            logger.warning(
                "Cannot build FLI interface for Modelsim/Questa - "
                "the mti.h header for '{}' was not found at '{}'.".format(
                    vsim_path, mti_path
                )
            )  # some Modelsim version does not include FLI.

    #
    # GHDL
    #
    ghdl_extra_lib = []
    ghdl_extra_lib_path = []
    logger.info("Compiling libraries for GHDL")
    if os.name == "nt":
        ghdl_extra_lib = ["ghdl"]
        ghdl_extra_lib_path = [share_def_dir]

    ghdl_vpi_ext = _get_vpi_lib_ext(
        include_dir=include_dir,
        share_lib_dir=share_lib_dir,
        sim_define="GHDL",
        extra_lib=ghdl_extra_lib,
        extra_lib_dir=ghdl_extra_lib_path,
    )
    ext.append(ghdl_vpi_ext)

    #
    # IUS
    #
    if os.name == "posix":
        logger.info("Compiling libraries for Incisive/Xcelium")
        ius_vpi_ext = _get_vpi_lib_ext(
            include_dir=include_dir, share_lib_dir=share_lib_dir, sim_define="IUS"
        )
        ext.append(ius_vpi_ext)

        ius_vhpi_ext = _get_vhpi_lib_ext(
            include_dir=include_dir, share_lib_dir=share_lib_dir, sim_define="IUS"
        )
        ext.append(ius_vhpi_ext)

    #
    # VCS
    #
    if os.name == "posix":
        logger.info("Compiling libraries for VCS")
        vcs_vpi_ext = _get_vpi_lib_ext(
            include_dir=include_dir, share_lib_dir=share_lib_dir, sim_define="VCS"
        )
        ext.append(vcs_vpi_ext)

    #
    # Aldec Riviera Pro
    #
    aldec_extra_lib = []
    aldec_extra_lib_path = []
    logger.info("Compiling libraries for Riviera")
    if os.name == "nt":
        aldec_extra_lib = ["aldec"]
        aldec_extra_lib_path = [share_def_dir]

    aldec_vpi_ext = _get_vpi_lib_ext(
        include_dir=include_dir,
        share_lib_dir=share_lib_dir,
        sim_define="ALDEC",
        extra_lib=aldec_extra_lib,
        extra_lib_dir=aldec_extra_lib_path,
    )
    ext.append(aldec_vpi_ext)

    aldec_vhpi_ext = _get_vhpi_lib_ext(
        include_dir=include_dir,
        share_lib_dir=share_lib_dir,
        sim_define="ALDEC",
        extra_lib=aldec_extra_lib,
        extra_lib_dir=aldec_extra_lib_path,
    )
    ext.append(aldec_vhpi_ext)

    #
    # Verilator
    #
    if os.name == "posix":
        logger.info("Compiling libraries for Verilator")
        verilator_vpi_ext = _get_vpi_lib_ext(
            include_dir=include_dir, share_lib_dir=share_lib_dir, sim_define="VERILATOR"
        )
        ext.append(verilator_vpi_ext)

    return ext


import numpy as np
from dash import Dash, html, dcc, Input, Output, dash_table, callback
import dash_bootstrap_components as dbc
import Data_countries as Data

# csv-files names and an index column
csv_names = ['missing_values', 'continent', 'annual-number-of-deaths-by-cause']
index = ['iso_code', 'iso_code', 'Code']

# define n_components of PCA, n_clusters of KMeans and AgglomerativeClustering
# a number of nearest neighbors and an effective minimum distance
# between embedded points in UMAP
n, nKM, nAC, nUmap, dist_Umap = 0.9, 3, 3, 30, 0.1

# create an object, load and clean a data
df = Data.Visualization(csv_names, index, n, nKM, nAC, nUmap, dist_Umap)
df_cl, df_labels = Data.load_prepr(df)
# data preprocessing, train models
Data.train_model(df)

# create a list with columns' names
fig_names = df_cl.columns.tolist()

# reassign clusters' numbers in order to visualize them in a similar way
df_labels['ClusterKM'] = np.where(
    df_labels['ClusterKM'] == 1, 0, (np.where(
        df_labels['ClusterKM'] == 2, 2, 1)
        )
    )

# create a layout
layout = html.Div(
    children=[
        html.H1( # title of a layout
            children=
                'Cluster Analysis of Countries by Death Causes and '+\
                    'Socioeconomic Indicators',
        style={'text-align':'center'}),
        dcc.Graph( # call a PCA plot
            id='pca_',
            figure=df.exp_pca(),
            style={'display' : 'inline-block', 'width':'50%'}
            ),
        dcc.Graph( # call a 'silhouette score' plot
            id='silhouette_score_',
            figure=df.silhouette(),
            style={'display' : 'inline-block', 'width':'50%'}
            ),
        html.Br(), # make a space
        html.Br(), # make a space
        dcc.RadioItems( # create an option button
            id='clusters_',
            options = [
                {'label': 'KMeans', 'value': 'ClusterKM'},
                {'label': 'AgglomerativeClustering', 'value': 'ClusterAC'}
                ],
            value='ClusterKM',
            style={'text-align': 'center'},
            ),

    # add a container for a plot and a table
    html.Div([
        html.Div([

            html.Div(id='fig_plot_')

        ], style={'width': '65%', 'display': 'inline-block'}),

        # table container
        html.Div([
            dbc.Label('Difference between the Clustering Algorithms'),

            dash_table.DataTable(
                df.compair_cl().to_dict('records'),
                [{"name": i, "id": i} for i in df.compair_cl().columns],
                style_data={ 'border': '1px solid blue' },
                style_header={'border': '2px solid pink',
                               'fontWeight': 'bold'},
                style_cell={'textAlign': 'left'},
                style_cell_conditional=[
                                     {
                                    'if': {'column_id': c},
                                    'textAlign': 'center'
                                     } for c in ['ClusterKM', 'ClusterAC']
                                 ],
                                 )

        ], style={
            'width': '30%',
            'display': 'inline-block',
            'marginTop' : '100px'},
            ),

    ], style={'display': 'flex'}),
    #https://stackoverflow.com/questions/69874454/plotly-dash-cannot-place-figure-and-table-side-by-side

    dcc.Dropdown( # cteate a dropdown menu
        id='fig_dropdown_',
        options=[{'label': x, 'value': x} for x in fig_names],
        value=fig_names[0],
        style={'text-align': 'center', 'width':'50%', 'display' : 'inline-block'}
        # use the first feature as the initial selection
    ),
    dcc.RadioItems( # create an option button
        id='graph_',
        options=[
            {'label': 'by Death Causes', 'value': 1},
            {'label': 'by Socioeconomic Indicators', 'value': 0}
            ],
        value=1,
        style={
            'text-align': 'center',
            'marginLeft' : '100px',
            'marginBottom' : '30px',
            'display' : 'inline-block'},
            ),
    html.Br(), # make a space
    html.Div( # create a container for a plot
        id='fig_plot_1_',
        style={
            'display' : 'inline-block', 'width':'50%'
            }
        ),
    html.Div( # create a container for a plot
        id='fig_plot_2_',
        style={
            'display' : 'inline-block',
            'width':'50%'
            }
        )
    ])

# define a callback for updating the plots
# based on the dropdown selection and the option buttons
@callback(
    [Output('fig_plot_', 'children'),
    Output('fig_plot_1_', 'children'),
    Output('fig_plot_2_', 'children')],
    [Input('fig_dropdown_', 'value'),
    Input('clusters_', 'value'),
    Input('graph_', 'value')]
    )

def update_figure(fig_name: str, cluster: str, graph_sel: int):
    """ call the plots
    Params: fig_name (str), cluster (str), graph_sel (int)
    Return: fig_treemap, fig_2, fig_m
    """
    fig_treemap = df.treemap(cluster)
    fig = df.umap_plot(fig_name, cluster)
    fig_m = df.cl_analys(graph_sel, cluster)

    return(dcc.Graph(figure=fig_treemap),
            dcc.Graph(figure=fig),
            dcc.Graph(figure=fig_m))


import time
import math
import sqlite3
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import re

#Define functions used
def PageNav_Main (EntryNum):
	#Page up / down until it is posible to access page containing entry with nav panel
	PageSource=str(driver.page_source)
	Min_NavButtons,Max_NavButtons=PageNav_GetNavOptions(PageSource)
	iteration=0
	while iteration<=10000 and not (EntryNum>=Min_NavButtons and EntryNum<=Max_NavButtons):
		if EntryNum>Max_NavButtons:
			Containstxt=str(Max_NavButtons-8)+' - '+str(Max_NavButtons+1)
			driver.find_element_by_xpath('//*[@id="_bld_result_frm:_result_tbl_paginator_bottom"]/span[4]/span[10]').click()
			WebDriverWait(driver, 30).until(EC.text_to_be_present_in_element((By.ID, '_bld_result_frm:_result_tbl_paginator_bottom'), Containstxt))
			iteration+=1
		elif EntryNum<Min_NavButtons:
			Containstxt=str(Min_NavButtons+1)+' - '+str(Min_NavButtons+10)
			driver.find_element_by_xpath('//*[@id="_bld_result_frm:_result_tbl_paginator_bottom"]/span[4]/span[1]').click()
			WebDriverWait(driver, 30).until(EC.text_to_be_present_in_element((By.ID, '_bld_result_frm:_result_tbl_paginator_bottom'), Containstxt))
			iteration+=1
		else:
			print('Error')
		PageSource=str(driver.page_source)
		Min_NavButtons,Max_NavButtons=PageNav_GetNavOptions(PageSource)

	# Determine which button should be pressed to access correct page and click that button
	Min_TableEntries,Max_TableEntries=PageNav_FindEntries(PageSource)

	if Min_TableEntries<=EntryNum and Max_TableEntries>=EntryNum:
		pass
	else:
		target_page=math.trunc(EntryNum/10)+1
		nav_target=target_page-(Min_NavButtons/10)
		Containstxt=str((target_page*10)-9)+' - '+str(target_page*10)
		target_page_xpath='//*[@id="_bld_result_frm:_result_tbl_paginator_bottom"]/span[4]/span['+str(nav_target)+']'
		driver.find_element_by_xpath(target_page_xpath).click()
		WebDriverWait(driver, 30).until(EC.text_to_be_present_in_element((By.ID, '_bld_result_frm:_result_tbl_paginator_bottom'), Containstxt))

def PageNav_GetNavOptions(SourceStr):
	#Obtain entries the can be accessed on the nav panel
	NavButtons = re.findall(r'([0-9]*)',SourceStr)
	NavButtons= [int(i) for i in NavButtons]
	Min_NavButtons=(min(NavButtons)-1)*10
	Max_NavButtons=(max(NavButtons)*10)-1
	return (Min_NavButtons,Max_NavButtons)

def PageNav_FindEntries(SourceStr):
	#Obtain entries currently on the page
	TableEntries = re.findall(r'id="_bld_result_frm:_result_tbl:([0-9]*):j_id_4k',SourceStr)
	TableEntries= [int(i) for i in TableEntries]
	Min_TableEntries=min(TableEntries)
	Max_TableEntries=max(TableEntries)
	return (Min_TableEntries,Max_TableEntries)

def Selenium_click(Click_ID,Wait_ID,Wait_Str):
	WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, Click_ID)))
	driver.find_element_by_id(Click_ID).click()
	#WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element((By.ID, Wait_ID), Wait_Str))

	try:
		WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element((By.ID, Wait_ID), Wait_Str))
	except:
		WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element((By.ID, Wait_ID), Wait_Str))

def AddressClick(Num):
	#Checks that the link is active and then click if it is
	id='_bld_result_frm:_result_tbl:'+str(Num)+':j_id_4k'
	try:
		Selenium_click(id,'_detail_form:j_id_2s','Name of Building')
	except:
		Selenium_click('j_id_n:j_id_1k','_detail_form:j_id_2s','大厦名称')
		Selenium_click('j_id_n:j_id_1c','_detail_form:j_id_2s','Name of Building')
def Return_List():
	#Checks that the link is active and then click if it is
	try:
		Selenium_click('j_id_n:j_id_1c','_detail_form:j_id_2s','Name of Building')
	except:
		Selenium_click('j_id_n:j_id_1k','_detail_form:j_id_2s','大厦名称')
		Selenium_click('j_id_n:j_id_1c','_detail_form:j_id_2s','Name of Building')

	Selenium_click('_detail_form:j_id_3w','_bld_result_frm:pnlBldSearchResult_header','Building Search Result')

def DB_Insert(add_matrix,build_list):
	c.executemany('INSERT INTO Address VALUES (?,?,?,?)',add_matrix)
	conn.commit()
	c.execute('INSERT INTO Building VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)',build_list)
	conn.commit()

def Scrape_Entry(Num):
	#Scrape entry in all three languages

	Details =[]
	Details.append(Num)
	Details .extend(re.findall(r'
([^\n]*)\n',str(driver.page_source))[:3])
	Address1 = re.findall(r'([^<]*)	',str(driver.page_source))[:5]

	try:
		Selenium_click('j_id_n:j_id_1g','_detail_form:j_id_2s','大廈名稱')
	except:
		Selenium_click('j_id_n:j_id_1c','_detail_form:j_id_2s','Name of Building')
		Selenium_click('j_id_n:j_id_1g','_detail_form:j_id_2s','大廈名稱')

	Details.extend(re.findall(r'
([^\n]*)\n',str(driver.page_source))[:3])
	Address2=re.findall(r'([^<]*)	',str(driver.page_source))[:5]

	try:
		Selenium_click('j_id_n:j_id_1k','_detail_form:j_id_2s','大厦名称')
	except:
		Selenium_click('j_id_n:j_id_1c','_detail_form:j_id_2s','Name of Building')
		Selenium_click('j_id_n:j_id_1k','_detail_form:j_id_2s','大厦名称')

	Details.extend(re.findall(r'
([^\n]*)\n',str(driver.page_source))[:7])
	Address3=re.findall(r'([^<]*)	',str(driver.page_source))[:5]

	counter=[]
	for x in Address1:
		counter.append(Num)
	add_matrix=list(zip(counter,Address1,Address2,Address3))
	DB_Insert(add_matrix,Details)

def Scrape_Main (Start, End):
	counter=Start
	while counter>=Start and counter<=End:
		PageNav_Main (counter)
		try:
			AddressClick(counter)
			Scrape_Entry(counter)
		except:
			print('Missed: '+str(counter))
		try:
			Return_List()
		except:
			print('Ret Failed: '+str(counter))
		counter+=1

def Scrape_Main_Single (counter):
	PageNav_Main (counter)
	try:
		AddressClick(counter)
		Scrape_Entry(counter)
	except:
		print('Missed: '+str(counter))
	try:
		Return_List()
	except:
		print('Ret Failed: '+str(counter))

#Load Firefox driver and go to the web database
driver = webdriver.Firefox()
driver.implicitly_wait(10)

#Connect to local SQLite DB
path='BuildingsDB.db'
conn=sqlite3.connect(path)
c=conn.cursor()

#Go to website
driver.get('https://bmis1.buildingmgt.gov.hk/bd_hadbiex/content/searchbuilding/building_search.jsf?renderedValue=true')
WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element((By.ID, '_bld_result_frm:pnlBldSearchResult_header'), 'Building Search Result'))

#Scrape loop
Scrape_Main (34802,44000)

#Close connection
c.close()
conn.close()


import numpy as np
import os
from astropy.io import fits

from specklepy.logging import logger
from specklepy.exceptions import SpecklepyTypeError
from specklepy.utils.time import default_time_stamp

class Outfile(object):

    def __init__(self, filename, path=None, data=None, shape=None, extensions=None, header=None, cards=None,
                 initialize=True, timestamp=False, header_card_prefix=None, verbose=True):
        """Instantiate a generic outfile.

        Args:
            filename (str):
                Name of the file that is always referred to in the background.
            path (str, optional):
                Target path under which the new file will be stored.
            data (np.ndarray, optional):
                If provided, the file will be initialized with these data and shape argument is ignored. Default is
                None.
            shape (tuple, dtype=int, optional):
                If provided, the file will be initialized with an empty array of this shape. This argument will be
                ignored if data argument is provided. Default is None.
            extensions (list of dict, optional):
                List of dictionaries of names (and data or shapes) for each extension that shall be initialized right
                away. Could look like this:
                [{'name': 'VAR', 'data': array},
                 {'name': 'SUB', 'shape': (1024, 1024)}]
                Default is None.
            header (fits.header, optional):
                Header that will be used to initialize the new Primary HDU. Default is None.
            cards (dict, optional):
                Dictionary of cards that will be added to the fits header. Default is None.
            initialize (bool, optional):
                Set to `False` to suppress creating a new file.
            timestamp (bool, optional):
                Set to True to automatically add a time stamp to the file name. Default is False.
            header_card_prefix (str, optional):
                Prefix of header cards. Default is None.
            verbose (bool, optional):
                Set to False to suppress logger output. Default is True.
        """

        # Input parameters
        if filename is None:
            raise RuntimeError("Outfile did not receive a filename!")
        else:
            self.filename = filename

        if path is None or isinstance(path, str):
            self.path = path
        else:
            raise SpecklepyTypeError('Outfile', argname='path', argtype=type(path), expected='str')

        if data is None or isinstance(data, np.ndarray):
            pass
        else:
            raise SpecklepyTypeError('Outfile', 'data', type(data), 'np.ndarray')

        if shape is None or isinstance(shape, tuple):
            self.shape = shape
        else:
            raise SpecklepyTypeError('Outfile', 'shape', type(shape), 'tuple')

        if shape is not None and data is not None:
            logger.info("Outfile instance ignores shape input as data is provided for initialization!")

        if isinstance(extensions, dict):
            self.extensions = [extensions]
        elif extensions is None or isinstance(extensions, list):
            self.extensions = extensions
        else:
            raise SpecklepyTypeError('Outfile', 'extensions', type(extensions), 'list of dict')

        if cards is None:
            self.cards = {}
        elif isinstance(cards, dict):
            self.cards = cards
        else:
            raise SpecklepyTypeError('Outfile', 'cards', type(cards), 'dict')

        if header is None or isinstance(header, fits.header.Header):
            pass
        else:
            raise SpecklepyTypeError('Outfile', 'header', type(header), 'fits.header')

        if not isinstance(timestamp, bool):
            raise SpecklepyTypeError('Outfile', 'timestamp', type(timestamp), 'bool')
        else:
            if timestamp:
                self.filename = filename.replace('.fits', '_{}.fits'.format(default_time_stamp()))

        if header_card_prefix is None:
            self.header_card_prefix = ""
        elif isinstance(header_card_prefix, str):
            # Assert that there are gaps between prefix and card keywords
            if header_card_prefix[-1] != ' ':
                header_card_prefix = header_card_prefix + ' '
            self.header_card_prefix = header_card_prefix
        else:
            raise SpecklepyTypeError('Outfile', 'header_card_prefix', type(header_card_prefix), 'str')

        if isinstance(verbose, bool):
            self.verbose = verbose
        else:
            raise SpecklepyTypeError('Outfile', 'verbose', type(verbose), 'bool')

        if initialize:
            self.initialize_file()

    def initialize_file(self, header=None, data=None):

        # Initialize primary HDU
        hdu = fits.PrimaryHDU(header=header)
        for key in self.cards:
            hdu.header.set(self.header_card_prefix + key, self.cards[key])
        if self.shape is not None and data is None:
            hdu.data = np.zeros(self.shape)
        if data is not None:
            hdu.data = data
        if 'DATE' not in hdu.header.keys():
            hdu.header.set('DATE', default_time_stamp())

        # Create a HDU list with the primary and write to file
        hdu_list = fits.HDUList([hdu])
        if self.verbose:
            logger.info(f"Initializing file {self.file_path!r}")
        hdu_list.writeto(self.file_path, overwrite=True)

        if self.extensions is not None:
            for extension in self.extensions:
                if 'name' not in extension:
                    raise ValueError(f"Outfile initialization with extensions failed, because extension did not "
                                     f"contain a name!")
                else:
                    name = extension['name']
                if 'data' in extension:
                    data = extension['data']
                elif 'shape' in extension:
                    data = np.empty(extension['shape'])
                else:
                    data = None
                if 'header' in extension:
                    header = extension['header']
                else:
                    header = None
                self.new_extension(name=name, data=data, header=header)

    @classmethod
    def from_file(cls, filename, path=None):
        return cls(filename=filename, path=path, initialize=False)

    @property
    def file_path(self):
        if self.path is None:
            return self.filename
        else:
            return os.path.join(self.path, self.filename)

    @property
    def data(self):
        return fits.getdata(self.file_path)

    @data.setter
    def data(self, data):
        with fits.open(self.file_path, mode='update') as hdu_list:
            hdu_list[0].data = data
            hdu_list[0].header.set('UPDATED', default_time_stamp())
            hdu_list.flush()
        if self.verbose:
            logger.info(f"Updating data in {self.file_path!r}")

    def __getitem__(self, extension):
        return fits.getdata(self.file_path, extension)  # [index]

    def __setitem__(self, extension, data):
        with fits.open(self.file_path, mode='update') as hdu_list:
            hdu_list[extension].data = data
            hdu_list[extension].header.set('UPDATED', default_time_stamp())
            hdu_list.flush()

    def update_frame(self, frame_index, data, extension=None):
        if extension is None:
            extension = 0
        with fits.open(self.file_path, mode='update') as hdu_list:
            hdu_list[extension].data[frame_index] = data
            hdu_list[extension].header.set('UPDATED', default_time_stamp())
            hdu_list.flush()
        # self.data[frame_index] = data

    def new_extension(self, name, data=None, header=None, index=None):
        """Create a new fits extension.

        All arguments are passed to the new HDU in the HDUlist of instances file, i.e. 'self.filename'.

        Args:
            name (str):
                Name of the new extension/ HDU.
            data (array, optional):
                Data array, passed to the new HDU.
            header (fits.header, optional):
                Fits header, passed to the new HDU.
            index (int, optional):
                Index of the extension, before which the new hdu is inserted. None translates into appending the new
                HDU at the end of the HDUlist.
        """

        with fits.open(self.file_path, mode='update') as hdu_list:
            hdu = fits.ImageHDU(data=data, name=name, header=header)
            if index is None:
                hdu_list.append(hdu=hdu)
            else:
                hdu_list.insert(index=index, hdu=hdu)
            hdu_list.flush()

    def update_extension(self, ext_name, data):
        if self.verbose:
            logger.info(f"Updating data in {self.file_path!r}[{ext_name}]")
        with fits.open(self.file_path, mode='update') as hdu_list:
            hdu_list[ext_name].data = data
            hdu_list.flush()

    def has_extension(self, ext_name):
        with fits.open(self.file_path) as hdu_list:
            return ext_name in hdu_list

    @staticmethod
    def extract_frame_shape(header):
        number_axes = header.get('NAXIS')
        if number_axes == 2 or number_axes == 3:
            frame_shape = (header.get('NAXIS1'), header.get('NAXIS2'))
        else:
            raise NotImplementedError(f"Frame shape extraction is not defined for FITS cubes with {number_axes!r} "
                                      f"axes!")
        return frame_shape

    @staticmethod
    def extract_frame_number(header):
        number_axes = header.get('NAXIS')
        if number_axes == 2:
            number_frames = 1
        elif number_axes == 3:
            number_frames = header.get('NAXIS3')
        else:
            raise NotImplementedError(f"Frame number extraction is not defined for FITS cubes with {number_axes!r} "
                                      f"axes!")
        return number_frames