# ============================================================================
# Physics-informed Neural Network functions using PyTorch
# Goal : Predict the kinetic constants of a microwave-assisted biodiesel process.
# Author : Valérie Bibeau, Polytechnique Montréal, 2023
# ============================================================================

# ---------------------------------------------------------------------------
# Libraries importation
import torch
import torch.autograd as autograd
import torch.nn as nn
import numpy as np
# ---------------------------------------------------------------------------

# Set seed
torch.set_default_dtype(torch.float)
torch.manual_seed(1234)
np.random.seed(1234)

# PINN architecture
class PINeuralNet(nn.Module):

    def __init__(self, device, E, A, neurons):
        """Constructor

        Args:
            device (string): CPU or CUDA
            E (list): Initial estimate of k''
            A (list): Initial estimate of k'
            neurons (int): Number of neurons in hidden layers
        """

        super().__init__()

        # Architecture
        self.activation = nn.Tanh()

        self.f1 = nn.Linear(2, neurons)
        self.f2 = nn.Linear(neurons, neurons)
        self.f3 = nn.Linear(neurons, neurons)
        self.out = nn.Linear(neurons, 6)

        # Parameters set up
        # Molar balances
        self.E1 = torch.tensor(E[0], requires_grad=True).float().to(device)
        self.E2 = torch.tensor(E[1], requires_grad=True).float().to(device)
        self.E3 = torch.tensor(E[2], requires_grad=True).float().to(device)
        self.E4 = torch.tensor(E[3], requires_grad=True).float().to(device)
        self.E5 = torch.tensor(E[4], requires_grad=True).float().to(device)
        self.E6 = torch.tensor(E[5], requires_grad=True).float().to(device)

        self.E1 = nn.Parameter(self.E1)
        self.E2 = nn.Parameter(self.E2)
        self.E3 = nn.Parameter(self.E3)
        self.E4 = nn.Parameter(self.E4)
        self.E5 = nn.Parameter(self.E5)
        self.E6 = nn.Parameter(self.E6)

        self.A1 = torch.tensor(A[0], requires_grad=True).float().to(device)
        self.A2 = torch.tensor(A[1], requires_grad=True).float().to(device)
        self.A3 = torch.tensor(A[2], requires_grad=True).float().to(device)
        self.A4 = torch.tensor(A[3], requires_grad=True).float().to(device)
        self.A5 = torch.tensor(A[4], requires_grad=True).float().to(device)
        self.A6 = torch.tensor(A[5], requires_grad=True).float().to(device)

        self.A1 = nn.Parameter(self.A1)
        self.A2 = nn.Parameter(self.A2)
        self.A3 = nn.Parameter(self.A3)
        self.A4 = nn.Parameter(self.A4)
        self.A5 = nn.Parameter(self.A5)
        self.A6 = nn.Parameter(self.A6)

        # Energy balance
        self.e = torch.tensor(0.5, requires_grad=True).float().to(device)
        self.c1 = torch.tensor(-0.1, requires_grad=True).float().to(device)
        self.c2 = torch.tensor(-0.1, requires_grad=True).float().to(device)

        self.e = nn.Parameter(self.e)
        self.c1 = nn.Parameter(self.c1)
        self.c2 = nn.Parameter(self.c2)

    def forward(self, x):
        """Forward pass

        Args:
            x (tensor): Input tensor

        Returns:
            tensor: Output tensor
        """
        if torch.is_tensor(x) != True:
            x = torch.from_numpy(x)

        a = x.float()

        z_1 = self.f1(a)
        a_1 = self.activation(z_1)
        z_2 = self.f2(a_1)
        a_2 = self.activation(z_2)
        z_3 = self.f3(a_2)
        a_3 = self.activation(z_3)

        a_4 = self.out(a_3)

        return a_4

# Full PINN to discover k
class Curiosity():

    def __init__(self, X, Y, Z, idx, idx_y0, idx_T, f_hat, learning_rate, E, A, neurons, regularization, penalization, device, prm):
        """Constructor

        Args:
            X (tensor): Input tensor
            Y (tensor): Output tensor (concentrations)
            Z (tensor): Temperature output tensor
            idx (list): Index of concentration data points
            idx_y0 (list): Index of IC data points
            idx_T (list): Index of temperature data points
            f_hat (tensor): Null tensor (for residuals)
            learning_rate (float): Learning rate for gradient descent
            E (list): Initial estimates of k''
            A (list): Initial estimates of k'
            neurons (int): Number of neurons in hidden layers
            regularization (float): Regularization parameters on MSE of molar balances
            penalization (float): Regularization parameter on MSE of experimental data
            device (string): CPU or CUDA
            prm (struct): Extra paramaters
        """

        def loss_function_ode(output, target):

            loss = torch.mean((output - target)**2)

            return loss

        def loss_function_data(output, target):

            loss = torch.mean((output[idx] - target[idx])**2)

            return loss

        def loss_function_IC(output, target):

            loss = torch.mean((output[idx_y0] - target[idx_y0])**2)

            return loss

        def loss_function_T(output, target):

            loss = torch.mean((output[idx_T] - target[idx_T])**2)

            return loss

        self.PINN = PINeuralNet(device, E, A, neurons).to(device)

        self.PINN.register_parameter('E1', self.PINN.E1)
        self.PINN.register_parameter('E2', self.PINN.E2)
        self.PINN.register_parameter('E3', self.PINN.E3)
        self.PINN.register_parameter('E4', self.PINN.E4)
        self.PINN.register_parameter('E5', self.PINN.E5)
        self.PINN.register_parameter('E6', self.PINN.E6)

        self.PINN.register_parameter('A1', self.PINN.A1)
        self.PINN.register_parameter('A2', self.PINN.A2)
        self.PINN.register_parameter('A3', self.PINN.A3)
        self.PINN.register_parameter('A4', self.PINN.A4)
        self.PINN.register_parameter('A5', self.PINN.A5)
        self.PINN.register_parameter('A6', self.PINN.A6)

        self.PINN.register_parameter('e', self.PINN.e)
        self.PINN.register_parameter('c1', self.PINN.c1)
        self.PINN.register_parameter('c2', self.PINN.c2)

        self.x = X
        self.y = Y
        self.z = Z

        self.loss_function_ode = loss_function_ode
        self.loss_function_data = loss_function_data
        self.loss_function_IC = loss_function_IC
        self.loss_function_T = loss_function_T
        self.f_hat = f_hat
        self.regularization = regularization
        self.penalization = penalization

        self.device = device
        self.lr = learning_rate
        self.params = list(self.PINN.parameters())
        self.optimizer = torch.optim.Adam(self.params, lr=self.lr)

        self.prm = prm

    def loss(self, x, y_train):
        """Loss function

        Args:
            x (tensor): Input tensor
            y_train (tensor): Output tensor (for training)

        Returns:
            float: Evaluation of the loss function
        """

        # Input
        g = x.clone()
        g.requires_grad = True

        # Output
        y = self.PINN(g)
        cTG = y[:,0].reshape(-1,1)
        cDG = y[:,1].reshape(-1,1)
        cMG = y[:,2].reshape(-1,1)
        cG = y[:,3].reshape(-1,1)
        cME = y[:,4].reshape(-1,1)
        T = y[:,5].reshape(-1,1)

        # Microwave power
        Q = g[:,1].reshape(-1,1)

        # Kinetic constants (using Taylor's linearization of Arrhenius equation)
        k1 = self.PINN.A1 + self.PINN.E1 * (T - torch.mean(T))
        k2 = self.PINN.A2 + self.PINN.E2 * (T - torch.mean(T))
        k3 = self.PINN.A3 + self.PINN.E3 * (T - torch.mean(T))
        k4 = self.PINN.A4 + self.PINN.E4 * (T - torch.mean(T))
        k5 = self.PINN.A5 + self.PINN.E5 * (T - torch.mean(T))
        k6 = self.PINN.A6 + self.PINN.E6 * (T - torch.mean(T))

        # Kinetic constants (using Arrhenius equation)
        # k1 = self.PINN.A1 * torch.exp(-self.PINN.E1 / T)
        # k2 = self.PINN.A2 * torch.exp(-self.PINN.E2 / T)
        # k3 = self.PINN.A3 * torch.exp(-self.PINN.E3 / T)
        # k4 = self.PINN.A4 * torch.exp(-self.PINN.E4 / T)
        # k5 = self.PINN.A5 * torch.exp(-self.PINN.E5 / T)
        # k6 = self.PINN.A6 * torch.exp(-self.PINN.E6 / T)

        # Automatic differentiation
        grad_cTG = autograd.grad(cTG, g, torch.ones(x.shape[0], 1).to(self.device), \
                                 retain_graph=True, create_graph=True) \
                                 [0][:,0].reshape(-1,1)
        grad_cDG = autograd.grad(cDG, g, torch.ones(x.shape[0], 1).to(self.device), \
                                 retain_graph=True, create_graph=True) \
                                 [0][:,0].reshape(-1,1)
        grad_cMG = autograd.grad(cMG, g, torch.ones(x.shape[0], 1).to(self.device), \
                                 retain_graph=True, create_graph=True) \
                                 [0][:,0].reshape(-1,1)
        grad_cG = autograd.grad(cG, g, torch.ones(x.shape[0], 1).to(self.device), \
                                retain_graph=True, create_graph=True) \
                                [0][:,0].reshape(-1,1)
        grad_cME = autograd.grad(cME, g, torch.ones(x.shape[0], 1).to(self.device), \
                                 retain_graph=True, create_graph=True) \
                                 [0][:,0].reshape(-1,1)
        grad_T = autograd.grad(T, g, torch.ones(x.shape[0], 1).to(self.device), \
                               retain_graph=True, create_graph=True) \
                               [0][:,0].reshape(-1,1)

        # MSE of ODEs (molar balances)
        self.loss_cTG_ode = self.loss_function_ode(grad_cTG + k1*cTG - k2*cDG*cME, self.f_hat)
        self.loss_cDG_ode = self.loss_function_ode(grad_cDG - k1*cTG + k2*cDG*cME \
                                                            + k3*cDG - k4*cMG*cME, self.f_hat)
        self.loss_cMG_ode = self.loss_function_ode(grad_cMG - k3*cDG + k4*cMG*cME \
                                                            + k5*cMG - k6*cG*cME, self.f_hat)
        self.loss_cG_ode = self.loss_function_ode(grad_cG - k5*cMG + k6*cG*cME, self.f_hat)
        self.loss_cME_ode = self.loss_function_ode(grad_cME - k1*cTG + k2*cDG*cME \
                                                            - k3*cDG + k4*cMG*cME \
                                                            - k5*cMG + k6*cG*cME, self.f_hat)

        # MSE of ODE (energy balance)
        self.loss_T_ode = self.loss_function_ode(self.prm.m_Cp*grad_T - self.PINN.e*Q \
                                                 - self.PINN.c1*T - self.PINN.c2, self.f_hat)

        # MSE of data points (concentration & temperature)
        self.loss_cTG_data = self.loss_function_data(cTG, y_train[:,0].reshape(-1,1))
        self.loss_cDG_data = self.loss_function_data(cDG, y_train[:,1].reshape(-1,1))
        self.loss_cMG_data = self.loss_function_data(cMG, y_train[:,2].reshape(-1,1))
        self.loss_cG_data = self.loss_function_IC(cG, y_train[:,3].reshape(-1,1))
        self.loss_cME_data = self.loss_function_IC(cME, y_train[:,4].reshape(-1,1))
        self.loss_T_data = self.loss_function_T(T, self.z)

        # Total loss function
        self.loss_c_data = self.loss_cTG_data + self.loss_cDG_data + self.loss_cMG_data + self.loss_cG_data + self.loss_cME_data
        self.loss_c_ode = self.loss_cTG_ode + self.loss_cDG_ode + self.loss_cMG_ode + self.loss_cG_ode + self.loss_cME_ode

        self.total_loss = self.regularization * self.loss_c_ode + self.loss_T_ode + self.penalization*self.loss_c_data + self.loss_T_data

        return self.total_loss

    def closure(self):
        """Forward and backward pass

        Returns:
            float: Evaluation of the loss function
        """

        self.optimizer.zero_grad()

        loss = self.loss(self.x, self.y)

        loss.backward()

        return loss


# -*- coding: utf-8 -*-
from rest_framework.exceptions import NotAcceptable

class ProcessLimitSkipFilter:
    error_msgs = {
        'invalid_type': "Parameter for '{property}' filter should be <type 'int'>, got - {type}",
        'less_than_zero': "Parameter for '{property}' filter should be positive number or zero",
        'limit_is_zero': "Parameter for 'limit' filter should be greater than zero"
    }

    def __init__(self, queryset, _filter):
        self.limit = _filter.get('limit', None)
        self.skip = _filter.get('skip', None)
        self.queryset = queryset

    def filter_queryset(self):
        self.validate_value('limit', self.limit)
        self.validate_value('skip', self.skip)
        queryset = self.queryset

        if self.skip and self.queryset.count() < self.skip:
            return queryset.none()

        if self.skip is not None:
            queryset = queryset[self.skip:]
        if self.limit is not None:
            queryset = queryset[:self.limit]

        return queryset

    def validate_value(self, property, value):
        if value is not None:
            if not isinstance(value, int):
                raise NotAcceptable(self.error_msgs['invalid_type'].format(
                    property=property,
                    type=type(value)
                ))
            if value < 0:
                raise NotAcceptable(self.error_msgs['less_than_zero'].format(
                    property=property
                ))

        if property == 'limit' and value == 0:
            raise NotAcceptable(self.error_msgs['limit_is_zero'])


# -*- coding: utf-8 -*-

# Define here the code for scraping content

from scrapy import Spider
from stackSpider.items import StackQuestion, StackAnswer
import re

class StackOverflowSpider(Spider):
    name = "stack"
    allowed_domains = ['stackoverflow.com']
    start_urls = ['http://stackoverflow.com/questions/3044580/multiprocessing-vs-threading-python']

    def parse(self, response):
        question = StackQuestion()

        question['type'] = 'question'
        id = re.search(r'^http:\/\/stackoverflow\.com\/questions\/(\d*)\/.*$', response.url)
        if id:
            question['id'] = id.group(1)
        else:
            question['id'] = response.url
        question['title'] = response.xpath("//h1/a[@class='question-hyperlink']/text()").extract()[0]
        question['author'] = \
            response.xpath("//td[@class='post-signature owner']/div/div[@class='user-details']/a/text()").extract()[0]
        question['date'] = \
            response.xpath("//td[@class='post-signature owner']//span[@class='relativetime']/@title").extract()[0]
        question['body'] = response.xpath("//div[@class='post-text']").extract()[0]
        question['answers'] = response.xpath("//span[@itemprop='answerCount']/text()").extract()[0]

        yield question

        answers_count = int(question['answers'])
        for i in range(1, answers_count + 1):
            answer = StackAnswer()

            answer['type'] = 'answer'
            answer['id'] = str(i)
            answer['qid'] = question['id']
            answer['author'] = response.xpath("//td/div/div[@class='user-details']/a/text()").extract()[i]
            answer['date'] = response.xpath("//td//span[@class='relativetime']/@title").extract()[i]
            answer['body'] = response.xpath("//div[@class='post-text']").extract()[i]

            yield answer


from django.urls import path
from .views import NewList, NewDetail, NewSearch, NewCreate, NewDelete, NewUpdate, upgrade_me, check_subscribe,\
    check_unsubscribe

urlpatterns = [
    path('', NewList.as_view()),
    path('<int:pk>', NewDetail.as_view(), name='new_detail'),
    path('search/', NewSearch.as_view()),
    path('add/', NewCreate.as_view(), name='new_create'),
    path('<int:pk>/edit', NewUpdate.as_view(), name='new_update'),
    path('<int:pk>/delete', NewDelete.as_view(), name='new_delete'),
    path('upgrade/', upgrade_me, name='upgrade'),
    path('subscribe/<int:pk>', check_subscribe, name='subscribe'),
    path('unsubscribe/<int:pk>', check_unsubscribe, name='unsubscribe')
]


import os
import sys
import time
from PyQt6 import QtWidgets
from PySide6 import *
import PySide6 as PySide6
from PyQt6.QtGui import *
from PyQt6.QtGui import QFileSystemModel
from PyQt6.QtWidgets import *
from PyQt6.QtWidgets import QApplication, QWidget, QHBoxLayout, QVBoxLayout, QTreeView, QListWidget, QListWidgetItem
from PyQt6.QtCore import QModelIndex
from PyQt6.QtCore import Qt as QtCore
from PyQt6 import uic
from O_cs_rv3 import *
import O_cs_rv3 as csr

erstellenbool = False
bid_num = []
log_Path_img = [None]
log_user_gui = []
log_user_lusd = []
log_user_sqlite = []
image_shueler = []
kopie_schueler = []
submit_schueler = []
pathCSVFileStorage = [None]

class CSVUploadWindow(QWidget):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi('GUI\CSV_Upload.ui', self)
            self.pushButton.clicked.connect(self.csv_upload)
            self.pushButton_2.clicked.connect(self.browserWindowOpen)
            self.diaolog = DialogDone()
            self.diaError = DialogImpError()
            try:
                element = pathCSVFileStorage[-1]
                self.label_3.setText(element)
            except:
                self.label_3.setText("")
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (Class:CSVUploadWindow())\n")
                log.close()

    def browserWindowOpen(self):
        try:
            self.browser_window = BrowserWindow()
            self.browser_window.show()
            self.close()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (browserWindowOpen())\n")
                log.close()

    def pbar(self):
        k = 500
        for i in range(k):
            self.progressBar.setValue(i)
            time.sleep(0.1)

    def csv_upload(self):
        try:
            filename = pathCSVFileStorage[-1]
            with open(filename, encoding="utf-8") as file:
                reader = csv.DictReader(file)
                for row in reader:
                    print(row)
                    csr.SchuelerErstellen("aktiv",row['Klassen_Klassenbezeichnung'],None,row['Schueler_Nachname'],row['Schueler_Vorname'],row['Schueler_Geburtsdatum'])
                file.close()
            with open("ImHis.txt", "a") as imhis:
                history = date.today()
                imhis.write(str(history) + "\n")
                imhis.close()
            self.close()
            self.diaolog.show()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Es ist etwas schief gelaufen beim erstellen der Schüler\n")
                log.close()

class BrowserWindow(QWidget):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi('GUI\FileBrowser.ui', self)
            self.treeView.clicked.connect(self.getPath)
            self.pushButton.clicked.connect(self.confirm)
            self.pushButton_2.clicked.connect(self.cancel)
            dir_path = ""
            tree = self.treeView
            model = QFileSystemModel(self.treeView)
            model.setRootPath(dir_path)
            model.setNameFilters(["*.csv"])
            model.setNameFilterDisables(False)
            tree.setAnimated(True)
            tree.setSortingEnabled(True)
            tree.setModel(model)
            tree.setRootIndex(model.index(dir_path))
            tree.hideColumn(1)
            tree.hideColumn(2)
            tree.hideColumn(3)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (Class: BrowserWindow())\n")
                log.close()
    def getPath(self):
        try:
            item = []
            for ix in self.treeView.selectedIndexes():
                try:
                    item.pop(0)
                except:
                    pass
                model = QFileSystemModel()
                pathCSVFile = model.filePath(ix)
                item.append(pathCSVFile)
                pathCSVFileStorage.append(pathCSVFile)
                self.label_2.setText(pathCSVFile)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (getPath())\n")
                log.close()

    def confirm(self):
        try:
            self.windowCSVUpload = CSVUploadWindow()
            self.windowCSVUpload.show()
            self.close()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (confirm())\n")
                log.close()

    def cancel(self):
        try:
            self.windowCSVUpload = CSVUploadWindow()
            self.windowCSVUpload.show()
            self.close()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (cancel())\n\n")
                log.close()

class DialogDone(QDialog):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi("GUI\progess.ui", self)
            self.label.setText("Import ist Fertig!")
            self.pushButton.clicked.connect(self.pressed)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (Class: DialogDone())\n")
                log.close()

    def pressed(self):
        self.close()

class DialogImpError(QDialog):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi("GUI\progess.ui", self)
            self.label.setText("Error: Import konnte nicht erfolgreich abgeschlossen werden!")
            self.pushButton.clicked.connect(self.pressed)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (Class: DialogImpError())\n")
                log.close()

    def pressed(self):
        self.close()

class DialogWindow(QDialog):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi("GUI\progess.ui", self)
            self.label.setText("Update: Datenbank neu erstellen!")
            self.pushButton.clicked.connect(self.pressed)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (DialogWindow())\n")
                log.close()

    def pressed(self):
        self.close()

class MainWindow(QWidget):
    def __init__(self):
        try:
            super().__init__()
            uic.loadUi('GUI\GUI.ui', self)
            self.schuelerCount()
            self.setAcceptDrops(False)
            csr.KlassenHinzufuegen()
            self.show_logWin()
            title = "Ausweis Verwaltung"
            self.dialo = DialogWindow()
            self.UpladWin = CSVUploadWindow()
            self.historyimp()
            self.pushButton.clicked.connect(self.changePhoto)
            self.pushButton_2.clicked.connect(self.del_schueler)
            self.pushButton_3.clicked.connect(self.erstellen)
            self.pushButton_5.clicked.connect(self.schueler_suche)
            self.pushButton_7.clicked.connect(self.del_img)
            self.pushButton_10.clicked.connect(self.show_db)
            self.treeView.clicked.connect(self.getPath)
            self.pushButton_4.clicked.connect(self.openUpload)
            self.dialo.show()
            self.paths()
            self.photoViewer = self.label_7
            auto_names = ["10LH-1","13LH-1","10BV01","10BV02"
            ,"10PS01","10PS02","10KFZ1","10KFZ2"
            ,"10KFZ3","10PA1","10PA2","10MR-0"
            ,"10ASH1","10ASH2","10MBX0"
            ,"10EAX0","10EEX0","10EG01","10EG02"
            ,"10BZX1","10BZX2","10ITS1","10ITS2"
            ,"10ITS3","10ME-0","10TIX0","10TZX1"
            ,"10TZX2","10IMX0","10ZMX0","11IMX1"
            ,"11IMX2","11WMX0","11ZMX0","11BGX1"
            ,"11FOX2","11FOX1","11FOI0","11BZX0","11FKMO","11ITS1"
            ,"11ITS2","11ITS3","11ME-0","11PDX0"
            ,"11TIX0","11EAX0","11EEX0","11EG-0","11KL-0","11MB-0"
            ,"11FL-0","11ZI-1","11ZI-2","11ASH1"
            ,"11ASH2","11KFZ1","11KFZ2","11KFZ3"
            ,"11PAB0","11SPX1","11SPX2",
            "11BFI0","11BGX2","11BGX3"
            ,"11BFE0","11BFF0","11BFH0","12BGPI"
            ,"12BGME","12BFI0",
            "12ASH1","12ASH2","12KFZ1","12KFZ2"
            ,"12KFZ3","12KL-1","12PAB1","12PAB2"
            ,"12SPX1","12SPX2","12FL-1","12ZI01"
            ,"12ZI02","12MB-1","12EEX0","12EG01"
            ,"12EG02","12BZX0","12ITS1","12ITS2"
            ,"12ITS3","12ME-0","12PDX0","12TIX1"
            ,"12TIX2","12FOX1","12FOX2","12FOB0"
            ,"12FOM0","12BGB0","12IMX1","12IMX2"
            ,"12WMX0","12ZMX0","13IMX1","13IMX2"
            ,"13ZMX0","13ME-0","13SPX0","13EAX0"
            ,"13EEX0","13EG01","13MB-0","13ASH1"
            ,"13ASH2","13KFZ1","13KFZ2","13KL-0"
            ,"13BGME","13BGPI","10BJH0",
            "01FSHK","02FSHK","10BVP1","10BVP2",
            "08MS-0","09MS-0","10MS-0",
            "10BFX1","10BFX2","10BFX4","10BFX3",
            "11LH01","11LH02","12LH01","12LH02",
            "10INT1","10INT2","10INT3"]
            completer = QCompleter(auto_names)
            self.lineEdit_7.setCompleter(completer)
            self.tableWidget.cellClicked.connect(self.cell_row_print)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("#####################")
                log.write("Error: (MainWindow())\n")
                log.close()

    def historyimp(self):
        try:
            with open("ImHis.txt", "r") as imhis:
                for i in imhis:
                    self.label_16.setText("Letzter Import:" + i )
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (historyimp())\n")
                log.close()

    def cell_row_print(self):
        try:
            data = []
            column = self.tableWidget.currentRow()
            self.tableWidget.selectRow(column)

            if (os.path.exists("temp.jpg")):
                os.remove("temp.jpg")

            try:
                for i in range(5):
                    self.QTableWidgetItem = self.tableWidget.selectedItems()[i]
                    data.append(self.QTableWidgetItem.text())
                self.lineEdit.setText(data[0])
                self.lineEdit_2.setText(data[1])
                self.lineEdit_3.setText(data[2])
                self.lineEdit_4.setText(data[3])
                connection = sqlite3.connect('schuelerausweis.sqlite')
                connection.row_factory = lambda cursor, row: row[0]
                cursor = connection.cursor()
                cursor.execute("SELECT foto FROM schueler WHERE vorname=? AND nachname=?", (data[0],data[1],))
                photo = cursor.fetchone()
                if (photo == None):
                    self.set_image("Avatar.jpg")
                else:
                    csr.SchuelerErstellen.writeBinarytoFile(photo,"temp.jpg")
                    self.set_image("temp.jpg")
                cursor.close()
            except:
                self.label_12.setText("Keine Daten in der Tabelle vorhanden")
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (cell_row_print())\n")
                log.close()

    def del_img(self):
        try:
            csr.SchuelerBildDel(self.lineEdit.text(),self.lineEdit_2.text(),self.lineEdit_3.text())
            pixmap = QPixmap("Avatar.jpg")
            small_pixmap = pixmap.scaled(281, 321)
            pix = self.photoViewer.setPixmap(small_pixmap)
            log_Path_img.append("Avatar.jpg")
            self.schueler_suche()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (del_img())\n")
                log.close()

    def del_schueler(self):
        try:
            csr.SchuelerLoeschen(self.lineEdit.text(),self.lineEdit_2.text(),self.lineEdit_3.text())
            self.lineEdit.clear()
            self.lineEdit_2.clear()
            self.lineEdit_3.clear()
            self.schueler_suche()
            self.show_logWin()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (del_schueler())\n")
                log.close()

    def paths(self):
        try:
            dir_path = self.lineEdit_6.text()
            model = QFileSystemModel()
            model.setRootPath(dir_path)
            model.setNameFilters(["*.jpg","*.png"])
            model.setNameFilterDisables(False)
            tree = self.treeView
            tree.setAnimated(True)
            tree.setSortingEnabled(True)
            tree.setModel(model)
            tree.setRootIndex(model.index(dir_path))
            tree.setDragEnabled(True)
            tree.hideColumn(1)
            tree.hideColumn(2)
            tree.hideColumn(3)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (paths())\n")
                log.close()

    def getPath(self):
        try:
            for ix in self.treeView.selectedIndexes():
                model = QFileSystemModel()
                pathCSVFile = model.filePath(ix)
                self.label_14.setText(log_Path_img[-1])
                try:
                    log_Path_img.pop(0)
                except:
                    with open("log/log_create_schueler.txt", "a") as logfile:
                        logfile.write("Erorr: (getPath)\n")
                        logfile.close()

                finally:
                    log_Path_img.append(pathCSVFile)
                    self.set_image(log_Path_img[-1])
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (getPath())\n")
                log.close()

    def erstellen(self):
        try:
            vorname = self.lineEdit.text()
            nachname = self.lineEdit_2.text()
            gebdat = self.lineEdit_3.text()
            gebdat = gebdat.replace(".","-")
            klasse = self.lineEdit_4.text()
            self.lineEdit_7.setText(klasse)
            status = self.lineEdit_5.text()
            foto = log_Path_img[-1]
            csr.SchuelerErstellen(status,klasse,foto,nachname,vorname,gebdat)
            self.schuelerCount()
            self.schueler_suche()
            self.show_logWin()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (erstellen())\n")
                log.close()

    def clicked_search(self):
        try:
            for n in range(40):
                for m in Range(4):
                    self.tableWidget.setItem(n,m, QTableWidgetItem)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (clicked_search())\n")
                log.close()

    def dragEnterEvent(self, event):
        if event.mimeData().hasImage:
            event.accept()
        else:
            event.ignore()

    def dragMoveEvent(self, event):
        if event.mimeData().hasImage:
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        if event.mimeData().hasImage:
            #event.setDropAction(QtG.CopyAction)
            file_path = event.mimeData().urls()[0].toLocalFile()
            self.set_image(file_path)
            log_Path_img.append(file_path)
            event.accept()
        else:
            event.ignore()

    def set_image(self, file_path):
        try:
            pixmap = QPixmap(file_path)
            small_pixmap = pixmap.scaled(281, 321)
            pix = self.photoViewer.setPixmap(small_pixmap)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (set_image())\n")
                log.close()

    def schueler_suche(self):
        try:
            self.tableWidget.clear()
            kid = self.lineEdit_7.text()
            name = self.lineEdit_8.text()
            self.tableWidget.setHorizontalHeaderLabels(["Vorname","Nachname","Geburtstag","Klasse","Gültigbis","Foto", "Status"])
            self.label_12.setText("")
            box_klasse = self.checkBox.isChecked()
            box_name = self.checkBox_2.isChecked()
            self.show_logWin()

            if (box_klasse == True):
                results = csr.SucheKlasse.suche_klasse(kid)
                for n, key in enumerate(results):
                    if(key[5]==None):
                        l = list(key)
                        l[5] = "None"
                        key = tuple(l)
                    else:
                        l = list(key)
                        l[5] = "Vorhanden"
                        key = tuple(l)
                    for m, item in enumerate(key):
                        self.tableWidget.setItem(n,m, QTableWidgetItem(item))
            elif (box_name == True):
                results = csr.SucheKlasse.suche_name(name)
                for n, key in enumerate(results):
                    if(key[5]==None):
                        l = list(key)
                        l[5] = "None"
                        key = tuple(l)
                    else:
                        l = list(key)
                        l[5] = "Vorhanden"
                        key = tuple(l)
                    for m, item in enumerate(key):
                        self.tableWidget.setItem(n,m, QTableWidgetItem(item))
            elif (box_name == True and box_klasse == True):
                results = csr.SucheKlasse.suchen(name,kid)
                for n, key in enumerate(results):
                    if(key[5]==None):
                        l = list(key)
                        l[5] = "None"
                        key = tuple(l)
                    else:
                        l = list(key)
                        l[5] = "Vorhanden"
                        key = tuple(l)
                    for m, item in enumerate(key):
                        self.tableWidget.setItem(n,m, QTableWidgetItem(item))
        except:
            self.label_6.setText("Klasse nicht gefunde/Error")
            with open("log/log_create_schueler.txt", "a") as logfile:
                logfile.write("Klasse nicht gefunden/Error F(schueler_suche())\n")
                logfile.close()

    def show_db(self):
        try:
            sql_DB = "schuelerausweis.sqlite"
            sql = "SELECT * FROM schueler"
            connection = sqlite3.connect(sql_DB)
            cursor = connection.cursor()
            cursor.execute(sql)
            results = cursor.fetchall()
            print(results)
            cursor.close()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (show_db())\n")
                log.close()

    def changePhoto(self):
        try:
            path = log_Path_img[-1]
            print(path)
            csr.AddPhoto(path,self.lineEdit.text(), self.lineEdit_2.text(),self.lineEdit_3.text())
            self.schueler_suche()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (changePhoto())\n)")
                log.close()

    def schuelerCount(self):
        try:
            sql_DB = "schuelerausweis.sqlite"
            connection = sqlite3.connect(sql_DB)
            cursor = connection.cursor()
            cursor.execute("SELECT bid FROM schueler")
            results = cursor.fetchall()
            count = 0
            for i in results:
                count+=1
            connection.commit()
            cursor.close()
            self.label_15.setText("Schüler in der Datenbank: " + str(count))
        except:
            with open("log/log_create_schueler.txt", "a") as logfile:
                logfile.write("Error: (schuelerCount)/Keine Schüler in der Datenbank\n")
                logfile.close()

    def openUpload(self):
        try:
            self.UpladWin.show()
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (openUpload())\n")
                log.close()

    def show_logWin(self):
        try:
            self.listWidget.clear()
            with open("log/log_create_schueler.txt", "r") as logfile:
                for i in logfile:
                    self.listWidget.addItem(i)
        except:
            with open("log/log_create_schueler.txt", "a") as log:
                log.write("Error: (show_logWin())\n")
                log.close()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    myApp = MainWindow()
    myApp.show()

    try:
        sys.exit(app.exec())
    except SystemExit:
        print('Closing Window...')


from flask import Blueprint, render_template, send_from_directory
from data.main_query import MainQuery
from src.index.forms import DataForm
from src.models import Individual, AccountSite, LinkSite
import json
import string

index_blueprint = Blueprint('index', __name__, template_folder='templates')

@index_blueprint.route('/', methods=['GET', 'POST'])
def index():
    form = DataForm()
    if form.validate_on_submit():
        usernames = [username.strip() for username in form.usernames.data.split(",")]
        name = form.name.data
        organisations = [organisation.strip() for organisation in form.organisations.data.split(",")]
        domains = [domain.strip() for domain in form.domains.data.split(",")]
        new_individual = Individual(
            usernames=usernames,
            name=name,
            organisations=organisations,
            domains=domains
        )
        print(new_individual.__dict__)
        query = MainQuery(new_individual).query()
        bing_results = query["bing_results"]
        sherlock_results = query["sherlock_results"]
        beenpwned_results = query["beenpwned_results"]
        # with open("sherlock_results.json", "w") as f:
        #     f.write(json.dumps(sherlock_results))
        #
        # with open("bing_results.json", "w") as f:
        #     f.write(json.dumps(bing_results))
        #
        # with open("beenpwned_results.json", "w") as f:
        #     f.write(json.dumps(beenpwned_results))

        # with open("bing_results.json", "r") as bing, open("sherlock_results.json") as sherlock, open("beenpwned_results.json") as beenpwned:
        #     bing_results = json.loads(bing.read())
        #     sherlock_results = json.loads(sherlock.read())
        #     beenpwned_results= json.loads(beenpwned.read())

        account_sites = []

        for user in sherlock_results:
            for url in user["urls"]:
                pwned = False
                breach_date = None
                pwn_description = None
                for account in beenpwned_results:
                    for site in account:
                        if site["id"] == url["id"]:
                            pwned = True
                            breach_date = site["breach_date"]
                            pwn_description = site["description"]
                account_sites.append(AccountSite(user["user"], url["url"], pwned=pwned, breach_date=breach_date, pwn_description=pwn_description))

        print(account_sites)

        linked_sites = []
        for page in bing_results["webPages"]["value"]:
            linked_sites.append(LinkSite(page["url"], page["name"], page["snippet"]))

        grouped_list = {}
        for site in linked_sites:
            dom = site.extracted_domain.domain + site.extracted_domain.suffix
            if dom not in grouped_list.keys():
                grouped_list[dom] = [site]
            else:
                grouped_list[dom].append(site)

        return render_template("data.html", account_sites=account_sites, str=str, string=string, grouped_list=grouped_list)

    return render_template('index.html', form=form)

@index_blueprint.route("/favicon.ico")
def favicon():
    return send_from_directory("static", "favicon.ico")


# r = read
# w = write and truncate
# a = append

# open requests from OS the file descriptor
import io

my_file = open("read_my_contents", "r")

# returns a list object
contents = my_file.readlines()

print(contents)
for line in contents:
    # print command by default add a new line "\n" we can tell print that the last char is ''
    # print(line)
    print(line, end='')

# close sending the file descriptor back to the OS
my_file.close()

my_file = open("names.txt", "w")
for i in range(3):
    current_name = input("enter your name: ")
    my_file.write(current_name + "\n")

my_file.close()

my_file = open("names.txt", "r")
for name in my_file.readlines():
    print(f"hello {name}", end='')

my_file.close()

try:
    my_file = open("names.txt", "r")
    my_file.write("foo haha")
except FileNotFoundError as e:
    print("file not found")
except io.UnsupportedOperation:
    print("file is not writeable")
my_file.close()


#BackEnd
from PyQt5.QtCore import QObject, pyqtSignal
from functools import reduce
import math

class PonderadorNotas(QObject):

    senal_notas = pyqtSignal(float, float, int)

    def __init__(self, *args, **kwargs):

        super().__init__(*args, **kwargs)

        self.__nota_minima = 39.5

    def recibir_datos(self, l_notas, ide):

        l_notas_corregido = list()
        try:
            for tupla in l_notas:
                if not tupla[0].isdecimal():
                    raise Exception("Ponderacion no es numerica")

                if not tupla[1].isdecimal() and tupla[1] != "":

                    raise Exception("Nota no es numerica o vacia")

                elif tupla[1] == "":

                    l_notas_corregido.append((int(tupla[0]), -1))

                else:

                    l_notas_corregido.append((int(tupla[0]), int(tupla[1])))

        except Exception as error:
            print("Error: ", error)

        else:

            ponderacion_total = reduce(lambda x,y: x + y[0], l_notas_corregido, 0)
            if ponderacion_total != 100:
                print("Error, ponderacion no suma 100")
                ########################################################

            else:
                nota_ponderada = self.obtener_nota_ponderada(l_notas_corregido)
                nota_pasar = self.obtener_nota_pasar(l_notas_corregido)

                self.senal_notas.emit(nota_ponderada, nota_pasar, ide)

    def obtener_nota_ponderada(self, notas):
        suma_ponderacion = float()
        nota = float()

        for conjunto in notas:
            if conjunto[1] == -1:
                pass
            else:
                nota += conjunto[0] * conjunto[1]
                suma_ponderacion += conjunto[0]

        nota_ponderada = nota / suma_ponderacion
        nota_ponderada = self.round_half_up(nota_ponderada, 1)
        return nota_ponderada

    def obtener_nota_pasar(self, notas):

        nota_decimal = float()
        ponderacion_decimal = float()

        for conjunto in notas:
            if conjunto[1] == -1:
                pass
            else:
                ponderacion = conjunto[0] / 100
                nota_decimal += ponderacion * conjunto[1]
                ponderacion_decimal += ponderacion

        try:
            nota_pasar = (self.__nota_minima - nota_decimal) / (1 - ponderacion_decimal)
        except ZeroDivisionError:
            nota_pasar = self.__nota_minima - nota_decimal
        return self.round_half_up(nota_pasar, 1)

    def round_half_up(self, n, decimals=0):
        multiplier = 10 ** decimals
        return math.floor(n*multiplier + 0.5) / multiplier


from typing import List, Union, Optional, Tuple
from shelley.parsers import shelley_lark_parser, ltlf_lark_parser
from pathlib import Path
from dataclasses import dataclass, field

from shelley.ast.devices import Device
import shelley.parsers.ltlf_lark_parser
from shelley.parsers.ltlf_lark_parser import (
    Variable,
    And,
    Equal,
    Action,
    NotAction,
    Not,
    Always,
    Next,
    Implies,
    Or,
    Until,
    Bool,
    Eventually,
    ExistsFinally,
    Formula,
    ltlf_to_ltl,
    LTL,
    LTL_F,
    CTL,
    EOS,
)

@dataclass
class Op:
    targets: list
    is_final: bool
    is_initial: bool

    @classmethod
    def make(cls, is_final=False, is_initial=False):
        return cls(targets=[], is_final=is_final, is_initial=is_initial)

    def add(self, name):
        self.targets.append(name)

@dataclass
class Spec:
    formulae: List[Union[CTL, LTL, LTL_F]] = field(default_factory=list)
    comment: Optional[str] = field(default=None)

    def add(self, formula):
        self.formulae.append(formula)

    def __len__(self):
        return len(self.formulae)

    def dump(
        self, fp, action_name: Optional[str] = None, eos_name: Optional[str] = None
    ):
        action = Variable("_action" if action_name is None else action_name)
        eos = Variable("_eos" if eos_name is None else eos_name)
        if self.comment is not None:
            fp.write("-- ")
            fp.write(self.comment)
            fp.write("\n")

        for f in self.formulae:
            is_ctl = isinstance(f, CTL)
            f = ltlf_to_ltl(f, eos=eos, action=action)
            if is_ctl:
                fp.write("SPEC ")
            else:
                fp.write("LTLSPEC ")
            ltlf_lark_parser.dump(formula=f, fp=fp)
            fp.write("\n")

    def dumps(self, action_name: Optional[str] = None, eos_name: Optional[str] = None):
        buffer = StringIO()
        self.dump(
            fp=buffer,
            action_name=action_name,
            eos_name=eos_name,
        )
        return buffer.getvalue()

def generate_system_spec(dev: Device) -> Spec:
    """
    @param dev: Shelley device instance
    @return:
    """
    specs = []
    for op in dev.events.list_str():
        specs.append(
            CTL(ExistsFinally(And(LTL_F(Action(op)), ExistsFinally(LTL_F(EOS)))))
        )
    return Spec(specs, comment=f"SYSTEM VALIDITY FOR: {dev.name}")

def generate_usage_validity(dev: Device, prefix: str) -> Spec:
    """
    Generates a spec that represents the usage validity of a given device.
    """
    mk_act = lambda name: LTL_F(Action(name=name))
    initials = [EOS]
    targets = dict()
    for (src, dst) in dev.behaviors.as_list_tuples():
        dsts = targets.get(src, None)
        if dsts is None:
            evt = dev.events.find_by_name(src)
            targets[src] = dsts = Op.make(
                is_final=evt.is_final,
                is_initial=evt.is_start,
            )
            if evt.is_start:
                initials.append(Action(name=src))
        dsts.add(dst)

    spec = Spec(comment=f"USAGE VALIDITY FOR {prefix}: {dev.name}")
    spec.add(LTL_F(Or.make(initials)))
    for (src, op) in targets.items():
        successors = Or.make(map(mk_act, op.targets))
        if op.is_final:
            successors = Or(successors, LTL_F(EOS))
        spec.add(LTL(Always(Implies(mk_act(src), Next(successors)))))
    return spec

def generate_enforce_usage(dev: Device, prefix: str) -> Spec:
    spec = Spec(formulae=[], comment=f"ENFORCE SPECS FOR {prefix}: {dev.name}")
    for f in dev.enforce_formulae:
        spec.formulae.append(LTL_F(f))
    return spec

def generate_subsystem_checks(
    subsystem_formulae: List[Tuple[str, Formula]], prefix: str
) -> Spec:
    spec = Spec(formulae=[], comment=f"SUBSYSTEM CHECKS FOR {prefix}")
    for (k, f) in subsystem_formulae:
        if k == prefix:
            spec.formulae.append(LTL_F(f))
    return spec

def parse_command():
    from argparse import ArgumentParser

    parser = ArgumentParser()
    subparsers = parser.add_subparsers(
        dest="command", help="Each command performs some functionality regarding LTL"
    )
    subparsers.required = True
    i_parser = subparsers.add_parser(
        "instance",
        aliases=["i"],
        help="Generate the integration checks for a given instance.",
    )
    i_parser.add_argument(
        "--spec", "-s", type=Path, help="A Shelley specification", required=True
    )
    i_parser.add_argument("--name", "-n", help="The instance name", required=True)
    # Formula mode
    f_parser = subparsers.add_parser(
        "formula", aliases=["f"], help="Convert LTLf to NuSMV LTL"
    )
    f_parser.add_argument("formulas", nargs="+", help="LTLf formula")
    f_parser.add_argument("--name", "-n", help="Set an instance name")
    parser.add_argument(
        "--var-action",
        default="_action",
        help="NuSMV variable used to represent the current FSM state. Default: %(default)s",
    )
    parser.add_argument(
        "--var-end-of-sequence",
        default="_eos",
        help="NuSMV variable used to represent the end of a sequence. Default: %(default)s",
    )

    return parser.parse_args()

def main():
    args = parse_command()
    if args.command in ["i", "instance"]:
        dev: Device = shelley_lark_parser.parse(args.spec)
        ltl_specs: List[str] = generate_instance_spec(
            dev=dev,
            prefix=args.name,
            eos=Action(args.var_end_of_sequence),
            var_action=Action(args.var_action),
        )
        for spec in ltl_specs:
            print(spec)
    elif args.command in ["f", "formula"]:
        ltlf_formulae: List[Formula] = parse_ltlf_formulae(args.formulas)
        ltl_formulae: List[str] = convert_ltlf_formulae(
            ltlf_formulae,
            name=args.name,
            eos=Action(args.var_end_of_sequence),
            var_action=Action(args.var_action),
        )
        for formula in ltl_formulae:
            print(formula)


import argparse
import json
import sys
import time
import gzip
import numpy as np
import pandas as pd

from utils import *

def read_gear_mutations(input_path):
    '''
    Parse a json file with GEAR file format.

    Parameters
    ----------
    input_path : str
        The json file path.

    Returns
    -------
    pandas.DataFrame
        a data frame with motif count information.
    '''

    # open the input file
    with gzip.open(input_path) as f:
        # load the data and create an empty list
        data = json.load(f)
        data_list = list()
        for i, item in enumerate(data):
            data_dict = {k: v for k, v in item.items() if not k in ["sbs", "indels"]}
            if "sbs" in item:
                for k, v in item["sbs"].items():
                    for vv in v:
                        data_list.append({**data_dict,
                                          **{"kind": "sbs", "pos": int(k), "ref_pos": vv["pos"], "ref_qv": vv["qv"],
                                             "mean_qv": vv["mean_qv"], "sbs": vv["value"],
                                             "is_trimmed": vv["is_trimmed"]}})
            if "indels" in item:
                for k, v in item["indels"].items():
                    for vv in v:
                        data_list.append({**data_dict, **{"kind": vv["kind"], "pos": int(k), "ref_pos": vv["pos"],
                                                          "mean_qv": vv["mean_qv"], "indel": vv["seq"],
                                                          "size": len(vv["seq"]), "is_trimmed": vv["is_trimmed"]}})
    return pd.DataFrame(data_list).drop_duplicates()

if __name__ == "__main__":

    # store start time for benchmarking
    start_time = pd.to_datetime(time.time(), unit="s")

    # setup arguments
    parser = argparse.ArgumentParser(description='region indexer for unknown chromosomes', epilog=epilog_text,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-s', '--silent', action='store_true', help='Starts in silent mode, no message will be output.')
    parser.add_argument('-d', '--debug', action='store_true', help='Shows debug info')
    parser.add_argument('-o', '--output', type=str, help='Output file', default="../data/")
    parser.add_argument('-i', '--input', type=str, help='Input directory', default="../data/GEAR_TELOMERE_MUTATION/")

    # parse arguments and set logger
    args = parser.parse_args()
    consoleHandler = logging.StreamHandler(sys.stdout)

    # setup logger
    root_logger, log_formatter = get_cmri_logger()

    if args.debug:
        root_logger.setLevel(logging.DEBUG)

    if args.silent:
        consoleHandler.setLevel(logging.ERROR)

    consoleHandler.setFormatter(log_formatter)
    root_logger.addHandler(consoleHandler)

    print_cmri_welcome("Mutation Analysis")

    directory_exists(args.input, True)

    data_list = list()
    for f in find_files(args.input, "json.gz",compressed=True):
        if "._" in f:
            continue
        logging.info("reading file: %s", f)
        df_tmp = read_gear_mutations(f)
        D1 = os.path.dirname(f)
        D2 = os.path.dirname(D1)
        df_tmp["sample"] = os.path.basename(D2) + os.path.basename(D1)

        data_list.append(df_tmp)
    df = pd.concat(data_list).reset_index(drop=True)

    variants = [c for c in df.columns if "G" in c]

    df_size = df[["mlen", "name", "seq", "sample"] + variants].drop_duplicates().groupby("sample")[
        variants + ["mlen"]].sum().reset_index()

    category_map = {
        "A1L1": "Con"
        , "A1L2": "Con"
        , "A2L1": "Con"
        , "A2L2": "Con"
        , "A3L1": "MSH6 KO"
        , "A3L2": "MSH6 KO"
        , "A4L1": "MSH6 KO"
        , "A4L2": "MSH6 KO"
    }
    sequence_size_map = {
        "A1L1": 178424347
        , "A1L2": 173957699
        , "A2L1": 171704752
        , "A2L2": 163338462
        , "A3L1": 171391614
        , "A3L2": 158500729
        , "A4L1": 192706327
        , "A4L2": 179876862
    }

    logging.info("Total computation time: %s", str(pd.to_datetime(time.time(), unit="s") - start_time))


from tensorflow.keras.layers import *
from tensorflow.keras.models import *
from tensorflow.python.framework.ops import Tensor
import tensorflow as tf
import numpy as np
from absl import app
from absl import flags

from data import get_dataset

# Random seed fixation
tf.random.set_seed(666)
np.random.seed(666)

FLAGS = flags.FLAGS
flags.DEFINE_integer(
    'global_batch_size', 2048,
    'Batch size for training/eval before distribution.')

flags.DEFINE_integer(
    'epochs', 100,
    'Number of epochs to train.')

flags.DEFINE_float(
    'learning_rate', 0.001,
    'Initial learning rate')

flags.DEFINE_float(
    'proj_head', 1,
    'Projection_Head using finetune.')

flags.DEFINE_string(
    'dataset', None,
    'Directory where dataset is stored.')

flags.DEFINE_string(
    'job_dir', None,
    'GCS path for training job management.')

flags.DEFINE_integer(
    'percentage', 100,
    'percentage of dataset for using finetune.')

flags.DEFINE_string(
    'num_classes', None,
    'Num of class label.')

def build_model(num_classes: int) -> (any, int):
    base_model = tf.keras.models.load_model(f"{FLAGS.job_dir}/pretrain/saved_model")
    base_model.trainable = True

    feature = None
    if FLAGS.proj_head == 0:
        feature = base_model.layers[-6].output
    elif FLAGS.proj_head == 1:
        feature = base_model.layers[-4].output
    elif FLAGS.proj_head == 2:
        feature = base_model.layers[-2].output

    head = Dense(num_classes, name="dense_head", activation="softmax")(feature)
    model = Model(base_model.input, head)
    return model, model.input_shape[1]

def read_tfrecord(example: Tensor, size: int):
    features = {
        "image": tf.io.FixedLenFeature([], tf.string),
        "label": tf.io.FixedLenFeature([], tf.int64),
    }
    # decode the TFRecord
    example = tf.io.parse_single_example(example, features)

    image = tf.image.decode_jpeg(example['image'], channels=3)
    image = tf.image.convert_image_dtype(image, tf.float32)
    w = tf.shape(image)[0]
    h = tf.shape(image)[1]
    resize_crit = (w * size) / (h * size)
    image = tf.cond(resize_crit < 1,
                    lambda: tf.image.resize(image, [w*size/w, h*size/w]),  # if true
                    lambda: tf.image.resize(image, [w*size/h, h*size/h])   # if false
                    )
    nw = tf.shape(image)[0]
    nh = tf.shape(image)[1]
    image = tf.image.crop_to_bounding_box(image, (nw - size) // 2, (nh - size) // 2, size, size)

    return image, example["label"]

def main(argv):
    if len(argv) > 1:
        raise app.UsageError('Too many command-line arguments.')

    # Setup tpu-cluster
    cluster = tf.distribute.cluster_resolver.TPUClusterResolver()
    tf.config.experimental_connect_to_cluster(cluster)
    tf.tpu.experimental.initialize_tpu_system(cluster)
    distribute_strategy = tf.distribute.TPUStrategy(cluster)

    with distribute_strategy.scope():
        model, input_size = build_model(num_classes=FLAGS.num_classes)
        optimizer = tf.keras.optimizers.Adam(learning_rate=FLAGS.learning_rate)
    model.compile(loss="sparse_categorical_crossentropy",
                  optimizer=optimizer,
                  metrics=["accuracy"])
    model.summary()

    tboard_callback = tf.keras.callbacks.TensorBoard(log_dir=f"{FLAGS.job_dir}/finetune/logs", histogram_freq=1)
    callbacks = [tboard_callback]

    train_ds = get_dataset(FLAGS.dataset, "train", read_tfrecord, FLAGS.global_batch_size, input_size, FLAGS.percentage)
    valid_ds = get_dataset(FLAGS.dataset, "valid", read_tfrecord, FLAGS.global_batch_size, input_size, FLAGS.percentage)
    for epoch in range(FLAGS.epochs):
        model.fit(train_ds, validation_data=valid_ds, callbacks=callbacks, initial_epoch=epoch, epochs=epoch+1)
        model.save(f"{FLAGS.job_dir}/finetune/checkpoints/{epoch+1}", include_optimizer=True)

    model.save(f"{FLAGS.job_dir}/finetune/saved_model", include_optimizer=False)

if __name__ == '__main__':
    app.run(main)


from tools.data_loader import TimeSeriesDataLoader

from tools.model_trainer_reader import SklearnRandomForestClassifierTrainer, RiverAdaRandomForestClassifier
from tools.model_evaluator import SklearnModelEvaluator, RiverModelEvaluator

from tools.model_perform_visualization import TrendPlot, PredictionProbabilityDist, RocCurve

import datetime

import json
import numpy as np
import pandas as pd

#=============================================================#
# Definition of functions for following data processing steps #
#=============================================================#
def combine_data_path(year_month) -> str:
    return DATA_HOME_PATH + "highway_traffic_eda_data_ready_for_ml_" + year_month + '.csv'

def prepare_dataloader_for_test(data_path, drop_feature_list: list) -> TimeSeriesDataLoader:
    data_loader = TimeSeriesDataLoader(
        data_path,
        time_series_column_name="DateTime", time_format="%yyyy-%mm-%dd %HH:%MM:%SS"
    )

    for i in drop_feature_list:
        try:
            data_loader.drop_feature(i)
        except KeyError:
            print("{} is not found, skip".format(i))

    # do one hot encoding if needed
    # data_loader.do_one_hot_encoding_by_col("Hour")

    return data_loader

#=================================#
# End of methods definition parts #
#=================================#

#----------------------------#
# Preparation of files paths #
#----------------------------#

# data paths
DATA_HOME_PATH = "../../../data/highway_etc_traffic/eda_data/"

# experimental title
TRAIN_EXTEND_NAME = '2016_12_first_day_weekdays'

# model direction
SKLEARN_MODEL_SAVE_DIR = '../../../model_store/sklearn/rfc/'
SKLEARN_MODEL_SAVE_NAME = 'sklearn_rfc_'+TRAIN_EXTEND_NAME+'.pickle'

RIVER_MODEL_SAVE_DIR = '../../../model_store/river/adarf/'
RIVER_MODEL_SAVE_NAME = 'river_adarf_'+TRAIN_EXTEND_NAME+'.pickle'

LABEL = "TrafficJam60MinLater"

# output plot direction
OUTPUT_DIR = '../../../output_plot/'

#----------------------------------------------------------#
# Start of Online ML Time Series Training/Testing workflow #
#----------------------------------------------------------#

DATA_YEAR_MONTH_LIST = [
    '2019_01', '2019_02', '2019_03', '2019_04', '2019_05', '2019_06', #0~6
    '2019_07', '2019_08', '2019_09', '2019_10', '2019_11', '2019_12', #6~12
    '2020_01', '2020_02', '2020_03', '2020_04', '2020_05', '2020_06', #12~18
    '2020_07', '2020_08', '2020_09', '2020_10', '2020_11', '2020_12', #18~24
    '2021_01', '2021_02', '2021_03', '2021_04', '2021_05', '2021_06', #24~30
    '2021_07', '2021_08', '2021_09', '2021_10', '2021_11', '2021_12',  #30~36
    '2016_weekdays', '2017_weekdays', '2018_weekend', #36~39
    '2020_weekend', '2021_weekend' #39~#41
    # '2021_01_weekdays', '2021_02_weekend', '2021_03_weekend', '2021_04_weekend', '2021_05_weekend', '2021_06_weekend' #42~#48
]

datapaths_training = list(map(lambda x : combine_data_path(DATA_YEAR_MONTH_LIST[x]), range(36, 37)))
feature_to_drop = [
    # "DayOfWeek",
    # "Hour",
    "TrafficJam",
    "TrafficJam30MinLater",
    "MeanSpeed",
    "MeanSpeed10MinAgo",
    "MeanSpeed30MinAgo",
    "MeanSpeed60MinAgo",
    "Upstream1MeanSpeed",
    "Upstream2MeanSpeed",
    "Upstream3MeanSpeed",
    "Downstream1MeanSpeed",
    "Downstream2MeanSpeed",
    "Downstream3MeanSpeed"
]

data_loader_for_training = prepare_dataloader_for_test(datapaths_training, feature_to_drop)

training_start_date = '2016-12-01'
training_end_date = '2016-12-02'

model_master_sklearn = SklearnRandomForestClassifierTrainer(
    data_loader=data_loader_for_training,
    model_saving_dir=SKLEARN_MODEL_SAVE_DIR,
    model_name=SKLEARN_MODEL_SAVE_NAME,
    n_tree=100, max_depth=20, criterion='gini',
    training_data_start_time=training_start_date, training_data_end_time=training_end_date
)

model_master_river = RiverAdaRandomForestClassifier(
    data_loader=data_loader_for_training,
    model_saving_dir=RIVER_MODEL_SAVE_DIR,
    model_name=RIVER_MODEL_SAVE_NAME,
    n_tree=100, max_depth=20, criterion='gini',
    training_data_start_time=training_start_date, training_data_end_time=training_end_date
)

model_sklearn = model_master_sklearn.get_model()
model_river = model_master_river.get_model()

# print("Tree basic structure measurements")
# trees = model_river.models
#
# print(json.dumps(trees[0].model.model_measurements, indent=2, default=str))
# print("Tree structure details inspection")
# print(json.dumps(trees[0].model.model_description(), default=str))
# g = trees[0].model.draw()
# g.render(OUTPUT_DIR + "AdaRF_tree1_Structure_do_parallel", format='png')

#====================================#
# End of Model Training/Preparation, #
# Going to do model validation.      #
#====================================#

datapaths_testing = list(map(lambda x: combine_data_path(DATA_YEAR_MONTH_LIST[x]), range(37, 39)))
data_loader_for_test = prepare_dataloader_for_test(datapaths_testing, feature_to_drop)

#===============================================================#
# Evaluating model by prediction probability distribution check #
#===============================================================#
def run_prediction_proba(pred_proba_list, y_true_list, save_fig_path=None):
    print("checking out prediction probability distribution by ground truth classified")
    if isinstance(pred_proba_list, list):
        print("input probability is list, casting into numpy array")
        pred_np_array = np.array(pred_proba_list)
        ground_true = pd.Series(y_true_list)
    else:
        print("input probability is numpy array")
        pred_np_array = pred_proba_list
        ground_true = y_true_list

    draw_pred_proba = PredictionProbabilityDist(pred_np_array, ground_true)
    print("going to draw probability distribution")
    draw_pred_proba.draw_proba_dist_by_true_false_class_seperated()
    print("finish to draw probability distribution")
    if save_fig_path is not None:
        draw_pred_proba.save_fig(save_fig_path)
        print("Saving prediction probability distribution plot at {} successfully".format(save_fig_path))

#--------------------------------------#
# initializing sklearn model evaluator #
#--------------------------------------#
sklearn_evaluator = SklearnModelEvaluator(
    model_sklearn, data_loader_for_test, LABEL
)
#-----------------------------------------------#
# check out prediction probability distribution #
#-----------------------------------------------#
predict_full_set, y_test_full_set = sklearn_evaluator.predict_proba_true_class_full_set()
run_prediction_proba(predict_full_set, y_test_full_set, OUTPUT_DIR + 'sklearn_pred_proba_plot.pdf')

#--------------------------------------------------------#
# Running Accuracy, recall-rate, and f1 score trend plot #
#--------------------------------------------------------#
sklearn_acc_trend_list = []
sklearn_recall_trend_list = []
sklearn_recall_uncertainty_list = []
sklearn_f1_score_list = []
sklearn_auc_score_list = []
for i_date in data_loader_for_test.get_distinct_date_set_list():
    #===========================================================================#
    # Running prediction probability by date and return daily acc, recall, etc. #
    #===========================================================================#
    pred_result, y_test = sklearn_evaluator.predict_proba_true_class_by_date(i_date)
    acc, recall, recall_uncertainty, f1_s, auc_score = sklearn_evaluator.get_model_score_by_daily_subset(pred_result, y_test, proba_cut=0.4)

    sklearn_acc_trend_list.append(acc * 100)
    sklearn_recall_trend_list.append(recall * 100)
    sklearn_recall_uncertainty_list.append(recall_uncertainty * 100)
    sklearn_f1_score_list.append(f1_s * 100)
    sklearn_auc_score_list.append(auc_score)

#==========================#
# Evaluating RiverML model #
#==========================#
# --------------------------------------#
# initializing riverml model evaluator #
# --------------------------------------#
river_evaluator = RiverModelEvaluator(
    model_river, data_loader_for_test, LABEL
)

#-----------------------------------------------#
# check out prediction probability distribution #
#-----------------------------------------------#
# predict_full_set, y_test_full_set = river_evaluator.predict_proba_true_class_full_set()

roc_curve_display = river_evaluator.roc_curve_displayer(predict_full_set, y_test_full_set, estimator_name="river ml")
RocCurve(roc_curve_display)

run_prediction_proba(predict_full_set, y_test_full_set, OUTPUT_DIR + 'river_pred_proba_plot.pdf')

river_acc_trend_list = []
river_recall_trend_list = []
river_recall_uncertainty_list = []
river_f1_score_list = []
river_auc_score_list = []

#------------------------------------------------------------------------------#
# accumulating prediction proba and corresponding target for plot distribution #
#------------------------------------------------------------------------------#
predict_set_appending_accumulating = []
y_test_set_appending_accumulating = []

for i_date in data_loader_for_test.get_distinct_date_set_list():
    #===========================================================================#
    # Running prediction probability by date and return daily acc, recall, etc. #
    #===========================================================================#

    print("running river accumulating training with date:{}".format(str(i_date)))

    pred_result, y_test = river_evaluator.predict_proba_true_class_by_date(i_date, do_online_training=True)

    predict_set_appending_accumulating.extend(pred_result)
    y_test_set_appending_accumulating.extend(y_test)
    print(len(predict_set_appending_accumulating))

    """
    for check, to be remove
    """
    dates_to_draw = [
        '2021-01-03', '2021-01-06', '2021-01-09', '2021-01-12', '2021-01-15', '2021-01-18', '2021-01-21', '2021-01-24', '2021-01-27', '2021-01-30'
    ]
    if str(i_date) in dates_to_draw:
        run_prediction_proba(predict_set_appending_accumulating, y_test_set_appending_accumulating, OUTPUT_DIR + 'river_pred_proba_plot_{}.pdf'.format(str(i_date)))
        predict_set_appending_accumulating = []
        y_test_set_appending_accumulating = []

    acc, recall, recall_uncertainty, f1_s, auc_score = river_evaluator.get_model_score_by_daily_subset(pred_result, y_test, proba_cut=0.4)

    # roc_curve_display = river_evaluator.roc_curve_displayer(pred_result, y_test, estimator_name="river ml")
    # RocCurve(roc_curve_display)

    river_acc_trend_list.append(acc * 100)
    river_recall_trend_list.append(recall * 100)
    river_recall_uncertainty_list.append(recall_uncertainty * 100)
    river_f1_score_list.append(f1_s * 100)
    river_auc_score_list.append(auc_score)

x_list = data_loader_for_test.get_distinct_date_set_list()

trend_plot_auc = TrendPlot(figsize_x=14, figsize_y=4, is_time_series=True)
trend_plot_auc.plot_trend(x_list, sklearn_auc_score_list, label="sklearn AUC")
trend_plot_auc.plot_trend(x_list, river_auc_score_list, label="river AUC")
trend_plot_auc.save_fig(title="AUC Trend Plot", x_label='date', y_label='AUC', save_fig_path=OUTPUT_DIR+'model_evaluation_trend_plot_auc.pdf')

trend_plot_f1_score = TrendPlot(figsize_x=14, figsize_y=4, is_time_series=True)
trend_plot_f1_score.plot_trend(x_list, sklearn_f1_score_list, label="sklearn f1 score")
trend_plot_f1_score.plot_trend(x_list, river_f1_score_list, label="river f1 score")
trend_plot_f1_score.save_fig(title="F1 score trend plot", x_label='date', y_label='f1 score', save_fig_path=OUTPUT_DIR+'model_evaluation_trend_plot_f1_score.pdf')

trend_plot_recall = TrendPlot(figsize_x=14, figsize_y=4, is_time_series=True)
trend_plot_recall.plot_trend(x_list, sklearn_recall_trend_list, label="sklearn recall")
trend_plot_recall.plot_trend(x_list, river_recall_trend_list, label="river recall")
trend_plot_recall.save_fig(title="recall rate trend plot", x_label='date', y_label='recall', save_fig_path=OUTPUT_DIR+'model_evaluation_trend_plot_recall.pdf')


#! /usr/bin/python

import argparse
import client
import tracker
import bencode
import peer
import logging
import message
from peer import Peer
import select

def main(torrent_file):
    with open(torrent_file, 'rb') as f:
        torrent_info = b""
        for each_line in f:
            torrent_info += each_line
    mycl = client.Client(torrent_info)
    print('starting client to download a file : %s'\
          %mycl.metainfo.get(b'info')[b'name'].decode('utf8'))
    mycl.connect_peers()
    mycl.handshake()
    mycl.send_interested_to_all()
    mycl.recv_message()
    reqs = client.MessageScheduler(mycl.pieces_needed)

    while mycl.connected_peers:
        readables, writables, execptions = select.select(mycl.connected_peers, mycl.connected_peers, [])
        mycl.recv_message(readables)
        reqs.schedule_messages(writables)
        for each_peer in writables:
            each_peer.send_scheduled_messages()
        if ( len(mycl.pieces_needed) is 0 ):
            mycl.combine_pieces()
            break

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('torrent', help = 'input torrent file')
    parser.add_argument('--logging', '-l', default='error', help = \
                        'level of displaying logging info (error, info, debug)',\
                        choices = ['debug', 'info', 'error'])
    arg = parser.parse_args()
    logging.basicConfig(level=arg.logging.upper())
    main(arg.torrent)


#!/usr/bin/env python3

'''
Student: Emil Waldemar Petersson
email: emil-waldemar.petersson.3310@student.uu.se
Reviewed by: Stephan Spengler
Date: 13.okt 2023
'''

from person import Person
from numba import njit
from time import perf_counter as pc
import matplotlib.pyplot as plt

def fib_py(n):
    if(n <= 1):
        return n
    else:
        return (fib_py(n-1) + fib_py(n-2))

@njit
def numba_fib(n: int):
    if n <= 1:
        return n
    else:
        return (numba_fib(n-1) + numba_fib(n-2))

def main():
	f = Person(5)
	print(f.get())
	f.set(7)
	print(f.get())

	test_set = list(range(20,31))
	py_fib = []
	nb_fib = []
	cpp_fib = []

	#print(numba_fib(47)) # --> 2971215073
	f = Person(47)
	print(f.fib()) # -->  2971215073
	'''
	for i in test_set:
		start_py = pc()
		fib_py(i)
		end_py = pc()
		py_fib.append(round(end_py - start_py, 6))
		print(f"Time with just Python for fib({i}): {round(end_py - start_py, 6)}")
	'''
	'''
	for i in test_set:
		start_nb = pc()
		numba_fib(i)
		end_nb = pc()
		nb_fib.append(round(end_nb - start_nb,6))
		print(f'Time with Numba: {round(end_nb - start_nb, 6)}')
	'''

	'''
	for i in test_set:
		f = Person(i)
		start_cpp = pc()
		f.fib()
		end_cpp = pc()
		cpp_fib.append(round(end_cpp - start_cpp,6))
		print(f'Time with C++: {round(end_cpp - start_cpp, 6)}')
	'''

	print('========== Python times ===========')
	print(py_fib)

	print('========== Numba times ===========')
	print(nb_fib)

	#print('========== C++ times ===========')
	#print(cpp_fib)

	'''
	# Plotting the data
	plt.figure(figsize=(10, 6))
	plt.plot(test_set, py_fib, marker='o', label='Python')
	plt.plot(test_set, nb_fib, marker='o', label='Numba')
	#plt.plot(test_set, cpp_fib, marker='o', label='C++')

	# Adding labels and title
	plt.xlabel('n')
	plt.ylabel('Time (seconds)')
	plt.title('Timing Comparisons between Python and Numba Implementations for Fibonacci Sequence')
	plt.legend()
	plt.grid(True)
	plt.yscale('log')  # Using a log scale due to the wide range of time values
	plt.xticks(test_set)  # Explicitly set x-ticks to ensure all n_values are labeled

	# Save the figure
	#plt.savefig('fibonacci_timing_comparison_numba_py.png')
	'''

if __name__ == '__main__':
	main()


# -*- coding: utf-8 -*-
# @Time    : 8/12/2021 11:04 PM
# @Author  : Chris.Wang
# @Site    :
# @File    : title_capitalizer.py
# @Software: PyCharm
# @Description: 用来处理标题大写
import re
from os.path import join
from features.resource.function import projpath
import pyperclip

available_lang = ['en', 'es','fr','de','it']

def capitalize(s,lang='en'):
    if lang not in available_lang:
        exit('unsupported language. ')

    with open(
            join(projpath(),'features','resource','word-list',f'{lang}.txt'),
            mode='r',
            encoding='UTF-8'
    ) as ignore_file:
        ignore_list = ignore_file.read().splitlines()

    sentence_group = s.split('\n')
    word_list = [sentence.split(' ') for sentence in sentence_group]
    remade_string = []
    for i in range(len(word_list)):
        for j in range(len(word_list[i])):
            pattern = r"[^.,()]+"
            try:
                raw_s = re.search(f'({pattern})', word_list[i][j])[1]
            except:
                raw_s = word_list[i][j]
            if j == 0:
                continue
            if raw_s.lower() in ignore_list and "\'" in raw_s:
                continue
            if raw_s.isupper():
                continue
            if raw_s.lower() not in ignore_list:
                # print(f"{word_list[i][j]} to {word_list[i][j].capitalize()}")
                raw_s_new = raw_s.capitalize()
                result = re.sub(pattern, raw_s_new, word_list[i][j], count=1)
                word_list[i][j] = result
            else:
                word_list[i][j] = word_list[i][j].lower()
        temp_s = ' '.join(word_list[i])
        remade_string.append(temp_s)
    return '\n'.join(remade_string)

def run():
    content = []
    lang = input('输入目标语言缩写(en/fr/de/es/it)：')
    if lang not in available_lang:
        exit(f'unsupported language: {lang}. ')

    line = input('请输入文本，按 Ctrl-D 完成输入确认：\n')
    content.append(line)
    while True:
        line = input()
        if line.endswith('\\\\'):
            content.append(line.strip('\\\\'))
            break
        else:
            content.append(line)

    s = '\n'.join(content)
    s = s.replace(b'\xc2\xa0'.decode('utf-8'),' ')
    print('输出结果：')
    result = capitalize(s,lang=lang)
    print(result)

    pyperclip.copy(result)
    print('\n')
    print('内容已复制到剪切板，程序结束。')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
import uuid
import boto3
import sys
from ebooklib import epub

from style.style import STYLE
from plugins.picture import PicturePlugin
from utils import (minio_make_bucket, put_file, get_metadata, es, put_book_info_2_es,
                   presigned_get_object, create_book_resource, send_result2tg, str2bool, md2html,
                   invalid_xml_remove)
import configs

def make_book_with_metadata(_metadata, _book_uuid):
    print('Building the book, got metadata: {}'.format(_metadata))

    book = epub.EpubBook()
    # add metadata
    print('Book identifier id: {}'.format(str(_book_uuid)))
    book.set_identifier(str(_book_uuid))
    book.set_title(_metadata['_source']['title'])
    book.set_language('zh')
    book.add_author('ee-book')

    # intro chapter
    c1 = epub.EpubHtml(title='Introduction', file_name='intro.xhtml', lang='zh')
    c1.content = _metadata['_source']['title']

    # about chapter, TODO, get from _INDEX/_TYPE/
    c2 = epub.EpubHtml(title='About this book', file_name='about.xhtml')
    c2.content = 'About this book'
    query = {
        'bool': {
            'must':[
                {
                    "terms": {
                        "dayTimestamp": [configs.DAY_TIME_STAMP]
                    }
                }
            ]
        }
    }
    query = _metadata['_source'].get('query') or query
    dsl_body = {
        "query": query
    }

    # TODO: Separate by volumes
    content_result = es.search(index=configs.ES_INDEX, doc_type=configs.ES_TYPE+':content', body=dsl_body, size=configs.CONTENT_SIZE, from_=0)
    content = content_result['hits']['hits']

    # add chapters to the book
    book.add_item(c1)
    book.add_item(c2)

    section = [c1, c2]
    book_spine = ['nav', c1, c2]

    for item in content:
        title = invalid_xml_remove(item['_source'].get('title') or 'No Title')
        if title is None or title == "":
            title = "No Title"
        print('Add chapter: {}'.format(title))
        chapter = epub.EpubHtml(title=title, file_name=title + '.xhtml')
        title_html = '
' + title + '
'
        author_html = '
Author: ' + item['_source']['author'] + '
'
        if isinstance(item['_source']['content'], list):
            content = ''
            for sub_item in item['_source']['content']:
                sub_author = '

Author: ' + sub_item['author'] + '
'
                sub_content = sub_item['content']
                content += (sub_author+str(sub_content))
            chapter.content = title_html + content
        else:
            chapter.content =  title_html + author_html + item['_source']['content']
        chapter.content = md2html(chapter.content, configs.CONTENT_IS_MARKDOWN)
        section.append(chapter)
        book_spine.append(chapter)
        book.add_item(chapter)

    # create table of contents
    book.toc = (epub.Link('intro.xhtml', 'Introduction', 'intro'),
                (epub.Section('Articles'),
                 tuple(section)
                )
    )
    # add navigation files
    book.add_item(epub.EpubNcx())
    book.add_item(epub.EpubNav())

    # define css style

    # add css file
    nav_css = epub.EpubItem(uid="style_nav", file_name="style/nav.css", media_type="text/css", content=STYLE)
    book.add_item(nav_css)

    # create spine
    book.spine = book_spine
    # epub options
    opts = {}
    if str2bool(os.getenv("REPLACE_PICTURE", "true")):
        opts = {'plugins': [PicturePlugin()]}

    # create epub file
    epub_name = 'ee-bookorg-' + _metadata['_source']['title'] + '-' + configs.DAY_TIME_STAMP + '-' + configs.CREATED_BY + '.epub' if configs.EPUB_NAME_FOR_DEBUG is None else configs.EPUB_NAME_FOR_DEBUG
    epub.write_epub('./' + epub_name, book, opts)
    return epub_name

def main():
    if configs.DEBUG_MODE is not True:
        minio_make_bucket('webeebook')
    else:
        print("DEBUG mode")
    metadata = get_metadata(_id=configs.EEBOOK_URL)
    book_uuid = uuid.uuid4()
    epub_name = make_book_with_metadata(metadata, book_uuid)
    file_path = '/src/' + epub_name

    if configs.DEBUG_MODE is not True:
        # TODO: size restriction
        s3 = boto3.client(
            's3',
            aws_access_key_id=configs.EPUB_S3_ACCESS_KEY,
            aws_secret_access_key=configs.EPUB_S3_SECRET_KEY
        )
        s3.upload_file(file_path, 'webeebook', 'books/'+epub_name)
        content_string = 'attachment; filename="' + epub_name + '"'
        put_file('webeebook', 'books/'+epub_name, file_path)
        download_url = presigned_get_object(bucket='webeebook',
                                            filename='books/'+epub_name,
                                            expire_days=1,
                                            content_string=content_string)
    else:
        download_url = 'DEBUG_MODE'
    # print("Download url: {}".format(download_url))
    # TODO, send book metadata to es, include book_name, created_by, book_basic_info,
    # TODO: update book href in each es doc, so we can search with book content
    # Just copy github, project->book, code->book content
    book_info_body = {
        'id': book_uuid,
        'title': epub_name,
        'type': 'eebook',
        'tags': [
            {
                'name': configs.ES_INDEX,
                'title': configs.ES_INDEX,
                'count': 1,
            },
        ],
        'created_by': configs.CREATED_BY,
        'created_time': 'create_time',  # TODO
        'updated_time': 'updated_time',
        'url': configs.EEBOOK_URL,
        'download_url': download_url
    }
    if configs.DEBUG_MODE is not True:
        response = create_book_resource(epub_name, str(book_uuid), configs.IS_PUBLIC)
        chat_id = os.getenv('_CHAT_ID', None)
        if response.status_code != 201:
            print("Got error...")
            print("status code: {}, response: {}".format(response.status_code, response.text))
            send_result2tg('', '', chat_id)
            return
        if chat_id is not None:
            send_book_response = send_result2tg(epub_name, download_url, chat_id)

    put_book_info_2_es(book_id=book_uuid, body=book_info_body)
    print('Successfully send book information to ee-book, book_id: {}, name: {}'.format(
        book_uuid, epub_name))

if __name__ == '__main__':
    main()


import numpy as np
from tqdm import tqdm
from kmeans import KMeans

SIGMA_CONST = 1e-6  # Only add SIGMA_CONST when sigma_i is not invertible
LOG_CONST = 1e-32

FULL_MATRIX = False  # Set False if the covariance matrix is a diagonal matrix

class GMM(object):
    def __init__(self, X, K, max_iters=100):  # No need to change
        """
        Args:
            X: the observations/datapoints, N x D numpy array
            K: number of clusters/components
            max_iters: maximum number of iterations (used in EM implementation)
        """
        self.points = X
        self.max_iters = max_iters

        self.N = self.points.shape[0]  # number of observations
        self.D = self.points.shape[1]  # number of features
        self.K = K  # number of components/clusters

    # Helper function for you to implement
    def softmax(self, logit):  # [5pts]
        """
        Args:
            logit: N x D numpy array
        Return:
            prob: N x D numpy array. See the above function.
        """

        temp = np.atleast_2d(logit)
        temp -= np.expand_dims(
            np.max(temp, axis=1),
            axis=1
        )
        temp = np.exp(temp)

        sig = np.expand_dims(
            np.sum(temp, axis=1),
            axis=1
        )

        return temp / sig

    def logsumexp(self, logit):  # [5pts]
        """
        Args:
            logit: N x D numpy array
        Return:
            s: N x 1 array where s[i,0] = logsumexp(logit[i,:]). See the above function
        """
        temp = list(np.asarray(logit).shape)
        temp[1] = 1
        top = np.asarray(logit).max(axis=1)
        total = np.log(
            np.exp(
                np.asarray(logit) - top.reshape(temp)
            ).sum(axis=1)
        )
        output = (top + total).reshape(temp[0], 1)
        return output

    # for undergraduate student
    def normalPDF(self, logit, mu_i, sigma_i):  # [5pts]
        """
        Args:
            logit: N x D numpy array
            mu_i: (D,) numpy array, the center for the ith gaussian.
            sigma_i: DxD numpy array, the covariance matrix of the ith gaussian.
        Return:
            pdf: (N,) numpy array, the probability density value of N data for the ith gaussian

        Hint:
            np.diagonal() should be handy.
        """
        temp = np.exp(
            -0.5 * (logit - mu_i) ** 2 / sigma_i.diagonal()
        ) / np.sqrt(
            2 * np.pi * sigma_i.diagonal()
        )
        return np.prod(temp, axis=1)

    # for grad students
    def multinormalPDF(self, logits, mu_i, sigma_i):  # [5pts]
        """
        Args:
            logit: N x D numpy array
            mu_i: (D,) numpy array, the center for the ith gaussian.
            sigma_i: DxD numpy array, the covariance matrix of the ith gaussian.
        Return:
            normal_pdf: (N,) numpy array, the probability density value of N data for the ith gaussian

        Hint:
            1. np.linalg.det() and np.linalg.inv() should be handy.
            2. The value in self.D may be outdated and not correspond to the current dataset,
            try using another method involving the current arguments to get the value of D
        """

        raise NotImplementedError

    def _init_components(self, **kwargs):  # [5pts]
        """
        Args:
            kwargs: any other arguments you want
        Return:
            pi: numpy array of length K, prior
            mu: KxD numpy array, the center for each gaussian.
            sigma: KxDxD numpy array, the diagonal standard deviation of each gaussian.
                You will have KxDxD numpy array for full covariance matrix case
        """
        pi = np.ones(self.K) * (1.0 / self.K)
        mu = self.points[
            np.random.choice(
                self.N,
                size=self.K,
                replace=False
            )
        ]
        sigma = np.array(
            [np.eye(self.points.shape[1]) for _ in range(self.K)]
        )
        return pi, mu, sigma

    def _ll_joint(self, pi, mu, sigma, full_matrix=FULL_MATRIX, **kwargs):  # [10 pts]
        """
        Args:
            pi: np array of length K, the prior of each component
            mu: KxD numpy array, the center for each gaussian.
            sigma: KxDxD numpy array, the diagonal standard deviation of each gaussian. You will have KxDxD numpy
            array for full covariance matrix case
            full_matrix: whether we use full covariance matrix in Normal PDF or not. Default is True.

        Return:
            ll(log-likelihood): NxK array, where ll(i, k) = log pi(k) + log NormalPDF(points_i | mu[k], sigma[k])
        """
        output = np.empty((len(self.points), len(mu)))
        for x in range(len(mu)):
            output[:, x] = np.log(pi[x] + 1e-32) + np.log(self.normalPDF(self.points, mu[x], sigma[x]) + 1e-32)
        return output

    def _E_step(self, pi, mu, sigma, full_matrix=FULL_MATRIX, **kwargs):  # [5pts]
        """
        Args:
            pi: np array of length K, the prior of each component
            mu: KxD numpy array, the center for each gaussian.
            sigma: KxDxD numpy array, the diagonal standard deviation of each gaussian.You will have KxDxD numpy
            array for full covariance matrix case
            full_matrix: whether we use full covariance matrix in Normal PDF or not. Default is True.
        Return:
            gamma(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignment) for each observation.

        Hint:
            You should be able to do this with just a few lines of code by using _ll_joint() and softmax() defined above.
        """
        return self.softmax(
            self._ll_joint(pi, mu, sigma)
        )

    def _M_step(self, gamma, full_matrix=FULL_MATRIX, **kwargs):  # [10pts]
        """
        Args:
            gamma(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignment) for each observation.
            full_matrix: whether we use full covariance matrix in Normal PDF or not. Default is True.
        Return:
            pi: np array of length K, the prior of each component
            mu: KxD numpy array, the center for each gaussian.
            sigma: KxDxD numpy array, the diagonal standard deviation of each gaussian. You will have KxDxD numpy
            array for full covariance matrix case

        Hint:
            There are formulas in the slides and in the Jupyter Notebook.
        """
        temp = np.sum(
            gamma,
            axis=0
        )

        mu = np.array(
            [
                np.sum(
                    gamma[:, x].reshape(
                        self.N,
                        1
                    ) * self.points,
                    axis=0
                ) / temp[x] for x in range(self.K)
            ]
        )

        sigma = np.array(
            [
                np.diag(
                    np.diag(
                        np.matmul(
                            (gamma[:, x].reshape(self.N, 1) * (self.points - mu[x].reshape(
                                1,
                                self.D
                            ))).T,
                            self.points - mu[x].reshape(
                                1,
                                self.D
                            )
                        ) / temp[x]
                    )
                ) for x in range(self.K)
            ]
        )

        pi = temp / self.N

        return pi, mu, sigma

    def __call__(self, full_matrix=FULL_MATRIX, abs_tol=1e-16, rel_tol=1e-16, **kwargs):  # No need to change
        """
        Args:
            abs_tol: convergence criteria w.r.t absolute change of loss
            rel_tol: convergence criteria w.r.t relative change of loss
            kwargs: any additional arguments you want

        Return:
            gamma(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignment) for each observation.
            (pi, mu, sigma): (1xK np array, KxD numpy array, KxDxD numpy array)

        Hint:
            You do not need to change it. For each iteration, we process E and M steps, then update the paramters.
        """
        pi, mu, sigma = self._init_components(**kwargs)
        pbar = tqdm(range(self.max_iters))

        for it in pbar:
            # E-step
            gamma = self._E_step(pi, mu, sigma, full_matrix)

            # M-step
            pi, mu, sigma = self._M_step(gamma, full_matrix)

            # calculate the negative log-likelihood of observation
            joint_ll = self._ll_joint(pi, mu, sigma, full_matrix)
            loss = -np.sum(self.logsumexp(joint_ll))
            if it:
                diff = np.abs(prev_loss - loss)
                if diff < abs_tol and diff / prev_loss < rel_tol:
                    break
            prev_loss = loss
            pbar.set_description('iter %d, loss: %.4f' % (it, loss))
        return gamma, (pi, mu, sigma)


def load_tests(loader, standard_tests, pattern):
    """ Custom test loading function that enables test filtering using regex
    exclusion pattern.

    Parameters
    ----------
    loader : unittest.TestLoader
        The instance of test loader
    standard_tests : unittest.TestSuite
        Tests that would be loaded by default from this module (no tests)
    pattern : str
        An inclusion pattern used to match test files (test*.py default)

    Returns
    -------
    filtered_package_tests : unittest.TestSuite
        TestSuite representing all package tests that did not match specified
        exclusion pattern.
    """
    from os import environ
    from os.path import dirname
    from pyface.util.testing import filter_tests
    from unittest import TestSuite

    # Make sure the right toolkit is up and running before importing tests
    from pyface.toolkit import toolkit_object  # noqa: F401

    this_dir = dirname(__file__)
    package_tests = loader.discover(start_dir=this_dir, pattern=pattern)

    # List of regular expression for filtering test using the test id.
    exclusion_patterns = []

    # Environment variable for skipping more tests.
    # e.g. etstool.py in the source tree root sets this to skip packages for
    # specific toolkit
    additional_exclude = environ.get("EXCLUDE_TESTS", None)
    if additional_exclude is not None:
        exclusion_patterns.append(additional_exclude)

    filtered_package_tests = TestSuite()
    for test_suite in package_tests:
        for exclusion_pattern in exclusion_patterns:
            test_suite = filter_tests(test_suite, exclusion_pattern)
        filtered_package_tests.addTest(test_suite)

    return filtered_package_tests


# coding:utf-8
import requests
import re
from bs4 import BeautifulSoup as bs
requests.packages.urllib3.disable_warnings()

verifyheaders = {
    "Host": "wangpan007.com",
    "Connection": "keep-alive",
    "Cache-Control": "max-age=0",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    "Upgrade-Insecure-Requests": "1",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Referer": "https://wangpan007.com/share/kw%E6%9D%83%E5%8A%9B%E7%9A%84",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",

}

baiduheaders = {
    "Connection": "keep-alive",
    "Cache-Control": "max-age=0",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9"
}
url = "https://wangpan007.com/share/file/4842079"

def verifyt(url):
    r = requests.get(url, headers=verifyheaders, verify=False)
    r.encoding = "utf-8"
    text = r.text
    soup = bs(text, "html.parser")
    div = soup.find("div", class_="mt15")
    link = div.find_all("a", class_="btn btn_redirect")[0]['href']
    waiturl = "https://wangpan007.com" + link  # 获取真实的百度云链接
    # 对获取的百度云链接进行验证，判断内容是否失效
    s = requests.Session()
    newr = s.get(waiturl, verify=False,
                        headers=baiduheaders, allow_redirects=True)
    newr.encoding = "utf-8"  # 修改编码
    baidutext = newr.text
    print(baidutext)
    baidudsoup = bs(baidutext, 'html.parser')
    baidutitle = baidudsoup.find("div",id_="tip_msg")
    print(baidutitle)
    print(baidutitle.find_all("p")[1])

    if (baidutitle.text == "百度网盘-链接不存在"):
        print(baidutitle)
        status = "no"
        return 0
    elif(baidutitle.text == "页面不存在"):
        print(baidutitle)
        status = "no"
        return 0
    elif(baidutitle.text == "百度网盘 个人专辑 - 百度网盘，让美好永远陪伴"):
        print(baidutitle)
        status = "no"
        return 0
    else:
        print(baidutitle)
        status = "yes"
        return url

verifyt(url)