from datetime import datetime
import re

from .pdfReader import PDFReader

class PDFParserSantander:
    def __init__(self, file):
        self.file = file
        self.cards = []
        self.results = []

        self._text = None
        self._type = None

    @staticmethod
    def __replace_separator(value_str_brl):
        return value_str_brl.replace(',', '.')

    def run(self):
        self.results.clear()

        reader = PDFReader()
        self._text = reader.run(self.file)

        pdf_type = self._get_type()

        if pdf_type == 'internet':
            self._run_internet_banking()
        elif pdf_type in ['standard', 'unique']:
            self._run()
        elif pdf_type == 'unknown':
            raise ValueError(f'Parser does not know how to handle this file: {self.file}')
        else:
            raise ValueError(f'Unexpected value for type: {pdf_type}')

        return self.results

    def _get_type(self):
        if self._type is None:
            if self._text is None:
                raise ValueError(
                    'Run parser before calling this method or check if text is being improperly assigned'
                )
            if self._text.startswith('Internet Banking'):
                self._type = 'internet'
            elif any(
                    card_type in self._text for card_type in [
                        'SANTANDER NACIONAL',
                        'SANTANDER STYLE PLATINUM',
                        'SANTANDER FREE',
                    ]
            ):
                self._type = 'standard'
            elif self._text.find('SANTANDER UNIQUE') != -1:
                self._type = 'unique'
            else:
                self._type = 'unknown'
        return self._type

    def _run_internet_banking(self):
        cash_flows_delimiter = 'Resumo das despesas'
        if cash_flows_delimiter in self._text:
            tables_and_footers, _ = self._text.split(cash_flows_delimiter)
            cash_date = self.__find_cash_date()
        else:
            tables_and_footers = self._text
            cash_date = None

        header_delimiter = 'Data\nDescrição\nValor (US$)\nValor (R$)'
        tables_and_footers_list = tables_and_footers.split(header_delimiter)

        footer_delimiter = ' Central de Atendimento Santander'
        table_list = [t.split(footer_delimiter)[0].strip() for t in tables_and_footers_list]
        tables = '\n'.join(table_list)

        cards_raw = tables.split('NªCartao')[1:]
        for raw in cards_raw:
            card = {}
            tokens = raw.strip().split('\n')
            card['last_digits'] = tokens[0].strip('Final:')
            card['owner'] = tokens[1].strip('Titular:')
            card['cash_flows'] = []
            for i in range(2, len(tokens), 4):
                cash_flow = {
                    'date': datetime.strptime(tokens[i], '%d/%m/%Y'),
                    'description': tokens[i + 1],
                    'value_usd': self.__replace_separator(tokens[i + 2].strip('US$ ')),
                    'value_brl': self.__replace_separator(tokens[i + 3].strip('R$ ')),
                }

                # TODO: process expenses when currency is usd
                self.results.append([
                    cash_flow['date'],
                    cash_flow['description'],
                    cash_flow['value_brl'],
                    card['last_digits'],
                    cash_date,
                ])

                card['cash_flows'].append(cash_flow)

            self.cards.append(card)

    def _run(self):
        cash_date = self.__find_cash_date()
        origin = self.__find_card_number()

        pages = self._text.split('Nº DO CARTÃO ')

        if self._type == 'unique':
            expense_pages = pages[2:]
        elif self._type == 'standard':
            expense_pages = pages[3:]
        else:
            raise ValueError(f'Could not run type={self._type}')

        expense_pages[0] = expense_pages[0].split('IOF e CET')[0]

        expense_history = ''.join(expense_pages)
        tokens = expense_history.split()
        start = False
        card_tokens = []
        for token in tokens:
            if token in ['Histórico', 'TransaçõesNacionais', 'TransaçõesInternacionais']:
                start = True
            elif token in ['DataDescrição', '(+)Despesas/DébitosnoBrasil']:
                start = False
            elif start is True:
                card_tokens.append(token)

        for i in range(len(card_tokens)):
            token = card_tokens[i]
            if re.match(r"\d{2}/\d{2}", token[:5]):
                date_str = f"{token[:5]}/{cash_date.year}"
                date = datetime.strptime(date_str, '%d/%m/%Y')

                description = token[5:]

                next_token = card_tokens[i + 1]
                if next_token.startswith('PARC'):
                    description = f"{description} {next_token}"
                    next_token = card_tokens[i + 2]

                value = self.__replace_separator(next_token)

                self.results.append([date, description, value, origin, cash_date])

    def __find_card_number(self):
        pos = self._text.find('Nº DO CARTÃO ')
        # format: Nº DO CARTÃO 1234 XXXX XXXX 4321
        return self._text[pos + 13:pos + 32]

    def __find_cash_date(self):
        if self._type == 'internet':
            delimiter = 'Data de vencimento:\n'
            pos = self._text.find(delimiter) + len(delimiter)
        elif self._type in ['unique', 'standard']:
            delimiter = '!Vencimento\n'
            pos = self._text.find(delimiter) + len(delimiter)
        else:
            raise ValueError('Could not find cash date.')

        # format dd/mm/YYYY (len=10)
        date_str = self._text[pos:pos + 10]
        return datetime.strptime(date_str, '%d/%m/%Y')


import json

import privatbank
import monobank

from notion_utils import insert_transactions_to_notion, get_last_table_update, get_transactions_after

if __name__ == '__main__':
    with open("config.json") as f:
        config = json.load(f)
    forward_window = config["forward_window"]
    notion_token = config["notion"]["token"]
    notion_url = config["notion"]["table_url"]
    cards = config["cards"]

    last_update = get_last_table_update(notion_url, "time", notion_token)
    get_transaction_to = last_update + forward_window

    transactions = []
    for card in cards:
        if card["type"] == "privat":
            transactions.extend(
                privatbank.get_transactions(
                    last_update,
                    get_transaction_to,
                    card["card_num"],
                    card["merchant_id"],
                    card["merchant_pass"],
                )
            )
        if card["type"] == "mono":
            transactions.extend(
                monobank.get_transactions(
                    last_update,
                    get_transaction_to,
                    card["card_id"],
                    card["limits_history"],
                    card["token"],
                )
            )

    transactions.sort(key=lambda x: x['time'])

    last_transactions = get_transactions_after(notion_url, "time", last_update, notion_token)
    last_transactions_hash = {
        f"{t['amount'] * 100:.0f}{t['rest'] * 100:.0f}{t['time'].start.timestamp():.0f}"
        for t in last_transactions
    }

    transactions = [
        t
        for t in transactions
        if f"{t['amount']}{t['rest']}{t['time']}" not in last_transactions_hash
    ]

    insert_transactions_to_notion(notion_url, transactions, notion_token)


from typing import Dict, Iterator, List, Optional, Tuple, Union

from .. import AddedToken, Tokenizer, decoders, pre_tokenizers, trainers
from ..models import BPE
from ..normalizers import BertNormalizer, Lowercase, Sequence, unicode_normalizer_from_str
from .base_tokenizer import BaseTokenizer

class CharBPETokenizer(BaseTokenizer):
    """Original BPE Tokenizer

    Represents the BPE algorithm, as introduced by Rico Sennrich
    (https://arxiv.org/abs/1508.07909)

    The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
    Sennrich subword-nmt implementation by the following options that you can deactivate:
        - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
            * removing any control characters and replacing all whitespaces by the classic one.
            * handle chinese chars by putting spaces around them.
            * strip all accents.
        - spitting on punctuation in addition to whitespaces (deactivate it with
          `split_on_whitespace_only=True`)
    """

    def __init__(
        self,
        vocab: Optional[Union[str, Dict[str, int]]] = None,
        merges: Optional[Union[str, Dict[Tuple[int, int], Tuple[int, int]]]] = None,
        unk_token: Union[str, AddedToken] = "<unk>",
        suffix: str = "</w>",
        dropout: Optional[float] = None,
        lowercase: bool = False,
        unicode_normalizer: Optional[str] = None,
        bert_normalizer: bool = True,
        split_on_whitespace_only: bool = False,
    ):
        if vocab is not None and merges is not None:
            tokenizer = Tokenizer(
                BPE(
                    vocab,
                    merges,
                    dropout=dropout,
                    unk_token=str(unk_token),
                    end_of_word_suffix=suffix,
                )
            )
        else:
            tokenizer = Tokenizer(BPE(unk_token=str(unk_token), dropout=dropout, end_of_word_suffix=suffix))

        if tokenizer.token_to_id(str(unk_token)) is not None:
            tokenizer.add_special_tokens([str(unk_token)])

        # Check for Unicode normalization first (before everything else)
        normalizers = []

        if unicode_normalizer:
            normalizers += [unicode_normalizer_from_str(unicode_normalizer)]

        if bert_normalizer:
            normalizers += [BertNormalizer(lowercase=False)]

        if lowercase:
            normalizers += [Lowercase()]

        # Create the normalizer structure
        if len(normalizers) > 0:
            if len(normalizers) > 1:
                tokenizer.normalizer = Sequence(normalizers)
            else:
                tokenizer.normalizer = normalizers[0]

        if split_on_whitespace_only:
            tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
        else:
            tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()

        tokenizer.decoder = decoders.BPEDecoder(suffix=suffix)

        parameters = {
            "model": "BPE",
            "unk_token": unk_token,
            "suffix": suffix,
            "dropout": dropout,
            "lowercase": lowercase,
            "unicode_normalizer": unicode_normalizer,
            "bert_normalizer": bert_normalizer,
            "split_on_whitespace_only": split_on_whitespace_only,
        }

        super().__init__(tokenizer, parameters)

    @staticmethod
    def from_file(vocab_filename: str, merges_filename: str, **kwargs):
        vocab, merges = BPE.read_file(vocab_filename, merges_filename)
        return CharBPETokenizer(vocab, merges, **kwargs)

    def train(
        self,
        files: Union[str, List[str]],
        vocab_size: int = 30000,
        min_frequency: int = 2,
        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
        limit_alphabet: int = 1000,
        initial_alphabet: List[str] = [],
        suffix: Optional[str] = "</w>",
        show_progress: bool = True,
    ):
        """Train the model using the given files"""

        trainer = trainers.BpeTrainer(
            vocab_size=vocab_size,
            min_frequency=min_frequency,
            special_tokens=special_tokens,
            limit_alphabet=limit_alphabet,
            initial_alphabet=initial_alphabet,
            end_of_word_suffix=suffix,
            show_progress=show_progress,
        )
        if isinstance(files, str):
            files = [files]
        self._tokenizer.train(files, trainer=trainer)

    def train_from_iterator(
        self,
        iterator: Union[Iterator[str], Iterator[Iterator[str]]],
        vocab_size: int = 30000,
        min_frequency: int = 2,
        special_tokens: List[Union[str, AddedToken]] = ["<unk>"],
        limit_alphabet: int = 1000,
        initial_alphabet: List[str] = [],
        suffix: Optional[str] = "</w>",
        show_progress: bool = True,
        length: Optional[int] = None,
    ):
        """Train the model using the given iterator"""

        trainer = trainers.BpeTrainer(
            vocab_size=vocab_size,
            min_frequency=min_frequency,
            special_tokens=special_tokens,
            limit_alphabet=limit_alphabet,
            initial_alphabet=initial_alphabet,
            end_of_word_suffix=suffix,
            show_progress=show_progress,
        )
        self._tokenizer.train_from_iterator(
            iterator,
            trainer=trainer,
            length=length,
        )


import graphviz
from ShExJSG.ShExJ import IRIREF, TripleConstraint, NodeConstraint, ObjectLiteral
from pyshex.utils.schema_loader import SchemaLoader

symbol = {
    "class": "oval",
    "datatype": "octagon",
    "literal": "rectangle",
    "iri": "diamond",
    "bnode": 'point',
    "oneof": 'record'
}

def shex2dot(shex, graphviz_name="default.png", rankdir="LR"):
    def process_tc(tc):
        dotschema.node(startshape.replace(":", ""), startshape, shape=symbol["iri"])
        if isinstance(tc, TripleConstraint):
            if tc.max == None and tc.min == None:
                arrowhead = "normal"
            elif tc.max == 1 and tc.min == 0:
                arrowhead = "teeodot"
            elif tc.max == -1 and tc.min == 1:
                arrowhead = "crowtee"
            elif tc.max == -1 and tc.min == 0:
                arrowhead = "crowdot"
            if isinstance(tc.valueExpr, IRIREF):
                node = tc.valueExpr
                predicate = tc.predicate
                for key in prefixmap.keys():
                    node = node.replace(key, prefixmap[key])
                    predicate = predicate.replace(key, prefixmap[key] + ":")
                dotschema.node(node, node, shape=symbol["iri"])
                dotschema.edge(shape.id.split("/")[-1], node, label=predicate, arrowhead=arrowhead)
            elif isinstance(tc.valueExpr, NodeConstraint):
                if tc.valueExpr.datatype:
                    datatype = tc.valueExpr.datatype
                    predicate = tc.predicate
                    for key in prefixmap.keys():
                        datatype = datatype.replace(key, prefixmap[key] + ":")
                        predicate = predicate.replace(key, prefixmap[key] + ":")
                    dotschema.node(
                        shape.id.split("/")[-1] + tc.valueExpr.datatype.split("/")[-1] + tc.predicate.split("/")[-1],
                        datatype, shape=symbol["datatype"])
                    dotschema.edge(shape.id.split("/")[-1],
                                   shape.id.split("/")[-1] + tc.valueExpr.datatype.split("/")[-1] +
                                   tc.predicate.split("/")[
                                       -1], label=predicate, arrowhead=arrowhead)
                elif tc.valueExpr.values:
                    oneofs = []
                    predicate = tc.predicate
                    for value in tc.valueExpr.values:
                        if isinstance(value, ObjectLiteral):
                            oneofs.append(value.value)
                        else:
                            for key in prefixmap.keys():
                                try:
                                    value = value.replace(key, prefixmap[key] + ":")
                                except:
                                    value = "a"
                            oneofs.append(value)
                    for key in prefixmap.keys():
                        predicate = predicate.replace(key, prefixmap[key] + ":")
                    dotschema.node(
                        shape.id.split("/")[-1] + "|".join(oneofs).replace(":", "") + tc.predicate.split("/")[-1],
                        "{" + "|".join(oneofs) + "}", shape=symbol["oneof"])
                    dotschema.edge(shape.id.split("/")[-1],
                                   shape.id.split("/")[-1] + "|".join(oneofs).replace(":", "") +
                                   tc.predicate.split("/")[
                                       -1], label=predicate)

                elif tc.valueExpr.nodeKind:
                    dotschema.node(tc.valueExpr.nodeKind, tc.valueExpr.nodeKind.split("/")[-1],
                                   shape=symbol[tc.valueExpr.nodeKind])
                    dotschema.edge(shape.id.split("/")[-1], tc.valueExpr.nodeKind.split("/")[-1],
                                   label=tc.predicate.split("/")[-1], arrowhead="teedot")
                elif tc.valueExpr.xone:

                    dotschema.node(tc.valueExpr.xone[0].identifier, tc.valueExpr.xone[0].identifier.split("/")[-1],
                                   shape=symbol["oneof"])
                    dotschema.edge(shape.id.split("/")[-1], tc.valueExpr.xone[0].identifier.split("/")[-1],
                                   label=tc.predicate.split("/")[-1], arrowhead="teedot")
                else:
                    pass
                    # print("No valueExpr")
            else:
                pass
                # print("No valueExpr")

    dotschema = graphviz.Digraph(graphviz_name, format="png")
    dotschema.attr(rankdir=rankdir)
    prefixmap = dict()
    for line in shex.splitlines():
        if line.startswith("PREFIX"):
            line = line.replace("PREFIX", "")
            prefix, uri = line.split(": ")
            prefix = prefix.strip()
            uri = uri.strip()
            prefixmap[uri.replace("<", "").replace(">", "")] = prefix
            dotschema.node(prefix, uri, shape="none", style="invis")

    loader = SchemaLoader()
    schema = loader.loads(shex)

    for shape in schema.shapes:
        if id in (dir(shape)):
            continue
        startshape = shape.id
        for key in prefixmap.keys():
            startshape = startshape.replace(key, prefixmap[key] + ":")
        if "expressions" in dir(shape.expression):
            for tc in shape.expression.expressions:
                process_tc(tc)
        else:
            tc = shape.expression
            process_tc(tc)

    return dotschema

def view_graphviz(dotschema):
    return dotschema.view()

def save_graphviz(dotschema, filename):
    return dotschema.render(filename)


from flask import Blueprint, redirect,request
from flask_login import current_user, login_user, logout_user, login_required
from app.models import db, Thread,User, Category, Post
from app.forms import ThreadForm
from .auth_routes import validation_errors_to_error_messages

post_routes = Blueprint('post', __name__)

@post_routes.route('/thread/<int:thread_id>', methods=['GET','POST'])
@login_required
def create_post(thread_id):
    user = User.query.get(current_user.id)

    form = ThreadForm()
    form['csrf_token'].data = request.cookies['csrf_token']

    if form.validate_on_submit():
        find_thread = Thread.query.get(thread_id)
        new_post = Post(
            subject=form.data["subject"],
            text=form.data["text"],
            user=user,
            thread=find_thread
            )
        db.session.add(new_post)
        db.session.commit()
        return new_post.to_dict()
    elif form.errors:
        return {'errors': validation_errors_to_error_messages(form.errors)}, 401

@post_routes.route('/<int:id>', methods=['GET','PUT'])
@login_required
def edit_post(id):
    post_to_edit = Post.query.get(id)

    form = ThreadForm()
    form['csrf_token'].data = request.cookies['csrf_token']
    print(form.data)

    if form.validate_on_submit():
        if form.data["subject"]:
            post_to_edit.subject = form.data["subject"]
        if form.data["text"]:
            post_to_edit.text = form.data["text"]

        db.session.commit()
        return post_to_edit.to_dict()
    elif form.errors:
        return {'errors': validation_errors_to_error_messages(form.errors)}, 401

@post_routes.route('/<int:id>', methods=['DELETE'])
@login_required
def delete_post(id):
    post = Post.query.get(id)
    if not post:
        return {'errors': "could not find post"}

    db.session.delete(post)
    db.session.commit()
    return {'success':'post was deleted'}


import collections
import os
import json
import ast
import string
import random
import copy
from json import JSONDecodeError
from os import path

class Database:
    db = None
    db_path = None
    db_collection = None
    db_default_path = 'jlod/databases'
    db_dictionary = None
    DBManager = None

    def __init__(self, db_path=db_default_path, dbi=db, collection=db_collection):
        if db_path.__eq__(self.db_default_path):
            self.db_path = self.db_default_path
        else:
            self.db_path = db_path
        self.db = dbi
        self.db_collection = collection
        self.DBManager = DatabaseManager(self.db_collection)
        self.db_dictionary = {
            "documents": [
            ]
        }

    @property
    def instance(self):
        return Database(self.db_path, self.db)

    def connect(self, db_name=None):
        db_connection = False
        if not str(db_name).endswith('.db'):
            self.db = self.db_path + '/' + db_name + '.db'

        try:
            if db_name is not None:
                if not os.path.exists(self.db):
                    os.makedirs(self.db)
                    console.log('Database [' + db_name + '] created at [' + self.db_path + ']')
                    db_connection = True
                else:
                    db_connection = True
            else:
                db_connection = False
                console.log('Oops! Unable to find database ' + db_name)

        except FileNotFoundError and OSError:
            console.log('Oops! Unable to initialize local database ' + db_name + ' at ' + self.db_path)
            return db_connection
        else:
            return Database(self.db_path, self.db)

    def collection(self, collection_name=""):
        if collection_name is not None:

            if not collection_name.endswith('.collection'):
                collection_name = collection_name + '.collection'

            col = self.db + '/' + collection_name
            self.db_collection = col
            open(col, 'a+')  # create collection

            if path.exists(col):
                return Database(self.db_path, self.db, col)
            else:
                console.log('Unable to create collection' + collection_name)
                return None
        else:
            print('Oops! No collection provided')

    def exportTo(self, mongo_collection: classmethod, conditions={}):
        self.DBManager = DatabaseManager(self.db_collection)
        try:
            import pymongo
            documents: dict
            if conditions.__len__() > 0:
                documents = self.DBManager.search(conditions)
            else:
                documents = self.DBManager.read()
            mongo_collection.insert_many(documents)
        except ImportError:
            console.log('Oops! Unable to find MangoDB')
            return False
        else:
            return True

    def importFrom(self, mongo_collection: classmethod, conditions: dict):
        self.DBManager = DatabaseManager(self.db_collection)
        doc: dict
        try:
            import pymongo
            documents: dict
            if conditions.__len__() > 0:
                documents = mongo_collection.find({}, conditions)
            else:
                documents = mongo_collection.find({}, conditions)
            for doc in documents:

                if '_id' in doc.keys():
                    doc.pop('_id')
                if 'id' in doc.keys():
                    doc.pop('id')
                if doc.__len__() > 0:
                    self.add(doc)
        except ImportError as error:
            console.log('Oop! Unable to find MangoDB')
            return [error]
        else:
            return self.DBManager.read()

    def override(self, obj: dict):
        document: dict
        if dict is not None:
            documentId = self.DBManager.generateId()
            obj.__setitem__('id', documentId)
            dictionary = self.db_dictionary.get('documents')
            dictionary.append(obj)
            self.DBManager.write(self.db_dictionary)
            return True
        else:
            print('No data found!')
            return False

    def add(self, obj: dict):
        document: dict
        if dict is not None:

            if os.path.getsize(self.db_collection) == 0:
                self.override(obj)
            else:
                document = self.DBManager.read()
                documentId = self.DBManager.generateId()
                obj.__setitem__('id', documentId)
                document.append(obj)
                dictionary = {
                    'documents': document
                }
                self.DBManager.write(dictionary)
            return True
        else:
            return False

    def addMany(self, documents: []):
        try:
            if documents.__len__() > 0:
                for document in documents:
                    self.add(document)
            console.log(str(len(documents)) + ' documents added')
        except ValueError as ex:
            return False
        else:
            return True

    @property
    def documents(self):
        document: dict
        document = self.DBManager.read()
        return document

    @property
    def count(self):
        document: dict
        document = self.DBManager.read()
        return document.__len__()

    @property
    def truncate(self):
        self.DBManager.truncate()
        return True

    @property
    def drop(self):
        try:
            os.remove(self.db_collection)
        except FileNotFoundError:
            print("Collection " + self.db_collection + "  not found ")
        else:
            return True

    @property
    def size(self):
        self.DBManager = DatabaseManager(self.db_collection)
        return self.DBManager.getSize()

    def update(self, keyAndReplacement: dict, conditions=None):
        if conditions is None:
            document = self.DBManager.read()
        else:
            document = self.DBManager.search(conditions)

        self.DBManager.update(document, keyAndReplacement)

    def removeAll(self):
        self.DBManager.truncate()

    def sort(self, sorts=None, conditions=None, limit=0):
        self.DBManager = DatabaseManager(self.db_collection)
        if conditions is None and sorts is None:
            documents = self.DBManager.read()
        else:
            if conditions is not None and conditions.__len__() > 0:
                documents = self.DBManager.search(conditions)
            else:
                documents = self.DBManager.read()

            if limit > 0:
                documents = self.distinct(documents, limit)
            else:
                if conditions:
                    documents = self.DBManager.search(conditions)

            if sorts.__len__() > 0:
                for key, val in sorts.items():
                    keyName = key
                    if val.__eq__(-1):
                        reverseFlag = False
                    else:
                        reverseFlag = True
                    documents.sort(key=lambda x: x.__getitem__(
                        keyName), reverse=reverseFlag)
        return documents

    def remove(self, conditions=None):
        if conditions is None:
            document = self.DBManager.read()
        else:
            document = self.DBManager.search(conditions)

        if conditions is None:
            return self.truncate
        else:
            return self.DBManager.remove(document)

    def getOne(self, keyList=None, conditions=None):
        resultSet = []
        if conditions is None:
            documents = self.DBManager.read()
        else:
            documents = self.DBManager.search(conditions)

        if documents.__len__() > 0:
            resultSet.append(documents[0])

        return self.DBManager.get(resultSet, keyList)

    def get(self, keyList=None, conditions=None):
        if conditions is None:
            document = self.DBManager.read()
        else:
            document = self.DBManager.search(conditions)

        return self.DBManager.get(document, keyList)

    def findOne(self, conditions=None):
        self.DBManager = DatabaseManager(self.db_collection)
        resultSet = []
        if conditions is not None:
            documents = self.DBManager.search(conditions)
        else:
            documents = self.DBManager.read()

        if documents.__len__() > 0:
            resultSet.append(documents[0])
        return resultSet

    def distinct(self, conditions=None, limit=0):
        keylookup = ""
        resultSet = []
        key_checker = {}
        already_added = []

        if isinstance(conditions, dict):
            for key, vlaue in conditions.items():
                valuelookup = vlaue
                keylookup = key
        else:
            valuelookup = keylookup = conditions

        if conditions is not None and conditions.__len__() > 0:
            document = self.DBManager.read()
            for doc in document:
                params = doc[keylookup]
                name = doc[keylookup]
                doc_val = doc[keylookup]
                if params in key_checker:
                    key_checker[params] = doc_val
                elif name not in key_checker:
                    key_checker[params] = doc_val
            for doc in document:
                name = doc[keylookup]
                doc_val = doc[keylookup]
                if (key_checker[name] == doc_val) and (name not in already_added):
                    resultSet.append(doc)
                    already_added.append(name)
        if limit > 0:
            resultSet = distinct(resultSet, limit)
        return resultSet

    def find(self, conditions=None, limit=0):
        self.DBManager = DatabaseManager(self.db_collection)
        if conditions is None and limit == 0:
            documents = self.DBManager.read()
            return documents
        else:
            if conditions is not None and conditions.__len__() > 0:
                documents = self.DBManager.search(conditions)
            else:
                documents = self.DBManager.read()
            if limit > 0:

                return documents
            else:
                if conditions:
                    return self.DBManager.search(conditions)
                else:
                    return documents

class DatabaseManager:
    db_collection = None

    def __init__(self, collection=db_collection):
        self.db_collection = collection

    def generateId(self):
        return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(30))

    def getSize(self):
        st = os.stat(self.db_collection)
        return st.st_size

    def truncate(self):
        collection = open(self.db_collection, 'r+')
        collection.truncate()
        collection.close()

    def write(self, document: dict):
        collectionWriter = open(self.db_collection, 'w')
        collectionWriter.write(str(document))
        collectionWriter.close()

    def read(self):
        try:
            collectionReader = open(self.db_collection, 'r')
            document = collectionReader.read()
            if document.__ne__(''):
                document = document.replace("'", "\"")
                document = json.loads(document)
            else:
                document = {
                    "documents": [
                    ]
                }
        except JSONDecodeError as error:
            return [error]
        else:
            return document.get('documents')

    def get(self, document: dict, keys=None):

        resultSet = []
        if keys is not None:
            if keys.__ne__("*"):
                for doc in document:
                    localDict = {
                        "$~": "init"
                    }
                    for key in keys:
                        if key in doc.keys():
                            if type(doc[key]) is string:
                                localDict.update({'' + key + '': '' + doc[key] + ''})
                            else:
                                localDict.update({'' + key + '':doc[key]})
                    localDict.pop("$~")
                    resultSet.append(localDict)
            else:
                resultSet = document
            return resultSet
        else:
            return document

    def extract(self, doc: dict, key: str, operator: str, value):
        resultSet = None
        if operator.__eq__("$eq"):
            if str(doc[key]).lower() == str(value).lower():
                resultSet = doc
        elif operator.__eq__("$ne"):
            if str(doc[key]).lower() != str(value).lower():
                resultSet = doc
        elif operator.__eq__("$gt"):
            if int(str(doc[key]).lower()) > int(str(value).lower()):
                resultSet = doc
        elif operator.__eq__("$gte"):
            if int(str(doc[key]).lower()) >= int(str(value).lower()):
                resultSet = doc
        elif operator.__eq__("$lt"):
            if int(str(doc[key]).lower()) < int(str(value).lower()):
                resultSet = doc
        elif operator.__eq__("$lte"):
            if int(str(doc[key]).lower()) <= int(str(value).lower()):
                resultSet = doc
        elif operator.__eq__("$in"):
            if str(doc[key]).lower() == str(value).lower():
                resultSet = doc

        elif operator.__eq__("$nin"):
            if str(doc[key]).lower() != str(value).lower():
                resultSet = doc

        return resultSet

    def query(self, documents: dict, logicKey: str, conditions: dict):
        resultSet = []
        if type(conditions) is not dict and isinstance(conditions, collections.Sequence):
            if logicKey.__eq__("$or"):
                for cond in conditions:
                    for citems in cond.items():
                        key = citems[0]
                        items = citems[1]

                        for citems in items.items():
                            operator = citems[0]
                            value = citems[1]
                            for doc in documents:
                                if key in doc:
                                    result = self.extract(doc, key, operator, value)
                                    if result is not None:
                                        resultSet.append(result)
            elif logicKey.__eq__("$and"):
                for doc in documents:
                    query = []
                    for cond in conditions:
                        for citems in cond.items():
                            key = citems[0]
                            items = citems[1]
                            for citems in items.items():
                                operator = citems[0]
                                value = citems[1]

                                if key in doc:
                                    result = self.extract(doc, key, operator, value)
                                    if result is not None:
                                        query.append(1)

                    if query.__len__() == conditions.__len__():
                        if doc not in resultSet:
                            resultSet.append(doc)
        else:
            for cond in conditions.items():
                key = str(cond[0])
                items = cond[1].items()
                for citems in items:
                    operator = citems[0]
                    value = citems[1]
                    for doc in documents:
                        if key in doc:
                            result = self.extract(doc, key, operator, value)
                            if result is not None:
                                resultSet.append(result)
        return resultSet

    def search(self, conditions: dict):
        documents = self.read()
        resultSet = []

        if type(conditions) is list:
            for condition in conditions:
                for cond in condition.items():
                    key = str(cond[0]).lower()
                    items = cond[1]
                    for citems in items.items():
                        operator = citems[0]
                        value = citems[1]
                        for doc in documents:
                            result = self.extract(doc, key, operator, value)
                            if result is not None:
                                if result not in resultSet:
                                    resultSet.append(result)

        elif type(conditions) is dict:
            for cond in conditions.items():
                key = str(cond[0]).lower()
                value = cond[1]

                if key.startswith("$"):
                    if type(value) is list:
                        for cdoc in value:
                            for condoc in cdoc.items():
                                key = str(condoc[0]).lower()
                                items = condoc[1]
                                for citems in items.items():
                                    operator = citems[0]
                                    value = citems[1]
                                    for doc in documents:
                                        result = self.extract(doc, key, operator, value)
                                        if result is not None:
                                            if result not in resultSet:
                                                resultSet.append(result)
                    else:
                        if self.query(documents, key, value).__len__() > 0:
                            resultSet = self.query(documents, key, value)[:]
                else:

                    if type(value) is dict:
                        for citems in value.items():
                            operator = citems[0]
                            value = citems[1]
                            for doc in documents:
                                if type(value) is not str and isinstance(value, collections.Sequence):
                                    for each in value:
                                        result = self.extract(doc, key, operator, each)
                                        if result is not None:
                                            if str(operator) == "$nin":
                                                if result not in resultSet:
                                                    if result[key] not in value:
                                                        lowerResult = map(str, value)
                                                        lowerResult = map(str.lower, lowerResult)
                                                        if str(result[key]).lower() not in lowerResult:
                                                            resultSet.append(result)
                                            else:
                                                if result not in resultSet:
                                                    resultSet.append(result)
                                else:
                                    result = self.extract(doc, key, operator, value)
                                    if result is not None:
                                        if result not in resultSet:
                                            resultSet.append(result)

                    else:
                        for doc in documents:
                            for cond in conditions.items():
                                key = str(cond[0])
                                value = str(cond[1])
                                if str(value).__contains__('[') and str(value).__contains__(']'):
                                    values = ast.literal_eval(value)
                                    for val in values:
                                        if key in doc:
                                            if str(doc[key]).lower() == str(val).lower():
                                                if doc not in resultSet:
                                                    resultSet.append(doc)
                                else:
                                    if str(doc[key]).lower() == str(value).lower():
                                        if doc not in resultSet:
                                            resultSet.append(doc)
        return resultSet

    def update(self, document: dict, keyAndReplacement: dict):
        if keyAndReplacement.__len__() > 0 and document.__len__() > 0:
            oldDocument = copy.deepcopy(document)
            for doc in document:
                for sar in keyAndReplacement.items():
                    key = sar[0]
                    value = sar[1]
                    if key in doc:
                        doc[key] = value
                    else:
                        doc.__setitem__(key, value)

            base = self.read()
            for oldDoc in oldDocument:
                base = [x for x in base if x != oldDoc]

            for newDoc in document:
                if newDoc is not None:
                    base.append(newDoc)

            dictionary = {
                'documents': base
            }
            self.write(dictionary)
            return True
        else:
            return False

    def remove(self, document: dict):

        base = self.read()
        for doc in document:
            base = [x for x in base if x != doc]

        dictionary = {
            'documents': base
        }
        self.write(dictionary)
        return True

class Console:
    def log(self, msg):
        print(msg)

console = Console()
database = Database()


from data.data import DatasetPlaceholder, chose_dataset_placeholder, DataInfo
from util.show_frames import show_frames

def show_dataset(args):
    placeholders = DatasetPlaceholder.list_database(args.database_directory)
    joined_dataset_infos = DataInfo.join(map(lambda ds_placeholder: ds_placeholder.data_info, placeholders))
    print('num samples: {}'.format(joined_dataset_infos.num_samples))
    dataset_placeholder = chose_dataset_placeholder(placeholders)

    dataset = dataset_placeholder.load()

    show_frames(dataset)


# -*- coding: utf-8 -*-
"""
Created on Fri Aug 30 13:55:31 2019

@author: espenfb
"""

import metaModel as mm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import metaRes as mr

idx = pd.IndexSlice

time_data = {
'start_date': pd.Timestamp(year = 2015, month = 1, day = 1),
'end_date': pd.Timestamp(year = 2016, month = 1, day = 1),
'ref_date': pd.Timestamp(year = 2015, month = 1, day = 1)}

dirs = {
'data_dir' : "Data\\",
'ctrl_data_file' : 'ctrl_data.csv',
#'res_dir' : 'NEEDS2\\Meta_results_100x\\'}
'res_dir' : 'Result_3\\Year\\'}

meta_data = {
'param': 'CO2_cost',
 'index': 'None',
 'kind': 'absolute',
 'range': np.arange(0.00,0.29,0.03)}

obj = mm.metaModel(time_data, dirs, meta_data)

obj.runMetaModel()

# Examples of result plotting:
# see file: savedRes.py for more information

plt.close('all') # close all plots

# Plot meta model results:

pt = 'bar'

if True: obj.meta_res = mr.metaRes(obj.res_dir, obj.meta_data, data = obj.model.data)

obj.meta_res.plotValueByType('prod', objects = 'POWER_PLANTS', kind = 'area',
                    lower_lim = 1000000)
obj.meta_res.plotValueByType('prod', objects = 'H2_PLANTS', kind = 'bar', xscale = 1000)

# Plot results for individual scenario:

res = obj.meta_res.res[-1] # results for the first parameter run of meta model

obj.meta_res.plotValueByType('cur', kind = 'bar', base_val = 'Load',
                    base_obj = 'EL_NODES', xscale=1000)

obj.meta_res.plotInvByType(objects = 'STORAGE', xscale = 1000,
                  lower_lim = 0.0, conH2power = True)

#res.plotEnergyByType()
res.getValueByType('prod',
                   objects = 'POWER_PLANTS',
                   lower_lim = 100).plot(kind = 'area',
                                        cmap = 'tab20c')

#res.plotInvByBus(lower_limit = 100.0)

res.plotValue('storage', objects= 'BATTERY_STORAGE')
res.plotValue('storage', objects= 'HYDROGEN_STORAGE')

mkr_scale = 500

res.plotMap(linetype = 'Cap', objects = 'BATTERY_STORAGE', mkr_scaling= 100, rel_lines = True)
res.plotMap(linetype = 'Cap', nodes = 'H2_NODES',
            objects = 'HYDROGEN_STORAGE',
            nodetype= 'energy', mkr_scaling= mkr_scale, line_lim = 1000)

res.plotMap(objects = 'PEMEL_PLANTS', nodetype = 'power', mkr_scaling= mkr_scale)

res.plotMap(objects = 'ONSHORE_WIND_POWER_PLANTS', nodetype = 'power', mkr_scaling= mkr_scale)
res.plotMap(objects = 'SOLAR_POWER_PLANTS', nodetype = 'power', mkr_scaling= mkr_scale)
res.plotMap(objects = 'SMR_PLANTS', nodetype = 'power', mkr_scaling= mkr_scale)
res.plotMap(objects = 'SMR_CCS_PLANTS', nodetype = 'power', mkr_scaling= mkr_scale)

total_load = res.opr_res.loc[idx[:],idx[res.EL_NODES,'Load']].sum().sum()
total_rat = res.opr_res.loc[idx[:],idx[res.EL_NODES,'rat']].sum().sum()
print('Total rationing: ','%.1f' % total_rat, ' MWh (%.2f percent of total load)' % ((total_rat/total_load)*100))
total_h2_load = res.opr_res.loc[idx[:],idx[res.H2_NODES,'Load']].sum().sum()
total_h2_rat = res.opr_res.loc[idx[:],idx[res.H2_NODES,'rat']].sum().sum()
print('Total H2 rationing: ','%.1f' % total_h2_rat, ' MWh (%.2f percent of total load)' % ((total_rat/total_load)*100))
wind_cur = res.opr_res.loc[idx[:],idx[res.WIND_POWER_PLANTS,'cur']].sum().sum()
print('Total wind power curtailment: ','%.1f' % wind_cur, ' MWh (%.2f percent of total load)' % ((wind_cur/total_load)*100))
solar_cur = res.opr_res.loc[idx[:],idx[res.SOLAR_POWER_PLANTS,'cur']].sum().sum()
print('Total solar power curtailment: ','%.1f' % solar_cur, ' MWh (%.2f percent of total load)' % ((solar_cur/total_load)*100))


#Jaspal Bainiwal
#Script to check DeAnza college in Cupertino, CA class availability
#class search url follows the following format
#"https://www.deanza.edu/schedule/listings.html?dept="+dept+"&t="+term

import bs4 as bsoup
import requests
import time
import api_keys as key

print("Welcome to course waitlist watcher")
dept = input("Enter department id: ").upper()
term = input("Enter quarter term: ").upper()
crn = input("Enter course number to watch: ")
class_open = False

def email_event(course, crn, prof):
  event = key.EVENT_NAME
  ifttt_key = key.KEY
  payload = {"value1":course,"value2":crn,"value3":prof}
  r = requests.post('https://maker.ifttt.com/trigger/' + event + '/with/key/' + ifttt_key, json=payload)
  print("Email sent")
  return

def class_checker():
  url = "https://www.deanza.edu/schedule/listings.html?dept="+dept+"&t="+term
  response = requests.get(url)
  soup = bsoup.BeautifulSoup(response.text, "html.parser")
  t_body = soup.find("tbody")
  tr = t_body.find_all("tr")

  for x in tr:
    td = x.find_all('td')
    if (td[0].get_text()) == crn:
      #if this is the correct course number
      if (td[3].get_text().upper()) == 'OPEN':
        course_name = td[4].find("a", href=True).get_text()
        prof_name = td[7].get_text()
        email_event(course_name, crn, prof_name)
        class_open = True
        return class_open

while class_open != True:
  class_open = class_checker()
  time.sleep(3600)


# pylint: disable=missing-function-docstring
import math
from typing import Tuple
import pytest
import torch
from pycave.bayes import MarkovChain

def test_fit_automatic_config():
    chain = MarkovChain()
    data = torch.randint(50, size=(100, 20))
    chain.fit(data)
    assert chain.model_.config.num_states == 50

@pytest.mark.flaky(max_runs=3, min_passes=1)
def test_sample_and_fit():
    chain = MarkovChain(2)
    initial_probs, transition_probs = _set_probs(chain)
    sample = chain.sample(1000000, 10)

    new = MarkovChain(2)
    new.fit(sample)

    assert torch.allclose(initial_probs, new.model_.initial_probs, atol=1e-3)
    assert torch.allclose(transition_probs, new.model_.transition_probs, atol=1e-3)

def test_score():
    chain = MarkovChain(2)
    test_data, expected = _set_sample_data(chain)
    actual = chain.score(test_data)
    assert math.isclose(actual, -expected.mean())

def test_score_samples():
    chain = MarkovChain(2)
    test_data, expected = _set_sample_data(chain)
    actual = chain.score_samples(test_data)
    assert torch.allclose(actual, -expected)

# -------------------------------------------------------------------------------------------------

def _set_probs(chain: MarkovChain) -> Tuple[torch.Tensor, torch.Tensor]:
    data = torch.randint(2, size=(100, 20))
    chain.fit(data)

    initial_probs = torch.as_tensor([0.8, 0.2])
    chain.model_.initial_probs.copy_(initial_probs)

    transition_probs = torch.as_tensor([[0.5, 0.5], [0.1, 0.9]])
    chain.model_.transition_probs.copy_(transition_probs)

    return initial_probs, transition_probs

def _set_sample_data(chain: MarkovChain) -> Tuple[torch.Tensor, torch.Tensor]:
    _set_probs(chain)

    test_data = torch.as_tensor(
        [
            [1, 1, 0, 1],
            [0, 1, 0, 1],
            [0, 0, 1, 1],
        ]
    )
    expected = torch.as_tensor(
        [
            math.log(0.2) + math.log(0.9) + math.log(0.1) + math.log(0.5),
            math.log(0.8) + math.log(0.5) + math.log(0.1) + math.log(0.5),
            math.log(0.8) + math.log(0.5) + math.log(0.5) + math.log(0.9),
        ]
    )
    return test_data, expected


'''
Program Name: load_to_METviewer_AWS.py
Contact(s): Mallory Row
Abstract: This is run at the end of all step1 scripts
          in scripts/.
          This scripts loads data to the METviewer AWS
          server.
              1) Create a temporary directory and
                 link the files that are to be
                 loaded
              2) Create XML that will load files
              3) Create database on AWS server
              4) Listing of METviewer datbases
'''

import datetime
import os
import subprocess

print("BEGIN: "+os.path.basename(__file__))

# Read in environment variables
KEEPDATA = os.environ['KEEPDATA']
machine = os.environ['machine']
DATA = os.environ['DATA']
NET = os.environ['NET']
RUN = os.environ['RUN']
RUN_abbrev = os.environ['RUN_abbrev']
RUN_type_list = os.environ[RUN_abbrev+'_type_list'].split(' ')
USHverif_global = os.environ['USHverif_global']
QUEUESERV = os.environ['QUEUESERV']
ACCOUNT = os.environ['ACCOUNT']
PARTITION_BATCH = os.environ['PARTITION_BATCH']
MET_version = os.environ['MET_version']
model_list = os.environ['model_list'].split(' ')
METviewer_AWS_scripts_dir = os.environ['METviewer_AWS_scripts_dir']
mv_database = os.environ[RUN_abbrev+'_mv_database_name']
mv_group = os.environ[RUN_abbrev+'_mv_database_group']
mv_desc = os.environ[RUN_abbrev+'_mv_database_desc']

# Set up walltime
transfer_walltime = '180'
walltime_seconds = datetime.timedelta(minutes=int(transfer_walltime)) \
        .total_seconds()
walltime = (datetime.datetime.min
           + datetime.timedelta(minutes=int(transfer_walltime))).time()

# Check current databases to see if it exists
current_database_info = subprocess.check_output(
    os.path.join(METviewer_AWS_scripts_dir, 'mv_db_size_on_aws.sh')+' '
    +os.environ['USER'].lower(), shell=True, encoding='UTF-8'
)
if mv_database in current_database_info:
    new_or_add = 'add'
else:
    new_or_add = 'new'

# Create linking file dir
link_file_dir = os.path.join(os.getcwd(), 'METviewer_AWS_files')
os.makedirs(link_file_dir, mode=0o755)

# Create load XML
load_xml_file = os.path.join(os.getcwd(), 'load_'+mv_database+'.xml')
print("Creating load xml "+load_xml_file)
if new_or_add == 'new':
    drop_index = 'false'
else:
    drop_index = 'true'
if os.path.exists(load_xml_file):
    os.remove(load_xml_file)
with open(load_xml_file, 'a') as xml:
    xml.write('<load_spec>\n')
    xml.write('  <connection>\n')
    xml.write('    <host>metviewer-dev-2-cluster.cluster-c0bl5kb6fffo.'
              +'us-east-1.rds.amazonaws.com:3306</host>\n')
    xml.write('    <database>'+mv_database+'</database>\n')
    xml.write('    <user>rds_user</user>\n')
    xml.write('    <password>rds_pwd</password>\n')
    xml.write('    <management_system>aurora</management_system>\n')
    xml.write('  </connection>\n')
    xml.write('\n')
    xml.write('  <met_version>V'+MET_version+'</met_version>\n')
    xml.write('\n')
    xml.write('  <verbose>true</verbose>\n')
    xml.write('  <insert_size>1</insert_size>\n')
    xml.write('  <mode_header_db_check>true</mode_header_db_check>\n')
    xml.write('  <stat_header_db_check>true</stat_header_db_check>\n')
    xml.write('  <drop_indexes>false</drop_indexes>\n')
    xml.write('  <apply_indexes>false</apply_indexes>\n')
    xml.write('  <load_stat>true</load_stat>\n')
    xml.write('  <load_mode>true</load_mode>\n')
    xml.write('  <load_mpr>true</load_mpr>\n')
    xml.write('  <load_orank>true</load_orank>\n')
    xml.write('  <force_dup_file>false</force_dup_file>\n')
    xml.write('  <group>'+mv_group+'</group>\n')
    xml.write('  <description>'+mv_desc+'</description>\n')
    xml.write('  <load_files>\n')
for RUN_type in RUN_type_list:
    gather_by = os.environ[RUN_abbrev+'_'+RUN_type+'_gather_by']
    for model in model_list:
        gather_by_RUN_type_model_dir = os.path.join(
            DATA, RUN, 'metplus_output', 'gather_by_'+gather_by,
            'stat_analysis', RUN_type, model
        )
        for file_name in os.listdir(gather_by_RUN_type_model_dir):
            os.link(
                os.path.join(gather_by_RUN_type_model_dir, file_name),
                os.path.join(link_file_dir, RUN_type+'_'+file_name)
            )
            with open(load_xml_file, 'a') as xml:
                xml.write('    <file>/base_dir/'
                          +RUN_type+'_'+file_name+'</file>\n')
with open(load_xml_file, 'a') as xml:
    xml.write('  </load_files>\n')
    xml.write('\n')
    xml.write('</load_spec>')

# Create job card file for:
#   Create database if needed and load data
#   mv_create_db_on_aws.sh agruments:
#      1 - username
#      2 - database name
#   mv_load_to_aws.sh agruments:
#      1 - username
#      2 - base dir
#      3 - XML file
#      4 (opt) - sub dir
AWS_job_filename = os.path.join(DATA, 'batch_jobs',
                                NET+'_'+RUN+'_load2METviewerAWS.sh')
with open(AWS_job_filename, 'a') as AWS_job_file:
    AWS_job_file.write('#!/bin/sh'+'\n')
    AWS_job_file.write('set -x'+'\n')
    if machine == 'WCOSS2':
        AWS_job_file.write('cd $PBS_O_WORKDIR\n')
    if new_or_add == 'new':
        AWS_job_file.write('echo "Creating database on METviewer AWS using '
                           +os.path.join(METviewer_AWS_scripts_dir,
                                         'mv_create_db_on_aws.sh')
                           +'"\n')
        AWS_job_file.write(
            os.path.join(METviewer_AWS_scripts_dir,
                         'mv_create_db_on_aws.sh')+' '
            +os.environ['USER'].lower()+' '
            +mv_database+'\n'
        )
    AWS_job_file.write('echo "Loading data to METviewer AWS using '
                       +os.path.join(METviewer_AWS_scripts_dir,
                                     'mv_load_to_aws.sh')
                       +'"\n')
    AWS_job_file.write(
        os.path.join(METviewer_AWS_scripts_dir, 'mv_load_to_aws.sh')+' '
        +os.environ['USER'].lower()+' '
        +link_file_dir+' '
        +load_xml_file+'\n'
    )
    AWS_job_file.write('echo "Check METviewer AWS database list using '
                       +os.path.join(METviewer_AWS_scripts_dir,
                                     'mv_db_size_on_aws.sh')
                      +'"\n')
    AWS_job_file.write(
        os.path.join(METviewer_AWS_scripts_dir, 'mv_db_size_on_aws.sh')+' '
        +os.environ['USER'].lower()
    )
    if KEEPDATA == 'NO':
        AWS_job_file.write('\n')
        AWS_job_file.write('cd ..\n')
        AWS_job_file.write('rm -rf '+RUN)

# Submit job card
os.chmod(AWS_job_filename, 0o755)
AWS_job_output = AWS_job_filename.replace('.sh', '.out')
AWS_job_name = AWS_job_filename.rpartition('/')[2].replace('.sh', '')
print("Submitting "+AWS_job_filename+" to "+QUEUESERV)
print("Output sent to "+AWS_job_output)
if machine == 'WCOSS2':
    os.system('qsub -V -l walltime='+walltime.strftime('%H:%M:%S')+' '
              +'-q '+QUEUESERV+' -A '+ACCOUNT+' -o '+AWS_job_output+' '
              +'-e '+AWS_job_output+' -N '+AWS_job_name+' '
              +'-l select=1:ncpus=1 '+AWS_job_filename)
elif machine in ['HERA', 'ORION', 'S4', 'JET']:
    os.system('sbatch --ntasks=1 --time='+walltime.strftime('%H:%M:%S')+' '
              +'--partition='+QUEUESERV+' --account='+ACCOUNT+' '
              +'--output='+AWS_job_output+' '
              +'--job-name='+AWS_job_name+' '+AWS_job_filename)
print("END: "+os.path.basename(__file__))


# Makes a new page for tracking items
import jinja_env
import logging
import webapp2
import time
from google.appengine.ext import ndb
from models import money_model
from google.appengine.api import users
class TrackItemHandler(webapp2.RequestHandler):
	def get(self):
		logging.info("TrackItemHandler")

		user=users.get_current_user()
		track = money_model.moneyModel.query(money_model.moneyModel.user_email == user.email()).get()

		if track == None:
			self.redirect("/second")
			return
		itemKey=track.key.urlsafe()

		html_params = {
			"title": "Tracked Item List",
			"content": "Selected Items Listed Below:",
			"totalCost": track.price,
			"modelKey": itemKey,
			"current": track.currentSavings,

		}

		template = jinja_env.env.get_template('templates/Track.html')
		self.response.out.write(template.render(html_params))

#Get information for comments and post of images
	def post(self):
		logging.info("USER SAID POST")
		# r_user = self.request.get("usrform")
		# r_form = self.request.get("fileToUpload")
		# r_comit = self.request.get("comment")
		r_currentSavings = self.request.get("form_currentSavings")
		modelKeyString = self.request.get("modelKey")
		modelKey = ndb.Key(urlsafe=modelKeyString)
		track = modelKey.get()

		track.currentSavings = float(r_currentSavings)
		track.put()
		time.sleep(1)

		# TrackItem = Save_Model. SaveModel(
		# 	username= r_user,)
		self.redirect("/progress")


from matplotlib import pyplot as plt
import math
import numpy as np
from scipy.stats import norm

def f():
    #change path as required
    # with open("C://Users//sapam//Desktop//PYPY//m1" + sign + "m2//v1.txt", "r") as v_1:
    #     data_v1 = [float(x) for x in v_1.read().split("\n")]
    # with open("C://Users//sapam//Desktop//PYPY//m1" + sign + "m2//vd.txt", "r") as v_d:
    #     data_vd = [float(x) for x in v_d.read().split("\n")]
    # res = [i / j for i, j in zip(data_vd, data_v1)]

    with open("C://Users//sapam//Desktop//PYPY//_physics_//fall22_lab//BP//y.txt", "r") as v_1:
        res = [float(x) for x in v_1.read().split("\n")]
    print(sorted(res))
    #res = [x+ 255.5 for x in res]

    def avg(ls):
        n = len(ls)
        if n <= 1:
            return ls[0]
        return sum(ls)/float(n)

    def std(data):
        n = len(data)
        if n <= 1:
            return 0.0
        sd = 0.0
        for el in data:
            sd += (float(el) - avg(data))**2
        return math.sqrt(sd / float(n-1))

    print()
    print(chr(92)+"item", "Average of data is " + str(round(avg(res),3)))
    print(chr(92)+"item", "SD of data is " + str(round(std(res),3)))

    outc, outc2 = 0, 0
    av, sd = avg(res), std(res)
    for r in res:
        if r < av-sd or r > av+sd:
            outc += 1
            if r < av-2*sd or r > av+2*sd:
                outc2 += 1

    #print(outc, outc2, len(res))
    print(chr(92)+"item", "Fraction Data outside average +/- SD: " + str(round(outc/len(res),3)))
    print(chr(92)+"item", "Fraction Data outside average +/- 2xSD: " + str(round(outc2/len(res),3)))

    # a = np.array(res)
    # fig, ax = plt.subplots(figsize =(10, 7))
    data = np.array(res)

    mu, std = norm.fit(data)
    # We can also use our mean and sd

    # Plot the histogram.
    plt.hist(data, bins=5, density=False, alpha=1, color='b')

    # Plot the PDF.
    xmin, xmax = plt.xlim()
    x = np.linspace(xmin, xmax, 100)
    p = 25*norm.pdf(x, mu, std)

    plt.plot(x, p, 'k', linewidth=2)
    #ax.hist(a, bins = list(range(255, 271, 2)))
    #ax.hist(a, bins = [0.43,0.44, 0.45, 0.46, 0.47, 0.48, 0.49, 0.5])
    #ax.hist(a, bins = [0.47, 0.48, 0.49, 0.50, 0.51, 0.52, 0.53, 0.54])
    plt.show()

f()


# !/usr/bin/env python
# -*-coding:utf-8 -*-

"""
# File       : app.py
# Time       ：2021/11/11 15:26
# Author     ：jiewei_yang
# version    ：python 3.7.9
# Description：
"""

from flask import Flask, render_template, jsonify, request
from icecream import ic

app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False

@app.route('/')
def index():
    # return render_template('layui_jquery_button.html') ## 增加了表格数据的异步加载
    return render_template('layui_jquery_form.html')  # form表单的异步加载数据

# /table/data?page=1&limit=10
@app.route('/table/data', methods=['GET'])
def login():
    page = request.args.get("page")
    limit = request.args.get("limit")
    ic(page, limit)
    data = {"code": 0, "msg": "", "count": 30, "data":
        [{"id": 10000, "username": "user-0", "sex": "女", "city": "城市-0", "sign": "签名-0", "experience": 255,
          "logins": 24, "wealth": 82830700, "classify": "作家", "score": 57},
         {"id": 10001, "username": "user-1", "sex": "男", "city": "城市-1", "sign": "签名-1", "experience": 884,
          "logins": 58, "wealth": 64928690, "classify": "词人", "score": 27},
         {"id": 10002, "username": "user-2", "sex": "女", "city": "城市-2", "sign": "签名-2", "experience": 650,
          "logins": 77, "wealth": 6298078, "classify": "酱油", "score": 31},
         {"id": 10003, "username": "user-3", "sex": "女", "city": "城市-3", "sign": "签名-3", "experience": 362,
          "logins": 157, "wealth": 37117017, "classify": "诗人", "score": 68},
         {"id": 10004, "username": "user-4", "sex": "男", "city": "城市-4", "sign": "签名-4", "experience": 807,
          "logins": 51, "wealth": 76263262, "classify": "作家", "score": 6},
         {"id": 10005, "username": "user-5", "sex": "女", "city": "城市-5", "sign": "签名-5", "experience": 173,
          "logins": 68, "wealth": 60344147, "classify": "作家", "score": 87},
         {"id": 10006, "username": "user-6", "sex": "女", "city": "城市-6", "sign": "签名-6", "experience": 982,
          "logins": 37, "wealth": 57768166, "classify": "作家", "score": 34},
         {"id": 10007, "username": "user-7", "sex": "男", "city": "城市-7", "sign": "签名-7", "experience": 727,
          "logins": 150, "wealth": 82030578, "classify": "作家", "score": 28},
         {"id": 10008, "username": "user-8", "sex": "男", "city": "城市-8", "sign": "签名-8", "experience": 951,
          "logins": 133, "wealth": 16503371, "classify": "词人", "score": 14}, ]}

    return jsonify(data)

import json
@app.route('/table/form', methods=[  'POST'])
def form():
    # 通过 ajax 进行数据的传递
    data = json.loads(request.form.get('data_front'))
    print(data)
    return jsonify({'front_return':'后端返回的数据'})

if __name__ == '__main__':
    app.run(debug=True)


from PySide6.QtWidgets import *
from PySide6 import QtGui
from uiScripts.ui_EditVM import Ui_Dialog
import sqlite3
import platform

if platform.system() == "Windows":
    import platformSpecific.windowsSpecific

else:
    import platformSpecific.unixSpecific

import subprocess
from dialogExecution.vmExistsDialog import VmAlreadyExistsDialog
import translations.de
import translations.uk
import translations.en
import translations.fr
import translations.es
import translations.ro
import translations.be
import translations.cz
import translations.ru
import translations.pt
import translations.it
import locale

class EditVMNewDialog(QDialog, Ui_Dialog):
    def __init__(self, parent=None):
        try:
            super().__init__(parent)

        except:
            super().__init__()

        self.setupUi(self)
        self.connectSignalsSlots()
        self.tabWidget.setCurrentIndex(0)
        self.langDetect()
        self.vmSpecs = self.readTempVmFile()

        try:
            self.setWindowIcon(QtGui.QIcon("EmuGUI.png"))

        except:
            pass

    def connectSignalsSlots(self):
        self.pushButton.clicked.connect(self.close)
        self.pushButton_2.clicked.connect(self.finishCreation)
        self.comboBox_2.currentTextChanged.connect(self.vhdAddingChange)
        self.comboBox.currentTextChanged.connect(self.archChanged)
        self.pushButton_3.clicked.connect(self.vhdBrowseLocation)
        self.pushButton_4.clicked.connect(self.extBiosFileLocation)
        self.pushButton_5.clicked.connect(self.linuxKernelBrowseLocation)
        self.pushButton_6.clicked.connect(self.linuxInitridBrowseLocation)

        # For new and existing
        self.lineEdit_2.setEnabled(True)
        self.pushButton_3.setEnabled(True)

        # For new
        self.comboBox_3.setEnabled(False)
        self.spinBox.setEnabled(False)
        self.comboBox_4.setEnabled(False)

    def langDetect(self):
        select_language = """
        SELECT name, value FROM settings
        WHERE name = "lang";
        """

        if platform.system() == "Windows":
            connection = platformSpecific.windowsSpecific.setupWindowsBackend()

        else:
            connection = platformSpecific.unixSpecific.setupUnixBackend()

        cursor = connection.cursor()

        try:
            cursor.execute(select_language)
            connection.commit()
            result = cursor.fetchall()

            # Language modes
            # system: language of OS
            # en: English
            # de: German
            langmode = "system"

            try:
                qemu_img_slot = str(result[0])

                if result[0][1] == "en":
                    langmode = "en"

                elif result[0][1] == "de":
                    langmode = "de"

                elif result[0][1] == "uk":
                    langmode = "uk"

                elif result[0][1] == "fr":
                    langmode = "fr"

                elif result[0][1] == "es":
                    langmode = "es"

                elif result[0][1] == "ro":
                    langmode = "ro"

                elif result[0][1] == "ru":
                    langmode = "ru"

                elif result[0][1] == "be":
                    langmode = "be"

                elif result[0][1] == "cz":
                    langmode = "cz"

                elif result[0][1] == "pt":
                    langmode = "pt"

                elif result[0][1] == "it":
                    langmode = "it"

                elif result[0][1] == "system":
                    langmode = "system"

                self.setLanguage(langmode)
                print("The query was executed successfully. The language slot already is in the database.")

            except:
                langmode = "system"
                self.setLanguage(langmode)
                print("The query was executed successfully. The language slot has been created.")

        except sqlite3.Error as e:
            print(f"The SQLite module encountered an error: {e}.")

    def setLanguage(self, langmode):
        if langmode == "system" or langmode == None:
            languageToUse = locale.getlocale()[0]

        else:
            languageToUse = langmode

        print(languageToUse)

        if languageToUse != None:
            if languageToUse.startswith("de"):
                translations.de.translateEditVMDE(self)

            elif languageToUse.startswith("uk"):
                translations.uk.translateEditVMUK(self)

            elif languageToUse.startswith("fr"):
                translations.fr.translateEditVMFR(self)

            elif languageToUse.startswith("es"):
                translations.es.translateEditVMES(self)

            elif languageToUse.startswith("ro"):
                translations.ro.translateEditVMRO(self)

            elif languageToUse.startswith("ru"):
                translations.ru.translateEditVMRU(self)

            elif languageToUse.startswith("be"):
                translations.be.translateEditVMBE(self)

            elif languageToUse.startswith("cz"):
                translations.cz.translateEditVMCZ(self)

            elif languageToUse.startswith("pt"):
                translations.pt.translateEditVMPT(self)

            elif languageToUse.startswith("it"):
                translations.it.translateEditVMIT(self)

            else:
                translations.en.translateEditVMEN(self)

        else:
            if platform.system() == "Windows":
                langfile = platformSpecific.windowsSpecific.windowsLanguageFile()

            else:
                langfile = platformSpecific.unixSpecific.unixLanguageFile()

            try:
                with open(langfile, "r+") as language:
                    languageContent = language.readlines()
                    languageToUse = languageContent[0].replace("\n", "")

                if languageToUse != None:
                    if languageToUse.startswith("de"):
                        translations.de.translateEditVMDE(self)

                    elif languageToUse.startswith("uk"):
                        translations.uk.translateEditVMUK(self)

                    elif languageToUse.startswith("fr"):
                        translations.fr.translateEditVMFR(self)

                    elif languageToUse.startswith("es"):
                        translations.es.translateEditVMES(self)

                    elif languageToUse.startswith("ro"):
                        translations.ro.translateEditVMRO(self)

                    elif languageToUse.startswith("ru"):
                        translations.ru.translateEditVMRU(self)

                    elif languageToUse.startswith("be"):
                        translations.be.translateEditVMBE(self)

                    elif languageToUse.startswith("cz"):
                        translations.cz.translateEditVMCZ(self)

                    elif languageToUse.startswith("pt"):
                        translations.pt.translateEditVMPT(self)

                    elif languageToUse.startswith("it"):
                        translations.it.translateEditVMIT(self)

                    else:
                        translations.en.translateEditVMEN(self)

            except:
                print("Translation can't be figured out. Using English language.")
                translations.en.translateEditVMEN(self)

    def machineCpuI386Amd64(self, machine, cpu):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_12.count():
            if letQemuDecideContent.__contains__(self.comboBox_12.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_12.setCurrentIndex(i)
                    break

            elif self.comboBox_12.itemText(i) == machine:
                self.comboBox_12.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_11.count():
            if letQemuDecideContent.__contains__(self.comboBox_11.itemText(i)):
                if cpu == "Let QEMU decide":
                    self.comboBox_11.setCurrentIndex(i)
                    break

            if self.comboBox_11.itemText(i) == "Icelake-Client (depreciated)":
                if cpu == "Icelake-Client":
                    self.comboBox_11.setCurrentIndex(i)
                    break

            if self.comboBox_11.itemText(i) == cpu:
                self.comboBox_11.setCurrentIndex(i)
                break

            i += 1

    def machineCpuPpc(self, machine, cpu):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_14.count():
            if letQemuDecideContent.__contains__(self.comboBox_14.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_14.setCurrentIndex(i)
                    break

            elif self.comboBox_14.itemText(i) == machine:
                self.comboBox_14.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_13.count():
            if letQemuDecideContent.__contains__(self.comboBox_13.itemText(i)):
                if cpu == "Let QEMU decide":
                    self.comboBox_13.setCurrentIndex(i)
                    break

            if self.comboBox_13.itemText(i) == cpu:
                self.comboBox_13.setCurrentIndex(i)
                break

            i += 1

    def machineCpuMips64el(self, machine, cpu):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_16.count():
            if letQemuDecideContent.__contains__(self.comboBox_16.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_16.setCurrentIndex(i)
                    break

            elif self.comboBox_16.itemText(i) == machine:
                self.comboBox_16.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_15.count():
            if letQemuDecideContent.__contains__(self.comboBox_15.itemText(i)):
                if cpu == "Let QEMU decide":
                    self.comboBox_15.setCurrentIndex(i)
                    break

            elif self.comboBox_15.itemText(i) == cpu:
                self.comboBox_15.setCurrentIndex(i)
                break

            i += 1

    def machineCpuAarch64(self, machine, cpu):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_18.count():
            if letQemuDecideContent.__contains__(self.comboBox_18.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_18.setCurrentIndex(i)
                    break

            elif self.comboBox_18.itemText(i) == machine:
                self.comboBox_18.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_17.count():
            if letQemuDecideContent.__contains__(self.comboBox_17.itemText(i)):
                if cpu == "Let QEMU decide":
                    self.comboBox_17.setCurrentIndex(i)
                    break

            elif self.comboBox_17.itemText(i) == cpu:
                self.comboBox_17.setCurrentIndex(i)
                break

            i += 1

    def machineSparc(self, machine):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_20.count():
            if letQemuDecideContent.__contains__(self.comboBox_20.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_20.setCurrentIndex(i)
                    break

            elif self.comboBox_20.itemText(i) == machine:
                self.comboBox_20.setCurrentIndex(i)
                break

            i += 1

    def machineSparc64(self, machine):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        i = 0

        while i < self.comboBox_21.count():
            if letQemuDecideContent.__contains__(self.comboBox_21.itemText(i)):
                if machine == "Let QEMU decide":
                    self.comboBox_21.setCurrentIndex(i)
                    break

            elif self.comboBox_21.itemText(i) == machine:
                self.comboBox_21.setCurrentIndex(i)
                break

            i += 1

    def vhdAddingChange(self):
        with open("translations/createnewvhd.txt", "r+", encoding="utf8") as creNewVhdFile:
            creNewVhdContent = creNewVhdFile.read()

        with open("translations/addexistingvhd.txt", "r+", encoding="utf8") as addExistVhdFile:
            addExistVhdContent = addExistVhdFile.read()

        with open("translations/addnovhd.txt", "r+", encoding="utf8") as noVhdFile:
            noVhdContent = noVhdFile.read()

        if creNewVhdContent.__contains__(self.comboBox_2.currentText()):
            # For new and existing
            self.lineEdit_2.setEnabled(True)
            self.pushButton_3.setEnabled(True)

            # For new
            self.comboBox_3.setEnabled(True)
            self.spinBox.setEnabled(True)
            self.comboBox_4.setEnabled(True)

        elif addExistVhdContent.__contains__(self.comboBox_2.currentText()):
            # For new and existing
            self.lineEdit_2.setEnabled(True)
            self.pushButton_3.setEnabled(True)

            # For new
            self.comboBox_3.setEnabled(False)
            self.spinBox.setEnabled(False)
            self.comboBox_4.setEnabled(False)

        elif noVhdContent.__contains__(self.comboBox_2.currentText()):
            # For new and existing
            self.lineEdit_2.setEnabled(False)
            self.pushButton_3.setEnabled(False)

            # For new
            self.comboBox_3.setEnabled(False)
            self.spinBox.setEnabled(False)
            self.comboBox_4.setEnabled(False)

    def vhdBrowseLocation(self):
        filename, filter = QFileDialog.getSaveFileName(parent=self, caption='Save/Open VHD file', dir='.', filter='Hard disk file (*.img);;VirtualBox disk image (*.vdi);;VMware disk file (*.vmdk);;Virtual hard disk file with extra features (*.vhdx);;All files (*.*)')

        if filename:
            self.lineEdit_2.setText(filename)

    def archChanged(self):
        if self.comboBox.currentText() == "i386" or self.comboBox.currentText() == "amd64":
            self.stackedWidget.setCurrentIndex(0)

        elif self.comboBox.currentText() == "ppc" or self.comboBox.currentText() == "ppc64":
            self.stackedWidget.setCurrentIndex(1)

        elif self.comboBox.currentText() == "mips" or self.comboBox.currentText() == "mipsel":
            self.stackedWidget.setCurrentIndex(2)

        elif self.comboBox.currentText() == "mips64" or self.comboBox.currentText() == "mips64el":
            self.stackedWidget.setCurrentIndex(2)

        elif self.comboBox.currentText() == "arm" or self.comboBox.currentText() == "aarch64":
            self.stackedWidget.setCurrentIndex(3)

        elif self.comboBox.currentText() == "sparc":
            self.stackedWidget.setCurrentIndex(4)

        elif self.comboBox.currentText() == "sparc64":
            self.stackedWidget.setCurrentIndex(5)

    def extBiosFileLocation(self):
        filename, filter = QFileDialog.getOpenFileName(parent=self, caption='Select BIOS file', dir='.', filter='BIN files (*.bin);;All files (*.*)')

        if filename:
            self.lineEdit_4.setText(filename)

    def linuxKernelBrowseLocation(self):
        filename, filter = QFileDialog.getOpenFileName(parent=self, caption='Select Linux kernel', dir='.', filter='All files (*.*)')

        if filename:
            self.lineEdit_5.setText(filename)

    def linuxInitridBrowseLocation(self):
        filename, filter = QFileDialog.getOpenFileName(parent=self, caption='Select Linux initrid image', dir='.', filter='IMG files (*.img);;All files (*.*)')

        if filename:
            self.lineEdit_6.setText(filename)

    def linuxInitridBrowseLocation(self):
        filename, filter = QFileDialog.getOpenFileName(parent=self, caption='Select Linux initrid image', dir='.', filter='IMG files (*.img);;All files (*.*)')

        if filename:
            self.lineEdit_6.setText(filename)

    def readTempVmFile(self):
        with open("translations/createnewvhd.txt", "r+", encoding="utf8") as creNewVhdFile:
            creNewVhdContent = creNewVhdFile.read()

        with open("translations/addexistingvhd.txt", "r+", encoding="utf8") as addExistVhdFile:
            addExistVhdContent = addExistVhdFile.read()

        with open("translations/addnovhd.txt", "r+", encoding="utf8") as noVhdFile:
            noVhdContent = noVhdFile.read()

        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideFile:
            letQemuDecideContent = letQemuDecideFile.read()

        # Searching temporary files
        if platform.system() == "Windows":
            tempVmDef = platformSpecific.windowsSpecific.windowsTempVmStarterFile()

        else:
            tempVmDef = platformSpecific.unixSpecific.unixTempVmStarterFile()

        vmSpecs = []

        with open(tempVmDef, "r+") as tempVmDefFile:
            vmSpecsRaw = tempVmDefFile.readlines()

        for vmSpec in vmSpecsRaw:
            vmSpecNew = vmSpec.replace("\n", "")
            vmSpecs.append(vmSpecNew)

        # Setting VM variables

        self.lineEdit.setText(vmSpecs[0])
        self.setWindowTitle(f"EmuGUI - Edit {vmSpecs[0]}")

        i = 0

        while i < self.comboBox.count():
            if self.comboBox.itemText(i) == vmSpecs[1]:
                self.comboBox.setCurrentIndex(i)
                break

            i += 1

        self.archChanged()

        if vmSpecs[1] == "i386" or vmSpecs[1] == "x86_64":
            self.machineCpuI386Amd64(vmSpecs[2], vmSpecs[3])
            self.spinBox_2.setValue(int(vmSpecs[4]))

        elif vmSpecs[1] == "mips64el" or vmSpecs[1] == "mipsel" or vmSpecs[1] == "mips64" or vmSpecs[1] == "mips":
            self.machineCpuMips64el(vmSpecs[2], vmSpecs[3])
            self.spinBox_4.setValue(int(vmSpecs[4]))

        elif vmSpecs[1] == "ppc" or vmSpecs[1] == "ppc64":
            self.machineCpuPpc(vmSpecs[2], vmSpecs[3])
            self.spinBox_3.setValue(int(vmSpecs[4]))

        elif vmSpecs[1] == "aarch64" or vmSpecs[1] == "arm":
            self.machineCpuAarch64(vmSpecs[2], vmSpecs[3])
            self.spinBox_5.setValue(int(vmSpecs[4]))

        elif vmSpecs[1] == "sparc":
            self.machineSparc(vmSpecs[2])
            self.spinBox_7.setValue(int(vmSpecs[4]))

        elif vmSpecs[1] == "sparc64":
            self.machineSparc64(vmSpecs[2])
            self.spinBox_8.setValue(int(vmSpecs[4]))

        if vmSpecs[5] != "NULL":
            self.lineEdit_2.setText(vmSpecs[5])
            i = 0

            while i < self.comboBox_2.count():
                if addExistVhdContent.__contains__(self.comboBox_2.itemText(i)): #self.comboBox_2.itemText(i) == "Add existing virtual hard drive" or self.comboBox_2.itemText(i) == "Existierende virtuelle Festplatte anfügen":
                    self.comboBox_2.setCurrentIndex(i)
                    break

                i += 1

        else:
            i = 0

            while i < self.comboBox_2.count():
                if noVhdContent.__contains__(self.comboBox_2.itemText(i)): #self.comboBox_2.itemText(i) == "Don't add a virtual hard drive" or self.comboBox_2.itemText(i) == "Keine virtuelle Festplatte anfügen":
                    self.comboBox_2.setCurrentIndex(i)
                    break

                i += 1

        self.vhdAddingChange()

        i = 0

        while i < self.comboBox_7.count():
            if vmSpecs[6] == "Let QEMU decide":
                if letQemuDecideContent.__contains__(self.comboBox_7.itemText(i)):
                    self.comboBox_7.setCurrentIndex(i)
                    break

            elif self.comboBox_7.itemText(i) == vmSpecs[6]:
                self.comboBox_7.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_8.count():
            if self.comboBox_8.itemText(i) == vmSpecs[7]:
                self.comboBox_8.setCurrentIndex(i)
                break

            i += 1

        if vmSpecs[8] == "1":
            self.checkBox_2.setChecked(True)

        self.lineEdit_3.setText(vmSpecs[10])

        if vmSpecs[9] == "1":
            self.checkBox_3.setChecked(True)

        self.lineEdit_8.setText(vmSpecs[11])

        i = 0

        while i < self.comboBox_10.count():
            if self.comboBox_10.itemText(i) == vmSpecs[12]:
                self.comboBox_10.setCurrentIndex(i)
                break

            i += 1

        self.lineEdit_5.setText(vmSpecs[13])
        self.lineEdit_6.setText(vmSpecs[14])
        self.lineEdit_7.setText(vmSpecs[15])

        i = 0

        while i < self.comboBox_5.count():
            if self.comboBox_5.itemText(i) == vmSpecs[16]:
                self.comboBox_5.setCurrentIndex(i)
                break

            i += 1

        self.lineEdit_4.setText(vmSpecs[18])
        self.spinBox_6.setValue(int(vmSpecs[17]))

        i = 0

        while i < self.comboBox_6.count():
            if self.comboBox_6.itemText(i) == vmSpecs[19]:
                self.comboBox_6.setCurrentIndex(i)
                break

            i += 1

        if vmSpecs[20] == "1":
            self.checkBox.setChecked(True)

        i = 0

        while i < self.comboBox_9.count():
            if self.comboBox_9.itemText(i) == vmSpecs[21]:
                self.comboBox_9.setCurrentIndex(i)
                break

            i += 1

        if vmSpecs[20] == "1":
            self.checkBox.setChecked(True)

        i = 0

        while i < self.comboBox_19.count():
            if self.comboBox_19.itemText(i) == vmSpecs[22]:
                self.comboBox_19.setCurrentIndex(i)
                break

            i += 1

        i = 0

        while i < self.comboBox_22.count():
            if self.comboBox_22.itemText(i) == vmSpecs[23]:
                self.comboBox_22.setCurrentIndex(i)
                break

            i += 1

        return vmSpecs

    def finishCreation(self):
        with open("translations/letqemudecide.txt", "r+", encoding="utf8") as letQemuDecideVariants:
            letQemuDecideVariantsStr = letQemuDecideVariants.read()

        with open("translations/systemdefault.txt", "r+", encoding="utf8") as sysDefFile:
            sysDefContent = sysDefFile.read()

        # This applies the changes to your VM.

        if platform.system() == "Windows":
            connection = platformSpecific.windowsSpecific.setupWindowsBackend()

        else:
            connection = platformSpecific.unixSpecific.setupUnixBackend()

        cursor = connection.cursor()

        if self.comboBox.currentText() == "i386" or self.comboBox.currentText() == "x86_64":
            machine = self.comboBox_12.currentText()
            cpu = self.comboBox_11.currentText()

            if cpu.startswith("Icelake-Client"):
                cpu = "Icelake-Client"

            ram = self.spinBox_2.value()

        elif self.comboBox.currentText() == "ppc" or self.comboBox.currentText() == "ppc64":
            machine = self.comboBox_14.currentText()
            cpu = self.comboBox_13.currentText()
            ram = self.spinBox_3.value()

        elif self.comboBox.currentText() == "mips64el" or self.comboBox.currentText() == "mipsel":
            machine = self.comboBox_16.currentText()
            cpu = self.comboBox_15.currentText()
            ram = self.spinBox_4.value()

        elif self.comboBox.currentText() == "mips64" or self.comboBox.currentText() == "mips":
            machine = self.comboBox_16.currentText()
            cpu = self.comboBox_15.currentText()
            ram = self.spinBox_4.value()

        elif self.comboBox.currentText() == "aarch64" or self.comboBox.currentText() == "arm":
            machine = self.comboBox_18.currentText()
            cpu = self.comboBox_17.currentText()
            ram = self.spinBox_5.value()

        elif self.comboBox.currentText() == "sparc":
            machine = self.comboBox_20.currentText()
            cpu = "Let QEMU decide"
            ram = self.spinBox_7.value()

        elif self.comboBox.currentText() == "sparc64":
            machine = self.comboBox_21.currentText()
            cpu = "Let QEMU decide"
            ram = self.spinBox_8.value()

        if letQemuDecideVariantsStr.__contains__(machine):
            machine = "Let QEMU decide"

        if letQemuDecideVariantsStr.__contains__(cpu):
            cpu = "Let QEMU decide"

        if self.lineEdit_2.text() == "" or self.lineEdit_2.isEnabled() == False:
            vhd = "NULL"

        else:
            vhd = self.lineEdit_2.text()

            if platform.system() == "Windows":
                tempVmDef = platformSpecific.windowsSpecific.windowsTempVmStarterFile()

            else:
                tempVmDef = platformSpecific.unixSpecific.unixTempVmStarterFile()

            with open(tempVmDef, "r+") as tempVmDefFile:
                vmSpecsRaw = tempVmDefFile.readlines()

            vhdAction = vmSpecsRaw[0]

            if self.comboBox_3.isEnabled():
                vhdAction = "overwrite"

            else:
                vhdAction = "keep"

            get_qemu_img_bin = """
            SELECT value FROM settings
            WHERE name = "qemu-img"
            """

            vhd_cmd = ""

            try:
                cursor.execute(get_qemu_img_bin)
                connection.commit()
                result = cursor.fetchall()
                qemu_binary = result[0][0]
                vhd_size_in_b = None

                if self.comboBox_4.currentText().startswith("K"):
                    vhd_size_in_b = self.spinBox.value() * 1024

                elif self.comboBox_4.currentText().startswith("M"):
                    vhd_size_in_b = self.spinBox.value() * 1024 * 1024

                elif self.comboBox_4.currentText().startswith("G"):
                    vhd_size_in_b = self.spinBox.value() * 1024 * 1024 * 1024

                print(vhd_size_in_b)

                if platform.system() == "Windows":
                    vhd_cmd = f"{qemu_binary} create -f {self.comboBox_3.currentText()} \"{vhd}\" {str(vhd_size_in_b)}"

                else:
                    vhd_cmd = f"{qemu_binary} create -f {self.comboBox_3.currentText()} {vhd} {str(vhd_size_in_b)}"

                if vhdAction.startswith("overwrite"):
                    subprocess.Popen(vhd_cmd)

                print("The query was executed and the virtual disk created successfully.")

            except sqlite3.Error as e:
                print(f"The SQLite module encountered an error: {e}.")

            except:
                print(f"The query was executed successfully, but the virtual disk couldn't be created. Trying to use subprocess.run")

                try:
                    vhd_cmd_split = vhd_cmd.split(" ")

                    if vhdAction.startswith("overwrite"):
                        subprocess.run(vhd_cmd_split)

                    print("The query was executed and the virtual disk created successfully.")

                except:
                    print("The virtual disk could not be created. Please check if the path and the QEMU settings are correct.")

        if letQemuDecideVariantsStr.__contains__(self.comboBox_7.currentText()):
            vga = "Let QEMU decide"

        else:
            vga = self.comboBox_7.currentText()

        if self.comboBox_8.currentText() == "none":
            networkAdapter = "none"

        else:
            networkAdapter = self.comboBox_8.currentText()

        if self.checkBox_2.isChecked():
            usbtablet = 1

        else:
            usbtablet = 0

        if self.checkBox_3.isChecked():
            win2k = 1

        else:
            win2k = 0

        ext_bios_dir = self.lineEdit_3.text()

        add_args = self.lineEdit_8.text()

        if self.checkBox_2.isChecked() or self.checkBox.isChecked() or self.comboBox_5.currentText() == "USB Mouse":
            usb_support = 1

        elif self.comboBox_5.currentText() == "USB Tablet Device" or self.comboBox_6.currentText() == "USB Keyboard":
            usb_support = 1

        else:
            usb_support = 0

        if sysDefContent.__contains__(self.comboBox_19.currentText()):
            kbdlayout = "en-us"

        else:
            kbdlayout = self.comboBox_19.currentText()

        insert_into_vm_database = f"""
        UPDATE virtualmachines
        SET name = "{self.lineEdit.text()}", architecture = "{self.comboBox.currentText()}", machine = "{machine}", cpu = "{cpu}",
        ram = {ram}, hda = "{vhd}", vga = "{vga}", net = "{networkAdapter}", usbtablet = {usbtablet},
        win2k = {win2k}, dirbios = "{ext_bios_dir}", additionalargs = "{add_args}", sound = "{self.comboBox_10.currentText()}",
        linuxkernel = "{self.lineEdit_5.text()}", linuxinitrid = "{self.lineEdit_6.text()}", linuxcmd = "{self.lineEdit_7.text()}",
        mousetype = "{self.comboBox_5.currentText()}", cores = {self.spinBox_6.value()}, filebios = "{self.lineEdit_4.text()}",
        keyboardtype = "{self.comboBox_6.currentText()}", usbsupport = {usb_support}, usbcontroller = "{self.comboBox_9.currentText()}",
        kbdtype = "{kbdlayout}", acceltype = "{self.comboBox_22.currentText()}"
        WHERE name = "{self.vmSpecs[0]}";
        """

        cursor = connection.cursor()

        try:
            cursor.execute(insert_into_vm_database)
            connection.commit()
            print("The query was executed successfully.")

        except sqlite3.Error as e:
            print(f"The SQLite module encountered an error: {e}.")

        self.close()


"""
PHDNet
(c) Schobs, Lawrence
"""
import torch
import torch.nn as nn
import torch.nn.functional as F

# from .preresnet import BasicBlock, Bottleneck

class PHDNet(nn.Module):
    """Implmentation of the PHDNet architecture [ref].
    Can use either the multi-branch scheme, heatmap branch only, or displacment branch only.

    """

    def __init__(self, branch_scheme):
        self.branch_scheme = branch_scheme
        super(PHDNet, self).__init__()

        padding = 1
        run_stats = True
        momentum = 0.1
        # if is_train:
        #     print("train mode")
        #     run_stats = True
        #     momentum = 0.1
        # else:
        #     print("eval mode")
        #     run_stats = False
        #    momentum = 0.1

        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, stride=2),
        )
        self.layer2 = nn.Sequential(
            nn.Conv2d(32, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, stride=2),
        )
        self.layer3 = nn.Sequential(
            nn.Conv2d(32, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(2, stride=2),
        )
        self.layer4 = nn.Sequential(
            nn.Conv2d(32, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
        )
        self.layer5 = nn.Sequential(
            nn.Conv2d(32, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
        )
        self.layer6 = nn.Sequential(
            nn.Conv2d(32, 32, kernel_size=3, stride=1, bias=True, padding=1),
            nn.BatchNorm2d(32, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
        )

        self.layer_reg = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=1, stride=1, bias=True),
            nn.BatchNorm2d(64, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 96, kernel_size=1, stride=1, bias=True),
            nn.BatchNorm2d(96, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
        )

        self.layer_class = nn.Sequential(
            nn.Conv2d(32, 64, kernel_size=1, stride=1, bias=True),
            nn.BatchNorm2d(64, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 96, kernel_size=1, stride=1, bias=True),
            nn.BatchNorm2d(96, momentum=momentum, track_running_stats=run_stats),
            nn.ReLU(inplace=True),
        )

        self.outReg = nn.Sequential(
            nn.Conv2d(96, 2, kernel_size=1, stride=1, bias=True)
            # nn.ReLU(inplace=True),
            # what acitivaton function for this??
        )

        self.outClass = nn.Sequential(
            nn.Conv2d(96, 1, kernel_size=1, stride=1, bias=True)
        )

        # dont apply sigmoid if using weighted loss as BCEwithlogits does sigmoid in it.

    def forward(self, x):

        # print("the shape of x is:", x.shape)
        # out = []
        # print(x.shape)
        x = self.layer1(x)
        # print(x.shape)
        x = self.layer2(x)
        # print(x.shape)

        x = self.layer3(x)
        # print(x.shape)

        x = self.layer4(x)
        # print(x.shape)

        x = self.layer5(x)
        # print(x.shape)

        x = self.layer6(x)
        # print(x.shape)

        if self.branch_scheme == "multi" or "displacement":

            x_reg = self.layer_reg(x)
            # print("reg, ", x_reg.shape)
            # unsqueeze because single landmark version does not give correct out dimensions
            out_reg = self.outReg(x_reg).unsqueeze(1)
            # print(out_reg.shape)

        if self.branch_scheme == "multi" or "heatmap":

            x_class = self.layer_class(x)
            # print("class, ", x_class.shape)

            out_class = self.outClass(x_class)

            # if sigmoid == True:
            #     s = nn.Sigmoid()
            #     out_class = s(out_class)

        #  print(out_class.shape)

        # out.append([out_class, out_reg])
        if self.branch_scheme == "multi":
            return [out_class, out_reg]
        elif self.branch_scheme == "heatmap":
            return out_class
        else:
            return out_reg


import argparse
import os

from config import Config
from migrations.database import Database
from migrations.migration import Migration

def main(checking):
    """ Migration run. """
    path = os.path.join('/migrations', 'versions')
    db = Database(conf=Config.POSTGRES).connect()

    migration = Migration(path=path, db=db)
    migrations = migration.get_migrations(checking=checking)

    if not checking:
        migration.migrate(migrations=migrations)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', action='store_true')
    args = parser.parse_args()

    main(checking=args.c)