#!/usr/bin/env python2.6

# Standard Lib Modules
import os
import logging
import argparse
import fnmatch
import re

def parse_file(file_obj, regex):
    print("Parsing file.")
    parsed_file = {
        "type": 'unknown',
        "content": {}
    }
    block_num = 0
    in_block = False
    content = dict()

    for line in file_obj:
        line = line.strip()
        line = line.replace('\t', ' ')  # Yes, lines could be tab separated. :(

        if regex["comment"].match(line) or regex["blankline"].match(line):
            continue

        if not in_block:
            if regex["host"].match(line):
                print("Host match. {0}".format(file_obj.name))
                parsed_file["type"] = "host"
                in_block = True
            elif regex["hostgroup"].match(line):
                print("Hostgroup match. {0}".format(file_obj.name))
                parsed_file["type"] = "hostgroup"
                in_block = True
            elif regex["hostdependency"].match(line):
                print("hostdependency match. {0}".format(file_obj.name))
                parsed_file["type"] = "hostdependency"
                in_block = True
            elif regex["hostescalation"].match(line):
                print("hostescalation match. {0}".format(file_obj.name))
                parsed_file["type"] = "hostescalation"
                in_block = True
            elif regex["hostextinfo"].match(line):
                print("hostextinfo match. {0}".format(file_obj.name))
                parsed_file["type"] = "hostextinfo"
                in_block = True
            elif regex["service"].match(line):
                print("Service match. {0}".format(file_obj.name))
                parsed_file["type"] = "service"
                in_block = True
            elif regex["servicegroup"].match(line):
                print("Servicegroup match. {0}".format(file_obj.name))
                parsed_file["type"] = "servicegroup"
                in_block = True
            elif regex["servicedependency"].match(line):
                print("servicedependency match. {0}".format(file_obj.name))
                parsed_file["type"] = "servicedependency"
                in_block = True
            elif regex["serviceescalation"].match(line):
                print("serviceescalation match. {0}".format(file_obj.name))
                parsed_file["type"] = "serviceescalation"
                in_block = True
            elif regex["serviceextinfo"].match(line):
                print("serviceextinfo match. {0}".format(file_obj.name))
                parsed_file["type"] = "serviceextinfo"
                in_block = True
            elif regex["contact"].match(line):
                print("contact match. {0}".format(file_obj.name))
                parsed_file["type"] = "contact"
                in_block = True
            elif regex["contactgroup"].match(line):
                print("contactgroup match. {0}".format(file_obj.name))
                parsed_file["type"] = "contactgroup"
                in_block = True
            elif regex["timeperiod"].match(line):
                print("timeperiod match. {0}".format(file_obj.name))
                parsed_file["type"] = "timeperiod"
                in_block = True
            elif regex["command"].match(line):
                print("command match. {0}".format(file_obj.name))
                parsed_file["type"] = "command"
                in_block = True
        elif in_block:
            if regex["end"].match(line):  # This is for files with multiple blocks.
                print("End of block.")
                # Added content to parsed_file
                parsed_file["content"][block_num] = content
                block_num += 1
                in_block = False
                content = dict()
            else:
                line = line.split(' ', 1)
                print("Line: {0}".format(line))
                if line[1] == '':
                   line[1] = ''
                content[line[0]] = line[1].strip()
    return parsed_file

def merge_files(file1, file2):
    print("Merging files.")
    print("Parsed File 1: {0}".format(file1))
    print("Parsed File 2: {0}".format(file2))
    for block in file2["content"]:  # Making the assumption the type is the same.
        if block in file1["content"]:
            for key in file2["content"][block]:
                if key in file1["content"][block]:
                    if file1["content"][block][key] != file2["content"][block][key]:
                        file1["content"][block][key] = file2["content"][block][key]
                    else:
                        file1["content"][block][key] = file1["content"][block][key]
                else:
                    file1["content"][block][key] = file2["content"][block][key]
        else:
            file1["content"][block] = file2["content"][block]

    print("Merged Files: {0}".format(file1))

    return file1

def file_walk(path, file_filter, regex, host_file={}, hostgroup_file={}, hostdependency_file={}, hostescalation_file={}, hostextinfo_file={}, service_file={}, servicegroup_file={}, servicedependency_file={}, serviceescalation_file={}, serviceextinfo_file={}, contact_file={}, contactgroup_file={}, timeperiod_file={}, command_file={}):
    for root, _, filelist in os.walk(path):
        for base_file_name in fnmatch.filter(filelist, file_filter):
            base_file_name = "/".join(
                [
                    root,
                    base_file_name
                ]
            )
            base_file = open(base_file_name, 'r')
            override_file_name = ".".join([base_file_name, "override"])
            # Possible race condition with this. A try/catch block with open
            # would be better, but I don't want to deal with that right now.
            if os.path.isfile(override_file_name):
                override_file = open(override_file_name, 'r')
                base_file_parsed = parse_file(base_file, regex)
                override_file_parsed = parse_file(override_file, regex)
                final_file = merge_files(base_file_parsed, override_file_parsed)
                if final_file["type"] == "host":
                    print("Adding host file.")
                    host_file[base_file_name] = final_file
                elif final_file["type"] == "hostgroup":
                    print("Adding hostgroup file.")
                    hostgroup_file[base_file_name] = final_file
                elif final_file["type"] == "hostdependency":
                    print("Adding hostdependency file.")
                    hostdependency_file[base_file_name] = final_file
                elif final_file["type"] == "hostescalation":
                    print("Adding hostescalation file.")
                    hostescalation_file[base_file_name] = final_file
                elif final_file["type"] == "hostextinfo":
                    print("Adding hostextinfo file.")
                    hostextinfo_file[base_file_name] = final_file
                elif final_file["type"] == "service":
                    print("Adding service file.")
                    service_file[base_file_name] = final_file
                elif final_file["type"] == "servicegroup":
                    print("Adding servicegroup file.")
                    servicegroup_file[base_file_name] = final_file
                elif final_file["type"] == "servicedependency":
                    print("Adding servicedependency file.")
                    servicedependency_file[base_file_name] = final_file
                elif final_file["type"] == "serviceescalation":
                    print("Adding serviceescalation file.")
                    serviceescalation_file[base_file_name] = final_file
                elif final_file["type"] == "serviceextinfo":
                    print("Adding serviceextinfo file.")
                    serviceextinfo_file[base_file_name] = final_file
                elif final_file["type"] == "contact":
                    print("Adding contact file.")
                    contact_file[base_file_name] = final_file
                elif final_file["type"] == "contactgroup":
                    print("Adding contactgroup file.")
                    contactgroup_file[base_file_name] = final_file
                elif final_file["type"] == "timeperiod":
                    print("Adding timeperiod file.")
                    timeperiod_file[base_file_name] = final_file
                elif final_file["type"] == "command":
                    print("Adding command file.")
                    command_file[base_file_name] = final_file
                override_file.close()
            else:
                final_file = parse_file(base_file, regex)
                if final_file["type"] == "host":
                    host_file[base_file_name] = final_file
                elif final_file["type"] == "hostgroup":
                    hostgroup_file[base_file_name] = final_file
                elif final_file["type"] == "hostdependency":
                    hostdependency_file[base_file_name] = final_file
                elif final_file["type"] == "hostescalation":
                    hostescalation_file[base_file_name] = final_file
                elif final_file["type"] == "hostextinfo":
                    hostextinfo_file[base_file_name] = final_file
                elif final_file["type"] == "service":
                    service_file[base_file_name] = final_file
                elif final_file["type"] == "servicegroup":
                    servicegroup_file[base_file_name] = final_file
                elif final_file["type"] == "servicedependency":
                    servicedependency_file[base_file_name] = final_file
                elif final_file["type"] == "serviceescalation":
                    serviceescalation_file[base_file_name] = final_file
                elif final_file["type"] == "serviceextinfo":
                    serviceextinfo_file[base_file_name] = final_file
                elif final_file["type"] == "contact":
                    contact_file[base_file_name] = final_file
                elif final_file["type"] == "contactgroup":
                    contactgroup_file[base_file_name] = final_file
                elif final_file["type"] == "timeperiod":
                    timeperiod_file[base_file_name] = final_file
                elif final_file["type"] == "command":
                    command_file[base_file_name] = final_file

            base_file.close()

    return host_file, hostgroup_file, hostdependency_file, hostescalation_file, hostextinfo_file, service_file, servicegroup_file, servicedependency_file, serviceescalation_file, serviceextinfo_file, contact_file, contactgroup_file, timeperiod_file, command_file

def build_config(cfg_file, cfg_data):
    for filename in cfg_data:
        source_dir, source_file = filename.split("/")[-2:]
        cfg_file.write("# Start {0}/{1}\n".format(source_dir, source_file))
        for block in cfg_data[filename]["content"]:
            cfg_file.write("define {0} {{\n".format(cfg_data[filename]["type"]))
            for key in cfg_data[filename]["content"][block]:
                cfg_file.write("    {0}    {1}\n".format(key, cfg_data[filename]["content"][block][key]))
            cfg_file.write("}\n")
        cfg_file.write("# End {0}/{1}\n".format(source_dir, source_file))

def main():
    description = "Walks a directory and consolidates config files"
    log_entry_format = ":".join(
        [
            '%(asctime)s',
            '%(levelname)s',
            '%(name)s',
            '%(message)s'
        ]
    )
    logging.basicConfig(
        format=log_entry_format,
        level=logging.INFO,
        filename="consolidation.log"
    )
    logger = logging.getLogger(__name__)

    logger.info("Starting consolidation work.")

    parser = argparse.ArgumentParser(description=description)
    parser.add_argument(
        "path",
        nargs='+',
        help="List of paths to search."
    )
#    parser.add_argument(
#        "-f",
#        "--filter",
#        dest='file_filter',
#        default='*',
#        help="Set the search filter for files, default \"*\"."
#    )
    parser.add_argument(
        "-o",
        "--host-file",
        dest='host_file',
        default='hosts.cfg',
        help="Changes the hosts config file."
    )
    parser.add_argument(
        "-O",
        "--hostgroup-file",
        dest='hostgroup_file',
        default='hostgroups.cfg',
        help="Changes the hostgroups config file."
    )
    parser.add_argument(
        "-D",
        "--hostdependency-file",
        dest='hostdependency_file',
        default='hostdependencies.cfg',
        help="Changes the hostdependencies config file."
    )
    parser.add_argument(
        "-E",
        "--hostescalation-file",
        dest='hostescalation_file',
        default='hostescalations.cfg',
        help="Changes the hostescalations config file."
    )
    parser.add_argument(
        "-F",
        "--hostextinfo-file",
        dest='hostextinfo_file',
        default='hostextinfo.cfg',
        help="Changes the hostextinfo config file."
    )
    parser.add_argument(
        "-s",
        "--service-file",
        dest='service_file',
        default='services.cfg',
        help="Changes the services config file."
    )
    parser.add_argument(
        "-S",
        "--servicegroup-file",
        dest='servicegroup_file',
        default='servicegroups.cfg',
        help="Changes the servicegroups config file."
    )
    parser.add_argument(
        "-H",
        "--servicedependency-file",
        dest='servicedependency_file',
        default='servicedependencies.cfg',
        help="Changes the servicedependencies config file."
    )
    parser.add_argument(
        "-I",
        "--serviceescalation-file",
        dest='serviceescalation_file',
        default='serviceescalations.cfg',
        help="Changes the serviceescalations config file."
    )
    parser.add_argument(
        "-J",
        "--serviceextinfo-file",
        dest='serviceextinfo_file',
        default='serviceextinfo.cfg',
        help="Changes the serviceextinfs config file."
    )
    parser.add_argument(
        "-c",
        "--contact-file",
        dest='contact_file',
        default='contacts.cfg',
        help="Changes the contacts config file."
    )
    parser.add_argument(
        "-C",
        "--contactgroup-file",
        dest='contactgroup_file',
        default='contactgroups.cfg',
        help="Changes the contactgroups config file."
    )
    parser.add_argument(
        "-t",
        "--timeperiod-file",
        dest='timeperiod_file',
        default='timeperiods.cfg',
        help="Changes the timeperiods config file."
    )
    parser.add_argument(
        "-p",
        "--command-file",
        dest='command_file',
        default='commands.cfg',
        help="Changes the commands config file."
    )
    parser.add_argument(
        "-a",
        "--append",
        default=False,
        action='store_true',
        help="Appends the files rather then overwriting."
    )
    args = parser.parse_args()
    # file_filter = args.file_filter
    file_filter = "*.cfg"
    host_file_name = args.host_file
    hostgroup_file_name = args.hostgroup_file
    hostdependency_file_name = args.hostdependency_file
    hostescalation_file_name = args.hostescalation_file
    hostextinfo_file_name = args.hostextinfo_file
    service_file_name = args.service_file
    servicegroup_file_name = args.servicegroup_file
    servicedependency_file_name = args.servicedependency_file
    serviceescalation_file_name = args.serviceescalation_file
    serviceextinfo_file_name = args.serviceextinfo_file
    contact_file_name = args.contact_file
    contactgroup_file_name = args.contactgroup_file
    timeperiod_file_name = args.timeperiod_file
    command_file_name = args.command_file
    if (args.append):
        open_mode = 'a'
    else:
        open_mode = 'w'

    regex_dict = {
        "host": re.compile("^define host( |){"),
        "hostgroup": re.compile("^define hostgroup( |){"),
        "hostdependency": re.compile("^define hostdependency( |){"),
        "hostescalation": re.compile("^define hostescalation( |){"),
        "hostextinfo": re.compile("^define hostextinfo( |){"),
        "service": re.compile("^define service( |){"),
        "servicegroup": re.compile("^define servicegroup( |){"),
        "servicedependency": re.compile("^define servicedependency( |){"),
        "serviceescalation": re.compile("^define serviceescalation( |){"),
        "serviceextinfo": re.compile("^define serviceextinfo( |){"),
        "contact": re.compile("^define contact( |){"),
        "contactgroup": re.compile("^define contactgroup( |){"),
        "timeperiod": re.compile("^define timeperiod( |){"),
        "command": re.compile("^define command( |){"),
        "end": re.compile("^}$"),
        "comment": re.compile("^#"),
        "blankline": re.compile("^\s*$"),
    }

    for path in args.path:
        host_file_dict, hostgroup_file_dict, hostdependency_file_dict, hostescalation_file_dict, hostextinfo_file_dict, service_file_dict, servicegroup_file_dict, servicedependency_file_dict, serviceescalation_file_dict, serviceextinfo_file_dict, contact_file_dict, contactgroup_file_dict, timeperiod_file_dict, command_file_dict = file_walk(
            path,
            file_filter,
            regex_dict
        )

    host_file_handler = open(host_file_name, open_mode)
    hostgroup_file_handler = open(hostgroup_file_name, open_mode)
    hostdependency_file_handler = open(hostdependency_file_name, open_mode)
    hostescalation_file_handler = open(hostescalation_file_name, open_mode)
    hostextinfo_file_handler = open(hostextinfo_file_name, open_mode)
    service_file_handler = open(service_file_name, open_mode)
    servicegroup_file_handler = open(servicegroup_file_name, open_mode)
    servicedependency_file_handler = open(servicedependency_file_name, open_mode)
    serviceescalation_file_handler = open(serviceescalation_file_name, open_mode)
    serviceextinfo_file_handler = open(serviceextinfo_file_name, open_mode)
    contact_file_handler = open(contact_file_name, open_mode)
    contactgroup_file_handler = open(contactgroup_file_name, open_mode)
    timeperiod_file_handler = open(timeperiod_file_name, open_mode)
    command_file_handler = open(command_file_name, open_mode)

    build_config(host_file_handler, host_file_dict)
    build_config(hostgroup_file_handler, hostgroup_file_dict)
    build_config(hostdependency_file_handler, hostdependency_file_dict)
    build_config(hostescalation_file_handler, hostescalation_file_dict)
    build_config(hostextinfo_file_handler, hostextinfo_file_dict)
    build_config(service_file_handler, service_file_dict )
    build_config(servicegroup_file_handler, servicegroup_file_dict)
    build_config(servicedependency_file_handler, servicedependency_file_dict)
    build_config(serviceescalation_file_handler, serviceescalation_file_dict)
    build_config(serviceextinfo_file_handler, serviceextinfo_file_dict)
    build_config(contact_file_handler, contact_file_dict )
    build_config(contactgroup_file_handler, contactgroup_file_dict)
    build_config(timeperiod_file_handler, timeperiod_file_dict)
    build_config(command_file_handler, command_file_dict)

    host_file_handler.close()
    hostgroup_file_handler.close()
    hostdependency_file_handler.close()
    hostescalation_file_handler.close()
    hostextinfo_file_handler.close()
    service_file_handler.close()
    servicegroup_file_handler.close()
    servicedependency_file_handler.close()
    serviceescalation_file_handler.close()
    serviceextinfo_file_handler.close()
    contact_file_handler.close()
    contactgroup_file_handler.close()
    timeperiod_file_handler.close()
    command_file_handler.close()

    return 0

if __name__ == '__main__':
    main()


from flask import Flask, jsonify
from external_database_functions import tagIterator_extern, get_add_data, get_asset_data

app_DB = Flask(__name__)
# app_DB.config['DEBUG']=True

@app_DB.route('/', methods = ['GET'])
def welcome():
    return 'Welcome to the External-Database.'

@app_DB.route('/database/availableAdds', methods = ['GET'])
def getAvailableAdds():
    response = tagIterator_extern(xml_file = 'add_directory.xml',
                                  asset_name = None)
    return jsonify(response)

@app_DB.route('/database/assets/<asset_id>', methods = ['GET'])
def getAssetData(asset_id):
    response = get_asset_data(asset_id)
    return jsonify(response)

@app_DB.route('/database/adds/<add_id>', methods = ['GET'])
def getAddData(add_id):
    response = get_add_data(add_id)
    return jsonify(response)

if __name__ == "__main__":
	app_DB.run(host='0.0.0.0', port=5001)


# -*- coding: utf-8 -*-
"""
Created on Sun Aug  9 22:16:40 2020

@author: LENOVO
"""
import socket
import numpy as np
import os, datetime
import torch
import cv2

class Load_Data(object):

    def __init__(self, file_list=None):
        self.file_list = file_list
        self.path = '/home/yj/Computer_Vision/Ther2RGB-Translation/T2R_Dataset/'

    def __len__(self):
        return len(self.file_list)

    def __getitem__(self, index):
        x = cv2.imread(self.path + 'train_A/' + self.file_list[index], cv2.IMREAD_GRAYSCALE)
        x = torch.from_numpy(x).float()
        x = x.unsqueeze(2)
        x = x.unsqueeze(0)
        x = x/255*2-1

        y = cv2.imread(self.path + 'train_B/' + 'V' + self.file_list[index][1:])
        y = torch.from_numpy(y).float()
        y = y.unsqueeze(0)
        y = y/255*2-1

        return x,y


from django.urls import path
from django.contrib.auth.views import LoginView

from .views import (
    query_view,
    schema_view,
    get_models,
    get_fields,
    sql_view,
    get_whitelist_view,
)

urlpatterns = (
    path("", query_view),
    path("schema/", schema_view),
    path("sql/", sql_view),
    path("models/<appname>/", get_models),
    path("fields/<modelname>/", get_fields),
    path("whitelist/", get_whitelist_view),
)


from django.urls import path
# from .views import Inbox, UserSearch, Directs, NewConversation, SendDirect
from . import views
# app_name = 'chats'
# urlpatterns = [
#    	path('', Inbox, name='inbox'),
#    	path('directs/<username>', Directs, name='directs'),
#    	path('new/', UserSearch, name='usersearch'),
#    	path('new/<username>', NewConversation, name='newconversation'),
#    	path('send/', SendDirect, name='send_direct'),

# ]
urlpatterns = [
   	path('', views.Inbox, name='inbox'),
   	path('directs/<str:username>', views.Directs, name='directs'),
   	path('new/', views.UserSearch, name='usersearch'),
   	path('new/<str:username>', views.NewConversation, name='newconversation'),
   	path('send/', views.SendDirect, name='send_direct'),

]


from argparse import ArgumentParser
from typing import Dict, Tuple

import torch
from pytorch_lightning import LightningModule
from torch import Tensor
from torch.nn import functional as F
from torchvision.models.segmentation import deeplabv3_resnet50
from torchvision.models.segmentation.deeplabv3 import DeepLabHead

class DeepLabV3_resnet50(LightningModule):
    def __init__(
        self,
        lr: float = 1e-4,
        pretrained: bool = True,
        num_classes: int = 3,
    ):
        super().__init__()

        self.num_classes = num_classes
        self.lr = lr

        model = deeplabv3_resnet50(pretrained=pretrained).train()

        self.backbone = model.backbone
        self.classifier = DeepLabHead(2048, num_classes)

        self.criterion = torch.nn.MSELoss(reduction="mean")

    def forward(self, x: Tensor) -> Dict[str, Tensor]:
        input_shape = x.shape[-2:]
        features = self.backbone(x)

        x = features["out"]
        x = self.classifier(x)
        x = F.interpolate(x, size=input_shape, mode="bilinear", align_corners=False)

        return x

    def training_step(self, batch: Tuple[Tensor, Tensor]):
        img, mask = batch
        mask.squeeze_(1)
        mask[mask > 21] = 0
        mask = F.one_hot(mask.long(), 22)[..., 1:].permute(0, 3, 1, 2).float()
        out = self(img)

        # plt.imshow(mask[0].argmax(0).cpu().detach().numpy())
        # plt.imshow(out[0].argmax(0).cpu().detach().numpy())

        loss_val = self.criterion(out, mask)
        log_dict = {"train_loss": loss_val}
        return {"loss": loss_val, "log": log_dict, "progress_bar": log_dict}

    def configure_optimizers(self):
        opt = torch.optim.Adam(self.parameters(), lr=self.lr)
        sch = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=10)
        return [opt], [sch]

    @staticmethod
    def add_model_specific_args(parent_parser):
        parser = ArgumentParser(parents=[parent_parser], add_help=False)
        parser.add_argument(
            "--lr", type=float, default=1e-4, help="adam: learning rate"
        )

        return parser


from statistics import harmonic_mean
from copy import deepcopy

from splink.charts import load_chart_definition, altair_if_installed_else_json
from splink.settings import Settings
import warnings

def _apply_aggregate_function(zipped_probs, aggregate_function):
    result = []
    for probs_list in zipped_probs:
        try:
            reduced = aggregate_function(probs_list)
        except Exception as e:
            warnings.warn(
                "The aggregation function produced an error when "
                f"operating on the following data: {probs_list}. "
                "The result of this aggreation has been set to None. "
                f"The error was {e}"
            )
            reduced = None
        result.append(reduced)
    return result

def _format_probs_for_report(probs):
    try:
        probs_as_strings = [f"{p:.4g}" for p in probs]
    except TypeError:
        probs_as_strings = [f"{p}" for p in probs]
    return f"{probs_as_strings}"

def _filter_nones(list_of_lists):
    def filter_none(sublist):
        return [item for item in sublist if item is not None]

    return [filter_none(sublist) for sublist in list_of_lists]

def _zip_m_and_u_probabilities(cc_estimates: list):
    """Groups together the different estimates of the same parameter.

    e.g. turns:
    [{"m_probabilities":[ma0,ma1],{"m_probabilities":[ua0,ua1],...},
        {"m_probabilities":[mb0,mb1],{"m_probabilities":[ub0,ub1],...}]

    into

    {"zipped_m": [(ma0,mb0), (ma1, mb1)],
        "zipped_u": [(ua0,ub0), (ua1, ub1)]}
    """

    zipped_m_probs = zip(*[cc["m_probabilities"] for cc in cc_estimates])
    zipped_m_probs = _filter_nones(zipped_m_probs)
    zipped_u_probs = zip(*[cc["u_probabilities"] for cc in cc_estimates])
    zipped_u_probs = _filter_nones(zipped_u_probs)

    return {"zipped_m": zipped_m_probs, "zipped_u": zipped_u_probs}

def combine_cc_estimates(cc_estimates: list, aggregate_function=harmonic_mean):
    """cc_estimates is a list of the different estaimtes for a single comparison column
    e.g. all of the different comparison columns for forename from params_list
    """

    zipped = _zip_m_and_u_probabilities(cc_estimates)

    m_probs = _apply_aggregate_function(zipped["zipped_m"], aggregate_function)
    u_probs = _apply_aggregate_function(zipped["zipped_u"], aggregate_function)

    cc = deepcopy(cc_estimates[0].column_dict)
    cc["m_probabilities"] = m_probs
    cc["u_probabilities"] = u_probs
    return cc

class ModelCombiner:
    def __init__(self, input_dicts: list):

        self.input_dicts = input_dicts
        self.named_settings_dict = {}
        for i in input_dicts:
            name = i["name"]
            settings = i["model"].current_settings_obj
            self.named_settings_dict[name] = settings

        self.settings_obj_list = list(self.named_settings_dict.values())

    def _groups_of_comparison_columns_by_name(self):
        """
        The user inputs a list of parameter estimates, each of which
        contains a settings dict

        If the input data is:
        Model list element 1:  Estimate name 'forename blocking'
            Comparison columns: [surname, dob, email]
        Model list element 2:  Estimate name 'surname blocking'
            Comparison columns: [forename, dob, email]
        Model list element 3:  Estimate name 'dob blocking'
            Comparison columns: [forename, surname, email]

        We want to group by comparison column name, respecting the fact
        that not all models have all comparison columns

        This function gives you back a dict in the form:
        {
            "forename": {"surname blocking": cc, "dob_blocking": cc},
            "surname": [etc]
            "dob": [etc]
            "email": [etc]

        }
        """
        combined_cc = {}
        # For each model which has been estimated
        for estimate_name, settings_estimate in self.named_settings_dict.items():
            # For each comparison column in this model
            for cc_name, cc in settings_estimate.comparison_column_dict.items():
                # Create or add to dict which contains the different estimate
                # for this column, using estimate names as keys
                if cc_name not in combined_cc:
                    combined_cc[cc_name] = {}
                combined_cc[cc_name][estimate_name] = cc
        return combined_cc

    def get_combined_settings_dict(self, aggregate_function=harmonic_mean):

        new_settings = deepcopy(self.settings_obj_list[0].settings_dict)

        new_comparison_columns = []
        gathered = self._groups_of_comparison_columns_by_name()

        # For each comparison column (first name, surname etc)
        for dict_of_ccs in gathered.values():
            ccs = list(dict_of_ccs.values())
            # Take the average of each parameter estimates
            combined = combine_cc_estimates(ccs, aggregate_function)
            new_comparison_columns.append(combined)

        new_settings["comparison_columns"] = new_comparison_columns

        new_blocking_rules = []
        for settings_dict in self.settings_obj_list:
            new_blocking_rules.extend(settings_dict["blocking_rules"])

        new_settings["blocking_rules"] = new_blocking_rules

        global_lambdas = []
        for input_dict in self.input_dicts:
            # If key exists, use in computation of global lambda
            # else ignore
            if "comparison_columns_for_global_lambda" in input_dict:
                gl = self._estimate_global_lambda_from_blocking_specific_lambda(
                    input_dict
                )
                global_lambdas.append(gl)
        if len(global_lambdas) > 0:
            global_lambda = aggregate_function(global_lambdas)
        else:
            global_lambda = None

        new_settings["proportion_of_matches"] = global_lambda

        return new_settings

    def _estimate_global_lambda_from_blocking_specific_lambda(self, model_dict):

        bayes_factor = 1
        for cc in model_dict["comparison_columns_for_global_lambda"]:
            m = cc["m_probabilities"][-1]
            u = cc["u_probabilities"][-1]
            b = m / u
            bayes_factor = bayes_factor * b

        # https://observablehq.com/@robinl/conditional-independence-and-repeated-application-of-bay
        # https://www.wolframalpha.com/input/?i=solve%5Bm%3Db*%CE%BB%2F%28b*%CE%BB+%2B+%281-%CE%BB%29%29%5D
        blocking_specific_lambda = model_dict["model"].current_settings_obj[
            "proportion_of_matches"
        ]

        global_lambda = blocking_specific_lambda / (
            bayes_factor
            + blocking_specific_lambda
            - bayes_factor * blocking_specific_lambda
        )

        return global_lambda

    def summary_report(self, aggregate_function=harmonic_mean, summary_name="combined"):

        lines = []
        gathered = self._groups_of_comparison_columns_by_name()

        combined_settings = self.get_combined_settings_dict(
            aggregate_function=aggregate_function
        )
        combined_settings = Settings(combined_settings)

        for cc_name, dict_of_ccs in gathered.items():

            lines.append(f"Column name: {cc_name}")

            lines.append(f"    m probabilities")
            for estimate_name, cc in dict_of_ccs.items():
                m_probs = _format_probs_for_report(cc["m_probabilities"])
                lines.append(f"        {estimate_name:<15}: {m_probs}")

            cc = combined_settings.get_comparison_column(cc_name)
            m_probs = _format_probs_for_report(cc["m_probabilities"])
            summary = f"{summary_name} value:"
            lines.append(f"        {summary:<15}: {m_probs}")

            lines.append(f"    u probabilities")

            for estimate_name, cc in dict_of_ccs.items():
                m_probs = _format_probs_for_report(cc["u_probabilities"])
                lines.append(f"        {estimate_name:<15}: {m_probs}")

            cc = combined_settings.get_comparison_column(cc_name)
            m_probs = _format_probs_for_report(cc["u_probabilities"])
            summary = f"{summary_name} value:"
            lines.append(f"        {summary:<15}: {m_probs}")

        return "\n".join(lines)

    def _estimates_as_rows(self):
        """A list of dicts represeting
        all the param estimates which can be passed
        t"""
        rows = []

        gathered = self._groups_of_comparison_columns_by_name()
        for dict_of_ccs in gathered.values():
            for estimate_name, cc in dict_of_ccs.items():
                new_rows = cc.as_rows()
                for r in new_rows:
                    r["estimate_name"] = estimate_name
                rows.extend(new_rows)

        return rows

    def comparison_chart(self):
        chart_def = load_chart_definition("compare_estimates.json")
        chart_def["data"]["values"] = self._estimates_as_rows()

        return altair_if_installed_else_json(chart_def)

    def _repr_pretty_(self, p, cycle):
        p.text(
            self.summary_report(
                summary_name="harmonic_mean", aggregate_function=harmonic_mean
            )
        )


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from openstack import exceptions
from openstack import resource
from openstack import utils

class ShareAccessRule(resource.Resource):
    resource_key = "share_access_rule"
    resources_key = "access_list"
    base_path = "/share-access-rules"

    # capabilities
    allow_create = True
    allow_fetch = True
    allow_commit = False
    allow_delete = True
    allow_list = True
    allow_head = False

    _query_mapping = resource.QueryParameters("share_id")

    _max_microversion = '2.45'

    #: Properties
    #: The access credential of the entity granted share access.
    access_key = resource.Body("access_key", type=str)
    #: The access level to the share.
    access_level = resource.Body("access_level", type=str)
    #: The object of the access rule.
    access_list = resource.Body("access_list", type=str)
    #: The value that defines the access.
    access_to = resource.Body("access_to", type=str)
    #: The access rule type.
    access_type = resource.Body("access_type", type=str)
    #: The date and time stamp when the resource was created within the
    #: service’s database.
    created_at = resource.Body("created_at", type=str)
    #: One or more access rule metadata key and value pairs as a dictionary
    #: of strings.
    metadata = resource.Body("metadata", type=dict)
    #: The UUID of the share to which you are granted or denied access.
    share_id = resource.Body("share_id", type=str)
    #: The state of the access rule.
    state = resource.Body("state", type=str)
    #: The date and time stamp when the resource was last updated within
    #: the service’s database.
    updated_at = resource.Body("updated_at", type=str)

    def _action(self, session, body, url, action='patch', microversion=None):
        headers = {'Accept': ''}

        if microversion is None:
            microversion = self._get_microversion(session, action=action)

        return session.post(
            url, json=body, headers=headers, microversion=microversion
        )

    def create(self, session, **kwargs):
        return super().create(
            session,
            resource_request_key='allow_access',
            resource_response_key='access',
            **kwargs
        )

    def delete(self, session, share_id, ignore_missing=True):
        body = {"deny_access": {"access_id": self.id}}
        url = utils.urljoin("/shares", share_id, "action")
        response = self._action(session, body, url)
        try:
            response = self._action(session, body, url)
            self._translate_response(response)
        except exceptions.ResourceNotFound:
            if not ignore_missing:
                raise
        return response


"""
Group: BugMaker
Game Features: Voice-activated Dragon, Rewards and punishments props., Challenging and fun
"""

import pygame
import os
import sys

from myVariables import *

def Initialize():

    pygame.init()
    pygame.mixer.init()

    # open the window in the centre of the screen
    os.environ['SDL_VIDEO_WINDOW_POS'] = "%d,%d" % ((screenResolution.current_w - SCREEN_WIDTH) / 2,
                                                    (screenResolution.current_h - SCREEN_HEIGHT) / 2)
    # set screen
    screen = pygame.display.set_mode([SCREEN_WIDTH, SCREEN_HEIGHT], pygame.DOUBLEBUF, 32)
    # set icon and caption
    pygame.display.set_icon(pygame.image.load('images/dragonicon.ico'))
    pygame.display.set_caption("Voice-activated Dragon Adventure")

    return screen

def draw_text(screen, text, y, size, colour="grey"):
    """
    Draw a black text bigger and a grey text smaller over it to create a 3D effect.
    """

    # set font
    font_black = pygame.font.Font("data/JE.TTF", size + 1)
    font_grey = pygame.font.Font("data/JE.TTF", size)

    # set colour and antialias
    text_black = font_black.render(str(text), 1, pygame.Color("black"))
    text_grey = font_grey.render(str(text), 1, pygame.Color(colour))

    # set x position
    x_black = (SCREEN_WIDTH - text_black.get_width()) / 2
    x_grey = (SCREEN_WIDTH - text_grey.get_width()) / 2

    # show the text in the screen
    screen.blit(text_black, (x_black + 5, y - 2))
    screen.blit(text_grey, (x_grey, y))

def end_the_game(screen, game_score):

    global high_score

    # draw a rectangle to show the result.
    pygame.draw.rect(screen, pygame.Color("black"), (43, SCREEN_HEIGHT / 2 - 77, 314, 154))
    pygame.draw.rect(screen, pygame.Color("lightgrey"), (45, SCREEN_HEIGHT / 2 - 75, 310, 150))

    draw_text(screen, "Your Score: " + str(game_score), 220, 35, "royalblue")

    # if there is a new record, replace it.
    if game_score > high_score:
        high_score = game_score
        draw_text(screen, "New Record!", 160, 35, "gold")

    draw_text(screen, "High score: " + str(high_score), 270, 35, "royalblue")
    draw_text(screen, "Click or Press {Space} to restart", 335, 20)
    draw_text(screen, "Press {Esc} to exit", 365, 20)

    pygame.display.flip()

    # Gets the keyboard events to see whether the user wants to continue to play
    while True:
        for e in pygame.event.get():
            if e.type == pygame.QUIT:
                pygame.quit()
                sys.exit()
            if e.type == pygame.KEYDOWN:
                if e.key == K_SPACE:
                    return False
                elif e.key == K_ESCAPE:
                    return True
            elif e.type == pygame.MOUSEBUTTONDOWN:
                return False


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.messages.views import SuccessMessageMixin
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic.edit import CreateView, DeleteView
from django.db.models import F, Q
from django.http import HttpResponse
import csv
from datetime import datetime, date, timedelta
from django.core.paginator import Paginator

from .models import Booking
from .forms import VenderBookingForm, CadastrarBookingForm, EditarBookingForm

# Create your views here.
@login_required(login_url='/login/')
def Index(request):
    bookings = Booking.objects.all()
    queryset = request.GET.get('q')
    if queryset:
        bookings = Booking.objects.filter(
            Q(number_booking__icontains=queryset))

    return render (request, 'booking/index.html', {'bookings': bookings})

class BookingCreate(SuccessMessageMixin,CreateView):
    today = date.today()
    model = Booking
    form_class = CadastrarBookingForm
    template_name = 'booking/cadastrar-booking.html'
    success_url = '/'
    success_message = 'Booking Cadastrado com sucesso!!!!'

    def form_valid(self, form):
        form.instance.status = 'Vazio'
        form.instance.cadastrado_por = self.request.user
        return super(BookingCreate, self).form_valid(form)

@login_required(login_url='/login/')
def EditarBookingDisponivel(request, id=None):
    booking = get_object_or_404(Booking, id=id)
    form = EditarBookingForm(request.POST or None, instance=booking)
    if form.is_valid():
        obj = form.save()
        obj.save()
        messages.success(request, 'Booking editado com sucesso.')
        return redirect('/')
    return render(request, 'booking/editar-booking-disponivel.html', {'form': form})

@login_required(login_url='/login/')
def ListaBookingDisponivel(request):
    from django.db.models.functions import ExtractDay
    from django.db.models import F, Func
    from django.db.models import Avg, F
    if request.user.funcionario.escritorio:
        bookings = Booking.objects.filter(status='Vazio').filter(escritorio=request.user.funcionario.escritorio)

        queryset = request.GET.get('q')
        if queryset:
            bookings = Booking.objects.filter(
                Q(number_booking__icontains=queryset)).filter(status='Vazio')
    else:
        bookings = Booking.objects.filter(status='Vazio')
        queryset = request.GET.get('q')
        if queryset:
            bookings = Booking.objects.filter(
                Q(number_booking__icontains=queryset)).filter(status='Vazio')

    paginator = Paginator(bookings, 10)  # Show 25 contacts per page

    page = request.GET.get('page')
    bookings = paginator.get_page(page)

    return render (request, 'booking/lista-booking-disponivel.html', {'bookings': bookings})

@login_required(login_url='/login/')
def RemoverBookingDisponivel(request, id=None):
    booking = get_object_or_404(Booking, id=id)
    if request.method == "POST":
        booking.delete()
        messages.success(request, 'Booking removido com sucesso!')
        return redirect('/booking/lista-booking-disponivel/')
    return render(request, 'booking/deletar-booking-disponivel.html', {'booking': booking})

@login_required(login_url='/login/')
def ListaBookingVendido(request):
    #Exibi a lista com apenas os bookings da filial do funcionario )
    if request.user.funcionario.escritorio:
        bookings = Booking.objects.filter(status='Vendido').filter(escritorio=request.user.funcionario.escritorio)
        queryset = request.GET.get('q')
        if queryset:
            bookings = Booking.objects.filter(
                Q(number_booking__icontains=queryset)).filter(status='Vendido')
    #Exibi a lista contendo todos os bookings
    else:
        bookings = Booking.objects.filter(status='Vendido')
        queryset = request.GET.get('q')
        if queryset:
            bookings = Booking.objects.filter(
                Q(number_booking__icontains=queryset)).filter(status='Vendido')
    return render (request, 'booking/lista-booking-vendido.html', {'bookings': bookings})

@login_required(login_url='/login/')
def RemoverBookingVendido(request, id=None):
    booking = get_object_or_404(Booking, id=id)
    if request.method == "POST":
        booking.delete()
        messages.success(request, 'Booking removido com sucesso!')
        return redirect('/booking/lista-booking-vendido/')
    return render(request, 'booking/deletar-booking-vendido.html', {'booking': booking})

@login_required(login_url='/login/')
def VenderBooking(request, id=None):
    booking = get_object_or_404(Booking, id=id)
    form = VenderBookingForm(request.POST or None, instance=booking)
    if form.is_valid():
        obj = form.save(commit=False)
        obj.status = 'Vendido'
        obj.vendido_por = request.user
        obj.save()
        messages.success(request, 'Booking vendido com sucesso')
        return redirect('/')
    return render(request, 'booking/vender-booking.html', {'form': form})

@login_required(login_url='/login/')
def EditarBookingVendido(request, id=None):
    booking = get_object_or_404(Booking, id=id)
    form = CadastrarBookingForm(request.POST or None, instance=booking)
    if form.is_valid():
        obj = form.save()
        obj.save()
        messages.success(request, 'Booking editado com sucesso.')
        return redirect('/')
    return render(request, 'booking/editar-booking-vendido.html', {'form': form})

@login_required(login_url='/login/')
def VisualizarBookingVendido(request):
    booking = request.GET.get('id')
    if booking:
        booking = Booking.objects.get(id=booking)
    return render(request, 'booking/visualizar-booking-vendido.html',{'booking': booking})

@login_required(login_url='/login/')
def VisualizarBookingDisponivel(request):
    booking = request.GET.get('id')
    if booking:
        booking = Booking.objects.get(id=booking)
    return render(request, 'booking/visualizar-booking-disponivel.html',{'booking': booking})

def ExportarCSV(request):
    # Create the HttpResponse object with the appropriate CSV header.
    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="relatorio-booking.csv"'

    bookings = Booking.objects.filter(escritorio=request.user.funcionario.escritorio)

    writer = csv.writer(response)
    writer.writerow(['id','number_booking','pol','pod','navio','commodity','eta','armador','quantidade','type','status',
                     'cotacao','shipper','contrato_venda','cadastrado_por','data_cadastro','vendido_por','data_venda',
                     'escritorio','data_ddl_draft','hora_ddl_draft','observacoes'
                     ])
    for booking in bookings:
        writer.writerow([booking.id ,booking.number_booking,booking.pol,booking.pod,booking.navio,booking.commodity,
                         booking.eta ,booking.armador, booking.quantidade, booking.type, booking.status, booking.cotacao,
                         booking.shipper, booking.contrato_venda, booking.cadastrado_por, booking.data_cadastro,
                         booking.vendido_por, booking.data_venda, booking.escritorio,booking.data_ddl_draft,
                         booking.hora_ddl_draft ,booking.observacoes
                         ])
    return response


from flask import Blueprint, render_template, request, redirect
import os

pages = Blueprint('pages', __name__)

@pages.route("/", methods = ["GET","POST"])
def home():
    if request.method == "POST":
        if request.files:

            csv = request.files["csv"]
            csv.save(os.path.join("uploads", csv.filename))
            return redirect(request.url)

    return render_template("index.html")


import logging
import httplib2
import json

import dataviz.geonames as geonames
import webapp2
from google.appengine.api import memcache
from apiclient.discovery import build
from oauth2client.appengine import AppAssertionCredentials

PROJECT_ID = ""
pkey = ""

MOST_FORKED_REPO_QUERY = """
SELECT repository_name, repository_owner, MAX(repository_forks) as maxi
FROM githubarchive:github.timeline
WHERE type = 'ForkEvent' AND
    LENGTH(actor_attributes_location) > 3 AND repository_forks > 200
GROUP BY repository_name, repository_owner
ORDER BY maxi DESC
LIMIT 20
"""

REPO_TIMELINE = """
SELECT created_at, actor_attributes_location, repository_forks
FROM githubarchive:github.timeline
WHERE type = 'ForkEvent' AND
    LENGTH(actor_attributes_location) > 3 AND
    repository_name = '%s'
    AND repository_owner = '%s' AND
    repository_fork = 'false'
ORDER BY repository_forks DESC
"""

# In seconds.
CACHE_EXPIRATION_TIME = 3600 * 24

queries = {
    "top": MOST_FORKED_REPO_QUERY,
    "repo": REPO_TIMELINE,
}

class Bigquery(webapp2.RequestHandler):
    def get(self, resource):
        self.response.headers['Content-Type'] = 'application/json'
        repo_name = self.request.get("r", "")
        repo_owner = self.request.get("o", "")
        key = "-".join([resource, repo_name, repo_owner])
        response = memcache.get(key)
        if response is None:
            logging.warn("Cache miss for GET.")
            if resource == "repo":
                response = get_repo_timeline(repo_name, repo_owner)
                logging.info("Called timeline")
            elif resource == "top":
                response = get_top_repositories()
                logging.info("Called top")
            else:
                response = []
                logging.warn("Bad query %r" % resource)
            memcache.add(key, response, CACHE_EXPIRATION_TIME)
        self.response.out.write(json.dumps(response))

def service():
    credentials = AppAssertionCredentials(
        scope='https://www.googleapis.com/auth/bigquery')
    http = credentials.authorize(httplib2.Http(memcache))
    return build("bigquery", "v2", http=http)

def get_top_repositories():
    query = queries['top']
    results = request(query)
    response = []
    for row in results['rows']:
        response.append({
            "name": row['f'][0]['v'],
            "owner": row['f'][1]['v'],
            "forks": row['f'][2]['v'],
        })
    return response

def get_repo_timeline(repo_name, repo_owner):
    query = queries['repo'] % (repo_name, repo_owner)
    results = request(query)
    response = []
    for row in results['rows']:
        city = geonames.locate(row['f'][1]['v'])
        response.append({
            "date": row['f'][0]['v'],
            "name": city.get('name', ""),
            "coord": city.get('coord', []),
            "totalForks": row['f'][2]['v'],
            "forks": 0,
        })
    return response

def request(query):
    results = memcache.get(query)
    if results is None:
        logging.warn("Cache miss for bigquery.")
        results = service().jobs().query(
            body={"query": query},
            projectId=PROJECT_ID
        ).execute()
        memcache.add(query, results, CACHE_EXPIRATION_TIME)
    return results


from django.db import models

# Create your models here.

class Device(models.Model):
    TYPES = [
        ('UK', 'Neznámé'),
        ('RT', 'Směrovač'),
        ('SW', 'Přepínač'),
        ('AP', 'Přístupový bod'),
        ('NS', 'Síťové úložiště'),
        ('PT', 'Tiskárna'),
        ('MC', 'Multimediální centrum'),
        ('PC', 'Stolní počítač'),
        ('NB', 'Notebook'),
        ('MB', 'Mobilní telefon'),
    ]
    type = models.CharField(
        max_length=2,
        choices=TYPES,
        default='UK',
    )
    ip_address = models.CharField(
        max_length=15,
        unique=True,
        null=False,
        blank=False,
    )
    name = models.CharField(
        max_length=80,
        null=False,
        blank=False,
    )
    description = models.CharField(
        max_length=150,
        null=True,
        blank=True,
    )
    default_url = models.OneToOneField(
        'Url',
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        related_name='default_for_device',
    )
    created_at = models.DateTimeField(
        auto_now_add=True,
    )
    updated_at = models.DateTimeField(
        auto_now=True,
    )

    def __str__(self):
        return str(type) + ' ' + str(self.name)

class Url(models.Model):
    device = models.ForeignKey(
        'Device',
        on_delete=models.CASCADE,
        related_name='urls',
        null=False,
        blank=False,
    )
    name = models.CharField(
        max_length=80,
        null=False,
        blank=False,
        default='Neznámá URL',
    )
    url = models.URLField(
        null=False,
        blank=False,
    )


from starlette.requests import Request

from .template import templates

async def bad_request(request: Request, exc: Exception):
    is_htmx = request.scope["htmx"].is_htmx()
    if is_htmx:
        template = "htmx/400.jinja2"
    else:
        template = "400.jinja2"
    return templates.TemplateResponse(template, {'request': request, "exc": exc}, status_code=400) # type: ignore

async def not_found(request: Request, exc: Exception):
    is_htmx = request.scope["htmx"].is_htmx()
    if is_htmx:
        template = "htmx/404.jinja2"
    else:
        template = "404.jinja2"
    return templates.TemplateResponse(template, {'request': request, "exc": exc}, status_code=404) # type: ignore

async def internal_error(request: Request, exc: Exception):
    is_htmx = request.scope["htmx"].is_htmx()
    if is_htmx:
        template = "htmx/500.jinja2"
    else:
        template = "500.jinja2"
    return templates.TemplateResponse(template, {'request': request, "exc": exc}, status_code=500) # type: ignore

async def cloudflare_error_page(request: Request):
    placeholder = request.query_params.get("placeholder")
    return templates.TemplateResponse('cf.jinja2', {'request': request, "placeholder": placeholder}) # type: ignore


import io
import os
import threading
import time
from tempfile import TemporaryFile

import h5py
import requests
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
from flask import Flask, json
from flask import request, Response
import time
from logging.config import dictConfig

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow import contrib

import tempfile
from tempfile import TemporaryFile

# from cryptography.fernet import Fernet
from ..utility.native_encryption import NativeEncryptor

tmpdir = tempfile.mkdtemp()

tf.enable_eager_execution()
tfe = contrib.eager
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
STORAGE_SVC_NAME = os.getenv("STORAGE_SVC_NAME")
SECUREML_NAMESPACE = os.getenv("SECUREML_NAMESPACE")
FRONTEND_SVC_NAME = os.getenv("FRONTEND_SVC_NAME")
MODEL_KEY = b"needstobesixteen"

encryptor_m = NativeEncryptor(MODEL_KEY)
# encryptor_m = Fernet(MODEL_KEY)

dictConfig({
    'version': 1,
    'formatters': {'default': {
        'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
    }},
    'handlers': {'wsgi': {
        'class': 'logging.StreamHandler',
        'formatter': 'default'
    }},
    'root': {
        'level': 'DEBUG',
        'handlers': ['wsgi']
    }
})

app = Flask(__name__)

LEARNING_RATE = 0.001
BATCH_SIZE = 32
WORKER_ID = 0

NEXT_ITERATION_URL = f'http://{FRONTEND_SVC_NAME}.{SECUREML_NAMESPACE}/start_iteration/'
STORAGE_URL = f'http://{STORAGE_SVC_NAME}.{SECUREML_NAMESPACE}/'

cce = keras.losses.SparseCategoricalCrossentropy()
optimizer = keras.optimizers.RMSprop(learning_rate=LEARNING_RATE)
global_step = tf.Variable(0)

global_model = None
gradient_list = []
start_time = None
started_lock = False

thread_lock = threading.Lock()

# Create the BlobServiceClient object which will be used to create a container client
blob_service_client = BlobServiceClient.from_connection_string(connect_str)
# Create a unique name for the container
gradient_container_client = blob_service_client.get_container_client('gradients')
model_container_client = blob_service_client.get_container_client('model')
# Create the container
# container_client = blob_service_client.create_container(container_name)

def uncompress_nparr(bytestring):
    """
    """
    return np.load(io.BytesIO(bytestring), allow_pickle=True)

def bytestream_nparr(nparr):
    """
    Returns the given numpy array as compressed bytestring,
    the uncompressed and the compressed byte size.
    """
    bytestream = io.BytesIO()
    np.save(bytestream, nparr)
    return bytestream

def download_blob(container_name, blob_name, local_name):
    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
    # outfile = TemporaryFile()
    # outfile.write(blob_client.download_blob().readall())
    # return outfile
    local_name = tmpdir + '/' + local_name
    with open(local_name, "wb") as download_file:
        download_file.write(blob_client.download_blob().readall())
    return local_name

def download_decrypt_blob(container_name, blob_name, local_name, encryptor):
    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
    # outfile = TemporaryFile()
    # outfile.write(blob_client.download_blob().readall())
    # return outfile
    local_name = tmpdir + '/' + local_name
    with open(local_name, "wb") as download_file:
        download_file.write(encryptor.decrypt(blob_client.download_blob().readall()))
    return local_name

def download_decrypt_gradients(container_name, blob_list, encryptor):
    def download_worker(_container_name, blob_name):
        blob_client = blob_service_client.get_blob_client(container=_container_name, blob=blob_name)
        bio = io.BytesIO()
        blob_client.download_blob().readinto(bio)
        return bio
    grad_streams = [download_worker(container_name, blob.name) for blob in blob_list]
    # grad_streams = Parallel(n_jobs=3, prefer="threads")(delayed(download_worker)(container_name, blob.name) for blob in blob_list)
    decrypted_s = [io.BytesIO(encryptor.decrypt(s.getvalue())) for s in grad_streams]
    return decrypted_s

def upload_blob_file(container_name, local_file, blob_name):
    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
    local_file.seek(0)
    blob_client.upload_blob(local_file)

def upload_blob(container_name, data: bytes, blob_name):
    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
    blob_client.upload_blob(data)

def upload_model_mongo(file_name, model):
    requests.post(STORAGE_URL+'upload_model/'+file_name, data=model,
                  headers={'Content-Type': 'application/octet-stream'})

def download_model_mongo(file_name, encryptor):
    response = requests.get(STORAGE_URL+'download_model/'+file_name)
    bio = io.BytesIO(encryptor.decrypt(response.content))
    bio = h5py.File(bio, mode='r')
    return bio

def download_decrypt_gradients_mongo(iter_num, worker_id, encryptor):
    response = requests.get(STORAGE_URL+'download_gradients/'+'gradients_{:03d}_{:03d}'.format(worker_id, iter_num))
    grads = uncompress_nparr(encryptor.decrypt(response.content))
    return grads

@app.route('/upload_gradient/<worker_id>', methods=['POST'])
def upload_gradient(worker_id):
    global gradient_list, thread_lock, start_time, started_lock
    app.logger.info("start receiving grads. worker: {:03d}".format(int(worker_id)))
    grads = uncompress_nparr(encryptor_m.decrypt(request.data))
    app.logger.info("grads decrypted. worker: {:03d}".format(int(worker_id)))
    thread_lock.acquire()
    gradient_list.append(grads)
    if started_lock == False:
        start_time = time.time()
        started_lock = True
    thread_lock.release()
    app.logger.info("grads saved. worker: {:03d}".format(int(worker_id)))

    return Response(status=200)

@app.route('/aggregate/<prefix>', methods=['POST'])
def aggregate(prefix):
    global global_model
    start_t = time.time()
    # getting masks matching prefix
    iter_num = int(request.json['iter_num'])

    blob_list = gradient_container_client.list_blobs(name_starts_with='masked_grads_{0:03}_'.format(iter_num))
    grad_files = [download_decrypt_blob('gradients', blob.name, blob.name, encryptor_m) for blob in blob_list]
    grad_list = [np.load(grad_file, allow_pickle=True) for grad_file in grad_files]

    sum_grads = np.sum(grad_list, axis=0)

    optimizer.apply_gradients(zip(sum_grads, global_model.trainable_variables), global_step)

    out_file = io.BytesIO()
    global_model.save(out_file)
    encrypted_model = encryptor_m.encrypt(out_file.getvalue())
    upload_blob('model', encrypted_model, 'mnist_{:03d}'.format(iter_num+1))
    # bytestream = bytestream_nparr(grads)
    elapsed_time = time.time() - start_t
    app.logger.debug('Finished in {:4f}s'.format(elapsed_time))
    return Response(status=200)

@app.route('/aggregate_inm/<prefix>', methods=['POST'])
def aggregate_inm(prefix):
    global global_model, thread_lock, gradient_list
    start_t = time.time()
    # getting masks matching prefix
    iter_num = int(request.json['iter_num'])

    thread_lock.acquire()
    sum_grads = np.sum(gradient_list, axis=0)
    gradient_list = []
    thread_lock.release()

    # optimizer = keras.optimizers.RMSprop(learning_rate=LEARNING_RATE)
    # global_step = tf.Variable(0)
    optimizer.apply_gradients(zip(sum_grads, global_model.trainable_variables), global_step)
    # out_file = TemporaryFile()
    out_file = io.BytesIO()
    global_model.save(out_file)
    encrypted_model = encryptor_m.encrypt(out_file.getvalue())
    upload_blob('model', encrypted_model, 'mnist_{:03d}'.format(iter_num+1))
    # bytestream = bytestream_nparr(grads)
    elapsed_time = time.time() - start_t
    app.logger.debug('Finished in {:4f}s'.format(elapsed_time))
    return Response(status=200)

@app.route('/finish_iteration/<iter_num>', methods=['POST'])
def finish_iteration(iter_num):
    global global_model

    start_t = time.time()
    app.logger.info("start to finish aggregation. iter_num: {:03d}".format(int(iter_num)))

    app.logger.info("start decrypting grads. iter_num: {:03d}".format(int(iter_num)))
    grads = uncompress_nparr(encryptor_m.decrypt(request.data))
    app.logger.info("grads decrypted. iter_num: {:03d}".format(int(iter_num)))

    app.logger.info("start applying grads. iter_num: {:03d}".format(int(iter_num)))
    optimizer.apply_gradients(
        zip(grads, global_model.trainable_variables), global_step)
    app.logger.info("grads applied. iter_num: {:03d}".format(int(iter_num)))

    app.logger.info("start serialize model. iter_num: {:03d}".format(int(iter_num)))
    out_file = io.BytesIO()
    global_model.save(out_file)
    app.logger.info("model serialized. iter_num: {:03d}".format(int(iter_num)))

    app.logger.info("start encrypting model. iter_num: {:03d}".format(int(iter_num)))
    encrypted_model = encryptor_m.encrypt(out_file.getvalue())
    app.logger.info("model encrypted. iter_num: {:03d}".format(int(iter_num)))

    app.logger.info("start uploading model. iter_num: {:03d}".format(int(iter_num)))
    upload_model_mongo('mnist_{:03d}'.format(int(iter_num)+1), encrypted_model)
    app.logger.info("model uploaded. iter_num: {:03d}".format(int(iter_num)))

    elapsed_time = time.time() - start_t
    app.logger.debug('Finished in {:4f}s'.format(elapsed_time))

    app.logger.info("start informing frontend to start next iteration. iter_num: {:03d}".format(int(iter_num)))
    response = requests.get(NEXT_ITERATION_URL + str(int(iter_num)+1))
    app.logger.info("inform done. iter_num: {:03d}".format(int(iter_num)))

    return Response(status=200)

@app.route('/aggregate_mongo/<prefix>', methods=['POST'])
def aggregate_mongo(prefix):
    global global_model, thread_lock, gradient_list, start_time, started_lock
    # start_t = time.time()
    # getting masks matching prefix
    iter_num = int(request.json['iter_num'])
    app.logger.info("starting aggregation. iter_num: {:03d}".format(iter_num))

    thread_lock.acquire()
    sum_grads = np.sum(gradient_list, axis=0)
    gradient_list = []
    thread_lock.release()
    app.logger.info("grads aggregated. iter_num: {:03d}".format(iter_num))

    # optimizer = keras.optimizers.RMSprop(learning_rate=LEARNING_RATE)
    # global_step = tf.Variable(0)
    optimizer.apply_gradients(
        zip(sum_grads, global_model.trainable_variables), global_step)
    app.logger.info("grads applied. iter_num: {:03d}".format(iter_num))
    # out_file = TemporaryFile()
    out_file = io.BytesIO()
    global_model.save(out_file)
    app.logger.info("model serialized. iter_num: {:03d}".format(iter_num))
    encrypted_model = encryptor_m.encrypt(out_file.getvalue())
    app.logger.info("model encrypted. iter_num: {:03d}".format(iter_num))
    upload_model_mongo('mnist_{:03d}'.format(iter_num+1), encrypted_model)
    app.logger.info("model uploaded. iter_num: {:03d}".format(iter_num))
    elapsed_time = time.time() - start_time
    app.logger.debug('Finished in {:4f}s'.format(elapsed_time))
    started_lock = False
    return Response(status=200)

if __name__ == "__main__":
    # print('lala')
    # threading.Thread(target=app.run).start()
    global_model = download_model_mongo('mnist_000', encryptor_m)
    global_model = keras.models.load_model(global_model)
    app.run(debug=False, port=80, host='0.0.0.0', threaded=True)


#!/usr/bin/env python3
import re
import sys
from collections import defaultdict
from typing import List, Dict, Set, Tuple

Colour = str
QuantityOfColour = Tuple[int, Colour]
ParentsMap = Dict[Colour, Set]
ChildrenMap = Dict[Colour, List[QuantityOfColour]]

def find_parents(colour: Colour, parents: ParentsMap, found_parents: Set[Colour]):
    for parent in parents[colour]:
        found_parents.add(parent)
        find_parents(parent, parents, found_parents)

def count_bags(colour: Colour, children: ChildrenMap) -> int:
    return sum(map(lambda k: k[0] * (count_bags(k[1], children) + 1), children[colour]))

def hash_parents_and_children_lookups(lines: List[str],) -> (ParentsMap, ChildrenMap):
    parents_map: ParentsMap = defaultdict(set)
    children_map: ChildrenMap = defaultdict(list)
    for line in lines:
        parent_colour = re.match(r"(.+?) bags contain", line)[1]
        for num, child_colour in re.findall(r"(\d+) (.+?) bags?[,.]", line):
            parents_map[child_colour].add(parent_colour)
            children_map[parent_colour].append((int(num), child_colour))
    return parents_map, children_map

def main():
    lines = [line.strip() for line in open("../inputs/07_01.txt", "r")]
    parents, children = hash_parents_and_children_lookups(lines)

    golden_parents = set()
    find_parents("shiny gold", parents, golden_parents)
    print(len(golden_parents))

    print(count_bags("shiny gold", children))

if __name__ == "__main__":
    sys.exit(main())


import re
import tweepy
import time
import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import SGDClassifier

from sklearn import metrics

import matplotlib.pyplot as plt

import mysql.connector
from textblob import TextBlob

# app = Flask(__name__)

# @app.route("/")
# def index():
#     return "Hello World!"

def main():

    # tweets = pd.DataFrame(columns=['id','term','text','sent'])

    term = ''

    while True:
        term = input('Enter a search term: ')
        if term in ['exit', 'e', 'stop']:
            break
        elif term == 'stream':

            strm.append((input('Enter a term to stream: ')))

            getStream(strm[-1])
        else:
            tweets = {
                'term': [],
                'text': [],
                'sent': [],
            }

            getTweets(term, tweets)
            tweet_df = pd.DataFrame.from_dict(tweets)
            print(tweet_df.head(10))
            barChart(tweets)

            # tweet_df = pd.DataFrame(columns=['id','term','text','sent'])
            # database_option = input("Save results to database? ")
            # if database_option in ['Y', 'y', 'yes']:
            #     connectSQL(tweets, table='tweets')

            action = input('Save tweets to file? ')
            if action in ['yes', 'y', 'save']:
                saveTweets(tweets)

class tweepStreamListener(tweepy.StreamListener):

    def on_status(self, status):

        try:
            streamTextSize(status.extended_tweet['full_text'])

        except AttributeError:
            streamTextSize(status.text)

        # if not status.truncated:
        # else:

    def on_error(self, status_code):
        return super().on_error(status_code)

def streamTextSize(text):
    clean_stream = cleanTweet(text)

    # stream_tweets["id"].append(id)
    stream_tweets['term'].append(strm[-1])
    stream_tweets['text'].append(clean_stream)
    stream_tweets['sent'].append(nbSentiment(clean_stream))

def getStream(strm):
    runtime = int(input('how long should the stream run?: '))
    print('Now istening for: ' + strm)
    stream_listener = tweepStreamListener()
    stream = tweepy.Stream(auth=api.auth, listener=stream_listener)
    stream.filter(track=strm, is_async=True)
    time.sleep(runtime)

    print('Time\'s Up! Stream Connection Terminated.')
    stream.disconnect()
    #print (stream_tweets)
    stream_df = pd.DataFrame(stream_tweets)
    print(stream_df.head(10))

    barChart(stream_tweets)
    # database_option = input("Save results to database? ")
    # if database_option in ['Y', 'y', 'yes']:
    #     connectSQL(stream_tweets, table='stream')

    save_option = input('Save stream results to file? ')
    if save_option in ['Y', 'y', 'yes']:
        saveTweets(stream_tweets)

# def connectSQL(tweets, table):

#     try:
#         connection = mysql.connector.connect(
#             user="root",
#             password="",
#             host="127.0.0.1",
#             database="pytwitter"
#         )

#         cursor = connection.cursor()

#         addtoTable(connection, tweets, table)

#     except connection.Error as err:
#         print(err)

# def addtoTable(connection, tweets, table):
#     x = connection.cursor()
#     for i in range(0, len(tweets['text'])):
#         if table == 'tweets':
#             x.execute('INSERT INTO tweets (search,text,sentiment) VALUES (%s,%s,%s)', (str(
#                 tweets['term'][i]), str(tweets['text'][i]), str(tweets['sent'][i])))

#         elif table == 'stream':
#             x.execute("INSERT INTO streamtweets (search,text,sentiment) VALUES (%s,%s,%s)", (str(
#                 tweets['term'][i]), str(tweets['text'][i]), str(tweets['sent'][i])))

#     connection.commit()

def getTweets(term, tweets):
    for t in tweepy.Cursor(api.search, q='%s -filter:retweets' % term, lang='en', tweet_mode='extended').items(50):
        clean_text = str(cleanTweet(t.full_text))
        if clean_text not in tweets['text']:

            # print(clean_text)
            # tweets = tweets.append({'id': t.id, 'term': term, 'text': clean_text , 'sent': tblobSentiment(clean_text)}, ignore_index=True)

            # tweets["id"].append(t.id)
            tweets['term'].append(term)
            tweets['text'].append(clean_text)
            tweets['sent'].append(nbSentiment(clean_text))
            # tweets["like"].append(t.retweeted_status.favorite_count)
            # tweets["rtwt"].append(t.retweeted_status.retweet_count)
            # tweets["date"].append(t.created_at)
            # tweets["geo"].append(t.geo)

    return tweets

def nbTrain(dataset):
    # dataset.head()
    pass
    # X_train_counts = count_vect.fit_transform(dataset.data)
    # X_train_counts.shape

def tblobSentiment(t):  # function for checking tweet polarity
    tw_sent = TextBlob(t)
    if tw_sent.sentiment.polarity > 0:
        return 'Positive'  # positive tweets
    elif tw_sent.sentiment.polarity == 0:
        return 'Neutral'  # negative tweets
    else:
        return 'Negative'  # tweets with no strong polarity

def cleanTweet(t):
    # clean_text=re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t]) |(\w+:\/\/\S+)", " ", t).split()
    t = re.sub('@[A-Za-z0-9]+', '', t)  # removes Twitter handles/@symbol
    t = re.sub(r'(?:\@|http?\://|https?\://|www)\S+', '', t)  # removes URLs
    t = t.replace("#", "").replace("_", " ")  # remove hashtags and underscores
    t = " ".join(t.split())
    return t

def saveTweets(tweets):
    # try:
    with open(input('Enter a file name: '), 'a+', encoding='utf-8') as tdata:
        tdata.seek(0)
        if tdata.read() == '':
            tdata.write('Search, Tweet, Sentiment\n')

        rows = len(tweets['term'])
        for i in range(0, len(tweets['term'])):
            tdata.write(str(tweets['term'][i]) + ',' + str(tweets['text']
                        [i]).replace(',', ' ')+','+str(tweets['sent'][i])+'\n')

        print('Saved ' + str(rows) + ' rows.\n')

    # # tdata.write(str(tweets["term"][i])+ "," + str(tweets["text"][i]).replace(","," ")+","+str(tweets["sent"][i])+","
    # # +str(tweets["like"][i])+","+str(tweets["rtwt"][i])+"\n")
    #     else:
    #         tdata.write(str(tweets["term"][i]) + "," + str(tweets["text"]
    #                     [i]).replace(",", " ")+","+str(tweets["sent"][i])+"\n")
    tdata.close()
    # except:
    #     print("Error with save.")

def barChart(t):
    bar_data = {'pos':0, 'neg':0, 'neu':0}
    bar_sent = []
    for s in t['sent']:
        bar_sent.append(str(s))

    for i in bar_sent:
        if i == 'Positive':
            bar_data['pos'] += 1
        elif i == 'Negative':
            bar_data['neg'] += 1
        else:
            bar_data['neu'] += 1

    bkey = bar_data.keys()
    bval = bar_data.values()
    # %matplotlib inline
    plt.bar(bkey,bval)
    plt.show()

if __name__ == "__main__":
    # app.run(host="127.0.0.1", port=8080, debug=True)
    # Twitter API keys
    consumer_key = ""
    consumer_secret = ""
    access_token = ""
    access_token_secret = ""

    auth = tweepy.auth.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)

    api = tweepy.API(auth)  # authorises and enables Tweepy API

    stream_tweets = {
        # "id": [],
        "term": [],
        "text": [],
        "sent": []
    }

    strm = []

    fields = ['tweet', 'polarity']
    # findcsv = open(,'r')
    # trainingcsv = findcsv.read()
    traning_data = pd.read_csv('d:/OneDrive/LSBU/Year 3/Final Year Project/mysite/business.csv',
                               usecols=[1, 2], low_memory=True, names=fields, dtype=str)

    csv_df = pd.DataFrame(traning_data)

    print(csv_df)

    tweets_corpus = []
    for i in traning_data['tweet']:
        tweets_corpus.append(cleanTweet(i))

    vect = TfidfVectorizer(stop_words='english', ngram_range=(1, 3))
    X = vect.fit_transform(tweets_corpus)
    y = traning_data['polarity']
    mnb = MultinomialNB(fit_prior=True).fit(X, y)

    def nbSentiment(t):
        t = [t]
        X_test = vect.transform(t)
        y_pred = mnb.predict(X_test)

        t = y_pred[-1]

        # if y_pred == 'Negative':
        #     t = 'Negative'
        # elif y_pred == '4':
        #     t = 'Positive'
        # else:
        #     t = 'Neutral'

        return t

    main()


"""
An application to process imaginary purchase transactions in real-time using Kafka
running on Quix platform.

In this application, we will simulate notifications for "Gold" accounts about
purchase events larger than $1000
"""

from dotenv import load_dotenv

from quixstreams import Application, MessageContext, State

# Reminder: the platform will have these values available by default so loading the
# environment would be unnecessary there.
load_dotenv("./bank_example/quix_platform_version/quix_vars.env")

def uppercase_source(value: dict, ctx: MessageContext):
    """
    Upper-case field "transaction_source" for each processed message

    :param value: message value, a dictionary with all deserialized message data
    :param ctx: message context, it contains message metadata like key, topic, timestamp
        etc.

    :return: this function must either return None or a new dictionary
    """
    print(f'Processing message with key "{ctx.key}"')
    value["transaction_source"] = value["transaction_source"].upper()
    return value

def count_transactions(value: dict, ctx: MessageContext, state: State):
    """
    Track the number of transactions using persistent state

    :param value: message value
    :param ctx: message context with key, timestamp and other Kafka message metadata
    :param state: instance of State store
    """
    total = state.get("total_transactions", 0)
    total += 1
    state.set("total_transactions", total)
    value["total_transactions"] = total

# Define your application and settings
# Quix application is automatically configured to work with Quix platform
app = Application.Quix(
    "qts__purchase_notifier",
    auto_offset_reset="earliest",
    auto_create_topics=True,  # Quix app has an option to auto create topics
)

# Define an input topic with Quix deserializer
input_topic = app.topic("qts__purchase_events", value_deserializer="quix")

# Define an output topic with Quix Timeseries serializer
output_topic = app.topic("qts__user_notifications", value_serializer="quix_timeseries")

# Create a StreamingDataFrame and start building your processing pipeline
sdf = app.dataframe(input_topic)

# Filter only messages with "account_class" == "Gold" and "transaction_amount" >= 1000
sdf = sdf[
    (sdf["account_class"] == "Gold")
    & (sdf["transaction_amount"].apply(lambda x, ctx: abs(x)) >= 1000)
]

# Drop all fields except the ones we need
sdf = sdf[["account_id", "transaction_amount", "transaction_source"]]

# Update the total number of transactions in state
sdf = sdf.apply(count_transactions, stateful=True)

# Transform field "transaction_source" to upper-case using a custom function
sdf = sdf.apply(uppercase_source)

# Add a new field with a notification text
sdf["customer_notification"] = "A high cost purchase was attempted"

# Print the transformed message to the console
sdf = sdf.apply(lambda val, ctx: print(f"Sending update: {val}"))

# Send the message to the output topic
sdf.to_topic(output_topic)

if __name__ == "__main__":
    # Start message processing
    app.run(sdf)