import torch
from torch import nn
from torch.nn import functional as F
from torch_geometric.nn import MessagePassing, global_sort_pool
from torch_geometric.utils import add_self_loops, degree
from sklearn.metrics import roc_auc_score
from ThisDataset import ThisDataset
import random
import yaml
random.seed(42)
def warn(*args, **kwargs):
    pass
import warnings
warnings.warn = warn
class DGCNN(nn.Module):
    """
    Uses fixed architecture
    """

    def __init__(self, dim_features, dim_target, config):
        super(DGCNN, self).__init__()

        self.ks = {'NCI1': { '0.6': 30, '0.9': 46 },
                   'PROTEINS_full': { '0.6': 32, '0.9': 81 },
                   'DD': {'0.6': 291, '0.9': 503 },
                   'ENZYMES': { '0.6': 36, '0.9': 48 },
                   'IMDB-BINARY': { '0.6': 18, '0.9': 31 },
                   'IMDB-MULTI': { '0.6': 11, '0.9': 22 },
                   'REDDIT-BINARY': { '0.6': 370, '0.9': 1002 },
                   'REDDIT-MULTI-5K': { '0.6': 469, '0.9': 1081 },
                   'COLLAB': { '0.6': 61, '0.9': 130 },
                   }

        self.k = 30#self.ks[config.dataset.name][str(config['k'])]
        self.embedding_dim = config['embedding_dim'][0]
        self.num_layers = config['num_layers'][0]

        self.convs = []
        for layer in range(self.num_layers):
            input_dim = dim_features if layer == 0 else self.embedding_dim
            self.convs.append(DGCNNConv(input_dim, self.embedding_dim))
        self.total_latent_dim = self.num_layers * self.embedding_dim

        # Add last embedding
        self.convs.append(DGCNNConv(self.embedding_dim, 1))
        self.total_latent_dim += 1

        self.convs = nn.ModuleList(self.convs)

        # should we leave this fixed?
        self.conv1d_params1 = nn.Conv1d(1, 16, self.total_latent_dim, self.total_latent_dim)
        self.maxpool1d = nn.MaxPool1d(2, 2)
        self.conv1d_params2 = nn.Conv1d(16, 32, 5, 1)

        dense_dim = int((self.k - 2) / 2 + 1)
        self.input_dense_dim = (dense_dim - 5 + 1) * 32

        self.hidden_dense_dim = config['dense_dim'][0]
        self.dense_layer = nn.Sequential(nn.Linear(self.input_dense_dim, self.hidden_dense_dim),
                                         nn.ReLU(),
                                         nn.Dropout(p=0.5),
                                         nn.Linear(self.hidden_dense_dim, dim_target))

    def forward(self, data):
        # Implement Equation 4.2 of the paper i.e. concat all layers' graph representations and apply linear model
        # note: this can be decomposed in one smaller linear model per layer
        x, edge_index, batch = data.x, data.edge_index, data.batch

        hidden_repres = []

        for conv in self.convs:
            x = torch.tanh(conv(x, edge_index))
            hidden_repres.append(x)

        # apply sortpool
        x_to_sortpool = torch.cat(hidden_repres, dim=1)
        x_1d = global_sort_pool(x_to_sortpool, batch, self.k)  # in the code the authors sort the last channel only

        # apply 1D convolutional layers
        x_1d = torch.unsqueeze(x_1d, dim=1)
        conv1d_res = F.relu(self.conv1d_params1(x_1d))
        conv1d_res = self.maxpool1d(conv1d_res)
        conv1d_res = F.relu(self.conv1d_params2(conv1d_res))
        conv1d_res = conv1d_res.reshape(conv1d_res.shape[0], -1)

        # apply dense layer
        out_dense = self.dense_layer(conv1d_res)
        return out_dense

class DGCNNConv(MessagePassing):
    """
    Extended from tuorial on GCNs of Pytorch Geometrics
    """

    def __init__(self, in_channels, out_channels):
        super(DGCNNConv, self).__init__(aggr='add')  # "Add" aggregation.
        self.lin = nn.Linear(in_channels, out_channels)

    def forward(self, x, edge_index):
        # x has shape [N, in_channels]
        # edge_index has shape [2, E]

        # Step 1: Add self-loops to the adjacency matrix.
        edge_index, _ = add_self_loops(edge_index, num_nodes=x.size(0))

        # Step 2: Linearly transform node feature matrix.
        x = self.lin(x)

        # Step 3-5: Start propagating messages.
        return self.propagate(edge_index, size=(x.size(0), x.size(0)), x=x)

    def message(self, x_j, edge_index, size):
        # x_j has shape [E, out_channels]

        # Step 3: Normalize node features.
        src, dst = edge_index  # we assume source_to_target message passing
        deg = degree(src, size[0], dtype=x_j.dtype)
        deg = deg.pow(-1)
        norm = deg[dst]

        return norm.view(-1, 1) * x_j  # broadcasting the normalization term to all out_channels === hidden features

    def update(self, aggr_out):
        # aggr_out has shape [N, out_channels]

        # Step 5: Return new node embeddings.
        return aggr_out

    def __repr__(self):
        return '{}({}, {})'.format(self.__class__.__name__, self.in_channels,
                                   self.out_channels)

with open("config_DGCNN.yml", "r") as f:
    config = yaml.load(f)
addFile = "/home/bcypher/IdeaProjects/Orbit/articleOrbits.csv"
dataset= ThisDataset(root="data/", n_graphs=5000,sampleWhite= 50, filepath="/home/bcypher/PycharmProjects/PytorchTutorial/data/Orbit/graphs/", addressFile=addFile)
dataset = dataset.shuffle()
from torch_geometric.loader import DataLoader
print(f'Dataset num node features: {dataset.num_node_features:.2f}')
print(f'Dataset num classes: {dataset.num_classes:.2f}')
print(f'Dataset num graphs: {len(dataset)}')
#print(f'Dataset classes: {dataset.vals}')

#train_dataset = dataset[len(dataset) // 10:]
#train_loader = DataLoader(train_dataset, 512, shuffle=True)

#test_dataset = dataset[:len(dataset) // 10]
#test_loader = DataLoader(test_dataset, 512)

def train(train_loader,model,criterion,optimizer):
    model.train()

    for data in train_loader:  # Iterate in batches over the training dataset.
         out = model(data)  # Perform a single forward pass.
         loss = criterion(out, data.y)  # Compute the loss.
         loss.backward()  # Derive gradients.
         optimizer.step()  # Update parameters based on gradients.
         optimizer.zero_grad()  # Clear gradients.

def test(test_loader, model):
    model.eval()

    correct = 0
    auc_score = 0
    total_samples = 0

    for data in test_loader:  # Iterate in batches over the training/test dataset.
        out = model(data)
        pred = out.argmax(dim=1)  # Use the class with the highest probability.

        correct += int((pred == data.y).sum().item())  # Check against ground-truth labels.
        total_samples += data.y.size(0)

        arr2 = out[:, 1].detach().numpy()
        arr1 = data.y.detach().numpy()
        auc_score += roc_auc_score(y_true=arr1, y_score=arr2, multi_class='ovr', average='weighted')

    accuracy = correct / total_samples
    auc_score /= len(test_loader)

    #print(f"Accuracy: {accuracy:.4f}, AUC Score: {auc_score:.4f}")

    return accuracy, auc_score

for duplication in range(0,5):
    train_size = int(0.8 * len(dataset))
    test_size = len(dataset) - train_size
    train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
    train_loader = DataLoader(train_dataset, 128, shuffle=True)
    test_loader = DataLoader(test_dataset, 128)
    model = DGCNN(dim_features=dataset.num_features, dim_target=dataset.num_classes, config=config)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
    criterion = torch.nn.CrossEntropyLoss()

    for epoch in range(0, 1001):
        train(train_loader,model,criterion,optimizer)
        scores_tr = test(train_loader,model)
        train_acc = scores_tr[0]
        train_auc = scores_tr[1]
        scores_te = test(test_loader,model)
        test_acc = scores_te[0]
        test_auc = scores_te[1]
        if epoch % 10 == 0:
            print(
                f"Duplicate\t{duplication}\tEpoch\t {epoch}\t Train Accuracy\t {train_acc:.4f}\t Train AUC Score\t {train_auc:.4f}\t Test Accuracy: {test_acc:.4f}\t Test AUC Score\t {test_auc:.4f}")


import os
import cubical
from setuptools import setup, find_packages

with open('README.md') as f:
    long_description = f.read()

try:
    import six
except ImportError:
    raise ImportError("Please install six before running install. If you're using pip 19 to install this package you should not be seeing this message.")

try:
    import numpy
except ImportError:
    raise ImportError("Please install numpy before running install. If you're using pip 19 to install this package you should not be seeing this message.")

# Check for readthedocs environment variable.

on_rtd = os.environ.get('READTHEDOCS') == 'True'

if on_rtd:
    requirements = ['numpy',
                    'matplotlib',
                    'scipy']
else:
    requirements = ['future',
                    'numpy',
                    'numba',
                    'python-casacore',
                    'sharedarray >= 3.2.1',
                    'matplotlib',
                    'scipy',
                    'astro-tigger-lsm',
                    'six',
                    'astropy>=3.0',
                    'psutil'
                    ]

setup(name='cubical',
      version=cubical.VERSION,
      description='Fast calibration implementation exploiting complex optimisation.',
      url='https://github.com/ratt-ru/CubiCal',
      classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Science/Research",
        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Python",
        "Topic :: Scientific/Engineering :: Astronomy"],
      author='Jonathan Kenyon',
      author_email='jonosken@gmail.com',
      license='GNU GPL v3',
      long_description=long_description,
      long_description_content_type='text/markdown',
      packages=find_packages(),
      python_requires=">=3.6",
      install_requires=requirements,
      include_package_data=True,
      zip_safe=False,
      scripts=['cubical/bin/print-cubical-stats',
               'cubical/bin/plot-leakage-solutions',
               'cubical/bin/plot-gain-solutions'],
      entry_points={'console_scripts': ['gocubical = cubical.main:main']},
      extras_require={
          'lsm-support': ['montblanc >= 0.6.4'],
          'degridder-support': ['ddfacet >= 0.6.1',
                                'regions < 0.5', # bug in new DS9 parser
                                'meqtrees-cattery >= 1.7.7']
      }
)


from network.base_model import ModelBase_ResNet18
from network.heads import (
    MoCoProjectionHead
)
import torch.nn.functional as F
import copy
from util.MemoryBankModule import MemoryBankModule
from util.utils import *
class SCE(nn.Module):
    def __init__(self, dim=256, K=4096, momentum=-1, tem=0.05, dataset='cifar10', bn_splits=8, symmetric=False):
        super(SCE, self).__init__()
        self.K = K
        self.momentum = momentum
        self.tem = tem
        self.coeff = 0.5
        self.symmetric = symmetric
        # create the encoders
        self.net               = ModelBase_ResNet18(dataset=dataset, bn_splits=bn_splits)
        self.backbone_momentum = copy.deepcopy(self.net)

        self.projection_head = MoCoProjectionHead(input_dim=512, hidden_dim=2048, output_dim=dim)
        self.projection_head_momentum = copy.deepcopy(self.projection_head)

        self.memory_bank = MemoryBankModule(size=self.K).cuda()

        deactivate_requires_grad(self.backbone_momentum)
        deactivate_requires_grad(self.projection_head_momentum)
        # self.max_entropy = np.log(self.K)

    def contrastive_loss(self, im_q, im_k, true_labels, update=False):

        # compute query features
        z_q = self.projection_head(self.net(im_q))  # queries: NxC

        with torch.no_grad():  # no gradient to keys
            # shuffle
            im_k_, shuffle = batch_shuffle(im_k)
            z_k = self.projection_head_momentum(self.backbone_momentum(im_k_))  # keys: NxC
            # undo shuffle
            z_k = batch_unshuffle(z_k, shuffle)

        # Nearest Neighbour,    queue: [feature_dim, self.K]
        _, bank, _ = self.memory_bank(output=z_k, labels=true_labels, update=update)
        # ================normalized==================
        z_q = nn.functional.normalize(z_q, dim=1)
        z_k = nn.functional.normalize(z_k, dim=1)
        bank = nn.functional.normalize(bank.t(), dim=1)
        # ================SCE==================
        batch_size = z_q.shape[0]

        # ================target similarity distribution==================
        one_hot_labels = torch.zeros(batch_size, dtype=torch.long).cuda()
        # [batch_size, 1+bank.shape[0]], except for the first column, which is 1, the rest is 0
        one_hot_labels = nn.functional.one_hot(one_hot_labels, 1 + bank.shape[0])

        sim_k_ktarget = torch.zeros(batch_size).unsqueeze(-1).cuda()
        sim_k_queue = torch.einsum("nc,mc->nm", z_k, bank).cuda()
        sim_k = torch.cat([sim_k_ktarget, sim_k_queue], dim=1).cuda()
        logits_k = sim_k / self.tem
        prob_k = nn.functional.softmax(logits_k, dim=1)
        w = nn.functional.normalize(self.coeff * one_hot_labels + (1 - self.coeff) * prob_k, p=1, dim=1)

        sim_q_ktarget = torch.einsum('nc,nc->n', [z_q, z_k]).unsqueeze(-1).cuda()
        sim_q_queue = torch.einsum("nc,mc->nm", z_q, bank).cuda()
        sim_q = torch.cat([sim_q_ktarget, sim_q_queue], dim=1)
        logits_q = sim_q / 0.1

        loss = -torch.sum(w * F.log_softmax(logits_q, dim=1), dim=1).mean(dim=0).cuda()
        return loss

    def forward(self, im1, im2, labels):
        """
        Input:
            im_q: a batch of query images
            im_k: a batch of key images
        Output:
            loss
        """
        # Updates parameters of `model_ema` with Exponential Moving Average of `model`
        update_momentum(model=self.net, model_ema=self.backbone_momentum, m=self.momentum)
        update_momentum(model=self.projection_head, model_ema=self.projection_head_momentum, m=self.momentum)

        if self.symmetric:  # symmetric loss
            loss_21 = self.contrastive_loss(im2, im1, update=False, true_labels=labels)

        loss_12 = self.contrastive_loss(im1, im2, update=True, true_labels=labels)

        loss = loss_12
        # compute loss
        if self.symmetric:  # symmetric loss
            # loss_21 = self.contrastive_loss(im2, im1, update=False, true_labels=labels)
            loss = (loss_12 + loss_21) * 1.0 / 2

        return loss


import motmetrics as mm
import numpy as np

class MOTMetric(object):

    def __init__(self, ground_truth, tracks, max_d2):
        assert len(ground_truth) == len(tracks)

        self.__has_truth = np.array([bool(d) for d in ground_truth]).any()
        self.__has_hyps = np.array([bool(d) for d in tracks]).any()

        self.acc = mm.MOTAccumulator(auto_id=True)

        for t, objects in enumerate(ground_truth):
            obj_ids = list(objects.keys())
            obj_states = np.array(list(objects.values()))
            hyp_ids = list(tracks[t].keys())
            hyp_states = np.array(list(tracks[t].values()))

            C = mm.distances.norm2squared_matrix(obj_states, hyp_states, max_d2)

            self.acc.update(
                obj_ids,
                hyp_ids,
                C
            )

    def summary(self):
        mh = mm.metrics.create()
        summary = mh.compute(self.acc, metrics=mm.metrics.motchallenge_metrics, name='metrics')
        return mm.io.render_summary(
            summary,
            formatters=mh.formatters,
            namemap=mm.io.motchallenge_metric_names
        )

    def MOTP(self):
        """
        Multiple Object Tracking Precision

        The total position error for all matched object-hypothesis, averaged by the total
        number of matches made. It shows the ability of the tracker to estimate precise
        object positions, independent of its skill at recognizing object configurations,
        keeping consistent trajectories, etc.

        returns a scalar <= 1.0
        """
        if self.__has_hyps and self.__has_truth:
            mh = mm.metrics.create()
            summary = mh.compute(self.acc, metrics=['motp'], return_dataframe=False)
            motp = summary['motp']
            if np.isnan(motp):
                return 0.0
            else:
                return 1.0 - motp
        elif self.__has_hyps:
            return 0.0
        elif self.__has_truth:
            return 0.0
        else:
            return 1.0

    def MOTA(self):
        """
        Multiple Object Tracking Accuracy

        The MOT A accounts for all object configuration errors made by the tracker:
        false positives, misses and mismatches over all object-hypothesis assignments.

        returns a scalar in (-inf, 1.0]
        """
        if self.__has_hyps and self.__has_truth:
            mh = mm.metrics.create()
            summary = mh.compute(self.acc, metrics=['mota'], return_dataframe=False)
            return summary['mota']
        elif self.__has_hyps:
            return 0.0
        elif self.__has_truth:
            return 0.0
        else:
            return 1.0


from django.contrib import admin
from django.urls.base import reverse
from django.urls.exceptions import NoReverseMatch
from django_audit_fields.admin import audit_fieldset_tuple
from edc_model_admin.dashboard import ModelAdminSubjectDashboardMixin
from edc_model_admin.model_admin_simple_history import SimpleHistoryAdmin

from ..admin_site import inte_screening_admin
from ..forms import SubjectRefusalForm
from ..models import SubjectRefusal

@admin.register(SubjectRefusal, site=inte_screening_admin)
class SubjectRefusalAdmin(ModelAdminSubjectDashboardMixin, SimpleHistoryAdmin):
    form = SubjectRefusalForm

    post_url_on_delete_name = "screening_listboard_url"
    subject_listboard_url_name = "screening_listboard_url"
    subject_dashboard_url_name = "screening_listboard_url"

    fieldsets = (
        [
            None,
            {
                "fields": (
                    "screening_identifier",
                    "report_datetime",
                    "reason",
                    "other_reason",
                    "comment",
                )
            },
        ],
        audit_fieldset_tuple,
    )

    list_display = (
        "screening_identifier",
        "report_datetime",
        "reason",
        "user_created",
        "created",
    )

    list_filter = ("report_datetime", "reason")

    search_fields = ("screening_identifier",)

    radio_fields = {"reason": admin.VERTICAL}

    def get_subject_dashboard_url_kwargs(self, obj):
        return dict(screening_identifier=obj.screening_identifier)

    def view_on_site(self, obj):
        try:
            url = reverse(
                self.get_subject_dashboard_url_name(),
                kwargs=self.get_subject_dashboard_url_kwargs(obj),
            )
        except NoReverseMatch as e:
            if callable(super().view_on_site):
                url = super().view_on_site(obj)
            else:
                raise NoReverseMatch(f"{e}. See subject_dashboard_url_name for {repr(self)}.")
        return url


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 22 14:32:28 2017

    Health impacts of PM2.5 - WHO-Europe method (HRAPIE reccomendations)
    to calculate mortality according to the Risk Rates

    Corresonding YLLs or days of life lost are calculated considering the
    distribution of mortality and population by age and by country,
    where data is not available in the ICD-10 format YLLs are calculated
    considering the average value for the countries that
    are availble.

    ASSUMPTION: baseline values (i.e. mortality and average years of life
    loss) are averaged in border cells between countries.

    NB: A positive delta means a reduction!

    INPUT:
        - path_healthbl: baseline values for the impact calculation \
          produced by precompute_healthia.py
        - path_config_json_test: configuration file that the SHERPA interface
          will also use (if it is not there default values are taken)
        - path_base_conc_cdf_test: optional argument (it is not needed when
          using the module from the interface)
    OUTPUT:
        - healthia.nc

    - Bibliography:

    [1] Estimating Local Mortality Burdens associated with Particulate Air
    Pollution 2014 Public Health England

    [2] World Health Organization Europe, 2013. Health risks of air pollution
    in Europe - HRAPIE project - Recommendations for concentration–response
    functions for cost–benefit analysis of particulate matter, ozone and
    nitrogen dioxide, Copenhagen Ø, Denmark.

    [3] Holland, M., 2014. Cost-benefit Analysis of Final Policy Scenarios
    for the EU Clean Air Package Version. Version 2

    [4] World Health Organization Europe, 2017. AirQ+: software tool for health
    risk assessment of air pollution.

    [5] Holland, M., 2014. Implementation of the HRAPIE Recommendations for
    European Air Pollution CBA work. EMRC. \

    [6] Data for baseline population

        ICD codes: ICD-10: A00-B99,C00-D48,D50-D89,E00-E88,F01-F99,G00-G98,
        H00-H59,H60-H93,I00-I99,J00-J98,K00-K92,L00-L98,M00-M99,N00-N98,
        O00-O99,P00-P96,Q00-Q99,R00-R99
        Age: '30 - 85 +'
        Sex: Both
        http://data.euro.who.int/dmdb/ [Accessed December 13, 2016].

    @author: peduzem
    """

from netCDF4 import Dataset
import numpy as np
import os as os
import json

#
#from sherpa_globals import (path_result_cdf_test,
#                            path_healthbl_test, path_config_json_test, path_base_conc_cdf_test,
#                            )

def health_impact(pop30plus, pm25_conc, ar_drate, ar_lyl, approx='l'):

    """
    Function that caclulates the health impact

    INPUT :
        - pop30plus = array with the distribution of the population over 30 \
                      years of age
        - pm25_conc = array with antrhopogenic concentration of PM2.5 \
                      (total)
        - ar_drate = array with the distribution of baseline death rate \
                     (from all cause mortality)
        - ar_lyl = array with the average years of life lost per death \
                    over 30 years of age
        - approx = 'e' for exponential and 'l' for linear

    OUTPUT :
        - mort = array with mortality (from lower bound to upper bound)
        - dll = array with the days of life lost per year \
                (from lower bound to upper bound)
        - dll_spec = array with the days of life lost per person per year \
                     (from lower bound to upper bound)

    @author: peduzem
    """
# -----------------------------------------------------------------------------
    # create empty arrays to store results
    mort = np.zeros(np.shape(pop30plus))
    dll = np.zeros(np.shape(pop30plus))

# -----------------------------------------------------------------------------
    # CONCENTRATION RESPONSE FUNCTION:
    # From [2] Table 1
    # Estimate of mortality, all-cause (natural) age 30+ years
    # PM2.5 Annual mean
    # RR = 1.062 (1.04-1.083) 95% CI per 10 microg/m3
    lrr = 1.04
    mrr = 1.062
    hrr = 1.083

    # From [4] Beta:
    lbeta = 0.003922071315328133  # lower bound
    mbeta = 0.006015392281974714  # average value
    hbeta = 0.007973496801885352  # higher bound
    # RR = e^(beta x)=(e^(beta*10))^(x/10) = 1.062^(x/10) CI = 1.04, 1.083
    # (we obtain the same values reported in [2])

    if approx == 'l':
    # Linear approximation: f(x-x0) = f(x0)+ f'(x0)(x-x0)
    #                       RR = 1 + e^(beta*10)^(x0/10)*ln(e^(beta*10))*(x-x0)/10
    #                       RR = 1 + ln(e^(beta*10))*(x-x0)/10 = 1 + coef*x/10
    # AF = (RR -1)/RR = coef*x/10 / (1+ coef*x/10)

    # linear approximation
        lcoef = np.log(lrr)
        mcoef = np.log(mrr)
        hcoef = np.log(hrr)

        coef = [lcoef, mcoef, hcoef]
        pt = len(coef)
        mort = mort + (np.where(np.isnan(pm25_conc), 0, (
                     [(coef[i]*pm25_conc/10)/(1+coef[i]*pm25_conc/10)*pop30plus*ar_drate
                     for i in range(len(coef))])))

    # exponential approximation
    elif approx == 'e':
    # AF = (RR -1)/RR = e^bx -1 / (e^bx) = 1 -e^(-bx)
        beta = [lbeta, mbeta, hbeta]
        pt = len(beta)
        mort = mort + (np.where(
                    np.isnan(pm25_conc), 0, (
                            [(1-(np.exp(-beta[i]*pm25_conc))) *
                             pop30plus * ar_drate
                             for i in range(len(beta))])))

# -----------------------------------------------------------------------------
    # ESTIMATE OF THE YLL (Not in the Guidelines!)
    # days of life lost per year
    dll = dll + (np.where(np.isnan(pm25_conc), 0,
                             [mort[i] * ar_lyl * 365
                             for i in range(pt)]))
    # days of life lost per person per year
    dll_spec = [np.divide(dll[i], pop30plus, out=np.zeros_like(dll[i]), where=pop30plus!=0) for i in range(pt)]

# -----------------------------------------------------------------------------
    # return results
    return mort, dll, dll_spec

def module8_healthia(path_healthbl, path_result_cdf, path_config_json, *path_base_conc_cdf):
    """
    Main functin that calculates the health impacts given the paths:
    input:
        - path_base_conc_cdf_test = base case concentration \
          optional input argument if value_conc is not in the results
        - path_dust_conc_cdf_test = path of baseline dust concentration
        - path_salt_conc_cdf_test = path of baseline salt concentration
        - path_healthbl = path where results are stored (health baseline)
        - path_result_cdf_test: path of the delta concentrations
           (output of module1)
    @author: peduzem
    """
    # value of concentration from dust and salt (i.e. natural)
    fh_pm25_natural = Dataset(path_healthbl, mode='r')
    pm25_natural = fh_pm25_natural.variables['conc'][:]
    fh_pm25_natural.close()

    # delta concentration from model resutls
    path_conc_nc = path_result_cdf + 'delta_concentration.nc'
    fh_deltapm25 = Dataset(path_conc_nc, mode='r')
    d_pm25_conc = fh_deltapm25.variables['delta_concentration'][:]
#       pm25_delta = fh_deltapm25.variables['conc'][:]
    fh_deltapm25.close()

    # SHERPA interface produces the scenario nc file..
    path_value_nc = path_result_cdf + 'value_conc.nc'
    # if it is not present the scenario concentration has to be calculated
    # from the base concentration
    if not os.path.exists(path_value_nc):
#        if path_base_conc_cdf[0]:
        if path_base_conc_cdf[0]:
            print('Calculating scenario value from base case concentration')
            fh_pm25_base = Dataset(path_base_conc_cdf[0], mode='r')
            pm25_base = fh_pm25_base.variables['conc'][:]
            fh_pm25_base.close()
            pm25_conc = pm25_base - d_pm25_conc
        else:
            print('Error')
    else:
    # if the scenario value is in the results
        fh_pm25_conc = Dataset(path_value_nc, mode='r')
        pm25_conc = fh_pm25_conc.variables['conc'][:]
        fh_pm25_conc.close()

    # Anthropogenic concentration: scenario values minus natural concentration
    # -- there are different views on this. At the moment natural background is
    # substracted to the concentration. See for example:
    # -- [1]	H. Fintan, A. Hunt, H. Cowie, M. Holland, B. Miller, S. Pye, and
    # -- P. Watkiss, “Service Contract for Carrying out Cost-Benefit Analysis
    # -- of Air Quality Related Issues, in particular in the Clean Air for
    # -- Europe (CAFE) Programme,” 2005.
    # -- [2]   The ALPHA Benefit Assessment Model, EC4MACS 2013

    sce_pm25_conc = pm25_conc - pm25_natural

    # get baseline data from nc file
    fh = Dataset(path_healthbl, mode='r')
    pop30plus = fh.variables['ppl30+'][:]
    fh.close()
    fh = Dataset(path_healthbl, mode='r')
    ar_drate = fh.variables['deathsppl30+'][:]
    fh.close()
    fh = Dataset(path_healthbl, mode='r')
    ar_lyl = fh.variables['lyl30+'][:]
    fh.close()

 # -----------------------------------------------------------------------------
    # calculate impacts
    sce_mort, sce_dll, sce_dll_spec = health_impact(pop30plus, sce_pm25_conc,
                                                    ar_drate, ar_lyl, approx='l')
    bc_pm25 = sce_pm25_conc + d_pm25_conc

    # this could be improved (the valuse are always the same)
    bc_mort, bc_dll, bc_dll_spec = health_impact(pop30plus, bc_pm25,
                                                    ar_drate, ar_lyl, approx='l')
    delta_mort = bc_mort - sce_mort
    delta_dll = bc_dll - sce_dll
    delta_dll_spec = np.array(bc_dll_spec) - np.array(sce_dll_spec)

# -----------------------------------------------------------------------------
    # Saving results:
    # default dictionary to save results:
    dflt_dict = {
	"d_mort":{
		"impact": "Mortality",
		"data": "Delta",
		"ci":["d_mort_lb", "d_mort", "d_mort_ub"],

		"long_description":["delta mortality lower bound", "delta mortality", "delta mortality upper bound"],
		"aggregation":"sum",
		"units":"people/year"},
	"v_mort":{
		"impact": "Mortality",
		"data": "Value",
		"ci":["v_mort_lb", "v_mort", "v_mort_ub"],

		"long_description":["mortality lower bound", "mortality", "mortality upper bound"],
		"aggregation":"sum",
		"units":"people/year"},
	"d_dll":
	{
		"impact": "Days of life loss",
		"data": "Delta",
		"ci":["d_dll_lb", "d_dll", "d_dll_ub"],

		"long_description":["delta days of life loss lower bound", "delta days of life loss", "delta days of life loss upper bound"],
		"aggregation":"sum",
		"units":"dll/year"},
	"v_dll":
	{
		"impact": "Days of life loss",
		"data": "Value",
		"ci":["v_dll_lb", "v_dll", "v_dll_ub"],

		"long_description":["days of life loss lower bound", "days of life loss", "days of life loss upper bound"],
		"aggregation":"sum",
		"units":"dll/year"},
	"d_dll_pp":
	{
		"impact": "Days of life loss per person",
		"data": "Delta",
		"ci":["d_dll_pp_lb", "d_dll_pp", "d_dll_pp_ub"],

		"long_description":["delta days of life loss per person lower bound", "delta days of life loss per person", "delta days of life loss per person upper bound"],
		"aggregation":"population weighted average",
		"units":"dll/(person year)"},
	"v_dll_pp":
	{
		"impact": "Days of life loss per person",
		"data": "Value",
		"ci":["v_dll_pp_lb", "v_dll_pp", "v_dll_pp_ub"],

		"long_description":["days of life loss per person lower bound", "days of life loss per person", "days of life loss per person upper bound"],
		"aggregation":"population weighted average",
		"units":"dll/(person year)"}
    }

    # If it exists we use the json config file which is used also by the
    # SHERPA interface
    if os.path.exists(path_config_json):
        print('Using stored json file')
        json_file = open(path_config_json)
        json_str = json_file.read()
        cfg_dct = json.loads(json_str)
    else:
        print('Not using stored json file')
        cfg_dct = dflt_dict

    # Generation of results files:
    outfile=path_result_cdf + 'healthimp.nc'
    if os.path.exists(outfile):
        os.remove(outfile)
    for key in cfg_dct.keys():
        if key == 'd_mort':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(delta_mort[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])
        if key == 'v_mort':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(sce_mort[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])
        if key == 'd_dll':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(delta_dll[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])
        if key == 'v_dll':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(sce_dll[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])
        if key == 'd_dll_pp':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(delta_dll_spec[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])
        if key == 'v_dll_pp':
            for it in enumerate(cfg_dct[key]['ci']):
                write_nc(sce_dll_spec[it[0]], outfile, it[1], cfg_dct[key]['units'], path_healthbl,
                     addnutsid=False, l_name=cfg_dct[key]['long_description'][it[0]])

## SUPPORT FUNCTIONS (IDEALLY IN THE AUXIALIARIES FILE)

def write_nc(array, path_nc, name_var, unit_var, path_healthbl, addnutsid=False, l_name=None):
    ''' Function to write an array in a netcdf file,
        if the file already exist it is going to write in append mode,
        otherwise in write mode.
        input:
            - array: data to write
            - path_nc: path of netcdf file
            - name_var: name for data in array
            - unit_var: units for data in array
            - path_healthbl: ncd file template (for lon and lat arrays)
            - addnutsid: if True the layer nuts_id is added so that the
                nectcdf file is consistent with the ones provided
                by terraria
        ouput:
            - nc file
    @author: peduzem
    '''
    rootgrp = Dataset(path_healthbl, 'r')
    lon_array = rootgrp.variables['longitude'][:]
    lat_array = rootgrp.variables['latitude'][:]
    rootgrp.close()

    if not os.path.exists(path_nc):
        mode = 'w'
        fh=Dataset(path_nc, mode=mode, format='NETCDF3_CLASSIC')
        fh.createDimension('latitude', len(lat_array))
        fh.createDimension('longitude', len(lon_array))
        latitude = fh.createVariable('latitude', 'f4', ('latitude',))
        longitude = fh.createVariable('longitude', 'f4', ('longitude',))
        if addnutsid is True:
#        fh.createDimension('z', 10)
            fh.createDimension('nuts_id', 1)
            var = fh.createVariable(name_var, 'f8',
                                    ('nuts_id', 'latitude', 'longitude',))
            nutsid = fh.createVariable('NUTS', 'i4', ('nuts_id',))
            longitude[:] = lon_array
            latitude[:] = lat_array
            nutsid[0] = 1
            var[0, :] = array
        elif addnutsid is False:
            longitude[:] = lon_array
            latitude[:] = lat_array
            var = fh.createVariable(name_var, 'f8', ('latitude', 'longitude'))
            var[:] = array
    else:
        mode = 'a'
        fh=Dataset(path_nc, mode=mode, format='NETCDF3_CLASSIC')
        if addnutsid is True:
            var = fh.createVariable(name_var, 'f8',
                                    ('nuts_id', 'latitude', 'longitude',))
            var[0, :] = array
        elif addnutsid is False:
            var = fh.createVariable(name_var, 'f8', ('latitude', 'longitude'))
            var[:] = array

    fh.variables[name_var].units = unit_var
    if l_name is not None:
            fh.variables[name_var].long_name =l_name
    fh.close()

if __name__ == '__main__':

#    module8_healthia(path_healthbl_test, path_result_cdf_test, path_config_json_test, path_base_conc_cdf_test)

    pass


import pygame
import random

pygame.font.init()

# Globale verdier
s_bredde = 600                                   # bredde på vindu
s_hoyde = 550                                    # lengde på vindu
blokk_storrelse = 20                             # størrelse på blokk
blokker_bredde = 10                              # antall blokker bortover
blokker_hoyde = 20                               # antall blokker nedover
poeng_level = 200                                # poeng som kreves per level

spill_bredde = blokker_bredde * blokk_storrelse  # spillvindu størrelse
spill_høyde = blokker_hoyde * blokk_storrelse    # spillvindu
top_venstre_x = (s_bredde - spill_bredde) // 2   # finne start posisjon
top_venstre_y = s_hoyde - spill_høyde - 50

max_score = 0

# Figurer

S = [['.....',
      '.....',
      '..00.',
      '.00..',
      '.....'],
     ['.....',
      '..0..',
      '..00.',
      '...0.',
      '.....']]

Z = [['.....',
      '.....',
      '.00..',
      '..00.',
      '.....'],
     ['.....',
      '..0..',
      '.00..',
      '.0...',
      '.....']]

I = [['..0..',
      '..0..',
      '..0..',
      '..0..',
      '.....'],
     ['.....',
      '0000.',
      '.....',
      '.....',
      '.....']]

O = [['.....',
      '.....',
      '.00..',
      '.00..',
      '.....']]

J = [['.....',
      '.0...',
      '.000.',
      '.....',
      '.....'],
     ['.....',
      '..00.',
      '..0..',
      '..0..',
      '.....'],
     ['.....',
      '.....',
      '.000.',
      '...0.',
      '.....'],
     ['.....',
      '..0..',
      '..0..',
      '.00..',
      '.....']]

L = [['.....',
      '...0.',
      '.000.',
      '.....',
      '.....'],
     ['.....',
      '..0..',
      '..0..',
      '..00.',
      '.....'],
     ['.....',
      '.....',
      '.000.',
      '.0...',
      '.....'],
     ['.....',
      '.00..',
      '..0..',
      '..0..',
      '.....']]

T = [['.....',
      '..0..',
      '.000.',
      '.....',
      '.....'],
     ['.....',
      '..0..',
      '..00.',
      '..0..',
      '.....'],
     ['.....',
      '.....',
      '.000.',
      '..0..',
      '.....'],
     ['.....',
      '..0..',
      '.00..',
      '..0..',
      '.....']]

figurer = [S, Z, I, O, J, L, T]
figur_farger = [(0, 255, 0, 0), (255, 0, 0, 0), (0, 255, 255, 0), (255, 255, 0, 0), (255, 165, 0, 0), (0, 0, 255, 0), (128, 0, 128, 0)]
# indeks 0 - 6 representerer figurene

class Blokk(object):                                             # definerer klassen "Blokk" med x-verdi, y-verdi, figur, farger og rotasjon
    def __init__(self, x, y, figur):
        self.x = x
        self.y = y
        self.figur = figur
        self.farge = figur_farger[figurer.index(figur)]
        self.rotasjon = 0
    def getHoyde(self):                                          # får ut høyden til en figur
        (ins, cf) = get_koordinat(self)                          # gjør om til koordinater
        minY = min(y for (x,y) in cf)
        maxY = max(y for (x,y) in cf)
        return maxY - minY + 1

def lage_grid(laast_pos={}):                                     # lager en spill ramme
    global blokker_bredde,blokker_hoyde
    grid = [[(0,0,0,0) for _ in range(blokker_bredde)] for _ in range(blokker_hoyde)]
    # lager et svart 10x20 rektangel

    for i in range(len(grid)):
        for j in range(len(grid[i])):
            if (j, i) in laast_pos:                              # lagrer fargen til ruter som har falt (låst posisjon)
                c = laast_pos[(j,i)]
                grid[i][j] = c
    return grid

def get_koordinat(figur):                                        # gjør om figurene til koordinater
    posisjoner = []
    format = figur.figur[figur.rotasjon % len(figur.figur)]      # få formen av figuren

    for i, line in enumerate(format):                            # splitter figur til lister med indeks til hver rad
        row = list(line)
        for j, column in enumerate(row):                         # splitter radene til enkelte ruter
            if column == '0':
                posisjoner.append((figur.x + j, figur.y + i))    # finner koordinatene til hver rute med farge

    for i, pos in enumerate(posisjoner):
        posisjoner[i] = (pos[0] - 2, pos[1] - 4)

    iSkjerm = all(y >= 0 for (x,y) in posisjoner)

    return (iSkjerm, posisjoner)

def gyldig_plass(figur, grid):                                   # sjekker om figurerne er innenfor rammen
    global blokker_bredde, blokker_hoyde
    # finner ledige ruter
    ledig_pos = [[(j, i) for j in range(blokker_bredde) if grid[i][j] == (0,0,0,0)] for i in range(blokker_hoyde)]
    ledig_pos = [j for sub in ledig_pos for j in sub]

    iSkjerm, formatted = get_koordinat(figur)

    for pos in formatted:                                        # sjekker om hver posisjon er gyldig
        if pos not in ledig_pos:
            if pos[1] > -1:
                return False
    return True

def sjekk_tapt(posisjoner):                                      # sjekker om figuren når toppen
    for pos in posisjoner:
        x, y = pos
        if y < 1:
            return True
    return False

def get_figur():                                                 # gir en tilfeldig figur
    return Blokk(5, 0, random.choice(figurer))

def get_skygge_Blokk(fallendeBlokk):                             # skaper en brikke som viser hvor den fallende brikken kommer til å lande
    skyggeBlokk = Blokk(fallendeBlokk.x, fallendeBlokk.y, fallendeBlokk.figur)
    skyggeBlokk.rotasjon = fallendeBlokk.rotasjon
    l = list(fallendeBlokk.farge)
    l[3] = 128                                                   # gjør figuren halvveis gjennomsiktig
    skyggeBlokk.farge = tuple(l)
    skyggeBlokk.y = fallendeBlokk.y + fallendeBlokk.getHoyde()
    return skyggeBlokk

def isSkygge(farge_tuple):                                       # sjekker om figur er en skygge figur
    l = list(farge_tuple)
    return l[3] == 128

def get_farge(farge_tuple):                                      # gjør figur halvveis gjennomsiktig
    l = list(farge_tuple)
    return tuple(l[:3])

def skygge(fallende_Blokk, gammel_skygge_Blokk, grid):           # lage skygge hvis mulig
    if ((gammel_skygge_Blokk != None)                            # sjekker om at den fallende figuren ikke er lik skyggen
        and (fallende_Blokk.y + fallende_Blokk.getHoyde() >= gammel_skygge_Blokk.y)
        and (fallende_Blokk.figur == gammel_skygge_Blokk.figur)
        and (fallende_Blokk.rotasjon == gammel_skygge_Blokk.rotasjon)
        and (fallende_Blokk.x == gammel_skygge_Blokk.x)):
            return fallende_Blokk;
    skyggeBlokk = get_skygge_Blokk(fallende_Blokk)               # skaper skygge figur
    funnetGyldig = False
    GyldigY1 = -1
    while (skyggeBlokk.y < 30) and (funnetGyldig == False):      # flytter skyggen nedover til første gyldige y-verdi uten å være lik den fallende figuren
        skyggeBlokk.y += 1
        funnetGyldig = gyldig_plass(skyggeBlokk, grid)           # sjekker om gyldig
    if (funnetGyldig):
        GyldigY1 = skyggeBlokk.y                                 # flytter skyggen til første gyldige y-verdi
        funnetIGyldig = False
        while (skyggeBlokk.y < 30) and (funnetIGyldig == False): # flytter skygge nedover til noe stopper den
            skyggeBlokk.y += 1
            funnetIGyldig = not gyldig_plass(skyggeBlokk, grid)
        if funnetIGyldig:                                        # stopper skyggeblokken
            skyggeBlokk.y -= 1
            return skyggeBlokk
        else:
            skyggeBlokk.y = GyldigY1
            return skyggeBlokk
    return fallende_Blokk

def tegne_tekst(overflate, tekst, storrelse, farge):             # funksjon som skriver ut tekster
    font = pygame.font.SysFont("comicsans", storrelse, bold=True)
    label = font.render(tekst, 1, farge)

    overflate.blit(label, (top_venstre_x + spill_bredde /2 - (label.get_width()/2), top_venstre_y + spill_høyde/2 - label.get_height()/2))

def tegne_ruter(surface, grid):                                  # lager rutene
    sx = top_venstre_x
    sy = top_venstre_y

    for i in range(len(grid)):
        pygame.draw.line(surface, (128,128,128), (sx, sy + i*blokk_storrelse), (sx+spill_bredde, sy+ i*blokk_storrelse))
        for j in range(len(grid[i])):
            pygame.draw.line(surface, (128, 128, 128), (sx + j*blokk_storrelse, sy),(sx + j*blokk_storrelse, sy + spill_høyde))

def slette_rader(grid, laast):                                       # tømmer rutene når en rad er fylt opp
    inc = 0
    rader_pop = []
    for i in range(len(grid)-1, -1, -1):                             # gjør om til rader
        rad = grid[i]
        if all( (c != (0,0,0,0) and not isSkygge(c)) for c in rad):  # sjekker om cellene er okkuperbare
            inc += 1
            rader_pop.append(i)
            for j in range(len(rad)):                                # tømme rader
                try:
                    del laast[(j,i)]
                except:
                    continue

    if inc > 0:                                                      # flytte rader om rader slettes
        for key in sorted(list(laast), key=lambda x: x[1])[::-1]:    # begynner fra bunnen
            x, y = key

            # flytter rader nedover i følge til hvor mange rander som slettes under
            rader_pop_lower = list(filter(lambda r: r > y, rader_pop))
            newKey = (x, y + len(rader_pop_lower))
            laast[newKey] = laast.pop(key)

    score_inc = {                                                    # lager et dictionary med poeng for antall slettede rader
        0: 0,
        1: 10,
        2: 30,
        3: 50,
        4: 80,
        }
    return score_inc.get(inc)

def tegne_neste_figur(figur, overflate):                     # viser den neste fallende figuren
    font = pygame.font.SysFont('comicsans', 30)
    label = font.render('Next figur', 1, (255,255,255))

    sx = top_venstre_x + spill_bredde + 50                   # plasserer figuren
    sy = top_venstre_y + spill_høyde/2 - 100
    format = figur.figur[figur.rotasjon % len(figur.figur)]

    for i, line in enumerate(format):                        # tegne neste figur
        row = list(line)
        for j, column in enumerate(row):
            if column == '0':
                pygame.draw.rect(overflate, figur.farge, (sx + j*blokk_storrelse, sy + i*blokk_storrelse, blokk_storrelse, blokk_storrelse), 0)

    overflate.blit(label, (sx + 10, sy - 30))                # plasserer tekst

def oppdatere_high_score(nscore):                            # opdaterer high score
    global max_score
    if max_score < nscore:
        max_score = nscore

def tegne_vindu(overflate, grid, score=0):                   # tegner vinduet
    overflate.fill((0, 0, 0))

    pygame.font.init()                                       # lager tittel
    font = pygame.font.SysFont('comicsans', 60)
    tittel = font.render('Tetris', 1, (255, 255, 255))

    overflate.blit(tittel, (top_venstre_x + spill_bredde / 2 - (tittel.get_width() / 2), 30))

    # viser score
    font = pygame.font.SysFont('comicsans', 30)
    tekst = font.render('Score: ' + str(score), 1, (255,255,255))

    sx = top_venstre_x + spill_bredde + 50
    sy = top_venstre_y + spill_høyde/2 - 100

    # viser high score
    overflate.blit(tekst, (sx + 20, sy + 160))
    tekst = font.render('High Score: ' + str(max_score), 1, (255,255,255))

    sx = top_venstre_x - 200
    sy = top_venstre_y + 200

    overflate.blit(tekst, (sx + 20, sy + 160))

    # fargelegging
    for i in range(len(grid)):
        for j in range(len(grid[i])):
            skygge_flate = pygame.Surface((blokk_storrelse, blokk_storrelse))
            if isSkygge(grid[i][j]):
                skygge_flate.set_alpha(128)                  # gjør skyggen halvveis gjennomsiktig
            skygge_flate.fill(get_farge(grid[i][j]))
            overflate.blit(skygge_flate, (top_venstre_x + j*blokk_storrelse, top_venstre_y + i*blokk_storrelse))

    # tegner spillruten
    pygame.draw.rect(overflate, (255, 0, 0), (top_venstre_x, top_venstre_y, spill_bredde, spill_høyde), 5)

    tegne_ruter(overflate, grid)

def level_up(fallende_hastighet):                            # fjerner ruter med farger og øker hastighet når spiller når nytt level
    global blokker_bredde,blokker_hoyde
    ny_hastighet = fallende_hastighet - 0.05                 # øker hastighet
    # fjerner alle blokker
    grid = [[(0,0,0,0) for _ in range(blokker_bredde)] for _ in range(blokker_hoyde)]

    return (ny_hastighet, grid, {})

def hoved(win):
    global poeng_level
    laast_posisjoner = {}                   # koordinatene til okkuperte celler
    grid = lage_grid(laast_posisjoner)      # 10x20 tabell som lagrer fargen til hver celle

    bytte_Blokk = False                     # om skal bytte figur
    run = True                              # holder spillet gående
    fallende_Blokk = get_figur()            # skaper fallende blokk
    neste_Blokk = get_figur()               # skaper neste blokk
    klokke = pygame.time.Clock()            # tar tiden
    fall_tid = 0                            # hvor lenge brikken har falt
    fall_hastighet = 0.5                    # hastigheten brikken faller på
    score = 0                               # poeng
    level = 1                               # level
    skygge_Blokk = None                     # skyggeblokk

    while run:
        grid = lage_grid(laast_posisjoner)  # lager spill skjerm (der brikkene faller)
        fall_tid += klokke.get_rawtime()    # stopper stoppeklokken for løkken
        klokke.tick()                       # starter stoppeklokken for løkken

        # flytter fallende blokk nedover når fall tiden er nådd
        if fall_tid/1000 > fall_hastighet:
            fall_tid = 0
            if (gyldig_plass(fallende_Blokk, grid)) == True:
                fallende_Blokk.y += 1
                # stopper blokken når hindret og bytter blokk
                if not(gyldig_plass(fallende_Blokk, grid)) and fallende_Blokk.y > 0:
                    fallende_Blokk.y -= 1
                    bytte_Blokk = True
            else:
                tegne_tekst(win, "GAME OVER", 80, (255,255,255))
                pygame.display.update()
                pygame.time.delay(1500)
                run = False

        # kontroller
        for event in pygame.event.get():
            # quit
            if event.type == pygame.QUIT:
                run = False
                pygame.display.quit()
            if event.type == pygame.KEYDOWN:
                # flytter den fallende blokken mot venstre
                if event.key == pygame.K_LEFT:
                    fallende_Blokk.x -= 1
                    if not(gyldig_plass(fallende_Blokk, grid)):
                        fallende_Blokk.x += 1
                # flytter den fallende blokken mot høyre
                if event.key == pygame.K_RIGHT:
                    fallende_Blokk.x += 1
                    if not(gyldig_plass(fallende_Blokk, grid)):
                        fallende_Blokk.x -= 1
                # flytter den fallende blokken nedover kjappere
                if event.key == pygame.K_DOWN:
                    fallende_Blokk.y += 1
                    if not(gyldig_plass(fallende_Blokk, grid)):
                        fallende_Blokk.y -= 1
                # roterer den fallende blokken
                if event.key == pygame.K_UP:
                    fallende_Blokk.rotasjon += 1
                    if not(gyldig_plass(fallende_Blokk, grid)):
                        fallende_Blokk.rotasjon -= 1
                # flytter den fallende blokken rett ned
                if event.key == pygame.K_SPACE:
                    while gyldig_plass(fallende_Blokk, grid):
                        fallende_Blokk.y += 1
                    fallende_Blokk.y -= 1

        # legger inn fallende figurer
        iSkjerm_fallende, figur_pos = get_koordinat(fallende_Blokk)
        for i in range(len(figur_pos)):
            x, y = figur_pos[i]
            if y > -1:
                grid[y][x] = fallende_Blokk.farge

        if iSkjerm_fallende:
            # Lage ny skygge når fallende blokk er byttet
            skygge_Blokk = skygge(fallende_Blokk, skygge_Blokk, grid)
            if (skygge_Blokk.y > fallende_Blokk.y):
                (iSkjerm_skygge, figur_pos_skygge) = get_koordinat(skygge_Blokk)
                # legger inn skygge hvis gyldig plass
                if (iSkjerm_skygge):
                    for i in range(len(figur_pos_skygge)):
                        x, y = figur_pos_skygge[i]
                        grid[y][x] = skygge_Blokk.farge

        # låser den fallende blokken og bytter til neste fallende blokk
        if bytte_Blokk:
            for pos in figur_pos:
                p = (pos[0], pos[1])
                laast_posisjoner[p] = fallende_Blokk.farge
            fallende_Blokk = neste_Blokk
            neste_Blokk = get_figur()
            bytte_Blokk = False
            score += slette_rader(grid, laast_posisjoner)
            # level up
            if score >= level*poeng_level:
                level += 1
                fall_hastighet, grid, laast_posisjoner = level_up(fall_hastighet)
                tegne_tekst(win, "Level up", 80, (255,255,255))
                pygame.display.update()
                pygame.time.delay(1500)

        oppdatere_high_score(score)
        tegne_vindu(win, grid, score)
        tegne_neste_figur(neste_Blokk, win)
        pygame.display.update()

        # sjekker om spillet er tapt
        if sjekk_tapt(laast_posisjoner):
            tegne_tekst(win, "GAME OVER", 80, (255,255,255))
            pygame.display.update()
            pygame.time.delay(1500)
            run = False

# hoved program
def hovedmeny(win):
    run = True
    while run:
        win.fill((0,0,0))
        tegne_tekst(win, 'Press Any Key To Play', 60, (255,255,255))
        pygame.display.update()
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
            if event.type == pygame.KEYDOWN:
                hoved(win)

    pygame.display.quit()

# setter opp vindu
win = pygame.display.set_mode((s_bredde, s_hoyde))
pygame.display.set_caption('Tetris')
hovedmeny(win)


#!/usr/bin/env python3

from datetime import datetime
from datetime import timedelta
import locale

LAST_DATE = "2022-09-26"
WEEK = timedelta(days=7)
PAYOUT = 0.5

def main():
    locale.setlocale(locale.LC_ALL, "de_DE")
    last_date = datetime.fromisoformat(LAST_DATE)

    pay_day = last_date + WEEK
    amount = PAYOUT

    while pay_day < datetime.now():
        print(f"pay out at {pay_day:%x}: €{PAYOUT:.2f}, sum: €{amount:.2f}")
        pay_day += WEEK
        amount += PAYOUT

if __name__ == "__main__":
    main()


import torch

# Training configuration
MODEL = ""

BATCH_SIZE = 128

LRATE = 1e-4

NUM_EPOCHS = 50

WEIGHT_DECAY = 0

TRAIN_SIZE = 33600

VAL_SIZE = 8400

NUM_FEATURES = 30
# Classifier configuration
TRAIN_PROPORTION = 0.9999

NUM_NEIGHBORS = 3

NUM_ESTIMATORS = 101

# General configuration
TRAIN_FILE = "../data/train.csv"

TEST_FILE = "../data/test.csv"

SUBMISSION_FILE = "result/submission.csv"

EVAL_FILE = "../eval.xlsx"

MODEL_STATE_DICT_FILE = "result/model.pt"

DTYPE = torch.float32

# Best
# n_neighbors = 5, num_features = 200


from __future__ import division
from future.utils import raise_from

import errno
import json
import os
import platform
import socket
import subprocess
import sys
from collections import OrderedDict
from time import sleep
from warnings import warn

import semver

from pennsieve.log import get_log_level, get_logger
from pennsieve.models import Collection, DataPackage, Dataset

logger = get_logger("pennsieve.agent")

try:
    from websocket import create_connection
except ModuleNotFoundError:
    logger.warn(
        "websocket-client is not installed - uploading with the Agent will not work"
    )

MINIMUM_AGENT_VERSION = semver.VersionInfo.parse("0.3.4")
DEFAULT_LISTEN_PORT = 11235

class AgentError(Exception):
    pass

def agent_cmd():
    if sys.platform == "darwin":
        return "/usr/local/opt/pennsieve/bin/pennsieve"

    elif sys.platform.startswith("linux"):
        return "/opt/pennsieve/bin/pennsieve"

    elif sys.platform in ["win32", "cygwin"]:
        return "C:/Program Files/Pennsieve/pennsieve.exe"

    raise AgentError("Platform {} is not supported".format(sys.platform))

def validate_agent_installation(settings):
    """
    Check whether the agent is installed and at least the minimum version.
    """
    try:
        env = agent_env(settings)
        env["PENNSIEVE_LOG_LEVEL"] = "ERROR"  # Avoid spurious output with the version
        version = subprocess.check_output([agent_cmd(), "version"], env=env)
    except (AgentError, subprocess.CalledProcessError, EnvironmentError) as e:
        raise AgentError(
            "Agent not installed. Visit https://developer.pennsieve.io/agent for installation directions."
        )

    try:
        agent_version = semver.VersionInfo.parse(version.decode().strip())
    except ValueError as e:
        raise_from(AgentError("Invalid version string"), e)

    if agent_version < MINIMUM_AGENT_VERSION:
        raise AgentError(
            "Agent not compatible: found version {}, need version {}".format(
                agent_version, MINIMUM_AGENT_VERSION
            )
        )

    logger.info("Agent version %s found", agent_version)

def agent_env(settings):
    """
    Configure the agent environment to mirror the Python client
    The "local" environment looks for the host in PENNSIEVE_API_LOC
    (this is configured down in pennsieve-rust)
    """
    env = {
        "PENNSIEVE_API_ENVIRONMENT": "local",
        "PENNSIEVE_API_LOC": settings.api_host,
        "PENNSIEVE_API_TOKEN": settings.api_token,
        "PENNSIEVE_API_SECRET": settings.api_secret,
        "PENNSIEVE_LOG_LEVEL": get_log_level(),
    }
    if sys.platform in ["win32", "cygwin"]:
        env["SYSTEMROOT"] = os.getenv("SYSTEMROOT")
    # On Windows, the SYSTEMROOT environment variable must be preserved for DLLs to correctly load.
    # ref: https://travis-ci.community/t/socket-the-requested-service-provider-could-not-be-loaded-or-initialized/1127

    logger.debug("Agent environment: %s", env)

    return env

class AgentListener(object):
    """
    Context manager that starts the agent in listen server mode.
    """

    def __init__(self, settings, port):
        self.settings = settings
        self.port = port
        self.proc = None
        self.devnull = None
        warn(
            f"Pennsieve is transitioning to the new agent. This class '{self.__class__.__name__}' will be deprecated; version=7.0.0; date=2022-11-01.",
            DeprecationWarning,
            stacklevel=2,
        )

    def __enter__(self):
        check_port(self.port)
        command = [agent_cmd(), "upload-status", "--listen", "--port", str(self.port)]

        self.devnull = open(os.devnull, "w")

        self.proc = subprocess.Popen(
            command,
            env=agent_env(self.settings),
            stdout=sys.stdout if get_log_level() == "DEBUG" else self.devnull,
            stderr=sys.stderr if get_log_level() == "DEBUG" else self.devnull,
        )
        return self.proc

    def __exit__(self, *exc):
        self.proc.kill()
        self.devnull.close()

def check_port(port):
    """
    Refuse to start up if the agent is already running in listen mode.
    This can cause problems with relative files paths and session credentials.
    """
    try:
        logger.debug("Checking port %s", port)
        create_connection(socket_address(port)).close()
    except socket.error as e:
        if e.errno == errno.ECONNREFUSED:  # ConnectionRefusedError for Python 3
            logger.debug("No agent found, port %s OK", port)
            return True
        else:
            raise
    else:
        raise AgentError(
            "The agent is already running. Please stop any running processes and try again"
        )

def socket_address(port):
    if platform.system() == "Windows":
        return "ws://127.0.0.1:{}".format(port)
    return "ws://0.0.0.0:{}".format(port)

def create_agent_socket(port):
    """
    Open a websocket connection to the agent

    If the agent is not available, wait using exponential backoff for it to
    come up and start responding to messages
    """
    for i in range(-2, 4):
        try:
            return create_connection(socket_address(port))
        except socket.error as e:
            if e.errno == errno.ECONNREFUSED:  # ConnectionRefusedError for Python 3
                sleep_time = 2**i
                logger.debug("Connection refused - sleeping for %s seconds", sleep_time)
                sleep(sleep_time)
            else:
                raise

    raise AgentError("Could not connect to Agent")

def agent_upload(
    destination, files, dataset, append, recursive, display_progress, settings
):
    """
    Push an upload through the agent.
    """
    directory_upload = any(os.path.isdir(f) for f in files)

    if directory_upload and len(files) > 1:
        raise AgentError(
            "Can only upload a single directory.\n"
            'Please pass a single directory argument: `pkg.upload("/experiment/dir")`'
        )

    if recursive and not directory_upload:
        raise AgentError(
            "Recursive uploads are only allowed with directories.\n"
            "Upload a directory or pass `recursive=False`."
        )

    if recursive and append:
        raise AgentError("Cannot use `recursive=True` when appending`")

    # Figure out what files the agent is going to upload.
    # We cannot count on the agent to send "upload queued" messages for
    # all files before it starts uploading, so we generate the files we
    # plan to wait for.
    if directory_upload:
        directory = files[0]
        if recursive:
            expected_files = []
            for dirpath, _, filenames in os.walk(directory):
                for f in filenames:
                    expected_files.append(os.path.join(dirpath, f))
        else:
            expected_files = []
            for f in os.listdir(directory):
                path = os.path.join(directory, f)
                if os.path.isfile(path):
                    expected_files.append(path)
    else:
        expected_files = files

    # Agent uses absolute paths
    expected_files = [os.path.abspath(f) for f in expected_files]

    if isinstance(destination, Dataset):
        dataset_id = destination.id
        package_id = None
    elif isinstance(destination, (Collection, DataPackage)):
        dataset_id = dataset.id
        package_id = destination.id
    else:
        raise ValueError("Can only upload to a Dataset, Package, or Collection")

    with AgentListener(settings, DEFAULT_LISTEN_PORT):
        try:
            ws = create_agent_socket(DEFAULT_LISTEN_PORT)

            ws.send(
                json.dumps(
                    {
                        "message": "queue_upload",
                        "body": {
                            "dataset": dataset_id,
                            "package": package_id,
                            "files": files,
                            "append": append,
                            "recursive": recursive,
                        },
                    }
                )
            )

            upload_manager = UploadManager(expected_files, display_progress)
            upload_manager.print_progress()

            for msg in ws:
                msg = json.loads(msg)

                if msg["message"] == "file_queued_for_upload":
                    upload_manager.set_queued(msg["path"], msg["import_id"])

                elif msg["message"] == "upload_progress":
                    upload_manager.set_progress(
                        msg["path"], msg["import_id"], msg["percent_done"], msg["done"]
                    )

                elif msg["message"] == "upload_complete":
                    upload_manager.set_complete(msg["import_id"])

                elif msg["message"] == "upload_error":
                    logger.error(msg["context"])
                    upload_manager.set_error(msg["import_id"])

                elif msg["message"] == "error":
                    raise AgentError(msg["context"])

                else:
                    logger.debug("Unknown message", msg)

                upload_manager.print_progress()
                if upload_manager.done:
                    break

        finally:
            try:
                ws.close()
            except UnboundLocalError:
                pass

def remove_prefix(text, prefix):
    return text[text.startswith(prefix) and len(prefix) :]

class UploadManager(object):
    """
    Manager for file status and messages.

    This is complicated by that fact that the agent sends status information
    for files that are already in the queue or started by other processes.
    This makes it possible for the same file to be queued twice, so we
    have to track both the filename and import id. We only want to wait for
    all "our" files to upload.
    """

    def __init__(self, files, display_progress):
        # Should we show progress bars?
        self.display_progress = display_progress

        # map of filepath -> list(FileProgress)
        self.uploads = OrderedDict()

        for file in files:
            self.track_file(file, import_id=None, ours=True)

        # Keep track of whether progress bars have already been rendered so
        # we know if/what to erase when re-drawing
        self.lines_on_screen = 0
        warn(
            f"Pennsieve is transitioning to the new agent. This class '{self.__class__.__name__}' will be deprecated; version=7.0.0; date=2022-11-01.",
            DeprecationWarning,
            stacklevel=2,
        )

    def track_file(self, file, import_id, ours):
        progress = FileProgress(file, import_id, ours)
        if file in self.uploads:
            self.uploads[file].append(progress)
        else:
            self.uploads[file] = [progress]
        return progress

    def get_tracked_file(self, file, import_id):
        if sys.platform in ["win32", "cygwin"]:
            file = remove_prefix(
                file, "\\\\?\\"
            )  # windows OS sometimes prefix the filepath with \\?\, so we remove it if we see if
        if file in self.uploads:
            for p in self.uploads[file]:
                if p.import_id == import_id:
                    return p

    def all_tracked_files(self):
        for filegroup in self.uploads.values():
            for progress in filegroup:
                yield progress

    def set_queued(self, file, import_id):
        # Update the unqueued version of the file with an import id
        progress = self.get_tracked_file(file, None)

        # This import is queued from a different process - we don't care
        if progress is None:
            return

        progress.queued = True
        progress.import_id = import_id

    def set_progress(self, file, import_id, percent_done, done):
        # Absorb any files that are in the DB/already queued
        if self.get_tracked_file(file, import_id) is None:
            self.track_file(file, import_id, ours=False)

        progress = self.get_tracked_file(file, import_id)
        progress.percent_done = percent_done

    def set_complete(self, import_id):
        for progress in self.all_tracked_files():
            if progress.import_id == import_id:
                progress.done = True

    def set_error(self, import_id):
        for progress in self.all_tracked_files():
            if progress.import_id == import_id:
                progress.errored = True

    @property
    def done(self):
        return all([fstat.done for fstat in self.all_tracked_files() if fstat.ours])

    def print_progress(self, width=24):
        if not self.display_progress:
            return

        # move cursor to relative beginning
        sys.stdout.write("\033[F" * self.lines_on_screen)

        for fstat in self.all_tracked_files():
            if fstat.done:
                state = "DONE"
            elif fstat.errored:
                state = "ERRORED"
            elif fstat.queued:
                state = "UPLOADING"
            else:
                state = "WAITING"

            text = " [ {bars}{dashes} ] {state:12s} {percent:05.1f}% {name}\n".format(
                bars="#" * int(fstat.progress * width),
                dashes="-" * (width - int(fstat.progress * width)),
                percent=fstat.percent_done,
                name=fstat.name,
                state=state,
            )

            sys.stdout.write("{}\r".format(text))
            sys.stdout.flush()

        self.lines_on_screen = len(list(self.all_tracked_files()))

class FileProgress(object):
    def __init__(self, filename, import_id, ours):
        # We only care about the state of uploads started by this process
        self.ours = ours
        self.filename = filename
        self.import_id = import_id
        self.name = os.path.basename(filename)
        self._percent_done = 0
        self.done = False
        self.errored = False
        self.queued = False
        warn(
            f"Pennsieve is transitioning to the new agent. This class '{self.__class__.__name__}' will be deprecated; version=7.0.0; date=2022-11-01.",
            DeprecationWarning,
            stacklevel=2,
        )

    @property
    def percent_done(self):
        if self.done:
            return 100
        return self._percent_done

    @percent_done.setter
    def percent_done(self, value):
        # Only increment progress (in case messages come out of order)
        if value > self._percent_done:
            self._percent_done = value

    @property
    def progress(self):
        return self.percent_done / 100


from sklearn.linear_model import Ridge
from sklearn.linear_model import LinearRegression
import numpy as np
from sklearn.preprocessing import PolynomialFeatures
import torch
class Poly_linear_regression_grad():
    '''
    import matplotlib.pyplot as plt
    from sklearn.linear_model import Ridge
    from model.Ridge_shift_regression import Ridge_Shift
    from model.model import polynomial_regression
    from model.linear_regression_grad import Poly_linear_regression_grad
    import numpy as np
    scale = 1
    r_scale = 2
    shift = 0.2
    num = 5
    xs = [ele / scale - shift for ele in range(num)]
    ys = [np.exp(-ele ) + np.random.random() / r_scale for ele in xs]

    model = Poly_linear_regression_grad(n_order= 4 ,lr = 0.00001,  momentum = 0.99, epoch= 100000, verbal= False, lambda_ = 0.1)
    model.fit(np.array(xs).reshape(-1, 1), np.array(ys), range_ = [-0.1, 4.2] )
    pred_x = np.arange(-0.1, 4.2, 0.1)
    pred_y = model.predict(pred_x.reshape(-1, 1))
    plt.scatter(xs, ys)
    plt.plot(pred_x, pred_y)
    plt.show()
    '''
    def  __init__(self, n_order, lr = 0.0001, momentum = 0.8, epoch = 1000, verbal = False, lambda_ = 0.01 ):
        """
        Example code
        n_samples, n_features = 3, 2
        """
        self.n_order = n_order
        self.lr = lr
        self.momentum = momentum
        self.epoch = epoch
        self.verbal = verbal
        self.lambda_ = lambda_
    def fit(self, x, y, range_ = None, direction = 'decreasing'):
        '''
        x : numpy : N_len x m_feature,
        y : numpy : N_len

        '''
        self._range = range_
        self._direction = direction
        self.poly = PolynomialFeatures(degree = self.n_order)
        x_trans = self.poly.fit_transform(x)
        self.n_features = x_trans.shape[1]
        LR =  LinearRegression(fit_intercept=False)
        LR.fit(x_trans, y)

        coef = np.array( LR.coef_.tolist()) / 1000#+ 1
        self.coef_ = torch.from_numpy(coef)
        self.coef_.requires_grad = True
        self.x = torch.from_numpy(x_trans)
        self.y = torch.from_numpy(y)

        # preparing for monotonic loss
        if self._range != None:
            left, right, step = self._range[0], self._range[1], (self._range[1] - self._range[0] ) / 100
            x_range = np.arange(left, right, step).reshape(-1, 1)
            poly__ = PolynomialFeatures(degree = self.n_order - 1)
            self.x_trans__ = poly__.fit_transform(x_range)
            self.x__ = torch.from_numpy(self.x_trans__)

        self.op = torch.optim.SGD([self.coef_], lr=self.lr, momentum = self.momentum)
        self.train(self.epoch)

    def train(self, n_epoch = 1000):
        for _ in range(n_epoch):
            # acc loss
            predict_y = self.x @ self.coef_.T
            loss1 =  torch.sum((predict_y - self.y) ** 2)

            if self._range != None:
                # breakpoint()
                f_derivative = self.x__ @ ( self.coef_[1:] * ( torch.arange(self.n_features - 1 ) + 1 ) )
                if self._direction == 'decreasing':
                    loss2 = torch.sum(f_derivative[f_derivative > 0 ] ** 2)
                else:
                    loss2 = torch.sum(-f_derivative[f_derivative < 0 ]** 2)
                # print('f_derivative', f_derivative)
            else:
                loss2 = 0
            # loss2 = loss1 * 0
            loss = loss1 + self.lambda_ *loss2
            self.op.zero_grad()
            loss.backward()
            # loss1.backward()
            self.op.step()

            if self.verbal and _%10 == 0 :
                print('loss: ', loss.item(), ' loss1: ', loss1.item(), ' loss2: ', loss2.item(), 'self.coef_', self.coef_.detach().numpy())
                # print('predict_y', predict_y)
                # print('self.y', self.y)
            # breakpoint()
        # loss2 =
        print('final training loss', loss1)
        # print('predict_y', predict_y)
        # print('self.y', self.y)
        # breakpoint()
    def predict(self, x):
        x_trans = self.poly.fit_transform(x)
        pred = x_trans @ self.coef_.detach().numpy()
        return pred

if __name__ == '__main__':
    PLRG = Poly_linear_regression_grad(3)
    rng = np.random.RandomState(0)
    n_samples = 3
    n_features = 1
    y = rng.randn(n_samples)
    X = rng.randn(n_samples, n_features)
    PLRG.fit(X,y)
    print(' PLRG.pred(X)',  PLRG.predict(X))
    breakpoint()


# importing the required libraries
import csv
import json

# defining the function to convert CSV file to JSON file
def convjson(csvFilename, jsonFilename):

    # creating a dictionary
    mydata = {}

    # reading the data from CSV file
    with open(csvFilename, encoding = 'utf-8') as csvfile:
        csvRead = csv.reader(csvfile, delimiter='~')

        # Converting rows into dictionary and adding it to data
        for rows in csvRead:
            print(rows)

    # dumping the data
    with open(jsonFilename, 'w', encoding = 'utf-8') as jsonfile:
        jsonfile.write(json.dumps(mydata, indent = 4))

# filenames
csvFilename = r'dataframe1.csv'
jsonFilename = r'dataframe1.json'

# Calling the convjson function
convjson(csvFilename, jsonFilename)

Диплом/Дипломный проект/Генерация базы для модели/Convert_CSV_to_json.py


import gym
gym.logger.set_level(40)  # nopep8
from gym.envs.registration import registry
from eagent.gym_evolving_locomotion_envs import EvolvingWalkerEnv  # noqa
from eagent.gym_evolving_manipulate_envs import EvolvingHandEnv  # noqa

def register(id, *args, **kvargs):
    if id in registry.env_specs:
        return
    else:
        return gym.envs.registration.register(id, *args, **kvargs)

# ----------------------------------
# Walking task
register(
    id="EvolvingWalkerEnv-v2",
    entry_point="eagent:EvolvingWalkerEnv",
    max_episode_steps=1000,  # Should be matched with num_steps_in_eval
    reward_threshold=99999.0,  # This might not be needed
)

# ----------------------------------
# EvolvingBlock
register(
    id="EvolvingHandBlockDense-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "fixed_random",
        "target_rotation": "xyz",
        "reward_type": "dense",
        "object_type": "block",
    },
)
register(
    id="EvolvingHandBlockSparse-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "fixed_random",
        "target_rotation": "xyz",
        "reward_type": "sparse",
        "object_type": "block",
    },
)
register(
    id="EvolvingHandBlockRotateZDense-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "ignore",
        "target_rotation": "z",
        "reward_type": "dense",
        "object_type": "block",
    },
)
register(
    id="EvolvingHandBlockRotateZSparse-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "ignore",
        "target_rotation": "z",
        "reward_type": "sparse",
        "object_type": "block",
    },
)

# ----------------------------------
# EvolvingEgg
register(
    id="EvolvingHandEggRotateZSparse-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "ignore",
        "target_rotation": "z",
        "reward_type": "sparse",
        "object_type": "egg",
    },
)
register(
    id="EvolvingHandEggRotateSparse-v1",
    entry_point="eagent:EvolvingHandEnv",
    max_episode_steps=100,
    kwargs={
        "target_position": "ignore",
        "target_rotation": "xyz",
        "reward_type": "sparse",
        "object_type": "egg",
    },
)


import time
import re
from cumulusci.robotframework.utils import capture_screenshot_on_error
from cumulusci.robotframework.pageobjects import BasePage
from cumulusci.robotframework.pageobjects import pageobject
from BaseObjects import BaseNPSPPage
from NPSP import npsp_lex_locators
from logging import exception

OID_REGEX = r"^(%2F)?([a-zA-Z0-9]{15,18})$"
@pageobject("Custom", "GE_Gift_Entry")
class GiftEntryPage(BaseNPSPPage, BasePage):

    def _go_to_page(self):
        """To go to Gift Entry page"""
        url_template = "{root}/lightning/n/{object}"
        name = self._object_name
        object_name = "{}{}".format(self.cumulusci.get_namespace_prefix(), name)
        url = url_template.format(root=self.cumulusci.org.lightning_base_url, object=object_name)
        self.selenium.go_to(url)
        self.salesforce.wait_until_loading_is_complete()
        self.selenium.wait_until_page_contains("Templates")

    def _is_current_page(self):
        """
        Verifies that current page is Gift Entry landing page
        """
        self.selenium.wait_until_location_contains("GE_Gift_Entry", timeout=60,
                                                   message="Current page is not Gift Entry landing page")
        self.selenium.wait_until_page_contains("Default Gift Entry Template")

    def click_gift_entry_button(self,title):
        """clicks on Gift Entry button identified with title"""
        locator=npsp_lex_locators["gift_entry"]["button"].format(title)
        self.selenium.wait_until_page_contains_element(locator)
        self.selenium.click_element(locator)

    def enter_value_in_field(self,**kwargs):
        """Enter value in specified field"""
        for key,value in kwargs.items():
            if key=='Description':
                locator=npsp_lex_locators["gift_entry"]["field_input"].format(key,"textarea")
                self.selenium.wait_until_page_contains_element(locator)
                self.salesforce._populate_field(locator, value)
            else:
                locator=npsp_lex_locators["gift_entry"]["field_input"].format(key,"input")
                self.selenium.wait_until_page_contains_element(locator)
                self.salesforce._populate_field(locator, value)

    def select_template_action(self,name,action):
        """From the template table, select template with name and select an action from the dropdown"""
        locator=npsp_lex_locators["gift_entry"]["actions_dropdown"].format(name)
        self.selenium.click_element(locator)
        element=self.selenium.get_webelement(locator)
        status=element.get_attribute("aria-expanded")
        if status=="false":
            self.selenium.wait_until_page_contains("Clone")
        self.selenium.click_link(action)
        if action=="Edit" or action=="Clone":
            self.selenium.wait_until_page_contains("Gift Entry Template Information")
        elif action=="Delete":
            self.selenium.wait_until_page_does_not_contain(name)

    def select_object_group_field(self,object_group,field):
        """Select the specified field under specified object group
           to add the field to gift entry form and verify field is added"""
        locator=npsp_lex_locators["gift_entry"]["form_object_dropdown"].format(object_group)
        self.selenium.scroll_element_into_view(locator)
        self.selenium.click_element(locator)
        element=self.selenium.get_webelement(locator)
        status=element.get_attribute("aria-expanded")
        if status=="false":
            time.sleep(2)
        field_checkbox=npsp_lex_locators["gift_entry"]["object_field_checkbox"].format(field)
        self.selenium.scroll_element_into_view(field_checkbox)
        self.selenium.click_element(field_checkbox)
        field_label=object_group+': '+field
        self.selenium.wait_until_page_contains(field_label)

    def verify_template_is_not_available(self,template):
        """Verify that a gift template is not available for selection while creating a new batch"""
        field=npsp_lex_locators["adv_mappings"]["field_mapping"].format("Template")
        self.selenium.click_element(field)
        element=self.selenium.get_webelement(field)
        status=element.get_attribute("aria-activedescendant")
        if status is not None:
            self.selenium.page_should_not_contain(template)
        else:
            self.selenium.wait_until_page_contains("Default Gift Entry Template")
            self.selenium.page_should_not_contain(template)
        self.selenium.click_button("Cancel")

    def get_template_record_id(self,template):
        """ Parses the current url to get the object id of the current record.
            Expects url format like: [a-zA-Z0-9]{15,18}
        """
        locator=npsp_lex_locators["link-text"].format(template)
        element = self.selenium.get_webelement(locator)
        e=element.get_attribute("href")
        print(f"url is {e}")
        for part in e.split("="):
            oid_match = re.match(OID_REGEX, part)
            if oid_match is not None:
                return oid_match.group(2)
        raise AssertionError("Could not parse record id from url: {}".format(e))

    def store_template_record_id(self,template):
        """ Parses the template href to get the object id of the current record.
            Expects url format like: [a-zA-Z0-9]{15,18}
        """
        id=self.get_template_record_id(template)
        self.salesforce.store_session_record("Form_Template__c",id)


from torch.utils.data.dataloader import DataLoader
from dataset import DataSet
from network import NetA, SubCNNA, NetA4
from torch import optim
from torch import nn, save, load, Tensor
from collections import Counter
from util import calculate_mcc
import time

def train(epoch, batch_size=128, lr=0.00001):
    model.cuda()
    model.train()
    optimizer = optim.AdamW(model.parameters(), lr=lr)
    lossfn = nn.MSELoss()
    print("=========Epoch-{}==========[lr:{}]".format(epoch, lr))
    dataset = DataSet("train_data_a.json", True)
    train_data = DataLoader(dataset, batch_size=batch_size, shuffle=True)
    tloss = 0
    avg_loss = 0
    start_time = time.time()
    for i, (left, right, label, angle) in enumerate(train_data):
        optimizer.zero_grad()
        left = left.cuda()
        right = right.cuda()
        label = label.cuda()
        angle = angle.cuda()
        preds = model(left, right, angle)
        loss = lossfn(preds, label)
        loss.backward()
        optimizer.step()
        tloss += loss
        avg_loss = round(float(tloss / i), 3)
        curtime = time.time()
        eta = (curtime - start_time) / (i + 1) * (len(train_data) - i)
        print('\r', "{}/{} current loss {}, avg loss {}, ETA {}s."
              .format(i * batch_size, len(dataset), loss, avg_loss, round(eta, 2)), end="")
    print("\nEpoch {} has finish. Epoch loss is {}".format(epoch, avg_loss))

def testCHK(**kwargs):
    chkname = kwargs.get("chkname")
    epoch = kwargs.get("epoch")
    model.eval()
    model.cuda()
    if chkname is not None:
        fchk = open(chkname, 'rb')
        model.load_state_dict(load(fchk))
        print("=========Test-{}==========".format(chkname))
    else:
        print("=========Test-{}==========".format(epoch))
    test_data = DataLoader(DataSet("test_data_a.json"), batch_size=128, shuffle=False)
    res = []
    corr = []
    for i, (left, right, label, angle) in enumerate(test_data):
        left = left.cuda()
        right = right.cuda()
        label = label.cuda()
        angle = angle.cuda()
        pred: Tensor = model(left, right, angle)
        res.extend(pred.tolist())
        corr.extend(label.tolist())
    print(res[2])
    res = list(map(softmax_convert, res))
    corr = list(map(softmax_convert, corr))
    tp, tn, fp, fn = 0, 0, 0, 0
    for x in range(len(res)):
        if res[x] == corr[x]:
            if corr[x] == [0, 1]:
                tp += 1
            if corr[x] == [1, 0]:
                tn += 1
        elif res[x] != corr[x]:
            if corr[x] == [0, 1]:
                fn += 1
            if corr[x] == [1, 0]:
                fp += 1
    print("labels: " + str(corr[:10]))
    print("result: {}".format(res[:10]))
    print(Counter(map(tuple, corr)))
    print(Counter(map(tuple, res)))
    print("tp:{},tn:{},fp:{},fn:{}".format(tp, tn, fp, fn))
    print("Accuracy:{}/{}".format(tp + tn, tp + tn + fn + fp))
    print("Precision:{}/{}".format(tp, tp + fp))
    print("Recall:{}/{}".format(tp, tp + fn))
    print("MCC:{}".format(calculate_mcc(tp, fp, fn, tn)))

def softmax_convert(tar):
    if tar[0] == max(tar):
        return [1, 0]
    else:
        return [0, 1]

def sigmod_convent(tar):
    if tar[0] >= 0:
        return 1
    else:
        return 0

if __name__ == '__main__':
    model = NetA(SubCNNA)
    model.cuda()
    init_lr = 0.0001
    # for e in range(1, 82):
    #     if e > 1 and (e - 1) % 20 == 0:
    #         init_lr *= 0.5
    #     train(e, lr=init_lr, batch_size=256)
    #     save(model.state_dict(), open("{}a_epcoh-checkpoint.chk".format(e), 'wb'))
    #     testCHK(epoch=e)
    testCHK(chkname="62a_epcoh-checkpoint.chk")


from Tkinter import *
import pandas as pd
from scipy.stats import mode
import tkFileDialog
import tkMessageBox
from decimal import *
from sklearn import metrics

class Analyser:
    train = None
    sructureDic = None
    numOfBins = None
    initDictionaryStruct = {}
    countRecotdsClass = {}
    struct = {}
    p = 0.0
    m = 2

#The function calculates the average and put the value at missind data
    def average(self, key):
        self.train[key].fillna(self.train[key].mean(), inplace=True)

#The function found the most common value and put the value at missind data
    def mostCommon(self, key):
        self.train[key].fillna(mode(self.train[key]).mode[0], inplace=True)

#The function handles missing numeric and category data
    def handleMissingData(self):
        for key, value in self.sructureDic.iteritems():
            if(value == "class"):
                continue
            if value == "NUMERIC":
                self.average(key)
            else:
                self.mostCommon(key)

#The function calculates equal width
    def EqualWidth(self, maxval, minval, cut_points):
        tempWidth = float(maxval) - float(minval)
        width =  float(tempWidth/ float(cut_points))
        i = 1
        binsArr = []
        while i < cut_points:
            binsArr.append(minval + (width * i))
            i += 1
        return binsArr

#The function devides to equal width
    def binning(self, col, cut_points, key):
        minval = col.min()
        maxval = col.max()
        binsArr = self.EqualWidth(maxval, minval, cut_points) #calculate bins
        labels = range(1, cut_points + 1)
        break_points = [minval] + binsArr + [maxval]
        self.initDictionaryStruct[key] = labels
        self.struct[key] = break_points
        colBin = pd.cut(col, bins=break_points, labels=labels, include_lowest=True)
        return colBin

#The function does discretization to the data set
    def Discretization(self):
        try:
            for key, value in self.sructureDic.iteritems(): #devide if numeric or categoty
                if value == "NUMERIC":
                    # print (key)
                    self.train[key] = self.binning(self.train[key], self.numOfBins, key)
                else:
                    valueList = str(value).replace("}", "").replace("{", "").split(",")
                    self.initDictionaryStruct[key] = valueList
                    self.struct[key] = valueList
        except:
            tkMessageBox.showerror("Naive Bayes Classifier", "the bins are not valid")

#The function calculates the probability of the attributes in category by the class
    def calculateProb(self, classValue, attribute, category):
        n = self.countRecotdsClass[classValue]
        countAtt = self.train[category] == attribute
        countClass = self.train['class'] == classValue
        count = self.train[countAtt & countClass]
        nc = len(count)
        result = (nc + (self.m * self.p)) / (n + self.m)
        return result

#The function calculates the probability of 'class' attribute
    def probClass(self):
        self.countRecotdsClass = self.train["class"].value_counts()

#The function creates class probability table for each class value
    def createClassProbTable(self, attribute, category):
        probDictionary = {}
        for classValue in self.initDictionaryStruct["class"]:
            prob = self.calculateProb(classValue, attribute, category)
            probDictionary[classValue] = prob
        return probDictionary

#The function create table category for each attribute
    def createTableCategory(self, attributesCategory, category):
        categoryTable = {}
        for attribute in attributesCategory:
            classProb = self.createClassProbTable(attribute, category)
            categoryTable[attribute] = classProb
        return categoryTable

#The function bulids table probability
    def buildTablesProb(self):
        bigTable = {}
        leng = len(str(self.sructureDic["class"]).replace("}", "").replace("{", "").split(","))
        self.p = float(float(1) / float(leng))
        for category, value in self.sructureDic.iteritems():
            if category == "class":
                continue
            attributesCategory = self.initDictionaryStruct[category] #take the attribue category
            categoryTable = self.createTableCategory(attributesCategory, category)
            bigTable[category] = categoryTable

        return bigTable

#The function builds the model
    def buildModel(self, sructureDic, train, numOfBins):
        self.train = train
        self.sructureDic = sructureDic
        self.numOfBins = numOfBins
        self.handleMissingData()
        self.Discretization()
        self.probClass()
        probTables = self.buildTablesProb()
        return probTables

class Classifier:
    test = None
    structIntervals = None
    structDic = None
    structLables = None

# The function does discretization to the test set
    def Discretization(self):
        for key, value in self.structDic.iteritems():
            if key == "class":
                continue
            if value == "NUMERIC":
                intervals = self.structIntervals[key]
                labels = self.structLables[key]
                self.test[key] = pd.cut(self.test[key], bins=intervals, labels=labels, include_lowest=True)

#The function writes to file the results
    def writeToFile(self, dirpath, classification):
        f = open(dirpath + "\\output.txt", "w")
        i = int(1)
        while i < len(classification):
            f.write(str(i) + " " + str(classification[int(i)]) + "\n") #write the number and the classification
            i = int(i) + int(1)
        f.close()

#The function calculates nult between probs
    def getMult(self, listProb):
        mult = Decimal(1)
        for prob in listProb:
            mult = Decimal(mult) * Decimal(prob)
        return mult

#The function gets the probability from the table
    def getProb(self, model, category, attribute, classAttribute):
        dictionary = model[category]
        classProb = dictionary[attribute]
        prob = classProb[classAttribute]
        return prob

#The function gets the classification of the row
    def getClassificationRow(self,probByVal):
        # print (probByVal)
        classification = None
        maxProb = 0
        for classVal, prob in probByVal.iteritems(): # for on probByVal
            if prob>maxProb:
                maxProb = prob
                classification = classVal
        return classification

#The function get the classification for the test
    def getClassification(self,model):
        classification = []
        for index, row in self.test.iterrows():
            probByVal = {}
            for classValue in self.structLables["class"]: #for on each class
                listProb  = []
                for category, value in self.structDic.iteritems():
                    if category =="class":
                         continue
                    attribute = row[category]
                    prob = self.getProb(model,category,attribute,classValue)
                    listProb.append(prob)
                probClassVal = self.getMult(listProb)
                probByVal[classValue] = probClassVal
            classificationRow = self.getClassificationRow(probByVal)
            classification.append(classificationRow)
        return classification

#The function classify the test by the model
    def classify(self, test, structIntervals, structDic, structLables, dirpath,model):
        self.test = test
        self.structIntervals = structIntervals
        self.structDic = structDic
        self.structLables = structLables
        self.Discretization()
        classification = self.getClassification(model)
        self.writeToFile(dirpath,classification)
        accurancy = metrics.accuracy_score(classification,test["class"])
        # print "Accuracy : %s" % "{0:.3%}".format(accurancy)
        tkMessageBox.showinfo("Naive Bayes Classifier", "classify is done!")
        exit()

class NaiveBayes:
    analyser = Analyser()
    classifier = Classifier()
    model = None

#The function give the user the search for the path
    def browse(self):
        dir = tkFileDialog.askdirectory()
        self.var.set(dir)

#The function load csv file
    def load_csv(self, filename):
        try:
            df = pd.read_csv(filename)
            return df
        except:
            tkMessageBox.showerror("Naive Bayes Classifier", "the path " + filename + " is not exist")

#The function load text file
    def load_txt(self, filename):
        try:
            d = {}
            with open(filename) as f:
                content = f.read().splitlines()
                for line in content:
                    arr = line.split(' ')
                    key = arr[1]
                    value = arr[2]
                    d[key] = value
            return d
        except:
            tkMessageBox.showerror("Naive Bayes Classifier", "the path " + filename + " is not exist") #print if the file is not exist

#The function build the model
    def build(self):
        dirPath = self.var.get()
        filename = dirPath + "\\Structure.txt"
        sructureDic = self.load_txt(filename)
        filename = dirPath + "\\train.csv"
        train = self.load_csv(filename)
        numOfBins = int(self.varBins.get())
        self.model = self.analyser.buildModel(sructureDic, train, numOfBins)
        tkMessageBox.showinfo("Naive Bayes Classifier", "Building classifier using train-set is done!")

#The function classify the test by the model
    def classify(self):
        dirPath = self.var.get()
        filename = dirPath + "\\test.csv"
        test = self.load_csv(filename)
        structIntervals = self.analyser.struct
        structDic = self.analyser.sructureDic
        structLabes = self.analyser.initDictionaryStruct
        self.classifier.classify(test, structIntervals, structDic, structLabes, dirPath,self.model)

#The function init the gui
    def __init__(self, master):
        self.master = master
        self.master.title("Naive Bayes Classifier")
        self.labelDir = Label(master, text="Directory Path")
        self.labelDisc = Label(text="Discretization Bins:")
        self.var = StringVar()
        dirname = Entry(master, textvariable=self.var)
        self.varBins = StringVar()
        bins = Entry(master, textvariable=self.varBins)
        self.browse_button = Button(master, text="Browse", command=lambda: self.browse()) #button browse
        self.build_button = Button(master, text="Build", command=lambda: self.build()) #button build
        self.classify_button = Button(master, text="Classify", command=lambda: self.classify())
        self.labelDir.grid(row=0, column=0, sticky=W)
        self.labelDisc.grid(row=1, column=0, sticky=E)
        dirname.grid(row=0, column=1, columnspan=2, sticky=W + E)
        bins.grid(row=1, column=1, columnspan=2, sticky=W + E)
        self.browse_button.grid(row=0, column=3)
        self.build_button.grid(row=2, column=1)
        self.classify_button.grid(row=3, column=1)

root = Tk()
my_gui = NaiveBayes(root)
root.mainloop()