import pandas
import numpy
import scipy.signal

# Helper function: given a signal it returns the (normed) autocorrelation
def autocorrelation(signal):
    x = numpy.correlate(signal, signal, mode="full")
    x = x[x.size // 2:]
    result = x / float(x.max())
    return result

# Helper function: compute bandpower
def bandpower(signal, fmin, fmax):
    f, spectral_density = scipy.signal.periodogram(signal, 50)
    ind_min = numpy.argmax(f > fmin) - 1
    ind_max = numpy.argmax(f > fmax) - 1
    return numpy.trapz(spectral_density[ind_min: ind_max], f[ind_min: ind_max])

def feature_extractor(signal):
    result = list()
    signal_ac = autocorrelation(signal)

    # Autocorrelation-realted features: number of peaks, prominent peaks and weak peaks, height of the first peak
    # after 0 and maximum height of the autocorrelation function
    n_peaks = len(scipy.signal.find_peaks(signal_ac)[0])
    result.append(n_peaks)
    n_prom_peaks = len(scipy.signal.find_peaks(signal_ac, prominence=0.17, distance=100)[0])
    result.append(n_prom_peaks)
    n_weak_peaks = len(scipy.signal.find_peaks(signal_ac, prominence=[0, 0.17], wlen=200)[0])
    result.append(n_weak_peaks)
    max_ac = max(signal_ac[1:])
    result.append(max_ac)
    try:
        max_peak = scipy.signal.find_peaks(signal_ac)[0][1]
    except:
        max_peak = 0
    result.append(max_peak)

    # Band Powers (10 features)
    bands_f = numpy.linspace(0.1, 25, 11)
    bp = list()
    for index in range(10):
        band = bandpower(signal, bands_f[index], bands_f[index + 1])
        bp.append(band)
    result = result + bp

    # Mean, standard deviation and variance
    mean = numpy.mean(signal)
    result.append(mean)
#    std = numpy.std(signal)
#    result.append(std)
    variance = numpy.var(signal)
    result.append(variance)

    # RMS amplitude
    f, spectral_density = scipy.signal.periodogram(signal, 50)
    rms = numpy.sqrt(spectral_density.max())
    result.append(rms)

    # RMS amplitude after cumulative summation (acceleration -> velocity)
    signal_cumsum = numpy.cumsum(signal)
    f, spectral_density = scipy.signal.periodogram(signal_cumsum, 50)
    rms_cumsum = numpy.sqrt(spectral_density.max())
    result.append(rms_cumsum)

    return result

def features(data):
    # Creating an empty dataframe with the right column names
#    feats = ["nPeaks", "nPromPeaks", "nWeakPeaks", "maxAc", "maxPeak", "BP1", "BP2", "BP3", "BP4", "BP5", "BP6", "BP7",
#             "BP8", "BP9", "BP10", "mean", "std", "variance", "rms", "rms_cumsum", "mean1", "std1", "var1", "rms1",
#             "mean2", "std2", "var2", "rms2"]

    feats = ["nPeaks", "nPromPeaks", "nWeakPeaks", "maxAc", "maxPeak", "BP1", "BP2", "BP3", "BP4", "BP5", "BP6", "BP7",
             "BP8", "BP9", "BP10", "mean", "variance", "rms", "rms_cumsum"]
    names = ["aX_Wrist", "aY_Wrist", "aZ_Wrist", "gX_Wrist", "gY_Wrist", "gZ_Wrist", "aX_Ankle", "aY_Ankle", "aZ_Ankle",
             "gX_Ankle", "gY_Ankle", "gZ_Ankle"]
    cols = []
    for name in names:
        for feat in feats:
            cols.append(name + feat)
    features = pandas.DataFrame(columns=cols)

    # Creating the rows of the dataframe
    for i in range(len(data.index)):
        newRow = []
        for column in data.columns:
            newRow = newRow + feature_extractor(data[column][i])
        features.loc[i] = newRow

    return features


# -*- coding: utf-8 -*-
"""
This module provides utilities for validating and preparing an annotation schema.
"""

from collections import OrderedDict
import json
from typing import Dict, List, Set

__classification__ = "UNCLASSIFIED"

class LabelSchema(object):
    """
    The basic structure for an annotation/labelling schema.

    The label names may certainly be modified in place through use of the `labels`
    property, without worry for causing errors. This is discouraged, because having two
    schemas with same version number/ids and differing names can likely lead to confusion.

    Any modification of label ids of sub-id structure should be performed by using the
    :func:`set_labels_and_subtypes` method, or difficult to diagnose runtime errors
    will likely be introduced.
    """

    __slots__ = (
        '_version', '_labels', '_subtypes', '_parent_types', '_confidence_values',
        '_permitted_geometries')

    def __init__(self, version, labels, subtypes=None, confidence_values=None, permitted_geometries=None):
        """

        Parameters
        ----------
        version : str
            The version of the schema.
        labels : Dict[str, str]
            The {<label id> : <label name>} pair dictionary. Each entry must be a string,
            and '' is not a valid label id.
        subtypes : None|Dict[str, List[str]]
            The {<label id> : <sub id list>} pairs. The root ids (i.e. those ids
            not belonging as sub-id to some other id) will be populated in the subtypes
            entry with empty string key (i.e. ''). Every key and entry of subtypes
            (excluding the subtypes root '') must correspond to an entry of labels,
            and no id can be a direct subtype of more than one id.
        confidence_values : None|List[Union[str, int]]
            The possible confidence values.
        permitted_geometries : None|List[str]
            The possible geometry types.
        """

        self._version = None
        self._labels = None
        self._subtypes = None
        self._parent_types = None
        self._confidence_values = None
        self._permitted_geometries = None

        self.confidence_values = confidence_values
        self.permitted_geometries = permitted_geometries
        self.set_labels_and_subtypes(version, labels, subtypes)

    @property
    def version(self):
        """
        The version of the schema.

        Returns
        -------
        str
        """

        return self._version

    @property
    def labels(self):
        """
        The complete label dictionary of the form `{label_id : label_name}`.

        Returns
        -------
        Dict[str, str]
        """

        return self._labels

    @property
    def subtypes(self):
        """
        The complete dictionary of subtypes of the form `{parent_id : <subids list>}`.

        Returns
        -------
        Dict[str, List[str]]
        """

        return self._subtypes

    @property
    def parent_types(self):
        """
        The dictionary of parent types of the form `{child_id : <set of parent ids>}`.
        It is canonically defined that an id is a parent of itself.

        Returns
        -------
        Dict[str, Set[str]]
        """

        return self._parent_types

    @property
    def confidence_values(self):
        """
        The list of confidence values.

        Returns
        -------
        List
            Each element should be a json type (most likely use cases are str or int).
        """

        return self._confidence_values

    @confidence_values.setter
    def confidence_values(self, conf_values):
        if conf_values is None:
            self._confidence_values = None
            return

        if not isinstance(conf_values, list):
            raise TypeError('confidence_values must be a list. Got type {}'.format(type(conf_values)))
        self._confidence_values = conf_values

    @property
    def permitted_geometries(self):
        """
        The collection of permitted geometry types. None corresponds to all.

        Returns
        -------
        None|List[str]
        """

        return self._permitted_geometries

    @permitted_geometries.setter
    def permitted_geometries(self, values):
        if values is None:
            self._permitted_geometries = None
            return
        if not isinstance(values, list):
            values = list(values)
        self._permitted_geometries = values

    def __str__(self):
        return json.dumps(self.to_dict(), indent=1)

    def __repr__(self):
        return json.dumps(self.to_dict())

    def set_labels_and_subtypes(self, version, labels, subtypes):
        """
        Set the labels and subtypes. This modification must be accompanied by a
        version number modification. **Note that subtypes may be modified in place.**

        Parameters
        ----------
        version : str
        labels : dict
        subtypes : None|dict

        Returns
        -------
        None
        """

        if not isinstance(version, str):
            raise TypeError('version is required to be a string. Got type {}'.format(type(version)))

        if not isinstance(labels, dict):
            raise TypeError('labels is required to be a dict. Got type {}'.format(type(labels)))

        if subtypes is None:
            subtypes = OrderedDict()
        elif not isinstance(subtypes, dict):
            raise TypeError('subtypes is required to be None or a dict. Got type {}'.format(type(subtypes)))

        # ensure that every key and value of labels are strings
        for key in labels:
            if not isinstance(key, str):
                raise TypeError(
                    'All keys of labels must be of type string. Got key `{}` of '
                    'type {}'.format(key, type(key)))
            if key == '':
                raise ValueError('The empty string is not a valid label id.')
            value = labels[key]
            if not isinstance(value, str):
                raise TypeError(
                    'All values of labels must be of type string. Got value {} '
                    'for key `{}` of type {}'.format(value, key, type(value)))

        # we need to check the reference count for each key in labels
        counts = OrderedDict((key, 0) for key in labels)

        # ensure that every key of subtypes is a string and every value is a list,
        # also that inclusion makes sense
        for key in subtypes:
            if not isinstance(key, str):
                raise TypeError(
                    'All keys of subtypes must be of type string. Got key `{}` of '
                    'type {}.'.format(key, type(key)))
            if key != '' and key not in labels:
                raise KeyError(
                    'All keys of subtypes must belong to labels. Got key `{}` '
                    'which is missing from labels.'.format(key))

            value = subtypes[key]
            if not isinstance(value, list):
                raise TypeError(
                    'All values of subtypes must be of type `list`. Got value {} '
                    'for key `{}` of type {}'.format(value, key, type(value)))
            for entry in value:
                if entry not in labels:
                    raise KeyError(
                        'All entries for each value of subtypes must belong to labels. '
                        'Got entry `{}` in key `{}` which is missing from labels.'.format(entry, key))
                counts[entry] += 1
        # create the root entry for subtypes
        if '' not in subtypes:
            subtypes[''] = []
        if isinstance(subtypes, OrderedDict):
            subtypes.move_to_end('', last=False)
        for key in counts:
            value = counts[key]
            if value > 1:
                raise ValueError('key {} is referenced in more than one subtype. This is invalid.'.format(key))
            if value == 0:
                subtypes[''].append(key)
        # now, we set the values
        self._version = version
        self._labels = labels
        self._subtypes = subtypes
        self._construct_parent_types()

    def _construct_parent_types(self):
        def iterate(key, entries):
            if key not in self._subtypes:
                self._parent_types[key].union(entries)
                return

            self._parent_types[key].union(entries)
            for entry in self._subtypes[key]:
                iterate(entry, {key, }.union(entries))
            return

        self._parent_types = OrderedDict((key, {key, }) for key in self._labels)
        for key in self._subtypes['']:
            iterate(key, set())

    @classmethod
    def from_file(cls, file_name):
        """
        Read schema from a file.

        Parameters
        ----------
        file_name : str

        Returns
        -------
        LabelSchema
        """

        with open(file_name, 'r') as fi:
            input_dict = json.load(fi)
        return cls.from_dict(input_dict)

    @classmethod
    def from_dict(cls, input_dict):
        """
        Construct from a dictionary.

        Parameters
        ----------
        input_dict : dict

        Returns
        -------
        LabelSchema
        """

        version = input_dict['version']
        labels = input_dict['labels']
        subtypes = input_dict.get('subtypes', None)
        conf_values = input_dict.get('confidence_values', None)
        perm_geometries = input_dict.get('permitted_geometries', None)
        return cls(
            version, labels, subtypes=subtypes, confidence_values=conf_values,
            permitted_geometries=perm_geometries)

    def to_dict(self):
        """
        Serialize to a dictionary representation.

        Returns
        -------
        dict
        """

        out = OrderedDict()
        out['version'] = self.version
        if self.confidence_values is not None:
            out['confidence_values'] = self.confidence_values
        if self.permitted_geometries is not None:
            out['permitted_geometries'] = self.permitted_geometries
        out['labels'] = self._labels
        out['subtypes'] = self._subtypes
        return out

    def to_file(self, file_name):
        """
        Write to a (json) file.

        Parameters
        ----------
        file_name : str

        Returns
        -------
        None
        """

        with open(file_name, 'w') as fi:
            json.dump(self.to_dict(), fi, indent=1)

    def is_valid_confidence(self, value):
        """
        Is the given value a valid confidence (i.e. is in `confidence_values`)?
        Note that `None` is always considered valid here.

        Parameters
        ----------
        value

        Returns
        -------
        bool
        """

        if self._confidence_values is None or value is None:
            return True
        else:
            return value in self._confidence_values

    def is_valid_geometry(self, value):
        """
        Is the given geometry type allowed (i.e. is in `permitted_geometries`)?
        Note that `None` is always considered valid here.

        Parameters
        ----------
        value
            If string, it should likely be the geometry type string (Point, Linestring, etc).
            For any other object, the exact name of the class will be used for the check.

        Returns
        -------
        bool
        """

        if self._permitted_geometries is None or value is None:
            return True
        elif isinstance(value, str):
            return value in self._permitted_geometries
        else:
            return value.__class__.__name__ in self._permitted_geometries


from PySide2.QtWidgets import *
from PySide2.QtGui import *
import time
import pandas as pd
from modules import functions
import os
import subprocess
import platform
import shutil
from modules import SettingsManager
settings = SettingsManager.settingsManager()

class tabMain(QWidget):

    global settings

    def __init__(self,parent):
        super().__init__(parent)

        self.instance_types = pd.read_csv(functions.resource_path('files/instances.csv'))

        # Acciones
        self.hbox_manipulate = QHBoxLayout()
        self.hbox_manipulate.addWidget(QLabel("Actions:"))

        self.on = QPushButton("Turn On")
        self.off = QPushButton("Turn Off")
        self.reset = QPushButton("Reset")

        self.grid_manipulate = QGridLayout()
        self.grid_manipulate.addWidget(self.on, 0, 0)
        self.grid_manipulate.addWidget(self.off, 0, 1)
        self.grid_manipulate.addWidget(self.reset, 0, 2)

        self.on.clicked.connect(self.fn_prender)
        self.off.clicked.connect(self.fn_apagar)
        self.reset.clicked.connect(self.fn_reset)

        # Tamaño
        self.grid_manipulate.addWidget(QLabel('Instance Size:'), 1, 0)
        self.instance_type = QComboBox()

        self.grid_manipulate.addWidget(self.instance_type, 1, 1)
        self.set_type = QPushButton("Set Size")
        self.grid_manipulate.addWidget(self.set_type, 1, 2)
        self.set_type.clicked.connect(self.fn_set_type)

        # Atributos
        self.hbox_attr = QHBoxLayout()
        self.hbox_attr.addWidget(QLabel("Instance Information:"))

        self.grid_attr = QGridLayout()
        self.grid_attr.addWidget(QLabel('Name'), 0, 0)
        self.name = QLineEdit()
        self.name.setReadOnly(True)
        self.grid_attr.addWidget(self.name, 0, 1)

        self.grid_attr.addWidget(QLabel('Type'), 1, 0)
        self.type = QLineEdit()
        self.type.setReadOnly(True)
        self.grid_attr.addWidget(self.type, 1, 1)

        self.grid_attr.addWidget(QLabel('USD / hour'), 2, 0)
        self.price = QLineEdit()
        self.price.setReadOnly(True)
        self.grid_attr.addWidget(self.price, 2, 1)

        self.grid_attr.addWidget(QLabel('IP'), 3, 0)
        self.ip = QLineEdit()
        self.ip.setReadOnly(True)
        self.grid_attr.addWidget(self.ip, 3, 1)

        self.grid_attr.addWidget(QLabel('Status'), 4, 0)
        self.status = QLineEdit()
        self.status.setReadOnly(True)
        self.grid_attr.addWidget(self.status, 4, 1)

        # Servicios
        self.hbox_serv = QHBoxLayout()
        self.hbox_serv.addWidget(QLabel("Launch Services:"))

        self.nomachine = QPushButton("Remote Desktop")
        self.rstudio = QPushButton("RStudio")
        self.jupyter = QPushButton("Jupyter")
        self.ssh = QPushButton("SSH")
        self.sftp = QPushButton("File Transfer")

        self.nomachine.clicked.connect(self.launch_nx)
        self.rstudio.clicked.connect(self.launch_rstudio)
        self.jupyter.clicked.connect(self.launch_jupyter)
        self.ssh.clicked.connect(self.launch_ssh)
        self.sftp.clicked.connect(self.launch_sftp)

        self.hbox_url = QGridLayout()
        self.hbox_url.addWidget(self.nomachine,0,0)
        self.hbox_url.addWidget(self.rstudio,0,1)
        self.hbox_url.addWidget(self.jupyter,1,0)
        self.hbox_url.addWidget(self.ssh, 1, 1)
        self.hbox_url.addWidget(self.sftp, 2, 0)

        self.refresh = QPushButton("Refresh")
        self.refresh.clicked.connect(self.fn_status)

        self.vbox = QVBoxLayout()
        self.vbox.addLayout(self.hbox_manipulate)
        self.vbox.addLayout(self.grid_manipulate)
        self.vbox.addLayout(self.hbox_attr)
        self.vbox.addLayout(self.grid_attr)
        self.vbox.addWidget(self.refresh)
        self.vbox.addLayout(self.hbox_serv)
        self.vbox.addLayout(self.hbox_url)

        self.setLayout(self.vbox)
        self.fn_status()

    def fn_status(self):

        global settings

        if settings.getParam('os') == 'Linux':
            self.rstudio.setEnabled(True)
            self.jupyter.setEnabled(True)
        elif settings.getParam('os') == 'Windows':
            self.rstudio.setEnabled(False)
            self.jupyter.setEnabled(False)

        self.vm_name = settings.getParam('vm_name')

        #self.i = settings.getInstance()
        if self.i == None:
            print("please setup api keys")

        try:
            state = self.i.state["Name"]
            self.status.setText(state)

            self.instance_type.clear()

            for x in self.instance_types.values:
                self.instance_type.addItem(x[1],x[2])

            current_type = self.instance_types.loc[self.instance_types.id ==self.i.instance_type].values
            self.instance_type.setCurrentText(current_type[0][1])

            tags = functions.tagsToDict(self.i.tags)
            #print(tags)
            self.name.setText(tags["Name"])
            self.type.setText(self.i.instance_type)
            self.price.setText(functions.get_price(self.i.instance_type))
        except:
            self.name.setText("Invalid ID")
        try:
            self.ip.setText(settings.getIP())
        except:
            self.ip.setText("")

    def fn_prender(self):
        self.i.start()
        while self.i.state["Name"] != "running":
            print(".", end="")
            self.status.setText(self.i.state["Name"])
            time.sleep(2)
            self.i.reload()
        self.fn_status()

    def fn_apagar(self):
        self.i.stop()
        time.sleep(2)
        self.i.reload()
        self.status.setText(self.i.state["Name"])

    def fn_reset(self):
        functions.run_script(self.i,'reboot')

    def fn_set_type(self):
        if (self.i.state["Name"] == "stopped"):
            #print(self.instance_type.currentData())
            self.i.modify_attribute(Attribute='instanceType', Value=self.instance_type.currentData())
        self.fn_status()

    def launch_nx(self):
        global settings
        if settings.getParam('os') == 'Linux':
            file = functions.setNxXML(self.ip.text())
            params =  [
                '--session-conf={}'.format(file),
                '--sessionid=20181207145907927',
                '--no-menu',
                '--no-session-edit',
                '--tray-icon',
                '--clipboard=both',
                '--dpi=96',
                '--add-to-known-hosts']
            if platform.system() == 'Windows':
                subprocess.Popen(["C:\\Program Files (x86)\\x2goclient\\x2goclient.exe"]+params + ["--disable-pulse"])
            else:
                subprocess.Popen(["x2goclient"] + params)
            pass
        else:
            params = ["/v:"+self.ip.text()]
            subprocess.Popen(["mstsc"] + params)

    def launch_rstudio(self):
        QDesktopServices.openUrl(
            "http://" + self.ip.text() + ":8787/")

    def launch_jupyter(self):
        QDesktopServices.openUrl(
            "http://" + self.ip.text() + ":8000/")

    def launch_ssh(self):
        ip =  self.ip.text()
        user = settings.getParam('user')

        if platform.system() == 'Windows':
            subprocess.Popen( [functions.resource_path(os.path.join('files','putty.exe')), user + '@' + ip] )
        else:
            cmd = ' ssh  -o "StrictHostKeyChecking no" ' + user + '@' + ip + ' &'
            if shutil.which('sshpass') != None:
                cmd = "sshpass -p '{}' ".format(settings.getParam('ec2_passwd')) + cmd
            if shutil.which('konsole') != None:
                os.system('konsole -e ' + cmd)
            elif shutil.which('xfce4-terminal') != None:
                os.system('xfce4-terminal -x ' + cmd)
            elif shutil.which('gnome-terminal') != None:
                os.system('gnome-terminal ' + cmd)
            elif shutil.which('xterm') != None:
                os.system('xterm -e ' + cmd)

    def launch_sftp(self):
        ip =  self.ip.text()
        if settings.getParam("os") == "Windows":
            user = "Administrator"
        else:
            user = settings.getParam('user')

        if platform.system() == 'Windows':
            subprocess.Popen( ["winscp", user +':' + settings.getParam("ec2_passwd") + '@' + ip] )
        else:
            os.system( 'dolphin sftp://'+ user +':' + settings.getParam("ec2_passwd") + '@' + ip + ' &')


# Automates the execution of OpenSim analyses.
#
# It is better to execute the analyses one by one in the OpenSim GUI and perform
# the necessary sanity checks to be sure. Once we are sure that all stages work
# properly (e.g., scaling), this script can be used to automate the
# process. Make sure that the initial and final time of the residual reduction
# algorithm corresponds to when the legs touch the force plates. To check this,
# plot the vertical force of the left and right legs (e.g.,
# experimental_data/walk_grf.mot) to identify the time interval.
#
# author: Dimitar Stanev <jimstanev@gmail.com>
##
import re
import os
from subprocess import PIPE, run, call
from utils import adjust_model_mass, subject_specific_isometric_force
from utils import plot_sto_file, replace_thelen_muscles_with_millard

##
# subject data

# 1.61m, 41.5 kg
subject_height = 1.610
generic_height = 1.70
subject_dir = os.path.abspath('../')
os.chdir(subject_dir)

# switches
adapt_mass = True
adapt_muscle_strength = False
calculate_muscle_activations = True
calculate_joint_reactions = False

##
# scale

os.chdir('scale/')
call(['opensim-cmd', 'run-tool', 'setup_scale.xml'])
os.chdir(subject_dir)

##
# inverse kinematics

os.chdir('inverse_kinematics/')
call(['opensim-cmd', 'run-tool', 'setup_ik.xml'])
plot_sto_file('task_InverseKinematics.mot', 'task_InverseKinematics.pdf', 3)
plot_sto_file('task_ik_marker_errors.sto', 'task_ik_marker_errors.pdf', 3)
plot_sto_file('task_ik_model_marker_locations.sto',
              'task_ik_model_marker_locations.pdf', 3)
os.chdir(subject_dir)

##
# residual reduction algorithm and model adjustment

os.chdir('residual_reduction_algorithm/')
rra_output = run(['opensim-cmd', 'run-tool', 'setup_rra.xml'],
                 stdout=PIPE, stderr=PIPE, universal_newlines=True)
print(rra_output.stdout)

# find mass change from RRA output (perform only when RRA does not suggest much
# mass change)
if adapt_mass:
    mass_change = float(re.findall(
        '[-+]?[.]?[\d]+(?:,\d\d\d)*[\.]?\d*(?:[eE][-+]?\d+)?',
        re.search('Total mass change: .?[0-9]+[.][0-9]+',
                  rra_output.stdout).group(0))[0])

    # load model and manually adjust body masses (RRA adjusts only CoM not body
    # masses)
    adjust_model_mass('model_adjusted.osim', mass_change)

# adjust max isometric force of subject-specific model based on height and
# weight regression model
if adapt_muscle_strength:
    subject_specific_isometric_force('../model/model_generic.osim',
                                     'model_adjusted.osim',
                                     generic_height,
                                     subject_height)

os.chdir(subject_dir)

##
# inverse dynamics

os.chdir('inverse_dynamics/')
call(['opensim-cmd', 'run-tool', 'setup_id.xml'])
plot_sto_file('task_InverseDynamics.sto', 'task_InverseDynamics.pdf', 3)
os.chdir(subject_dir)

##
# muscle analysis

os.chdir('muscle_analysis/')
replace_thelen_muscles_with_millard(
    '../residual_reduction_algorithm/model_adjusted.osim', '.')
call(['opensim-cmd', 'run-tool', 'setup_ma.xml'])
plot_sto_file('task_MuscleAnalysis_NormalizedFiberLength.sto',
              'task_MuscleAnalysis_NormalizedFiberLength.pdf', 3)
plot_sto_file('task_MuscleAnalysis_NormFiberVelocity.sto',
              'task_MuscleAnalysis_NormFiberVelocity.pdf', 3)
os.chdir(subject_dir)

##
# static optimization

if calculate_muscle_activations:
    os.chdir('static_optimization/')
    call(['opensim-cmd', 'run-tool', 'setup_so.xml'])
    plot_sto_file('task_StaticOptimization_activation.sto',
                  'task_StaticOptimization_activation.pdf', 3)
    plot_sto_file('task_StaticOptimization_force.sto',
                  'task_StaticOptimization_force.pdf', 3)
    os.chdir(subject_dir)

##
# joint reaction analysis

if calculate_muscle_activations and calculate_joint_reactions:
    os.chdir('joint_reaction_analysis/')
    call(['opensim-cmd', 'run-tool', 'setup_jra.xml'])
    plot_sto_file('task_JointReaction_ReactionLoads.sto',
                  'task_JointReaction_ReactionLoads.pdf', 3)
    os.chdir(subject_dir)

##
# computed muscle controls (takes more time)

if calculate_muscle_activations:
    os.chdir('computed_muscle_controls/')
    call(['opensim-cmd', 'run-tool', 'setup_cmc.xml'])
    # plot_sto_file('task_states.sto', 'task_states.pdf', 3)
    plot_sto_file('task_Actuation_force.sto', 'task_Actuation_force.pdf', 3)
    plot_sto_file('task_MuscleAnalysis_NormalizedFiberLength.sto',
                  'task_MuscleAnalysis_NormalizedFiberLength.pdf', 3)
    plot_sto_file('task_MuscleAnalysis_NormFiberVelocity.sto',
                  'task_MuscleAnalysis_NormFiberVelocity.pdf', 3)
    os.chdir(subject_dir)

##


from sstcam_simulation import Photoelectrons
from sstcam_simulation.camera.spe import single_gaussian, sipm_gentile_spe, \
    SPESpectrum, optical_crosstalk_probability, SiPMDelayed
from ctapipe.core import non_abstract_children
import numpy as np
import pytest

def test_pmt():
    x = np.linspace(0, 3, 10000)
    pdf = single_gaussian(x, 0.2)
    avg = np.average(x, weights=pdf)
    var = np.average((x - avg) ** 2, weights=pdf)
    np.testing.assert_allclose(avg, 1, rtol=1e-5)
    np.testing.assert_allclose(var, 0.04, rtol=1e-5)

def test_optical_crosstalk_probability():
    k = 1
    assert optical_crosstalk_probability(k, 0.3) == 1-0.3

    k = np.arange(1, 250)
    assert optical_crosstalk_probability(k, 0.2).sum() == 1

    assert optical_crosstalk_probability(0, 0.3) == 0

def test_sipm_gentile():
    x = np.linspace(0, 10, 10000)
    pdf = sipm_gentile_spe(x, 0.2, 0.3)
    avg = np.average(x, weights=pdf)
    var = np.average((x - avg) ** 2, weights=pdf)
    np.testing.assert_allclose(avg, 1.428433, rtol=1e-5)
    np.testing.assert_allclose(var, 0.668078, rtol=1e-5)

def _get_result_photoelectrons(spectrum, rng):
    n_events = 1000
    pe = []
    charge = []
    for iev in range(n_events):
        n_photoelectrons = 1
        photoelectrons = Photoelectrons(
            pixel=np.zeros(n_photoelectrons, dtype=np.int),
            time=np.zeros(n_photoelectrons),
            charge=np.ones(n_photoelectrons),
            metadata=dict(test="test"),
        )
        result = spectrum.apply(photoelectrons, rng)
        pe.append(result.get_photoelectrons_per_pixel(1)[0])
        charge.append(result.get_charge_per_pixel(1)[0])
    return np.array(pe), np.array(charge)

@pytest.mark.parametrize("spectrum_class", non_abstract_children(SPESpectrum))
def test_spe_spectra(spectrum_class):
    rng = np.random.RandomState(seed=3)

    spectrum = spectrum_class(normalise_charge=True)
    pe, charge = _get_result_photoelectrons(spectrum, rng)
    np.testing.assert_allclose(spectrum.average, 1, rtol=2e-2)
    np.testing.assert_allclose(pe.mean(), 1, rtol=2e-2)
    np.testing.assert_allclose(charge.mean(), 1, rtol=2e-2)
    # np.testing.assert_allclose(1+charge.std()**2, spectrum.excess_noise_factor, rtol=1e-2)

    spectrum = spectrum_class(normalise_charge=False)
    pe, charge = _get_result_photoelectrons(spectrum, rng)
    np.testing.assert_allclose(pe.mean(), 1, rtol=2e-2)
    np.testing.assert_allclose(charge.mean(), spectrum.average, rtol=2e-2)
    # np.testing.assert_allclose(1+charge.std()**2, spectrum.excess_noise_factor, rtol=1e-2)

def test_delayed():
    n_photoelectrons = 1000000
    photoelectrons = Photoelectrons(
        pixel=np.zeros(n_photoelectrons, dtype=np.int),
        time=np.full(n_photoelectrons, 10.),
        charge=np.ones(n_photoelectrons),
    )

    rng = np.random.RandomState(seed=1)

    spectrum_template = SiPMDelayed(spe_sigma=0.1, opct=0.2, time_constant=20)
    result = spectrum_template.apply(photoelectrons, rng)
    time = result.time[~result.initial]
    assert (time > 10).all()
    np.testing.assert_allclose(time.mean(), 10 + 20, rtol=1e-2)
    np.testing.assert_allclose(time.std(), 20, rtol=1e-2)


# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

# flake8: noqa
import logging
import threading

import azure.functions as func

async def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse:
    logging.info('Before threads.')

    t1 = threading.Thread(target=thread_function, args=(context, 'Thread1 used.'))
    t2 = threading.Thread(target=thread_function, args=(context, 'Thread2 used.'))
    t3 = threading.Thread(target=thread_function, args=(context, 'Thread3 used.'))

    t1.start()
    t2.start()
    t3.start()

    t1.join()
    t2.join()
    t3.join()

    logging.info('After threads.')

    return func.HttpResponse('This HTTP triggered function executed successfully.', status_code=200)

def thread_function(context: func.Context, message: str):
    context.thread_local_storage.invocation_id = context.invocation_id
    logging.info(message)


import pandas as pd
#import tia.bbg.datamgr as dm
import numpy as np
import scipy as sp
#from googlefinance.client import get_price_data, get_prices_data, get_prices_time_data
from pandas_datareader import data
from alpha_vantage.timeseries import TimeSeries
import datetime
import os
import signals
from pandas.tseries.offsets import BMonthEnd

def get_tickers_from_xls(excel_file, index):
    return pd.read_excel(excel_file, index_col = index)

def get_last_business_day_of_month(date_to_find):
    offset = BMonthEnd()
    return offset.rollforward(date_to_find).date()

def generate_list_of_business_days(last_year):
    d = datetime.date.today()
    last_b_days = []
    while (d.year>=last_year):
        last_b_days = last_b_days + [get_last_business_day_of_month(d)]
        d = d - datetime.timedelta(days = 30)
    return sorted(last_b_days)

def get_all_done(excel_file, index, provider):
    tick_list = get_tickers_from_xls(excel_file, index).index.values
    name_list = get_tickers_from_xls(excel_file, index)['Indicator'].values
    df_hist   = signals.get_data(tick_list,provider).fillna(method='ffill')
    index_lis = pd.DatetimeIndex(generate_list_of_business_days(2015))
    print (index_lis)
    filter_df = df_hist.ix[index_lis]
    filter_df.columns = name_list
    return filter_df.dropna().pct_change(periods = 12)

filter_df = get_all_done('macroeconomics.xlsx', 'Ticker', 'fred').tail(12).T

signals.write_to_excel(filter_df, "Macro-Economic Dashboard.xlsx")
#signals.write_to_excel(get_all_done('macroeconomics.xlsx', 'Ticker', 'fred'), "testing eco 2.xlsx")


from gamegym.games import RockPaperScissors, Goofspiel
from gamegym.strategy import FixedStrategy, UniformStrategy
from gamegym.algorithms import BestResponse
from gamegym.distribution import Explicit
import pytest

def test_best_response_rps():

    bart_simpson_strategy = FixedStrategy(Explicit([1, 0, 0], values=["R", "P", "S"]))
    game = RockPaperScissors()
    strategy = BestResponse(game, 0, {1: bart_simpson_strategy})
    assert list(strategy.best_responses.values())[0].probability("R") == 0.0
    assert list(strategy.best_responses.values())[0].probability("P") == 1.0
    assert list(strategy.best_responses.values())[0].probability("S") == 0.0
    assert strategy.value == pytest.approx(1.0)

    strategy = BestResponse(game, 1, {0: bart_simpson_strategy})
    assert list(strategy.best_responses.values())[0].probability("R") == 0.0
    assert list(strategy.best_responses.values())[0].probability("P") == 1.0
    assert list(strategy.best_responses.values())[0].probability("S") == 0.0
    assert strategy.value == pytest.approx(1.0)

def test_best_response_goofspiel():

    for n_cards, br_value in [(3, pytest.approx(4/3)), (4, pytest.approx(2.5))]:
        game = Goofspiel(n_cards, Goofspiel.Scoring.ZEROSUM)
        strategy = BestResponse(game, 0, {1: UniformStrategy()})
        for k, v in strategy.best_responses.items():
            reward = k[1][-1]
            assert reward not in v.values() or v.probability(reward) == 1.0
        assert strategy.value == br_value


import flask, flask_login
import requests
from .. import db, login_manager
from flask import request, flash, redirect, session, url_for, current_app, render_template
from . import learning_hub_bp
from . import forms, models
from ..static import constant_functions as const_func
from ..static import enums
import jwt
from ..user_managment import models as user_models

@learning_hub_bp.route("/free-for-all")
def free_for_all_questions():
    return render_template("free_for_all_q.html")

@learning_hub_bp.route("/get_q_ids_free_for_all", methods=["POST", "GET"])
def get_q_ids_free_for_all():
    is_timed = request.json
    if is_timed:
        questions = models.Questions.query.filter(models.Questions.time_for_completion > "0",
                                                  models.Questions.is_exam == "0",
                                                  models.Questions.is_test_exam == "0").all()

    else:
        questions = models.Questions.query.filter(models.Questions.is_exam == "0",
                                                  models.Questions.is_test_exam == "0").all()

    return create_response(questions, is_timed)

@learning_hub_bp.route("/get_q_by_id", methods=["POST", "GET"])
def get_q_by_id():
    response = ""

    q = models.Questions.query.filter_by(id=request.json).first()

    return build_q_response(q)

def build_q_response(q):
    if q.qtype == enums.QuestionType.SINGLE:
        response = {"id": q.id,
                    "title": q.question_title,
                    "body": q.question_text,
                    "answer": q.answer,
                    "type": q.qtype.value,
                    "notion": q.notion.notion,
                    "subnotion": q.sub_notion.sub_notion,
                    "level": q.level.value,
                    "time": q.time_for_completion}
    else:
        response = {"id": q.id,
                    "title": q.question_title,
                    "body": q.question_text,
                    "answer": q.answer,
                    "wrong1": q.wrong_answer1,
                    "wrong2": q.wrong_answer2,
                    "wrong3": q.wrong_answer3,
                    "type": q.qtype.value,
                    "notion": q.notion.notion,
                    "subnotion": q.sub_notion.sub_notion,
                    "level": q.level.value,
                    "time": q.time_for_completion}
    return response

@learning_hub_bp.route("/exe_by_notion", methods=["POST", "GET"])
def exe_by_notion():
    notions = models.Notions.query.all()
    subnotions = models.SubNotions.query.all()
    return render_template("exercise_by_notion.html", notions=notions, subnotions=subnotions)

@learning_hub_bp.route("/get_q_by_notion", methods=["POST", "GET"])
def get_q_by_notion():
    notion = models.Notions.query.filter_by(notion=request.json[0]).first()
    sub_notion = models.SubNotions.query.filter_by(sub_notion=request.json[1]).first()
    is_timed = request.json[2]
    if is_timed:
        questions = models.Questions.query.filter(models.Questions.time_for_completion > "0", \
                                                  models.Questions.is_exam == "0", \
                                                  models.Questions.is_test_exam == "0", \
                                                  models.Questions.notion_id == notion.id, \
                                                  models.Questions.sub_notion_id == sub_notion.id).all()

    else:
        questions = models.Questions.query.filter(models.Questions.is_exam == 0, \
                                                  models.Questions.is_test_exam == 0, \
                                                  models.Questions.notion_id == notion.id, \
                                                  models.Questions.sub_notion_id == sub_notion.id).all()

    if questions:
        return create_response(questions, is_timed)
    else:
        return "false"

def create_response(questions, timer):
    response = {}
    if timer:
        for number, q in enumerate(questions):
            response[number] = [q.id, q.time_for_completion]
    else:
        for number, q in enumerate(questions):
            response[number] = [q.id]
    return response

@learning_hub_bp.route("/exams")
def exams_main():
    if session["role"] != enums.UserType.STUDENT:
        flash("This is student area, teachers and admins belong in management", category="student_vs_teachers")
        return render_template("exams.html")
    exams = ""
    student_email = session['user_email']
    student_class = user_models.User.query.filter_by(email=student_email).first()
    if not student_class.class_name:
        flash("You are not assigned to class", category="error")

    elif not student_class.class_name.exams:
        flash("You are not assigned an exam", category="error")
    else:
        exams = student_class.class_name.exams
    return render_template("exams.html", exams=exams)

@learning_hub_bp.route("/get_exam", methods=["POST", "GET"])
def get_exam():
    exam_id = request.json

    exam = models.Exams.query.filter_by(id=exam_id).first()
    response = {"id": exam.id,
                "name": exam.exam_title,
                "time": exam.time_for_completion}

    return response

@learning_hub_bp.route("/get_questions_by_exam", methods=["POST", "GET"])
def get_questions_by_exam():
    exam_id = request.json
    exam = models.Exams.query.filter_by(id=exam_id).first()
    questions = models.Questions.query.filter(models.Questions.exams.contains(exam)).all()
    response = {}
    return create_response(questions, False)

@learning_hub_bp.route("/submit_exam", methods=['POST', 'GET'])
def submit_exam():
    calculate_exam_score(request.json)
    return "True"

def calculate_exam_score(answer_sheet):
    """
    The score, for now is calculated by equal amount - 100 / number of questions
    """
    exam = models.Exams.query.filter_by(id=answer_sheet["exam_id"]).first()
    student = user_models.User.query.filter_by(email=session['user_email']).first()
    number_of_questions = len(exam.questions)
    number_of_correct_questions = 0
    exam_questions = exam.questions
    for answer in answer_sheet['answers']:
        for q in exam_questions:
            if answer['answer'] == q.answer:
                number_of_correct_questions += 1

    score = (number_of_correct_questions / number_of_questions) * 100

    exam_score = models.ExamScores()
    exam_score.score = score
    exam_score.student = student.id
    exam_score.exam = exam.id
    db.session.add(exam_score)
    db.session.commit()

    return True

@learning_hub_bp.route("/practice_exam")
def practice_exam():
    return redirect(url_for("main.under_const"))


import sqlite3

from exceptions.zipcode import Zip_codesCreationError

class ZipcodesService:
    """Сервис взаимодействия с таблицей zipcode"""
    def __init__(self, connection):
        self.connection = connection

    def _check_existence(self, zip_code: str) -> bool:
        """Проверка существования индекса в базе"""
        query = (
            """
            SELECT *
            FROM zipcode
            WHERE zip_code = ?
            """
        )
        params = (zip_code,)

        cursor = self.connection.execute(query, params)
        if cursor.fetchone() is not None:
            return True
        return False

    def create(self, zip_code_data: dict):
        """Создание новой связи в базе"""
        zip_code = zip_code_data["zip_code"]
        city_id = zip_code_data["city_id"]

        query = (
            """
            INSERT INTO zipcode (zip_code, city_id) VALUES (?, ?)
            """
        )

        params = (
            zip_code,
            city_id,
        )

        if self._check_existence(zip_code):
            return

        try:
            self.connection.execute(query, params)
            self.connection.commit()
        except sqlite3.IntegrityError:
            raise Zip_codesCreationError

    def update(self, data: dict):
        """Частичное редактирование записи в базе"""
        zip_code = data.get("zip_code")
        city_id = data.get("city_id")

        if not all((zip_code, city_id)):
            return

        if self._check_existence(zip_code):
            self.connection.execute(
                'UPDATE zipcode SET city_id = ? WHERE zip_code = ?',
                (city_id, zip_code)
            )

        else:
            self.create(
                {
                    "zip_code": zip_code,
                    "city_id": city_id,
                }
            )

Lesson 13/final v 2.0/src/services/zipcode.py


# -*- coding: utf-8 -*-
"""
Created on Sun Feb  6 08:32:00 2022

@author: SHERIF ATITEBI O
"""

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import confusion_matrix, accuracy_score
#%%

dataset = pd.read_csv("Social_Network_Ads.csv")
x = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values
#%%

#Splitting into training set and test set
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.25, random_state=0)
#%%

#feature scaling
sc = StandardScaler()
x_train = sc.fit_transform(x_train)
x_test = sc.transform(x_test)
#%%

#Logistic Regression Model
classifier = GaussianNB()
classifier.fit(x_train, y_train)
#%%

# Predicting the test set
y_pred = classifier.predict(x_test)
print(np.concatenate((y_pred.reshape(len(y_pred), 1), y_test.reshape(len(y_test), 1)), 1))
#%%

#Confusion Matrix
cm = confusion_matrix(y_test, y_pred)
print(cm)
print(accuracy_score(y_test, y_pred))
#%%


import torch
import torch.nn as nn
from torch.nn import init
import numpy as np
import functools
import torch.nn.functional as F
from torch.optim import lr_scheduler
import util.util as util
from .InnerShiftTriple import InnerShiftTriple
from .InnerCos import InnerCos

###############################################################################
# Functions
###############################################################################
def get_norm_layer(norm_type='instance'):
    if norm_type == 'batch':
        norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
    elif norm_type == 'instance':
        norm_layer = functools.partial(nn.InstanceNorm2d, affine=True)
    elif norm_type == 'none':
        norm_layer = None
    else:
        raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
    return norm_layer

def get_scheduler(optimizer, opt):
    if opt.lr_policy == 'lambda':
        def lambda_rule(epoch):
            lr_l = 1.0 - max(0, epoch + 1 + opt.epoch_count - opt.niter) / float(opt.niter_decay + 1)
            return lr_l
        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule)
    elif opt.lr_policy == 'step':
        scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_iters, gamma=0.1)
    elif opt.lr_policy == 'plateau':
        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5)
    elif opt.lr_policy == 'cosine':
        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=opt.niter, eta_min=0)
    else:
        return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy)
    return scheduler

def init_weights(net, init_type='normal', gain=0.02):
    def init_func(m):
        classname = m.__class__.__name__
        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
            if init_type == 'normal':
                init.normal_(m.weight.data, 0.0, gain)
            elif init_type == 'xavier':
                init.xavier_normal_(m.weight.data, gain=gain)
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal_(m.weight.data, gain=gain)
            else:
                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)
        elif classname.find('BatchNorm2d') != -1:
            init.normal_(m.weight.data, 1.0, gain)
            init.constant_(m.bias.data, 0.0)

    print('initialize network with %s' % init_type)
    net.apply(init_func)

def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]):
    if len(gpu_ids) > 0:
        assert(torch.cuda.is_available())
        net.to(gpu_ids[0])
        net = torch.nn.DataParallel(net, gpu_ids)
    init_weights(net, init_type, gain=init_gain)
    return net

def define_G(input_nc, output_nc, ngf, which_model_netG, opt, mask_global, norm='batch', use_dropout=False, init_type='normal', gpu_ids=[], init_gain=0.02):
    netG = None
    norm_layer = get_norm_layer(norm_type=norm)

    innerCos_list = []
    shift_list = []

    # Here we need to initlize an artificial mask_global to construct the init model.
    # When training, we need to set mask for special layers(mostly for Shift layers) first.
    # If mask is fixed during training, we only need to set mask for these layers once,
    # else we need to set the masks each iteration, generating new random masks and mask the input
    # as well as setting masks for these special layers.

    if which_model_netG == 'unet_256':
        netG = UnetGenerator(input_nc, output_nc, 8, ngf, norm_layer=norm_layer, use_dropout=use_dropout)
    elif which_model_netG == 'unet_shift_triple':
        netG = UnetGeneratorShiftTriple(input_nc, output_nc, 8, opt, innerCos_list, shift_list, mask_global, \
                                                         ngf, norm_layer=norm_layer, use_dropout=use_dropout)
    else:
        raise NotImplementedError('Generator model name [%s] is not recognized' % which_model_netG)

    print('Constraint in netG:')
    print(innerCos_list)

    print('Shift in netG:')
    print(shift_list)

    return init_net(netG, init_type, init_gain, gpu_ids), innerCos_list, shift_list

def define_D(input_nc, ndf, which_model_netD,
             n_layers_D=3, norm='batch', use_sigmoid=False, init_type='normal', gpu_ids=[], init_gain=0.02):
    netD = None
    norm_layer = get_norm_layer(norm_type=norm)

    if which_model_netD == 'basic':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers=3, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    elif which_model_netD == 'n_layers':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers_D, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    else:
        print('Discriminator model name [%s] is not recognized' %
              which_model_netD)
    return init_net(netD, init_type, init_gain, gpu_ids)

##############################################################################
# Classes
##############################################################################

# Defines the GAN loss which uses either LSGAN or the regular GAN.
# When LSGAN is used, it is basically same as MSELoss,
# but it abstracts away the need to create the target label tensor
# that has the same size as the input
class GANLoss(nn.Module):
    def __init__(self, gan_type='wgan_gp', target_real_label=1.0, target_fake_label=0.0):
        super(GANLoss, self).__init__()
        self.register_buffer('real_label', torch.tensor(target_real_label))
        self.register_buffer('fake_label', torch.tensor(target_fake_label))
        if gan_type == 'wgan_gp':
            self.loss = nn.MSELoss()
        elif gan_type == 'lsgan':
            self.loss = nn.MSELoss()
        elif gan_type == 'vanilla':
            self.loss = nn.BCELoss()
        else:
            raise ValueError("GAN type [%s] not recognized." % gan_type)

    def get_target_tensor(self, input, target_is_real):
        if target_is_real:
            target_tensor = self.real_label
        else:
            target_tensor = self.fake_label
        return target_tensor.expand_as(input)

    def __call__(self, input, target_is_real):
        target_tensor = self.get_target_tensor(input, target_is_real)
        return self.loss(input, target_tensor)

################################### ***************************  #####################################
################################### This the original Shift_net  #####################################
################################### ***************************  #####################################
# Defines the Unet generator.
# |num_downs|: number of downsamplings in UNet. For example,
# if |num_downs| == 7, image of size 128x128 will become of size 1x1
# at the bottleneck
class UnetGeneratorShiftTriple(nn.Module):
    def __init__(self, input_nc, output_nc,  num_downs, opt, innerCos_list, shift_list, mask_global, ngf=64,
                 norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetGeneratorShiftTriple, self).__init__()

        # construct unet structure
        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)
        print(unet_block)
        for i in range(num_downs - 5):  # The innner layers number is 3 (sptial size:512*512), if unet_256.
            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)
        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)

        unet_shift_block = UnetSkipConnectionShiftTriple(ngf * 2, ngf * 4, opt, innerCos_list, shift_list, mask_global, input_nc=None, \
                                                         submodule=unet_block, norm_layer=norm_layer)  # passing in unet_shift_block
        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_shift_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)

        self.model = unet_block

    def forward(self, input):
        return self.model(input)

# Mention: the TripleBlock differs in `upconv` defination.
# 'cos' means that we add a `innerCos` layer in the block.
class UnetSkipConnectionShiftTriple(nn.Module):
    def __init__(self, outer_nc, inner_nc, opt, innerCos_list, shift_list, mask_global, input_nc, \
                 submodule=None, shift_layer=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetSkipConnectionShiftTriple, self).__init__()
        self.outermost = outermost
        if input_nc is None:
            input_nc = outer_nc

        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,
                             stride=2, padding=1)
        downrelu = nn.LeakyReLU(0.2, True)
        downnorm = norm_layer(inner_nc, affine=True)
        uprelu = nn.ReLU(True)
        upnorm = norm_layer(outer_nc, affine=True)

        # As the downconv layer is outer_nc in and inner_nc out.
        # So the shift define like this:
        shift = InnerShiftTriple(opt.threshold, opt.fixed_mask, opt.shift_sz, opt.stride, opt.mask_thred, opt.triple_weight)

        shift.set_mask(mask_global, 3, opt.threshold)
        shift_list.append(shift)

        # Add latent constraint
        # Then add the constraint to the constrain layer list!
        innerCos = InnerCos(strength=opt.strength, skip=opt.skip)
        innerCos.set_mask(mask_global, opt)  # Here we need to set mask for innerCos layer too.
        innerCos_list.append(innerCos)

        # Different position only has differences in `upconv`
            # for the outermost, the special is `tanh`
        if outermost:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downconv]
            up = [uprelu, upconv, nn.Tanh()]
            model = down + [submodule] + up
            # for the innermost, the special is `inner_nc` instead of `inner_nc*2`
        elif innermost:
            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv]  # for the innermost, no submodule, and delete the bn
            up = [uprelu, upconv, upnorm]
            model = down + up
            # else, the normal
        else:
            # shift triple differs in here. It is `*3` not `*2`.
            upconv = nn.ConvTranspose2d(inner_nc * 3, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv, downnorm]
            # shift should be placed after uprelu
            # NB: innerCos are placed before shift. So need to add the latent gredient to
            # to former part.
            up = [uprelu, innerCos, shift, upconv, upnorm]

            if use_dropout:
                model = down + [submodule] + up + [nn.Dropout(0.5)]
            else:
                model = down + [submodule] + up

        self.model = nn.Sequential(*model)

    def forward(self, x):
        if self.outermost:  # if it is the outermost, directly pass the input in.
            return self.model(x)
        else:
            x_latter = self.model(x)
            _, _, h, w = x.size()
            if h != x_latter.size(2) or w != x_latter.size(3):
                x_latter = F.interpolate(x_latter, (h, w), mode='bilinear')
            return torch.cat([x_latter, x], 1)  # cat in the C channel

# Defines the Unet generator.
# |num_downs|: number of downsamplings in UNet. For example,
# if |num_downs| == 7, image of size 128x128 will become of size 1x1
# at the bottleneck
class UnetGenerator(nn.Module):
    def __init__(self, input_nc, output_nc, num_downs, ngf=64,
                 norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetGenerator, self).__init__()

        # construct unet structure
        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)
        for i in range(num_downs - 5):
            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)
        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)

        self.model = unet_block

    def forward(self, input):
        return self.model(input)

# construct network from the inside to the outside.
# Defines the submodule with skip connection.
# X -------------------identity---------------------- X
#   |-- downsampling -- |submodule| -- upsampling --|
class UnetSkipConnectionBlock(nn.Module):
    def __init__(self, outer_nc, inner_nc, input_nc,
                 submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetSkipConnectionBlock, self).__init__()
        self.outermost = outermost

        if input_nc is None:
            input_nc = outer_nc

        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,
                             stride=2, padding=1)
        downrelu = nn.LeakyReLU(0.2, True)
        downnorm = norm_layer(inner_nc, affine=True)
        uprelu = nn.ReLU(True)
        upnorm = norm_layer(outer_nc, affine=True)

        # Different position only has differences in `upconv`
            # for the outermost, the special is `tanh`
        if outermost:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downconv]
            up = [uprelu, upconv, nn.Tanh()]
            model = down + [submodule] + up
            # for the innermost, the special is `inner_nc` instead of `inner_nc*2`
        elif innermost:
            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv]  # for the innermost, no submodule, and delete the bn
            up = [uprelu, upconv, upnorm]
            model = down + up
            # else, the normal
        else:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv, downnorm]
            up = [uprelu, upconv, upnorm]

            if use_dropout:
                model = down + [submodule] + up + [nn.Dropout(0.5)]
            else:
                model = down + [submodule] + up

        self.model = nn.Sequential(*model)

    def forward(self, x):
        if self.outermost:  # if it is the outermost, directly pass the input in.
            return self.model(x)
        else:
            x_latter = self.model(x)
            _, _, h, w = x.size()
            if h != x_latter.size(2) or w != x_latter.size(3):
                x_latter = F.interpolate(x_latter, (h, w), mode='bilinear')
            return torch.cat([x_latter, x], 1)  # cat in the C channel

################################### This is for D ###################################
# Defines the PatchGAN discriminator with the specified arguments.
class NLayerDiscriminator(nn.Module):
    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, use_sigmoid=False):
        super(NLayerDiscriminator, self).__init__()
        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm2d
        else:
            use_bias = norm_layer == nn.InstanceNorm2d

        kw = 4
        padw = 1
        sequence = [
            nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw),
            nn.LeakyReLU(0.2, True)
        ]

        nf_mult = 1
        nf_mult_prev = 1
        for n in range(1, n_layers):
            nf_mult_prev = nf_mult
            nf_mult = min(2**n, 8)
            sequence += [
                nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                          kernel_size=kw, stride=2, padding=padw, bias=use_bias),
                norm_layer(ndf * nf_mult),
                nn.LeakyReLU(0.2, True)
            ]

        nf_mult_prev = nf_mult
        nf_mult = min(2**n_layers, 8)
        sequence += [
            nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                      kernel_size=kw, stride=1, padding=padw, bias=use_bias),
            norm_layer(ndf * nf_mult),
            nn.LeakyReLU(0.2, True)
        ]

        sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]

        if use_sigmoid:
            sequence += [nn.Sigmoid()]

        self.model = nn.Sequential(*sequence)

    def forward(self, input):
        return self.model(input)


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

def prepare_data(seq, num_steps, train_size=0.8, dev_size=0.1, num_preds=1):
    last_window_start = len(seq) - num_steps - num_preds + 1
    X = np.array([seq[i: i + num_steps] for i in range(last_window_start)])
    y = np.array([seq[i + num_steps:i + num_steps + num_preds, 0] for i in range(last_window_start)])
    train_end = int(len(X) * (train_size))
    dev_end = int(len(X) * (dev_size + train_size))
    train_X, train_y = X[:train_end], y[:train_end]
    dev_X, dev_y = X[train_end:dev_end], y[train_end:dev_end]
    test_X, test_y = X[train_end+dev_end:], y[train_end+dev_end:]

    return train_X, train_y, dev_X, dev_y, test_X, test_y, y

def plot_results(predicted_data, true_data):
    plt.plot(true_data, label='True Data')
    plt.plot(predicted_data, label='Prediction')
    plt.legend()
    plt.show()

def plot_results_multiple(predicted_data, true_data, prediction_len):
    fig = plt.figure(facecolor='white')
#     ax = fig.add_subplot(111)
    plt.plot(true_data, label='True Data')
    # Pad the list of predictions to shift it in the graph to it's correct start
    for i, data in enumerate(predicted_data):
        padding = [None for p in range(i * prediction_len)]
        plt.plot(padding + data, label='Prediction')
        plt.legend()
    plt.show()

def direction_accuracy(true_data, predicted_data):
    labels = (true_data[:-1] - true_data[1:]) >= 0
    predicted = (predicted_data[:-1] - predicted_data[1:]) >= 0
    acc = np.sum(labels == predicted) / labels.size
    return acc

# def prepare_data_many_to_one(seq, num_steps, test_ratio, num_preds=1):
#     last_window_start = len(seq) - num_steps - num_preds + 1
#     X = np.array([seq[i: i + num_steps] for i in range(last_window_start)])
#     y = np.array([seq[i + num_steps:i + num_steps + num_preds, 0] for i in range(last_window_start)])
#     print(X.shape, y.shape)
#     train_size = int(len(X) * (1.0 - test_ratio))
#     train_X, test_X = X[:train_size], X[train_size:]
#     train_y, test_y = y[:train_size], y[train_size:]

#     return train_X, train_y, test_X, test_y

# def prepare_data_many_to_many(seq, num_steps, test_ratio):
#     X = np.array([seq[i: i + num_steps] for i in range(len(seq) - num_steps - 1)])
#     y = np.array([seq[i+1:i + num_steps + 1, 0] for i in range(len(seq) - num_steps - 1)])
#     train_size = int(len(X) * (1.0 - test_ratio))
#     train_X, test_X = X[:train_size], X[train_size:]
#     train_y, test_y = y[:train_size, :, [0]].squeeze(), y[train_size:, :, [0]].squeeze()

#     return train_X, train_y, test_X, test_y


import argparse
import typing
from os import path
from pathlib import Path, PurePath
from zipfile import ZipFile, is_zipfile

import shtab
import yaml
from yaml import YAMLError

from opera.error import DataError, ParseError, OperaError
from opera.parser import tosca
from opera.parser.tosca.csar import CloudServiceArchive
from opera.storage import Storage

def add_parser(subparsers):
    parser = subparsers.add_parser(
        "init",
        help="Initialize the deployment environment for the TOSCA service template or CSAR"
    )
    parser.add_argument(
        "--instance-path", "-p",
        help="Storage folder location (instead of default .opera)"
    ).complete = shtab.DIR
    parser.add_argument(
        "--inputs", "-i", type=argparse.FileType("r"),
        help="YAML or JSON file with inputs",
    ).complete = shtab.FILE
    parser.add_argument(
        "--clean", "-c", action="store_true",
        help="Clean storage by removing previously initialized TOSCA service template or CSAR",
    )
    parser.add_argument(
        "--verbose", "-v", action="store_true",
        help="Turns on verbose mode",
    )
    parser.add_argument(
        "csar", type=argparse.FileType("r"),
        help="TOSCA YAML service template file or CSAR"
    ).complete = shtab.FILE
    parser.set_defaults(func=_parser_callback)

def _parser_callback(args):
    print("Warning: 'opera init' command is deprecated and will probably be removed within one of the next releases. "
          "Please use 'opera deploy' to initialize and deploy service templates or compressed CSARs.")
    if args.instance_path and not path.isdir(args.instance_path):
        raise argparse.ArgumentTypeError("Directory {} is not a valid path!".format(args.instance_path))

    storage = Storage.create(args.instance_path)
    try:
        inputs = yaml.safe_load(args.inputs) if args.inputs else {}
    except YAMLError as e:
        print("Invalid inputs: {}".format(e))
        return 1

    try:
        if is_zipfile(args.csar.name):
            init_compressed_csar(args.csar.name, inputs, storage, args.clean)
            print("CSAR was initialized")
        else:
            init_service_template(args.csar.name, inputs, storage, args.clean)
            print("Service template was initialized")
    except ParseError as e:
        print("{}: {}".format(e.loc, e))
        return 1
    except DataError as e:
        print(str(e))
        return 1
    except OperaError as e:
        print("Invalid CSAR: {}".format(e))
        return 1

    return 0

def init_compressed_csar(csar_name: str, inputs: typing.Optional[dict], storage: Storage, clean_storage: bool):
    if storage.exists("root_file"):
        if clean_storage:
            storage.remove_all()
        else:
            print("Looks like service template or CSAR has already been initialized. "
                  "Use the --clean/-c flag to clear the storage.")
            return

    if inputs is None:
        inputs = {}
    storage.write_json(inputs, "inputs")

    csars_dir = Path(storage.path) / "csars"
    csars_dir.mkdir(exist_ok=True)

    csar = CloudServiceArchive.create(PurePath(csar_name))
    csar.validate_csar()
    tosca_service_template = csar.get_entrypoint()

    # unzip csar and save the path to storage
    csar_dir = csars_dir / Path("csar")
    ZipFile(csar_name, "r").extractall(csar_dir)
    csar_tosca_service_template_path = csar_dir / tosca_service_template
    storage.write(str(csar_tosca_service_template_path), "root_file")

    # try to initiate service template from csar
    ast = tosca.load(Path(csar_dir), Path(tosca_service_template))
    template = ast.get_template(inputs)
    template.instantiate(storage)

def init_service_template(service_template: str, inputs: typing.Optional[dict],
                          storage: Storage, clean_storage: bool):
    if storage.exists("root_file"):
        if clean_storage:
            storage.remove_all()
        else:
            print("Looks like service template or CSAR has already been initialized. "
                  "Use --clean/-c flag to clear the storage.")
            return

    if inputs is None:
        inputs = {}
    storage.write_json(inputs, "inputs")
    storage.write(service_template, "root_file")

    ast = tosca.load(Path.cwd(), PurePath(service_template))
    template = ast.get_template(inputs)
    template.instantiate(storage)


import discord

def colour(*args):
    """Returns a discord Colour object.
    Pass one as an argument to define colour:
        `str` match common colour names.
        `discord.Guild` bot's guild colour.
        `None` light grey.
    """
    arg = args[0] if args else None
    if isinstance(arg, str):
        color = arg
        try:
            return getattr(discord.Colour, color)()
        except AttributeError:
            return discord.Colour.lighter_grey()
    if isinstance(arg, discord.Guild):
        return arg.me.colour
    else:
        return discord.Colour.lighter_grey()

def make_embed(
    msg_type='', title=None, icon=discord.Embed.Empty, content=None, msg_colour=None, guild=None,
    title_url=discord.Embed.Empty, thumbnail='', image='', fields=None, footer=None, footer_icon=None, inline=False,
    subtitle=None, subtitle_url=None
):
    """
    Helper for generating a formatted embed.

    Types available:
    error, warning, info, success, help.
    """

    embed_types = {
        'error': {
            'icon': 'https://i.imgur.com/juhq2uJ.png',
            'colour': 'red'
        },
        'warning': {
            'icon': 'https://i.imgur.com/4JuaNt9.png',
            'colour': 'gold'
        },
        'info': {
            'icon': 'https://i.imgur.com/wzryVaS.png',
            'colour': 'blue'
        },
        'success': {
            'icon': 'https://i.imgur.com/ZTKc3mr.png',
            'colour': 'green'
        },
        'help': {
            'icon': 'https://i.imgur.com/kTTIZzR.png',
            'colour': 'blue'
        }
    }

    if msg_type in embed_types.keys():
        msg_colour = embed_types[msg_type]['colour']
        icon = embed_types[msg_type]['icon']

    if guild and not msg_colour:
        msg_colour = colour(guild)
    else:
        if not isinstance(msg_colour, discord.Colour):
            msg_colour = colour(msg_colour)

    embed = discord.Embed(description=content, colour=msg_colour, title=subtitle, url=subtitle_url)

    if title:
        embed.set_author(name=title, icon_url=icon, url=title_url)

    if thumbnail:
        embed.set_thumbnail(url=thumbnail)

    if image:
        embed.set_image(url=image)

    fields = fields or {}
    for key, value in fields.items():
        ilf = inline
        if not isinstance(value, str):
            ilf = value[0]
            value = value[1]
        embed.add_field(name=key, value=value, inline=ilf)

    if footer:
        footer = {'text': footer}

        if footer_icon:
            footer['icon_url'] = footer_icon

        embed.set_footer(**footer)
    return embed

def bold(msg: str):
    """Format to bold markdown text"""
    return f'**{msg}**'

def italics(msg: str):
    """Format to italics markdown text"""
    return f'*{msg}*'

def bolditalics(msg: str):
    """Format to bold italics markdown text"""
    return f'***{msg}***'

def code(msg: str):
    """Format to markdown code block"""
    return f'```{msg}```'

def pycode(msg: str):
    """Format to code block with python code highlighting"""
    return f'```py\n{msg}```'

def ilcode(msg: str):
    """Format to inline markdown code"""
    return f'`{msg}`'

def convert_to_bool(argument):
    lowered = argument.lower()
    if lowered in ('yes', 'y', 'true', 't', '1', 'enable', 'on'):
        return True
    elif lowered in ('no', 'n', 'false', 'f', '0', 'disable', 'off'):
        return False
    else:
        return None

def bitround(x):
    return max(min(1 << int(x).bit_length() - 1, 1024), 16)


import numpy as np
import astropy.units as u

def radial_profile(diagnostic, steady_state_start, steady_state_end):

    # Develop radial profile; can use in ex. neutrals.py where needed
    linear_diagnostic_profile = linear_profile(diagnostic, steady_state_start, steady_state_end)
    pass

def linear_profile(diagnostic, steady_state_start, steady_state_end):

    if validate_dimensions(diagnostic.sizes):
        time = diagnostic.coords['time']
        return diagnostic.squeeze().where(
            np.logical_and(time >= steady_state_start.to(u.s).value, time <= steady_state_end.to(u.s).value), drop=True
        ).mean(dim='time', keep_attrs=True)

def validate_dimensions(da_sizes):
    # check if diagnostic is 1D; needed for radial profile
    if da_sizes['x'] == da_sizes['y'] == 1:
        raise ValueError("Diagnostic data has no spatial dimension. One-dimensional data needed for linear profiles.")
    elif da_sizes['x'] > 1 and da_sizes['y'] > 1:
        print("Linear profiles not defined for two-dimensional (areal) data.")
        return False
    else:
        return True

def get_spatial_dimensions(data_xarray):

    spatial_dimensions = ('x', 'y')
    return tuple(dimension for dimension in spatial_dimensions if data_xarray.sizes[dimension] >= 1)


#!/usr/bin/env python3

###
# PASA Confidentiality Notice:
# This source code and information contained herewith may be legally privileged and confidential
# Any dissemination, distribution or copy of this source code is strictly prohibited.
#
# Copyright (C) 2019, Panasonic Automotive Systems Company of America
#                     All Rights Reserved
#
#
# @file: list_ssm_parameters.py
#
# @author: Panasonic, developer
#
##

import re
import asyncio
import boto3
import argparse

def get_args():
    parser = argparse.ArgumentParser(description='List AWS SSM parameters with given prefix.')
    parser.add_argument(
        '--prefix', required=True, help='Parameter name prefix')
    parser.add_argument(
        '--noop', action="store_true", help='No operations mode')
    parser.add_argument(
        '--decrypt', action="store_true", help='Decrypt secure values')
    args = parser.parse_args()
    return args

def describe_parameters(client, prefix):
    parameters = []
    response = client.describe_parameters(
        Filters=[
            {
                'Key': 'Name',
                'Values': [
                    prefix,
                ]
            },
        ],
        MaxResults=50
    )
    parameters += response["Parameters"]
    while response.get("NextToken", False):
        response = client.describe_parameters(
            Filters=[
                {
                    'Key': 'Name',
                    'Values': [
                        prefix,
                    ]
                },
            ],
            MaxResults=50,
            NextToken=response["NextToken"]
        )
        parameters += response["Parameters"]
    return parameters

def get_parameter(client, p, noop=False, decrypt=False):
    if noop:
        print("Trying to get {}".format(p["Name"]))
    else:
        r = client.get_parameters(Names=[p["Name"]], WithDecryption=decrypt)
        value = r["Parameters"][0]["Value"]
        if "\n" in r["Parameters"][0]["Value"]:
            value = "{0} // Value is truncated".format(value.split("\n")[0])
        print("{0}={1}".format(r["Parameters"][0]["Name"], value))

async def get_parameters(client, parameters, noop=False, decrypt=False):
    loop = asyncio.get_event_loop()
    properties = []
    for p in parameters:
        loop.run_in_executor(None, get_parameter, client, p, noop, decrypt)

def main():
    args = get_args()
    client = boto3.client('ssm')
    parameters = describe_parameters(client, args.prefix)
    loop = asyncio.get_event_loop()
    loop.run_until_complete(get_parameters(client, parameters, noop=args.noop, decrypt=args.decrypt))

if __name__ == "__main__":
    main()


import tensorflow as tf
import numpy as np
from keras.utils import np_utils
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense

# 데이터 처리
xy = np.loadtxt('./data/data-04-zoo.csv',encoding='utf-8',delimiter=',')

x_data = xy[:, 0:-1]
y_data = xy[:,-1:]
y_data = np_utils.to_categorical(y_data)

x_train , x_test, y_train , y_test = train_test_split(x_data, y_data, test_size=0.2, random_state= 66)

print(x_train.shape,y_train.shape) #(80,16),(80,7)
print(x_test.shape,y_test.shape) #(21,7),(21,7)

# 모델 생성
model = Sequential()
model.add(Dense(7 , activation='softmax', input_shape = (16,)))

# 훈련
model.compile(optimizer='adadelta',
                loss='categorical_crossentropy',
                metrics=['accuracy'])

model.fit(x_train,y_train, validation_data=(x_test,y_test), epochs=500, batch_size=4)

# 검증
loss, acc = model.evaluate(x_test, y_test, batch_size=3)
print('acc :', acc)


import webapp2
import logging
import urllib
from webapp2_extras import jinja2

# this library is for decoding json responses
from webapp2_extras import json

# BaseHandler subclasses RequestHandler so that we can use jinja
class BaseHandler(webapp2.RequestHandler):

    @webapp2.cached_property
    def jinja2(self):
        # Returns a Jinja2 renderer cached in the app registry.
        return jinja2.get_jinja2(app=self.app)

        # This will call self.response.write using the specified template and context.
        # The first argument should be a string naming the template file to be used.
        # The second argument should be a pointer to an array of context variables
        #  that can be used for substitutions within the template
    def render_response(self, _template, **context):
        # Renders a template and writes the result to the response.
        rv = self.jinja2.render_template(_template, **context)
        self.response.write(rv)

class Viz3(BaseHandler):
	def get(self):
		# logic for map vizualizaiton
		try:
			mapyear = int(self.request.get('mapyear'))
		except ValueError:
			mapyear = 0

		if not mapyear:
			mapyear = 0

		logging.info(mapyear)
		state_num = []
		try:
			fp = open("data/state_numbers.txt")
			lines = fp.readlines()
			line = lines[mapyear]
			line = line.split(":")
			line = line[:len(line)-1]
			state_nums = line
		except IOError:
			logging.info("failed to load file")

		if mapyear == 0:
			mapyear = '2014'
		elif mapyear == 1:
			mapyear = '2013'
		elif mapyear == 2:
			mapyear = '2012'
		elif mapyear == 3:
			mapyear = '2011'
		else:
			mapyear = '2010'

		variables = {'state_nums': state_nums,
        			 'map_year': mapyear}

		self.render_response('viz3.html', **variables)

class MainPage(BaseHandler):

	def get(self):
		#logging.info(results)
		# add it to the context being passed to jinja
		try:
			year = int(self.request.get('year'))
		except ValueError:
			year = 0

		if not year:
			year = 0

		data = []
		countries = []
		should_read = False
		try:
			fp = open("data/studies_cleaned.txt")
			for line in fp:
				if (line[0] == str(year)):
					should_read = True
					continue
				elif (len(line) == 2):
					should_read = False
				if (should_read):
					data += [line.split(",")]
			fp.close()
		except IOError:
			logging.info("failed to load file")

		for i in xrange(0,len(data)):
			countries += [data[i][0]]
			data[i] = data[i][1:len(data[i])]
			for j in xrange(0,len(data[i])):
				data[i][j] = float(data[i][j].strip("\n"))

		majors = ['Business/Mgmt.', 'Education', 'Engineering', 'Fine/Applied Arts', 'Health Professions', 'Humanities', 'Intensive English', 'Math/Computer Science', 'Physical/Life Sciences', 'Social Sciences', 'Other', 'Undeclared']
		variables = {'countries': countries,
					 'majors': majors,
				     'percentages': data,
					 'current_year': year}

		self.render_response('viz2.html', **variables)

application = webapp2.WSGIApplication([
    ('/visualization2/', MainPage),
	('/visualization3/', Viz3),
], debug=True)


import pandas as pd
import numpy as np

from tools import str2num

converters = {'Price':str2num,
              'Rank':str2num,
              'Rating':str2num,
              'Sales':str2num,
              'Revenue':str2num,
              'Reviews':str2num
             }
try:
    data = pd.read_csv('data.csv', converters=converters, header=7, index_col=0)
except BaseException as e:
    print(e)


# -*- coding: utf-8 -*-
from __future__ import absolute_import
import os
import sys
import logging
from datetime import datetime, timedelta

# add up one level dir into sys path
sys.path.append(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
os.environ['DJANGO_SETTINGS_MODULE'] = 'luckyplatform.settings'

from luckycommon.db.account import get_account
from luckycommon.account.model.account import Account
from luckycommon.stats import MG as mg
from luckycommon.db.missed_vips import batch_insert_missed_vips
from luckycommon.model.missed_vips import MissedVips
from luckycommon.model import orm
from luckycommon.db import account as account_db
from luckycommon.utils.tz import utc_to_local_str

_LOGGER = logging.getLogger(__name__)

missed_users = {
    1: set(),
    2: set(),
    3: set()
}
for k in (1, 2, 3):
    items = orm.session.query(MissedVips.uid).filter(
        MissedVips.type == k).all()
    for item in items:
        missed_users[k].add(item[0])

missed_users[1] -= missed_users[2]
missed_users[2] -= missed_users[3]

all_missed = missed_users[1] | missed_users[2] | missed_users[3]

def dump_vip_users(lost_days=3, missed_times=0):
    now = datetime.now()
    start_date = now - timedelta(days=lost_days)
    start_date = start_date.replace(hour=16, minute=0, second=0, microsecond=0)
    end_date = start_date + timedelta(days=1)

    filters = {'updated_at': {'$lte': end_date, '$gte': start_date},
               'recharge.total': {'$gt': 100}}

    if missed_times == 0:
        filters['_id'] = {'$nin': list(all_missed)}
    else:
        filters['_id'] = {'$in': list(missed_users[missed_times])}

    user_stats = mg.user_stats.find(filters).sort(
        'recharge.total', -1).limit(100)
    return user_stats

def dump_new_users():
    now = datetime.now()
    updated_at_upper = now - timedelta(days=3)
    created_at_lower = now - timedelta(days=10)
    created_at_lower = created_at_lower.replace(
        hour=16, minute=0, second=0, microsecond=0)
    created_at_upper = created_at_lower + timedelta(days=1)
    filters = {
        'created_at': {'$gte': created_at_lower, '$lt': created_at_upper},
        'recharge.total': {'$gt': 0, '$lte': 100},
        'updated_at': {'$lt': updated_at_upper}}
    user_stats = mg.user_stats.find(filters).sort(
        'recharge.total', -1)
    insert_list = []
    forbidden_users = orm.session.query(Account.id).filter(
        Account.status != 0).all()
    forbidden_users = [x[0] for x in forbidden_users]
    forbidden_users = set(forbidden_users)
    for user_stat in user_stats:
        if user_stat['_id'] not in forbidden_users:
            insert_list.append(user_stat)
    return insert_list

def get_insert_list(user_stats, lost_days, type, user_type=0):
    insert_list = []
    for user_stat in user_stats:
        user_id = int(user_stat['_id'])
        register_chn = user_stat.get('chn') or u'未知'
        created_at = user_stat.get('created_at')
        if not created_at:
            created_at = get_account(user_id).created_at
        created_at = utc_to_local_str(created_at)
        updated_at = utc_to_local_str(user_stat['updated_at'])
        recharge_amount = int(user_stat['recharge']['total'])
        pay_count = user_stat.get('pay', {}).get('count', 0)
        if 'win' in user_stat:
            win_count = user_stat['win']['count']
            win_amount = user_stat['win']['total']
        else:
            win_count = 0
            win_amount = 0
        active_count = mg.daily_stats.aggregate([
            {"$match": {"user_id": user_id}},
            {"$group": {"_id": None, "count": {"$sum": 1}}}
        ])
        active_days = active_count.next().get(
            'count', 0) if active_count.alive else 0
        account = account_db.get_account(user_id)
        user_info = {
            'uid': user_id,
            'nick_name': account.nick_name,
            'phone': account.phone[2:],
            'chn': register_chn,
            'type': type,
            'user_type': user_type,
            'created_time': created_at,
            'active_days': active_days,
            'updated_time': updated_at,
            'lost_days': lost_days,
            'recharge_amount': recharge_amount,
            'pay_count': pay_count,
            'win_count': win_count,
            'win_amount': win_amount
        }
        insert_list.append(user_info)
    return insert_list

try:
    for k in (3, 7, 14):
        user_stats = dump_vip_users(k, 0)
        insert_list = get_insert_list(user_stats, k, 1)
        batch_insert_missed_vips(insert_list)
    user_stats = dump_vip_users(7, 1)
    insert_list = get_insert_list(user_stats, 7, 2)
    batch_insert_missed_vips(insert_list)
    user_stats = dump_vip_users(30, 2)
    insert_list = get_insert_list(user_stats, 30, 3)
    batch_insert_missed_vips(insert_list)

    user_stats = dump_new_users()
    insert_list = get_insert_list(user_stats, 3, 1, 1)
    batch_insert_missed_vips(insert_list)
except Exception as e:
    _LOGGER.exception(e)


import sqlite3
import mysql.connector
import time

class SqlDB(object):
    def __init__(self, dbtype='sqlite3', **kwargs):
        """
        MySQL
            Password is given by the user at the time of installing the MySQL
            database. If you are using root then you won’t need the password.

        ref: https://pynative.com/python-mysql-insert-data-into-database-table/
        """

        self.dbtype = dbtype

        if dbtype == 'sqlite3':
            print("Connecting sqlite3 database...")
            assert 'path' in kwargs.keys()

            self.conn = sqlite3.connect(kwargs['path'])

            # Create table
            self.conn.execute('''CREATE TABLE IF NOT EXISTS PATTERNS
                (ID INTEGER PRIMARY KEY     AUTOINCREMENT,
                PATTERN        TEXT     NOT NULL,
                ENT            TEXT     NOT NULL,
                TIMESTAMP      INT      NOT NULL,
                FREQ           INT      NOT NULL);''')
            print("PATTERNS table created successfully!")
        elif dbtype == 'mysql':
            print("Connecting MySQL database...")
            assert 'host' in kwargs.keys()
            assert 'user' in kwargs.keys()
            assert 'database' in kwargs.keys()

            self.conn = mysql.connector.connect(host=kwargs['host'],
                user=kwargs['user'], database=kwargs['database'])

            print("MySQL connected:", self.conn.is_connected())

            self.cursor = self.conn.cursor()
            self.cursor.execute('''CREATE TABLE IF NOT EXISTS PATTERNS
                (ID INT PRIMARY KEY     AUTO_INCREMENT,
                PATTERN        TEXT     NOT NULL,
                ENT            TEXT     NOT NULL,
                TIMESTAMP      INT      NOT NULL,
                FREQ           INT      NOT NULL);''')
            print("PATTERNS table created successfully!")
        else:
            raise ValueError(dbtype + ' database not supported')

    @classmethod
    def fromconfig(cls, config):
        if config['DBTYPE'] == 'sqlite3':
            return cls(dbtype=config['DBTYPE'], path=config['PATH'])
        elif config['DBTYPE'] == 'mysql':
            return cls(dbtype=config['DBTYPE'],
                host=config['HOST'],
                user=config['USER'],
                database=config['DATABASE'])

        raise ValueError(config['DBTYPE'] + ' database not supported')

    def execute(self, query, params=None):
        if params is None:
            if self.dbtype == 'sqlite3':
                return self.conn.execute(query)
            elif self.dbtype == 'mysql':
                self.cursor.execute(query)
                return self._mysql_fetch()
            else:
                raise ValueError(self.dbtype + ' database not supported')

        if self.dbtype == 'sqlite3':
            return self.conn.execute(query, params)
        elif self.dbtype == 'mysql':
            # MySQL parameterized arguments uses %s instead of ?
            query = query.replace('?', '%s')
            self.cursor.execute(query, params)
            return self._mysql_fetch()

        raise ValueError(self.dbtype + ' database not supported')

    def executemany(self, query, params, max_retries=4):
        if self.dbtype == 'sqlite3':
            i = 0
            while i < max_retries:
                try:
                    return self.conn.executemany(query, params)
                except sqlite3.OperationalError:
                    time.sleep(0.5)
                    i += 1
            return

        elif self.dbtype == 'mysql':
            # MySQL parameterized arguments uses %s instead of ?
            query = query.replace('?', '%s')
            i = 0
            while i < max_retries:
                try:
                    self.cursor.executemany(query, params)
                    return self._mysql_fetch()
                except mysql.connector.errors.DatabaseError:
                    time.sleep(0.5)
                    i += 1

        raise ValueError(self.dbtype + ' database not supported')

    def commit(self):
        self.conn.commit()

    def close(self):
        self.conn.close()

        if self.dbtype == 'mysql':
            self.cursor.close()

    def delete(self):
        if self.dbtype == 'sqlite3':
            self.conn.execute("DROP TABLE IF EXISTS PATTERNS")
        elif self.dbtype == 'mysql':
            self.cursor.execute("DROP TABLE IF EXISTS PATTERNS")
        else:
            raise ValueError(self.dbtype + ' database not supported')

        print("SQL table deleted successfully!")

    def _mysql_fetch(self):
        try:
            return self.cursor.fetchall()
        except mysql.connector.errors.InterfaceError:
            return None


# Brain Tumor Classification
# Script for None Preprocessing
# Author: Qixun Qu
# Create on: 2017/11/28
# Modify on: 2017/11/28

#     ,,,         ,,,
#   ;"   ';     ;'   ",
#   ;  @.ss$$$$$$s.@  ;
#   `s$$$$$$$$$$$$$$$'
#   $$$$$$$$$$$$$$$$$$
#  $$$$P""Y$$$Y""W$$$$$
#  $$$$  p"$$$"q  $$$$$
#  $$$$  .$$$$$.  $$$$'
#   $$$DaU$$O$$DaU$$$'
#    '$$$$'.^.'$$$$'
#       '&$$$$$&'

'''

Class BTCNonePreprocess

'''

import os
import numpy as np
import nibabel as nib
from btc_settings import *
from multiprocessing import Pool, cpu_count

# Helper function to do multiprocessing of
# BTCNonePreprocess._merge_to_one_volume
def unwrap_merge_to_one_volume(arg, **kwarg):
    return BTCNonePreprocess._merge_to_one_volume(*arg, **kwarg)

class BTCNonePreprocess():

    def __init__(self, input_dir, output_dir):

        if not os.path.isdir(input_dir):
            raise IOError("Cannot find input directory.")

        if not os.path.isdir(output_dir):
            os.makedirs(output_dir)

        self.volume_no = os.listdir(input_dir)
        self.full_folder = os.path.join(output_dir, FULL_FOLDER)
        self.mask_folder = os.path.join(output_dir, MASK_FOLDER)

        self._create_folders()
        self._merge_to_one_volume_multi(input_dir, output_dir)

        return

    def _merge_to_one_volume_multi(self, input_dir, output_dir):
        '''_MERGE_TO_ONE_VOLUME_MULTI

            Main function of merging four types volumes and saving outputs
            to map tasks on different cpus to accelerate processing speed.
            The number of subprocesses equals to the number of cpus.

            Inputs:
            -------
            - input_dir: path of the directory which keeps mask volumes
            - output_dir: path of directory which keeps the outputs

        '''

        print("\nMerge flair, t1, t1Gd and t2 into One Volume\n")
        volume_no_len = len(self.volume_no)
        paras = zip([self] * volume_no_len,
                    [input_dir] * volume_no_len,
                    [output_dir] * volume_no_len,
                    self.volume_no)
        pool = Pool(processes=cpu_count())
        pool.map(unwrap_merge_to_one_volume, paras)

        return

    def _create_folders(self):
        '''_CREATE_FOLDERS

            Create folders for temporary files and outputs.
            All folders are as below.

            Folders for outputs:
            ----- output_dir
              |----- full
              |----- mask

        '''

        if not os.path.isdir(self.mask_folder):
            os.makedirs(self.mask_folder)

        if not os.path.isdir(self.full_folder):
            os.makedirs(self.full_folder)

        return

    def _merge_to_one_volume(self, input_dir, output_dir, vno):
        '''_MERGE_TO_ONE_VOLUME

            Merge normalized flair, t1, t1Gd and t2 volumes of one patient
            to one volume. Remove surrounding backgrounds, and save output
            into output folder as the result of preprocessing.

            Inputs:
            -------
            - input_dir: path of the directory which keeps mask volumes
            - output_dir: path of directory which keeps the outputs
            - vno: serial number of volumes, which is also the folder name
                   of one patient's volumes

        '''

        print("NO." + vno + ": Save brain volume and mask volume")
        full_volume = np.zeros(FULL_SHAPE)
        volume_folder = os.path.join(input_dir, vno)
        for file in os.listdir(volume_folder):
            for i in range(len(VOLUME_TYPES)):
                if VOLUME_TYPES[i] in file:
                    volume_path = os.path.join(volume_folder, file)
                    volume = nib.load(volume_path).get_data()
                    volume = np.rot90(volume, 3, axes=(0, 1))
                    # full_volume[..., 0] <== flair volume
                    # full_volume[..., 1] <== t1 volume
                    # full_volume[..., 2] <== t1Gd volume
                    # full_volume[..., 3] <== t2 volume
                    full_volume[..., i] = volume
            if MASK_NAME in file:
                # Load relevant mask
                mask_path = os.path.join(volume_folder, file)
                mask_volume = nib.load(mask_path).get_data()
                mask_volume = np.rot90(mask_volume, 3, axes=(0, 1))

        # Remove surrounding backgrounds from ensemble volume and mask volume
        full_volume, mask_volume = self._keep_minimum_volume(full_volume, mask_volume)

        # Save volume into output folders
        full_volume_path = os.path.join(self.full_folder, vno + TARGET_EXTENSION)
        mask_volume_path = os.path.join(self.mask_folder, vno + TARGET_EXTENSION)

        np.save(full_volume_path, full_volume)
        np.save(mask_volume_path, mask_volume)

        return

    def _keep_minimum_volume(self, full, mask):
        '''_KEEP_MINIMUM_VOLUME

            Remove surrounding backgrounds from ensemble volume
            and mask volume to keep the minimum volume.
            Based on input volumes, compute the range of indices
            of three axes, extract sub-volume and return it back.

            Inputs:
            -------
            - full: ensemble volume
            - mask: relevant mask volume

            Outputs:
            --------
            - new_full: new ensemble volume after being processed
            - new_mask: new mask volume after being processed

        '''

        # Function to extract sub-array from given array
        # according to ranges of indices of three axes
        def sub_array(arr, index_begin, index_end):
            return arr[index_begin[0]:index_end[0],
                       index_begin[1]:index_end[1],
                       index_begin[2]:index_end[2]]

        # Compute background value of volume
        full_sum = np.sum(full, axis=3)
        min_full_sum = np.min(full_sum)

        # Compute range of indices of each axes
        non_bg_index = np.where(full_sum > min_full_sum)
        dims_begin = [np.min(nzi) for nzi in non_bg_index]
        dims_end = [np.max(nzi) + 1 for nzi in non_bg_index]

        # Add a bit more space around the minimum brain volume
        for i in range(len(dims_begin)):
            dims_begin[i] = dims_begin[i] - EDGE_SPACE
            # if the beginning index is lower than 0
            if dims_begin[i] < 0:
                dims_begin[i] = 0
            dims_end[i] = dims_end[i] + EDGE_SPACE
            # if the ending index is larger than the maximum index
            if dims_end[i] > BRAIN_SHAPE[i] - 1:
                dims_end[i] = BRAIN_SHAPE[i] - 1

        # Obtain sub-volumes from input volumes
        new_full = sub_array(full, dims_begin, dims_end)
        new_full = new_full.astype(np.float32)
        new_mask = sub_array(mask, dims_begin, dims_end)

        return new_full, new_mask

if __name__ == "__main__":

    parent_dir = os.path.dirname(os.getcwd())
    input_dir = os.path.join(parent_dir, DATA_FOLDER, ORIGINAL_FOLDER)
    output_dir = os.path.join(parent_dir, DATA_FOLDER, NONEPREPROCESSED_FOLDER)

    BTCNonePreprocess(input_dir, output_dir)