"""
Copyright (c) 2017, Battelle Memorial Institute
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
   this list of conditions and the following disclaimer in the documentation
   and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies,
either expressed or implied, of the FreeBSD Project.

This material was prepared as an account of work sponsored by an agency of the
United States Government. Neither the United States Government nor the United
States Department of Energy, nor Battelle, nor any of their employees, nor any
jurisdiction or organization that has cooperated in the development of these
materials, makes any warranty, express or implied, or assumes any legal
liability or responsibility for the accuracy, completeness, or usefulness or
any information, apparatus, product, software, or process disclosed, or
represents that its use would not infringe privately owned rights.

Reference herein to any specific commercial product, process, or service by
trade name, trademark, manufacturer, or otherwise does not necessarily
constitute or imply its endorsement, recommendation, or favoring by the
United States Government or any agency thereof, or Battelle Memorial Institute.
The views and opinions of authors expressed herein do not necessarily state or
reflect those of the United States Government or any agency thereof.

PACIFIC NORTHWEST NATIONAL LABORATORY
operated by
BATTELLE
for the
UNITED STATES DEPARTMENT OF ENERGY
under Contract DE-AC05-76RL01830
"""
import datetime
from datetime import timedelta as td
import logging
import sys
import os
from volttron.platform.agent.math_utils import mean
from volttron.platform.agent.driven import Results, AbstractDrivenAgent
from volttron.platform.agent.utils import (setup_logging)

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '/diagnostics/'))
from .diagnostics.temperature_sensor_dx import TempSensorDx
from .diagnostics.economizer_dx import EconCorrectlyOn, EconCorrectlyOff
from .diagnostics.ventilation_dx import ExcessOA, InsufficientOA

__version__ = "1.0.8"

ECON1 = "Temperature Sensor Dx"
ECON2 = "Not Economizing When Unit Should Dx"
ECON3 = "Economizing When Unit Should Not Dx"
ECON4 = "Excess Outdoor-air Intake Dx"
ECON5 = "Insufficient Outdoor-air Intake Dx"
DX = "/diagnostic message"
EI = "/energy impact"
dx_list = [ECON1, ECON2, ECON3, ECON4, ECON5]

FAN_OFF = -99.3
OAF = -89.2
OAT_LIMIT = -79.2
RAT_LIMIT = -69.2
MAT_LIMIT = -59.2
TEMP_SENSOR = -49.2

setup_logging()
_log = logging.getLogger(__name__)
logging.basicConfig(level=logging.debug, format='%(asctime)s   %(levelname)-8s %(message)s',
                    datefmt='%m-%d-%y %H:%M:%S')

def create_table_key(table_name, timestamp):
    return "&".join([table_name, timestamp.isoformat()])

def data_builder(value_tuple, point_name):
    value_list = []
    for item in value_tuple:
        value_list.append(item[1])
    return value_list

class Application(AbstractDrivenAgent):
    """
    Application to detect and correct operational problems for AHUs/RTUs.

    This application uses metered data from zones server by an AHU/RTU
    to detect operational problems and where applicable correct these problems
    by modifying set points.  When auto-correction cannot be applied then
    a message detailing the diagnostic results will be made available to
    the building operator.
    """

    def __init__(self, economizer_type="DDB", econ_hl_temp=65.0,
                 device_type="AHU", cooling_enabled_threshold=5.0,
                 temp_band=1.0, data_window=30, no_required_data=15,
                 open_damper_time=5, low_supply_fan_threshold=15.0,
                 mat_low_threshold=50.0, mat_high_threshold=90.0,
                 oat_low_threshold=30.0, oat_high_threshold=110.0,
                 rat_low_threshold=50.0, rat_high_threshold=90.0,
                 temp_difference_threshold=4.0, temp_damper_threshold=90.0,
                 open_damper_threshold=80.0, oaf_temperature_threshold=5.0,
                 minimum_damper_setpoint=20.0, desired_oaf=10.0,
                 rated_cfm=6000.0, eer=10.0, constant_volume=False,
                 sensitivity="default", **kwargs):
        def get_or_none(name):
            value = kwargs["point_mapping"].get(name, None)
            if value:
                value = value.lower()
            return value

        if sensitivity is not None and sensitivity == "custom":
            oaf_temperature_threshold = max(5., min(oaf_temperature_threshold, 15.))
            cooling_enabled_threshold = max(5., min(cooling_enabled_threshold, 50.))
            temp_difference_threshold = max(2., min(temp_difference_threshold, 6.))
            mat_low_threshold = max(40., min(mat_low_threshold, 60.))
            mat_high_threshold = max(80., min(mat_high_threshold, 90.))
            rat_low_threshold = max(40., min(rat_low_threshold, 60.))
            rat_high_threshold = max(80., min(rat_high_threshold, 90.))
            oat_low_threshold = max(20., min(oat_low_threshold, 40.))
            oat_high_threshold = max(90., min(oat_high_threshold, 125.))
            open_damper_threshold = max(60., min(open_damper_threshold, 90.))
            minimum_damper_setpoint = max(0., min(minimum_damper_setpoint, 50.))
            desired_oaf = max(5., min(desired_oaf, 30.))
        else:
            oaf_temperature_threshold = 5.
            cooling_enabled_threshold = 5.
            temp_difference_threshold = 4.
            mat_low_threshold = 50.
            mat_high_threshold = 90.
            rat_low_threshold = 50.
            rat_high_threshold = 90.
            oat_low_threshold = 30.
            oat_high_threshold = 110.
            open_damper_threshold = 80.
            minimum_damper_setpoint = 20.
            desired_oaf = 10.

        econ_hl_temp = max(50., min(econ_hl_temp, 75.))
        temp_band = max(0.5, min(temp_band, 10.))

        self.device_type = device_type.lower()
        if self.device_type not in ("ahu", "rtu"):
            _log.error('device_type must be specified as "AHU" or "RTU" in configuration file.')
            sys.exit()

        if economizer_type.lower() not in ("ddb", "hl"):
            _log.error('economizer_type must be specified as "DDB" or "HL" in configuration file.')
            sys.exit()

        Application.analysis = analysis = kwargs["analysis_name"]

        # data point name mapping
        self.fan_status_name = get_or_none("supply_fan_status")
        self.fan_sp_name = get_or_none("supply_fan_speed")

        if self.fan_sp_name is None and self.fan_status_name is None:
            _log.error("SupplyFanStatus or SupplyFanSpeed are required to verify AHU status.")
            sys.exit()

        self.oat_name = get_or_none("outdoor_air_temperature")
        self.rat_name = get_or_none("return_air_temperature")
        self.mat_name = get_or_none("mixed_air_temperature")
        self.oad_sig_name = get_or_none("outdoor_damper_signal")
        self.cool_call_name = get_or_none("cool_call")

        # Precondition flags
        self.oaf_condition = None
        self.unit_status = None
        self.sensor_limit = None
        self.temp_sensor_problem = None

        # Time based configurations
        self.data_window = data_window = td(minutes=data_window)
        open_damper_time = td(minutes=open_damper_time)
        no_required_data = no_required_data

        # diagnostic threshold parameters
        self.economizer_type = economizer_type.lower()
        self.econ_hl_temp = float(econ_hl_temp) if self.economizer_type == "hl" else None
        self.constant_volume = constant_volume

        self.cooling_enabled_threshold = cooling_enabled_threshold
        self.low_supply_fan_threshold = low_supply_fan_threshold
        self.oaf_temperature_threshold = oaf_temperature_threshold
        self.oat_thresholds = [oat_low_threshold, oat_high_threshold]
        self.rat_thresholds = [rat_low_threshold, rat_high_threshold]
        self.mat_thresholds = [mat_low_threshold, mat_high_threshold]
        self.temp_band = temp_band
        cfm = float(rated_cfm)
        eer = float(eer)

        oat_mat_check = {
            'low': max(temp_difference_threshold * 1.5, 6.0),
            'normal': max(temp_difference_threshold*1.25, 5.0),
            'high': max(temp_difference_threshold, 4.0)
        }
        temp_difference_threshold = {
            'low': temp_difference_threshold + 2.0,
            'normal': temp_difference_threshold,
            'high': max(1.0, temp_difference_threshold - 2.0)
        }
        oaf_economizing_threshold = {
            'low': open_damper_threshold - 30.0,
            'normal': open_damper_threshold - 20.0,
            'high': open_damper_threshold - 10.0
        }
        open_damper_threshold = {
            'low': open_damper_threshold - 10.0,
            'normal': open_damper_threshold,
            'high': open_damper_threshold + 10.0
        }
        excess_damper_threshold = {
            'low': minimum_damper_setpoint*2.0,
            'normal': minimum_damper_setpoint,
            'high':  minimum_damper_setpoint*0.5
        }
        excess_oaf_threshold = {
            'low': minimum_damper_setpoint*2.0 + 10.0,
            'normal': minimum_damper_setpoint + 10.0,
            'high': minimum_damper_setpoint*0.5 + 10.0
        }
        ventilation_oaf_threshold = {
            'low': desired_oaf*0.75,
            'normal': desired_oaf*0.5,
            'high': desired_oaf*0.25
        }
        self.sensitivity = ['low', 'normal', 'high']
        self.econ1 = TempSensorDx(data_window, no_required_data,
                                  temp_difference_threshold, open_damper_time,
                                  oat_mat_check, temp_damper_threshold,
                                  analysis)

        self.econ2 = EconCorrectlyOn(oaf_economizing_threshold,
                                     open_damper_threshold,
                                     minimum_damper_setpoint,
                                     data_window, no_required_data,
                                     cfm, eer, analysis)

        self.econ3 = EconCorrectlyOff(data_window, no_required_data,
                                      minimum_damper_setpoint,
                                      excess_damper_threshold,
                                      desired_oaf, cfm, eer, analysis)

        self.econ4 = ExcessOA(data_window, no_required_data,
                              excess_oaf_threshold,
                              minimum_damper_setpoint,
                              excess_damper_threshold,
                              desired_oaf, cfm, eer, analysis)

        self.econ5 = InsufficientOA(data_window, no_required_data,
                                    ventilation_oaf_threshold, desired_oaf,
                                    analysis)

    def run(self, cur_time, points):
        """
        Main run method that is called by the DrivenBaseClass.

        run receives a dictionary of data 'points' and an associated timestamp
        for the data cur_time'.  run then passes the appropriate data to
        each diagnostic when calling
        the diagnostic message.
        :param cur_time:
        :param points:
        :return:
        """
        device_dict = {}
        dx_result = Results()

        for point, value in points.items():
            point_device = [name.lower() for name in point.split("&")]
            if point_device[0] not in device_dict:
                device_dict[point_device[0]] = [(point_device[1], value)]
            else:
                device_dict[point_device[0]].append((point_device[1], value))

        damper_data = []
        oat_data = []
        mat_data = []
        rat_data = []
        cooling_data = []
        fan_sp_data = []
        fan_status_data = []
        missing_data = []

        for key, value in device_dict.items():
            data_name = key
            if value is None:
                continue
            if data_name == self.fan_status_name:
                fan_status_data = data_builder(value, data_name)
            elif data_name == self.oad_sig_name:
                damper_data = data_builder(value, data_name)
            elif data_name == self.oat_name:
                oat_data = data_builder(value, data_name)
            elif data_name == self.mat_name:
                mat_data = data_builder(value, data_name)
            elif data_name == self.rat_name:
                rat_data = data_builder(value, data_name)
            elif data_name == self.cool_call_name:
                cooling_data = data_builder(value, data_name)
            elif data_name == self.fan_sp_name:
                fan_sp_data = data_builder(value, data_name)

        if not oat_data:
            missing_data.append(self.oat_name)
        if not rat_data:
            missing_data.append(self.rat_name)
        if not mat_data:
            missing_data.append(self.mat_name)
        if not damper_data:
            missing_data.append(self.oad_sig_name)
        if not cooling_data:
            missing_data.append(self.cool_call_name)
        if not fan_status_data and not fan_sp_data:
            missing_data.append(self.fan_status_name)
        if missing_data:
            dx_result.log("Missing data from publish: {}".format(missing_data))
            return dx_result

        current_fan_status, fan_sp = self.check_fan_status(fan_status_data, fan_sp_data, cur_time)
        dx_result = self.check_elapsed_time(dx_result, cur_time, self.unit_status, FAN_OFF)

        if not current_fan_status:
            dx_result.log("Supply fan is off: {}".format(cur_time))
            return dx_result
        else:
            dx_result.log("Supply fan is on: {}".format(cur_time))

        if fan_sp is None and self.constant_volume:
            fan_sp = 100.0

        oat = mean(oat_data)
        rat = mean(rat_data)
        mat = mean(mat_data)
        oad = mean(damper_data)

        self.check_temperature_condition(oat, rat, cur_time)
        dx_result = self.check_elapsed_time(dx_result, cur_time, self.oaf_condition, OAF)

        if self.oaf_condition:
            dx_result.log("OAT and RAT readings are too close.")
            return dx_result

        limit_condition = self.sensor_limit_check(oat, rat, mat, cur_time)
        dx_result = self.check_elapsed_time(dx_result, cur_time, self.sensor_limit, limit_condition[1])
        if limit_condition[0]:
            dx_result.log("Temperature sensor is outside of bounds: {} -- {}".format(limit_condition, self.sensor_limit))
            return dx_result

        dx_result, self.temp_sensor_problem = self.econ1.econ_alg1(dx_result, oat, rat, mat, oad, cur_time)
        econ_condition, cool_call = self.determine_cooling_condition(cooling_data, oat, rat)
        _log.debug("Cool call: {} - Economizer status: {}".format(cool_call, econ_condition))

        if self.temp_sensor_problem is not None and not self.temp_sensor_problem:
            dx_result = self.econ2.econ_alg2(dx_result, cool_call, oat, rat, mat,
                                             oad, econ_condition, cur_time, fan_sp)

            dx_result = self.econ3.econ_alg3(dx_result, oat, rat, mat, oad,
                                             econ_condition, cur_time, fan_sp)

            dx_result = self.econ4.econ_alg4(dx_result, oat, rat, mat, oad,
                                             econ_condition, cur_time, fan_sp)

            dx_result = self.econ5.econ_alg5(dx_result, oat, rat, mat, cur_time)
        elif self.temp_sensor_problem:
            self.pre_conditions(dx_list[1:], TEMP_SENSOR, cur_time, dx_result)
            self.econ2.clear_data()
            self.econ2.clear_data()
            self.econ3.clear_data()
            self.econ4.clear_data()
            self.econ5.clear_data()
        return dx_result

    def clear_all(self):
        """
        Reinitialize all data arrays for diagnostics.
        :return:
        """
        self.econ1.clear_data()
        self.econ2.clear_data()
        self.econ2.clear_data()
        self.econ3.clear_data()
        self.econ4.clear_data()
        self.econ5.clear_data()
        self.temp_sensor_problem = None
        self.unit_status = None
        self.oaf_condition = None
        self.sensor_limit = None
        return

    def determine_cooling_condition(self, cooling_data, oat, rat):
        """
        Determine if the unit is in a cooling mode and if conditions
        are favorable for economizing.
        :param cooling_data:
        :param oat:
        :param rat:
        :return:
        """
        if self.device_type == "ahu":
            clg_vlv_pos = mean(cooling_data)
            cool_call = True if clg_vlv_pos > self.cooling_enabled_threshold else False
        elif self.device_type == "rtu":
            cool_call = int(max(cooling_data))

        if self.economizer_type == "ddb":
            econ_condition = (rat - oat) > self.temp_band
        else:
            econ_condition = (self.econ_hl_temp - oat) > self.temp_band

        return econ_condition, cool_call

    def check_fan_status(self, fan_status_data, fan_sp_data, cur_time):
        """
        :param fan_status_data:
        :param fan_sp_data:
        :param cur_time:
        :return:
        """
        supply_fan_status = int(max(fan_status_data)) if fan_status_data else None

        fan_speed = mean(fan_sp_data) if fan_sp_data else None
        if supply_fan_status is None:
            supply_fan_status = 1 if fan_speed > self.low_supply_fan_threshold else 0

        if not supply_fan_status:
            if self.unit_status is None:
                self.unit_status = cur_time
        else:
            self.unit_status = None
        return supply_fan_status, fan_speed

    def check_temperature_condition(self, oat, rat, cur_time):
        """
        Ensure the OAT and RAT have minimum difference to allow
        for a conclusive diagnostic.
        :param oat:
        :param rat:
        :param cur_time:
        :return:
        """
        if abs(oat - rat) < self.oaf_temperature_threshold:
            if self.oaf_condition is None:
                self.oaf_condition = cur_time
        else:
            self.oaf_condition = None
        return

    def check_elapsed_time(self, dx_result, cur_time, condition, message):
        """
        Check for persistence of failure to meet pre-conditions for diagnostics.
        :param dx_result:
        :param cur_time:
        :param condition:
        :param message:
        :return:
        """
        elapsed_time = cur_time - condition if condition is not None else td(minutes=0)
        if elapsed_time >= self.data_window:
            dx_result = self.pre_conditions(dx_list, message, cur_time, dx_result)
            self.clear_all()
        return dx_result

    def pre_conditions(self, diagnostics, message, cur_time, dx_result):
        """
        Publish pre-conditions not met message.
        :param diagnostics:
        :param message:
        :param cur_time:
        :param dx_result:
        :return:
        """
        dx_msg = {}
        for sensitivity in self.sensitivity:
            dx_msg[sensitivity] = message

        for diagnostic in diagnostics:
            dx_table = {diagnostic + DX: dx_msg}
            table_key = create_table_key(self.analysis, cur_time)
            dx_result.insert_table_row(table_key, dx_table)
        return dx_result

    def sensor_limit_check(self, oat, rat, mat, cur_time):
        """
        Check temperature limits on sensors.
        :param oat:
        :param rat:
        :param mat:
        :param cur_time:
        :return:
        """
        sensor_limit = (False, None)
        if oat < self.oat_thresholds[0] or oat > self.oat_thresholds[1]:
            sensor_limit = (True, OAT_LIMIT)
        elif mat < self.mat_thresholds[0] or mat > self.mat_thresholds[1]:
            sensor_limit = (True, MAT_LIMIT)
        elif rat < self.rat_thresholds[0] or rat > self.rat_thresholds[1]:
            sensor_limit = (True, RAT_LIMIT)

        if sensor_limit[0]:
            if self.sensor_limit is None:
                self.sensor_limit = cur_time
        else:
            self.sensor_limit = None
        return sensor_limit

    def output_format(cls, input_object):
        """
        The output object takes the resulting input object as a argument
        so that it may give correct topics to it's outputs if needed.

        output schema description
            {TableName1: {name1:OutputDescriptor1, name2:OutputDescriptor2,...},....}

            eg: {'OAT': {'Timestamp':OutputDescriptor('timestamp', 'foo/bar/timestamp'),'OAT':OutputDescriptor('OutdoorAirTemperature', 'foo/bar/oat')},
                'Sensor': {'SomeValue':OutputDescriptor('integer', 'some_output/value'),
                'SomeOtherValue':OutputDescriptor('boolean', 'some_output/value),
                'SomeString':OutputDescriptor('string', 'some_output/string)}}

        Should always call the parent class output_format and update the dictionary returned from
        the parent.

        result = super().output_format(input_object)
        my_output = {...}
        result.update(my_output)
        return result
        """
        return {}


#standard
import math
import sys
import unittest
import time
import logging
import os
from xml.dom import minidom
import xml.etree.ElementTree as ET

#get current directory of file
curdir = os.path.dirname(os.path.realpath(__file__)) + os.sep
control_values =  {
            'OutputLevel' : [0, 'Default', '', 'int', 'OutputLevel'],
            'Title' : ['', 'Default', '', 'None', 'Title'],
            'Seed' :[123345, 'Default', '', 'int', 'Seed'],
            'SimulationSeconds' :[0, 'Default', '', 'int', 'SimulationSeconds'],
            'FileType' : ['CSV', 'Default', '', 'Upper', 'InputFiles/TrajectoryFile/FileType'],
            'TrajectoryFileName' : [None, 'Default', '', 'file', 'InputFiles/TrajectoryFile/FileName'],
            'TypeColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/Type'],
            'XColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/X'],
            'YColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/Y'],
            'TimeColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/Time'],
            'IDColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/ID'],
            'SpdColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/Speed'],
            'AccelColumn' : [None, 'Default', '', 'None', 'InputFiles/TrajectoryFile/CSVTrajectoryFileFields/Accel'],

            'PDMMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMMarketPenetration'],
            'PDMVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMVehicleIDs'],
            'PDMVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMVehicleTypes'],

            'BSMMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/BSMMarketPenetration'],
            'BSMVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMVehicleIDs'],
            'BSMVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMVehicleTypes'],

            'DualPDMBSMMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/DualPDMBSMMarketPenetration'],
            'DualPDMBSMVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/DualPDMBSMVehicleIDs'],
            'DualPDMBSMVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/DualPDMBSMVehicleTypes'],

            'CAMMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/CAMMarketPenetration'],
            'CAMVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/CAMVehicleIDs'],
            'CAMVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/CAMVehicleTypes'],
            'SPOTMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/SPOTMarketPenetration'],
            'SPOTVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/SPOTVehicleIDs'],
            'SPOTVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/SPOTVehicleTypes'],

            'PDMDSRCMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMEquipped/DSRC/MarketPenetration'],
            'PDMDSRCVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/DSRC/VehicleTypes'],
            'PDMDSRCVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/DSRC/VehicleIDs'],
            'PDMCellularMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMEquipped/Cellular/MarketPenetration'],
            'PDMCellularVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/Cellular/VehicleTypes'],
            'PDMCellularVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/Cellular/VehicleIDs'],
            'PDMDualCommMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMEquipped/DualComm/MarketPenetration'],
            'PDMDualCommVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/DualComm/VehicleTypes'],
            'PDMDualCommVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMEquipped/DualComm/VehicleIDs'],
            'BSMDSRCMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/BSMEquipped/DSRC/MarketPenetration'],
            'BSMDSRCVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/DSRC/VehicleTypes'],
            'BSMDSRCVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/DSRC/VehicleIDs'],
            'BSMCellularMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/BSMEquipped/Cellular/MarketPenetration'],
            'BSMCellularVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/Cellular/VehicleTypes'],
            'BSMCellularVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/Cellular/VehicleIDs'],
            'BSMDualCommMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/BSMEquipped/DualComm/MarketPenetration'],
            'BSMDualCommVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/DualComm/VehicleTypes'],
            'BSMDualCommVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/BSMEquipped/DualComm/VehicleIDs'],
            'PDMBSMDSRCMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMBSMEquipped/DSRC/MarketPenetration'],
            'PDMBSMDSRCVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/DSRC/VehicleTypes'],
            'PDMBSMDSRCVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/DSRC/VehicleIDs'],
            'PDMBSMCellularMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMBSMEquipped/Cellular/MarketPenetration'],
            'PDMBSMCellularVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/Cellular/VehicleTypes'],
            'PDMBSMCellularVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/Cellular/VehicleIDs'],
            'PDMBSMDualCommMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/PDMBSMEquipped/DualComm/MarketPenetration'],
            'PDMBSMDualCommVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/DualComm/VehicleTypes'],
            'PDMBSMDualCommVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/PDMBSMEquipped/DualComm/VehicleIDs'],

            'CAMDSRCMarketPenetration' : [0, 'Default', '', 'int', 'EquippedVehicles/CAMEquipped/DSRC/MarketPenetration'],
            'CAMDSRCVehicleTypes' : [[], 'Default', '', 'List_int', 'EquippedVehicles/CAMEquipped/DSRC/VehicleTypes'],
            'CAMDSRCVehicleIDs' : [[], 'Default', '', 'List_int', 'EquippedVehicles/CAMEquipped/DSRC/VehicleIDs'],

            'RSELocationFile' : [None, 'Default', '', 'file', 'InputFiles/RSELocationFile'],
            'SPOTLocationFile' : [None, 'Default', '', 'file', 'InputFiles/SPOTLocationFile'],
            'StrategyFile' : [None, 'Default', '', 'file', 'InputFiles/StrategyFile'],
            'RegionsFile' : [None, 'Default', '', 'file', 'InputFiles/RegionsFile'],
            'PDMAllFile' : ['PDM_All.csv', 'Default', '', 'file', 'OutputFiles/PDMAllFile'],
            'PDMTransFile' : ['PDM_Trans.csv', 'Default', '', 'file', 'OutputFiles/PDMTransFile'],
            'BSMTransFile' : ['BSM_Trans.csv', 'Default', '', 'file', 'OutputFiles/BSMTransFile'],
            'CAMTransFile' : ['CAM_Trans.csv', 'Default', '', 'file', 'OutputFiles/CAMTransFile'],
            'SPOTTravelFile' : ['SPOT_TravelRecords.csv', 'Default', '', 'file', 'OutputFiles/SPOTTravelFile'],
            'SPOTBehaviorFile' : ['SPOT_BehaviorRecords.csv', 'Default', '', 'file', 'OutputFiles/SPOTBehaviorFile'],

            'BSMTransColor' : [[0,0,204], 'Default', '', 'List_int', 'Color/BSMTransColor'], # Default: dark blue
            'PDMPeriodicColor' : [[204,0,204], 'Default', '', 'List_int', 'Color/PDMPeriodicColor'], # Default: light purple
            'PDMStopColor' : [[204,0,0], 'Default', '', 'List_int', 'Color/PDMStopColor'], # Default: red
            'PDMStartColor' : [[102,204,0], 'Default', '', 'List_int', 'Color/PDMStartColor'], # Default: green
            'PDMDSRCTransColor' : [[0,0,0], 'Default', '', 'List_int', 'Color/PDMDSRCTransColor'], # Default: black
            'PDMCellularTransColor' : [[0,204,204], 'Default', '', 'List_int', 'Color/PDMCellularTransColor'], # Default: light blue
            'SpotBehaviorColor' : [[102,0,204], 'Default', '', 'List_int', 'Color/SpotBehaviorColor'], # Default: dark purple
            'SpotTravelColor' : [[0,102,204], 'Default', '', 'List_int', 'Color/SpotTravelColor'], # Default: blue
            'SpotTransColor' : [[255,255,71], 'Default', '', 'List_int', 'Color/SpotTransColor'], # Default: yellow
            'CAMTransColor' : [[204,102,0], 'Default', '', 'List_int', 'Color/CAMTransColor'], # Default: orange
            'DualPDMBSMColor' : [[255,255,255], 'Default', '', 'List_int', 'Color/DualPDMBSMColor'], # Default: white
            'PDMDefaultColor' : [[255,255,255], 'Default', '', 'List_int', 'Color/PDMDefaultColor'], # Default: white
            'BSMDefaultColor' : [[255,255,255], 'Default', '', 'List_int', 'Color/BSMDefaultColor'], # Default: white
            'DefaultColor' : [[255,255,255], 'Default', '', 'List_int', 'Color/DefaultColor'], # Default: white
            'ColorDisplay' : [False, 'Default', '', 'Boolean', 'Color/ColorDisplay'],
            'ColorDisplayDuration' : [1, 'Default', '', 'int', 'Color/ColorDisplayDuration'],
        }

strategy_values = {
            'Title' : ['', 'Default', '', 'None', 'Title'],
            'TimeBeforePDMCollection' : [120, 'Default', '', 'int', 'Inputs/PDM/TimeBeforePDMCollection' ],
            'DistanceBeforePDMCollection' : [1680, 'Default', '', 'int', 'Inputs/PDM/DistanceBeforePDMCollection' ],
            'GapMinTime': [3, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/GapInformation/MinTime'],
            'GapMaxTime': [13, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/GapInformation/MaxTime'],
            'GapMinDistance': [164, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/GapInformation/MinDistance'],
            'GapMaxDistance': [820, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/GapInformation/MaxDistance'],

            'TimeBetweenPSNSwitches' : [120, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/TimeBetweenPSNSwitches' ],
            'DistanceBetweenPSNSwitches' : [3281, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/DistanceBetweenPSNSwitches'],
            'Gap' : [0, 'Default', '', 'int', 'Inputs/PDM/PSNStrategy/Gap'],

            'StopStartStrategy' : [1, 'Default', '', 'int', 'Inputs/PDM/StopStartStrategy/StrategyID'],
            'StopThreshold' : [5, 'Default', '', 'int', 'Inputs/PDM/StopStartStrategy/StopThreshold'],
            'StopLag' : [15, 'Default', '', 'int', 'Inputs/PDM/StopStartStrategy/StopLag'],
            'StartThreshold' : [10, 'Default', '', 'int', 'Inputs/PDM/StopStartStrategy/StartThreshold'],
            'MultipleStops' : [0, 'Default', '', 'int', 'Inputs/PDM/StopStartStrategy/MultipleStops'],

            'PeriodicStrategy' : [1, 'Default', '', 'int', 'Inputs/PDM/PeriodicStrategy/StrategyID'],
            'LowSpeedThreshold' : [20, 'Default', '', 'int', 'Inputs/PDM/PeriodicStrategy/LowSpeedThreshold'],
            'ShortSpeedInterval' : [4, 'Default', '', 'int', 'Inputs/PDM/PeriodicStrategy/ShortSpeedInterval'],
            'HighSpeedThreshold' : [60, 'Default', '', 'int', 'Inputs/PDM/PeriodicStrategy/HighSpeedThreshold'],
            'LongSpeedInterval' : [20, 'Default', '', 'int', 'Inputs/PDM/PeriodicStrategy/LongSpeedInterval'],
            'MaxDeltaSpeed' : [0.10, 'Default', '', 'float', 'Inputs/PDM/PeriodicStrategy/MaxDeltaSpeed'],

            'TotalCapacity' : [30, 'Default', '', 'int', 'Inputs/PDM/BufferStrategy/TotalCapacity'],
            'SSRetention' : [4, 'Default', '', 'int', 'Inputs/PDM/BufferStrategy/SSRetention'],

            'MinRSERange': [492, 'Default', '', 'int', 'Inputs/DSRC/RSEInformation/MinRSERange'],
            'MaxRSERange': [492, 'Default', '', 'int', 'Inputs/DSRC/RSEInformation/MaxRSERange'],
            'TimeoutRSE': [200, 'Default', '', 'int', 'Inputs/DSRC/RSEInformation/TimeoutRSE'],
            'MinNumberofPDMtoTransmitViaDSRC' : [4, 'Default', '', 'int', 'Inputs/DSRC/RSEInformation/MinNumberofPDMtoTransmitViaDSRC'],

            'SPOTdeviceRange' : [32.8, 'Default', '', 'float', 'Inputs/SPOT/DeviceRange'], #SPOT devices usually cover 20 meters (10 meter radius or 32.8 feet) of roadway
            'SPOTBehaviorSensingFrequency' : [0.3, 'Default', '', 'float', 'Inputs/SPOT/BehaviorSensingFrequency'],
            'SPOTaccelThreshold' : [-0.25, 'Default', '', 'float', 'Inputs/SPOT/AccelThreshold'],
            'SPOTyawrateThreshold' : [9.8, 'Default', '', 'float', 'Inputs/SPOT/YawrateThreshold'],

            'BrakeThreshold': [-0.2, 'Default', '', 'float', 'Inputs/BSM/BrakeThreshold'],

            'PDMFrequencyDSRC': [0.0, 'Default', '', 'float', 'Inputs/PDM/DSRCFrequency'],
            'PDMFrequencyCellular': [0.0, 'Default', '', 'float', 'Inputs/PDM/CellularFrequency'],
            'BSMFrequencyCellular': [0.0, 'Default', '', 'float', 'Inputs/BSM/CellularFrequency'],
            'BSMFrequencyDSRC': [0.0, 'Default', '', 'float', 'Inputs/BSM/DSRCFrequency'],
        }

def write_default(filename, filetype, parameters):
    root = ET.Element(filetype)
    for tag, value in parameters.iteritems():
        children = value[4].split('/')
        length = len(children)-1
        found_path = False
        path = value[4].rsplit('/',1)[0]
        while length > 0 and found_path == False:
              if root.findall("." + path):
                    found_path = True
              else:
                    path = path.rsplit('/',1)[0]
                    length -=1
        if length > 0:
            subelement = root.find(path)
        else:
            subelement = root
        for r in range(length, len(children)-1):
              x = ET.SubElement(subelement, children[r])
              subelement = x
        ET.SubElement(subelement, children[-1]).text = str(value[0])

    root.find('Title').text = "Default" + filetype

    rough_string = ET.tostring(root, 'utf-8')
    reparsed = minidom.parseString(rough_string).toprettyxml(indent = '    ')

    with open(filename, 'wb') as f_out:
            f_out.write(reparsed)

def write_default_regions(filename):
    with open(filename, 'wb') as f_out:
            f_out.write("""<?xml version="1.0" encoding="UTF-8" ?>
<Regions>
    <Cell_Regions>
      <!-- Default snapshot transmission loss percentage for cellular transmissions outside defined cellular regions. -->
      <DefaultLossPercent>2</DefaultLossPercent>
      <!-- Default latency (seconds) for cellular transmissions outside defined cellular regions -->
      <DefaultLatency>1</DefaultLatency>
      <!-- Default mininum number of PDMs needed in the vehicle buffer for cellular transmission -->
      <MinPDMtoTransmit>4</MinPDMtoTransmit>
      <Cell_Region>
          <Title>Cellular_Region1</Title>
          <!-- Upper left x and y coordinates of the region rectangle -->
          <UpperLeftPoint>
            <X>-5314</X>
            <Y>-2530</Y>
          </UpperLeftPoint>
          <!-- Lower right x and y coordinates of the region rectangle -->
          <LowerRightPoint>
            <X>-4917</X>
            <Y>-2705</Y>
          </LowerRightPoint>
          <!-- Transmission loss percentage of cellular transmissons made within the region -->
          <LossPercent>5</LossPercent>
          <!-- Transmission latency (seconds) of cellular transmissions made within the region -->
          <Latency>2</Latency>
      </Cell_Region>
    </Cell_Regions>
    <Event_Regions>
        <Region>
          <Title>AirTempHigh</Title>
          <UpperLeftPoint>
            <X>-6157</X>
            <Y>-2530</Y>
          </UpperLeftPoint>
          <LowerRightPoint>
            <X>-5624</X>
            <Y>-2705</Y>
          </LowerRightPoint>
          <TimePeriods>
            <Period>
                <!-- Start and end time that the event region is active -->
                <StartTime>0</StartTime>
                <EndTime>1200</EndTime>
             </Period>
          </TimePeriods>
          <Events>
              <Event>
                <Title>AirTemp</Title>
                <!-- Standard deviation probability requires a mean and a standard deviation (SD) -->
                <Mean>7</Mean>
                <SD>4</SD>
                <!-- Static recheck value (seconds) determines how often the TCA checks this event -->
                <Recheck>10</Recheck>
              </Event>
          </Events>
        </Region>
        <Region>
          <Title>HeavyRain</Title>
          <UpperLeftPoint>
            <X>-5314</X>
            <Y>-2530</Y>
          </UpperLeftPoint>
          <LowerRightPoint>
            <X>-4917</X>
            <Y>-2705</Y>
          </LowerRightPoint>
          <TimePeriods>
              <Period>
                <StartTime>50</StartTime>
                <EndTime>100</EndTime>
              </Period>
              <Period>
                <StartTime>300</StartTime>
                <EndTime>800</EndTime>
              </Period>
          </TimePeriods>
          <Events>
              <Event>
                <Title>Wipers</Title>
                <!-- Static probability (%) of occurrence -->
                <Probability>85</Probability>
                <!-- The mean value for generating a random recheck value (seconds) using a Poisson distribution -->
                <RecheckPoisson>7</RecheckPoisson>
              </Event>
              <Event>
                <Title>TractionControl</Title>
                <Probability>5</Probability>
                <Recheck>2</Recheck>
              </Event>
          </Events>
        </Region>
    </Event_Regions>
</Regions>"""
)

def set_logger(console_level = logging.INFO, file_level = logging.DEBUG, include_file = True, append_file = False):
    """
    Creates the log file

    :param console_level: level to log items to screen
    :param file_level:  level to log items to file
    :param include_file:  include log file
    :param append_file: append to last file
    :return: logger object
    """

    if not append_file:
        # os.remove(curdir + 'tca2.log')
        try:
            os.remove(curdir + 'tca2.log')
        except:
            pass

    logger = logging.getLogger(curdir + 'tca2')
    logger.setLevel(file_level)

    ch = logging.StreamHandler()
    ch.setFormatter(logging.Formatter('%(message)s'))
    ch.setLevel(console_level)
    logger.addHandler(ch)

    if include_file:
        fh = logging.FileHandler(curdir + 'tca2.log')
        # fh.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
        fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
        fh.setLevel(file_level)
        logger.addHandler(fh)

    return logger

logger = set_logger(include_file = True, file_level = logging.DEBUG)

#-------------------------------------------------------------------------
def SPOT_time(tp, interval):
    """
    Determines if the timeperiod is valid for generating an ITS Spot message

    :param tp: float of current time period
    :param interval: float of the frequency for checking ITS Spot behavior record triggers
    :return: True is the tp is valid, otherwise False
    """

    l = [str(x) for x in range(0, 10, int(str(interval)[-1]))]

    if str(tp)[-1] in l:
        return True

    return False

#-------------------------------------------------------------------------
def Get_Heading_Change(heading_last, heading_current):
    """
    determines the change in heading

    :param heading_last: float of previous handing
    :param heading_current: float of current heading
    :return: float of the difference in heading
    """
    r = heading_current - heading_last + 180
    return (r % 360) - 180

#-------------------------------------------------------------------------
def Chk_Range(x1, y1, x2, y2):

    """
    Determine the range between two points

    :param x1: float value for X for first point
    :param y1: float value for Y for first point
    :param x2: float value for X for 2nd point
    :param y2: float value for Y for 2nd point
    :return: float distance between the two points
    """
    unitValue = 1 #1 = linear distance #2 - Spherical distance

    ran = None

    if unitValue ==1:
        ran = math.sqrt((float(y2) - float(y1)) ** 2 + (float(x2) - float(x1)) ** 2)
    elif unitValue==2:

        rad = 3958.76 #Earth radius in Miles

        #Convert to radians
        p1lat = float(x1) / 180 * math.pi
        p1long = float(y1) / 180 * math.pi
        p2lat = float(x2) / 180 * math.pi
        p2long = float(y2) / 180 * math.pi

        if (p1lat == p2lat) and (p1long == p2long):
            ran = float(0)
        else:
            ran = math.acos(math.sin(p1lat) * math.sin(p2lat) + math.cos(p1lat) * math.cos(p2lat)
                  * math.cos(p2long - p1long)) * rad

    return ran

def file_fzp_start(filename):
    """
    finds the start of the fzp data

    :param filename: string of the fzp file name
    :return: number of lines to skip at top of file
    """

    with open(filename) as in_f:
        c= 0
        cols = []
        #find start of VISSIM data
        line = in_f.readline()
        while 'VehNr;' not in line:
            line = in_f.readline()
            cols = [x.strip() for x in line.split(';')][:-1]
            c +=1

        return {'lines_to_skip' : c, 'header_cols' : cols}

#-------------------------------------------------------------------------
# (x1,y1) : Top left of rectangle, (x2,y2) : Bottom right of rectangle, (x3,y3) : Point in question
def Chk_Cellular_Range(x1, y1, x2, y2, x3, y3):

    """
    determines in point is in a rectangle

    :param x1: float value for X for first point
    :param y1: float value for Y for first point
    :param x2: float value for X for 2nd point
    :param y2: float value for Y for 2nd point
    :param x3: float value for X for 3rd point
    :param y3: float value for Y for 3rd point
    :return: boolean value True if in, False if out
    """
    if((x3>=x1) & (y3<=y1) & (x3<=x2) & (y3>=y2)):
        return True
    else:
        return False

#-------------------------------------------------------------------------
def Get_Heading(x1, y1, x2, y2):
    """
    Returns the Heading based on two points

    :param x1: float value for X for first point
    :param y1: float value for Y for first point
    :param x2: float value for X for 2nd point
    :param y2: float value for Y for 2nd point
    :return: float new heading value
    """

    heading = 0
    dx = x2 - x1
    dy = y2 - y1

    if dx != 0:
      heading = (90 - math.degrees(math.atan2(dy,dx)) + 360) % 360

    elif dy > 0: heading = 0

    elif dy < 0: heading = 180

    return heading

#-------------------------------------------------------------------------
def clear_old_files(CF):

        files = [
            CF.Control["PDMTransFile"],
            CF.Control["PDMAllFile"],
            CF.Control["BSMTransFile"],
            CF.Control["CAMTransFile"],
            CF.Control['SPOTTravelFile'],
            CF.Control['SPOTBehaviorFile']]

        for f in files:
            try:
                os.remove(f)
            except:
                pass

#-------------------------------------------------------------------------
def report_errors(errors):
    """
    prints out any errors found

    :param errors: list of errors
    """
    if len(errors) > 0:
        for error in errors:
            logger.debug(error)
        sys.exit(0)

class Timer:

   def __init__(self, enabled=False):
        """
        Create Timer class

        :param enabled: Boolean if used
        """
        self.timers = {}
        self.last_start = ''
        self.enabled = enabled

   def valid_title(self, title):
       """
       Checks if title of timer item is validate

       :param title: string of title
       :return: boolean True if validate, False if not
       """
       if title in self.timers.keys() and isinstance(title, str) and self.timers[title]['count']>0:
           return True
       else:
           return False

   def __getitem__(self, title):
        """
        prints out summary information for timer

        :param title: title of time name
        :return: string of summary information about that timer
        """
        if self.valid_title(title):
                return title + ', ' + \
                       str(self.stats(title)) + ', ' + \
                       str(self.stats(title, type='max')) + ', ' + \
                       str(self.stats(title, type='avg')) + ', ' + \
                       str(self.stats(title, type='count')) + ', ' + \
                       str(self.stats(title, type='last'))
        else:
            print ('Timer Error %s not created or stopped' % title)

   def current(self, title):
       """
       Returns the current timer title

       :param title: title of timer
       :return: name of current timer or 0
       """
       if self.valid_title(title):
            return self.timers[title]['last']
       else:
            return 0

   def stats(self, title, type='SUM'):
        """
        returns the sum, max, average, count, or last value of a time title.

        :param title: name of timer value
        :param type: type of stats to produce
        :return: stat of the given timer value
        """
        if self.valid_title(title):
                if type.upper() == 'SUM':
                    return self.timers[title]['sum']
                elif type.upper() == 'MAX':
                    return self.timers[title]['max']
                elif type.upper() == 'AVG':
                    return self.timers[title]['sum'] / self.timers[title]['count']
                elif type.upper() == 'COUNT':
                    return self.timers[title]['count']
                elif type.upper() == 'LAST':
                    return  self.timers[title]['last']
        else:
            print('Error timer %s was never stopped' % title)

   def start(self, title):
       """
       starts a given timer

       :param title: string name of the timer
       """
       if self.enabled:
            if title not in self.timers.keys():
                self.timers[title] = {'st' : time.time(),
                                      'sum' : 0.0,
                                      'count' : 0,
                                      'max' : 0.0,
                                      'last' : None,
                                     }
            else:
                self.timers[title]['st'] = time.time()

            self.last_start = title

   def stop(self, title=''):
        """
        stops a given timer title

        :param title: name of the timer
        """
        if self.enabled:
            if title == '':
               title = self.last_start
            if self.timers.has_key(title):
               new_time = time.time() - self.timers[title]['st']
               self.timers[title]['sum'] +=  new_time
               self.timers[title]['count'] += 1
               self.timers[title]['last'] = new_time
               if new_time > self.timers[title]['max']:
                   self.timers[title]['max'] = new_time

   def drop(self,title):
        """
        removes timer from list

        :param title: name of a timer
        """

        if self.enabled:
            del self.timers[title]

   def header(self):
       """
       :return: string of header for timer file
       """
       return 'title,sum,max,avg,len,last\n'

   def write(self):
        """
        writes all timers values to string

        :return: string of the timer list
        """
        if self.enabled and len(self.timers) > 0:
            l = []
            for title in sorted(self.timers):
                if self.valid_title(title):
                    l.append(self.__getitem__(title))
            return '\n'.join(l)
        else:
            return ''

#*************************************************************************
class TestTimer(unittest.TestCase):

    def test_timer(self):
        t1=Timer(enabled = True)

        t1.start('test1')
        time.sleep(2)
        t1.stop()
        assert  ( round(t1.current('test1'), 2) >= 2.0) and (t1.current('test1') < 2.1)

        t1.start('test2')
        time.sleep(3)
        t1.stop('test2')

        assert  (round(t1.current('test2'), 2) >= 3.0) and (t1.current('test2') < 3.1)

        t1.start('test2')
        time.sleep(2)
        t1.stop()
        assert  (round (t1.stats('test2'), 2) >= 5.0) and (t1.stats('test2') < 5.1)

class CoreTests(unittest.TestCase):

    def test_range(self):
        assert 1 == int(Chk_Range(1,1,2,2))
        assert 5.0 == Chk_Range(-2,1,1,5)
        assert 0.0 == Chk_Range(1,1,1,1)

    def test_errors(self):
        e = []
        report_errors(e)

    def test_cellular_range(self):
        assert True == Chk_Cellular_Range(-2,1,0,0,-1,1)
        assert False == Chk_Cellular_Range(-2,1,0,0,1,0)

    def test_heading(self):
        assert 0 == Get_Heading(0,-1,0,0)
        assert 45 == Get_Heading(1,1,3,3)
        assert 90 == Get_Heading(-1,0,0,0)
        assert 180 == Get_Heading(0,0,0,-1)
        assert 225 == Get_Heading(-1,-1,-5,-5)
        assert 270 == Get_Heading(0,0,-1,0)

    def test_SPOT(self):
        assert True == SPOT_time(1.3, 0.3)
        assert False == SPOT_time(1.4, 0.3)
        assert True == SPOT_time(2.0, 0.3)

if __name__ == '__main__':
    unittest.main()


from django.core.urlresolvers import reverse
import datetime, time

from django.utils.translation import ugettext_lazy as _, ugettext
from django.forms import models as modelforms
from django.forms.fields import DateTimeField
from django.db.models import Q
from django.forms.util import ValidationError
from django.forms.models import save_instance
from django.conf.urls.defaults import patterns, url
from django.utils.safestring import mark_safe
from django.template.defaultfilters import date
from django.conf import settings

from ella.core.models import Author, Source, Category, Listing, HitCount, Placement, Related, Publishable
from ella.core.conf import core_settings

from ella import newman
from ella.newman import options, fields
from ella.newman.filterspecs import CustomFilterSpec, NewmanSiteFilter

class ListingForm(modelforms.ModelForm):
    def clean(self):
        d = super(ListingForm, self).clean()
        if not self.is_valid():
            return d
        if d['publish_to'] and d['publish_from'] > d['publish_to']:
            raise ValidationError(_('Publish to must be later than publish from.'))
        if d['priority_from'] and d['priority_to'] and d['priority_from'] > d['priority_to']:
            raise ValidationError(_('Priority to must be later than priority from.'))
        return d

    class Meta:
        model = Listing

from ella.core.admin import PlacementForm as DjangoAdminPlacementForm

class PlacementForm(DjangoAdminPlacementForm):
    # create the field here to pass validation
    listings =  fields.ListingCustomField(Category.objects.all(), label=_('Category'), cache_choices=True, required=False)

    def get_publish_date(self, pub_from):
        " Tries to save publish_from field specified either by POST parameter or by Placement (self.instance). "
        if pub_from:
            dt_field = DateTimeField()
            return dt_field.clean(pub_from)
        return self.instance.publish_from

    def save(self, commit=True):
        cleaned_list_cats = self.cleaned_data.pop('listings')
        list_cats = []
        # Order of items should be preserved (in cleaned_data is order not preserved)
        for pk in self.data.getlist( self.get_part_id('') ):
            list_cats.append(Category.objects.get(pk=int(pk)))
        publish_from_fields = self.data.getlist(self.get_part_id('publish_from'))
        commercial_fields = self.data.getlist(self.get_part_id('commercial'))
        # commercial_fields are optionul thus
        if len(commercial_fields) != len(list_cats):
            commercial_fields = [ None for c in list_cats ]
        else:
            commercial_fields = [ int(f) for f in commercial_fields ]

        instance = self.instance

        def save_them():
            if not list_cats:
                return
            listings = dict([ (l.category, l) for l in Listing.objects.filter(placement=instance.pk) ])
            forloop_counter = 0 # used for counting delete checkboxes
            for c, pub, commercial in zip(list_cats, publish_from_fields, commercial_fields):
                forloop_counter += 1
                delete_listing = self.data.get(self.get_part_id('%d-DELETE' % forloop_counter), 'off')
                if delete_listing == 'on':
                    # skip following for-cycle body, so the listing will be deleted
                    continue
                publish_from = self.get_publish_date(pub)
                if not c in listings:
                    # create listing
                    l = Listing(
                        placement=instance,
                        category=c,
                        publish_from=publish_from
                    )
                    if commercial is not None:
                        l.commercial = commercial
                    l.save()
                else:
                    del listings[c]
                    lst = Listing.objects.filter(placement=instance, category=c)
                    if not lst:
                        continue
                    l = lst[0]
                    # if publish_from differs, modify Listing object
                    if l.publish_from != publish_from or (commercial is not None and l.commercial != commercial):
                        l.publish_from = publish_from
                        if commercial is not None:
                            l.commercial = commercial
                        l.save()
            for l in listings.values():
                l.delete()

        if commit:
            save_them()
        else:
            save_m2m = getattr(self, 'save_m2m', None)
            def save_all():
                if save_m2m:
                    save_m2m()
                save_them()
            self.save_m2m = save_all
        instance.category = self.cleaned_data['category']
        instance.publish_from = self.cleaned_data['publish_from']
        instance.publish_to = self.cleaned_data['publish_to']
        instance.slug = self.cleaned_data['slug']
        instance.static = self.cleaned_data['static']
        if not commit:
            return instance
        if self.instance.pk is None:
            fail_message = 'created'
        else:
            fail_message = 'changed'
        return save_instance(self, instance, self._meta.fields,
                             fail_message, commit, exclude=self._meta.exclude)

    def get_part_id(self, suffix):
        id_part = self.data.get('placement_listing_widget')
        if not suffix:
            return id_part
        return '%s-%s' % (id_part, suffix)

    def listings_clean(self, placement_publish_from, data):
        # get listing category, publish_from and publish_to
        pub_from = data.getlist(self.get_part_id('publish_from'))
        listings = self.cleaned_data['listings']
        if pub_from and len(pub_from) != len(listings):
            raise ValidationError(_('Duplicate listings'))
        for lst, pub in zip(listings, pub_from):
            if not pub:
                #raise ValidationError(_('This field is required'))
                continue
            dt_field = DateTimeField()
            publish_from = dt_field.clean(pub)
            if publish_from < placement_publish_from:
                raise ValidationError(_('No listing can start sooner than main listing'))

    def clean(self):
        # no data - nothing to validate
        if not self.is_valid() or not self.cleaned_data or not self.instance or not self.cleaned_data['publishable']:
            return self.cleaned_data

        obj = self.instance
        cat = None
        if obj.pk:
            cat = getattr(obj, 'category', None)
        obj_slug = getattr(obj, 'slug', obj.pk)
        # if Placement has no slug, slug from Publishable object should be considered in following checks:
        if not obj_slug:
            obj_slug = self.cleaned_data['publishable'].slug

        main = None
        d = self.cleaned_data
        # empty form
        if not d:
            return self.cleaned_data
        #if cat and cat == cat and cat: # should be equiv. if cat:...
        if cat:
            main = d

        if d['publish_to'] and d['publish_from'] > d['publish_to']:
            raise ValidationError(_('Publish to must be later than publish from.'))

        d['slug'] = obj_slug
        # try and find conflicting placement
        qset = Placement.objects.filter(
            category=d['category'],
            slug=d['slug'],
            #publishable=obj,
            static=d['static']
        )
        if d['static']: # allow placements that do not overlap
            q = Q(publish_to__lt=d['publish_from'])
            if d['publish_to']:
                q |= Q(publish_from__gt=d['publish_to'])
            qset = qset.exclude(q)
        # check for same date in URL
        if not d['static']:
            qset = qset.filter(
                publish_from__year=d['publish_from'].year,
                publish_from__month=d['publish_from'].month,
                publish_from__day=d['publish_from'].day,
            )
        # exclude current object from search
        if d['id']:
            qset = qset.exclude(pk=d['id'].pk)
        if qset:
            plac = qset[0]
            # raise forms.ValidationError(
            raise ValidationError(
                    _('''There is already a Placement object published in
                    category %(category)s with the same URL referring to %(target)s.
                    Please change the slug or publish date.''') % {
                        'category' : plac.category,
                        'target' : plac.publishable,
                    })

        if cat and not main:
            # raise forms.ValidationError(_('If object has a category, it must have a main placement.'))
            raise (_('If object has a category, it must have a main placement.'))

        self.listings_clean(d['publish_from'], self.data)
        return self.cleaned_data

class PlacementInlineFormset(options.NewmanInlineFormSet):

    def __init__(self, data=None, files=None, instance=None, save_as_new=None, prefix=None):
        self.can_delete = True
        self.show_commercial_switch = core_settings.LISTING_USE_COMMERCIAL_FLAG

        super(PlacementInlineFormset, self).__init__(instance=instance, data=data, files=files, prefix=prefix)

class ListingInlineAdmin(newman.NewmanTabularInline):
    model = Listing
    extra = 2
    suggest_fields = {'category': ('__unicode__', 'title', 'slug',)}
    form = ListingForm
    fieldsets = ((None, {'fields' : ('category','publish_from', 'publish_to', 'priority_from', 'priority_to', 'priority_value', 'commercial',)}),)

class PlacementInlineAdmin(newman.NewmanTabularInline):
    exclude = ('slug',)
    template = 'newman/edit_inline/placement.html'
    model = Placement
    max_num = 1
    suggest_fields = {'category': ('__unicode__', 'title', 'slug',)}

    form = PlacementForm
    formset = PlacementInlineFormset
    '''
    ct_field = 'target_ct'
    ct_fk_field = 'target_id'
    fieldsets = ((None, {'fields' : ('category', 'publish_from', 'publish_to', 'slug', 'static', 'listings',)}),)

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'category':
            kwargs['widget'] = widgets.ListingCategoryWidget
        return super(PlacementInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)
    '''

class HitCountInlineAdmin(newman.NewmanTabularInline):
    model = HitCount
    extra = 0

class ListingAdmin(newman.NewmanModelAdmin):
    form = ListingForm
    '''
    list_display = ('target_admin', 'target_ct', 'publish_from', 'category', 'placement_admin', 'target_hitcounts', 'target_url',)
    list_display_links = ()
    list_filter = ('publish_from', 'category__site', 'category', 'placement__target_ct',)
    raw_id_fields = ('placement',)
    date_hierarchy = 'publish_from'
    '''

class CategoryAdmin(newman.NewmanModelAdmin):
    list_display = ('__unicode__', 'title', 'tree_path')
    list_filter = ('site',)
    search_fields = ('title', 'slug',)
    prepopulated_fields = {'slug': ('title',)}
    ordering = ('site', 'tree_path',)
    suggest_fields = {'tree_parent': ('__unicode__', 'title', 'slug')}

class HitCountAdmin(newman.NewmanModelAdmin):
    list_display = ('target', 'hits', 'publish_from', 'target_url')
#    list_filter = ('placement__category', 'placement__publish_from')
    raw_id_fields = ('placement',)
    search_fields = ('placement__category__title', 'placement__publishable__pk', 'placement__publishable__title', 'placement__publishable__slug' )
    ordering = ('-hits', '-last_seen',)

    def publish_from(self, object):
        return object.placement.publish_from
    publish_from.short_description = _('Publish from')
    publish_from.allow_tags = True
    publish_from.order_field = 'placement__publish_from'

    def target_url(self, object):
        target = object.target()
        return mark_safe('%s' % (target.get_absolute_url(), target))
    target_url.short_description = _('View on site')
    target_url.allow_tags = True

    def get_urls(self):

        info = self.admin_site.name, self.model._meta.app_label, self.model._meta.module_name

        urlpatterns = patterns('',
            url(r'^by-category/$',
                self.by_category_view,
                name='%sadmin_%s_%s_by_category' % info),
        )
        urlpatterns += super(HitCountAdmin, self).get_urls()
        return urlpatterns

    def by_category_view(self, request, extra_context={}):
        pass

class AuthorAdmin(newman.NewmanModelAdmin):
    list_display = ('name', 'user', 'email',)
    prepopulated_fields = {'slug': ('name',)}
    search_fields = ('name', 'email',)

class SourceAdmin(newman.NewmanModelAdmin):
    list_display = ('name', 'url',)
    search_fields = ('name',)

class RelatedInlineAdmin(newman.GenericTabularInline):
    extra = 3
    model = Related
    ct_field = 'related_ct'
    ct_fk_field = 'related_id'
    suggest_fields = {'publishable': ('__unicode__', 'title', 'slug')}

class IsPublishedFilter(CustomFilterSpec):
    " Published/Nonpublished objects filter"
    lookup_var = 'publish_from'
    PUBLISH_FROM_WHEN_EMPTY = core_settings.PUBLISH_FROM_WHEN_EMPTY.strftime('%Y-%m-%d')
    CAPTION_ALL_WITH_PLACEMENT = _('All with placement')
    CAPTION_YES = _('Yes')
    CAPTION_NO = _('No')

    def title(self):
        return _('Is published?')

    def get_lookup_kwarg(self):
        for param in self.request_get:
            if param.startswith('%s__gt' % self.lookup_var) or param.startswith('%s__lt' % self.lookup_var):
                self.selected_lookup = param
                return param
        return ''

    def filter_func(self):
        # nepublikovany = nemaji placement (datum 3000) ci maji placement v budoucnu
        # ?publish_from__exact=2008-10-10
        lookup_var_not_published = '%s__gt' % self.lookup_var
        lookup_var_published = '%s__lte' % self.lookup_var
        lookup_var_has_placement = '%s__lt' % self.lookup_var
        now = time.strftime('%Y-%m-%d')
        link = ( self.CAPTION_NO, {lookup_var_not_published: now})
        self.links.append(link)
        link = ( self.CAPTION_YES, {lookup_var_published: now})
        self.links.append(link)
        link = (
            self.CAPTION_ALL_WITH_PLACEMENT,
            {
                lookup_var_has_placement: self.PUBLISH_FROM_WHEN_EMPTY
            }
        )
        self.links.append(link)
        self.remove_from_querystring = [lookup_var_published, lookup_var_not_published, lookup_var_has_placement]
        return True

    def generate_choice(self, **lookup_kwargs):
        param = self.get_lookup_kwarg()
        if not param:
            return None
        if lookup_kwargs[param] == self.PUBLISH_FROM_WHEN_EMPTY:
            return self.CAPTION_ALL_WITH_PLACEMENT
        elif param.startswith('%s__gt' % self.lookup_var):
            return self.CAPTION_NO
        elif param.startswith('%s__lte' % self.lookup_var):
            return self.CAPTION_YES
        return None

class PublishFromFilter(CustomFilterSpec):
    " Publish from customized filter. "
    published_from_field_path = 'publish_from'

    def title(self):
        return _('Publish from')

    def get_lookup_kwarg(self):
        out = [
            '%s__day' % self.published_from_field_path,
            '%s__month' % self.published_from_field_path,
            '%s__year' % self.published_from_field_path,
        ]
        return out

    def filter_func(self):
        # SELECT created FROM qs._meta.dbtable  GROUP BY created
        #qs = fspec.model_admin.queryset(fspec.request)
        #dates =  qs.dates(fspec.field_path, 'day', 'DESC')[:365]
        # Article.objects.filter(placement__listing__publish_from__gte='2012-01-01')
        YEAR = 365*24*60*60
        ts = time.time() - YEAR
        last_year = time.strftime('%Y-%m-%d %H:%M', time.localtime(ts))
        qs = Placement.objects.filter(listing__publish_from__gte=last_year)
        dates = qs.dates('publish_from', 'day', 'DESC')
        for date in dates:
            lookup_dict = dict()
            lookup_dict['%s__day' % self.published_from_field_path] = date.day
            lookup_dict['%s__month' % self.published_from_field_path] = date.month
            lookup_dict['%s__year' % self.published_from_field_path] = date.year
            link_text = '%d. %d. %d' % (date.day, date.month, date.year)
            link = ( link_text, lookup_dict)
            self.links.append(link)
        return True

    def generate_choice(self, **lookup_kwargs):
        args = self.get_lookup_kwarg()
        day = lookup_kwargs[args[0]]
        month = lookup_kwargs[args[1]]
        year = lookup_kwargs[args[2]]
        return u'%s. %s. %s' % (day, month, year)

class PlacementAdmin(newman.NewmanModelAdmin):
    list_display = ('publishable', 'category', 'publish_from',)
    list_filter = ('category',)
    unbound_list_filter = (NewmanSiteFilter, PublishFromFilter,)
    search_fields = ('publishable__title', 'category__title',)
    suggest_fields = {'category': ('__unicode__', 'title', 'slug',), 'publishable': ('title',)}

    inlines = [ListingInlineAdmin]

class PublishableAdmin(newman.NewmanModelAdmin):
    """ Default admin options for all publishables """

    exclude = ('content_type',)
    list_display = ('admin_link', 'category', 'photo_thumbnail', 'publish_from_nice', 'placement_link', 'site_icon', 'fe_link',)
    list_filter = ('category', 'content_type')
    unbound_list_filter = (NewmanSiteFilter, PublishFromFilter, IsPublishedFilter,)
    search_fields = ('title', 'description', 'slug', 'authors__name', 'authors__slug',) # FIXME: 'tags__tag__name',)
    raw_id_fields = ('photo',)
    prepopulated_fields = {'slug' : ('title',)}
    rich_text_fields = {'small': ('description',)}
    ordering = ('-publish_from',)

    suggest_fields = {
        'category': ('__unicode__', 'title', 'slug', 'tree_path'),
        'authors': ('name', 'slug', 'email',),
        'source': ('name', 'url',),
    }

    inlines = [PlacementInlineAdmin]

    def admin_link(self, object):
        ct = object.content_type
        return mark_safe('%s' % (ct.model, ct.app_label, ct.model, object.pk, object))
    admin_link.allow_tags = True
    admin_link.short_description = _('Publishable object')

    def site_icon(self, object):
        return mark_safe('%s' % object.category.site.name)
    site_icon.short_description = _('site')
    site_icon.allow_tags = True

    def fe_link(self, obj):
        if obj.publish_from.year < 3000:
            kwargs = {'content_type_id': obj.content_type.id, 'object_id': obj.pk}
            return mark_safe(
                'www' %
                reverse('newman:obj-redirect', kwargs=kwargs)
            )
        else:
            return '---'
    fe_link.short_description = _('WWW')
    fe_link.allow_tags = True

    def publish_from_nice(self, obj):
        span_str = '%s'
        date_str = date(obj.publish_from, settings.DATETIME_FORMAT)

        if obj.publish_from.year >= 3000:
            return mark_safe(span_str % ('unpublished', ugettext('No placement')))
        elif obj.publish_from > datetime.datetime.now():
            return span_str % ('unpublished', date_str)
        return span_str % ('published', date_str)
    publish_from_nice.short_description = _('Publish from')
    publish_from_nice.admin_order_field = 'publish_from'
    publish_from_nice.allow_tags = True

    def photo_thumbnail(self, object):
        if object.photo:
            return mark_safe(object.photo.thumb())
        else:
            return mark_safe('%s' % ugettext('No main photo!'))
    photo_thumbnail.allow_tags = True
    photo_thumbnail.short_description = _('Photo')

    def placement_link(self, object):
        if object.main_placement:
            return mark_safe('%s' % (object.main_placement.pk, object.main_placement.category))
        return mark_safe('%s' % (object.id, ugettext('No main placement')))
    placement_link.allow_tags = True
    placement_link.short_description = _('Main placement')

    # TODO: check speed with select_related()
#    def queryset(self, request):
#        qs = super(PublishableAdmin, self).queryset(request)
#        return qs.select_related()

newman.site.register(HitCount, HitCountAdmin)
newman.site.register(Category, CategoryAdmin)
newman.site.register(Source, SourceAdmin)
newman.site.register(Author, AuthorAdmin)
newman.site.register(Placement, PlacementAdmin)
#newman.site.register(Listing, ListingAdmin)
newman.site.register(Publishable, PublishableAdmin)


import os

import numpy as np
import pandas as pd
from scipy import stats
from pprint import pprint
import operator
import random
import statistics

fields = ["number_of_holes", "horizental_symetry", "vertical_symetry", "mean_vtd",
          "stddev_vtd", "mean_htd", "stddev_htd", "bottom_line_density", "hu7", "Class"]

singletons = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '']

classes = {
    'number_of_holes': ['123457', '069', '8'],
    'horizental_symetry': ['5', '7', '4', '6', '02389', '1236'],
    'vertical_symetry': ['46', '179', '13', '279', '024578'],
    'mean_vtd': ['0147', '047', '023476', '235689'],
    'stddev_vtd': ['4', '0147', '017', '236', '5689', '235689'],
    'mean_htd': ['0', '12', '12357', '3456', '345', '69', '45', '7', '17', '08', '689', '2345'],
    'stddev_htd': ['0689', '35', '2345', '123457', '147', '1247', '7'],
    'bottom_line_density': ['12', '0345689', '7']
}

def precision_recall_fmeasure(conf_mat):
    true_pos = np.diag(conf_mat1)
    false_pos = np.sum(conf_mat1, axis=0) - true_pos
    false_neg = np.sum(conf_mat1, axis=1) - true_pos

    precision = (np.sum(true_pos / (true_pos + false_pos))) / 10
    recall = (np.sum(true_pos / (true_pos + false_neg))) / 10
    fmeasure = 2 * precision * recall / (precision + recall)

    return precision, recall, fmeasure

def train_test_on_one_fold(train, test, classes):
    def find_mass_and_combine(image, classes):

        mass_values = {'number_of_holes': {
            '123457': 0,
            '069': 0,
            '8': 0
        }}
        # calculate 'number of holes' mass function:
        # fill with appropriate values according to the image

        if int(image['number_of_holes']) == 0:
            mass_values['number_of_holes']['123457'] = 1
        if int(image['number_of_holes']) == 1:
            mass_values['number_of_holes']['069'] = 1
        if int(image['number_of_holes']) == 2:
            mass_values['number_of_holes']['8'] = 1

        for attribute in classes:
            if attribute == 'number_of_holes':
                continue
            subclasses = classes[attribute]
            mass_values_per_class = {}
            for subclass in subclasses:
                if subclass in singletons:  # simple class, not a compound class
                    kde = stats.gaussian_kde(train[attribute][train['Class'] == int(subclass)])
                    mass_values_per_class[subclass] = float(kde(image[attribute]))
                else:
                    # here we sum up the instances of the classes in 'subclasses' and then we estimate the density
                    number_of_subclasses = len(subclass)
                    format = []
                    for single_class in subclass:
                        format.append(train[attribute][train['Class'] == int(single_class)])
                    total_df = pd.concat(format, ignore_index=True)
                    kde = stats.gaussian_kde(total_df)
                    mass_values_per_class[subclass] = float(kde(image[attribute]))
            mass_values[attribute] = mass_values_per_class

        # normalisation:
        for attribute in mass_values.keys():
            for classes in mass_values[attribute].keys():
                factor = 1.0 / sum(mass_values[attribute].values())
                for k in mass_values[attribute]:
                    mass_values[attribute][k] = mass_values[attribute][k] * factor

        # pprint(mass_values)

        # combination by dempster shafer rule:
        def intersect(d1, d2):
            result = ""
            for element in d1:
                if element in d2:
                    result = result + element

            return result

        combined = mass_values
        number_of_keys = len(combined.keys())
        while number_of_keys > 1:
            # select two attributes to combine
            # print(number_of_keys)
            keys = list(combined.keys())
            m12_phi = 0
            new_masses = {}
            '''
            pprint(combined[keys[0]])
            print('******')
            pprint(combined[keys[1]])
            '''
            for class0 in combined[keys[0]]:
                for class1 in combined[keys[1]]:
                    new_class = intersect(class0, class1)
                    if new_class == "":  # empty intersection, mass conflectuelle:
                        m12_phi += combined[keys[0]][class0] * combined[keys[1]][class1]
                    else:
                        if new_class in new_masses:
                            new_masses[new_class] += combined[keys[0]][class0] * combined[keys[1]][class1]
                        else:
                            new_masses[new_class] = combined[keys[0]][class0] * combined[keys[1]][class1]
            if m12_phi > 0.00001:
                new_masses[""] = m12_phi
            combined.pop(keys[0])
            combined.pop(keys[1])
            combined[keys[0] + '/' + keys[1]] = new_masses
            number_of_keys -= 1

        # pprint(combined)
        key = list(combined.keys())
        new_keys = list(combined[key[0]].keys())
        final_mass_values = combined[key[0]]
        all_singletons = True
        for key in new_keys:
            if key not in singletons:
                # apply max of plausibility
                all_singletons = False

        if all_singletons:
            masse_conflictuelle = final_mass_values.pop('')
            predicted_class = max(final_mass_values.items(), key=operator.itemgetter(1))[0]
            #print('masse de la classe predite: ', final_mass_values[predicted_class], ' masse conf: ',
                  #masse_conflictuelle)

        else:

            masses_of_singletons = {
                '0': 0,
                '1': 0,
                '2': 0,
                '3': 0,
                '4': 0,
                '5': 0,
                '6': 0,
                '7': 0,
                '8': 0,
                '9': 0
            }
            masse_conflictuelle = final_mass_values.pop('')
            for key, mass in zip(final_mass_values.keys(), final_mass_values.values()):
                for element in key:
                    masses_of_singletons[element] += mass / len(key)
            print('need for max of plausibility')

            predicted_class = max(masses_of_singletons.items(), key=operator.itemgetter(1))[0]

        return predicted_class, masse_conflictuelle

    success = 0
    count = 0
    conf_mat = np.zeros([10, 10], int)
    ms_phi = []
    for index, image in test.iterrows():

        predicted_class, mass_conflictuelle = find_mass_and_combine(image, classes)
        ms_phi.append(mass_conflictuelle)
        if int(predicted_class) == int(image['Class']):
            success += 1
        else:
            print('true class is: ', image['Class'], 'predicted class is: ', predicted_class)

        conf_mat[int(image['Class'])][int(predicted_class)] += 1
        count += 1
        # print('true class is: ', image['Class'], 'predicted class is: ', predicted_class)
    mean = statistics.mean(ms_phi)
    success_rate = success / count
    print('mean is :', mean, 'and success rate is: ', success_rate)

    return success_rate, conf_mat

data_file = r'attributes.csv'

df = pd.read_csv(data_file).dropna()

avg_accuracy = 0
accuracies = []
number_of_folds = 1
precisions = 0
recalls = 0
fmeasures = 0
for i in range(number_of_folds):
    rand = random.sample(range(1, 1000), 5)
    train1 = df.sample(frac=0.9, random_state=rand[0])  # random state is a seed value
    test1 = df.drop(train1.index)
    train2 = df.sample(frac=0.9, random_state=rand[1])  # random state is a seed value
    test2 = df.drop(train2.index)
    train3 = df.sample(frac=0.9, random_state=rand[2])  # random state is a seed value
    test3 = df.drop(train3.index)
    train4 = df.sample(frac=0.9, random_state=rand[3])  # random state is a seed value
    test4 = df.drop(train4.index)
    train5 = df.sample(frac=0.9, random_state=rand[4])  # random state is a seed value
    test5 = df.drop(train5.index)

    accuracy1, conf_mat1 = train_test_on_one_fold(train1, test1, classes)
    precision1, recall1, fmeasure1 = precision_recall_fmeasure(conf_mat1)

    accuracy2, conf_mat2 = train_test_on_one_fold(train2, test2, classes)
    precision2, recall2, fmeasure2 = precision_recall_fmeasure(conf_mat2)

    accuracy3, conf_mat3 = train_test_on_one_fold(train3, test3, classes)
    precision3, recall3, fmeasure3 = precision_recall_fmeasure(conf_mat1)

    accuracy4, conf_mat4 = train_test_on_one_fold(train4, test4, classes)
    precision4, recall4, fmeasure4 = precision_recall_fmeasure(conf_mat1)

    accuracy5, conf_mat5 = train_test_on_one_fold(train5, test5, classes)
    precision5, recall5, fmeasure5 = precision_recall_fmeasure(conf_mat1)

    precision = (precision1 + precision2 + precision3 + precision4 + precision5) / 5
    recall = (recall1 + recall2 + recall3 + recall4 + recall5) / 5
    fmeasure = (fmeasure1 + fmeasure2 + fmeasure3 + fmeasure4 + fmeasure5) / 5

    precisions += precision
    recalls += recall
    fmeasures += fmeasure
    print(conf_mat5)
    print(conf_mat4)
    print(conf_mat3)
    print(conf_mat2)
    print(conf_mat1)
    print(precision)
    print(recall)
    print(fmeasure)

    print(accuracy1, accuracy2, accuracy3, accuracy4, accuracy5)
    total_accuracy = (accuracy1 + accuracy2 + accuracy3 + accuracy4 + accuracy5) / 5
    print('total accuracy is: ', total_accuracy)

    avg_accuracy += total_accuracy
    accuracies.append(total_accuracy)

avg_accuracy = avg_accuracy / number_of_folds

avg_precision = precisions / number_of_folds
avg_recall = recalls / number_of_folds
avg_fmeasure = fmeasures / number_of_folds

print('the average accuracy for ', number_of_folds, ' fold is: ', avg_accuracy)
print('the average precision for ', number_of_folds, ' fold is: ', avg_precision)
print('the average recall for ', number_of_folds, ' fold is: ', avg_recall)
print('the average f-measure for ', number_of_folds, ' fold is: ', avg_fmeasure)

print('the maximum accuracy is: ', max(accuracies))

'''
    rand = random.randint(1, 1000)
    p1 = df.sample(frac=0.2, random_state=rand)
    p2 = p1.sample(frac=0.25, random_state=rand)
    p3 = p2.sample(frac=0.333, random_state=rand)
    p4 = p3.sample(frac=0.5, random_state=rand)
    p5 = p3.drop(p4.index)

    train1 = pd.concat([p2, p3, p4, p5])
    test1 = p1
    train2 = pd.concat([p1, p3, p4, p5])
    test2 = p2
    train3 = pd.concat([p1, p2, p4, p5])
    test3 = p3
    train4 = pd.concat([p1, p2, p3, p5])
    test4 = p4
    train5 = pd.concat([p1, p2, p3, p4])
    test5 = p5
    '''


import numpy
from keras.datasets import imdb
from keras.layers import Dense, Flatten, SpatialDropout1D
from keras.layers.embeddings import Embedding
from keras.models import Sequential
from keras.preprocessing.sequence import pad_sequences

seed = 7
numpy.random.seed(seed)
top_words = 5000
(X1_train, Y1_train), (X1_test, Y1_test) = imdb.load_data(num_words=top_words)

max_words = 500
X1_train = pad_sequences(X1_train, maxlen=max_words)
X1_test = pad_sequences(X1_test, maxlen=max_words)

print(X1_train.shape)

model = Sequential()
model.add(Embedding(top_words, 32, input_length=max_words))
model.add(SpatialDropout1D(0.4))
model.add(Flatten())
model.add(Dense(250, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
              optimizer='adam', metrics=['accuracy'])
print(model.summary())

model.fit(X1_train, Y1_train, validation_data=(X1_test, Y1_test), epochs=2, verbose=2)

scores = model.evaluate(X1_test, Y1_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))


import pygame
import random

from board import *
from snake import *
from food import *

pygame.init()

pygame.mixer.init()
devour_sound = pygame.mixer.Sound('sounds/devour.wav')

pygame.display.set_caption('Snake')

clock = pygame.time.Clock()

board = Board(0, 0, 500, 700, 20)
snake = Snake(board)
food = Food(board, snake)

screen = pygame.display.set_mode([500, 700])
snake.screen = screen
snake.devour_sound = devour_sound
food.screen = screen
board.screen = screen

snake.place()
food.place()

game_running = True

while game_running:

    # event loop
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            game_running = False
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_DOWN and prev_y_speed != -1:
                snake.x_speed = 0.
                snake.y_speed = 1.
            if event.key == pygame.K_UP and prev_y_speed != 1:
                snake.x_speed = 0.
                snake.y_speed = -1.
            if event.key == pygame.K_LEFT and prev_x_speed != 1:
                snake.x_speed = -1.
                snake.y_speed = 0.
            if event.key == pygame.K_RIGHT and prev_x_speed != -1:
                snake.x_speed = 1.
                snake.y_speed = 0.

    # game loop
    snake.slither()

    if snake.collided():
        snake.place()

    if food.eaten():
        food.place()
        snake.devour()

    prev_x_speed = snake.x_speed
    prev_y_speed = snake.y_speed

    # draw loop
    screen.fill([0,0,0])

    board.draw()
    snake.draw()
    food.draw()

    pygame.display.flip()

    clock.tick(board.height/board.scale//4)

pygame.quit()


import unittest
import random

from tqdm import tqdm
import IPython
import torch
import numpy as np
import pandas as pd
import pytorch_lightning as pl
from pytorch_lightning import loggers as pl_loggers
from torch.utils.data import DataLoader

from models import BERTModelLightning
from dataset import ToyDataset

def generate_dataset(n=100, isLeft=True, dimension=32):
    outputs, images, sequences = [], [], []
    halved = int(dimension/2)
    if isLeft:
        names= ['left', 'right']
    else:
        names = ['top', 'bottom']
    for idx in range(n):
        input_npy = np.zeros((dimension, dimension))
        if random.random() < 0.5:
            if isLeft:
                input_npy[:,:halved] = np.random.randint(2, size=((dimension,halved)))
            else:
                input_npy[:halved,:] = np.random.randint(2, size=((halved, dimension)))
            true = 0
        else:
            if isLeft:
                input_npy[:,halved:] = np.random.randint(2, size=((dimension,halved)))
            else:
                input_npy[halved:,:] = np.random.randint(2, size=((halved,dimension)))
            true = 1
        for idx_d, direction in enumerate(names):
            images.append(input_npy)
            sequences.append(direction)
            outputs.append(int(idx_d == true))
    return np.stack(images, axis=0), np.array(sequences), np.array(outputs)

def predict(bbm, dataloader, save_path='logs/train_results.csv', device='cpu'):
    bbm = bbm.to(device)
    all_zeros, all_ones, all_outs = [], [], []
    for seq_ids, imgs, outs in dataloader:

        probs = bbm(seq_ids.to(device), imgs.to(device))
        tostore= bbm.pred_softmax(probs).tolist()
        zeros, ones = map(list, zip(*tostore))

        # save
        all_zeros += zeros
        all_ones += ones
        all_outs += outs.argmax(dim=1).tolist()
    df = pd.DataFrame({'prob_zero': all_zeros, 'prob_one': all_ones, 'correct': all_outs})
    df.to_csv(save_path)

class TestAdjConv(unittest.TestCase):
    def test_basic(self):
        dims = [16, 32, 64, 128]
        # ns = [100, 200, 300, 400]
        ns = [400, 400, 400, 400]

        for dim, n in zip(dims, ns):
            bbm = BERTModelLightning([1, 16, 32, 64, 128], output_channels=64, last_size=dim//2, device='cuda')
            bbm.to('cuda')
            train_ds = ToyDataset(*generate_dataset(n, dimension=dim))
            valid_ds = ToyDataset(*generate_dataset(n, dimension=dim))

            train_dl = DataLoader(train_ds, batch_size=16)
            valid_dl = DataLoader(valid_ds, batch_size=16)

            tb_logger = pl_loggers.CSVLogger('logs/', name=f'dim{dim}')
            trainer = pl.Trainer(max_epochs=10, logger=tb_logger, log_every_n_steps=1, gpus=1)

            trainer.fit(bbm, train_dl, valid_dl)

            predict(bbm, train_dl, save_path=f'logs/dim{dim}/train_results.csv', device='cuda')
            predict(bbm, valid_dl, save_path=f'logs/dim{dim}/valid_results.csv', device='cuda')

        # train on top/bottom
        # test_ds = ToyDataset(*generate_dataset(isLeft=False, n=10))
        # test_dl = DataLoader(test_ds, batch_size=1)
        #
        # test_ds2 = ToyDataset(*generate_dataset(isLeft=False, n=50))
        # test_dl2 = DataLoader(test_ds2, batch_size=1)
        #
        # test_logger = pl_loggers.CSVLogger('test_logs/')
        # test_trainer = pl.Trainer(max_epochs=5, logger=test_logger, log_every_n_steps=1)
        # test_trainer.fit(bbm, test_dl, test_dl2)
        # predict(bbm, test_ds, save_path='test_logs/test_results.csv')

if __name__ == "__main__":
    unittest.main()


import openpyxl as px
import warnings

def isFull(dorm):
    if (len(dorm) == 8):
        return True
    else:
        return False

def avg(data_List):
    if len(data_List) == 0:
        return 0
    else:
        return sum(data_List)/len(data_List)
def dorm_assign(Dorm1,Dorm2,Dorm3,age):
    Dorm_Status = [isFull(Dorm1),isFull(Dorm2),isFull(Dorm3)]
    if Dorm_Status == [False,False,False]:
        check_list = [ abs(avg(Dorm1)-age),abs(avg(Dorm2)-age),abs(avg(Dorm3)-age)]
        if max(check_list) == abs(avg(Dorm1)-age):
            Dorm1.append(age)
            return 1
        elif max(check_list) == abs(avg(Dorm2)-age):
            Dorm2.append(age)
            return 2
        else:
            Dorm3.append(age)
            return 3
    elif Dorm_Status == [True,False,False]:
        if abs(avg(Dorm2)-age) > abs(avg(Dorm3)-age):
            Dorm2.append(age)
            return 2
        else:
            Dorm3.append(age)
            return 3
    elif Dorm_Status == [False,True,False]:
        if abs(avg(Dorm1)-age) > abs(avg(Dorm3)-age):
            Dorm1.append(age)
            return 1
        else:
            Dorm3.append(age)
            return 3
    elif Dorm_Status == [False,False,True]:
        if abs(avg(Dorm1)-age) > abs(avg(Dorm2)-age):
            Dorm1.append(age)
            return 1
        else:
            Dorm2.append(age)
            return 2
    elif Dorm_Status == [False,True,True]:
        Dorm1.append(age)
        return 1
    elif Dorm_Status == [True,False,True]:
        Dorm2.append(age)
        return 2
    elif Dorm_Status == [True,True,False]:
        Dorm3.append(age)
        return 3
    elif Dorm_Status == [True,True,True]:
        return 4

def start_assign():
    url = "DD.xlsx"
     # Fill in Data .
    warnings.simplefilter("ignore")
    wb=px.load_workbook(url)
    ws=wb.get_sheet_by_name("Sheet1")
    sheet = wb.active
    DormM1 = []
    DormM2 = []
    DormM3 = []
    DormF1 = []
    DormF2 = []
    DormF3 = []

    nRow = 2
    nlRow = ws.max_row+1
    # Capturing each row of data from columns.
    for row in range(nRow, nlRow):
        # Only targeting Accepted candidates
        status = ws['L' + str(row)].value
        if status=="A":
            band = ws['']
            gender = (ws['E' + str(row)].value)
            age = int((ws['F' + str(row)].value))
            #Dorm assign:
            if gender == 'M':
                dorm_result = dorm_assign(DormM1,DormM2,DormM3,age)
                if dorm_result == 1 :
                    sheet['O' + str(row)] = "M1"
                elif dorm_result == 2:
                    sheet['O' + str(row)] = "M2"
                elif dorm_result == 3:
                    sheet['O' + str(row)] = "M3"
            if gender == 'F':
                dorm_result = dorm_assign(DormF1,DormF2,DormF3,age)
                if dorm_result == 1 :
                    sheet['O' + str(row)] = "F1"
                elif dorm_result == 2:
                    sheet['O' + str(row)] = "F2"
                elif dorm_result == 3:
                    sheet['O' + str(row)] = "F3"
    wb.save(url)


from parse import parse
from itertools import product

instructions = []
with open('day14.txt') as f:
	for l in f.readlines():
		if 'mask' in l:
			instructions.append(parse('{} {} {}', l.rstrip()).fixed)
		else:
			instructions.append(parse('{}[{:d}] = {:d}', l.rstrip()).fixed)

def part_one():
	set_zeros = 0
	set_ones = 0
	memory = {}
	for cmd, idx, value in instructions:
		if cmd == 'mask':
			set_zeros = int(''.join('0' if c == '0' else '1' for c in value), 2)
			set_ones = int(''.join('1' if c == '1' else '0' for c in value), 2)
		else:
			memory[idx] = (value & set_zeros) | set_ones
	return sum(memory.values())

def part_two():
	set_ones = 0
	x_vals = []
	memory = {}
	for cmd, idx, value in instructions:
		if cmd == 'mask':
			set_ones = int(''.join('1' if c == '1' else '0' for c in value), 2)
			x_vals = [2 ** (35 - i) for i, c in enumerate(value) if c == 'X']
		else:
			address = idx | set_ones
			for fl in product((0, 1), repeat=len(x_vals)):
				float_mask = sum(a * b for a, b in zip(fl, x_vals))
				memory[address ^ float_mask] = value

	return sum(memory.values())

print('Part one', part_one())
print('Part two', part_two())


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('yuesao_choice', '0003_yuesaoinfo_company'),
    ]

    operations = [
        migrations.AlterField(
            model_name='yuesaoinfo',
            name='company',
            field=models.ForeignKey(default=True, to='yusao_user.CompanyInfo'),
        ),
    ]


from django.conf.urls import patterns, include, url
import views

urlpatterns = patterns('',
    url(r'^main/',views.main, name="main" ),

    url(r'^maingm/', views.maingm, name="maingm" ),
    url(r'^create/crtskill/$', views.crtSkill, name="createskill"),
    url(r'^create/crtprof/$', views.crtProf, name="createprofession"),
    url(r'^create/crteffect/$', views.crtEff, name="createeffect"),
    url(r'^login/$', views.char_login, name="login"),
    url(r'^logout/$', views.char_logout, name="logout"),
    url(r'^log/$', views.show_log, name="log"),
    url(r'^roll/$', views.roll, name="roll"),
    url(r'^statup/(\w+)/$', views.statup, name="statup"),
)


__author__ = 'binarycat.ru'

import platform
import os
import os.path
import getpass
from datetime import datetime

class Logger(object):
	"""Logger"""

	currentUser = getpass.getuser();

	def __init__(self, logFile, test):
		self.test = test
		self.msg = ""

		nSystem = platform.system().lower()
		if(nSystem == "linux"  or nSystem == "java"):
			nFileName = logFile.replace('~/',os.path.expanduser('~/'))
		else:
			nFileName = logFile

		if logFile is None:
			self.logFile = None
		elif(os.path.exists(nFileName)):
			if(os.path.isdir(nFileName)):
				self.logFile = None
			else:
				self.logFile = nFileName
		else:
			self.logFile = nFileName

		return;

	def LogStart(self):
		self.__LogToFile(self.logFile, "run")

		return;

	def LogEnd(self):
		self.__LogToFile(self.logFile, "finish")

		return;

	def Log(self,msg):
		self.__LogToFile(self.logFile, msg)

		return;

	def __LogToFile(self, logFile, msg):

		msg = str(datetime.now()) + " | user: "+Logger.currentUser+" | " + msg

		if(self.test is True):
			print(msg)
		elif(logFile is None):
			print(msg)

		try:
			mode = 'a' if os.path.exists(logFile) else 'w'
			f = open(logFile, mode)
			f.write(msg+'\n')
			f.close()
		except Exception as e:
			print(str(e))

		return;


# (C) Datadog, Inc. 2019
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import os

import pytest
import requests

from datadog_checks.dev.conditions import CheckEndpoints
from datadog_checks.dev.kube_port_forward import port_forward
from datadog_checks.dev.terraform import terraform_run
from datadog_checks.dev.utils import get_here

try:
    from contextlib import ExitStack
except ImportError:
    from contextlib2 import ExitStack

DEPLOYMENTS = [
    ('istio-citadel', 15014),
    ('istio-galley', 15014),
    ('istio-pilot', 15014),
    ('istio-telemetry', 15014),
    ('istio-telemetry', 42422),
    ('istio-ingressgateway', 80),
]

@pytest.fixture(scope='session')
def dd_environment():
    with terraform_run(os.path.join(get_here(), 'terraform')) as outputs:
        kubeconfig = outputs['kubeconfig']['value']
        with ExitStack() as stack:
            ip_ports = [
                stack.enter_context(port_forward(kubeconfig, 'istio-system', deployment, port))
                for (deployment, port) in DEPLOYMENTS
            ]
            instance = {
                'citadel_endpoint': 'http://{}:{}/metrics'.format(*ip_ports[0]),
                'galley_endpoint': 'http://{}:{}/metrics'.format(*ip_ports[1]),
                'pilot_endpoint': 'http://{}:{}/metrics'.format(*ip_ports[2]),
                'mixer_endpoint': 'http://{}:{}/metrics'.format(*ip_ports[3]),
                'istio_mesh_endpoint': 'http://{}:{}/metrics'.format(*ip_ports[4]),
            }
            page = 'http://{}:{}/productpage'.format(*ip_ports[5])
            # Check a bit to make sure it's available
            CheckEndpoints([page], wait=5)()
            for _ in range(5):
                # Generate some traffic
                requests.get(page)
            yield instance