import unittest

import bitstamp.client
import mock
import requests
import hmac
import hashlib

from .fake_response import FakeResponse

class TradingTests(unittest.TestCase):

    def setUp(self):
        self.username = 'USERNAME'
        self.key = 'KEY'
        self.secret = 'SECRET'
        self.client = bitstamp.client.Trading(
            self.username, self.key, self.secret)

    def test_bad_response(self):
        response = FakeResponse(b'''{"error": "something went wrong"}''')
        with mock.patch('requests.post', return_value=response):
            self.assertRaises(
                bitstamp.client.BitstampError, self.client.account_balance)

    def test_nonjson_response(self):
        response = FakeResponse(b'''Hey wait, this isn't JSON!''')
        with mock.patch('requests.post', return_value=response):
            self.assertRaises(
                bitstamp.client.BitstampError, self.client.account_balance)

    def test_404_response(self):
        response = FakeResponse(status_code=404)
        with mock.patch('requests.post', return_value=response):
            self.assertRaises(requests.HTTPError, self.client.account_balance)

    def test_nonce(self):
        # Each call to .nonce increases it.
        with mock.patch('time.time', return_value=1):
            self.assertEqual(self.client.get_nonce(), 1)
            self.assertEqual(self.client.get_nonce(), 2)
            self.assertEqual(self.client.get_nonce(), 3)
        # But if the unix time is greater, use that instead.
        with mock.patch('time.time', return_value=10):
            self.assertEqual(self.client.get_nonce(), 10)

    def test_500_response(self):
        response = FakeResponse(status_code=500)
        with mock.patch('requests.post', return_value=response):
            self.assertRaises(requests.HTTPError, self.client.account_balance)

    def test_signing(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response) as mocker:
            self.client._post('test')
        kwargs = mocker.call_args[1]
        self.assertIn('data', kwargs)
        self.assertIn('nonce', kwargs['data'])
        self.assertIn('signature', kwargs['data'])
        self.assertEqual(kwargs['data'].get('key'), 'KEY')

    def test_account_balance(self):
        response = FakeResponse(b'''{
            "usd_balance": "100.00",
            "btc_balance": "2.001",
            "usd_reserved": "40",
            "btc_reserved": "1",
            "usd_available": "60.00",
            "btc_available": "1.001"}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.account_balance()
        self.assertIsInstance(result, dict)

    def test_user_transactions(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response):
            result = self.client.user_transactions()
        self.assertIsInstance(result, list)

    def test_open_orders(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response):
            result = self.client.open_orders()
        self.assertIsInstance(result, list)

    def test_all_open_orders(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response):
            result = self.client.all_open_orders()
        self.assertIsInstance(result, list)

    def test_order_status(self):
        response = FakeResponse(b'''{"status": "Open", "transactions": []}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.order_status(0000000000)
        self.assertIsInstance(result, dict)

    def test_bitcoin_deposit_address(self):
        response = FakeResponse(b'"1ARfAEqUzAtbnuJLUxm5KKfDJqrGi27hwA"')
        with mock.patch('requests.post', return_value=response):
            result = self.client.bitcoin_deposit_address()
        self.assertEqual(result, '1ARfAEqUzAtbnuJLUxm5KKfDJqrGi27hwA')

    def test_litecoin_deposit_address(self):
        response = FakeResponse(b'"MVYt6Mw6XHp7UE1gDn6pa8ZbsdtWqWiZUp"')
        with mock.patch('requests.post', return_value=response):
            result = self.client.litecoin_deposit_address()
        self.assertEqual(result, 'MVYt6Mw6XHp7UE1gDn6pa8ZbsdtWqWiZUp')

    def test_ethereum_deposit_address(self):
        response = FakeResponse(b'"0xbd8c4ffcb30c1fed0facf716bc9fd5849539e1c2"')
        with mock.patch('requests.post', return_value=response):
            result = self.client.ethereum_deposit_address()
        self.assertEqual(result, '0xbd8c4ffcb30c1fed0facf716bc9fd5849539e1c2')

    def test_xrp_deposit_address(self):
        response = FakeResponse(b'{"destination_tag": 53965834, "address":"rDsbeamaa4FFwbQTJp9Rs84Q56vCiWCaBx"}')
        with mock.patch('requests.post', return_value=response):
            result = self.client.xrp_deposit_address()
        self.assertDictEqual(result, {u'destination_tag': 53965834, u'address': u'rDsbeamaa4FFwbQTJp9Rs84Q56vCiWCaBx'})

    def test_buy_limit_order(self):
        response = FakeResponse(b'''
            {"amount": "0.1",
             "datetime": "2014-01-22 01:20:23.226788",
             "id": 55507211,
             "price": "90",
             "type": 0}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.buy_limit_order('0.1', '90')
        self.assertIsInstance(result, dict)

    def test_sell_limit_order(self):
        response = FakeResponse(b'''
            {"amount": "1",
             "datetime": "2014-01-22 02:20:23.226788",
             "id": 55507212,
             "price": "9000",
             "type": 1}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.sell_limit_order('1', '9000')
        self.assertIsInstance(result, dict)

    def test_buy_market_order(self):
        response = FakeResponse(b'''
            {"amount": "0.1",
             "datetime": "2014-01-22 01:20:23.226788",
             "id": 55507211,
             "price": "90",
             "type": 0}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.buy_market_order('0.1')
        self.assertIsInstance(result, dict)

    def test_sell_market_order(self):
        response = FakeResponse(b'''
            {"amount": "1",
             "datetime": "2014-01-22 02:20:23.226788",
             "id": 55507212,
             "price": "9000",
             "type": 1}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.sell_market_order('1')
        self.assertIsInstance(result, dict)

    def test_cancel_order(self):
        response = FakeResponse(b'true')
        with mock.patch('requests.post', return_value=response):
            result = self.client.cancel_order('15807214')
        self.assertTrue(result)

    def test_cancel_order_v2(self):
        response = FakeResponse(b'{}')
        with mock.patch('requests.post', return_value=response):
            result = self.client.cancel_order('15807214', version=2)
        self.assertIsInstance(result, dict)

    def test_cancel_all_orders(self):
        response = FakeResponse(b'true')
        with mock.patch('requests.post', return_value=response):
            result = self.client.cancel_all_orders()
        self.assertTrue(result)

    def test_withdrawal_requests(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response):
            result = self.client.withdrawal_requests()
        self.assertIsInstance(result, list)

    def test_unconfirmed_bitcoin_deposits(self):
        response = FakeResponse(b'[]')
        with mock.patch('requests.post', return_value=response):
            result = self.client.unconfirmed_bitcoin_deposits()
        self.assertIsInstance(result, list)

    def test_default_data(self):
        """
        GitHub Issue #9
        """
        ret_data = self.client._default_data()
        msg = str(ret_data['nonce']) + self.username + self.key
        signature = hmac.new(
            self.secret.encode('utf-8'), msg=msg.encode('utf-8'),
            digestmod=hashlib.sha256).hexdigest().upper()
        self.assertEqual(signature, ret_data['signature'])

    def test_bitcoin_withdrawal(self):
        response = FakeResponse(b'''{"id": "1"}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.bitcoin_withdrawal(1, '3J98a1Wpaf73CNmQviecrnyiWrnqRhWNLy')
        self.assertIsInstance(result, dict)

    def test_bch_withdrawal(self):
        response = FakeResponse(b'''{"id": "1"}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.bch_withdrawal(1, '17xu2tUdikA2grgqnVoxQxAKRuDuwmPDjU')
        self.assertIsInstance(result, dict)

    def test_xrp_withdrawal(self):
        response = FakeResponse(b'''{"id": "1"}''')
        with mock.patch('requests.post', return_value=response):
            result = self.client.xrp_withdrawal(1, "rDsbeamaa4FFwbQTJp9Rs84Q56vCiWCaBx")
        self.assertEqual(result, '1')

    def test_transfer_to_main(self):
        response = FakeResponse(b'{"status": "ok"}')
        with mock.patch('requests.post', return_value=response):
            result = self.client.transfer_to_main(1, "btc")
        self.assertIsInstance(result, dict)
        self.assertEqual(result["status"], "ok")

    def test_transfer_from_main(self):
        response = FakeResponse(b'{"status": "ok"}')
        with mock.patch('requests.post', return_value=response):
            result = self.client.transfer_from_main(1, "btc", "")
        self.assertIsInstance(result, dict)
        self.assertEqual(result["status"], "ok")

if __name__ == '__main__':
    unittest.main()


# coding: utf-8

# # Amazon and Rotten Tomatoes Entity Resolution
# ### Adam Coviensky (ac4092), Ian Johnson (icj2103)
# ### Instabase submission: icj2103
# ### GitHub repo: https://github.com/adamianaml/aml-entity-resolution

# Our entity resolution technique involved joining the Amazon and Rotten Tomatoes databases for the ID’s listed in the training set. We then matched the schemas for the features contained in both of the datasets and created new features from these. These features included the absolute value of the difference in time. Boolean features indicating whether or not the directors of the two movies are the same, the runtimes are the same, and whether or not the runtimes are the same within some range (we used 3 minutes). We then created features for the number of matching actors and for the percentage of matching actors. Finally, we performed cross-validation with a customized gridsearch using the training set to determine which features we should be keeping in our final model and which classifier to use between a decision tree, a random forest, and the gradient boosting classifier.
#
# Our model had a precision, recall, and F1-scores of 0.97, 0.97 and 0.96 respectively for the training set. We obtained these scores by using train_test_split on our training set and fitting the model to the training portion and testing it on the testing portion.
#
#
# The most important features were determined to be ‘time_same_2’ which is a binary variable indicating whether the two runtimes are equivalent within a range of 3 minutes. Also, ‘directors_same’, indicated whether the two directors are the same.  ‘num_match_stars’ and ‘percent_match_stars’ indicated the number of matching stars and the percentage of the stars matched from the Amazon dataset respectively. These were the features we used to train our final model.
#
# We avoided pairwise comparison of all movies across both datasets by simply joining the id’s listed in the training set to create a model. We trained a random forest classifier on this training data with our newly created features. This allowed us to predict whether two new movies matched using our trained random forest on a new combination of two movies.

# In[18]:

import re
import pandas as pd
import numpy as np

from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier

np.random.seed(42)

ama = pd.read_csv('./data/amazon.csv')
rt = pd.read_csv('./data/rotten_tomatoes.csv')

holdout = pd.read_csv('./holdout.csv')
test = pd.read_csv('./test.csv')
train = pd.read_csv('./train.csv')

# ### Define Columns

# In[19]:

ama.columns = [
    'id_left',
    'time_left',
    'director_left',
    'star_left',
    'cost_left'
]

rt.columns = [
    'id_right',
    'time_right',
    'director_right',
    'year_right',
    'star1_right',
    'star2_right',
    'star3_right',
    'star4_right',
    'star5_right',
    'star6_right',
    'rotten_tomatoes_right',
    'audience_rating_right',
    'review1_right',
    'review2_right',
    'review3_right',
    'review4_right',
    'review5_right'
]

# ### Feature Creation Function

# In[20]:

def create_features(dataset):
    new_data = pd.DataFrame(dtype=str)

    for index, row in dataset.iterrows():
        amarow = ama[ama['id_left'] == row['id1']]
        rtrow = rt[rt['id_right'] == row['id 2']]

        amarow.reset_index(drop=True, inplace=True)
        rtrow.reset_index(drop=True, inplace=True)

        new_row = pd.concat([amarow, rtrow], axis=1)
        new_data = pd.concat([new_data, new_row])

    new_data.fillna('0', inplace = True)
    new_data.dropna()

    # Compute directors match column
    new_data['directors_same'] = (new_data['director_left'] == new_data['director_right']).astype(int)

    # Compute time columns
    new_data['time_left'] = new_data['time_left'].astype(str)
    new_data['time_right'] = new_data['time_right'].astype(str)
    new_data['time_norm_left'] = new_data['time_left'].apply(compute_time_norm)
    new_data['time_norm_right'] = new_data['time_right'].apply(compute_time_norm)
    new_data['time_same'] = (new_data['time_norm_left'].astype(int) == new_data['time_norm_right'].astype(int)).astype(int)
    new_data['time_same_2'] = (abs((new_data['time_norm_left'].astype(int) - new_data['time_norm_right'].astype(int)).astype(int)) <= 3).astype(int)
    new_data['time_diff'] = (new_data['time_norm_left'].astype(int) - new_data['time_norm_right'].astype(int)).astype(int)
    new_data['time_diff_abs'] = abs(new_data['time_diff']).astype(int)

    # Compute actors columns
    actors_split = new_data['star_left'].str.split(', ', expand=True)
    for i in range(6 - actors_split.shape[1]):
        actors_split[str(i)] = ""
    actors_split.columns = ['star_' + str(i) for i in range(6)]
    new_data = pd.concat([new_data, actors_split], axis=1)

    # Number actors match
    cols = list(new_data.loc[:,'star_0':'star_4']) + list(new_data.loc[:,'star1_right':'star6_right'])
    new_data['num_match_stars'] = new_data[cols].apply(compute_number_stars_match, axis = 1)

    # Percent actors match
    new_data['percent_match_stars'] = new_data[cols].apply(compute_percent_stars_match, axis = 1)

    return new_data

# ### Helper Feature Creation Functions

# In[21]:

def compute_number_stars_match(row):
    actors_left = ['star_0', 'star_1', 'star_2', 'star_3', 'star_4']
    actors_right = ['star1_right', 'star2_right', 'star3_right', 'star4_right', 'star5_right', 'star6_right']
    list_left = row.loc[actors_left].tolist()
    list_right = row.loc[actors_right].tolist()

    # Avoid matching nulls
    list_left = filter(None, list_left)
    list_right = filter(None, list_right)

    return len(np.intersect1d(list_left, list_right))

def compute_percent_stars_match(row):
    actors_left = ['star_0', 'star_1', 'star_2', 'star_3', 'star_4']
    actors_right = ['star1_right', 'star2_right', 'star3_right', 'star4_right', 'star5_right', 'star6_right']
    list_left = row.loc[actors_left].tolist()
    list_left = filter(None, list_left)
    list_right = row.loc[actors_right].tolist()
    x = float(len(np.intersect1d(list_left, list_right)))
    ama_num = len(list_left)
    return x / ama_num

regex = re.compile(r'[0-9]*')

def compute_time_norm(row):
    row = str(row)

    if '/' in row:
        # Invalid time entry
        print(row)
        return 0

    match = regex.findall(row)
    temp = filter(None, match)
    if len(temp) == 2:
        return 60*int(temp[0]) + int(temp[1])
    if len(temp) == 1:
        return temp[0]

def remove_bad_samples(data):

    train.reset_index(drop=True, inplace=True)
    data.reset_index(drop=True, inplace=True)
    data = pd.concat([data, train], axis=1)

    # Remove bad training rows
    data = data[data['id_left'] != 199]
    data = data[data['id_left'] != 680]
    data = data[data['id_left'] != 487]
    data = data[data['id_left'] != 756]
    data = data[data['id_left'] != 770]
    data = data[data['id_left'] != 1701]

    return data

# ### Predict With Training Data

# In[23]:

features_cols = [
    'time_same_2',
    'directors_same',
    'num_match_stars',
    'percent_match_stars',
]

train_data = create_features(train)
train_data = remove_bad_samples(train_data)

x_train, x_test, y_train, y_test = train_test_split(train_data[features_cols], train_data['gold'], random_state=10)

clf = RandomForestClassifier(random_state=42)
clf.fit(x_train, y_train)
scores = clf.score(x_test, y_test)

preds = clf.predict(x_test)

# F1 score:
from sklearn.metrics import classification_report
print(classification_report(y_test, preds))

# ### Final Predictions

# In[24]:

features_cols = [
    'time_same_2',
    'directors_same',
    'num_match_stars',
    'percent_match_stars',
]

# Test Predictions
train_data = create_features(train)
train_data = remove_bad_samples(train_data)
test_data = create_features(test)

clf = RandomForestClassifier()
clf.fit(train_data[features_cols], train_data['gold'])
test_preds = clf.predict(test_data[features_cols])

test_preds = pd.DataFrame(test_preds)
test_preds.columns = ['gold']
test_preds.to_csv('test_gold.csv', index=False)

# Holdout Predictions
train_data = create_features(train)
train_data = remove_bad_samples(train_data)
holdout_data = create_features(holdout)

clf = RandomForestClassifier()
clf.fit(train_data[features_cols], train_data['gold'])
holdout_preds = clf.predict(holdout_data[features_cols])

holdout_preds = pd.DataFrame(holdout_preds)
holdout_preds.columns = ['gold']
holdout_preds.to_csv('holdout_gold.csv', index=False)

# In[ ]:


import os
import pulumi
import pulumi_aws as aws
import json
from pulumi_terraform import state

config = pulumi.Config()
statefile = config.require("statefile")
script_dir = os.path.dirname(os.path.realpath(__file__))

# read outputs from Terraform
s = state.RemoteStateReference(
    "localstate",
    "local",
    state.LocalBackendArgs(path=os.path.join(f"{script_dir}/../tf", statefile)),
)

terraform_owned = aws.ec2.Vpc(
    "terraform_owned",
    cidr_block="10.0.0.0/16",
    enable_dns_hostnames=True,
    tags={
        "Name": "lbriggs",
    },
    opts=pulumi.ResourceOptions(protect=True),
)

# return the outputs from the terraform state
vpc_id = s.get_output("vpc_id")
private_subnet_ids = s.get_output("private_subnets")
public_subnet_ids = s.get_output("public_subnets")

pulumi.export("vpcId", vpc_id)
pulumi.export("publicSubnetIds", private_subnet_ids)
pulumi.export("privateSubnetIds", public_subnet_ids)

#
# Time to use Pulumi!
#

# Create an ECS cluster to run a container-based service.
cluster = aws.ecs.Cluster("cluster")

# create a security group referencing the VPC
group = aws.ec2.SecurityGroup(
    "web-secgrp",
    vpc_id=vpc_id,
    description="Enable HTTP access",
    ingress=[
        aws.ec2.SecurityGroupIngressArgs(
            protocol="tcp",
            from_port=80,
            to_port=80,
            cidr_blocks=["0.0.0.0/0"],
        )
    ],
    egress=[
        aws.ec2.SecurityGroupEgressArgs(
            protocol="-1",
            from_port=0,
            to_port=0,
            cidr_blocks=["0.0.0.0/0"],
        )
    ],
)

# Create a load balancer to listen for HTTP traffic on port 80.
# Reads the subnet IDs from Terraform state
alb = aws.lb.LoadBalancer(
    "app-lb",
    security_groups=[group.id],
    subnets=public_subnet_ids,
)

# create a target group
atg = aws.lb.TargetGroup(
    "app-tg",
    port=80,
    protocol="HTTP",
    target_type="ip",
    vpc_id=vpc_id,
)

# create a listener
wl = aws.lb.Listener(
    "web",
    load_balancer_arn=alb.arn,
    port=80,
    default_actions=[
        aws.lb.ListenerDefaultActionArgs(
            type="forward",
            target_group_arn=atg.arn,
        )
    ],
)

assume_policy_document = aws.iam.get_policy_document(
    statements=[
        aws.iam.GetPolicyDocumentStatementArgs(
            sid="AssumeRolePolicy",
            effect="Allow",
            principals=[
                aws.iam.GetPolicyDocumentStatementPrincipalArgs(
                    type="Service",
                    identifiers=["ecs-tasks.amazonaws.com"],
                )
            ],
            actions=["sts:AssumeRole"],
        )
    ]
)

# Create an IAM role that can be used by our service's task.
role = aws.iam.Role(
    "task-exec-role",
    assume_role_policy=assume_policy_document.json,
)

rpa = aws.iam.RolePolicyAttachment(
    "task-exec-policy",
    role=role.name,
    policy_arn="arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
)

# Spin up a load balanced service running our container image.
task_definition = aws.ecs.TaskDefinition(
    "app-task",
    family="terraform-state-demo",
    cpu="256",
    memory="512",
    network_mode="awsvpc",
    requires_compatibilities=["FARGATE"],
    execution_role_arn=role.arn,
    container_definitions=json.dumps(
        [
            {
                "name": "nginx",
                "image": "nginx",
                "portMappings": [
                    {"containerPort": 80, "hostPort": 80, "protocol": "tcp"}
                ],
            }
        ]
    ),
)

service = aws.ecs.Service(
    "app-svc",
    cluster=cluster.arn,
    desired_count=3,
    launch_type="FARGATE",
    task_definition=task_definition.arn,
    network_configuration=aws.ecs.ServiceNetworkConfigurationArgs(
        assign_public_ip=True,
        subnets=private_subnet_ids,
        security_groups=[group.id],
    ),
    load_balancers=[
        aws.ecs.ServiceLoadBalancerArgs(
            target_group_arn=atg.arn,
            container_name="nginx",
            container_port=80,
        )
    ],
    opts=pulumi.ResourceOptions(depends_on=[wl]),
)
pulumi.export("url", alb.dns_name)


from typing import Dict, List

from aws_cdk import aws_ec2 as ec2
from aws_cdk import aws_logs as logs
from aws_cdk.core import CfnTag, Fn, NestedStack, Stack
from constructs import Construct

from pcluster.aws.aws_api import AWSApi
from pcluster.config.cluster_config import SharedStorageType, SlurmClusterConfig, SlurmComputeResource, SlurmQueue
from pcluster.constants import (
    DEFAULT_EPHEMERAL_DIR,
    NODE_BOOTSTRAP_TIMEOUT,
    OS_MAPPING,
    PCLUSTER_COMPUTE_RESOURCE_NAME_TAG,
    PCLUSTER_QUEUE_NAME_TAG,
)
from pcluster.templates.cdk_builder_utils import (
    CdkLaunchTemplateBuilder,
    ComputeNodeIamResources,
    create_hash_suffix,
    dict_to_cfn_tags,
    get_common_user_data_env,
    get_custom_tags,
    get_default_instance_tags,
    get_default_volume_tags,
    get_queue_security_groups_full,
    get_shared_storage_ids_by_type,
    get_user_data_content,
    scheduler_is_slurm,
    to_comma_separated_string,
)
from pcluster.templates.slurm_builder import SlurmConstruct
from pcluster.utils import get_attr, get_http_tokens_setting

class QueuesStack(NestedStack):
    """Stack encapsulating a set of queues and the associated resources."""

    def __init__(
        self,
        scope: Construct,
        id: str,
        queues: List[SlurmQueue],
        slurm_construct: SlurmConstruct,
        cluster_config: SlurmClusterConfig,
        log_group: logs.CfnLogGroup,
        shared_storage_infos: Dict,
        shared_storage_mount_dirs: Dict,
        shared_storage_attributes: Dict,
        compute_security_group,
        cluster_hosted_zone,
        dynamodb_table,
        head_eni,
    ):
        super().__init__(scope, id)
        self._queues = queues
        self._slurm_construct = slurm_construct
        self._config = cluster_config
        self._shared_storage_infos = shared_storage_infos
        self._shared_storage_mount_dirs = shared_storage_mount_dirs
        self._shared_storage_attributes = shared_storage_attributes
        self._compute_security_group = compute_security_group
        self._log_group = log_group
        self._cluster_hosted_zone = cluster_hosted_zone
        self._dynamodb_table = dynamodb_table
        self._head_eni = head_eni
        self._launch_template_builder = CdkLaunchTemplateBuilder()
        self._add_resources()

    @staticmethod
    def _get_placement_group_for_compute_resource(queue, managed_placement_groups, compute_resource) -> str:
        placement_group_settings = queue.get_placement_group_settings_for_compute_resource(compute_resource)
        placement_group_key = placement_group_settings.get("key")
        managed = placement_group_settings.get("is_managed")
        return managed_placement_groups[placement_group_key].ref if managed else placement_group_key

    @property
    def stack_name(self):
        """Name of the CFN stack."""
        return Stack.of(self.nested_stack_parent).stack_name

    def _add_resources(self):
        self._add_compute_iam_resources()
        self._add_placement_groups()
        self._add_launch_templates()

    def _add_placement_groups(self):
        self.managed_placement_groups = {}
        for queue in self._queues:
            for key in queue.get_managed_placement_group_keys():
                self.managed_placement_groups[key] = ec2.CfnPlacementGroup(
                    self,
                    f"PlacementGroup{create_hash_suffix(key)}",
                    strategy="cluster",
                )

    def _add_compute_iam_resources(self):
        iam_resources = {}
        for queue in self._queues:
            iam_resources[queue.name] = ComputeNodeIamResources(
                self,
                f"ComputeNodeIamResources{queue.name}",
                self._config,
                queue,
                self._shared_storage_infos,
                queue.name,
            )
        self._compute_instance_profiles = {k: v.instance_profile for k, v in iam_resources.items()}
        self.managed_compute_instance_roles = {k: v.instance_role for k, v in iam_resources.items()}
        if scheduler_is_slurm(self._config):
            self._slurm_construct.register_policies_with_role(
                scope=Stack.of(self),
                managed_compute_instance_roles=self.managed_compute_instance_roles,
            )

    def _add_launch_templates(self):
        self.compute_launch_templates = {}
        for queue in self._queues:
            self.compute_launch_templates[queue.name] = {}
            queue_lt_security_groups = get_queue_security_groups_full(self._compute_security_group, queue)

            for resource in queue.compute_resources:
                self.compute_launch_templates[queue.name][resource.name] = self._add_compute_resource_launch_template(
                    queue,
                    resource,
                    queue_lt_security_groups,
                    self._get_placement_group_for_compute_resource(queue, self.managed_placement_groups, resource),
                    self._compute_instance_profiles,
                    self._config.is_detailed_monitoring_enabled,
                )

    def _get_custom_compute_resource_tags(self, queue_config, compute_resource_config):
        """Compute resource tags and Queue Tags value on Cluster level tags if there are duplicated keys."""
        tags = get_custom_tags(self._config, raw_dict=True)
        queue_tags = get_custom_tags(queue_config, raw_dict=True)
        compute_resource_tags = get_custom_tags(compute_resource_config, raw_dict=True)
        return dict_to_cfn_tags({**tags, **queue_tags, **compute_resource_tags})

    def _add_compute_resource_launch_template(
        self,
        queue,
        compute_resource,
        queue_lt_security_groups,
        placement_group,
        instance_profiles,
        is_detailed_monitoring_enabled,
    ):
        # LT network interfaces
        compute_lt_nw_interfaces = [
            ec2.CfnLaunchTemplate.NetworkInterfaceProperty(
                device_index=0,
                associate_public_ip_address=queue.networking.assign_public_ip,
                interface_type="efa" if compute_resource.efa and compute_resource.efa.enabled else None,
                groups=queue_lt_security_groups,
                subnet_id=queue.networking.subnet_ids[0]
                if isinstance(compute_resource, SlurmComputeResource)
                else None,
            )
        ]

        for network_interface_index in range(1, compute_resource.max_network_interface_count):
            compute_lt_nw_interfaces.append(
                ec2.CfnLaunchTemplate.NetworkInterfaceProperty(
                    device_index=0,
                    network_card_index=network_interface_index,
                    associate_public_ip_address=False,
                    interface_type="efa" if compute_resource.efa and compute_resource.efa.enabled else None,
                    groups=queue_lt_security_groups,
                    subnet_id=queue.networking.subnet_ids[0]
                    if isinstance(compute_resource, SlurmComputeResource)
                    else None,
                )
            )

        conditional_template_properties = {}
        if compute_resource.is_ebs_optimized:
            conditional_template_properties.update({"ebs_optimized": True})
        if isinstance(compute_resource, SlurmComputeResource):
            conditional_template_properties.update({"instance_type": compute_resource.instance_types[0]})

        return ec2.CfnLaunchTemplate(
            self,
            f"LaunchTemplate{create_hash_suffix(queue.name + compute_resource.name)}",
            launch_template_name=f"{self.stack_name}-{queue.name}-{compute_resource.name}",
            launch_template_data=ec2.CfnLaunchTemplate.LaunchTemplateDataProperty(
                block_device_mappings=self._launch_template_builder.get_block_device_mappings(
                    queue.compute_settings.local_storage.root_volume,
                    AWSApi.instance().ec2.describe_image(self._config.image_dict[queue.name]).device_name,
                ),
                # key_name=,
                network_interfaces=compute_lt_nw_interfaces,
                placement=ec2.CfnLaunchTemplate.PlacementProperty(group_name=placement_group),
                image_id=self._config.image_dict[queue.name],
                iam_instance_profile=ec2.CfnLaunchTemplate.IamInstanceProfileProperty(
                    name=instance_profiles[queue.name]
                ),
                instance_market_options=self._launch_template_builder.get_instance_market_options(
                    queue, compute_resource
                ),
                instance_initiated_shutdown_behavior="terminate",
                capacity_reservation_specification=self._launch_template_builder.get_capacity_reservation(
                    queue,
                    compute_resource,
                ),
                metadata_options=ec2.CfnLaunchTemplate.MetadataOptionsProperty(
                    http_tokens=get_http_tokens_setting(self._config.imds.imds_support)
                ),
                user_data=Fn.base64(
                    Fn.sub(
                        get_user_data_content("../resources/compute_node/user_data.sh"),
                        {
                            **{
                                "EnableEfa": "efa" if compute_resource.efa and compute_resource.efa.enabled else "NONE",
                                "RAIDSharedDir": to_comma_separated_string(
                                    self._shared_storage_mount_dirs[SharedStorageType.RAID]
                                ),
                                "RAIDType": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.RAID]["Type"]
                                ),
                                "DisableMultiThreadingManually": "true"
                                if compute_resource.disable_simultaneous_multithreading_manually
                                else "false",
                                "BaseOS": self._config.image.os,
                                "SharedStorageType": self._config.head_node.shared_storage_type.lower(),  # noqa: E501  pylint: disable=line-too-long
                                "EFSIds": get_shared_storage_ids_by_type(
                                    self._shared_storage_infos, SharedStorageType.EFS
                                ),
                                "EFSSharedDirs": to_comma_separated_string(
                                    self._shared_storage_mount_dirs[SharedStorageType.EFS]
                                ),
                                "EFSEncryptionInTransits": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.EFS]["EncryptionInTransits"],
                                    use_lower_case=True,
                                ),
                                "EFSIamAuthorizations": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.EFS]["IamAuthorizations"],
                                    use_lower_case=True,
                                ),
                                "FSXIds": get_shared_storage_ids_by_type(
                                    self._shared_storage_infos, SharedStorageType.FSX
                                ),
                                "FSXMountNames": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.FSX]["MountNames"]
                                ),
                                "FSXDNSNames": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.FSX]["DNSNames"]
                                ),
                                "FSXVolumeJunctionPaths": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.FSX]["VolumeJunctionPaths"]
                                ),
                                "FSXFileSystemTypes": to_comma_separated_string(
                                    self._shared_storage_attributes[SharedStorageType.FSX]["FileSystemTypes"]
                                ),
                                "FSXSharedDirs": to_comma_separated_string(
                                    self._shared_storage_mount_dirs[SharedStorageType.FSX]
                                ),
                                "Scheduler": self._config.scheduling.scheduler,
                                "EphemeralDir": queue.compute_settings.local_storage.ephemeral_volume.mount_dir
                                if isinstance(queue, SlurmQueue)
                                and queue.compute_settings.local_storage.ephemeral_volume
                                else DEFAULT_EPHEMERAL_DIR,
                                "EbsSharedDirs": to_comma_separated_string(
                                    self._shared_storage_mount_dirs[SharedStorageType.EBS]
                                ),
                                "ClusterDNSDomain": str(self._cluster_hosted_zone.name)
                                if self._cluster_hosted_zone
                                else "",
                                "ClusterHostedZone": str(self._cluster_hosted_zone.ref)
                                if self._cluster_hosted_zone
                                else "",
                                "OSUser": OS_MAPPING[self._config.image.os]["user"],
                                "ClusterName": self.stack_name,
                                "SlurmDynamoDBTable": self._dynamodb_table.ref if self._dynamodb_table else "NONE",
                                "LogGroupName": self._log_group.log_group_name
                                if self._config.monitoring.logs.cloud_watch.enabled
                                else "NONE",
                                "IntelHPCPlatform": "true" if self._config.is_intel_hpc_platform_enabled else "false",
                                "CWLoggingEnabled": "true" if self._config.is_cw_logging_enabled else "false",
                                "LogRotationEnabled": "true" if self._config.is_log_rotation_enabled else "false",
                                "QueueName": queue.name,
                                "ComputeResourceName": compute_resource.name,
                                "EnableEfaGdr": "compute"
                                if compute_resource.efa and compute_resource.efa.gdr_support
                                else "NONE",
                                "CustomNodePackage": self._config.custom_node_package or "",
                                "CustomAwsBatchCliPackage": self._config.custom_aws_batch_cli_package or "",
                                "ExtraJson": self._config.extra_chef_attributes,
                                "UsePrivateHostname": str(
                                    get_attr(self._config, "scheduling.settings.dns.use_ec2_hostnames", default=False)
                                ).lower(),
                                "HeadNodePrivateIp": self._head_eni.attr_primary_private_ip_address,
                                "DirectoryServiceEnabled": str(self._config.directory_service is not None).lower(),
                                "Timeout": str(
                                    get_attr(
                                        self._config,
                                        "dev_settings.timeouts.compute_node_bootstrap_timeout",
                                        NODE_BOOTSTRAP_TIMEOUT,
                                    )
                                ),
                                "ComputeStartupTimeMetricEnabled": str(
                                    get_attr(
                                        self._config,
                                        "dev_settings.compute_startup_time_metric_enabled",
                                        default=False,
                                    )
                                ),
                            },
                            **get_common_user_data_env(queue, self._config),
                        },
                    )
                ),
                monitoring=ec2.CfnLaunchTemplate.MonitoringProperty(enabled=is_detailed_monitoring_enabled),
                tag_specifications=[
                    ec2.CfnLaunchTemplate.TagSpecificationProperty(
                        resource_type="instance",
                        tags=get_default_instance_tags(
                            self.stack_name, self._config, compute_resource, "Compute", self._shared_storage_infos
                        )
                        + [CfnTag(key=PCLUSTER_QUEUE_NAME_TAG, value=queue.name)]
                        + [CfnTag(key=PCLUSTER_COMPUTE_RESOURCE_NAME_TAG, value=compute_resource.name)]
                        + self._get_custom_compute_resource_tags(queue, compute_resource),
                    ),
                    ec2.CfnLaunchTemplate.TagSpecificationProperty(
                        resource_type="volume",
                        tags=get_default_volume_tags(self.stack_name, "Compute")
                        + [CfnTag(key=PCLUSTER_QUEUE_NAME_TAG, value=queue.name)]
                        + [CfnTag(key=PCLUSTER_COMPUTE_RESOURCE_NAME_TAG, value=compute_resource.name)]
                        + self._get_custom_compute_resource_tags(queue, compute_resource),
                    ),
                ],
                **conditional_template_properties,
            ),
        )


from typing import cast

from meiga import (
    Error,
    Result,
)
from PyQt5.QtGui import (
    QIcon,
)
from PyQt5.QtWidgets import (
    QMessageBox,
    QWidget,
)

from jenot import (
    logo,
    PollResult,
)
from jenot.args import Args
import jenot.notify as jnotify
from jenot.qt.processor import Processor

class MainWidget(QWidget):
    args: Args
    processors: list[Processor] = []

    def __init__(self) -> None:
        super().__init__()
        try:
            self.args = Args.parse(build_required=False)
        except SystemExit:
            QMessageBox.critical(None, 'Startup failed',
                "Could not parse some of required parameters, check your .jenotrc")
            raise
        self.jenot_icon = QIcon(logo('png'))

    def notify(self, result: Result[PollResult, Error], url: str) -> None:
        # have to store msgbox somewhere so python won't dispose of it
        self.msgbox = jnotify.qt(url, result)
        self.msgbox.show()

        if self.args.telegram:
            jnotify.telegram(url, result)

    def register(self, p: Processor) -> None:
        self.processors.append(p)
        p.finished.connect(self.notify)
        p.finished.connect(lambda: self.unregister(p))
        p.start()

    def unregister(self, p: Processor) -> None:
        self.processors.remove(p)


import os, logging
import unittest

import rmt.rmt

this_dir   = os.path.dirname(__file__)
models_dir = os.path.join( os.path.join(this_dir, os.pardir), 'sample', 'models') # i.e.: cd ../sample/models

logger = logging.getLogger(None) # get the "root" logger

class SampleRobotModel:
    '''
    Utility base class to load models contained in the `sample/` folder
    '''

    def __init__(self, name, extension):
        '''
        - `name`: the name of the robot model to load
        - `extension`: the model file extension, e.g. "kindsl", "urdf", ...

        This function will try to load the model `sample/<name>/<name>.<extension>`
        '''
        rpath = os.path.join( name, name+'.'+extension ) # <name>/<name>.<extension>
        rfile = os.path.join( models_dir, rpath )
        logger.debug("Trying to load {0}".format(rfile))
        self.connectivity, self.ordering, self.frames, self.geometry, self.inertia, _ = rmt.rmt.getmodels(rfile)

class RobotTestBase(unittest.TestCase):
    def setUp(self):
        '''
        Expect robot-data to be in the class, and copy a reference to the
        fields in the instance (i.e. self). This way the test code can access
        the robot model data and ground-truth data via self.
        '''
        cls = self.__class__
        self.robot = cls.robotdata.ordering
        self.connectivity = cls.robotdata.connectivity
        self.frames = cls.robotdata.frames
        self.treeu = cls.robotdata.treeutils
        self.groundtruth  = cls.robotdata.groundtruth

def main():
    formatter = logging.Formatter('%(levelname)s (%(name)s) : %(message)s')
    handler   = logging.StreamHandler()
    handler.setFormatter(formatter)
    logger.setLevel(logging.INFO)
    logger.addHandler(handler)
    unittest.main()


# name=Import field data
# displayinmenu=true
# displaytouser=true
# displayinselector=true

from monitoring import importdata
import toolbox as tb
import toolbox.util as tbu
from voluptuous import Schema, Optional

class ImportTool(tb.Tool):
    schema = Schema({
        'folder': unicode,
        'files': [
            unicode
        ],
        'site': unicode,
        'version': unicode,
        'columns': {
            'date': {
                'title': unicode,
                'format': unicode
            },
            Optional('time'): {
                'title': unicode
            },
            'location': {
                'title': unicode
            }
        },
        'mapping': {
            unicode: unicode
        },
        'params': {
            unicode: {
                'unit': unicode
            }
        }
    }, required=True)

    refreshCatalogue = 1

    def main(self):
        records = importdata.locationsDown(self.config)
        imported = tbu.saveRecords(records, self.dssFilePath)
        self.message = "{} Records imported.".format(imported)

tool = ImportTool()
tool.run()


import argparse
import time
import math
import numpy as np
import pickle
import random

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import torch.autograd as autograd
from torch.autograd import Variable

from utils import to_gpu, LoadEmbeddingsFromText, Corpus, batchify, truncate
from models import LanguageModel, RNNDiscriminator

parser = argparse.ArgumentParser(description='PyTorch Sequence GAN for Text')
parser.add_argument('--data', type=str, default='./snli_lm',
                    help='location of the data corpus')
parser.add_argument('--rnn_type', type=str, default='LSTM',
                    help='type of recurrent net (LSTM, GRU)')
parser.add_argument('--embedding_file', type=str, default='',
                    help='path to pretrained embedding file')
parser.add_argument('--emsize', type=int, default=50,
                    help='size of word embeddings')
parser.add_argument('--nhidden', type=int, default=50,
                    help='number of hidden units per layer')
parser.add_argument('--nlayers', type=int, default=1,
                    help='number of layers')
parser.add_argument('--lowercase', type=bool,  default=True,
                    help='whether to lowercase')
parser.add_argument('--k', type=int, default=1,
                    help='number of layers')
parser.add_argument('--pretrain-lr', type=float, default=20,
                    help='initial learning rate')
parser.add_argument('--lr', type=float, default=0.01,
                    help='initial learning rate')
parser.add_argument('--beta1', type=float, default=0.5,
                    help='beta1 for adam. default=0.5')
parser.add_argument('--clip', type=float, default=0.25,
                    help='gradient clipping')
parser.add_argument('--pretrain-epochs', type=int, default=1,
                    help='pretraining epochs limit')
parser.add_argument('--epochs', type=int, default=2,
                    help='upper epoch limit')
parser.add_argument('--pretrain-batch_size', type=int, default=80, metavar='N',
                    help='batch size')
parser.add_argument('--batch_size', type=int, default=100, metavar='N',
                    help='batch size')
parser.add_argument('--bidirectional', action='store_true',
                    help='bidirectional discriminator')
parser.add_argument('--dropout', type=float, default=0.2,
                    help='dropout applied to layers (0 = no dropout)')
parser.add_argument('--tied', action='store_true',
                    help='tie the word embedding and softmax weights')
parser.add_argument('--seed', type=int, default=111,
                    help='random seed')
parser.add_argument('--pretrain', action='store_true',
                    help='pretrains the generator on a language model objective')
parser.add_argument('--reinforce', action='store_true',
                    help='use REINFORCE log-likelihood reward instead of Maximum Likelihood augmented reward')
parser.add_argument('--cuda', action='store_true',
                    help='use CUDA')
parser.add_argument('--log-interval', type=int, default=200, metavar='N',
                    help='report interval')
parser.add_argument('--save', type=str,  default='netG.pt',
                    help='path to save the final model')
parser.add_argument('--outf', type=str, default='.')
parser.add_argument('--netG', default='', help="path to netG (to continue training)")
parser.add_argument('--netD', default='', help="path to netD (to continue training)")
parser.add_argument('--sample', action='store_true',\
                    help='will sample for generation')
parser.add_argument('--temp', type=float, default=1.0,\
                    help='softmax temperaturen')
args = parser.parse_args()

with open(args.outf+'/args.txt', 'w') as f:
    f.write(str(args))

# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
    if not args.cuda:
        print("WARNING: You have a CUDA device, so you should probably run with --cuda")
    else:
        torch.cuda.manual_seed(args.seed)

###############################################################################
# Load data
###############################################################################

corpus = Corpus(args.data, lowercase=args.lowercase)

eval_batch_size = 10
val_data = batchify(corpus.valid, eval_batch_size, max_len=150)
test_data = batchify(corpus.test, eval_batch_size, max_len=150)

print("Loaded data!")

###############################################################################
# Build the generator
###############################################################################

ntokens = len(corpus.dictionary.word2idx)

if args.embedding_file == "":
    vocab = None
else:
    # Load pretrained word embeddings
    vocab = LoadEmbeddingsFromText(corpus.word2idx, embedding_dim=args.emsize, path=args.embedding_file)

netG = LanguageModel(rnn_type=args.rnn_type,
                     emsize=args.emsize,
                     nhidden=args.nhidden,
                     ntokens=ntokens,
                     nlayers=args.nlayers,
                     dropout=args.dropout,
                     tied=args.tied,
                     max_unroll=50,
                     initial_vocab=vocab,
                     gpu=args.cuda)
if args.netG != "":
    print('Loading generator from '+args.netG)
    netG.load_state_dict(torch.load(args.netG))

print(netG.max_unroll)

if args.cuda:
    netG.cuda()

lm_criterion = nn.CrossEntropyLoss()

###############################################################################
# Pretraining code
###############################################################################

def lm_train():
    # Turn on training mode which enables dropout.
    netG.train()
    total_loss = 0
    start_time = time.time()
    ntokens = len(corpus.dictionary.word2idx)
    vocab = corpus.dictionary.idx2word

    for i, batch in enumerate(train_data):
        source, target, lengths = batch
        source = to_gpu(args.cuda, Variable(source))
        target = to_gpu(args.cuda, Variable(target))

        # Create sentence length mask over padding
        mask = target.gt(0)
        masked_target = target.masked_select(mask)
        flat_mask = mask.view(-1, 1)
        output_mask = flat_mask.view(-1, 1).expand(flat_mask.size(0), ntokens) # examples x ntokens

        netG.zero_grad()
        output = netG(teacher_forcing=True, input_=source) # output: batch x seq_len x ntokens
        flattened_output = output.view(-1, ntokens)

        masked_output = flattened_output.masked_select(output_mask).view(-1, ntokens)
        loss = lm_criterion(masked_output, masked_target)
        loss.backward()

        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
        torch.nn.utils.clip_grad_norm(netG.parameters(), args.clip)
        for p in netG.parameters():
            p.data.add_(-lr, p.grad.data)

        total_loss += loss.data

        if i % args.log_interval == 0 and i > 0:
            cur_loss = total_loss[0] / args.log_interval
            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.2f} | ms/batch {:5.2f} | '
                    'loss {:5.2f} | ppl {:8.2f}'.format(
                epoch, i, len(train_data), lr,
                elapsed * 1000 / args.log_interval, cur_loss, math.exp(cur_loss)))
            total_loss = 0
            start_time = time.time()

    return total_loss

def lm_evaluate(data_source):
    # Turn on evaluation mode which disables dropout.
    netG.eval()
    total_loss = 0
    ntokens = len(corpus.dictionary.word2idx)
    vocab = corpus.dictionary.idx2word
    for i, batch in enumerate(data_source):
        source, target, lengths = batch
        source = to_gpu(args.cuda, Variable(source, volatile=True))
        target = to_gpu(args.cuda, Variable(target, volatile=True))

        mask = target.gt(0)
        masked_target = target.masked_select(mask)
        flat_mask = mask.view(-1, 1)
        output_mask = flat_mask.view(-1, 1).expand(flat_mask.size(0), ntokens) # examples x ntokens

        netG.zero_grad()
        output = netG(teacher_forcing=True, input_=source) # output: batch x seq_len x ntokens
        flattened_output = output.view(-1, ntokens)

        masked_output = flattened_output.masked_select(output_mask).view(-1, ntokens)

        total_loss += lm_criterion(masked_output, masked_target).data

    return total_loss[0] / len(data_source)

# Loop over epochs.
lr = args.pretrain_lr
best_val_loss = None
stats = []

if args.pretrain:
    print("Pretraining generator with language model objective...")
    # At any point you can hit Ctrl + C to break out of training early.
    try:
        for epoch in range(1, args.pretrain_epochs+1):
            train_data = batchify(corpus.train, args.pretrain_batch_size, shuffle=True, max_len=120)
            epoch_start_time = time.time()
            train_loss = lm_train()
            val_loss = lm_evaluate(val_data)
            print('-' * 89)
            print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '
                    'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),
                                            val_loss, math.exp(val_loss)))
            print('-' * 89)
            stats.append((epoch, train_loss, val_loss, math.exp(val_loss)))
            # Save the model if the validation loss is the best we've seen so far.
            if not best_val_loss or val_loss < best_val_loss:
                print("Saving with best val loss", val_loss)
                with open(args.save, 'wb') as f:
                    torch.save(netG.state_dict(), f)
                best_val_loss = val_loss
            else:
                # Anneal the learning rate if no improvement has been seen in the validation dataset.
                lr *= 0.25
            pickle.dump(stats, open("stats_"+args.save[:-3]+".p", "wb"))

    except KeyboardInterrupt:
        print('-' * 89)
        print('Exiting from training early')

        # Load the best saved model.
        with open(args.save, 'rb') as f:
            netG.load_state_dict(torch.load(f))
        # Run on test data.
        test_loss = lm_evaluate(test_data)
        print('=' * 89)
        print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
            test_loss, math.exp(test_loss)))
        print('=' * 89)
else:
    print("Not pretraining generator with language model objective")

###############################################################################
# Build the discriminator
###############################################################################

netD = RNNDiscriminator(rnn_type=args.rnn_type,
                        emsize=args.emsize,
                        nhidden=args.nhidden,
                        ntokens=ntokens,
                        nlayers=args.nlayers,
                        bidirectional=args.bidirectional,
                        dropout=args.dropout,
                        initial_vocab=vocab,
                        gpu=args.cuda)
if args.netD != "":
    print('Loading discriminator from '+args.netD)
    netD.load_state_dict(torch.load(args.netD))

if args.cuda:
    netD.cuda()

###############################################################################
# Training setup code
###############################################################################

# Setup Optimizers
optimizerD = optim.SGD(netD.parameters(), lr=args.lr)
# optimizerD = optim.Adam(netD.parameters(), lr=args.lr, betas=(args.beta1, 0.999))
optimizerG = optim.SGD(netG.parameters(), lr=args.lr)
# optimizerG = optim.Adam(netG.parameters(), lr=args.lr, betas=(args.beta1, 0.999))

# input image, noise, fixed noise for comparing learning improvement over epochs, label (real/fake)
input_ = torch.FloatTensor(args.batch_size, 9, args.emsize)
noise = torch.FloatTensor(args.nlayers, args.batch_size, args.nhidden)
fixed_noise = torch.FloatTensor(args.nlayers, args.batch_size, args.nhidden).normal_(0, 1)
label = torch.FloatTensor(args.batch_size)
ones = torch.FloatTensor(args.batch_size)
real_label = 1
fake_label = 0

input_ = to_gpu(args.cuda, Variable(input_))
label = to_gpu(args.cuda, Variable(label))
noise = to_gpu(args.cuda, Variable(noise))
fixed_noise = to_gpu(args.cuda, Variable(fixed_noise))
ones = to_gpu(args.cuda, Variable(ones))

# Binary cross entropy loss
criterion = nn.BCELoss()

if args.cuda:
    netD.cuda()
    netG.cuda()
    criterion.cuda()
    input_, label = input_.cuda(), label.cuda()
    noise, fixed_noise = noise.cuda(), fixed_noise.cuda()

# Main Training Loop
g_losses = []
d_losses = []

# Discriminator and Generator share embedding weights
netD.embedding._parameters['weight'] = netG.embedding._parameters['weight']

# do checkpointing
torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (args.outf, 0))
torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (args.outf, 0))
last_D_x = 0.5
last_D_z = None
best_test_loss = None

train_data = batchify(corpus.train, args.batch_size, shuffle=True, max_len=50)

best_G = '%s/netG_epoch_%d.pth' % (args.outf, 0)

###############################################################################
# Language Model Generation
###############################################################################

print("\nLanguage Model Generation")
# Generate fake sentences
fake_probs, list_indices = netG(teacher_forcing=False, input_=fixed_noise,
                                sample=args.sample, temp=args.temp)
indices = torch.cat(list_indices, 1)

with open("%s/language_model.txt" % args.outf, "w") as f:
    fake, fake_lengths = truncate(indices, args.cuda, sort=False)
    for ex in fake.data:
        chars = " ".join([corpus.dictionary.idx2word[x] for x in ex])
        f.write(chars)
        f.write("\n")

###############################################################################
# Training loop
###############################################################################

# At any point you can hit Ctrl + C to break out of training early.
try:
    print("\nTraining...")
    # Loop through epochs
    for epoch in range(1, args.epochs+1):

        # Loop through batches
        for i, (data, target, lengths) in enumerate(train_data):
            ############################
            # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
            ###########################
            data = data[:, 1:]
            netD.zero_grad()
            batch_size = data.size(0)

            # Train with Real Examples ----------------------------
            label.data.resize_(batch_size).fill_(real_label)
            data = to_gpu(args.cuda, Variable(data))

            output = netD(data, lengths)
            errD_real = criterion(output, label)
            errD_real.backward()
            D_x = output.data.mean()

            # Train with Generated Examples ----------------------------

            # Resample noise for generator
            noise.data.resize_(args.nlayers, batch_size, args.nhidden)
            noise.data.normal_(0, 1)

            # Generate fake text
            fake_probs, list_indices = netG(teacher_forcing=False, input_=noise, sample=True, temp=args.temp)
            indices = torch.cat(list_indices, 1)

            fake, fake_lengths = truncate(indices, args.cuda)
            label.data.resize_(len(fake_lengths)).fill_(fake_label)

            # detach b/c discriminator does NOT backpropagate into the generator (adversarial objective)
            output = netD(fake.detach(), fake_lengths)
            errD_fake = criterion(output, label)
            errD_fake.backward(retain_graph=True)
            D_G_z1 = output.data.mean()

            errD = errD_real + errD_fake

            if last_D_z is None:
                last_D_z = D_G_z1

            if D_x < 0.9:
                # Sum losses from training on real and fake examples
                optimizerD.step()
                last_D_x = D_x
                last_D_z = D_G_z1

            netG.embedding._parameters['weight'] = netD.embedding._parameters['weight']
            d_losses.append((epoch, i, errD.data.cpu()[0]))

            if epoch != 1 or i > args.k:
                ############################
                # (2) Update G network: maximize log(D(G(z)))
                ###########################
                netG.zero_grad()
                label.data.fill_(real_label)  # fake labels are real for generator cost
                # generator gets signal from discriminator, don't detach output
                prob_real = netD(fake, fake_lengths)
                generated_lengths = to_gpu(args.cuda, Variable(torch.from_numpy(np.array(fake_lengths))))

                # Regular REINFORCE: reward is log probability of objective (tricking discriminator)
                #log_prob = prob_real.log()

                # Maximum-Likelihood Augmented
                ones.data.resize_(len(fake_lengths)).fill_(1)
                weight = torch.div(prob_real, ones - prob_real)
                total = torch.sum(weight)
                mlreward = torch.div(weight, total.data[0])

                if mlreward.size(0) < batch_size:
                    ones.data.resize_(batch_size-mlreward.size(0)).fill_(0)
                    mlreward = torch.cat([mlreward, ones], 0)

                # Policy gradient reward
                total_reward = 0
                for j, action in enumerate(list_indices):
                    # Create sentence length mask over padding
                    mask = generated_lengths.gt(j).float()
                    if mask.size(0) < batch_size:
                        mask = torch.cat([mask, ones], 0)
                    if args.reinforce:
                        reward = log_prob * mask
                    else:
                        reward = mlreward * mask
                    reward = reward.unsqueeze(1)
                    action.reinforce(reward.data)
                    total_reward += torch.sum(reward)

                optimizerG.zero_grad()
                autograd.backward(list_indices, [None for _ in list_indices], retain_graph=True)
                optimizerG.step()

                total_reward = total_reward.data.cpu()[0] / batch_size
                g_losses.append((epoch, i, total_reward))
                netD.embedding._parameters['weight'] = netG.embedding._parameters['weight']

                if i % 25 == 0:
                    print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f'
                        % (epoch, args.epochs, i, len(train_data),
                            errD.data[0], total_reward, D_x, D_G_z1))

                    with open("%s/%d-%d-epoch_real.txt" % (args.outf, i, epoch), "w") as f:
                        indices = data.data.cpu().numpy()
                        for ex in indices:
                            chars = " ".join([corpus.dictionary.idx2word[x] for x in ex])
                            f.write(chars)
                            f.write("\n")

                    with open("%s/%d-%d-epoch_fake.txt" % (args.outf, i, epoch), "w") as f:
                        fake_embeddings, list_indices = netG(teacher_forcing=False,
                                                             input_=fixed_noise,
                                                             sample=args.sample,
                                                             temp=args.temp)
                        indices = torch.cat(list_indices, 1)
                        fake, fake_lengths = truncate(indices, args.cuda, sort=False)
                        for ex in fake.data:
                            chars = " ".join([corpus.dictionary.idx2word[x] for x in ex])
                            f.write(chars)
                            f.write("\n")

            if i % 100 == 0:
                # Run on test data.
                test_loss = lm_evaluate(test_data)
                print('=' * 89)
                print('| test loss {:5.2f} | test ppl {:8.2f}'.format(
                    test_loss, math.exp(test_loss)))
                print('=' * 89)

                if best_test_loss is None or best_test_loss > test_loss:
                    print("saving with new best loss", test_loss)
                    # do checkpointing
                    best_G = '%s/netG_epoch_%d_best.pth' % (args.outf, epoch)
                    torch.save(netG.state_dict(), '%s/netG_epoch_%d_best.pth' % (args.outf, epoch))
                    torch.save(netD.state_dict(), '%s/netD_epoch_%d_best.pth' % (args.outf, epoch))
                    pickle.dump(g_losses, open("%s/netG_stats_epoch_%d_best.p" % (args.outf, epoch), "wb"))
                    pickle.dump(d_losses, open("%s/netD_stats_epoch_%d_best.p" % (args.outf, epoch), "wb"))
                    best_test_loss = test_loss

        # do checkpointing
        torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (args.outf, epoch))
        torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (args.outf, epoch))
        pickle.dump(g_losses, open("%s/netG_stats_epoch_%d.p" % (args.outf, epoch), "wb"))
        pickle.dump(d_losses, open("%s/netD_stats_epoch_%d.p" % (args.outf, epoch), "wb"))

except KeyboardInterrupt:
    print('-' * 89)
    print('Exiting from training early')

    # # Load the best saved model.
    with open(best_G, 'rb') as f:
        netG.load_state_dict(torch.load(f))

    # Run on test data.
    test_loss = lm_evaluate(test_data)
    print('=' * 89)
    print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
        test_loss, math.exp(test_loss)))
    print('=' * 89)
    exit(0)

print('-' * 89)
print('Exiting from training early')

# Load the best saved model.
with open(best_G, 'rb') as f:
    netG.load_state_dict(torch.load(f))

# Run on test data.
test_loss = lm_evaluate(test_data)
print('=' * 89)
print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(
    test_loss, math.exp(test_loss)))
print('=' * 89)


import ast
import functools

def compare(l, r):
    if type(l) == type(r) == int:
        return l - r
    elif type(l) == type(r) == list:
        for ll, rr in zip(l, r):
            result = compare(ll, rr)
            if result != 0:
                return result
        return len(l) - len(r)
    elif type(l) == int:
        return compare([l], r)
    else:
        return compare(l, [r])

def adv13a():
    count = 0
    index = 1

    with open("input13.txt", "r") as f:
        inputs = []
        for line in f.readlines():
            if line.strip():
                inputs.append(line)
            else:
                inputs = []
                index += 1

            if len(inputs) == 2:
                left, right = inputs
                left = ast.literal_eval(left)
                right = ast.literal_eval(right)

                result = compare(left, right)
                if result < 0:
                    count += index

    print(count)

def adv13b():
    data = [[[2]], [[6]]]
    with open("input13.txt", "r") as f:
        inputs = []
        for line in f.readlines():
            if line.strip():
                inputs.append(line)
            else:
                inputs = []

            if len(inputs) == 2:
                left, right = inputs
                left = ast.literal_eval(left)
                right = ast.literal_eval(right)
                data.append(left)
                data.append(right)

    data.sort(key=functools.cmp_to_key(compare))
    i1 = data.index([[2]]) + 1
    i2 = data.index([[6]]) + 1

    print(i1 * i2)

if __name__ == "__main__":
    adv13b()


from datetime import datetime, date

import pandas as pd
import plotly.express as px
import dash
from dash import Input, Output, callback, html, dcc
from dash_bootstrap_templates import load_figure_template
import dash_bootstrap_components as dbc

import plotly.graph_objs as go
from plotly.subplots import make_subplots
import finnhub

load_figure_template("SUPERHERO")

dash.register_page(__name__, name='Multi-select i korelacija')

education = (pd
             .read_csv("assets/states_all.csv")
             .iloc[:, 1:]
             .rename({
    "AVG_MATH_4_SCORE": "matematika 4ti razred",
    "AVG_MATH_8_SCORE": "matematika 8mi razred",
    "AVG_READING_4_SCORE": "Srpski 4ti razred",
    "AVG_READING_8_SCORE": "Srpski 8mi razred"
}, axis=1)
             .assign(expenditure_per_student=lambda x: x["TOTAL_EXPENDITURE"] / x["GRADES_ALL_G"])
             )

layout = html.Div([
    dbc.Row(html.H1("Obrazovni učinak i rashodi u SAD", style={"text-align": "center"})),
    html.P(children=["Prelaskom kursora preko Dijagrama rasejanja(levo),",html.Br(),
           "možete videti korelaciju troškova na Linijskom grafikonu(desno)"],
           style={"color": "yellow", "textAlign":"center", "font-size":"18",'font-style': 'italic'}),
    html.P("Izaberite X i Y kolonu ispod:",
           style={"color": "yellow", "textAlign":"left", "font-size":"18",'font-style': 'italic'}),
    dbc.Row([
        dbc.Col([
            dbc.Card([
                dcc.Markdown("X kolona",style={'color':'yellow', 'text-align':'center'}),
                dcc.RadioItems(
                    id="score-radio",
                    options=["matematika 4ti razred", "matematika 8mi razred", "Srpski 4ti razred", "Srpski 8mi razred"],
                    value="matematika 8mi razred"
                ),
                html.Hr(),
                dcc.Markdown("Y kolona", style={'color':'yellow', 'text-align':'center'}),
                dcc.RadioItems(
                    id="score-radio2",
                    options=["matematika 4ti razred", "matematika 8mi razred", "Srpski 4ti razred", "Srpski 8mi razred"],
                    value="Srpski 8mi razred"
                )], style={'border-radius': '12px'})
        ], width=2),
        dbc.Col(
            dcc.Graph(id="cross-filter-scatter", hoverData={'points': [{'customdata': ['CALIFORNIA']}]}),width=5,
        ),
        dbc.Col(dcc.Graph(id="x-line"),width=5,)
    ])
])

@callback(
    Output("cross-filter-scatter", "figure"),
    Input("score-radio", "value"),
    Input("score-radio2", "value"))
def score_scatter(x, y):
    fig = px.scatter(
        education.query("YEAR == 2013"),
        x=x,
        y=y,
        hover_name="STATE",
        custom_data=["STATE"]
    ).update_layout(
        # title="Plot Title",
        xaxis_title=f"Bodovi iz: {x}",
        yaxis_title=f"Bodovi iz: {y}",
        # legend_title="Legend Title",
        font=dict(
            family="Calibri",
            size=14,
            color="white"
        ))

    return fig

@callback(
    Output("x-line", "figure"),
    Input("cross-filter-scatter", "hoverData"))
def update_line(hoverData):
    state_name = hoverData["points"][0]["customdata"][0]
    df = education.query("STATE == @state_name")

    fig = px.line(
        df,
        x="YEAR",
        y="expenditure_per_student",
        title=f"Troškovi po studentu u {state_name.title()}"
    ).update_xaxes(showgrid=False,).update_layout(
        title={'y': 0.9,
               'x': 0.5,
               'xanchor': 'center',
               },
        xaxis_title="Godina",
        yaxis_title="Troškovi po studentu u hiljadama",
        # legend_title="Legend Title",
        font=dict(
            family="Calibri",
            size=14,
            color="white"
        ))

    return fig


import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

service_obj = Service("C:/Meera/chromedriver_win32/chromedriver.exe")
driver = webdriver.Chrome(service= service_obj)
driver.get("https://www.amazon.com/")

#enter item to search in search box
keyword = "hair dryer"
search = driver.find_element(By.XPATH, '//*[(@id = "twotabsearchtextbox")]')
search.send_keys(keyword)

# click search button
search_button = driver.find_element(By.ID, 'nav-search-submit-button')
search_button.click()

#apply filter by brand
driver.implicitly_wait(5)

brandlist = []

Brandnames = driver.find_elements(By.XPATH, "//span[@class='a-list-item']")
#Brandnames = driver.find_elements(By.ID, "brandsRefinements")

for brand in Brandnames:
       if brand.text == "Conair":
        brand.click()
        break
        print(brand.text)

#filter product by review
reviews = driver.find_elements(By.ID, "reviewsRefinements")
for review in reviews:
    #print(review.text)
    if review.find_element(By.XPATH, "//section[@aria-label='4 Stars & Up']"):
        print(review.find_element(By.XPATH, "//section[@aria-label='4 Stars & Up']").text)
        review.click()
        break

wait = WebDriverWait(driver, 10)  # explicit wait
totalitem = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='a-section a-spacing-base']")))

#final products after applying  all filters
for item in totalitem:
    productname = item.find_element(By.XPATH, "div/div/h2/a/span")
    if productname.text == "Conair Double Ceramic Hair Dryer, 1875W Hair Dryer with Ionic Conditioning":
        productname.click()
        break

driver.find_element(By.XPATH, "//*[@id='add-to-cart-button']").click()#addd to cart
errmsg= driver.find_element(By.XPATH, "//div[@class='a-fixed-left-grid-inner']").text

print(errmsg)
#check msg
assert "Added to Cart" in errmsg

driver.find_element(By.XPATH, "//*[@id='sw-gtc']").click()
subtotal =driver.find_element(By.XPATH, "//div[contains(@data-name,'Subtotals')]").text
print(subtotal)#print subtotal of item
assert subtotal

driver.find_element(By.ID, "sc-buy-box-ptc-button").click()

driver.find_element(By.XPATH,'//*[@id="ap_email"]').send_keys('+16476875247')

driver.find_element(By.ID, 'continue').click()
time.sleep(2)
driver.find_element(By.XPATH, '//*[@id="ap_password"]').send_keys('Chauhan@1095')
driver.find_element(By.XPATH, '//*[@id="signInSubmit"]').click()

driver.quit()


import common

nums = common.to_int(common.read_file('2021/01/data.txt').splitlines())

# part 1
acc = 0
for i in range(1, len(nums)):
    if nums[i-1] < nums[i]:
        acc += 1

print(acc)

# part 2
acc = 0
for i in range(3, len(nums)):
    if nums[i-3] < nums[i]:
        acc += 1

print(acc)


# -*- coding: utf-8 -*-
"""
@author: t.maybour
"""

import os
import h5py
import numpy as np

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt

def main():

    plots = cplots()

    c = np.float64(299792458) # speed of light
    epsilon0 = np.float(8.85418782e-12)
    mu0 = np.float(1.25663706e-6)
    nu = np.sqrt(mu0 / epsilon0)

#########################################################################

    with h5py.File('fdtd_data.h5', "r") as hdf:

        ################### Inputs ###################
        bw_nopo = np.uint32(hdf.get('bw_nopo'))
        tv_nopo_input = np.uint32(hdf.get('tv_nopo_input'))
        xv_nopo_input = np.uint32(hdf.get('xv_nopo_input'))
        res = np.uint32(hdf.get('res'))
        ##############################################
        l0 = np.float64(hdf.get('l0'))
        pfbw = np.float64(hdf.get('pfbw'))
        ##############################################
        glen = np.float64(hdf.get('glen'))
        gstart = np.float64(hdf.get('gstart'))
        dn = np.float64(hdf.get('dn'))
        ds = np.float64(hdf.get('ds'))
        gramp_scale = np.float64(hdf.get('gramp_scale'))
        ##############################################
        tlen = np.float64(hdf.get('tlen'))
        mu = np.float64(hdf.get('mu'))
        tstart = np.float64(hdf.get('tstart'))
        delta_t = np.float64(hdf.get('delta_t'))
        ton = np.float64(hdf.get('ton'))
        ##############################################

        ################### DATA #####################
        tv_nopo = np.uint32(hdf.get('tv_nopo'))
        xv_nopo = np.uint32(hdf.get('xv_nopo'))
        n0 = np.float64(hdf.get('n0'))
        gfstart = np.float64(hdf.get('gfstart'))
        gfend = np.float64(hdf.get('gfend'))
        gend = np.float64(hdf.get('gend'))
        ##############################################
        kv = np.array(hdf.get('kv'), dtype=np.float64)
        fk = np.array(hdf.get('fk'), dtype=np.float64)
        xv = np.array(hdf.get('xv'), dtype=np.float64)
        apod = np.array(hdf.get('apod'), dtype=np.float64)
        gm = np.array(hdf.get('gm'), dtype=np.float64)
        tv = np.array(hdf.get('tv'), dtype=np.float64)
        twindow = np.array(hdf.get('twindow'), dtype=np.float64)
        source_real = np.array(hdf.get('source_real'), dtype=np.float64)
        source_imag = np.array(hdf.get('source_imag'), dtype=np.float64)
        efield_mat_real = np.array(hdf.get('efield_mat_real'), dtype=np.float64)
        efield_mat_imag = np.array(hdf.get('efield_mat_imag'), dtype=np.float64)
        hfield_mat_real = np.array(hdf.get('hfield_mat_real'), dtype=np.float64)
        hfield_mat_imag = np.array(hdf.get('hfield_mat_imag'), dtype=np.float64)
        ##############################################

#########################################################################

    efield_mat = np.zeros((tv_nopo,xv_nopo), dtype=np.complex128)
    hfield_mat = np.zeros((tv_nopo,xv_nopo), dtype=np.complex128)

    for ii in range(0, tv_nopo):
        for jj in range(0, xv_nopo):
            efield_mat[ii,jj] = efield_mat_real[ii*xv_nopo+jj] + 1j*efield_mat_imag[ii*xv_nopo+jj]
            hfield_mat[ii,jj] = hfield_mat_real[ii*xv_nopo+jj] + 1j*hfield_mat_imag[ii*xv_nopo+jj]

    edensity = np.zeros((tv_nopo, xv_nopo), dtype=np.float64)
    edensity[:,:] = np.real((n0**2)*np.conj(efield_mat[:,:])*(efield_mat[:,:]) + (c**2)*np.conj(hfield_mat[:,:])*(hfield_mat[:,:]))
    edensity[:,:] = edensity[:,:] / np.max(edensity[:,:])

    source = np.zeros(tv_nopo, dtype=np.complex128)
    source[:] = source_real[:] + 1j*source_imag[:]

#######################################################################
############################## Graphs #################################
#######################################################################

    plots.x_label = 'Wavenumber (1/m)'
    plots.line_plot(kv, fk/np.max(fk), filename='fk')

    plots.x_scale = 1e3
    plots.x_label = 'Length (mm)'

    plots.line_plot(xv, apod, filename='apod')
    plots.line_plot(xv, gm, filename='gm')

    plots.x_scale = 1e12
    plots.x_label = 'Time (ps)'

    plots.line_plot(tv, twindow, filename='twindow')
    plots.line_plot(tv, np.abs(source)**2/np.max(np.abs(source)**2), filename='source')

########################################################################

    plots.width = 32
    plots.height = 16
    plots.axes_label_size = 50
    plots.tick_label_size = 45
    plots.major_tick_length = 8
    plots.major_tick_width = 3

    plots.x_scale = 1e12
    plots.x_label = 'Time (ps)'
    plots.y_scale = 1e3
    plots.y_label = 'Position (mm)'

    plots.z_label = "Energy Density (a.u.)"

    plots.heat_map(tv, xv, np.transpose(edensity), filename='edensity_plot')

#################################################################################
#################################################################################
#################################################################################

class cplots():

    def __init__(self):
        self.filename = ''
        self.save_dir = ''
        self.width = 20
        self.height = 16
        self.axes_label_size = 35
        self.tick_label_size = 35
        self.axes_label_padding = 35
        self.margins = False
        self.x_label = ''
        self.y_label = ''
        self.z_label = ''
        self.x_scale = 1
        self.y_scale = 1
        self.z_scale = 1
        self.linewidth = 6
        self.major_tick_length = 5
        self.major_tick_width = 3
        self.tick_label_padding = 15

    def set_save_dir(self):
        if (len(self.save_dir) > 0) :
            if (self.save_dir[-1] != '/'):
                self.save_dir = self.save_dir + '/'
        else:
            self.save_dir = 'graphs/'
        self.create_dir(self.save_dir)

    def create_dir(self, dir_in):
        if (dir_in[-1] != '/'):
            dir_in = dir_in + '/'
        if not os.path.exists(dir_in):
            os.makedirs(dir_in)

    def axis_setup(self):
        plt.rc('axes', labelsize=self.axes_label_size)
        plt.rc('xtick', labelsize=self.tick_label_size)
        plt.rc('ytick', labelsize=self.tick_label_size)

    def set_kwargs(self, kwargs):
        self.filename = kwargs.get('filename', self.filename)
        self.save_dir = kwargs.get('save_dir', self.save_dir)

    def line_plot(self, x, data, *args, **kwargs):

        self.set_kwargs(kwargs)
        self.set_save_dir()
        self.axis_setup()

        y = np.array(data)

        plt.figure(figsize=(self.width, self.height))
        plt.ticklabel_format(useOffset=False)
        if self.margins == False:
            plt.margins(x=0)
        plt.xlabel(self.x_label, labelpad=self.axes_label_padding)
        plt.ylabel(self.y_label, labelpad=self.axes_label_padding)

        plt.plot(self.x_scale*x,self.y_scale*y,linewidth=self.linewidth)

        plt.tight_layout()
        plt.savefig(self.save_dir + self.filename + ".png")
        plt.close()

    def heat_map(self, x, y, z, *args, **kwargs):

        self.set_kwargs(kwargs)
        self.set_save_dir()
        self.axis_setup()

        u, v = np.meshgrid(x, y)

        fig, ax1 = plt.subplots(constrained_layout=True)

        ax1.tick_params(direction='out', length=self.major_tick_length, width=self.major_tick_width, labelsize=self.axes_label_size, pad=self.tick_label_padding)
        im = ax1.pcolormesh(self.x_scale*u, self.y_scale*v, self.z_scale*z, cmap='viridis')

        ax1.set_xlabel(self.x_label, labelpad=self.axes_label_padding)
        ax1.set_ylabel(self.y_label, labelpad=self.axes_label_padding)

        cbar = fig.colorbar(im, ax=ax1) #need a colorbar to show the intensity scale
        cbar.set_label(self.z_label, labelpad=self.axes_label_padding)
        cbar.ax.tick_params(labelsize=self.axes_label_size)

        #################################################
        w=self.width
        h=self.height
        ax=plt.gca()
        l = ax.figure.subplotpars.left
        r = ax.figure.subplotpars.right
        t = ax.figure.subplotpars.top
        b = ax.figure.subplotpars.bottom
        figw = float(w)/(r-l)
        figh = float(h)/(t-b)
        ax.figure.set_size_inches(figw, figh)
        #################################################

        plt.savefig(self.save_dir + self.filename + ".png")
        plt.close()

#################################################################################
#################################################################################
#################################################################################

if __name__ == '__main__':
    main()


import seaborn as sns
import matplotlib.pyplot as plt

def plot_data(data, timestamp, mask=None):
    if mask is None:
        mask=data==1.0000000e+04
    fig, ax = plt.subplots(figsize=(10,2))
    sns.heatmap(data, cmap="jet", mask=mask, vmax=500 ,vmin=-500)
    ax.invert_yaxis()
    ax.set(ylabel='rgate_no_1')
    ax.spines['right'].set_visible(True)
    ax.spines['top'].set_visible(True)
    ax.spines['left'].set_visible(True)
    ax.spines['bottom'].set_visible(True)
    ax.collections[0].colorbar.set_label('Doppler velocity[m/s]')
    ax.set(xticklabels=[])
    # ax[0].tick_params(bottom=False)
    idxs = [int(xtick) for xtick in ax.get_xticks()]
    xlabels = [timestamp[idx] for idx in idxs]
    ax.set_xticklabels(xlabels)
    ax.set(xlabel='epoch')
    plt.subplots_adjust(hspace=0.08)
    plt.show()

def plot_list_data(list_data, timestamp=None, mask_value=None):
    fig, ax = plt.subplots(len(list_data), sharey='all', figsize=(10,2*len(list_data)))
    for i, d in enumerate(list_data):
        if mask_value is None:
            mask=d==1.0000000e+04
        else:
            mask=d>=mask_value
        sns.heatmap(d, ax=ax[i], cmap="jet", mask=mask, vmax=500 ,vmin=-500)
        ax[i].invert_yaxis()
        ax[i].set(ylabel='rgate_no_1')
        ax[i].spines['right'].set_visible(True)
        ax[i].spines['top'].set_visible(True)
        ax[i].spines['left'].set_visible(True)
        ax[i].spines['bottom'].set_visible(True)
        ax[i].collections[0].colorbar.set_label('Doppler velocity[m/s]')
        ax[i].set(xticklabels=[])
        # ax[i].tick_params(bottom=False)
    if timestamp is not None:
        idxs = [int(xtick) for xtick in ax[-1].get_xticks()]
        xlabels = [timestamp[idx] for idx in idxs]
        ax[-1].set_xticklabels(xlabels)
    ax[-1].set(xlabel='epoch')
    plt.subplots_adjust(hspace=0.08)
    plt.show()


import datetime
import logging
import os
from collections import deque
from functools import partialmethod
from random import randint

import numpy as np
import pynmea2
import serial
from pyubx import Manager

try:
    from utility import get_pn_uuid, UTC
except ImportError:
    from app.src.utility import get_pn_uuid, UTC

FILE_DIR = os.path.dirname(os.path.abspath(__file__))
LOG_DIR = os.path.join(FILE_DIR, "..", "..", "logs")
GPS_LOG = os.path.join(LOG_DIR, 'gps.log')

logger = logging.getLogger('gps')
logfile = logging.FileHandler(GPS_LOG)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s:: %(message)s')
logfile.setFormatter(formatter)
logger.addHandler(logfile)

class CoordinateService(Manager):
    def __init__(self, ser, debug=False, maxlen_vel=11, vel_avg_seconds=10,
                 vel_inst_seconds=10, s_i_max=55, s_a_max=55, t_i_max=30,
                 t_a_max=12.5, ref_spd=40, ref_spd_mod=20, gen_fake_vel=False):
        if not debug:
            logger.setLevel(logging.INFO)
        else:
            logger.setLevel(logging.DEBUG)

        logger.info("Beginning GPS service setup...")
        try:
            logger.info("Setting up serial connection to GPS device...")
            if isinstance(ser, bytes) or isinstance(ser, str):
                ser = serial.Serial(ser)
        except Exception:
            logger.exception("\n"
                             "********************\n"
                             "**GPS device error**\n"
                             "********************")
            raise

        logger.info("Setting up PyUBX Manager...")
        Manager.__init__(self, ser, debug)
        self._dumpNMEA = False
        self.latest_fix = None
        self.latest_vel = None
        self.msg_alarm = 0
        logger.info("Manager set up. Initializing variables...")

        self.gen_fake_vel = gen_fake_vel

        # Initialize values for velocity checking
        # we need enough values to satisfy average requirements - assume 1/sec
        maxlen_vel = max(maxlen_vel, vel_avg_seconds + 1)
        self.vel_array = deque(maxlen=maxlen_vel)  # velocity = (speed, track)
        self.vel_avg_seconds = vel_avg_seconds  # how long to consider in avgs
        self.vel_inst_seconds = vel_inst_seconds  # time ago to calc inst diff

        # Initialize parameters against which to check velocity
        self.s_i_max = s_i_max  # speed-instant trigger
        self.s_a_max = s_a_max  # speed-average trigger
        self.t_i_max = t_i_max  # track-instant trigger
        # We compare sine values - so compute this now
        # It's the max difference in sine value vs angular value
        self.t_a_max = 7  # init greater than sine is possible
        self.set_tamax(t_a_max)  # track-average trigger
        self.spd_holder = 0.0  # init outside func for use in comparison
        # Used to calculate a dynamic t_i_max based on speed
        self.ref_spd = ref_spd  # Speed to start decreasing t_i_max
        self.ref_spd_mod = ref_spd_mod  # Used to tune t_i_max descent

        logger.info("GPS service initialized.")

    @staticmethod
    def hdg_diff(init, final):
        if init > 360 or init < 0 or final > 360 or final < 0:
            raise Exception("out of range. init: {} // final: {}".format(init, final))
        diff = final - init
        abs_diff = abs(diff)

        if abs_diff == 180:
            return abs_diff
        elif abs_diff < 180:
            return diff
        elif final > init:
            return abs_diff - 360
        else:
            return 360 - abs_diff

    @staticmethod
    def _vel_proc(vel_dict, v_or_k, index, funcname):
        """
        :param v_or_k: str Either 'keys' or 'values' - what do we want?
        :param index: int The index of the key/value we want (typically 0 or 1)
        :param vel_dict: dict The velocity dict: { timestamp: ( speed, track ) }
        :return: float The requested value
        """
        # Check errors / log
        if not isinstance(vel_dict, dict):
            logger.error("***{} was passed a non-dict parameter") \
                .format(funcname)
            return -1
        if v_or_k not in ["values", "keys"]:
            logger.error("***{} was passed {}, not 'keys' or 'values'") \
                .format(funcname, v_or_k)
            return -1
        # Do actual work
        try:
            if v_or_k == "keys":
                return list(getattr(vel_dict, v_or_k)())[index]
            else:
                return list(getattr(vel_dict, v_or_k)())[0][index]
        except KeyError:
            logger.error("***{} encountered KeyError for {}" \
                         .format(funcname, vel_dict))
            return -1

    _spd = partialmethod(_vel_proc, v_or_k="values", index=0, funcname="_spd")
    _trk = partialmethod(_vel_proc, v_or_k="values", index=1, funcname="_trk")
    _ts = partialmethod(_vel_proc, v_or_k="keys", index=0, funcname="_ts")

    @staticmethod
    def avg_sin(angles):
        """
        Given a list of angles in degrees, return the average of the
          individual sine values.
        :param angles: list A list of angles in degrees
        :return: float Average sine value for the list
        """
        avg = sum(np.sin(np.radians(angles))) / len(angles)
        return round(avg, 4)

    def set_tamax(self, new_max):  # Alternative is self.t_a_max as property
        try:
            self.t_a_max = np.sin(np.radians(new_max))
        except Exception:
            logger.error("*****Error computing new t_a_max for value {}"
                         .format(new_max))

    def _check_velocity(self):
        try:
            now = list(self.vel_array[0])[0]  # Get first key (timestamp)
        except IndexError:
            return

        def speed_alarm():
            s_val = self._spd(self.vel_array[0])  # { now: (speed, track) }
            self.spd_holder = s_val
            """
            This gets the least-recent speed adhering to the vel_inst_seconds.
            The idea is to check less recently than the fastest rate of msgs.
            It will be the same for track (below).
            """
            s_i_check_val = [self._spd(x) for x in self.vel_array if
                             self._ts(x) >= (now - self.vel_inst_seconds)][-1]
            s_a_check_list = [self._spd(x) for x in self.vel_array if
                              self._ts(x) >= (now - self.vel_avg_seconds)]
            s_a_check_val = sum(s_a_check_list) / len(s_a_check_list)
            alarm = (abs(s_val - s_i_check_val) > self.s_i_max or
                     abs(s_val - s_a_check_val) > self.s_a_max)
            return alarm

        def track_alarm():
            # Reduce allowable instant track variance with increasing speed
            try:
                if self.spd_holder >= self.ref_spd:
                    current_t_i_max = min([
                        self.t_i_max,
                        (self.ref_spd/(self.spd_holder + self.ref_spd_mod)) * self.t_i_max
                    ])
                else:
                    current_t_i_max = self.t_i_max
            except:
                current_t_i_max = self.t_i_max
                logger.exception("***Error calculating speed-dependent t_i_max")

            t_val = self._trk(self.vel_array[0])  # { now: (speed, track) }
            t_i_check_val = [self._trk(x) for x in self.vel_array if
                             self._ts(x) >= (now - self.vel_inst_seconds)][-1]
            # Compute average heading over time
            t_a_check_val = self.avg_sin([self._trk(x) for x in self.vel_array if
                                          self._ts(x) >= (now - self.vel_avg_seconds)])
            alarm = (self.hdg_diff(t_val, t_i_check_val) > current_t_i_max or
                     abs(np.sin(np.radians(t_val)) - t_a_check_val) > self.t_a_max)
            return alarm

        try:
            # speed at rest 0-2.5
            if speed_alarm() or (self.spd_holder > 2.5 and track_alarm()):
                try:
                    self.msg_alarm = 1
                    self.latest_vel = self._constr_vel()
                    self.vel_array.clear()
                except:
                    logger.exception("***Error getting velocity alarms")
        except KeyError:
            logger.exception("\n"
                             "********************\n"
                             "***velocity error***\n"
                             "********************")
        except Exception:
            logger.exception("\n"
                             "********************\n"
                             "***velocity error***\n"
                             "********************")

    # Override onNMEA from parent class to do work
    def onNMEA(self, buffer):
        msg = pynmea2.parse(buffer)
        if msg.__class__ is pynmea2.GGA:  # position msg
            if msg.gps_qual > 0:
                self.latest_fix = msg
        elif msg.__class__ is pynmea2.VTG:  # velocity msg
            try:
                speed = msg.spd_over_grnd_kmph
                track = msg.true_track
                if self.gen_fake_vel:  # To test other stuff
                    speed = 50 + randint(-25, 25)
                    track = 180 + randint(-35, 35)
                if speed and track:
                    # now = timestamp in seconds
                    now = datetime.datetime.timestamp(datetime.datetime.now(tz=UTC))
                    self.vel_array.appendleft({now: (speed, track)})
                    self._check_velocity()
            except AttributeError:
                logger.exception("***Bad properties for VTG Sentence***")
        else:
            # Potential to expand to more message types here
            pass

    def get_latest_fix(self):
        if self.latest_fix is not None:
            return (
                self.latest_fix.latitude,
                self.latest_fix.longitude,
                self.latest_fix.altitude,
                self.latest_fix.timestamp
            )
        else:
            return 0

    def get_latest_velocity(self):
        if self.latest_vel is not None:
            return self.latest_vel
        else:
            return 0

    def _constr_vel(self):
        try:
            for k, v in self.vel_array[0].items():
                return (
                    v[0], v[1],  # speed in kph, track direction (true)
                    datetime.datetime.fromtimestamp(k, tz=UTC)  # Python DT obj
                )
        except IndexError:
            return 0
        except Exception:
            logger.exception("\n"
                             "********************\n"
                             "**constr_vel error**\n"
                             "********************")
            return 0


import os
import configparser as config

class ProjectManager:
    FILE_LOG_DATA = []
    CONSTANT = {}

    def __init__(self, fname):
        self.fname = fname

        # load file if exist
        if os.path.isfile(fname+".gai"):
            with open(fname+".gai", "r") as fbj:
                fdt = fbj.readlines()

        # load .ini file
        cfg = config.ConfigParser()  # load config parser
        cfg.optionxform = str
        cfg.read("meta.ini")
        for sect in cfg.sections():
            ProjectManager.CONSTANT[sect] = {}
            for key, val in cfg.items(sect):  # test out each type; default is string
                if val.isdigit(): ProjectManager.CONSTANT[sect][key] = int(val)
                elif val == "true": ProjectManager.CONSTANT[sect][key] = True
                elif val == "false": ProjectManager.CONSTANT[sect][key] = False
                else: ProjectManager.CONSTANT[sect][key] = val

    @classmethod
    def insert_node(cls, pos, fx):
        cls.FILE_LOG_DATA.append(["%NODE", pos, fx])

    @classmethod
    def insert_bias(cls, nodes, weight):
        cls.FILE_LOG_DATA.append(["%BIAS", nodes[0], nodes[1], weight])

    def flush(self):  # write files from the log data
        with open(self.fname+".gai", "a") as fbj:
            pass

    class Mouse:
        @staticmethod
        def OnTrigger(first, last): return True if first != last and last is False else False  # useless

"""
flushed-based file
.gai File system:

%world posx posy
$var %node posx posy fx
$nodes %node posx posy fx
$var1 %bias node1 node2 weight

"""