import glob
import os
import sys
import random
import time
import numpy as np

try:
    sys.path.append(glob.glob('../carla/dist/carla-*%d.%d-%s.egg' % (
        sys.version_info.major,
        sys.version_info.minor,
        'win-amd64' if os.name == 'nt' else 'linux-x86_64'))[0])
except IndexError:
    pass

import carla

from env.functions import CrossingRate, EnableTwoWheel, SafyDis, DrawArea
from env.functions import INSIDE, draw_area, get_area, get_PED_area

class Vehicles:

    def __init__(self, world, client, scene, waypoints, ego_car):
        self.world = world
        self.client = client
        self.scene = scene
        self.waypoints = waypoints
        self.ego_car = ego_car

        self.number = scene['NumOfVeh']
        self.AREA = get_area(self.scene)

        if DrawArea:
            draw_area(self.world.debug, self.AREA)

        self.bps = self.world.get_blueprint_library().filter('vehicle.*.*')

        if not EnableTwoWheel:
            self.bps = [x for x in self.bps if int(x.get_attribute('number_of_wheels')) == 4]

        self.spawns = self.world.get_map().get_spawn_points()
        self.spawns = [x for x in self.spawns if self.ego_car.start.location.distance(x.location) > SafyDis]

        inner_spawns = [x for x in self.spawns if INSIDE(x.location, self.AREA)]
        outer_spawns = [x for x in self.spawns if not INSIDE(x.location, self.AREA)]
        density = (self.number[0] + self.number[1]) / 2 / len(inner_spawns)

        self.vehicles = []
        self.generate_vehicles(inner_spawns, round(density * len(inner_spawns)))
        self.generate_vehicles(outer_spawns, round(density * len(outer_spawns)))

    def generate_vehicles(self, spawns, total):
        random.shuffle(spawns)

        for i in range(total):
            bp = random.choice(self.bps)
            vehicle = self.world.spawn_actor(bp, spawns[i])
            self.vehicles.append(vehicle)

    def start(self):
        for x in self.vehicles:
            x.set_autopilot(True)
        self.client.apply_batch_sync([carla.command.SetAutopilot(x, True) for x in self.vehicles])

    def step(self):
        self.start()
        #vehs = [x for x in self.vehicles if INSIDE(x.get_location(), self.AREA)]
        #print('Available Car: ', len(vehs), ' number: ', self.number)

        '''
        if len(inner_vehs) < self.number[0]:
            #self.destroy()
            self.generate_vehicles(1) #self.number[1] - len(inner_vehs))

            if outer_vehs == []:
                self.generate_vehicles(self.outer_spawns, len(self.outer_spawns))
                self.last_supply = frame

            elif frame % UpdateFrame == 0:
                self.vehicles = [x for x in self.vehicles if INSIDE(x.get_location(), self.inner)]
                self.client.apply_batch([carla.command.DestroyActor(x) for x in outer_vehs])
            else:
                print('hhhh')
        '''

    def destroy(self):
        self.client.apply_batch([carla.command.DestroyActor(x) for x in self.vehicles])
        self.vehicles = []

class Walkers:

    def __init__(self, world, client, scene, waypoints):
        self.world = world
        self.client = client
        self.scene = scene
        self.waypoints = waypoints

        self.number = scene['NumOfWal']
        self.AREA = get_PED_area(self.scene, self.waypoints)

        self.bps = self.world.get_blueprint_library().filter("walker.pedestrian.*")
        self.control_bp = self.world.get_blueprint_library().find('controller.ai.walker')

        self.walkers = []
        self.controllers = []
        self.generate_walkers(self.number[1])

        self.world.set_pedestrians_cross_factor(CrossingRate)

    def generate_walkers(self, total):
        for i in range(total):
            bp = random.choice(self.bps)
            if bp.has_attribute('is_invincible'):
                bp.set_attribute('is_invincible', 'false')

            while True:
                location = self.world.get_random_location_from_navigation()
                if not INSIDE(location, self.AREA):
                    continue

                spawn_point = carla.Transform()
                spawn_point.location = location
                walker = self.world.try_spawn_actor(bp, spawn_point)

                if walker is not None:
                    break

            controller = self.world.spawn_actor(self.control_bp, carla.Transform(), walker)
            self.walkers.append(walker)
            self.controllers.append(controller)

    def start(self):
        for controller in self.controllers:
            while True:
                location = self.world.get_random_location_from_navigation()
                if INSIDE(location, self.AREA):
                    break

            controller.start()
            controller.go_to_location(location)
            controller.set_max_speed(1.3 + random.random() * 0.5)

    def step(self):
        pass
        #wals = [x for x in self.walkers if INSIDE(x.get_location(), self.AREA)]
        #print('Available Walker: ', len(wals), ' number: ', self.number)

    def destroy(self):
        for x in self.controllers:
            x.stop()

        self.client.apply_batch([carla.command.DestroyActor(x) for x in self.controllers])
        self.client.apply_batch([carla.command.DestroyActor(x) for x in self.walkers])

        self.walkers = []
        self.controllers = []


import pathlib
from unittest import mock

from django.core import mail
from django.core.management import call_command
from django.test import override_settings
import responses

from mtp_common.notify import NotifyClient, TemplateError
from mtp_common.notify.templates import NotifyTemplateRegistry
from mtp_common.test_utils import silence_logger
from mtp_common.test_utils.notify import (
    GOVUK_NOTIFY_TEST_API_KEY, GOVUK_NOTIFY_TEST_REPLY_TO_PUBLIC, GOVUK_NOTIFY_TEST_REPLY_TO_STAFF,
    fake_template,
    mock_all_templates_response, mock_send_email_response,
    NotifyBaseTestCase,
)

class NotifyTestCase(NotifyBaseTestCase):
    @override_settings(GOVUK_NOTIFY_API_KEY='')
    def test_missing_settings(self):
        # client cannot be created if api key is missing
        with responses.RequestsMock(), self.assertRaises(AssertionError):
            NotifyClient.shared_client()

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_notify_client_duplicate_templates(self):
        # if multiple templates have the same name, the client cannot be created
        with responses.RequestsMock() as rsps, self.assertRaises(TemplateError):
            mock_all_templates_response(rsps, templates=[
                fake_template('0000', 'generic', required_personalisations=['message']),
                fake_template('11', 'test-template'),
                fake_template('12', 'test-template'),
            ])
            NotifyClient.shared_client()

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_notify_client_loads_templates(self):
        # client is created and loads all template name-ids pairs
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
        self.assertEqual(client.get_template_id_for_name('test-template'), '11')
        self.assertEqual(client.get_template_id_for_name('test-template2'), '12')
        with self.assertRaises(TemplateError):
            client.get_template_id_for_name('test-template3')

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_email_to_one_address(self):
        # can send a simple email by checking params posted to GOV.UK Notify
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '11', 'sample@localhost')
            message_ids = client.send_email('test-template', 'sample@localhost')
        self.assertEqual(len(message_ids), 1)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_email_to_several_addresses(self):
        # can send simple emails by checking params posted to GOV.UK Notify
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '12', 'sample@localhost')
            mock_send_email_response(rsps, '12', 'sample2@localhost')
            message_ids = client.send_email('test-template2', ['sample@localhost', 'sample2@localhost'])
        self.assertEqual(len(message_ids), 2)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_personalised_email(self):
        # can send a personalised email by checking params posted to GOV.UK Notify
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps, templates=[
                fake_template('0000', 'generic', required_personalisations=['message']),
                fake_template('11', 'test-template', required_personalisations=['first name']),
            ])
            client = NotifyClient.shared_client()
            personalisation = {
                'first name': 'Sample',
            }
            mock_send_email_response(rsps, '11', 'sample@localhost', personalisation=personalisation)
            client.send_email('test-template', 'sample@localhost', personalisation=personalisation)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_personalised_email_with_reference(self):
        # can send a personalised email with a reference by checking params posted to GOV.UK Notify
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps, templates=[
                fake_template('0000', 'generic', required_personalisations=['message']),
                fake_template('11', 'test-template', required_personalisations=['first name']),
            ])
            client = NotifyClient.shared_client()
            personalisation = {
                'first name': 'Sample',
                'last name': 'Name',
            }
            reference = 'registration-code-12345'
            mock_send_email_response(
                rsps, '11', 'sample@localhost', personalisation=personalisation, reference=reference,
            )
            client.send_email(
                'test-template', 'sample@localhost', personalisation=personalisation, reference=reference,
            )

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY,
                       GOVUK_NOTIFY_REPLY_TO_STAFF=GOVUK_NOTIFY_TEST_REPLY_TO_STAFF,
                       GOVUK_NOTIFY_REPLY_TO_PUBLIC=GOVUK_NOTIFY_TEST_REPLY_TO_PUBLIC)
    def test_choose_public_reply_to_address_by_default(self):
        # public site should default to using public reply-to address, but allows overriding
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '11', 'sample1@localhost', staff_email=False)
            client.send_email('test-template', 'sample1@localhost')
            mock_send_email_response(rsps, '11', 'sample1@localhost', staff_email=True)
            client.send_email('test-template', 'sample1@localhost', staff_email=True)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY,
                       GOVUK_NOTIFY_REPLY_TO_STAFF=GOVUK_NOTIFY_TEST_REPLY_TO_STAFF,
                       GOVUK_NOTIFY_REPLY_TO_PUBLIC=GOVUK_NOTIFY_TEST_REPLY_TO_PUBLIC,
                       MOJ_INTERNAL_SITE=True)
    def test_choose_staff_reply_to_address_if_internal_app(self):
        # internal sites should default to using staff reply-to address, but allow overriding
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '11', 'sample1@localhost', staff_email=True)
            client.send_email('test-template', 'sample1@localhost')
            mock_send_email_response(rsps, '11', 'sample1@localhost', staff_email=False)
            client.send_email('test-template', 'sample1@localhost', staff_email=False)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_plain_text_email(self):
        # can send a plain text email by checking params posted to GOV.UK Notify
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            subject = 'Test subject'
            message = 'Unformatted message\nwithout links'
            mock_send_email_response(rsps, '0000', 'sample@localhost', personalisation={
                'subject': subject,
                'message': message,
            })
            client.send_plain_text_email('sample@localhost', subject, message)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
    def test_send_email_with_files(self):
        # can send an email attaching files and byte content (GOV.UK Notify converts these to download links)
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps, templates=[
                fake_template('0000', 'generic', required_personalisations=['message']),
                fake_template('333', 'two-files', required_personalisations=['open file', 'byte content']),
            ])
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '333', 'sample@localhost', personalisation={
                'open file': {
                    'file': 'AAAAAAA=', 'is_csv': False,
                    'confirm_email_before_download': False, 'retention_period': '52 weeks',
                },
                'byte content': {
                    'file': 'MTIzNDU=', 'is_csv': False,
                    'confirm_email_before_download': False, 'retention_period': '52 weeks',
                },
            })
            client.send_email('two-files', 'sample@localhost', personalisation={
                # passes an open file handle
                'open file': (pathlib.Path(__file__).parent / 'test_notify-5-null-bytes.bin').open('rb'),
                # passes byte content directly
                'byte content': b'12345',
            })

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY,
                       GOVUK_NOTIFY_BLOCKED_DOMAINS={'mtp.local', 'localhost'},
                       ENVIRONMENT='test')
    @mock.patch('mtp_common.notify.client.logger')
    def test_emails_not_sent_to_blocked_domains(self, mock_logger):
        # in non-prod environments, specified email domain will not have emails sent to them
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            message_ids = client.send_email('test-template', ['sample@mtp.local', 'root@localhost'])
        self.assertSequenceEqual(message_ids, [None, None])
        self.assertEqual(mock_logger.warning.call_count, 2)

        NotifyClient.shared_client.cache_clear()
        mock_logger.reset_mock()

        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '12', 'sample@outside.local')
            message_ids = client.send_email('test-template2', ['sample@outside.local', 'root@localhost'])
        self.assertEqual(len(message_ids), 2)
        self.assertIsNotNone(message_ids[0])
        self.assertIsNone(message_ids[1])
        self.assertEqual(mock_logger.warning.call_count, 1)

    @override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY,
                       GOVUK_NOTIFY_BLOCKED_DOMAINS={'mtp.local', 'localhost'},
                       ENVIRONMENT='prod')
    @mock.patch('mtp_common.notify.client.logger')
    def test_emails_still_sent_to_blocked_domains_in_production(self, mock_logger):
        # in prod environment, emails are always sent
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '11', 'sample@mtp.local')
            mock_send_email_response(rsps, '11', 'root@localhost')
            message_ids = client.send_email('test-template', ['sample@mtp.local', 'root@localhost'])
        self.assertEqual(len(message_ids), 2)
        self.assertNotIn(None, message_ids)

        NotifyClient.shared_client.cache_clear()

        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            client = NotifyClient.shared_client()
            mock_send_email_response(rsps, '12', 'sample@outside.local')
            mock_send_email_response(rsps, '12', 'root@localhost')
            message_ids = client.send_email('test-template2', ['sample@outside.local', 'root@localhost'])
        self.assertEqual(len(message_ids), 2)
        self.assertNotIn(None, message_ids)

        mock_logger.warning.assert_not_called()

@override_settings(EMAIL_BACKEND='mtp_common.notify.email_backend.NotifyEmailBackend',
                   GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
class NotifyEmailBackendTestCase(NotifyBaseTestCase):
    def test_email_is_sent_using_notify(self):
        # subject and body both appear in the plain text message body of the email
        with responses.RequestsMock() as rsps, silence_logger():
            mock_all_templates_response(rsps)
            mock_send_email_response(rsps, '0000', 'sample@localhost', personalisation={
                'subject': 'Email subject',
                'message': 'Unformatted message\nwithout links',
            })
            mail.EmailMessage(
                subject='Email subject', body='Unformatted message\nwithout links',
                to=['sample@localhost'],
            ).send()
        self.assertEqual(len(mail.outbox), 0)

    def test_email_is_sent_to_all_recipients(self):
        # CCed and BBCed addresses also get sent an individual email (GOV.UK Notify does not permit multiple recipients)
        personalisation = {
            'subject': 'Email title',
            'message': 'Body text\r\nFooter\r\n',
        }
        with responses.RequestsMock() as rsps, silence_logger():
            mock_all_templates_response(rsps)
            mock_send_email_response(rsps, '0000', 'sample@localhost', personalisation=personalisation)
            mock_send_email_response(rsps, '0000', 'sample1@localhost', personalisation=personalisation)
            mock_send_email_response(rsps, '0000', 'sample2@localhost', personalisation=personalisation)
            mock_send_email_response(rsps, '0000', 'sample3@localhost', personalisation=personalisation)
            mail.EmailMessage(
                subject='Email title', body='Body text\r\nFooter\r\n',
                to=['sample@localhost', 'sample1@localhost'], cc=['sample2@localhost'], bcc=['sample3@localhost'],
            ).send()
        self.assertEqual(len(mail.outbox), 0)

@override_settings(GOVUK_NOTIFY_API_KEY=GOVUK_NOTIFY_TEST_API_KEY)
class NotifyTemplateRegistryTestCase(NotifyBaseTestCase):
    # used to pretend that only the generic template is registered
    # so that this test case does not need to be updated for each new template registered with mtp-common
    _pretend_registered_templates = {'generic': {
        'subject': '((subject))',
        'body': '((message))',
        'personalisation': ['subject', 'message'],
    }}

    @responses.activate
    def test_templates_are_registered(self):
        # this test actually runs autodiscovery to register templates, all others mock out autodiscovery
        NotifyTemplateRegistry.get_all_templates.cache_clear()
        templates = NotifyTemplateRegistry.get_all_templates()
        self.assertIn('generic', templates)
        for template_details in templates.values():
            self.assertSetEqual(set(template_details.keys()), {'subject', 'body', 'personalisation'})

    @mock.patch.object(NotifyTemplateRegistry, 'get_all_templates', return_value=_pretend_registered_templates)
    def test_successful_template_checking(self, _mock_get_all_templates):
        # pretend that only the generic template exists and check that matching api response produces no errors
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            error, messages = NotifyTemplateRegistry.check_notify_templates()
            self.assertFalse(error)
            self.assertEqual(len(messages), 0)

    @mock.patch.object(NotifyTemplateRegistry, 'get_all_templates', return_value={'generic': {
        'subject': 'Prisoner money: ((subject))',
        'body': '((message))\n-----\nPrisoner money team',
        'personalisation': ['subject', 'message'],
    }})
    def test_template_checking_with_warnings(self, _mock_get_all_templates):
        # pretend that the expected subject and body for the generic email was different
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            error, messages = NotifyTemplateRegistry.check_notify_templates()
            self.assertFalse(error)
            self.assertListEqual(
                messages,
                ['Email template ‘generic’ has different subject',
                 'Email template ‘generic’ has different body copy']
            )

    @mock.patch.object(NotifyTemplateRegistry, 'get_all_templates', return_value={'generic': {
        'subject': '((subject))',
        'body': '((message))\n-----\n((footer))',
        'personalisation': ['subject', 'message', 'footer'],
    }})
    def test_template_checking_with_missing_personalisation(self, _mock_get_all_templates):
        # pretend that the generic email expected more personalisation
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            error, messages = NotifyTemplateRegistry.check_notify_templates()
            self.assertTrue(error)
            self.assertListEqual(
                messages,
                ['Email template ‘generic’ has different body copy',
                 'Email template ‘generic’ is missing required personalisation: footer']
            )

    @mock.patch.object(NotifyTemplateRegistry, 'get_all_templates', return_value={'generic': {
        'subject': '((subject))',
        'body': 'No personalised content in the body',
        'personalisation': ['subject'],
    }})
    def test_template_checking_with_unexpected_personalisation(self, _mock_get_all_templates):
        # pretend that the generic email expected less personalisation
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            error, messages = NotifyTemplateRegistry.check_notify_templates()
            self.assertTrue(error)
            self.assertListEqual(
                messages,
                ['Email template ‘generic’ has different body copy',
                 'Email template ‘generic’ requires unexpected personalisation: message']
            )

    @mock.patch.object(NotifyTemplateRegistry, 'get_all_templates', return_value=_pretend_registered_templates)
    def test_management_command(self, _mock_get_all_templates):
        with responses.RequestsMock() as rsps:
            mock_all_templates_response(rsps)
            call_command('check_notify_templates', verbosity=0)


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from . import views

router = DefaultRouter()
router.register('hello-viewset', views.HelloViewSet, basename='Hello-viewset')
router.register('profile-viewset', views.UserProfileViewSet)
router.register('feed', views.ProfileFeedViewSet)
urlpatterns = [
    path('hello-view/', views.HelloApi.as_view(), name='api'),
    path('',include(router.urls)),
    path('login/', views.UserLoginApi.as_view()),
]


import collections
from typing import Any, TypeVar

import tensorflow as tf

from tensorflow_federated.python.common_libs import structure
from tensorflow_federated.python.learning.optimizers import optimizer

_BETA_1_KEY = 'beta_1'
_BETA_2_KEY = 'beta_2'
_EPSILON_KEY = 'epsilon'
_STEP_KEY = 'step'
_ACCUMULATOR_KEY = 'accumulator'
_PRECONDITIONER_KEY = 'preconditioner'
_HPARAMS_KEYS = [
    optimizer.LEARNING_RATE_KEY,
    _BETA_1_KEY,
    _BETA_2_KEY,
    _EPSILON_KEY,
]

State = TypeVar('State', bound=collections.OrderedDict[str, Any])
Hparams = TypeVar('Hparams', bound=collections.OrderedDict[str, float])

class _Adam(optimizer.Optimizer[State, optimizer.Weights, Hparams]):
  """Adam optimizer, see `build_adam` for details."""

  def __init__(
      self,
      learning_rate: optimizer.Float,
      beta_1: optimizer.Float = 0.9,
      beta_2: optimizer.Float = 0.999,
      epsilon: optimizer.Float = 1e-7,
  ):
    """Initializes Adam optimizer."""
    if learning_rate < 0.0:
      raise ValueError(
          f'Adam `learning_rate` must be nonnegative, found {learning_rate}.'
      )
    if beta_1 < 0.0 or beta_1 > 1.0:
      raise ValueError(
          f'Adam `beta_1` must be in the range [0.0, 1.0], found {beta_1}.'
      )
    if beta_2 < 0.0 or beta_2 > 1.0:
      raise ValueError(
          f'Adam `beta_2` must be in the range [0.0, 1.0], found {beta_2}.'
      )
    if epsilon < 0.0:
      raise ValueError(f'Adam `epsilon` must be nonnegative, found {epsilon}.')
    self._lr = learning_rate
    self._beta_1 = beta_1
    self._beta_2 = beta_2
    self._epsilon = epsilon

  def initialize(self, specs: Any) -> State:
    initial_accumulator = tf.nest.map_structure(
        lambda s: tf.zeros(s.shape, s.dtype), specs
    )
    initial_preconditioner = tf.nest.map_structure(
        lambda s: tf.zeros(s.shape, s.dtype), specs
    )
    state = collections.OrderedDict([
        (optimizer.LEARNING_RATE_KEY, self._lr),
        (_BETA_1_KEY, self._beta_1),
        (_BETA_2_KEY, self._beta_2),
        (_EPSILON_KEY, self._epsilon),
        (_STEP_KEY, 0),
        (_ACCUMULATOR_KEY, initial_accumulator),
        (_PRECONDITIONER_KEY, initial_preconditioner),
    ])
    return state

  def next(
      self, state: State, weights: optimizer.Weights, gradients: Any
  ) -> tuple[State, optimizer.Weights]:
    gradients = optimizer.handle_indexed_slices_gradients(gradients)
    optimizer.check_weights_gradients_match(weights, gradients)
    lr = state[optimizer.LEARNING_RATE_KEY]
    beta_1 = state[_BETA_1_KEY]
    beta_2 = state[_BETA_2_KEY]
    epsilon = state[_EPSILON_KEY]
    step = state[_STEP_KEY] + 1
    accumulator = state[_ACCUMULATOR_KEY]
    preconditioner = state[_PRECONDITIONER_KEY]
    optimizer.check_weights_state_match(weights, accumulator, 'accumulator')
    optimizer.check_weights_state_match(
        weights, preconditioner, 'preconditioner'
    )

    updated_accumulator = tf.nest.map_structure(
        lambda a, g: a + (g - a) * (1 - beta_1), accumulator, gradients
    )
    updated_preconditioner = tf.nest.map_structure(
        lambda s, g: s + (tf.math.square(g) - s) * (1 - beta_2),
        preconditioner,
        gradients,
    )
    normalized_lr = (
        lr
        * tf.math.sqrt((1 - tf.math.pow(beta_2, tf.cast(step, tf.float32))))
        / (1 - tf.math.pow(beta_1, tf.cast(step, tf.float32)))
    )
    updated_weights = tf.nest.map_structure(
        lambda w, g, a, s: w - normalized_lr * a / (tf.math.sqrt(s) + epsilon),
        weights,
        gradients,
        updated_accumulator,
        updated_preconditioner,
    )

    updated_state = collections.OrderedDict([
        (optimizer.LEARNING_RATE_KEY, lr),
        (_BETA_1_KEY, beta_1),
        (_BETA_2_KEY, beta_2),
        (_EPSILON_KEY, epsilon),
        (_STEP_KEY, step),
        (_ACCUMULATOR_KEY, updated_accumulator),
        (_PRECONDITIONER_KEY, updated_preconditioner),
    ])
    return updated_state, updated_weights

  def get_hparams(self, state: State) -> Hparams:
    return collections.OrderedDict([(k, state[k]) for k in _HPARAMS_KEYS])

  def set_hparams(self, state: State, hparams: Hparams) -> State:
    # TODO: b/245962555 - Find an alternative to `update_struct` if it
    # interferes with typing guarantees.
    # We use `tff.structure.update_struct` (rather than something like
    # `copy.deepcopy`) to ensure that this can be called within a
    # `tff.Computation`.
    return structure.update_struct(state, **hparams)

def build_adam(
    learning_rate: optimizer.Float,
    beta_1: optimizer.Float = 0.9,
    beta_2: optimizer.Float = 0.999,
    epsilon: optimizer.Float = 1e-7,
) -> optimizer.Optimizer:
  """Returns a `tff.learning.optimizers.Optimizer` for Adam.

  The Adam optimizer is based on [Adam: A Method for Stochastic Optimization](
  https://arxiv.org/pdf/1412.6980.pdf)

  The update rule given learning rate `lr`, epsilon `eps`, accumulator `acc`,
  preconditioner `s`, iteration `t`, weights `w` and gradients `g` is:

  ```
  acc = beta_1 * acc + (1 - beta_1) * g
  s = beta_2 * s + (1 - beta_2) * g**2
  normalized_lr = lr * sqrt(1 - beta_2**t) / (1 - beta_1**t)
  w = w - normalized_lr * acc / (sqrt(s) + eps)
  ```

  Args:
    learning_rate: A positive `float` for learning rate.
    beta_1: A `float` between `0.0` and `1.0` for the decay used to track the
      previous gradients.
    beta_2: A `float` between `0.0` and `1.0` for the decay used to track the
      magnitude (second moment) of previous gradients.
    epsilon: A small non-negative `float`, used to maintain numerical stability.
  """
  return _Adam(learning_rate, beta_1, beta_2, epsilon)


from django.test import TestCase
from .models import City, Reservation
from .views import total_commission, monthly_commission, city_commissions
import datetime

class ModelsTestCase(TestCase):
    def test_city_has_name(self):
        city = City.objects.create(city_name="TheName")
        city.rate = 0.10
        city.save()
        self.assertEqual(city.city_name, "TheName")

    def test_city_has_rate(self):
        city = City.objects.create(rate=0.10)
        city.city_name = "TheName"
        city.save()
        self.assertEqual(city.rate, 0.10)

    def test_reservation_has_all_data(self):
        city = City.objects.create(city_name="TheName", rate=0.10)
        city.save()

        checkin = datetime.datetime.fromtimestamp(1623849246)
        checkout = datetime.datetime.fromtimestamp(1626441246)
        reservation = Reservation.objects.create(
            reservation='HM123',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat',
            income=1900,
            city=city
        )
        reservation.save()

        self.assertEqual(reservation.reservation, 'HM123')
        self.assertEqual(reservation.checkin_date, checkin)
        self.assertEqual(reservation.checkout_date, checkout)
        self.assertEqual(reservation.flat, 'Flat')
        self.assertEqual(reservation.income, 1900)
        self.assertEqual(reservation.city.city_name, "TheName")
        self.assertEqual(reservation.city.rate, 0.10)

    def test_total_commission(self):
        city1 = City.objects.create(city_name="TheName", rate=0.10)
        city1.save()

        city2 = City.objects.create(city_name="TheName2", rate=0.20)
        city2.save()

        checkin = datetime.datetime.fromtimestamp(1623849246)
        checkout = datetime.datetime.fromtimestamp(1626441246)
        reservation1 = Reservation.objects.create(
            reservation='HM123',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat',
            income=1000,
            city=city1
        )
        reservation1.save()

        reservation2 = Reservation.objects.create(
            reservation='HM124',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat1',
            income=1000,
            city=city2
        )
        reservation2.save()

        reservations = [reservation1, reservation2]
        self.assertEquals(total_commission(reservations), 0.1*1000+0.2*1000)

    def test_city_commission(self):
        city1 = City.objects.create(city_name="THENAME", rate=0.10)
        city1.save()

        checkin = datetime.datetime.fromtimestamp(1623849246)
        checkout = datetime.datetime.fromtimestamp(1626441246)
        reservation1 = Reservation.objects.create(
            reservation='HM123',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat',
            income=700,
            city=city1
        )
        reservation1.save()

        reservation2 = Reservation.objects.create(
            reservation='HM124',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat1',
            income=900,
            city=city1
        )
        reservation2.save()

        reservations = [reservation1, reservation2]
        self.assertEquals(city_commissions(reservations, "THENAME"), 0.1*700+0.1*900)

    def test_monthly_commission(self):
        city1 = City.objects.create(city_name="THENAME", rate=0.10)
        city1.save()

        checkin = datetime.datetime.fromtimestamp(1623849246)
        checkout = datetime.datetime.fromtimestamp(1626441246)
        reservation1 = Reservation.objects.create(
            reservation='HM123',
            checkin_date=checkin,
            checkout_date=checkout,
            flat='Flat',
            income=700,
            city=city1
        )
        reservation1.save()

        checkout2 = datetime.datetime.fromtimestamp(1624194846)
        reservation2 = Reservation.objects.create(
            reservation='HM124',
            checkin_date=checkin,
            checkout_date=checkout2,
            flat='Flat1',
            income=900,
            city=city1
        )
        reservation2.save()

        reservations = [reservation1, reservation2]
        month_dict = monthly_commission(reservations)
        for k,v in month_dict.items():
            month_dict[k] = round(v,2)
        self.assertEquals(month_dict, {'7/2021': 70.0, '6/2021': 90.0})

class ViewsTestCase(TestCase):
    def test_index_loads_properly(self):
        response = self.client.get('')
        self.assertEqual(response.status_code, 200)

    def test_upload_loads_properly(self):
        response = self.client.get('/upload')
        self.assertEqual(response.status_code, 200)

    def test_random_loads_404(self):
        response = self.client.get('/randomurl')
        self.assertEqual(response.status_code, 404)


import unittest
from uuid import uuid1
from pyvcloud.vcd.client import ApiVersion
from pyvcloud.vcd.client import E
from pyvcloud.vcd.client import EntityType
from pyvcloud.vcd.client import find_link
from pyvcloud.vcd.client import GatewayBackingConfigType
from pyvcloud.vcd.client import NSMAP
from pyvcloud.vcd.client import QueryResultFormat
from pyvcloud.vcd.client import RelationType
from pyvcloud.vcd.client import ResourceType
from pyvcloud.vcd.client import TaskStatus
from pyvcloud.vcd.extension import Extension
from pyvcloud.system_test_framework.base_test import BaseTestCase
from pyvcloud.system_test_framework.environment import CommonRoles
from pyvcloud.system_test_framework.environment import Environment
from pyvcloud.system_test_framework.constants.gateway_constants import \
    GatewayConstants
from pyvcloud.vcd.gateway import Gateway
from pyvcloud.vcd.platform import Platform
from pyvcloud.vcd.utils import netmask_to_cidr_prefix_len

class TestGateway(BaseTestCase):
    """Test Gateway functionalities implemented in pyvcloud."""
    # All tests in this module should be run as System Administrator.
    _client = None
    _name = (GatewayConstants.name + str(uuid1()))[:34]
    _description = GatewayConstants.description + str(uuid1())
    _gateway = None
    _rate_limit_start = '101.0'
    _rate_limit_end = '101.0'

    # Firewall Rule
    _firewall_rule_name = 'Rule Name Test'

    # DHCP pool
    _pool_ip_range = '30.20.10.110-30.20.10.112'

    # DHCP binding
    _mac_address = '00:14:22:01:23:45'
    _host_name = 'xyzName'
    _binding_ip_address = '10.20.30.40'
    _syslog_server_ip1 = '10.40.40.40'
    _ip_address_for_config_ip_setting = '2.2.3.3'
    _ip_address_for_ip_range = '2.2.3.4-2.2.3.4'

    def test_0000_setup(self):
        """Setup the gateway required for the other tests in this module.

        Create a gateway as per the configuration stated
        above.

        This test passes if the gateway is created successfully.
        """
        TestGateway._client = Environment.get_sys_admin_client()
        TestGateway._vdc = Environment.get_test_vdc(TestGateway._client)

        TestGateway._org_client = Environment.get_client_in_default_org(
            CommonRoles.ORGANIZATION_ADMINISTRATOR)
        TestGateway._config = Environment.get_config()
        TestGateway._api_version = TestGateway._config['vcd']['api_version']

        external_network = Environment.get_test_external_network(
            TestGateway._client)

        ext_net_resource = external_network.get_resource()
        ip_scopes = ext_net_resource.xpath(
            'vcloud:Configuration/vcloud:IpScopes/vcloud:IpScope',
            namespaces=NSMAP)
        first_ipscope = ip_scopes[0]
        gateway_ip = first_ipscope.Gateway.text
        prefix_len = netmask_to_cidr_prefix_len(gateway_ip,
                                                first_ipscope.Netmask.text)
        subnet_addr = gateway_ip + '/' + str(prefix_len)
        ext_net_to_participated_subnet_with_ip_settings = {
            ext_net_resource.get('name'): {
                subnet_addr: self._ip_address_for_config_ip_setting
            }
        }

        ext_net_to_subnet_with_ip_range = {
            ext_net_resource.get('name'): {
                subnet_addr: [self._ip_address_for_ip_range]
            }
        }
        ext_net_to_rate_limit = {ext_net_resource.get('name'): {100: 100}}
        if float(TestGateway._api_version) <= float(
                ApiVersion.VERSION_30.value):
            TestGateway._gateway = \
                TestGateway._vdc.create_gateway_api_version_30(
                    self._name, [ext_net_resource.get('name')], 'compact',
                    None,
                    True, ext_net_resource.get('name'), gateway_ip, True,
                    False,
                    False, False, True,
                    ext_net_to_participated_subnet_with_ip_settings, True,
                    ext_net_to_subnet_with_ip_range, ext_net_to_rate_limit)
        elif float(TestGateway._api_version) == float(
                ApiVersion.VERSION_31.value):
            TestGateway._gateway = \
                TestGateway._vdc.create_gateway_api_version_31(
                    self._name, [ext_net_resource.get('name')], 'compact',
                    None, True, ext_net_resource.get('name'), gateway_ip,
                    True, False, False, False, True,
                    ext_net_to_participated_subnet_with_ip_settings, True,
                    ext_net_to_subnet_with_ip_range, ext_net_to_rate_limit)
        elif float(TestGateway._api_version) >= float(
                ApiVersion.VERSION_32.value):
            TestGateway._gateway = \
                TestGateway._vdc.create_gateway_api_version_32(
                    self._name, [ext_net_resource.get('name')], 'compact',
                    None, True, ext_net_resource.get('name'), gateway_ip,
                    True, False, False, False, True,
                    ext_net_to_participated_subnet_with_ip_settings, True,
                    ext_net_to_subnet_with_ip_range, ext_net_to_rate_limit)

        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=TestGateway._gateway.Tasks.Task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

        TestGateway._extension = Extension(TestGateway._client)
        TestGateway._extension.get_resource()
        link = find_link(TestGateway._extension.resource, RelationType.DOWN,
                         EntityType.SYSTEM_SETTINGS.value)
        settings = TestGateway._client.get_resource(link.href)
        syslog_server_settings = settings.GeneralSettings.SyslogServerSettings

        if hasattr(syslog_server_settings,
                   '{' + NSMAP['vcloud'] + '}SyslogServerIp1'):
            return
        syslog_server_settings.append(
            E.SyslogServerIp1(TestGateway._syslog_server_ip1))
        TestGateway._client.put_resource(link.href, settings,
                                         EntityType.SYSTEM_SETTINGS.value)
        TestGateway._extension.reload()
        settings = TestGateway._client.get_resource(link.href)
        self.assertTrue(
            hasattr(syslog_server_settings, '{' + NSMAP['vcloud'] +
                    '}SyslogServerIp1'))

    def test_0001_convert_to_advanced(self):
        """Convert the legacy gateway to advance gateway.

        Invoke the convert_to_advanced method for gateway.
        """
        if float(TestGateway._api_version) >= float(
                ApiVersion.VERSION_32.value):
            return
        gateway_obj = Gateway(TestGateway._org_client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.convert_to_advanced()
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0002_enable_dr(self):
        """Enable the Distributed routing.

        Invoke the enable_distributed_routing method for the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.enable_distributed_routing(True)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0003_modify_form_factor(self):
        """Modify form factor.

        Invoke the modify_form_factor method for the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.modify_form_factor(
            GatewayBackingConfigType.FULL.value)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0004_list_external_network_ip_allocations(self):
        """List external network ip allocations.

        Invoke the list_external_network_ip_allocations of the gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            ip_allocations = gateway_obj.list_external_network_ip_allocations()
            self.assertTrue(bool(ip_allocations))

    def test_0005_redeploy(self):
        """Redeploy the gateway.

        Invoke the redeploy function of gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            task = gateway_obj.redeploy()
            result = TestGateway._client.get_task_monitor().wait_for_success(
                task=task)
            self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0006_sync_syslog_settings(self):
        """Sync syslog settings of the gateway.

        Invoke the sync_syslog_settings function of gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            task = gateway_obj.sync_syslog_settings()
            result = TestGateway._client.get_task_monitor().wait_for_success(
                task=task)
            self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0010_set_tenant_syslog_server_ip(self):
        """Set Tenant syslog server IP of the gateway.

        Invoke the set_tenant_syslog_server_ip function of gateway.
        """

        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.set_tenant_syslog_server_ip('192.168.5.6')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0015_list_external_network_config_ip_allocations(self):
        """List external network configure ip allocations.

        Invoke the list_gateways_configure_ip_settings of the gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            ip_allocations = gateway_obj.list_configure_ip_settings()
            platform = Platform(TestGateway._client)
            external_networks = platform.list_external_networks()
            self.assertTrue(bool(ip_allocations))
            exnet = ip_allocations[0].get('external_network')
            self.assertEqual(external_networks[0].get('name'), exnet)

    def _create_external_network(self):
        """Creates an external network from the available portgroup."""
        vc_name = TestGateway._config['vc']['vcenter_host_name']
        name_filter = ('vcName', vc_name)
        query = TestGateway._client.get_typed_query(
            ResourceType.PORT_GROUP.value,
            query_result_format=QueryResultFormat.RECORDS,
            equality_filter=name_filter)

        port_group = None
        for record in list(query.execute()):
            if record.get('networkName') == '--':
                if not record.get('name').startswith('vxw-'):
                    port_group = record.get('name')
                    break

        if port_group is None:
            raise Exception(
                'None of the port groups are free for new network.')

        name = 'external_network_' + str(uuid1())
        platform = Platform(TestGateway._client)
        ext_net = platform.create_external_network(
            name=name,
            vim_server_name=vc_name,
            port_group_names=[port_group],
            gateway_ip='10.10.30.1',
            netmask='255.255.255.0',
            ip_ranges=['10.10.30.101-10.10.30.150'],
            description=name,
            primary_dns_ip='8.8.8.8',
            secondary_dns_ip='8.8.8.9',
            dns_suffix='example.com')

        task = ext_net['{' + NSMAP['vcloud'] + '}Tasks'].Task[0]
        TestGateway._client.get_task_monitor().wait_for_success(task=task)
        TestGateway._external_network2 = ext_net
        return ext_net

    def _delete_external_network(self, network):
        logger = Environment.get_default_logger()
        platform = Platform(TestGateway._client)
        task = platform.delete_external_network(network.get('name'))
        TestGateway._client.get_task_monitor().wait_for_success(task=task)
        logger.debug('Deleted external network ' + network.get('name') + '.')

    def test_0020_add_external_network(self):
        """Add an exernal netowrk to the gateway.

        Invoke the add_external_network function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))

        extNw2 = self._create_external_network()
        ip_scopes = extNw2.xpath(
            'vcloud:Configuration/vcloud:IpScopes/vcloud:IpScope',
            namespaces=NSMAP)
        first_ipscope = ip_scopes[0]
        gateway_ip = first_ipscope.Gateway.text
        prefixlen = netmask_to_cidr_prefix_len(gateway_ip,
                                               first_ipscope.Netmask.text)
        subnet_addr = gateway_ip + '/' + str(prefixlen)

        task = gateway_obj.add_external_network(
            extNw2.get('name'), [(subnet_addr, 'Auto')])
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0025_remove_external_network(self):
        """Remove an exernal netowrk from the gateway.

        Invoke the remove_external_network function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.remove_external_network(
            TestGateway._external_network2.get('name'))
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

        self._delete_external_network(TestGateway._external_network2)

    def test_0030_edit_gateway(self):
        """Edit the gateway name.

        Invokes the edit_gateway of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.edit_gateway(newname='gateway2')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        '''resetting back to original gateway name'''
        task = gateway_obj.edit_gateway(TestGateway._name)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0035_edit_config_ipaddress(self):
        """It edits the config ip settings of gateway.
        In this user can only modify Subnet participation and config Ip address
        of gateway's external network.

        Invokes the edit_config_ip_settings of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ipconfig = dict()
        subnet_participation = dict()
        subnet = dict()
        subnet_participation['enable'] = True
        subnet_participation['ip_address'] = ip_allocation.get('ip_address')[0]
        subnet[ip_allocation.get('gateway')[0]] = subnet_participation

        ipconfig[ip_allocation.get('external_network')] = subnet
        task = gateway_obj.edit_config_ip_settings(ipconfig)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def __get_subnet_participation(self, gateway, ext_network):
        for gatewayinf in \
                gateway.Configuration.GatewayInterfaces.GatewayInterface:
            if gatewayinf.Name == ext_network:
                return gatewayinf.SubnetParticipation

    def __validate_ip_range(self, IpRanges, _ip_range1):
        """ Validate if the ip range present in the existing ip ranges """
        _ip_ranges = _ip_range1.split('-')
        _ip_range1_start_address = _ip_ranges[0]
        _ip_range1_end_address = _ip_ranges[1]
        for ip_range in IpRanges.IpRange:
            if ip_range.StartAddress == _ip_range1_start_address:
                self.assertEqual(ip_range.EndAddress, _ip_range1_end_address)

    def test_0040_add_sub_allocated_ip_pools(self):
        """It adds the sub allocated ip pools to gateway.

        Invokes the add_sub_allocated_ip_pools of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        config = TestGateway._config['external_network']
        gateway_sub_allocated_ip_range = \
            config['gateway_sub_allocated_ip_range']
        ip_range_list = [gateway_sub_allocated_ip_range]

        task = gateway_obj.add_sub_allocated_ip_pools(ext_network,
                                                      ip_range_list)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        subnet_participation = self.__get_subnet_participation(
            gateway_obj.get_resource(), ext_network)
        ip_ranges = gateway_obj.get_sub_allocate_ip_ranges_element(
            subnet_participation)
        self.__validate_ip_range(ip_ranges, gateway_sub_allocated_ip_range)

    def test_0045_edit_sub_allocated_ip_pools(self):
        """It edits the sub allocated ip pools of gateway.

        Invokes the edit_sub_allocated_ip_pools of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        config = TestGateway._config['external_network']
        gateway_sub_allocated_ip_range = \
            config['gateway_sub_allocated_ip_range']

        gateway_sub_allocated_ip_range1 = \
            config['new_gateway_sub_allocated_ip_range']

        task = gateway_obj.edit_sub_allocated_ip_pools(
            ext_network, gateway_sub_allocated_ip_range,
            gateway_sub_allocated_ip_range1)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        subnet_participation = self.__get_subnet_participation(
            gateway_obj.get_resource(), ext_network)
        ip_ranges = gateway_obj.get_sub_allocate_ip_ranges_element(
            subnet_participation)
        self.__validate_ip_range(ip_ranges, gateway_sub_allocated_ip_range)

    def test_0050_remove_sub_allocated_ip_pools(self):
        """Remove the sub allocated ip pools of gateway.

        Invokes the remove_sub_allocated_ip_pools of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        config = TestGateway._config['external_network']
        gateway_sub_allocated_ip_range1 = \
            config['new_gateway_sub_allocated_ip_range']

        task = gateway_obj.remove_sub_allocated_ip_pools(
            ext_network, [gateway_sub_allocated_ip_range1])
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        subnet_participation = self.__get_subnet_participation(
            gateway_obj.get_resource(), ext_network)
        """removed the IpRanges form subnet_participation."""
        is_ip_range_found = False
        if hasattr(subnet_participation, 'IpRanges'):
            for ip_range in subnet_participation.IpRanges.IpRange:
                if gateway_sub_allocated_ip_range1 == \
                        ip_range.StartAddress.text+'-'+ip_range.EndAddress\
                        .text:
                    is_ip_range_found = True
                    break
        self.assertFalse(is_ip_range_found)

    def test_0055_edit_rate_limit(self):
        """Edits existing rate limit of gateway.

        Invokes the edit_rate_limits of the gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        config = dict()
        config[ext_network] = [self._rate_limit_start, self._rate_limit_end]

        task = gateway_obj.edit_rate_limits(config)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        for gateway_inf in \
                gateway_obj.get_resource()\
                        .Configuration.GatewayInterfaces.GatewayInterface:
            if gateway_inf.Name == ext_network:
                self.assertEqual(self._rate_limit_start,
                                 gateway_inf.InRateLimit.text)
                self.assertEqual(self._rate_limit_end,
                                 gateway_inf.OutRateLimit.text)

    def test_0060_list_syslog_settings(self):
        """List Tenant syslog server of the gateway.

        Invoke the list_syslog_server_ip function of gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            tenant_syslog_server = gateway_obj.list_syslog_server_ip()
            self.assertGreaterEqual(len(tenant_syslog_server), 1)

    def test_0065_list_rate_limit(self):
        """List rate limit of the gateway.

        Invoke the list_rate_limits function of gateway.
        """
        for client in (TestGateway._client, TestGateway._org_client):
            gateway_obj = Gateway(client, self._name,
                                  TestGateway._gateway.get('href'))
            rate_limit = gateway_obj.list_rate_limits()
            self.assertTrue(len(rate_limit) > 0)

    def __get_gateway_interface(self, gateway, ext_network):
        for gateway_inf in \
                gateway.Configuration.GatewayInterfaces.GatewayInterface:
            if gateway_inf.Name == ext_network:
                return gateway_inf

    def test_0070_disable_rate_limit(self):
        """Disable rate limit of the gateway.

        Invoke the disable_rate_limits function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.disable_rate_limits([ext_network])
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        # verification
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        gateway_interface = self.__get_gateway_interface(
            gateway_obj.get_resource(), ext_network)
        """removed the InRateLimit form gateway_interface."""
        self.assertFalse(hasattr(gateway_interface, 'InRateLimit'))

    def test_0075_configure_gateway(self):
        """configures the gateway.

        Invoke the configure_default_gateway function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        gateway = ip_allocation.get('gateway')
        gateway_ip = gateway[0].split('/')
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.configure_default_gateway(ext_network,
                                                     gateway_ip[0], 'true')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        # verification
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        gateway_interface = self.__get_gateway_interface(
            gateway_obj.get_resource(), ext_network)
        self.assertTrue(gateway_interface.UseForDefaultRoute)

    def test_0080_enable_dns_relay_gateway(self):
        """enables the dns relay of the gateway.

        Invoke the configure_dns_default_gateway function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.configure_dns_default_gateway('true')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        # verification
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        self.assertTrue(gateway_obj.get_resource()
                        .Configuration.UseDefaultRouteForDnsRelay)
        task = gateway_obj.configure_dns_default_gateway('false')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0085_list_configure_default_gateway(self):
        """list configured default gateway.

        Invoke the list_configure_default_gateway function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        default_gateways = gateway_obj.list_configure_default_gateway()
        self.assertTrue(len(default_gateways) > 0)

    def test_0090_disable_configure_gateway(self):
        """configures the gateway.

        Invoke the configure_default_gateway function of gateway.
        """
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        ip_allocations = gateway_obj.list_configure_ip_settings()
        ip_allocation = ip_allocations[0]
        ext_network = ip_allocation.get('external_network')
        gateway = ip_allocation.get('gateway')
        gateway_ip = gateway[0].split('/')
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        task = gateway_obj.configure_default_gateway(ext_network,
                                                     gateway_ip[0], 'false')
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)
        # verification
        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        gateway_interface = self.__get_gateway_interface(
            gateway_obj.get_resource(), ext_network)
        self.assertTrue(gateway_interface.UseForDefaultRoute.text == 'false')

    def test_0095_add_firewall_rule(self):
        """Add Firewall Rule's in the gateway."""

        gateway_obj = Gateway(TestGateway._client, self._name,
                              TestGateway._gateway.get('href'))
        gateway_obj.add_firewall_rule(TestGateway._firewall_rule_name)
        firewall_rule = gateway_obj.get_firewall_rules()
        # Verify
        matchFound = False
        for firewallRule in firewall_rule.firewallRules.firewallRule:
            if firewallRule['name'] == TestGateway._firewall_rule_name:
                matchFound = True
                break
        self.assertTrue(matchFound)

    def test_0100_add_dhcp_pool(self):
        """Add DHCP pool in the gateway.
         Invokes the add_dhcp_pool of the gateway.
        """

        gateway_obj = Gateway(
            TestGateway._client, self._name,
            Environment.get_test_gateway(Environment.get_sys_admin_client())
            .get('href'))
        gateway_obj.add_dhcp_pool(TestGateway._pool_ip_range)
        dhcp_resource = gateway_obj.get_dhcp()
        # Verify
        matchFound = False
        for ipPool in dhcp_resource.ipPools.ipPool:
            if ipPool.ipRange.text == TestGateway._pool_ip_range:
                matchFound = True
                break
        self.assertTrue(matchFound)

    def test_0105_add_dhcp_binding(self):
        """Add DHCP Binding in the gateway.

         Invokes the add_dhcp_binding of the gateway.
        """
        gateway_obj = Gateway(
            TestGateway._client, self._name,
            Environment.get_test_gateway(Environment.get_sys_admin_client())
            .get('href'))
        gateway_obj.add_dhcp_binding(TestGateway._mac_address,
                                     TestGateway._host_name,
                                     TestGateway._binding_ip_address)
        dhcp_resource = gateway_obj.get_dhcp()
        # Verify
        matchFound = False
        for static_binding in dhcp_resource.staticBindings.staticBinding:
            if static_binding.macAddress.text == TestGateway._mac_address:
                matchFound = True
                break
        self.assertTrue(matchFound)

    def test_1000_teardown(self):
        """Test the method System.delete_gateway().

        Invoke the method for the gateway created by setup.

        This test passes if no errors are generated while deleting the gateway.
        """
        vdc = Environment.get_test_vdc(TestGateway._client)
        task = vdc.delete_gateway(TestGateway._name)
        result = TestGateway._client.get_task_monitor().wait_for_success(
            task=task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_1010_cleanup(self):
        """Release all resources held by this object for testing purposes."""
        TestGateway._client.logout()

if __name__ == '__main__':
    unittest.main()


import random
import sys
from bisect import bisect_right
from typing import Iterator, Optional, Tuple, List

import pyarrow as pa
import pandas as pd
from pyarrow.dataset import ParquetFileFragment
from pyarrow.lib import RecordBatch
from pyarrow.parquet import ParquetDataset, ParquetFile
from torch.utils.data import Dataset

class PyArrowDataset(Dataset):
    def __init__(self, source: str, shuffle: bool = False, seed: int = 123):
        random.seed(seed)
        self.source = source
        self.seed = seed

        # Pyarrow
        self.dataset = ParquetDataset(source, use_legacy_dataset=False)
        self.parquet_indices: List[Tuple[int, int, int, ParquetFile, int, int]] = []
        self._cur_meta = None
        self._df: Optional[pd.DataFrame] = None

        # Debug (Memory Profiling)
        self.read_cnt = 0

        self.init_parquet_indexing(shuffle)

    def init_parquet_indexing(self, shuffle: bool = False):
        fragments = self.dataset.fragments
        if shuffle:
            random.shuffle(fragments)

        idx = 0
        parquet_indices = []
        for frag in fragments:
            parquet_file = ParquetFile(frag.path)
            for i, row_group in enumerate(frag.row_groups):
                start_idx = idx  # inclusive
                end_idx = idx + row_group.num_rows  # exclusive
                parquet_indices.append((i, start_idx, end_idx, parquet_file, row_group.id, row_group.num_rows))
                idx += row_group.num_rows

        self.parquet_indices.clear()
        self.parquet_indices = parquet_indices

    def __len__(self):
        if not self.parquet_indices:
            return 0
        return self.parquet_indices[-1][-1]

    def __getitem__(self, idx: int):
        meta_idx = self._binary_search(idx)

        if self._cur_meta is not None and meta_idx == self._cur_meta[0]:
            start_idx = self._cur_meta[1]
            # print(f'total: {len(self)} | {idx - start_idx}')
            assert (idx - start_idx) >= 0, f'{idx} - {start_idx} = {idx - start_idx} <- should not be negative.'
            return self._df.iloc[idx - start_idx]

        # Clear memory references
        del self._df
        del self._cur_meta
        self.read_cnt += 1
        # Read a new Parquet File
        self._cur_meta = self.parquet_indices[meta_idx]
        start_idx = self._cur_meta[1]
        parquet_file = self._cur_meta[3]
        row_id = self._cur_meta[4]
        table: pa.Table = parquet_file.read_row_group(row_id)
        self._df = table.to_pandas()

        return self._df.iloc[idx - start_idx]

    def _binary_search(self, target: int):
        arr = self.parquet_indices
        n = len(arr)
        left, right = 0, n

        while left <= right:
            mid = (left + right) // 2
            _, start_idx, end_idx, _, _, _ = arr[mid]
            if target == start_idx:
                return mid
            elif target == end_idx:
                return mid + 1
            elif start_idx <= target < end_idx:
                return mid
            elif target <= end_idx:
                right = mid - 1
            else:
                left = mid + 1
        return left


import numpy as np
import numba

from pax import plugin, datastructure, recarray_tools, dsputils

class SumWaveform(plugin.TransformPlugin):

    def startup(self):
        self.detector_by_channel = dsputils.get_detector_by_channel(self.config)

    def transform_event(self, event):

        # Initialize empty waveforms for each detector
        # One with only hits, one with raw data
        for postfix in ('', '_raw'):
            for detector, chs in self.config['channels_in_detector'].items():
                event.sum_waveforms.append(datastructure.SumWaveform(
                    samples=np.zeros(event.length(), dtype=np.float32),
                    name=detector + postfix,
                    channel_list=np.array(list(chs), dtype=np.uint16),
                    detector=detector
                ))

        # Make dictionary mapping pulse -> non-rejected hits found in pulse
        # Assumes hits are still sorted by pulse
        event.all_hits = np.sort(event.all_hits, order='found_in_pulse')
        hits_per_pulse = recarray_tools.dict_group_by(event.all_hits[True ^ event.all_hits['is_rejected']],
                                                      'found_in_pulse')

        # Add top and bottom tpc sum waveforms
        for q in ('top', 'bottom'):
            event.sum_waveforms.append(datastructure.SumWaveform(
                samples=np.zeros(event.length(), dtype=np.float32),
                name='tpc_%s' % q,
                channel_list=np.array(self.config['channels_%s' % q], dtype=np.uint16),
                detector='tpc'
            ))

        for pulse_i, pulse in enumerate(event.pulses):
            channel = pulse.channel

            # Do some initialization only when we switch channel
            # The 'current_channel' variable can't be pulled outside the loop, that would hurt performance
            # (trust me, try it and time it)
            if pulse_i == 0 or channel != current_channel:      # noqa
                current_channel = channel                       # noqa
                detector = self.detector_by_channel[channel]
                adc_to_pe = dsputils.adc_to_pe(self.config, channel)

                if detector == 'tpc':
                    if channel in self.config['channels_top']:
                        sum_w = event.get_sum_waveform('tpc_top')
                    else:
                        sum_w = event.get_sum_waveform('tpc_bottom')
                else:
                    sum_w = event.get_sum_waveform(detector)

            # Don't consider dead channels
            if self.config['gains'][channel] == 0:
                continue

            # Get the pulse waveform in pe/bin
            baseline_to_subtract = self.config['digitizer_reference_baseline'] - pulse.baseline
            w = baseline_to_subtract - pulse.raw_data.astype(np.float64)
            w *= adc_to_pe

            sum_w_raw = event.get_sum_waveform(detector+'_raw').samples
            sum_w_raw[pulse.left:pulse.right+1] += w

            hits = hits_per_pulse.get(pulse_i, None)
            if hits is None:
                continue

            w_hits_only = w.copy()
            # Obtain an array of same length as w, indicating whether the sample is in a hit or not
            mask = np.zeros(len(w), dtype=np.bool)
            set_if_in_ranges(mask, hits['left'] - pulse.left, hits['right'] - pulse.left)
            w_hits_only[True ^ mask] = 0

            sum_w.samples[pulse.left:pulse.right+1] += w_hits_only

        # Sum the tpc top and bottom tpc waveforms
        event.get_sum_waveform('tpc').samples = event.get_sum_waveform('tpc_top').samples + \
            event.get_sum_waveform('tpc_bottom').samples

        return event

@numba.jit(numba.void(numba.bool_[:], numba.int64[:], numba.int64[:]), nopython=True)
def set_if_in_ranges(w, left, right):
    """Set samples in w to True between ranges indicated by left and right (both inclusive)"""
    for i in range(len(left)):
        w[left[i]:right[i]+1] = True


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import logging
import time
import random

import numpy as np
import pandas as pd
import torch
from torch_geometric.data import Data
from feature_engineer import dim_reduction, feature_generation
from torch_geometric.utils import degree, is_undirected
from sklearn.preprocessing import StandardScaler

logger = logging.getLogger('code_submission')

def fix_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True

def generate_pyg_data(data, n_class, time_budget, use_dim_reduction=True, use_feature_generation=True,
                      use_label_distribution=False, use_node_degree=False, use_node_degree_binary=False,
                      use_node_embed=True, use_one_hot_label=False):
    other_needed = dict()

    x = data['fea_table']

    df = data['edge_file']
    edge_index = df[['src_idx', 'dst_idx']].to_numpy()
    edge_index = sorted(edge_index, key=lambda d: d[0])
    edge_index = torch.tensor(edge_index, dtype=torch.long).transpose(0, 1)

    edge_weight = df['edge_weight'].to_numpy()
    edge_weight = torch.tensor(edge_weight, dtype=torch.float32)

    num_nodes = x.shape[0]
    y = n_class * torch.ones(num_nodes, dtype=torch.long)
    inds = data['train_label'][['node_index']].to_numpy()
    train_y = data['train_label'][['label']].to_numpy()
    y[inds] = torch.tensor(train_y, dtype=torch.long)

    train_indices = data['train_indices']
    test_indices = data['test_indices']

    flag_directed_graph = not is_undirected(edge_index)

    ###   feature engineering  ###
    flag_none_feature = False
    if use_dim_reduction:
        x, flag_none_feature = dim_reduction(x)
    else:
        x = x.to_numpy()
        flag_none_feature = (x.shape[1] == 1)

    if use_feature_generation:
        other_needed["x"] = x
        other_needed["y"] = y
        other_needed["n_class"] = n_class
        other_needed["edge_index"] = edge_index
        other_needed["edge_weight"] = edge_weight
        other_needed["time_budget"] = time_budget
        other_needed["flag_none_feature"] = flag_none_feature
        other_needed["flag_directed_graph"] = flag_directed_graph
        other_needed["use_label_distribution"] = use_label_distribution
        other_needed["use_node_degree"] = use_node_degree
        other_needed["use_node_degree_binary"] = use_node_degree_binary
        other_needed["use_node_embed"] = use_node_embed
        other_needed["use_one_hot_label"] = use_one_hot_label
        added_features = feature_generation(x, y, n_class, edge_index, edge_weight,
                                            flag_none_feature, flag_directed_graph, time_budget,
                                            use_label_distribution=use_label_distribution,
                                            use_node_degree=use_node_degree,
                                            use_node_degree_binary=use_node_degree_binary,
                                            use_node_embed=use_node_embed,
                                            use_one_hot_label=use_one_hot_label)
        if added_features:
            x = np.concatenate([x]+added_features, axis=1)

    only_one_hot_id = False
    if x.shape[1] != 1:
        #remove raw node_index
        x = x[:,1:]
    else:
        #one hot encoder of node_index (backup plan)
        x = np.eye(num_nodes)
        only_one_hot_id = True

    logger.info('x.shape after feature engineering: {}'.format(x.shape))
    x = torch.tensor(x, dtype=torch.float)

    non_zero_index = torch.nonzero(edge_weight).reshape(-1)
    edge_weight = edge_weight[non_zero_index]
    edge_index = edge_index[:,non_zero_index]
    data = Data(x=x, edge_index=edge_index, y=y, edge_weight=edge_weight)

    data.num_nodes = num_nodes
    data.train_indices = train_indices
    data.test_indices = test_indices

    train_mask = torch.zeros(num_nodes, dtype=torch.bool)
    train_mask[train_indices] = 1
    data.train_mask = train_mask

    test_mask = torch.zeros(num_nodes, dtype=torch.bool)
    test_mask[test_indices] = 1
    data.test_mask = test_mask

    data["directed"] = flag_directed_graph  # used for directed DGL-GCN

    return data, other_needed, only_one_hot_id

def get_label_weights(train_label, n_class):
    """
        return label balanced weights according to the label distribution
    """
    unique, counts = np.unique(train_label, return_counts=True)
    inversed_counts = 1.0 / counts
    # is_major = (counts > 100).astype(float)
    # inversed_counts = 1.0 / counts * is_major + 100.0 * (1.0 - is_major)

    normalize_factor = inversed_counts.sum()
    inversed_counts = inversed_counts / normalize_factor

    T = 0.7
    inversed_counts = np.power(inversed_counts, T)

    # return [1.0 / n_class] * n_class  # the same weights for all label class
    return inversed_counts

def generate_pyg_data_without_transform(data):
    x = data['fea_table']
    if x.shape[1] == 1:
        x = x.to_numpy()
        x = x.reshape(x.shape[0])
        x = np.array(pd.get_dummies(x))
    else:
        x = x.drop('node_index', axis=1).to_numpy()

    x = torch.tensor(x, dtype=torch.float)

    df = data['edge_file']
    edge_index = df[['src_idx', 'dst_idx']].to_numpy()
    edge_index = sorted(edge_index, key=lambda d: d[0])
    edge_index = torch.tensor(edge_index, dtype=torch.long).transpose(0, 1)

    edge_weight = df['edge_weight'].to_numpy()
    edge_weight = torch.tensor(edge_weight, dtype=torch.float32)

    num_nodes = x.size(0)
    y = torch.zeros(num_nodes, dtype=torch.long)
    inds = data['train_label'][['node_index']].to_numpy()
    train_y = data['train_label'][['label']].to_numpy()
    y[inds] = torch.tensor(train_y, dtype=torch.long)

    train_indices = data['train_indices']
    test_indices = data['test_indices']

    data = Data(x=x, edge_index=edge_index, y=y, edge_weight=edge_weight)

    data.num_nodes = num_nodes
    data.train_indices = train_indices
    data.test_indices = test_indices

    train_mask = torch.zeros(num_nodes, dtype=torch.bool)
    train_mask[train_indices] = 1
    data.train_mask = train_mask

    test_mask = torch.zeros(num_nodes, dtype=torch.bool)
    test_mask[test_indices] = 1
    data.test_mask = test_mask
    return data

def get_performance(valid_info):
    # the larger, the better
    # naive implementation
    # return -valid_info['logloss']+0.1*valid_info['accuracy']
    return valid_info['accuracy']

def hyperparam_space_tostr(hyperparam_space):
    hyperparam_space_str = '\n'
    for k, v in hyperparam_space.items():
        hyperparam_space_str = hyperparam_space_str + "%-15s: %s\n" % (k, v)
    return hyperparam_space_str

def divide_data(data, split_rates, device):
    # divide training data into several partitions
    indices = np.array(data.train_indices)
    np.random.shuffle(indices)

    split_thred = []
    accumulated_rate = 0
    for r in split_rates:
        accumulated_rate += r
        split_thred.append(int(len(indices)*accumulated_rate/np.sum(split_rates)))

    all_indices = list()
    prev = 0
    for i, end in enumerate(split_thred):
        part_indices = indices[prev:end] if i < len(split_thred)-1 else indices[prev:]
        prev = end
        all_indices.append(part_indices)

    masks = list()
    for i in range(len(all_indices)):
        part_masks = torch.zeros(data.num_nodes, dtype=torch.bool)
        part_masks[all_indices[i]] = 1
        masks.append(part_masks.to(device))
    return tuple(masks)

def calculate_config_dist(tpa, tpb):
    """Trivially calculate the distance of two configs"""

    ca, cb = tpa[0], tpb[0]
    num_diff_field = 0
    for k in ca:
        if ca[k] != cb[k]:
            num_diff_field += 1
    return num_diff_field

def divide_data_label_wise(data, split_rates, device, n_class, train_y):
    # divide training data into several partitions, according to the label distribution

    train_indices_label_wise = [[] for _ in range(n_class)]

    for i, train_idx in enumerate(data.train_indices):
        data_i_y = int(train_y[i])
        train_indices_label_wise[data_i_y].append(train_idx)

    all_indices = [[] for _ in range(len(split_rates))]
    for data_of_y_i in train_indices_label_wise:
        indices = np.array(data_of_y_i)
        np.random.shuffle(indices)

        split_thred = []
        accumulated_rate = 0
        for r in split_rates:
            accumulated_rate += r
            split_thred.append(int(len(indices)*accumulated_rate/np.sum(split_rates)))

        prev = 0
        for i, end in enumerate(split_thred):
            part_indices = indices[prev:end] if i < len(split_thred)-1 else indices[prev:]
            prev = end
            # y_i_indices.append(part_indices)
            all_indices[i].extend(part_indices)

    masks = list()
    for i in range(len(all_indices)):
        part_masks = torch.zeros(data.num_nodes, dtype=torch.bool)
        part_masks[all_indices[i]] = 1
        masks.append(part_masks.to(device))
    return tuple(masks)

def get_imbalanced_task_type(train_label, n_class):
    unique, counts = np.unique(train_label, return_counts=True)

    logger.info(str(' '.join([str(i) for i in list(unique)])))
    logger.info(str(' '.join([str(i) for i in list(counts)])))

    sorted_counts = np.sort(counts)
    # one major class
    if sorted_counts[-1] >= (0.8 * np.sum(sorted_counts)):
        return 1, (counts < sorted_counts[-1]).astype(np.float)
    # two major classes
    elif n_class >= 3 and ((sorted_counts[-1] + sorted_counts[-2]) >= (0.9 * np.sum(sorted_counts))):
        return 2, None
    else:
        return 0, None


from django.urls import path

from users.views import ProfileDetail, ProfileCreate, ProfileUpdate, ProfileDetailMe

urlpatterns = [
    path('create/', ProfileCreate.as_view()),
    path('detail/<int:pk>/', ProfileDetail.as_view()),
    path('detail/me/', ProfileDetailMe.as_view()),
    path('update/<int:pk>/', ProfileUpdate.as_view()),

]


from django.contrib import admin
from .models import PreRegistration
admin.site.site_header = "किताब खाना"
admin.site.site_title = 'किताब खाना'
admin.site.index_title ="किताब खाना"
# Register your models here.

@admin.register(PreRegistration)
class PreRegistrationAdmin(admin.ModelAdmin):
    list_display = ['username','first_name']


import csv
import json

csvfile = open('oregonstateuniversity.csv', 'r')
jsonfile = open('oregonstateuniversity.json', 'w')

fieldnames = ("edge_liked_by.count","edge_media_to_caption.edges","edge_media_to_comment.count","id", "owner.id", "tags", "taken_at_timestamp")
reader = csv.DictReader( csvfile, fieldnames)
out = json.dumps( [ row for row in reader ] )
jsonfile.write(out)


from enum import Enum
from PyQt5.QtCore import *

from modules.MapModelGeneral import *

class HouseSquareType(str, Enum):
    Empty       = 'Empty'
    Corner      = 'Corner'
    Wall        = 'Wall'
    Door        = 'Door'
    Window      = 'Window'
    Floor       = 'Floor'
    CornerInt   = 'CornerInternal'
    WallInt     = 'WallInternal'
    DoorInt     = 'DoorInternal'
    Wardrobe    = 'Wardrobe'
    Bed2p       = 'Bed2p'
    Drawer      = 'Drawer'
    SinkOven    = 'SinkOven'
    Fridge      = 'Fridge'
    Counter     = 'Counter'
    Sofa        = 'Sofa'
    TableChairs = 'TableChairs'
    Bookshelf   = 'Bookshelf'
    Light       = 'Light'

class HouseSquareRotation(str, Enum):
    deg0    = '0'
    deg90   = '90'
    deg180  = '180'
    deg270  = '270'

class HouseMapSquareModel(MapObjectModelGeneral, QObject):
    changed = pyqtSignal()

    def __init__(self):
        MapObjectModelGeneral.__init__(self, 0, 0)
        QObject.__init__(self)

        self.classnames['type']      = HouseSquareType
        self.classnames['rotation']  = HouseSquareRotation

        self.properties['type']      = HouseSquareType.Empty
        self.properties['rotation']  = HouseSquareRotation.deg0

    def setProperty(self, name, value):
        print("setProperty " + name + ": " + str(value))
        variableClass = self.classnames[name]
        self.properties[name] = variableClass(value);
        self.changed.emit()

class HouseMapModel(MapModelGeneral, QObject):
    updatedEntireMap = pyqtSignal()
    def __init__(self):
        MapModelGeneral.__init__(self, HouseMapSquareModel)
        QObject.__init__(self)

        self.setUpdatedCallback(self._updateeEntireMap)

    def _updateeEntireMap(self):
        self.updatedEntireMap.emit()


import pickle
import tensorflow as tf
import numpy as np
from random import *
import os
from simpleNN import simpleNN
import argparse

# tensorflow session environment setting
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.9
# config.gpu_options.allow_growth = True
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

# parse arguments
parser = argparse.ArgumentParser() # ArgumentParser 객체 생성
parser.add_argument("--dim_latent", type=int, default=32, help="latent dimension")
parser.add_argument("--batch_size", type=int, default=100, help="batch size")
parser.add_argument("--n_epoch", type=int, default=100, help="batch size")
parser.add_argument("--train", type=int, default=1, help="train flag")
parser.add_argument("--debug", type=int, default=0, help="debug flag")
args = parser.parse_args()

# preprocessing
PREPRO_DIR = './data/amazon/preprocessed'
SAVE_DIR = './save'
SAVE_FILE = os.path.join(SAVE_DIR, 'model')
with open(os.path.join(PREPRO_DIR, 'amazon.para'),'rb') as f:
	para = pickle.load(f)
with open(os.path.join(PREPRO_DIR, 'amazon.train'),'rb') as f:
	train = pickle.load(f)
with open(os.path.join(PREPRO_DIR, 'amazon.test'),'rb') as f:
	test = pickle.load(f)
train = np.array(train)
test = np.array(test)
num_user = para['num_user']
num_item = para['num_item']
dim_latent = args.dim_latent
batch_size = args.batch_size
n_epoch = args.n_epoch

""" load graph"""
model = simpleNN(num_user, num_item, dim_latent)
""" optimizer """
# Minimize
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(model.loss)

# declare Saver object in order to save tensorflow graph
saver = tf.train.Saver()

if __name__ == '__main__':

	""" training phase """
	if args.train == True:
		with tf.Session(config=config) as sess:
			sess.run(tf.global_variables_initializer())

			for epoch in range(n_epoch):
				loss_epoch = 0
				num_step = int(len(train) / batch_size)
				for batch_num in range(num_step):
					# build minibatch
					start_index = batch_num * batch_size
					end_index = min((batch_num + 1) * batch_size, len(train))
					batches_train = train[start_index: end_index]
					uid = batches_train[:,0].astype('int32')
					iid = batches_train[:,1].astype('int32')
					ratings =batches_train[:,2].astype('float32')
					batches = {model.U:uid, model.I:iid, model.R:ratings}
					loss_step, _, = sess.run([model.loss, train_op], feed_dict=batches)
					loss_epoch += loss_step
				loss_epoch = loss_epoch/num_step

				if epoch % 10 == 0:
					""" test phase """
					loss_epoch_test = 0
					num_step = int(len(test) / batch_size)
					for batch_num in range(num_step):
						# build minibatch
						start_index = batch_num * batch_size
						end_index = min((batch_num + 1) * batch_size, len(test))
						batches_test = test[start_index: end_index]
						uid = batches_test[:,0].astype('int32')
						iid = batches_test[:,1].astype('int32')
						ratings =batches_test[:,2].astype('float32')
						batches = {model.U:uid, model.I:iid, model.R:ratings}
						loss_step = sess.run(model.loss, feed_dict=batches)
						loss_epoch_test += loss_step

					loss_epoch_test = loss_epoch_test/num_step
					print("epoch:", epoch, "\t |test:", loss_epoch_test, "\t |train:", loss_epoch)

			# save computational tensorflow graph
			print("Learning finished, saved")
			saver.save(sess, SAVE_FILE, global_step=epoch+1)

	if args.debug == True:
		with tf.Session() as sess:
			b_test = int(input('select minibatch size for test'))
			num_model = input('select model epoch for test')
			shuffle_indices = np.random.permutation(np.arange(len(test)))
			shuffled_test = test[shuffle_indices]
			start = randrange(len(test)-b_test)

			""" load model test """
			# pick a random batch
			batches_test = test[start:start+b_test]
			uid = batches_test[:,0].astype('int32')
			iid = batches_test[:,1].astype('int32')
			ratings =batches_test[:,2].astype('float32')
			saver.restore(sess, os.path.join(SAVE_DIR, 'model-{}'.format(num_model)))
			print("rating={}, prediction={}".format(ratings,
													model.pred.eval(feed_dict={model.U:uid ,
																			   model.I:iid})))


from django.core.context_processors import csrf
from django.shortcuts import render_to_response
from django.http import HttpResponseRedirect
from models import Category, CashFlow, CategoryForm, CashFlowForm, DjowDateHandler
from django.contrib.auth.decorators import login_required
from django.forms.models import modelformset_factory
from datetime import *

@login_required(login_url='/login/')
def index(request):
    user = request.user
    cf = CashFlow()

    if 'year' in request.GET:
        year = request.GET['year']
    else:
        year = datetime.now().year

    if 'month' in request.GET:
        month = request.GET['month']
    else:
        month = datetime.now().month

    cash_flow = CashFlow.objects.filter(user_id=user.id, date_due__month=month, date_due__year=year)
    amount_planned_sum = CashFlow.get_amount_planned_sum_by_month(cf, month, year)
    amount_payed_sum = CashFlow.get_amount_payed_sum_by_month(cf, month, year)

    month = int(month)
    year = int(year)
    month_now = datetime(year, month, 1)

    date_handler = DjowDateHandler.month_year_iterator(month_now)

    return render_to_response('cashflow.html', {
        'cash_flow' : cash_flow,
        'user' : user,
        'amount_planned_sum' : amount_planned_sum,
        'amount_payed_sum' : amount_payed_sum,
        'month_now' : date_handler['month_now'],
        'month_last' : date_handler['month_last'],
        'month_next' : date_handler['month_next'],
    })

@login_required(login_url='/login/')
def category(request):
    user = request.user
    category = Category.objects.filter(user_id=user.id)
    return render_to_response('category.html', {
        'category' : category,
        'user' : user,
    })

@login_required(login_url='/login/')
def cashflow_edit(request):
    user = request.user
    id = 0

    if request.method == 'POST':
        form = CashFlowForm(user, request.POST)
        if form.is_valid():
            id = form.cleaned_data['id']
            user = form.cleaned_data['user']
            category = form.cleaned_data['category']
            description = form.cleaned_data['description']
            installments = form.cleaned_data['installments']
            amount_planned = form.cleaned_data['amount_planned']
            amount_payed = form.cleaned_data['amount_payed']
            date_due = form.cleaned_data['date_due']
            date_payed = form.cleaned_data['date_payed']

            if installments > 0:
                installments_string = ' (1/' + str(installments) + ')'
            else:
                installments_string = ''

            if id > 0:
                c = CashFlow.objects.get(pk=id)
            else:
                c = CashFlow()

            c.user = user
            c.category = category
            c.description = description + installments_string
            c.amount_planned = amount_planned
            c.amount_payed = amount_payed
            c.date_due = date_due
            c.date_payed = date_payed
            c.save()

            if installments > 0 and id <= 0:
                for num in range(1, installments):
                    installments_string = ' (' + str(num+1) + '/' + str(installments) + ')'

                    if num == 1:
                        new_date_due = date_due

                    new_date_due = DjowDateHandler.month_increment(new_date_due)

                    cf = CashFlow()
                    cf.user = user
                    cf.category = category
                    cf.parent = c
                    cf.description = description + installments_string
                    cf.installments = installments
                    cf.amount_planned = amount_planned
                    cf.date_due = new_date_due
                    cf.save()

            return HttpResponseRedirect('/')
    else:
        if 'id' in request.GET:
            id = request.GET['id']
            cash_flow = CashFlow.objects.get(pk=id)
        else:
            cash_flow = CashFlow()

        form = CashFlowForm(user, instance=cash_flow, initial={'date_due': datetime.now(), 'date_payed': datetime.now()})

    c = {'id' : id}
    c.update({'form' : form})
    c.update({'user' : user,})
    c.update(csrf(request))
    return render_to_response('cashflow_edit.html', c)

@login_required(login_url='/login/')
def category_edit(request):
    user = request.user
    id = 0

    if request.method == 'POST':
        form = CategoryForm(user, request.POST)
        if form.is_valid():
            id = form.cleaned_data['id']
            user = form.cleaned_data['user']
            parent = form.cleaned_data['parent']
            description = form.cleaned_data['description']

            if id > 0:
                c = Category.objects.get(pk=id)
            else:
                c = Category()

            c.user = user
            c.parent = parent
            c.description = description
            c.save()

            return HttpResponseRedirect('/category')
    else:
        if 'id' in request.GET:
            id = request.GET['id']
            category = Category.objects.get(pk=id)
        else:
            category = Category()

        form = CategoryForm(user, instance=category)

    c = {'id' : id}
    c.update({'form' : form})
    c.update({'user' : user,})
    c.update(csrf(request))
    return render_to_response('category_edit.html', c)

@login_required(login_url='/login/')
def cashflow_remove(request):
    if request.method == 'GET':
        if 'id' in request.GET:
            id = request.GET['id']
            user = request.user
            CashFlow.objects.filter(parent=id, user_id=user).delete()
            CashFlow.objects.get(pk=id, user_id=user).delete()
            return HttpResponseRedirect('/')

@login_required(login_url='/login/')
def category_remove(request):
    if request.method == 'GET':
        if 'id' in request.GET:
            id = request.GET['id']
            user = request.user
            category = Category.objects.get(pk=id, user_id=user)
            CashFlow.objects.filter(category=category, user_id=user).delete()
            category.delete()
            return HttpResponseRedirect('/category')


import requests
from app.celery import celery_app
from celery.utils.log import get_task_logger
from app.models import db
from app.models.user import User
logger = get_task_logger(__name__)

@celery_app.task
def log_test(a):
    logger.info('put file %s', a)

@celery_app.task
def net_test(a):
    logger.info('connect %s', a)
    requests.get('http://192.168.0.1/api/test')

@celery_app.task
def db_test():
    from app.celery.app import app
    with app.app_context():
        a = db.session.query(User).first()
        logger.info('album %s', a.serializer())
        a.final_votes += 1
        db.session.commit()