# telescope_device.py - A virtual device to serve as an intermediate layer between SEO and
# MaxIm DL.  We only need to implement methods relevant to pulse guide, with everything else
# already managed.

from threading import Timer
from threading import Lock
from logging import Logger
import subprocess
import time

class TelescopeDevice:
    def __init__(self, logger: Logger):
        self._lock = Lock()
        self.name: str = 'SEO Telescope v2'
        self.logger = logger
        #
        # Telescope Constants
        #
        self._alignment_mode = 0  # Alt/Az alignment
        self._aperture_area = 0.196  # Area of the SEO Telescope mirror
        self._aperture_diameter = 0.5  # Aperture diameter of SEO telescope
        self._can_find_home = False  # Not implemented in this driver
        self._can_park = False  # Not implemented in this driver
        self._can_pulse_guide = True  # To be implemented by us
        self._can_set_declination_rate = False  # Not implemented in this driver
        self._can_set_guide_rates = True  # To be implemented by us
        self._can_set_park = False  # TODO: Check with Marc and Dick
        self._can_set_pier_side = False  # Not implemented in this driver
        # TODO: Check if we can actually set the RA rate
        self._can_set_right_ascension_rate = False
        self._can_set_tracking = False  # Not implemented by us
        self._can_slew = True
        self._can_slew_alt_az = False
        self._can_slew_alt_az_async = False
        self._can_slew_async = True
        self._can_sync = True
        self._can_unpark = False  # Not implemented in this driver
        self._declination_rate = 0  # Always 0 due to cansetdeclinationrate being False
        # SEO does not do atmospheric refraction to coordinates
        self._does_refraction = False
        self._equatorial_system = 2  # J2000
        self._focal_length = 2  # TODO: Check for SEO
        self._right_ascension_rate = 0  # 0 because canSetRightAscensionRate is False
        self._side_of_pier = 0  # Cannot be changed
        self._site_elevation = 144  # TODO: Get this from Marc
        self._site_latitude = 50  # TODO: Get real value
        self._site_longitude = 70  # TODO: Get real value
        self._slew_settling_time = 0.5  # TODO: Chose what value to set this to
        self._tracking_rate = 0  # Sidereal Tracking (15.041 arcs/s)
        self._tracking_rates = [0]  # We only have 1 rate
        #
        # Telescope state variables
        #
        self._altitude = 0
        self._at_home = True
        self._at_park = True
        self._azimuth = 0
        self._connected = False
        self._declination = 0
        self._guide_rate_declination = 0.005
        self._guide_rate_right_ascension = 0.005
        self._is_pulse_guiding = False
        self._right_ascension = 0
        self._sidereal_time = 0.1
        self._slewing = False
        self._tracking = True
        self._target_declination = 0
        self._target_right_ascension = 0
        self._utc_date = None

    # Connector methods
    @property
    def connected(self) -> bool:
        self._lock.acquire()
        res = self._connected
        self._lock.release()
        return res

    @connected.setter
    def connected(self, connected: bool):
        self._lock.acquire()
        if (not connected) and self._connected and self._slewing:
            self._lock.release()
            # Yes you could call Halt() but this is for illustration
            raise RuntimeError('Cannot disconnect while telescope is moving')
        elif (not connected) and self._connected and self._is_pulse_guiding:
            raise RuntimeError('Cannot disconnect while guider is tracking')
        self._connected = connected
        self._lock.release()
        if connected:
            self.logger.info('[connected]')
        else:
            self.logger.info('[disconnected]')

    #
    # Guarded properties
    #

    @property
    def alignment_mode(self) -> int:
        self._lock.acquire()
        res = self._alignment_mode
        self._lock.release()
        return res

    @property
    def aperture_area(self) -> float:
        self._lock.acquire()
        res = self._aperture_area
        self._lock.release()
        return res

    @property
    def aperture_diameter(self) -> float:
        self._lock.acquire()
        res = self._aperture_diameter
        self._lock.release()
        return res

    @property
    def can_find_home(self) -> bool:
        self._lock.acquire()
        res = self._can_find_home
        self._lock.release()
        return res

    @property
    def can_park(self) -> bool:
        self._lock.acquire()
        res = self._can_park
        self._lock.release()
        return res

    @property
    def can_pulse_guide(self) -> bool:
        self._lock.acquire()
        res = self._can_pulse_guide
        self._lock.release()
        return res

    @property
    def can_set_declination_rate(self) -> bool:
        self._lock.acquire()
        res = self._can_set_declination_rate
        self._lock.release()
        return res

    @property
    def can_set_guide_rates(self) -> bool:
        self._lock.acquire()
        res = self._can_set_guide_rates
        self._lock.release()
        return res

    @property
    def can_set_park(self) -> bool:
        self._lock.acquire()
        res = self._can_set_park
        self._lock.release()
        return res

    @property
    def can_set_pier_side(self) -> bool:
        self._lock.acquire()
        res = self._can_set_pier_side
        self._lock.release()
        return res

    @property
    def can_set_right_ascension_rate(self) -> bool:
        self._lock.acquire()
        res = self._can_set_right_ascension_rate
        self._lock.release()
        return res

    @property
    def can_set_tracking(self) -> bool:
        self._lock.acquire()
        res = self._can_set_tracking
        self._lock.release()
        return res

    @property
    def can_slew(self) -> bool:
        self._lock.acquire()
        res = self._can_slew
        self._lock.release()
        return res

    @property
    def can_slew_alt_az(self) -> bool:
        self._lock.acquire()
        res = self._can_slew_alt_az
        self._lock.release()
        return res

    @property
    def can_slew_alt_az_async(self) -> bool:
        self._lock.acquire()
        res = self._can_slew_alt_az_async
        self._lock.release()
        return res

    @property
    def can_slew_async(self) -> bool:
        self._lock.acquire()
        res = self._can_slew_async
        self._lock.release()
        return res

    @property
    def can_sync(self) -> bool:
        self._lock.acquire()
        res = self._can_sync
        self._lock.release()
        return res

    @property
    def can_unpark(self) -> bool:
        self._lock.acquire()
        res = self._can_unpark
        self._lock.release()
        return res

    @property
    def declination_rate(self) -> float:
        self._lock.acquire()
        res = self._declination_rate
        self._lock.release()
        return res

    @property
    def does_refraction(self) -> bool:
        self._lock.acquire()
        res = self._does_refraction
        self._lock.release()
        return res

    @property
    def equatorial_system(self) -> int:
        self._lock.acquire()
        res = self._equatorial_system
        self._lock.release()
        return res

    @property
    def focal_length(self) -> float:
        self._lock.acquire()
        res = self._focal_length
        self._lock.release()
        return res

    @property
    def right_ascension_rate(self) -> float:
        self._lock.acquire()
        res = self._right_ascension_rate
        self._lock.release()
        return res

    @property
    def side_of_pier(self) -> int:
        self._lock.acquire()
        res = self._side_of_pier
        self._lock.release()
        return res

    @property
    def site_elevation(self) -> float:
        self._lock.acquire()
        res = self._site_elevation
        self._lock.release()
        return res

    @property
    def site_latitude(self) -> float:
        self._lock.acquire()
        res = self._site_latitude
        self._lock.release()
        return res

    @property
    def site_longitude(self) -> float:
        self._lock.acquire()
        res = self._site_longitude
        self._lock.release()
        return res

    @property
    def slew_settling_time(self) -> float:
        self._lock.acquire()
        res = self._slew_settling_time
        self._lock.release()
        return res

    @property
    def tracking_rate(self) -> int:
        self._lock.acquire()
        res = self._tracking_rate
        self._lock.release()
        return res

    @property
    def tracking_rates(self) -> list:
        self._lock.acquire()
        res = self._tracking_rates
        self._lock.release()
        return res

    #
    # State properties
    #

    @property
    def altitude(self) -> float:
        self._lock.acquire()
        # Make a call to aster to get the altitude
        command = 'tx where'
        process = subprocess.run(command.split(), stdout=subprocess.PIPE)
        output = process.stdout.decode('utf-8')
        # Parse the output

        self._lock.release()
        return res

    @altitude.setter
    def altitude(self, altitude: float):
        self._lock.acquire()
        self._altitude = altitude
        self._lock.release()

    @property
    def at_home(self) -> bool:
        self._lock.acquire()
        res = self._at_home
        self._lock.release()
        return res

    @at_home.setter
    def at_home(self, at_home: bool):
        self._lock.acquire()
        self._at_home = at_home
        self._lock.release()

    @property
    def azimuth(self) -> float:
        self._lock.acquire()
        res = self._azimuth
        self._lock.release()
        return res

    @azimuth.setter
    def azimuth(self, azimuth: float):
        self._lock.acquire()
        self._azimuth = azimuth
        self._lock.release()

    @property
    def at_park(self) -> bool:
        self._lock.acquire()
        res = self._at_park
        self._lock.release()
        return res

    @at_park.setter
    def at_park(self, at_park: bool):
        self._lock.acquire()
        self._at_park = at_park
        self._lock.release()

    @property
    def declination(self) -> float:
        self._lock.acquire()
        res = self._declination
        self._lock.release()
        return res

    @declination.setter
    def declination(self, declination: float):
        self._lock.acquire()
        self._declination = declination
        self._lock.release()

    @property
    def guide_rate_declination(self) -> float:
        self._lock.acquire()
        res = self._guide_rate_declination
        self._lock.release()
        return res

    @guide_rate_declination.setter
    def guide_rate_declination(self, guide_rate_declination: float):
        self._lock.acquire()
        self._guide_rate_declination = guide_rate_declination
        self._lock.release()

    @property
    def guide_rate_right_ascension(self) -> float:
        self._lock.acquire()
        res = self._guide_rate_right_ascension
        self._lock.release()
        return res

    @guide_rate_right_ascension.setter
    def guide_rate_right_ascension(self, guide_rate_right_ascension: float):
        self._lock.acquire()
        self._guide_rate_right_ascension = guide_rate_right_ascension
        self._lock.release()

    @property
    def is_pulse_guiding(self) -> bool:
        self._lock.acquire()
        res = self._is_pulse_guiding
        self._lock.release()
        return res

    @is_pulse_guiding.setter
    def is_pulse_guiding(self, ispulseguiding: bool):
        self._lock.acquire()
        self._is_pulse_guiding = ispulseguiding
        self._lock.release()

    @property
    def right_ascension(self) -> float:
        self._lock.acquire()
        res = self._right_ascension
        self._lock.release()
        return res

    @right_ascension.setter
    def right_ascension(self, right_ascension: float):
        self._lock.acquire()
        self._right_ascension = right_ascension
        self._lock.release()

    @property
    def sidereal_time(self) -> float:
        self._lock.acquire()
        res = self._sidereal_time
        self._lock.release()
        return res

    @sidereal_time.setter
    def sidereal_time(self, sidereal_time: float):
        self._lock.acquire()
        self._sidereal_time = sidereal_time
        self._lock.release()

    @property
    def slewing(self) -> bool:
        self._lock.acquire()
        res = self._slewing
        self._lock.release()
        return res

    @slewing.setter
    def slewing(self, slewing: bool):
        self._lock.acquire()
        self._slewing = slewing
        self._lock.release()

    @property
    def tracking(self) -> bool:
        self._lock.acquire()
        res = self._tracking
        self._lock.release()
        return res

    @tracking.setter
    def tracking(self, tracking: bool):
        self._lock.acquire()
        self._tracking = tracking
        self._lock.release()

    @property
    def target_declination(self) -> float:
        self._lock.acquire()
        res = self._target_declination
        self._lock.release()
        return res

    @target_declination.setter
    def target_declination(self, target_declination: float):
        self._lock.acquire()
        self._target_declination = target_declination
        self._lock.release()

    @property
    def target_right_ascension(self) -> float:
        self._lock.acquire()
        res = self._target_right_ascension
        self._lock.release()
        return res

    @target_right_ascension.setter
    def target_right_ascension(self, target_right_ascension: float):
        self._lock.acquire()
        self._target_right_ascension = target_right_ascension
        self._lock.release()

    @property
    def utc_date(self) -> str:
        self._lock.acquire()
        res = self._utc_date
        self._lock.release()
        return res

    #
    # Methods
    #

    def abort_slew(self):
        pass

    def action(self, action: str, parameters: list):
        pass

    def axis_rates(self, axis: int) -> list:
        pass

    def can_move_axis(self, axis: int) -> bool:
        pass

    def pulse_guide(self, direction: int, duration: int):
        self._is_pulse_guiding = True
        if direction == 0:
            self.logger.info("Pulse guide: N")
        elif direction == 1:
            self.logger.info("Pulse guide: S")
        elif direction == 2:
            self.logger.info("Pulse guide: E")
        elif direction == 3:
            self.logger.info("Pulse guide: W")
        self.logger.info("Pulse guide: {} ms".format(duration))
        time.sleep(duration / 1000)
        self._is_pulse_guiding = False


import os
import numpy as np
import nibabel as nib
from tqdm import tqdm
import argparse
import monai

def nifti_to_2d_slices(input_folder: str, output_folder: str, axis: int, filtered, resize):
    for fname in tqdm(sorted(os.listdir(input_folder))):

        if not fname.endswith("nii.gz"):
            continue

        n_file = os.path.join(input_folder, fname)
        nifti = nib.load(n_file)

        np_data = nifti.get_fdata()
        np_data = np_data.astype(np.float16)

        f_basename = fname.split(".")[0]

        for i in range(np_data.shape[axis]):
            slc = [slice(None)] * len(np_data.shape)
            slc[axis] = i
            image = np_data[slc]

            if resize:
                tr = monai.transforms.Resize((resize, resize))
                image = tr(image[None])[0]

            if filtered:
                brain_mask = image > 0
                if brain_mask.sum() < 4000:
                    continue

            np.save(os.path.join(output_folder, f"{f_basename}_{i}.npy"), image)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("-i", "--input_dir",
                        default='./data/original/brain_train/',
                        help='input_dir')
    parser.add_argument("-o", "--output_dir",
                        default='./data/preprocessed/brain_train/2d_axis_2',
                        help='input_dir')
    parser.add_argument("-a", "--axis", type=int,
                        default=2,
                        help='axis')
    parser.add_argument("-f", "--do_not_filter", action='store_true',
                        default=False,
                        help='Default behavior: do not save slices where # of non zero pixels < 4000. \n'
                             'Use this flag to disable this behavior')
    parser.add_argument("-r", "--resize", type=int,
                        default=256,
                        help='Resize image while saving')

    args = parser.parse_args()

    input_dir = args.input_dir
    output_dir = args.output_dir
    axis = args.axis
    do_filter = (args.do_not_filter == False)
    resize = args.resize
    os.makedirs(output_dir, exist_ok=True)

    nifti_to_2d_slices(input_dir, output_dir, axis, do_filter, resize)


from fastapi import APIRouter, Depends, Response
from fastapi_jwt_auth import AuthJWT

from server.models import User, UserOut, UserUpdate
from server.utils.current_user import current_user

router = APIRouter(
    prefix="/users",
    tags=["users"]
)

@router.get("", response_model=UserOut)
async def get_user(user: User = Depends(current_user)):
    return user

@router.patch("", response_model=UserOut)
async def update_user(update: UserUpdate, user: User = Depends(current_user)):
    user = user.copy(update=update.dict(exclude_unset=True))
    await user.save()
    return user

@router.delete("")
async def delete_user(auth: AuthJWT = Depends()):
    auth.jwt_required()
    await User.find_one(User.email == auth.get_jwt_subject()).delete()
    return Response(status_code=204)


# -*- coding: utf-8 -*-

"""
@Author: lyzhang
@Date:
@Description:
"""
import torch
from config import *
import torch.nn as nn
torch.manual_seed(SEED)

class MLP(nn.Module):
    def __init__(self, input_size=None, output_size=None, hidden_size=None, num_layers=None, use_sig=True):
        nn.Module.__init__(self)
        input_size = input_size if input_size is not None else mlp_input_size
        hidden_size = hidden_size if hidden_size is not None else int(input_size / 2)
        num_layers = num_layers if num_layers is not None else MLP_LAYERS
        output_size = output_size if output_size is not None else Transition_num
        self.use_sig = use_sig

        # part one
        # linear probability
        self.linear_logits = nn.Linear(input_size, output_size)

        # part two
        # input to first hidden
        self.input_linear = nn.Linear(input_size, hidden_size)
        self.input_dropout = nn.Dropout(p=mlp_dropout)
        self.input_activation = nn.ReLU()

        # multi hidden layers
        self.linears = nn.ModuleList([nn.Linear(hidden_size, hidden_size) for _ in range(num_layers - 2)])
        self.dropouts = nn.ModuleList([nn.Dropout(p=mlp_dropout) for _ in range(num_layers - 2)])
        self.activations = nn.ModuleList([nn.ReLU() for _ in range(num_layers - 2)])

        # probabilities
        self.logits = nn.Linear(hidden_size, output_size)

    def forward(self, input_values):
        if MLP_LAYERS == 1:
            output = self.linear_logits(input_values)
        else:
            hidden = self.input_linear(input_values)
            hidden = self.input_dropout(hidden)
            hidden = self.input_activation(hidden)
            # hidden layers
            for linear, dropout, activation in zip(self.linears, self.dropouts, self.activations):
                hidden = linear(hidden)
                hidden = dropout(hidden)
                hidden = activation(hidden)
            # sigmoid output layer
            output = self.logits(hidden)
        sig_output = output if not self.use_sig else torch.tanh(output)
        return sig_output


"""Publicly accessible classes/routines."""

from typing import TYPE_CHECKING

if TYPE_CHECKING:
    from . import ensembles, lnpienergy, segment  # noqa: TCH004
    from .lnpidata import lnPiMasked  # noqa: TCH004
    from .lnpiseries import lnPiCollection  # noqa: TCH004
    from .options import OPTIONS, set_options  # noqa: TCH004
    from .segment import PhaseCreator  # noqa: TCH004
else:
    import lazy_loader as lazy

    __getattr__, __dir__, _ = lazy.attach(
        __name__,
        submodules=["ensembles", "lnpienergy", "segment"],
        submod_attrs={
            "lnpidata": ["lnPiMasked"],
            "lnpiseries": ["lnPiCollection"],
            "options": ["OPTIONS", "set_options"],
            "segment": ["PhaseCreator"],
        },
    )

# updated versioning scheme
try:
    from ._version import __version__
except Exception:
    __version__ = "999"

__author__ = """William P. Krekelberg"""
__email__ = "wpk@nist.gov"

__all__ = [
    "__author__",
    "__email__",
    "__version__",
    "lnPiMasked",
    "lnPiCollection",
    "segment",
    "ensembles",
    "lnpienergy",
    "PhaseCreator",
    "set_options",
    "OPTIONS",
]


from __future__ import annotations

import importlib
import sys
from copy import copy
from pathlib import Path

import mock
import pytest

from volttron.utils import get_class, get_module, get_subclasses

# sys_path_with holds a sys.path that includes the path to
# tests directory on the path.

sys_path_with = copy(sys.path)
sys_path_with.insert(0, str(Path(__file__).parent.parent))

def test_get_module_import_error_raised():
    with pytest.raises(ModuleNotFoundError):
        get_module("non_existing_module.module_data")

def test_get_class_import_error_raised():
    with pytest.raises(ModuleNotFoundError):
        get_class("non_existing_module.module_data", 'SecondaryClass')

def test_get_module():
    with mock.patch('sys.path', sys_path_with):
        module_name = "module_for_testing.module_data"
        actual_module = get_module("module_for_testing.module_data")

        assert actual_module.__name__ == module_name

def test_get_klass_str():
    with mock.patch('sys.path', sys_path_with):
        klass = get_class("module_for_testing.module_data", 'SecondaryClass')
        assert klass.__name__ == 'SecondaryClass'

def test_get_klass_mod():
    with mock.patch('sys.path', sys_path_with):
        module = importlib.import_module("module_for_testing.module_data")
        klass = get_class(module, "MyHelperClass")
        assert klass.__name__ == 'MyHelperClass'

def test_get_klass_attribute_error_raised():
    with pytest.raises(AttributeError) as err:
        get_class("module_for_testing.module_data", 'RandomClass')

def test_get_subclasses_value_error_raised():
    with mock.patch('sys.path', sys_path_with):
        with pytest.raises(ValueError) as err:

            module = importlib.import_module("module_for_testing.module_no_classes_data")
            klass = getattr(importlib.import_module("module_for_testing.module_data"),
                            "MyHelperClass")
            get_subclasses(module, klass)

@pytest.mark.parametrize("return_all, expected_len", [(False, 1), (True, 2)])
def test_get_subclasses_on_all_str(return_all, expected_len):
    with mock.patch('sys.path', sys_path_with):
        expected_subclasses = ['SecondaryClass', 'ThirdClass']
        module = "module_for_testing.module_data"
        parent_class = "MyHelperClass"

        subclasses = get_subclasses(module, parent_class, return_all)

        assert len(subclasses) == expected_len
        for subclass in subclasses:
            assert subclass.__name__ in expected_subclasses

@pytest.mark.parametrize("return_all, expected_len", [(False, 1), (True, 2)])
def test_get_subclasses_on_all_mod(return_all, expected_len):
    with mock.patch('sys.path', sys_path_with):
        expected_subclasses = ['SecondaryClass', 'ThirdClass']
        module = importlib.import_module("module_for_testing.module_data")
        parent_class = getattr(module, "MyHelperClass")

        subclasses = get_subclasses(module, parent_class, return_all=return_all)

        assert len(subclasses) == expected_len
        for subclass in subclasses:
            assert subclass.__name__ in expected_subclasses

@pytest.mark.parametrize("return_all, expected_len", [(False, 1), (True, 2)])
def test_get_subclasses_on_module_str_parent_class_mod(return_all, expected_len):
    with mock.patch('sys.path', sys_path_with):
        expected_subclasses = ['SecondaryClass', 'ThirdClass']
        module = "module_for_testing.module_data"
        parent_class = getattr(importlib.import_module(module), "MyHelperClass")

        subclasses = get_subclasses(module, parent_class, return_all=return_all)

        assert len(subclasses) == expected_len
        for subclass in subclasses:
            assert subclass.__name__ in expected_subclasses

@pytest.mark.parametrize("return_all, expected_len", [(False, 1), (True, 2)])
def test_get_subclasses_on_module_mod_parent_class_str(return_all, expected_len):
    with mock.patch('sys.path', sys_path_with):
        expected_subclasses = ['SecondaryClass', 'ThirdClass']
        module = importlib.import_module("module_for_testing.module_data")
        parent_class = "MyHelperClass"

        subclasses = get_subclasses(module, parent_class, return_all=return_all)

        assert len(subclasses) == expected_len
        for subclass in subclasses:
            assert subclass.__name__ in expected_subclasses


import logging

from funcy import log_calls, log_errors

from server.apps.products.models import ProductAllotment, ProductAvailability

@log_calls(logging.info, errors=False)
@log_errors(logging.exception)
def delete_availability_records(allotment_record: ProductAllotment):
    logging.info(
        f"Deleting availabilities for allotment {allotment_record.allotment_id}"
    )
    _delete_availabilities_from_deleted_allotment(allotment_record)

    logging.info(f"Availability {allotment_record.allotment_id} deleted")

def _delete_availabilities_from_deleted_allotment(allotment_record: ProductAllotment):
    ProductAvailability.objects.filter(
        allotments_id__contains=[allotment_record.allotment_id]
    ).delete()


from django.urls import path

from .views import StuffList, RentalList, StuffDetailView, RentalDetailView, StuffCreateView, \
    MyRentalListView, MyStuffListView, index, categories_posts, StuffUpdateView, StuffDeleteView, RentalDeleteView, \
    RentalUpdateView, RequestRentalList

app_name = 'bilyeo'

urlpatterns = [
    path('', index, name='index'),
    path('stuff_list/', StuffList.as_view(), name='stuff_list'),
    path('stuff/create/', StuffCreateView.as_view(), name='stuff_create'),
    path('stuff/<int:pk>/', StuffDetailView.as_view(), name='stuff_detail'),
    path('stuff/<int:pk>/update/', StuffUpdateView.as_view(), name='stuff_update'),
    path('stuff/<int:pk>/delete/', StuffDeleteView, name='stuff_delete'),
    path('stuff/category/<str:slug>/', categories_posts, name='category_list'),
    path('rental/<int:pk>/', RentalDetailView.as_view(), name='rental_detail'),
    path('rental/<int:pk>/delete/', RentalDeleteView, name='rental_delete'),
    path('rental/<int:pk>/update/', RentalUpdateView.as_view(), name='rental_update'),
    path('rental_list/', RentalList.as_view(), name='rental_list'),
    path('request_list/', RequestRentalList.as_view(), name='request_list'),
    path('my_rental_list/', MyRentalListView.as_view(), name='my_rental_list'),
    path('my_stuff_list/', MyStuffListView.as_view(), name='my_stuff_list'),
]


#!/usr/bin/python

from math import sqrt
import sys
import argparse
from collections import deque
from struct import pack, unpack
import numpy as np

from osgeo import gdal
import math
import decimal

from concavehull import ConcaveHull

gdal.UseExceptions()
gdal.TermProgress = gdal.TermProgress_nocb

#
# NormalVector
#
# Calculate the normal vector of a triangle. (Unit vector perpendicular to
# triangle surface, pointing away from the "outer" face of the surface.)
# Computed using 32-bit float operations for consistency with other tools.
#
# Parameters:
#  triangle vertices (nested x y z tuples)
#
# Returns:
#  normal vector (x y z tuple)
#
def NormalVector(t):
    (ax, ay, az) = t[0]
    (bx, by, bz) = t[1]
    (cx, cy, cz) = t[2]

    # first edge
    e1x = np.float32(ax) - np.float32(bx)
    e1y = np.float32(ay) - np.float32(by)
    e1z = np.float32(az) - np.float32(bz)

    # second edge
    e2x = np.float32(bx) - np.float32(cx)
    e2y = np.float32(by) - np.float32(cy)
    e2z = np.float32(bz) - np.float32(cz)

    # cross product
    cpx = np.float32(e1y * e2z) - np.float32(e1z * e2y)
    cpy = np.float32(e1z * e2x) - np.float32(e1x * e2z)
    cpz = np.float32(e1x * e2y) - np.float32(e1y * e2x)

    # return cross product vector normalized to unit length
    mag = np.sqrt(np.power(cpx, 2) + np.power(cpy, 2) + np.power(cpz, 2))
    return cpx / mag, cpy / mag, cpz / mag

def cal_ang(point_1, point_2, point_3):
    """
    根据三点坐标计算夹角
    :param point_1: 点1坐标
    :param point_2: 点2坐标
    :param point_3: 点3坐标
    :return: 返回任意角的夹角值，这里只是返回点2的夹角
    """
    a = decimal.Decimal(math.sqrt(
        (point_2[0] - point_3[0]) * (point_2[0] - point_3[0]) + (point_2[1] - point_3[1]) * (point_2[1] - point_3[1])))
    b = decimal.Decimal(math.sqrt(
        (point_1[0] - point_3[0]) * (point_1[0] - point_3[0]) + (point_1[1] - point_3[1]) * (point_1[1] - point_3[1])))
    c = decimal.Decimal(math.sqrt(
        (point_1[0] - point_2[0]) * (point_1[0] - point_2[0]) + (point_1[1] - point_2[1]) * (point_1[1] - point_2[1])))
    # print(point_1, point_2, point_3, a, b, c)
    A = math.degrees(math.acos(decimal.Decimal((a * a - b * b - c * c) / (-2 * b * c))))
    B = math.degrees(math.acos(decimal.Decimal((b * b - a * a - c * c) / (-2 * a * c))))
    # print((c * c - a * a - b * b) / (-2 * a * b))
    C = math.degrees(math.acos(decimal.Decimal((c * c - a * a - b * b) / (-2 * a * b))))

    return [A, B, C]

# stlwriter is a simple class for writing binary STL meshes.
# Class instances are constructed with a predicted face count.
# The output file header is overwritten upon completion with
# the actual face count.
class stlwriter():

    # path: output binary stl file path
    # facet_count: predicted number of facets
    def __init__(self, path, facet_count=0):
        self.f = open(path, 'wb')

        # track number of facets actually written
        self.written = 0

        # write binary stl header with predicted facet count
        self.f.write(str.encode('\0' * 80))
        # (facet count is little endian 4 byte unsigned int)
        self.f.write(pack('<I', facet_count))

    # t: ((ax, ay, az), (bx, by, bz), (cx, cy, cz))
    def add_facet(self, t):
        # facet normals and vectors are little endian 4 byte float triplets
        # strictly speaking, we don't need to compute NormalVector,
        # as other tools could be used to update the output mesh.
        self.f.write(pack('<3f', *NormalVector(t)))
        for vertex in t:
            self.f.write(pack('<3f', *vertex))
        # facet records conclude with two null bytes (unused "attributes")
        self.f.write(str.encode('\0\0'))
        self.written += 1

    def done(self):
        # update final facet count in header before closing file
        self.f.seek(80)
        self.f.write(pack('<I', self.written))
        self.f.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.done()

def fail(msg):
    print(msg)
    exit(1)

def log(msg):
    if args.verbose:
        print(msg)

ap = argparse.ArgumentParser(description='Convert a GDAL raster (like a GeoTIFF heightmap) to an STL terrain surface.')
ap.add_argument('-x', action='store', default=0.0, type=float, help='Fit output x to extent (mm)')
ap.add_argument('-y', action='store', default=0.0, type=float, help='Fit output y to extent (mm)')
optg_z = ap.add_mutually_exclusive_group()
optg_z.add_argument('-z', action='store', default=None, type=float,
                    help='Z scale expressed as a vertical scale factor (1)')
optg_z.add_argument('-s', action='store', default=None, type=float,
                    help='Z scale expressed as a ratio of vertical units per horizontal unit (1)')
ap.add_argument('-b', '--base', action='store', default=0.0, type=float, help='Base height (0)')
ap.add_argument('-c', '--clip', action='store_true', default=False, help='Clip z to minimum elevation')
ap.add_argument('-v', '--verbose', action='store_true', default=False, help='Print log messages')
ap.add_argument('--band', action='store', default=1, type=int, help='Raster data band (1)')
ap.add_argument('-m', '--minimum', action='store', default=None, type=float,
                help='Omit vertices below minimum elevation')
ap.add_argument('-M', '--maximum', action='store', default=None, type=float,
                help='Omit vertices above maximum elevation')
optg_region = ap.add_mutually_exclusive_group()
optg_region.add_argument('-w', '--window', action='store', default=None, type=float, nargs=4,
                         help='Opposing corner coordinates in geographic CRS')
optg_region.add_argument('-p', '--pixels', action='store', default=None, type=float, nargs=4,
                         help='Opposing corner coordinates in pixel coordinates')
ap.add_argument('RASTER', help='Input heightmap image')
ap.add_argument('STL', help='Output STL path')
args = ap.parse_args()

try:
    img = gdal.Open(args.RASTER)
except RuntimeError as e:
    fail(str(e).strip())

# input raster dimensions
w = img.RasterXSize
h = img.RasterYSize
log("raster dimensions = (%s, %s)" % (str(w), str(h)))

# get default transformation from image coordinates to world coordinates
# note that since we obtain this transformation before making any region
# selection, and since this transformation is used for output, subset
# window regions will be aligned correctly in output stl coordinates.
t = img.GetGeoTransform()

git = gdal.InvGeoTransform(t)[1]
log("geo->pixel transform: %s" % str(git))

if args.window is not None:
    # if a geographic window is specified, convert it to a pixel window in input raster coordinates

    # apply inverse geo transform to window points
    px0, py0 = gdal.ApplyGeoTransform(git, args.window[0], args.window[1])
    px1, py1 = gdal.ApplyGeoTransform(git, args.window[2], args.window[3])

    # set arg.pixels to obtained pixel points
    args.pixels = [px0, py0, px1, py1]

if args.pixels is None:
    # if no pixel extent window is specified, use whole input raster.
    xmin = 0
    ymin = 0
    ww = w
    wh = h
else:
    # if a pixel extent window is specified (either directly with --pixels or
    # derived from a geographic --window), clip it to the input raster extent.

    xmin = int(round(min(args.pixels[0], args.pixels[2])))
    ymin = int(round(min(args.pixels[1], args.pixels[3])))

    xmax = int(round(max(args.pixels[0], args.pixels[2])))
    ymax = int(round(max(args.pixels[1], args.pixels[3])))

    if xmin >= w:
        fail("Region of interest lies entirely outside raster (xmin)")

    if ymin >= h:
        fail("Region of interest lies entirely outside raster (ymin")

    if xmax <= 0:
        fail("Region of interest lies entirely outside raster (xmax)")

    if ymax <= 0:
        fail("Region of interest lies entirely outside raster (ymax)")

    # if we passed those tests, at least part of the window overlaps the raster,
    # so we can safely clip to the raster extent and still have something

    if xmin < 0:
        xmin = 0

    if ymin < 0:
        ymin = 0

    if xmax > w:
        xmax = w

    if ymax > h:
        ymax = h

    ww = xmax - xmin
    wh = ymax - ymin

log("xmin, ymin = %d, %d" % (xmin, ymin))
log("ww, wh = %d, %d" % (ww, wh))

# output mesh dimensions are one row and column less than raster window
mw = ww - 1
mh = wh - 1

# save x pixel size if needed for scaling
xyres = abs(t[1])

# Apply z scale factor, if any
if args.z is not None:
    zscale = args.z
elif args.s is not None:
    zscale = 1.0 / args.s
else:
    zscale = 1.0

# recalculate z scale and xy transform if different dimensions are requested
if args.x != 0.0 or args.y != 0.0:

    # recaculate xy scale based on requested x or y dimension
    # if both x and y dimension are set, select smaller scale
    if args.x != 0.0 and args.y != 0.0:
        pixel_scale = min(args.x / mw, args.y / mh)
    elif args.x != 0.0:
        pixel_scale = args.x / mw
    elif args.y != 0.0:
        pixel_scale = args.y / mh

    # adjust z scale to maintain proportions with new xy scale
    zscale *= pixel_scale / xyres

    # revise transformation matrix
    # image: 0,0 at top left corner of top left pixel (0.5,0.5 at pixel center)
    t = (
        -pixel_scale * mw / 2.0,  # 0 left edge of top left pixel
        pixel_scale,  # 1 pixel width
        0,  # 2
        pixel_scale * mh / 2.0,  # 3 top edge of top left pixel
        0,  # 4
        -pixel_scale  # 5 pixel height
    )

log("transform = %s" % str(t))

band = img.GetRasterBand(args.band)
nd = band.GetNoDataValue()

# map GDAL pixel data type to corresponding struct format character
typemap = {
    gdal.GDT_Byte: 'B',
    gdal.GDT_UInt16: 'H',
    gdal.GDT_Int16: 'h',
    gdal.GDT_UInt32: 'I',
    gdal.GDT_Int32: 'i',
    gdal.GDT_Float32: 'f',
    gdal.GDT_Float64: 'd'
}

typeName = gdal.GetDataTypeName(band.DataType)
if band.DataType not in typemap:
    fail('Unsupported data type: %s' % typeName)

# rowformat is used to unpack a row of raw image data to numeric form
rowformat = typemap.get(band.DataType) * ww
log("data type = %s" % typeName)
log("type format = %s" % typemap.get(band.DataType))

# min, max, mean, sd; min used for z clipping
stats = band.GetStatistics(True, True)
log("min, max, mean, sd = %s" % str(stats))

# zmin is subtracted from elevation values
if args.clip == True:
    zmin = stats[0]
    zmax = stats[1]
else:
    zmin = stats[0]
    zmax = stats[1]
log("zmin = %s, zmax= %s" % (str(zmin), str(zmax)))

# Rolling pixel buffer has space for two rows of image data.
# Old data is automatically discarded as new data is loaded.
pixels = deque(maxlen=(2 * ww))
log("buffer size = %s" % str(pixels.maxlen))

# Initialize pixel buffer with first row of data from the image window.
pixels.extend(unpack(rowformat, band.ReadRaster(xmin, ymin, ww, 1, ww, 1, band.DataType)))

# Precalculate output mesh size (STL is 50 bytes/facet + 84 byte header)
# Actual facet count and file size may differ (be less) if pixels are skipped as nodata or out of range.
facetcount = mw * mh * 2
filesize = (facetcount * 50) + 84
log("predicted (max) facet count = %s" % str(facetcount))
log("predicted (max) STL file size = %s bytes" % str(filesize))

# skip(v) tests if elevation value v should be omitted from output.
# It returns True if v matches the nodata value, if v is less than
# the minimum allowed elevation, or if v is greater than the
# maximum allowed elevation. Otherwise it returns False.
def skip(v):
    global nd
    global args
    if v == nd:
        return True
    if args.minimum is not None and v < args.minimum:
        return True
    if args.maximum is not None and v > args.maximum:
        return True
    return False

points_3d = []

for y in range(mh):
    pixels.extend(unpack(rowformat, band.ReadRaster(xmin, ymin + y + 1, ww, 1, ww, 1, band.DataType)))
    for x in range(mw):
        av = pixels[x]

        if skip(av):
            continue

        points_3d.append([t[0] + ((xmin + x) * t[1]) + ((ymin + y) * t[2]),
                          t[3] + ((xmin + x) * t[4]) + ((ymin + y) * t[5]),
                          zscale * float(av)])

points_3d = np.asarray(points_3d)
points_2d = points_3d[:, :2]

from scipy.spatial import Delaunay

tri = Delaunay(points_2d)

mask = []
for simple in tri.simplices:
    in_degrees = cal_ang(points_2d[simple[0]], points_2d[simple[1]], points_2d[simple[2]])
    dmax = max(in_degrees)
    # dind = in_degrees.index(dmax)
    mask.append(dmax <= 90.0001)

with stlwriter(args.STL, facetcount) as mesh:
    for simple in tri.simplices[mask]:
        a = points_3d[simple[0]]
        b = points_3d[simple[1]]
        c = points_3d[simple[2]]
        mesh.add_facet((a, b, c))
        # print(a, b, c)

    for simple in tri.simplices[mask]:
        a1 = [points_3d[simple[0]][0], points_3d[simple[0]][1], points_3d[simple[0]][2]]
        a1[2] = zscale * zmin + args.base
        b1 = [points_3d[simple[1]][0], points_3d[simple[1]][1], points_3d[simple[1]][2]]
        b1[2] = zscale * zmin + args.base
        c1 = [points_3d[simple[2]][0], points_3d[simple[2]][1], points_3d[simple[2]][2]]
        c1[2] = zscale * zmin + args.base
        mesh.add_facet((a1, b1, c1))
        # print(a1, b1, c1)

    tri.simplices = tri.simplices[mask]

    ch = ConcaveHull()

    ch.loadpoints(points_2d)
    ch.calculatehull()
    hull_points = np.vstack(ch.boundary.exterior.coords.xy).T.tolist()
    # print(hull_points[:2])

    points_2d = points_2d.tolist()
    # print(points_2d[:2])
    for i, hp in enumerate(hull_points):
        if i == len(hull_points)-1:
            continue
        # print(hp)
        ind = points_2d.index(hp)
        a = (hp[0], hp[1], points_3d[ind][2])
        a1 = (hp[0], hp[1], zscale * zmin + args.base)
        hp_1 = hull_points[i+1]
        ind = points_2d.index(hp_1)
        b = (hp_1[0], hp_1[1], points_3d[ind][2])
        b1 = (hp_1[0], hp_1[1], zscale * zmin + args.base)

        # print(a, a1, b, b1)
        mesh.add_facet((a, a1, b))
        mesh.add_facet((b, b1, a1))
    # log("actual facet count: %s" % str(mesh.written))


import matplotlib.pyplot as plt
import numpy as np # Bibliothèque scientifique

plt.clf()
X = np.array([0, 0.021, 0.039, 0.063])
Y = np.array([0, 0.84, 1.45, 2.4])

plt.plot(X, Y, 'x', color='blue')  # On trace

plt.xlabel("Légende des abscisses")
plt.ylabel("Légende des ordonnées")
plt.title("Titre du graphique")

plt.grid()

# Demande à Numpy de trouver les coefficients a et b pour y = a x + b
fit = np.polyfit(X, Y, 1)

coefficient_directeur = fit[0] #a dans y = ax + b
print("coefficient directeur :", coefficient_directeur)

ordonnee_origine = fit[1] #b dans y = ax + b
print("ordonnée à l'origine:", ordonnee_origine)

print("Equation : y = " + str(coefficient_directeur) + ' x + ' + str(ordonnee_origine) )
Modelisation = coefficient_directeur * X + ordonnee_origine
plt.plot(X, Modelisation, color='red')

plt.show()


from django.contrib.auth.models import ContentType, Group, Permission
from rest_framework import serializers
from rest_framework.serializers import ModelSerializer

VALID_APP_LABEL_LIST = ['users', 'events']

class ContentTypeSerializer(ModelSerializer):
    class Meta:
        model = ContentType
        fields = ['app_label', 'model']

class AuthPermissionSerializer(ModelSerializer):
    content_type = ContentTypeSerializer()

    class Meta:
        model = Permission
        fields = ['id', 'name', 'content_type', 'codename']

class GroupPermissionSerializer(ModelSerializer):
    permissions = serializers.PrimaryKeyRelatedField(
        queryset=Permission.objects.filter(content_type__app_label__in=VALID_APP_LABEL_LIST), many=True,
        write_only=True)

    class Meta:
        model = Group
        fields = ['name', 'permissions']

    def to_representation(self, instance):
        response = super().to_representation(instance)
        response['permissions'] = AuthPermissionSerializer(instance.permissions.all(), many=True).data
        return response


import seaborn as sns
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

cmap = sns.color_palette("coolwarm", as_cmap=True).reversed() #sns.cm.rocket_r

# TODO: not make it full
# ['pop5', 'pop3', 'random']
# [[ 0.          1.486       0.60266667]
#  [ 0.          0.         -0.049     ]
#  [ 0.          0.          0.        ]]

# CRITICAL - ['pop5', 'pop3', 'random', 'latest'] (SelfPlayTesting.py:448)
# CRITICAL -
# [[ 0.          1.486       0.60266667  0.60266667]
#  [ 0.          0.         -0.049      -0.049     ]
#  [ 0.          0.          0.         -0.31933333]
#  [ 0.          0.          0.          0.        ]] (SelfPlayTesting.py:449)

# 28.05.2022 EVO (gain1-gain2)
# pop5 > delta5 > random > pop3 > latest
# CRITICAL - ['pop5', 'pop3', 'random', 'latest', 'delta5'] (SelfPlayTesting.py:449)
# CRITICAL -
# [[ 0.          0.23733333  0.60266667  0.81566667  0.62133333]
#  [ 0.          0.         -0.049       0.16733333 -0.36466667]
#  [ 0.          0.          0.          0.43666667 -0.00266667]
#  [ 0.          0.          0.          0.         -0.10466667]
#  [ 0.          0.          0.          0.          0.        ]] (SelfPlayTesting.py:450)

# 28.05.2022 EVO (gain1+gain2)
# CRITICAL - ['pop5', 'pop3', 'random', 'latest', 'delta5'] (SelfPlayTesting.py:449)
# CRITICAL -
# [[ 0.          1.486       0.60266667  2.28233333  0.52666667]
#  [ 0.          0.         -0.049       1.63666667  1.48666667]
#  [ 0.          0.          0.         -0.43666667  1.88      ]
#  [ 0.          0.          0.          0.          0.52066667]
#  [ 0.          0.          0.          0.          0.        ]] (SelfPlayTesting.py:450)

# Create a dataset
N = 9
# a = np.random.rand(N, N)*8 - 4
# a = np.zeros((N,N))
# a[]
# Not correct interpretation: -1, as it was saved that the columns vs rows -> but it should be
# a = -1*np.load("crosstest_mat_evo1.npy").T
a = np.load("crosstest_mat_pz1_try1.npy").T

print(a)
# a = np.load("crosstest_pz.npy").T
# a = np.load("crosstest_evo.npy").T
# a = np.array([
#     [0,1,2, 3],
#     [0,0,4, 5],
#     [0,0,0,6],
#     [0,0,0,0],
# ]).T
# for i in range(4):
#     a[2,i] = abs(a[2,i])
# To have lower matrix are true values and the opposite in upper
mat = -np.tril(a) + np.tril(a, -1).T
print(mat)

# To have upper matrix are true values and the opposite in lower
# mat = np.tril(a) - np.tril(a, -1).T
# To have symmetric matrix
# mat = np.tril(a) + np.tril(a, -1).T

# mat = np.random.random((4,4))-0.5
# for i in range(4):
#     mat[]

# triangular mask
# mask = np.triu(np.ones_like(mat, dtype=bool)).T
# diagonal mask
# mask = np.eye(N, dtype=bool)
# No mask
mask = np.zeros((N,N), dtype=bool)

# print(mask)
labels = ["Naive", "Random", "Cyclic", "\u03B4=0.5", "\u03B4=0.7", "\u03B4=0.9", "\u03C1=3", "\u03C1=5", "\u03C1=8"][:N]
# labels = ["Latest", "Random", "\u03C1=5"]

mat = np.flip(mat,0)
mask = np.flip(mask,0)
# print(mat)

# for i in range(mat.shape[0]):
#     for j in range(mat.shape[1]):
#         mat[i,j] /= 4

df = pd.DataFrame(mat, reversed(labels), columns=labels)
# df = pd.DataFrame(np.load("crosstest_mat_evo1.npy"), labels, columns=labels)
# plot a heatmap with annotation
ax = sns.heatmap(df, annot=True,fmt=".3f", annot_kws={"size": 10}, vmin=-1, vmax=1, center=0, cmap=cmap, mask=mask,
            #   linewidths=0.1, linecolor='black'
                cbar_kws={'label': "Normalized gain"},

                )
ax.set_xticklabels(ax.get_xmajorticklabels(), fontsize = 17)
ax.set_yticklabels(ax.get_ymajorticklabels(), fontsize = 17)
plt.xticks(rotation=np.pi/2)

ax.figure.axes[-1].yaxis.label.set_size(17)
plt.show()


import json
from typing import Dict, Union, List

from easytext.data import Instance
from easytext.data.dataset import Dataset
from easytext.component.register import ComponentRegister

@ComponentRegister.register(name_space="mrc_ner")
class MSRAFlatNerDataset(Dataset):
    """
    MSRA flat ner dataset
    """

    def __init__(self, is_training: bool, dataset_file_path: str):
        super(MSRAFlatNerDataset, self).__init__(is_training=is_training)

        self._instances: List[Instance] = list()

        if self._is_training:
            with open(dataset_file_path, encoding="utf-8") as f:

                all_data = json.load(f)

                for item in all_data:
                    instance = self.create_instance(item)
                    self._instances.append(instance)

    def create_instance(self, input_data: Dict) -> Union[Instance, List[Instance]]:
        instance = Instance()

        instance["context"] = "".join(input_data["context"].split())
        instance["query"] = input_data["query"]

        instance["entity_label"] = input_data.get("entity_label", None)
        instance["impossible"] = input_data.get("impossible", None)

        instance["start_positions"] = input_data.get("start_position", None)
        instance["end_positions"] = input_data.get("end_position", None)

        return instance

    def __getitem__(self, index: int) -> Instance:
        return self._instances[index]

    def __len__(self) -> int:
        return len(self._instances)


from telegram import (ReplyKeyboardMarkup, ReplyKeyboardRemove)
from telegram.ext import (Updater, CommandHandler, MessageHandler,
                          Filters, RegexHandler, ConversationHandler)

import logging
import time
import Beauty
import Cardcode

# Enable logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)

logger = logging.getLogger(__name__)
TOKEN = '455989974:AAG1_pt549X9YV64asFXixuaMl-lchzidgk'
PTT_URL = 'https://www.ptt.cc'

START, BEAUTY, DECK, GUESS, END = range(5)

def showBeauty(bot, update):
    global index
    global beauty_articles
    global beauty_img_urls

    if index >= len(beauty_articles):
        update.message.reply_text("there is no more beauty today")
        return START

    while(beauty_articles[index]['push_count']<0):
        index = index + 1

    article = beauty_articles[index]
    page = Beauty.get_web_page(PTT_URL + article['href'])
    if page:
        beauty_img_urls = Beauty.parse_img(page)

    img_url = beauty_img_urls
    chat_id = update.message.chat_id
    bot.send_photo(chat_id=chat_id, photo=img_url[0])

    # reply and show keyboard choice
    reply_keyboard = [['OK', 'next']]

    update.message.reply_text(
        "do you like this one ?",
        reply_markup=ReplyKeyboardMarkup(
            reply_keyboard, one_time_keyboard=True)
    )

    index = index + 1
    return BEAUTY

def allBeauty(bot, update):
    global beauty_img_urls

    chat_id = update.message.chat_id
    for url in beauty_img_urls:
        if url == beauty_img_urls[0]:
            continue
        bot.send_photo(chat_id=chat_id, photo=url)

    return END

def getDeck(bot, update):
    deck_codes = Cardcode.get_deck_code()
    chat_id = update.message.chat_id

    for code in deck_codes:
        bot.send_message(chat_id=chat_id, text=code)

    bot.send_message(chat_id=chat_id, text="say something to me")
    return END

def guess(bot, update):
    reply_keyboard = [['0', '5', '10']]
    update.message.reply_text("哪個殺手只會講英文 ?? (給你五秒唷)")
    time.sleep(5)
    update.message.reply_text("銀翼殺手")
    update.message.reply_text(
        "請給分",
        reply_markup=ReplyKeyboardMarkup(reply_keyboard,
                                         one_time_keyboard=True)
    )
    return END

def start(bot, update):
    # reply and show keyboard choice
    reply_keyboard = [['beauty', 'code', 'guess']]

    update.message.reply_text(
        "/cancel to stop conversation.\n",
        reply_markup=ReplyKeyboardMarkup(
            reply_keyboard, one_time_keyboard=True)
    )

    return START

def end(bot, update):
    text = update.message.text
    update.message.reply_text("you say %s, i say goodbye." % text)
    return ConversationHandler.END

def cancel(bot, update):
    user = update.message.from_user
    logger.info("User %s canceled the conversation." % user.first_name)
    update.message.reply_text('Bye! I hope we can talk again some day.',
                              reply_markup=ReplyKeyboardRemove())

    return ConversationHandler.END

def error(bot, update, error):
    logger.warn('Update "%s" caused error "%s"' % (update, error))

def main():
    # init variable for Beauty
    global index
    global beauty_articles

    index = 0
    beauty_articles = Beauty.get_today_articles()

    # set parameter for bot
    updater = Updater(TOKEN)
    dp = updater.dispatcher

    conv_handler = ConversationHandler(
        # initial state (recieve '/start', then go start())
        entry_points=[CommandHandler('start', start)],

        states={
            START: [RegexHandler('^beauty$', showBeauty),
                    RegexHandler('^code$', getDeck),
                    RegexHandler('^guess$', guess)],

            BEAUTY: [RegexHandler('^OK$', allBeauty),
                    RegexHandler('^next$', showBeauty)],

            DECK: [MessageHandler(Filters.text, end)],
            GUESS:[MessageHandler(Filters.text, end)],

            END: [MessageHandler(Filters.text, end)]
            },
        fallbacks=[CommandHandler('cancel', cancel)],
    )

    dp.add_handler(conv_handler)
    dp.add_error_handler(error)

    updater.start_polling()
    updater.idle()

if __name__=='__main__':
    main()


import argparse
import os
import requests
import zipfile

MOVIELEN_DOMAIN = 'http://files.grouplens.org/datasets/movielens'
MOVIELEN_SMALL_DS = 'ml-latest-small.zip'
MOVIELEN_FULL_DS = 'ml-latest.zip'

def download_file(url, outputdir, chunk_size=8192):
    '''
    Download file and write it into outputdir

    Args:
        url (str): link to download
        outputdir (str): where to place new file
        chunk_size (int): size of chunks to write on disk default: 4Kb
    Return:
        (str) Name of downloaded File
    '''

    local_filename = url.split('/')[-1]
    local_filename = f'{outputdir}/{local_filename}'
    with requests.get(url, stream=True) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
    return local_filename

def unzip_file(filename, outputdir, delete=True):
    '''
    Unzip file and delete it if necessary

    Args:
        filename (str): file to unzip
        outputdir (str): where to unpack
        delete (bool): whatever delete or not after unziping [default: True]

    Return:
        None
    '''
    with zipfile.ZipFile(filename, 'r') as zip_ref:
        zip_ref.extractall(outputdir)
    if delete:
        os.remove(filename)

def main():
    parser = argparse.ArgumentParser(
        description='Downloads dataset with movies ratings')
    parser.add_argument('outdir', type=str, help='Output dir for dataset')
    parser.add_argument('--small',
                        action='store_true',
                        help='Download small version of dataset')
    args = parser.parse_args()
    if args.small:
        download_link = f'{MOVIELEN_DOMAIN}/{MOVIELEN_SMALL_DS}'
    else:
        download_link = f'{MOVIELEN_DOMAIN}/{MOVIELEN_FULL_DS}'

    archive_filename = download_file(download_link, outputdir=args.outdir)
    unzip_file(archive_filename, args.outdir)

if __name__ == '__main__':
    main()


#!/usr/bin/env python

import matplotlib.pyplot as plt
import pandas as pd
import matplotlib.dates as mdates

#aesthetics
#plt.style.use('bmh')

#Dataframe variable for reading file
df = pd.read_csv('counts.csv')

#data sorting
df['receivetimeAEST'] = pd.to_datetime(df['receivetimeAEST'])
df.sort_values('receivetimeAEST', inplace=True)

#date formatting
date_format = mdates.DateFormatter('%b, %d %Y')
plt.gca().xaxis.set_major_formatter(date_format)

#Specify axis
x = df['receivetimeAEST']
y = df['counts']

#Line Graph
plt.xlabel('receivetimeAEST', fontsize = 18)
plt.ylabel('counts', fontsize = 18)

#plt.scatter(x, y)
plt.plot_date(x, y, linestyle='solid')
plt.gcf().autofmt_xdate()

#Display graph
plt.show()


import logging
import os

from typing import List, Dict

from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from airflow.sensors.base_sensor_operator import apply_defaults
from dop.component.configuration.env import env_config
from dop.airflow_module.operator import dbt_operator_helper

# List of files generated by dbt docs generate
# https://docs.getdbt.com/reference/commands/cmd-docs
DBT_DOC_FILES = ["index.html", "manifest.json", "catalog.json"]
DBT_DOC_FOLDER = "target"
DBT_USER = "dbtuser"
DBT_RUN_RESULTS_PATH = "target/run_results.json"

# See: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
node_pool_affinity = {
    "nodeAffinity": {
        # requiredDuringSchedulingIgnoredDuringExecution means in order
        # for a pod to be scheduled on a node, the node must have the
        # specified labels. However, if labels on a node change at
        # runtime such that the affinity rules on a pod are no longer
        # met, the pod will still continue to run on the node.
        "requiredDuringSchedulingIgnoredDuringExecution": {
            "nodeSelectorTerms": [
                {
                    "matchExpressions": [
                        {
                            # When nodepools are created in Google Kubernetes
                            # Engine, the nodes inside of that nodepool are
                            # automatically assigned the label
                            # 'cloud.google.com/gke-nodepool' with the value of
                            # the nodepool's name.
                            "key": "cloud.google.com/gke-nodepool",
                            "operator": "In",
                            "values": ["kubernetes-task-pool"],
                        }
                    ]
                }
            ]
        }
    }
}

def retrieve_commit_hash():
    with open(
        os.path.sep.join([env_config.service_project_path, ".commit-hash"])
    ) as fp:
        return fp.read()

class DbtK8Operator(KubernetesPodOperator):
    template_fields = (
        "action",
        "target",
        "dbt_project_name",
        "image_tag",
        "dbt_arguments",
        "gcr_pull_secret_name",
        "arguments",
    )
    ui_color = "#FF694B"

    @apply_defaults
    def __init__(
        self,
        dbt_project_name: str,
        dbt_version: str,
        dbt_arguments: List[Dict],
        *args,
        **kwargs,
    ):
        """
        :param dbt_project_name: the name for the dbt project name inline with what's defined in `.dbt-project-repos.json`
        :param dbt_version: Not used
        :param args:
        :param kwargs: must contain the Task entity
        """

        task = kwargs["task"]
        self.dbt_project_name = dbt_project_name
        self.dbt_version = "N/A, this is fixed in the docker image"
        self.action = task.kind.action
        self.target = task.kind.target
        self.dbt_arguments = dbt_arguments
        self.gcr_pull_secret_name = env_config.gcr_pull_secret_name
        self.image_tag = retrieve_commit_hash()

        self._full_refresh = (
            False  # used to trigger DBT full refresh, modified via execute() override
        )

        self.arguments = [self.parse_bash_command()]

        super(DbtK8Operator, self).__init__(
            name=kwargs["task_id"],
            cmds=["/bin/bash", "-c"],
            arguments=self.arguments,
            get_logs=True,
            namespace="default",
            image=f"eu.gcr.io/{env_config.infra_project_id}/dop-dbt:{self.image_tag}",
            is_delete_operator_pod=True,
            env_vars={
                "DOP_PROJECT_ID": env_config.project_id,
                "DOP_LOCATION": env_config.location,
            },
            image_pull_secrets=self.gcr_pull_secret_name,
            affinity=node_pool_affinity,
            *args,
            **kwargs,
        )

    def execute(self, context):
        """
        Override the parent method to ingest required contexts
        """
        dag_run_conf = context["dag_run"].conf if context["dag_run"].conf else {}
        full_refresh = dag_run_conf.get("full_refresh", False)

        self._full_refresh = full_refresh

        logging.info(f"### IS FULL REFRESH ENABLED: {self._full_refresh}")

        self.arguments = [self.parse_bash_command(context=context)]

        logging.info(f"### Updated arguments: {self.arguments}")

        super(DbtK8Operator, self).execute(context=context)

    def parse_bash_command(self, context=None):
        full_refresh_cmd = ""
        if self.target != "run":
            full_refresh_cmd = ""
        elif self.dbt_arguments:
            if self._full_refresh and "--full-refresh" not in [
                arg.get("option") for arg in self.dbt_arguments
            ]:
                full_refresh_cmd = "--full-refresh"
        elif self._full_refresh:
            full_refresh_cmd = "--full-refresh"

        cmd_for_additional_arguments = ""

        # docs arguments are only used to copy files to GCS, not in the task execution
        if self.dbt_arguments and self.target != "docs generate":
            cmd_for_additional_arguments = dbt_operator_helper.implode_arguments(
                dbt_arguments=self.dbt_arguments
            )

        cmd_to_run_dbt = (
            f"pipenv run dbt --no-use-colors {self.target} --project-dir ./{self.dbt_project_name}"
            f" --vars {dbt_operator_helper.parsed_cmd_airflow_context_vars(context=context)}"
            f" {cmd_for_additional_arguments}"
            f" {full_refresh_cmd};"
            f" gsutil cp /home/{DBT_USER}/{self.dbt_project_name}/{DBT_RUN_RESULTS_PATH} gs://{os.getenv('GCS_BUCKET')}/dbt/{DBT_RUN_RESULTS_PATH}"
        )

        if self.target == "docs generate":
            command = self.copy_docs_to_gcs_command()
            if command:
                cmd_to_run_dbt += f"; {command}"

        return cmd_to_run_dbt

    def copy_docs_to_gcs_command(self):
        """
        Generate gsutil command line to copy doc files generated with dbt docs generate to GCS
        """
        command = []
        gcs_bucket = dbt_operator_helper.extract_argument(
            self.dbt_arguments, "--bucket"
        )
        if not gcs_bucket:
            logging.warning("No bucket argument provided. Skipping copy to GCS")
            return ""
        gcs_path = dbt_operator_helper.extract_argument(
            self.dbt_arguments, "--bucket-path", ""
        )

        for doc_file in DBT_DOC_FILES:
            doc_file_path = (
                f"/home/{DBT_USER}/{self.dbt_project_name}/{DBT_DOC_FOLDER}/{doc_file}"
            )
            logging.info(f"Copying {doc_file} to gs://{gcs_bucket}/{gcs_path}")
            command.append(
                f"gsutil cp {doc_file_path} gs://{gcs_bucket}/{gcs_path}/{doc_file}"
            )
        return ";".join(command)

    def post_execute(self, context, result=None):
        """
        This hook is triggered right after self.execute() is called.
        It is passed the execution context and any results returned by the
        operator.
        """
        dbt_operator_helper.save_run_results_in_bq(
            env_config.project_id,
            self.dbt_project_name,
            f"gs://{os.getenv('GCS_BUCKET')}/dbt/{DBT_RUN_RESULTS_PATH}",
        )


import matplotlib.pyplot as plt
import numpy as np
import math
from PIL import Image
#import cPickle as pkl
import _pickle as cPickle
import time
import tensorflow as tf
import tensorflow.examples.tutorials.mnist.input_data as input_data
import glob
#%matplotlib inline
print ("Packages loaded")

############ LOAD DATASET #############################################

dirpath = "/home/saurabh/Documents/data/iccv09Data/images/"
height = 240
width  = 320
resize_ratio = 3
nr_img = 0
fileList = glob.glob(dirpath + '*.jpg')
for i, file in enumerate(fileList):
    img = Image.open(file)
    array = np.array(img)
    if array.shape[0] == height and array.shape[1] == width:
        nr_img = nr_img + 1
        rgb = array.reshape(1, height, width, 3)
        imglow = img.resize((int(width/resize_ratio)
                ,int(height/resize_ratio)), Image.BICUBIC)
        imglow = imglow.resize((width, height), Image.BICUBIC)
        rgblow = np.array(np.float32(imglow)/255.)
        rgblow = rgblow.reshape(1, height, width, 3)
        rgb = np.reshape(rgb, [1, -1])
        rgblow = np.reshape(rgblow, [1, -1])
        if nr_img == 1:
            data = rgb
            datalow = rgblow
        else:
            data = np.concatenate((data, rgb), axis=0)
            datalow = np.concatenate((datalow, rgblow), axis=0)

print ("nr_img is %d" % (nr_img))
print ("Shape of 'data' is %s" % (data.shape,))
print ("Shape of 'datalow' is %s" % (datalow.shape,))

########################## DIVIDE in 2 TRAIN/ TEST SETS ##########################
##########################(xtrain, ytrain) and (xtest, ytest) ####################

randidx = np.random.permutation(nr_img)
nrtrain = int(nr_img*0.7)
nrtest  = nr_img - nrtrain
xtrain  = datalow[randidx[0:nrtrain], :]
ytrain  = data[randidx[0:nrtrain], :]
xtest   = datalow[randidx[nrtrain:nr_img], :]
ytest   = data[randidx[nrtrain:nr_img], :]
print ("Shape of 'xtrain' is %s" % (xtrain.shape,))
print ("Shape of 'ytrain' is %s" % (ytrain.shape,))
print ("Shape of 'xtest' is %s" % (xtest.shape,))
print ("Shape of 'ytest' is %s" % (ytest.shape,))

########################## DEFINE THE NETWORK #####################################

n1 = 32
n2 = 64
n3 = 64
n4 = 64
n5 = 64
n6 = 3
ksize = 3
weights = {
    'ce1': tf.Variable(tf.random_normal([ksize, ksize, 3, n1],  stddev=0.01)),
    'ce2': tf.Variable(tf.random_normal([ksize, ksize, n1, n2], stddev=0.01)),
    'ce3': tf.Variable(tf.random_normal([ksize, ksize, n2, n3], stddev=0.01)),
    'ce4': tf.Variable(tf.random_normal([ksize, ksize, n3, n4], stddev=0.01)),
    'ce5': tf.Variable(tf.random_normal([ksize, ksize, n4, n5], stddev=0.01)),
    'ce6': tf.Variable(tf.random_normal([ksize, ksize, n5, n6],  stddev=0.01))
}
biases = {
    'be1': tf.Variable(tf.random_normal([n1], stddev=0.01)),
    'be2': tf.Variable(tf.random_normal([n2], stddev=0.01)),
    'be3': tf.Variable(tf.random_normal([n3], stddev=0.01)),
    'be4': tf.Variable(tf.random_normal([n4], stddev=0.01)),
    'be5': tf.Variable(tf.random_normal([n5], stddev=0.01)),
    'be6': tf.Variable(tf.random_normal([n6], stddev=0.01))
}
def srn(_X, _W, _b, _keepprob):
    _input_r = tf.reshape(_X, shape=[-1, height, width, 3])
    # Encoder
    _ce1 = tf.nn.relu(tf.add(tf.nn.conv2d(_input_r, _W['ce1']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be1']))
    _ce1 = tf.nn.dropout(_ce1, _keepprob)
    _ce2 = tf.nn.relu(tf.add(tf.nn.conv2d(_ce1, _W['ce2']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be2']))
    _ce2 = tf.nn.dropout(_ce2, _keepprob)
    _ce3 = tf.nn.relu(tf.add(tf.nn.conv2d(_ce2, _W['ce3']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be3']))
    _ce3 = tf.nn.dropout(_ce3, _keepprob)

    _ce4 = tf.nn.relu(tf.add(tf.nn.conv2d(_ce3, _W['ce4']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be4']))
    _ce4 = tf.nn.dropout(_ce4, _keepprob)
    _ce5 = tf.nn.relu(tf.add(tf.nn.conv2d(_ce4, _W['ce5']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be5']))
    _ce5 = tf.nn.dropout(_ce5, _keepprob)
    _ce6 = tf.nn.relu(tf.add(tf.nn.conv2d(_ce5, _W['ce6']
        , strides=[1, 1, 1, 1], padding='SAME'), _b['be6']))
    _out = _ce6 + _input_r
    return {'input_r': _input_r, 'ce1': _ce1, 'ce2': _ce2, 'ce3': _ce3
        , 'ce4': _ce4, 'ce5': _ce5, 'ce6': _ce6
        , 'layers': (_input_r, _ce1, _ce2, _ce3, _ce4, _ce5, _ce6)
        , 'out': _out}
print ("Network ready")

########################## DEFINE THE FUNCTIONS #####################################

dim = height*width*3
x = tf.placeholder(tf.float32, [None, dim])
y = tf.placeholder(tf.float32, [None, dim])
keepprob = tf.placeholder(tf.float32)
pred = srn(x, weights, biases, keepprob)['out']
cost = tf.reduce_mean(tf.square(srn(x, weights, biases, keepprob)['out']
            - tf.reshape(y, shape=[-1, height, width, 3])))
learning_rate = 0.001
optm = tf.train.AdamOptimizer(learning_rate, 0.9).minimize(cost)
init = tf.initialize_all_variables()
print ("Functions ready")

########################## RUN TF ###################################################

sess = tf.Session()
sess.run(init)
# Fit all training data
batch_size = 1
n_epochs   = 10000
recon_images = []
train_error = []
test_error = []
print("Strart training..")
randidx = np.random.randint(nrtrain, size=batch_size)
batch_xs = xtrain[randidx, :]
batch_ys = ytrain[randidx, :]
for epoch_i in range(n_epochs):
    for batch_i in range(nrtrain // batch_size):
        randidx = np.random.randint(nrtrain, size=batch_size)
        batch_xs = xtrain[randidx, :]
        batch_ys = ytrain[randidx, :]
        sess.run(optm, feed_dict={x: batch_xs
            , y: batch_ys, keepprob: 0.7})
    #if (epoch_i % 10) == 0:
        #print("==========>")
        #print ("[%02d/%02d] cost: %.4f" % (epoch_i, n_epochs
         #   , sess.run(cost, feed_dict={x: batch_xs
        #  , y: batch_ys, keepprob: 1.})))

        # TO DO : Put the epoch no and Cost in a dict and plot via matplot
    if (epoch_i % 10) == 0:
        print ("[%02d/%02d] cost: %.4f" % (epoch_i, n_epochs
            , sess.run(cost, feed_dict={x: batch_xs
            , y: batch_ys, keepprob: 1.})))
        n_examples = 2
        print ("Training dataset")
        randidx = np.random.randint(nrtrain, size=n_examples)
        train_xs = xtrain[randidx, :]
        train_ys = ytrain[randidx, :]
        recon = sess.run(pred, feed_dict={x: train_xs, keepprob: 1.})

        ##### Training error ################
        #err1 = sess.run(cost, feed_dict = {x:batch_xs, y: batch_ys, keepprob: 1.})
        err1 = sess.run(cost, feed_dict = {x:train_xs, y: train_ys, keepprob: 1.})
        train_error.append(err1)
        #print("The train error is:", train_error)

        fig, axs = plt.subplots(3, n_examples, figsize=(15, 20))
        for example_i in range(n_examples):
            axs[0][example_i].imshow(np.reshape(
                train_xs[example_i, :], (height, width, 3)))
            axs[1][example_i].imshow(np.reshape(
                recon[example_i, :], (height, width, 3)))
            axs[2][example_i].imshow(np.reshape(
                train_ys[example_i, :], (height, width, 3)))
        plt.show()

        print ("Test dataset")
        randidx = np.random.randint(nrtest, size=n_examples)
        test_xs = xtest[randidx, :]
        test_ys = ytest[randidx, :]
        recon = sess.run(pred, feed_dict={x: test_xs, keepprob: 1.})

        # Test Error
        err2 = sess.run(cost, feed_dict = {x: test_xs, y: test_ys, keepprob: 1.})
        test_error.append(err2)
        #print("The test err is:",test_error)

        fig, axs = plt.subplots(3, n_examples, figsize=(15, 20))
        for example_i in range(n_examples):
            axs[0][example_i].imshow(np.reshape(
                test_xs[example_i, :], (height, width, 3)))
            axs[1][example_i].imshow(np.reshape(
                recon[example_i, :], (height, width, 3)))
            #recon_images.append(axs[1])
            axs[2][example_i].imshow(np.reshape(
                test_ys[example_i, :], (height, width, 3)))
        plt.show()
        #for i in range(0, recon_images):
            #print(recon_images[i])
            #print(i)
print("Training done. ")
print("the train error is......", err1)
plt.plot(train_error)
print("the TEST error is:......", err2)
plt.plot(test_error)
plt.grid(True)
plt.show()


import argparse
import traceback
from keras_segmentation.train import train
from keras_segmentation.models.all_models import model_from_name

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Like the default 'train' sub-command, but with more options exposed.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--model_name", type=str, required=True, help="The name of the pretrained model to use: %s" % str(', '.join(sorted(list(model_from_name.keys())))))
    parser.add_argument("--train_images", type=str, required=True, help="The directory with the training images.")
    parser.add_argument("--train_annotations", type=str, required=True, help="The directory with the training annotations.")
    parser.add_argument("--n_classes", type=int, required=True, help="The number of classes, including background.")
    parser.add_argument("--input_height", type=int, default=None, help="The height to scale the images to (multiple of 32).")
    parser.add_argument("--input_width", type=int, default=None, help="The width to scale the images to (multiple of 32)")
    parser.add_argument('--not_verify_dataset', action='store_false', help="Whether to skip verifying the dataset.")
    parser.add_argument("--checkpoints_path", type=str, default=None, help="The directory to store the checkpoint files and _config.json in.")
    parser.add_argument("--epochs", type=int, default=5, help="The number of epochs to train.")
    parser.add_argument("--batch_size", type=int, default=2, help="The batch size.")
    parser.add_argument('--validate', action='store_true', help="Whether to validate using a separate validation set.")
    parser.add_argument("--val_images", type=str, default="", help="The images of the validation set.")
    parser.add_argument("--val_annotations", type=str, default="", help="The annotations of the validation set.")
    parser.add_argument("--val_batch_size", type=int, default=2, help="The batch size for the validation.")
    parser.add_argument("--load_weights", type=str, default=None, help="The pretrained model to use.")
    parser.add_argument('--auto_resume_checkpoint', action='store_true', help="Whether to resume from the last checkpoint available from the checkpoints directory.")
    parser.add_argument("--steps_per_epoch", type=int, default=512, help="The number of steps per epoch.")
    parser.add_argument("--val_steps_per_epoch", type=int, default=512, help="The number of steps per validation epoch.")
    parser.add_argument('--use_multi_processing', action='store_true', help="Whether to use process-based multi-processing.")
    parser.add_argument('--ignore_zero_class', action='store_true', help="Defines the loss function, either 'categorical_crossentropy' or 'masked_categorical_crossentropy' (if ignoring zero class)")
    parser.add_argument("--optimizer_name", type=str, default="adadelta", help="The name of the optimizer to use (https://www.tensorflow.org/versions/r1.14/api_docs/python/tf/keras/optimizers).")
    parser.add_argument('--do_augment', action='store_true', help="Whether to augment the images.")
    parser.add_argument("--augmentation_name", type=str, default="aug_all", help="How to augment the images: aug_all, aug_all2, aug_geometric, aug_non_geometric")
    args = parser.parse_args()

    try:
        train(model=args.model_name,
              train_images=args.train_images,
              train_annotations=args.train_annotations,
              input_height=args.input_height,
              input_width=args.input_width,
              n_classes=args.n_classes,
              verify_dataset=args.not_verify_dataset,
              checkpoints_path=args.checkpoints_path,
              epochs=args.epochs,
              batch_size=args.batch_size,
              validate=args.validate,
              val_images=args.val_images,
              val_annotations=args.val_annotations,
              val_batch_size=args.val_batch_size,
              auto_resume_checkpoint=args.auto_resume_checkpoint,
              load_weights=args.load_weights,
              steps_per_epoch=args.steps_per_epoch,
              val_steps_per_epoch=args.val_steps_per_epoch,
              optimizer_name=args.optimizer_name,
              gen_use_multiprocessing=args.use_multi_processing,
              ignore_zero_class=args.ignore_zero_class,
              do_augment=args.do_augment,
              augmentation_name=args.augmentation_name)
    except Exception as e:
        print(traceback.format_exc())


import json.decoder

import grpc
import bankingsystem_pb2
import bankingsystem_pb2_grpc
from concurrent import futures
import multiprocessing
from google.protobuf.json_format import MessageToDict
import json
from BankingSystemUtility import log_msg, initializeLogging, parseInputFile, getProcessId, parseLocalArgs, \
    translateEntityToEnum
from time import sleep

class Branch(bankingsystem_pb2_grpc.TransactionServicer):
    # constants for sub-events names
    WITHDRAW_REQUEST = 'withdraw_request'
    WITHDRAW_EXECUTE = 'withdraw_execute'
    WITHDRAW_PROPAGATE_REQUEST = 'withdraw_propagate_request'
    WITHDRAW_PROPAGATE_EXECUTE = 'withdraw_propagate_execute'
    WITHDRAW_PROPAGATE_RESPONSE = 'withdraw_propagate_response'
    WITHDRAW_RESPONSE = 'withdraw_response'

    DEPOSIT_REQUEST = 'deposit_request'
    DEPOSIT_EXECUTE = 'deposit_execute'
    DEPOSIT_PROPAGATE_REQUEST = 'deposit_propagate_request'
    DEPOSIT_PROPAGATE_EXECUTE = 'deposit_propagate_execute'
    DEPOSIT_PROPAGATE_RESPONSE = 'deposit_propagate_response'
    DEPOSIT_RESPONSE = 'deposit_response'

    def __init__(self, id, balance, branches):
        # unique ID of the Branch
        self.id = id
        # replica of the Branch's balance
        self.balance = balance
        # the list of process IDs of the branches
        self.branches = branches
        # the list of Client stubs to communicate with the branches
        self.stubList = list()
        # a list of received messages used for debugging purpose
        self.recvMsg = list()
        # This will keep track of the branch's internal logical clock as per Lamport algorithm.
        self.clock = 0
        # This dictionary will record all the sub-events happening on the incoming requests along with the timestamps.
        self.branchSubEventData = {'pid': self.id, 'data': list()}

    def MsgDelivery(self, request, context):
        """
        The RPC call to handle the incoming requests.
        :param request: un-marsheled BankRequest received by the RPC call.
        :param context: Server context
        :return: BankResponse
        """
        log_msg('request::' + json.dumps(MessageToDict(request)))
        responseList = list()
        # protobuf does not translate ZERO values. This is to make sure if that happens we have a default request
        # clock value so we can increment it.
        if not request.clock:
            request.clock = 0
        eventId = 0
        eventInterface = 0
        for event in request.events:
            # we are only interested in deposit and withdraw interface for propagation. This will keep track of that
            if bankingsystem_pb2.deposit == event.interface or bankingsystem_pb2.withdraw == event.interface:
                eventInterface = event.interface
                eventId = event.id
            # Query interface
            if bankingsystem_pb2.query == event.interface:
                # Mandatory sleep after query interface so the results can be propagated.
                sleep(3)
                responseList.append(self.query())
            # deposit interface
            elif bankingsystem_pb2.deposit == event.interface:
                # syncing clock on the eventReceive sub-event
                self.clockEventRequest(request.clock)
                # logging the sub-event
                self.logBranchSubEventData(eventId=event.id, subEventName=self.DEPOSIT_REQUEST)
                # executing the deposit interface
                depositResponse = self.deposit(event)
                # adding the response of deposit to list to construct the final response for this request
                responseList.append(depositResponse)
                # calling the function to propagate if the response of the deposit was successful.
                self.propagateIfOperationSuccessful(depositResponse, event, bankingsystem_pb2.propagate_deposit)
            # withdraw interface
            elif bankingsystem_pb2.withdraw == event.interface:
                # syncing clock on the eventReceive sub-event
                self.clockEventRequest(request.clock)
                # logging the sub-event
                self.logBranchSubEventData(eventId=event.id, subEventName=self.WITHDRAW_REQUEST)
                # executing the withdraw interface
                withdrawRes = self.withdraw(event)
                # adding the response of withdraw to list to construct the final response for this request
                responseList.append(withdrawRes)
                # calling the function to propagate if the response of the deposit was successful.
                self.propagateIfOperationSuccessful(withdrawRes, event, bankingsystem_pb2.propagate_withdraw)
            # propagate deposit interface
            elif bankingsystem_pb2.propagate_deposit == event.interface:
                # syncing clock on the Propagate_Request sub-event
                self.clockPropagateRequest(request.clock)
                # logging the sub-event
                self.logBranchSubEventData(eventId=event.id, subEventName=self.DEPOSIT_PROPAGATE_REQUEST)
                # adding the response to list to construct the final response for this request
                responseList.append(self.handlePropagateDeposit(event))
            # propagate withdraw interface
            elif bankingsystem_pb2.propagate_withdraw == event.interface:
                # syncing clock on the Propagate_Request sub-event
                self.clockPropagateRequest(request.clock)
                # logging the sub-event
                self.logBranchSubEventData(eventId=event.id, subEventName=self.WITHDRAW_PROPAGATE_REQUEST)
                # adding the response to list to construct the final response for this request
                responseList.append(self.handlePropagateWithdraw(event))

        log_msg('going to log {0} for event id {1}'.format(str(eventInterface), str(eventId)))
        # if the interface is withdraw or deposit then we sync the clock for eventResponse and log the sub-events
        if bankingsystem_pb2.withdraw == eventInterface:
            log_msg('Logging withdraw')
            self.clockEventResponse()
            self.logBranchSubEventData(eventId=eventId, subEventName=self.WITHDRAW_RESPONSE)
        elif bankingsystem_pb2.deposit == eventInterface:
            log_msg('Logging deposit')
            self.clockEventResponse()
            self.logBranchSubEventData(eventId=eventId, subEventName=self.DEPOSIT_RESPONSE)

        # writing the branch output to the file as json
        self.writeServerOutPut()
        # constructing the final response
        bankResponse = bankingsystem_pb2.BankResponse(id=request.id, recv=responseList, clock=self.clock)

        log_msg('Response from Branch#{0}:{1}'.format(self.id, json.dumps(MessageToDict(bankResponse))))
        log_msg('*' * 20)
        log_msg(json.dumps(self.branchSubEventData))
        log_msg('*' * 20)
        return bankResponse

    def writeServerOutPut(self):
        """
        Writes the subevent data is desired format to the output json file.
        :return: None
        """
        log_msg('^' * 20)
        dataToWrite = list()
        outFileName = "serverOut.json"
        branchFound = False
        # this list will contain all the subevents fro the branches, we can sort it later on based on clock values
        events = list()
        try:
            with open(outFileName, "r") as file:
                existingData = json.load(file)
                log_msg('Existing data in {0} file ->{1}'.format(outFileName, existingData))
                for d in existingData:
                    # the output json file can contain two type of objects, one for branches and other for events
                    # at this point we are trying to gather the branch data only. We can construct
                    # the event data based on the branch data later on
                    if 'pid' in d:
                        # update the 'data' field only if the branch id is same as pid
                        if d['pid'] == self.id:
                            d['data'] = self.branchSubEventData['data']
                            branchFound = True
                        dataToWrite.append(d)
                        # add to list so we can construct event data later on
                        events.extend(d['data'])
                    else:
                        log_msg('Not branch data{0}'.format(d))
                # it means the existing data in json file is not for the branch currently running. We need to
                # add it to final data
                if not branchFound:
                    dataToWrite.append(self.branchSubEventData)
                    events.extend(self.branchSubEventData['data'])
        except Exception as err:
            # First time around we might not have the file, so will get file not found exception here
            log_msg('------>{0}'.format(err))
            # but we need to add data so we can write it for the first time
            dataToWrite.append(self.branchSubEventData)
            events.extend(self.branchSubEventData['data'])
            pass
        # adding the subevent data sorted by clock to the final dict to be written to the file
        dataToWrite.extend(self.makeSortedEventData(events))
        # writing to file
        outfile = open(outFileName, 'w')
        json.dump(dataToWrite, outfile)
        outfile.close()
        log_msg('^' * 20)
        # cautionary sleep after writing to the file
        sleep(1)

    def makeSortedEventData(self, events):
        """
        This function will receive the subevents data from the all the branches happened so far read from the output
        json file. We will sort the available data as per the clock values and group them by the event IDs
        :param events: List of all the subevents
        :return: list of dictionaries grouped by events and sorted by clock values
        """
        finalResult = {}
        # sorting the subevents by the clock values regardless to the eventIds
        events = sorted(events, key=lambda i: i['clock'])
        for eventData in events:
            # making the event Id as key in order to create the unique entries
            mainKey = str(eventData['id'])
            # if a event was not already in the list we will need to create it
            if mainKey not in finalResult:
                finalResult[mainKey] = {'eventid': eventData['id'], 'data': list()}

            # adding the subevent data
            finalResult[mainKey]['data'].append({'clock': eventData['clock'], 'name': eventData['name']})

        return list(finalResult.values())

    def logBranchSubEventData(self, eventId, subEventName):
        """
        This will populate the self.branchSubEventData for all the subevents along with the eventId and clock details
        :param eventId: Unique ID of the event
        :param subEventName: Name of sub event
        :return: None
        """
        self.branchSubEventData['data'].append({'id': eventId, 'name': subEventName, 'clock': self.clock})
        self.branchSubEventData['data'] = sorted(self.branchSubEventData['data'], key=lambda i: i['clock'])

    def query(self):
        """

        :return:
        """
        log_msg('-----QUERY-------')
        res = {'interface': bankingsystem_pb2.query, 'result': bankingsystem_pb2.success, 'money': self.balance}
        return res

    def deposit(self, event):
        """
        Depositing the money to the branch. And calling the eventExecute sub-interface. This function will also log the
        subevent.
        :param event: The incoming event which contains the money and event IDs
        :return: outputEvent in dictionary form
        """
        log_msg('-----DEPOSIT-------')
        if event.money > 0:
            self.balance += event.money
            opResult = bankingsystem_pb2.success
        else:
            log_msg('Deposit amount negative.Operation failed.')
            opResult = bankingsystem_pb2.failure

        # logging and calling the eventExecute interface
        self.clockEventExecute()
        self.logBranchSubEventData(eventId=event.id, subEventName=self.DEPOSIT_EXECUTE)

        return {'interface': bankingsystem_pb2.deposit, 'result': opResult}

    def withdraw(self, event):
        """
        Withdrawing money from the branch. And calling the eventExecute sub-interface. This function will also log the
        subevent.
        :param event: The incoming event which contains the money and event IDs
        :return: outputEvent in dictionary form
        """
        log_msg('-----WITHDRAW-------')
        if event.money < 0 or self.balance - event.money < 0:
            log_msg('Insufficient funds for Withdraw.Operation failed.')
            opResult = bankingsystem_pb2.failure
        else:
            self.balance -= event.money
            opResult = bankingsystem_pb2.success

        # logging and calling the eventExecute interface
        self.clockEventExecute()
        self.logBranchSubEventData(eventId=event.id, subEventName=self.WITHDRAW_EXECUTE)

        return {'interface': bankingsystem_pb2.withdraw, 'result': opResult}

    def propagateIfOperationSuccessful(self, operationResponse, event, propagateOperation):
        """
        Propage the interface if operation was successful. And advances the clock
        :param operationResponse:
        :param event:
        :param propagateOperation:
        :return:
        """
        if operationResponse['result'] == bankingsystem_pb2.success:
            self.propagateEvent(propagateOperation, event.money, event.id)
        else:
            log_msg('{0} operation failed. No propagation required.'.format(event.interface))

    def handlePropagateDeposit(self, event):
        """
        Handle the deposit propagation request and call the subevent Propagate_Execute.It will also log
            deposit_propagate_execute
        :param event:
        :return:
        """
        log_msg('-----Handling Propagated Deposit-------')
        self.balance += event.money
        # calling and logging Propagate_Execute
        self.clockPropagateExecute()
        self.logBranchSubEventData(eventId=event.id, subEventName=self.DEPOSIT_PROPAGATE_EXECUTE)
        return {'interface': bankingsystem_pb2.propagate_deposit, 'result': bankingsystem_pb2.success}

    def handlePropagateWithdraw(self, event):
        """
        Handle the withdraw propagation request and call the subevent Propagate_Execute.It will also log
        withdraw_propagate_execute
        :param event:
        :return:
        """
        log_msg('-----Handling Propagated Withdraw-------')
        self.balance -= event.money
        # calling and logging Propagate_Execute
        self.clockPropagateExecute()
        self.logBranchSubEventData(eventId=event.id, subEventName=self.WITHDRAW_PROPAGATE_EXECUTE)
        return {'interface': bankingsystem_pb2.propagate_withdraw, 'result': bankingsystem_pb2.success}

    def propagateEvent(self, eventType, amount, eventId):
        """
        This function is called when withdraw or deposit operation is successful. This will propagate that operation
        to other branches. This function will also call and log the subinterface PROPAGATE_RESPONSE when response is
        received from propagation.
        :param eventType:
        :param amount:
        :return:
        """
        log_msg(f'-----Propagating Event {eventType} for amount {amount}-------')
        for bid in self.branches:
            portId = getProcessId(bid)

            log_msg('Propagating to branch {0} on port {1}'.format(bid, portId))
            with grpc.insecure_channel(f'localhost:{str(portId)}') as channel:
                stub = bankingsystem_pb2_grpc.TransactionStub(channel)
                req = [{"id": eventId, "interface": eventType, "money": amount}]
                response = stub.MsgDelivery(
                    bankingsystem_pb2.BankRequest(id=bid, type=bankingsystem_pb2.branch, events=req,
                                                  clock=self.clock))
                log_msg('Response on propagation {0} from branch {1}'.format(json.dumps(MessageToDict(response)), bid))

                # calling the PROPAGATE_RESPONSE sub event and loggin it
                self.clockPropagateResponse(response.clock)
                if eventType == bankingsystem_pb2.propagate_withdraw:
                    self.logBranchSubEventData(eventId=eventId, subEventName=self.WITHDRAW_PROPAGATE_RESPONSE)
                elif eventType == bankingsystem_pb2.propagate_deposit:
                    self.logBranchSubEventData(eventId=eventId, subEventName=self.DEPOSIT_PROPAGATE_RESPONSE)

    def __str__(self):
        return "BRANCH[id = {0}, balance = {1} , branches = {2} , tick={3}".format(self.id,
                                                                                self.balance,
                                                                                str(self.branches), str(self.clock))

    def clockEventRequest(self, remoteClock):
        """
        This is implementation of first subinterface Event_Request and propagationRecieved. This subevent happens
        when the Branch process receives a request from the Customer process.
        The Branch process selects the larger value between the local clock
        and the remote clock from the message, and increments one from the selected value.
        :param remoteClock: Remote clock
        :return: None
        """
        self.selectMaxClockAndAdvance(remoteClock=remoteClock, subEventName="Event_Request")

    def clockEventExecute(self):
        """
        This is implementation of second subinterface Event_Execute.This subevent happens when the Branch process
        executes the event after the subevent  “Event Request”. The Branch process increments one from its local clock.
        :return: None
        """
        self.advanceClock("Event_Execute")

    def clockPropagateRequest(self, remoteClock):
        """
        Implantation of third subinterface Propagate_Request.This subevent happens when the
        Branch process recieves the propagation request from its fellow branch processes.
        The Branch process selects the larger value between the local clock
        and the remote clock from the message, and increments one from the selected value.
        :param remoteClock: Remote clock
        :return: None
        """
        self.selectMaxClockAndAdvance(remoteClock=remoteClock, subEventName="Propagate_Request")

    def clockPropagateResponse(self, remoteClock):
        """
        Implantation of fourth subinterface Propagate_Response.
        This subevent happens when the Branch receives the result of the subevent
        “Propogate_Execute” from its fellow branches.
        The Branch process selects the biggest value between the
        local clock and the remote clock from the message, and increments one from the selected value.
        :param remoteClock: Remote clock
        :return: None
        """
        self.selectMaxClockAndAdvance(remoteClock=remoteClock, subEventName="Propagate_Response")

    def clockPropagateExecute(self):
        """
        Implantation of fifth subinterface Propogate_Execute.
        This subevent happens when the Branch process executes the event after the subevent “Propogate_Request”.
        The Branch process increments one from its local clock.
        :return: None
        """
        self.advanceClock("Propogate_Execute")

    def clockEventResponse(self):
        """
        Implantation of sixth subinterface Event_Response.
        This subevent happens after all the propagation responses are returned from the branches.
        The branch returns success / fail back to the Customer process.
        The Branch process increments one from its local clock.
        :return: None
        """
        self.advanceClock("Event_Response")

    def advanceClock(self, subEventName):
        """
        Advance the clock by 1 tick
        :param subEventName: sub-event name for logging purposes
        :return: None
        """
        self.clock += 1
        log_msg("CLOCK:{0}:{1}".format(subEventName, str(self.clock)))

    def selectMaxClockAndAdvance(self, remoteClock, subEventName):
        """
        Take the maximum of the remote clock and local clock and advance by 1 tick.
        :param remoteClock: Remote clock value
        :param subEventName: sub-event name for logging purposes
        :return: None
        """
        self.clock = max(self.clock, remoteClock)
        self.advanceClock(subEventName)

def extractBranchData(parsedData):
    branches = list()
    branchIds = list()
    for data in parsedData:
        log_msg(data)
        if bankingsystem_pb2.branch == translateEntityToEnum(data['type']):
            log_msg('its a branch with id : %s' % data['id'])
            branches.append(Branch(id=data['id'], balance=data['balance'], branches=None))
            branchIds.append(data['id'])

    for branch in branches:
        branch.branches = [id for id in branchIds if branch.id != id]
        log_msg(branch)

    return branches

def startBranch(branch):
    portNumber = getProcessId(branch.id)
    log_msg(
        'Starting Branch server for {0} with balance {1} on port {2}'.format(branch.id, branch.balance, portNumber))
    options = (('grpc.so_reuseport', 1),)
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), options=options)
    bankingsystem_pb2_grpc.add_TransactionServicer_to_server(branch, server)
    server.add_insecure_port('[::]:{0}'.format(portNumber))
    server.start()
    server.wait_for_termination()
    log_msg("--------" * 40)
    log_msg(branch.branchEventData)
    log_msg("--------" * 40)

if __name__ == '__main__':
    initializeLogging()
    inputFile = parseLocalArgs()
    branches = extractBranchData(parseInputFile(inputFile[0]))

    workers = []
    for branch in branches:
        # NOTE: It is imperative that the worker subprocesses be forked before
        # any gRPC servers start up. See
        # https://github.com/grpc/grpc/issues/16001 for more details.
        worker = multiprocessing.Process(target=startBranch,
                                         args=(branch,))
        worker.start()
        workers.append(worker)
    for worker in workers:
        worker.join()